diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d67c04e..d0d8cc14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + # The release gates read the September 22c base files from git + # (scripts/finish_adds0928.py BASE_COMMIT), so fetch full history. + fetch-depth: 0 - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json index 59de2416..a20a3160 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json @@ -1,13 +1,14 @@ { "schema_version": 2, "note": "Developer adjudications of judge verdicts. An entry either resolves a non-final judge class to a final one (llm_error, parse_contract_failure) or affirms prompt_ambiguity, which is allowed only together with excluded_from_scoring: true, meaning the output is removed from scoring for every model and listed in the run's reference_exclusions.json (the freezer verifies the two records agree). Each entry keeps the judge's verdict verbatim beside the decision and the reasoning. Applied to the row annotations and case notes by scripts/apply_adjudications.py before export.", + "date_conventions": "adjudicated_on names the audit wave that made the decision (2026-09-05, 2026-09-22 or 2026-09-29). It carries no time of day or time zone. The 2026-09-05 and 2026-09-22 waves' decisions were written up to the day each wave's release was committed (2026-09-05 and 2026-09-23), and the 2026-09-29 wave's decisions were written on 2026-09-29 UTC, after its reference sweep began. judged_on_utc, judge_rejudged_on and each judge_previous item's judged_on are UTC days, read from the verdict's sidecar (verdict.meta.json judged_at_utc). The top-level judge fields name the case's verdict in this release's audit tree, dated by judge_rejudged_on when a later wave judged the case again and by judged_on_utc otherwise, and judge_previous keeps the verdict that re-judge replaced. A wave's own judge runs could finish after one of its decisions was first written, so a recorded verdict can postdate adjudicated_on up to the last day its wave's decisions were written. A flag an earlier judge run raised stays recorded and names that run in judge_reference_suspect_source, at the top level or on the judge_previous item it belongs to. Where a later wave replaced the verdict a decision reviewed without keeping it, adjudicated_verdict gives that verdict as the decision's own release recorded it.", "adjudications": [ { "country": "us", "scenario_id": "scenario_007", "variable": "head_medicare_eligible", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-23", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -16,14 +17,21 @@ "unlisted_input": "months_receiving_social_security_disability", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "adjudicated_verdict": { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-05", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_073", "variable": "head_medicare_eligible", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-23", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -32,14 +40,21 @@ "unlisted_input": "months_receiving_social_security_disability", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "adjudicated_verdict": { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-05", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_074", "variable": "head_medicare_eligible", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -48,14 +63,32 @@ "unlisted_input": "months_receiving_social_security_disability", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-05" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] }, { "country": "us", "scenario_id": "scenario_079", "variable": "head_medicare_eligible", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -64,14 +97,39 @@ "unlisted_input": "months_receiving_social_security_disability", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-05", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_084", "variable": "head_medicare_eligible", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", @@ -80,79 +138,210 @@ "unlisted_input": "months_receiving_social_security_disability", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-05", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_023", "variable": "snap", - "judge_model": "gpt-5.6-sol", - "judged_on_utc": "2026-09-01", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", - "judge_failure_subtype": "categorical_eligibility", + "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", "excluded_from_scoring": true, "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04).", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "gpt-5.6-sol", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judged_on": "2026-09-01", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_057", "variable": "snap", - "judge_model": "gpt-5.6-sol", - "judged_on_utc": "2026-09-01", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", - "judge_failure_subtype": "categorical_eligibility", + "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", "excluded_from_scoring": true, "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62).", + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "gpt-5.6-sol", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judged_on": "2026-09-01", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_100", "variable": "snap", - "judge_model": "gpt-5.6-sol", - "judged_on_utc": "2026-09-01", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", - "judge_failure_subtype": "categorical_eligibility", + "judge_failure_subtype": "taxable_income_or_deductions", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", "excluded_from_scoring": true, "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "gpt-5.6-sol", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judged_on": "2026-09-01", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_057", "variable": "ssi", - "judge_model": "gpt-5.6-sol", - "judged_on_utc": "2026-09-01", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", "judge_failure_source": "llm_error", - "judge_failure_subtype": "age_disability", + "judge_failure_subtype": "categorical_eligibility", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", "excluded_from_scoring": true, "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading.", + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "gpt-5.6-sol", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-01", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_064", "variable": "ssi", - "judge_model": "claude-opus-5", - "judged_on_utc": "2026-09-05", - "judge_failure_source": "prompt_ambiguity", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", + "judge_failure_source": "llm_error", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", @@ -160,15 +349,40 @@ "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support.", + "judge_reference_suspect": false, + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "claude-opus-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-05", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } }, { "country": "us", "scenario_id": "scenario_067", "variable": "ssi", - "judge_model": "gpt-5.6-sol", - "judged_on_utc": "2026-09-01", - "judge_failure_source": "llm_error", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", + "judge_failure_source": "reference_data_issue_fixed", "judge_failure_subtype": "age_disability", "adjudicated_failure_source": "prompt_ambiguity", "adjudicated_failure_subtype": "age_disability", @@ -176,7 +390,1837 @@ "unlisted_input": "meets_ssi_disability_criteria", "adjudicated_on": "2026-09-05", "adjudicator": "developer", - "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of $1,368 (six exactly); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion." + "reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion.", + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_data_issue_fixed", + "judge_failure_subtype": "age_disability", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ], + "adjudicated_verdict": { + "judge_model": "gpt-5.6-sol", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "age_disability", + "judged_on": "2026-09-01", + "recorded_in": "dashboard-data-20260905c (the adjudication record merged with PR #164, commit 7db59dab00)" + } + }, + { + "country": "us", + "scenario_id": "scenario_002", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "reasoning": "The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "judged_on_utc": "2026-09-05" + }, + { + "country": "us", + "scenario_id": "scenario_003", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "reasoning": "The frozen reference is 22,154.70; the corrected value is 22,400.65. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076", + "reasoning": "The frozen reference is 41,051.51; the corrected value is 40,920.40. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R", + "reasoning": "The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_008", + "variable": "payroll_tax", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "payroll_tax_base", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates", + "reasoning": "The frozen reference is 2,162.76; the corrected value is 2,276.66. PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "other", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say", + "reasoning": "The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "Cal. R&TC 17024.5 (conformity date), 17076", + "reasoning": "The frozen reference is 2,439.65; the corrected value is 1,968.81. PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_039", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "reasoning": "The frozen reference is 1,345.51; the corrected value is 8,596.03. PolicyEngine leaves estate income out of gross income while counting it as qualified business income.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_039", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "reasoning": "The frozen reference is 514.50; the corrected value is 1,975.80. PolicyEngine leaves estate income out of gross income while counting it as qualified business income.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", + "reasoning": "The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_049", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 30,543.91; the corrected value is 30,702.59. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_052", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "Idaho Code 63-3022P; Idaho Form 39R", + "reasoning": "The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_056", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies", + "reasoning": "The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_062", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "taxability of survivor_benefits", + "reasoning": "The frozen reference is 0.00; the alternative is 4,328.39. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "prompt_ambiguity", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_064", + "variable": "dependent1_medicaid_eligible", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "household_unit_or_filing_status", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "the relationship of an adult tax dependent to the filers who claim them", + "reasoning": "The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", + "judge_rejudged_on": "2026-09-29", + "judge_previous": [ + { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": false, + "judged_on": "2026-09-05" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_064", + "variable": "dependent2_medicaid_eligible", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "household_unit_or_filing_status", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "the relationship of an adult tax dependent to the filers who claim them", + "reasoning": "The frozen reference is 0.00; the alternative is 1.00. The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-09-05" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_064", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "reasoning": "The frozen reference is 4,439.29; the corrected value is 4,441.46. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "reasoning": "The frozen reference is 4,606.00; the corrected value is 4,599.62. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_067", + "variable": "dependent1_medicaid_eligible", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "household_unit_or_filing_status", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "the relationship of an adult tax dependent to the filers who claim them", + "reasoning": "The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-29", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_080", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04)", + "reasoning": "The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_081", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21", + "reasoning": "The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes.", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 9,563.05; the corrected value is 9,564.92. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", + "reasoning": "The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_099", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "credit_phaseout", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 10,679.75; the corrected value is 10,711.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076", + "reasoning": "The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_100", + "variable": "federal_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "reasoning": "The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judged_on_utc": "2026-09-30", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-22" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_100", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "reasoning": "The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_107", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "reasoning": "The frozen reference is 0.00; the alternative is 211.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "judged_on_utc": "2026-09-23" + }, + { + "country": "us", + "scenario_id": "scenario_108", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits", + "reasoning": "The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_110", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "reasoning": "The frozen reference is 23,897.44; the corrected value is 25,747.77. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_110", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "reasoning": "The frozen reference is 4,057.47; the corrected value is 4,236.81. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_112", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "unlisted_input", + "reference_basis": "weekly_hours_worked_before_lsr", + "reasoning": "The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_118", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "whether the listed home mortgage interest is on the home the SNAP household occupies", + "reasoning": "The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_119", + "variable": "federal_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "credit_phaseout", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "reasoning": "The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_119", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "credit_phaseout", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": true, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "reasoning": "The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_120", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "reference_engine_defect", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "engine_defect", + "reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "reasoning": "The frozen reference is 11,917.18; the corrected value is 11,919.06. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_121", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "reasoning": "The frozen reference is 0.00; the alternative is 875.34. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "judged_on_utc": "2026-09-05" + }, + { + "country": "us", + "scenario_id": "scenario_121", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "reasoning": "The frozen reference is 0.00; the alternative is 212.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "judged_on_utc": "2026-09-05" + }, + { + "country": "us", + "scenario_id": "scenario_028", + "variable": "child3_chip_eligible", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "health_coverage", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "health_coverage", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "affirmed", + "reference_basis": "42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7", + "reasoning": "Child 3 has employer-sponsored group coverage, which bars CHIP; the household's income passes Pennsylvania's CHIP limit, so the judge's income-limit hypothesis fails.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-29", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "health_coverage", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_056", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "credit_phaseout", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "credit_phaseout", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "affirmed", + "reference_basis": "N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32", + "reasoning": "New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails.", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_engine_defect", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "reference_model_issue_fixed", + "judge_failure_subtype": "credit_phaseout", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_086", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "affirmed", + "reference_basis": "26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12", + "reasoning": "The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law.", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-29", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_076", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "affirmed", + "reference_basis": "Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601", + "reasoning": "Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount.", + "judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_118", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "affirmed", + "reference_basis": "N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025", + "reasoning": "From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-23" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_030", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)", + "reasoning": "The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_042", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7", + "reasoning": "The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "household_unit_or_filing_status", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_051", + "variable": "state_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "thresholds_rates", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7", + "reasoning": "The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_066", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "thresholds_rates", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14)", + "reasoning": "The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judged_on_utc": "2026-09-23" + }, + { + "country": "us", + "scenario_id": "scenario_104", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "N.Y. Tax Law 606(e)(7)(D); Form IT-214", + "reasoning": "The frozen 375.00 is regenerated as 0.00: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9301 (issue #9298) and PolicyEngine/policyengine-us#9313 (the 2025 flat credit tables the sandbox module also applies), after the reference freeze): A renter whose adjusted rent averages more than $450 a month gets no real property tax credit.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judged_on_utc": "2026-09-23" + }, + { + "country": "us", + "scenario_id": "scenario_109", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "thresholds_rates", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": true, + "reference_verdict": "regenerated", + "reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A)", + "reasoning": "The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-23", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + }, + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": true, + "judged_on": "2026-09-29", + "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "reasoning": "The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).", + "judged_on_utc": "2026-09-29" + }, + { + "country": "us", + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "reasoning": "The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).", + "judged_on_utc": "2026-09-30", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "reasoning": "The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).", + "judged_on_utc": "2026-09-30", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] + }, + { + "country": "us", + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "categorical_eligibility", + "adjudicated_failure_source": "prompt_ambiguity", + "adjudicated_failure_subtype": "categorical_eligibility", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "excluded_from_scoring": true, + "judge_reference_suspect": false, + "reference_verdict": "unlisted_input", + "reference_basis": "meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require", + "reasoning": "The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release.", + "judged_on_utc": "2026-09-29" + }, + { + "country": "us", + "scenario_id": "scenario_007", + "variable": "federal_income_tax_before_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "taxable_income_or_deductions", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "judge_reference_suspect": false, + "reasoning": "The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.", + "judged_on_utc": "2026-09-29" + }, + { + "country": "us", + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "state_local_rule", + "adjudicated_on": "2026-09-29", + "adjudicator": "developer", + "judge_reference_suspect": false, + "reasoning": "The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.", + "judged_on_utc": "2026-09-30", + "judge_rejudged_on": "2026-09-30", + "judge_previous": [ + { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "state_local_rule", + "judge_reference_suspect": false, + "judged_on": "2026-09-29" + } + ] } ] } diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv index 26ee3849..b404e7b9 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv @@ -1,51 +1,55 @@ country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation -us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Applied qualifying-surviving-spouse status to a filer with no dependent child, so it used the $32,000/$44,000 joint provisional-income thresholds and got $11,542 of taxable Social Security instead of the single-threshold $18,542.70, then subtracted a $32,300 joint standard deduction plus $1,650 aged add-on. The single filer's deductions are $18,341.33 itemized plus the $6,000 senior deduction, so taxable income is $26,095.78, not the $3,486 whose 10% gives its $349." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Computed taxable Social Security correctly at $18,542 off the single $34,000 threshold but then paired it with a $32,200 married-filing-jointly standard deduction plus a $1,650 aged add-on, a combination no filing status yields. The correct deduction is $18,341.33 of itemized deductions (full $16,470 real estate tax under the $40,400 OBBBA SALT cap plus the Texas sales-tax deduction) plus $6,000, leaving $26,095.78 taxable rather than $10,586." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Included the $48,000 of veterans benefits in gross income, which is excluded from federal gross income under 38 U.S.C. 5301, inflating AGI to about $93,000, and then abandoned its own $6,600 computation to assert zero liability. Correct AGI is $50,437.11 and deductions total $24,341.33, leaving $26,095.78 taxable and $2,883.49 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Got single filing status right but misapplied the second-tier Social Security worksheet as min(0.85*SS, 0.5*SS + 0.85*(PI-$34,000)), hitting the 85% cap at $22,182; the lesser-of term caps the 50% component at $4,500, giving $18,542.70. It also took the $18,150 aged standard deduction instead of the $18,341.33 itemized total, so its taxable income of $29,926 overstates the correct $26,095.78 by $3,830." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Applied the flat 85% Social Security cap of $22,182 rather than the worksheet's $18,542.70, used joint standard-deduction figures for a filer who files single without a dependent child, and then discarded its own roughly $20,000 taxable-income estimate to report zero. Deductions of $24,341.33 against $50,437.11 of AGI leave $26,095.78 taxable and $2,883.49 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the 85% Social Security cap ($22,182) instead of the worksheet result of $18,542.70 and took the standard deduction rather than the $18,341.33 itemized total, derived a tax of about $3,300 from those figures, then submitted $1,176, a number no step of its own derivation produces. The correct chain is $50,437.11 AGI less $24,341.33 of deductions taxed at 10%/12% for $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Filed as qualifying surviving spouse — unavailable without a dependent child — taking a $31,550 joint standard deduction and the flat 85% Social Security cap of $22,182 instead of the single-threshold worksheet result of $18,542.70. It also omitted the $6,000 senior deduction and dismissed itemizing, though $18,341.33 of itemized deductions exceeds the $18,150 aged single standard deduction." -us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Used a $33,850 surviving-spouse standard deduction and the 85% Social Security cap of $22,182 rather than the single filer's $18,542.70, produced $2,023 of tax from those inputs, then inflated it to $3,204 for unspecified 'higher bracket effects.' The single-filer chain gives $26,095.78 of taxable income and $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Derived AGI of $50,436 and taxable Social Security of $18,542 correctly, then deducted only $17,450: it understated the 2026 aged single standard deduction ($16,100 + $2,050 = $18,150), omitted the $6,000 OBBBA senior deduction entirely, and never compared against $18,341.33 of itemized deductions. That left $32,986 taxable instead of $26,095.78." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Applied the 85% Social Security cap of $22,182 instead of the worksheet result of $18,542.70 and deducted only a $17,350 standard deduction, omitting both the $6,000 senior deduction and the $18,341.33 itemized total. Its taxable income of $36,726 exceeds the correct $26,095.78 by $10,630." -us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied the pre-TCJA regime that OBBBA permanently repealed — a $5,300 personal exemption and a 15% second bracket — and counted the $8,389 of employer-sponsored insurance premiums as itemizable medical expenses above the 7.5% floor while omitting the $6,000 senior deduction. For 2026 there is no personal exemption, the brackets are 10% to $12,400 then 12%, and itemized deductions are $18,341.33 from real estate tax plus the Texas sales-tax deduction." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA expired after 2025, restoring a $5,300 personal exemption and a 15% bracket; OBBBA made the TCJA rate schedule permanent, so 2026 income above $12,400 is taxed at 12% and no personal exemption exists. It also added $4,606 of employer-sponsored insurance premiums as a medical deduction and omitted the $6,000 senior deduction, giving $24,060 of taxable income instead of $26,095.78." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Reported taxable income of $37,990 — $11,894 above the correct $26,095.78 — yet a tax of $1,109, an amount no 2026 bracket schedule produces from that base. The correct derivation is $50,437.11 of AGI less $18,341.33 itemized and $6,000 senior deductions, taxed at 10% to $12,400 and 12% above, for $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Deducted a roughly $5,400 personal exemption and applied pre-TCJA 10%/15% brackets, both eliminated for 2026 by OBBBA's permanent extension, and omitted the $6,000 senior deduction while itemizing only the $16,470 real estate tax without the Texas sales-tax add-on. Correct deductions total $24,341.33, leaving $26,095.78 taxed at 10%/12%." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied post-sunset law — a $5,050 personal exemption and a 15% bracket above $11,000 — instead of the permanent TCJA schedule of 10% to $12,400 then 12%, and omitted the $6,000 senior deduction. Its $28,916 of taxable income overstates the correct $26,095.78, and no personal exemption is allowed in 2026." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed total income falls below the aged filer's deductions without computing either; AGI is $50,437.11 against $24,341.33 of deductions, leaving $26,095.78 taxable. The $24,000 pension and $7,849 IRA distribution alone exceed the deduction package before the $18,542.70 of taxable Social Security is added." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Subtracted a personal exemption alongside the $16,470 itemized real estate tax and taxed the remainder at a 15% rate; the 2026 personal exemption is zero under OBBBA's permanent TCJA extension and the second bracket is 12% above $12,400. The allowable deduction is $18,341.33 itemized plus the $6,000 senior deduction, so taxable income is $26,095.78, not $28,666." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used a personal exemption and the pre-TCJA 10%/15% brackets for 2026, both superseded by OBBBA, and left out the $6,000 senior deduction. With $24,341.33 of deductions and the 10%-to-$12,400-then-12% schedule, taxable income is $26,095.78 and tax is $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Computed AGI and taxable Social Security correctly but deducted a flat $17,000 standard deduction, omitting the $6,000 senior deduction and never comparing against $18,341.33 of itemized deductions. That left $33,436 taxable rather than $26,095.78, a $7,340 overstatement worth roughly $880 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Derived $50,436 of AGI and single status correctly, then capped the state and local tax deduction at $10,000 and rejected itemizing; OBBBA raised the 2026 SALT cap to $40,400, so the full $16,470 of real estate tax plus the Texas optional sales-tax deduction gives $18,341.33 of itemized deductions that beat the $18,150 aged standard deduction. It also omitted the $6,000 senior deduction, leaving $32,926 taxable instead of $26,095.78." -us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Itemized $21,076 by adding a $4,606 medical deduction for the $8,389 of employer-sponsored insurance premiums, which are not medical expenses in this computation, and omitted the $6,000 senior deduction. The correct itemized total is $18,341.33 — real estate tax plus the Texas sales-tax deduction — with the $6,000 senior deduction stacked on top, giving $26,095.78 of taxable income rather than $29,360." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that the age-65 standard deduction and surviving-spouse rate structure fully offset income without computing anything. AGI is $50,437.11, including $18,542.70 of taxable Social Security, against $24,341.33 of deductions, leaving $26,095.78 taxed at 10%/12% for $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Declared zero liability on the claim that a large aged standard deduction offsets the Social Security, pension, IRA, and interest income. Those sources produce $50,437.11 of AGI against $24,341.33 of deductions ($18,341.33 itemized plus the $6,000 senior deduction), leaving $26,095.78 taxable and $2,883.49 of tax." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse/joint provisional-income thresholds, producing $11,542 of taxable Social Security instead of the single-threshold $18,542.70, and paired that with a $32,200 joint standard deduction. With no dependent child the filer is single, deductions total $24,341.33, and taxable income is $26,095.78 rather than $3,586." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Took the $18,150 aged single standard deduction plus $6,000 rather than itemizing, missing that the $16,470 real estate tax plus the Texas optional sales-tax deduction gives $18,341.33 of itemized deductions, and overstated taxable Social Security by about $50. That left $26,336 taxable instead of $26,095.78, exactly the $28.85 by which its tax exceeds $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Deducted $24,150 — the $18,150 aged single standard deduction plus the $6,000 senior deduction — instead of the $24,341.33 allowed, because itemized deductions of $18,341.33 (the full $16,470 real estate tax under the $40,400 OBBBA SALT cap plus the Texas sales-tax deduction) exceed the standard deduction by $191.33. Every other step matched, so that omitted $191.33 taxed at 12% is its entire $22.83 error." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Applied surviving-spouse filing status with its joint standard and age deductions to a filer with no dependent child, who files single; that also implies joint Social Security thresholds. The single-filer chain gives $18,542.70 of taxable benefits, $24,341.33 of deductions, and $26,095.78 of taxable income, not the roughly $3,590 whose 10% produces its $359." -us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Computed AGI of $50,436 and taxable Social Security of $18,542 correctly, then applied a $33,850 surviving-spouse age-adjusted standard deduction plus the $6,000 senior deduction. The single filer's deduction is $18,341.33 of itemized deductions plus $6,000, so taxable income is $26,095.78 rather than $10,586." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no computation and asserted that deductions and 'senior credits' exceed taxable income. AGI of $50,437.11 less $24,341.33 of deductions leaves $26,095.78 taxable, and no nonrefundable credit applies here — the credit for the elderly is fully phased out at this income." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied post-sunset 2026 law — a $16,600 standard deduction, $1,600 age add-on, a $5,300 personal exemption, and a 15% bracket above $23,200 — none of which exist under OBBBA's permanent TCJA extension, and omitted both itemizing and the $6,000 senior deduction. Its $2,880 lands near the reference only because its $23,500 deduction total and inflated rate offset each other; the correct chain is $24,341.33 of deductions with 10%/12% brackets on $26,095.78." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Deducted a personal exemption on top of the $16,470 real estate tax and taxed the balance in pre-TCJA 10%/15% brackets. For 2026 the personal exemption is zero, the second bracket is 12% above $12,400, and the $6,000 senior deduction applies on top of $18,341.33 of itemized deductions, giving $26,095.78 of taxable income instead of $28,516." -us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Added $4,606 of 'excess medical' for the $8,389 of employer-sponsored insurance premiums to itemized deductions, omitted the $6,000 senior deduction, and taxed the balance at a 15% bracket. The 2026 schedule is 10% to $12,400 then 12%, and itemized deductions are $18,341.33 from real estate tax plus the Texas sales-tax deduction, so taxable income is $26,095.78, not $29,360." -us,scenario_000,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Took the $18,150 aged single standard deduction plus the $6,000 senior deduction and explicitly concluded there was 'no itemized benefit'; itemized deductions of $18,341.33 — the full $16,470 real estate tax plus the Texas optional sales-tax deduction — exceed the standard deduction. Correct taxable income is $26,095.78 rather than the $26,286 it used." -us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for this output, so no substantive computation was submitted. The required chain is $50,437.11 of AGI less $18,341.33 of itemized deductions and the $6,000 senior deduction, taxed at 10% to $12,400 then 12%, for $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Matched every step — single status, $18,542 of taxable Social Security, $50,436 AGI, the $6,000 senior deduction — except it concluded that the $24,150 standard-deduction package 'beats itemized real-estate taxes of $16,470.' Itemized deductions are $18,341.33 because the Texas optional sales-tax deduction stacks on the real estate tax, exceeding the $18,150 aged standard deduction, and that $191.33 shortfall taxed at 12% is its entire $22.83 error." -us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Treated all $26,096 of Social Security as taxable against a roughly $24,450 deduction, then subtracted a $750 credit for the elderly that is fully phased out at this income and reported zero even though its own figures give about $1,450. Taxable Social Security is $18,542.70, deductions are $24,341.33, no credits apply, and the tax is $2,883.49." -us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Botched the Social Security worksheet by excluding half the benefits from provisional income (using $37,472 rather than $50,520) and dropping the first-tier 50% component, yielding $2,951.20 of taxable benefits instead of $18,542.70. That one error understated AGI by $15,591, and combined with taking the $24,150 standard package instead of $24,341.33 it understated tax by $1,813.97." -us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Declared Social Security entirely nontaxable when provisional income is $50,520, $16,520 above the $34,000 single upper threshold, so $18,542.70 belongs in AGI; it also applied a $31,150 qualifying-widow standard deduction to a filer with no dependent child. AGI is $50,437.11 rather than $31,894 and taxable income is $26,095.78 rather than $744." -us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted the aged single standard deduction wipes out taxable income without computing AGI. The $16,100 base plus $2,050 age add-on — superseded here by $18,341.33 of itemized deductions — plus the $6,000 senior deduction totals $24,341.33 against $50,437.11 of AGI, leaving $26,095.78 taxable and $2,883.49 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Treated the head as a Qualifying Surviving Spouse even though there is no dependent child. It then applied the joint $32,000/$44,000 Social Security thresholds, which gave taxable SS of $11,542, plus the joint $32,300 standard deduction. Filing Single gives taxable SS of $18,542, total deductions of $24,150 and taxable income of $26,287." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the joint/QSS $32,200 standard deduction, $1,650 aged add-on and joint brackets. QSS requires a dependent child, so the head files Single with a $16,100 + $2,050 + $6,000 deduction and pays 12% on income above $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Counted the $48,000 of nontaxable VA benefits in gross income, took the full 85% of Social Security, and subtracted the ESI premiums even though there are no wages. It then set tax to $0 without support. The correct AGI is $50,437, and after the $24,150 of single/aged/senior deductions, $26,287 of taxable income remains." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Used the full 0.5 × SS ($13,048) as the first tier of the taxable Social Security formula instead of the lesser-of amount capped at $4,500. That produced the $22,182 cap instead of $18,542. It also ended the 10% bracket at $12,150 rather than the 2026 figure of $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Filed as QSS/MFJ without a qualifying dependent, taxed Social Security at the 85% cap, and then declared taxable income near zero even though its own figures left about $14,000 taxable. Single status with $24,150 of deductions leaves $26,287 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Chose Single status and the $24,150 of deductions correctly, but taxed Social Security at the 85% cap ($22,182) instead of $4,500 + 0.85 × $16,520 = $18,542. It then cut its own ~$3,300 result to $1,176 with no computation behind it." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Used the joint/QSS $32,200 standard deduction, $1,650 aged add-on and joint 10% bracket for a surviving spouse with no dependent child. That surviving spouse must file Single, with $24,150 of total deductions and the 12% rate above $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Assumed QSS status, which requires a dependent child, and so used MFJ Social Security thresholds, the 85% cap and a joint standard deduction of about $31,550. It also left out the $6,000 OBBBA senior deduction. Single filing gives AGI of $50,437 less $24,150 of deductions." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Filed as QSS with MFJ thresholds and a $33,850 joint deduction, and left out the $6,000 senior deduction. It then raised its own $2,023 result to $3,204 with a made-up 'higher bracket' adjustment. The correct Single computation gives $26,287 taxable and $2,906.45." +us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Computed Social Security with the single thresholds but took the joint/QSS $32,200 + $1,650 standard deduction and joint brackets. With no dependent child the head files Single: $16,100 + $2,050 + $6,000." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"AGI was right at $50,436, but it used the pre-OBBBA $15,400 single standard deduction and left out the $6,000 senior deduction. The correct 2026 total is $16,100 + $2,050 + $6,000 = $24,150. It also ended the 10% bracket at $12,025 instead of $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Taxed Social Security at the full 85% cap ($22,182) instead of the $18,542 the single-threshold formula gives. It also used a $17,350 standard deduction with no $6,000 senior deduction, which overstated taxable income by about $10,400." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA sunset and applied a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has no exemption, uses 10%/12% brackets, and adds the $18,150 aged standard deduction plus the $6,000 senior deduction. It also wrongly counted the ESI premiums as a deductible medical expense." +us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"AGI and the $18,150 single aged standard deduction were right, but it left out the $6,000 OBBBA senior deduction for filers 65 and older. It also ended the 10% bracket at $12,200 instead of $12,400, so it taxed $32,286 instead of $26,287." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA expired in 2026 and applied a personal exemption, 15% brackets and a medical deduction built from ESI premiums. OBBBA made the TCJA rates and standard deduction permanent and added the $6,000 senior deduction, which gives $24,150 of deductions taxed at 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of $37,990 leaves out the $6,000 senior deduction. Its $1,109 tax also does not follow from applying 2026 single brackets to that figure. The correct derivation is $50,437 AGI less $24,150 of deductions, giving $26,287 taxable and $2,906.45 of tax." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied reverted pre-TCJA law: itemized real estate taxes, a roughly $5,400 personal exemption and a 15% bracket. Under OBBBA-permanent law there is no exemption, the single aged standard deduction of $18,150 plus the $6,000 senior deduction beats itemizing, and the second bracket is 12%." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Assumed 2026 reverted to pre-TCJA rules, with a $5,050 personal exemption and a 15% bracket above $11,000. OBBBA made the TCJA structure permanent, so the $24,150 of standard, aged and senior deductions applies with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed income was below the deductions, but AGI including $18,542 of taxable Social Security is $50,437, against total single aged and senior deductions of only $24,150. That leaves $26,287 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Itemized real estate taxes and subtracted a personal exemption as if the TCJA had sunset. Under 2026 OBBBA law there is no personal exemption, the $18,150 standard deduction plus the $6,000 senior deduction applies, and rates are 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used pre-TCJA 2026 rules: itemized property tax, a personal exemption and 10%/15% brackets. OBBBA-permanent law instead gives $24,150 of standard, aged and senior deductions with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"AGI was right at $50,436, but it used a $17,000 single aged standard deduction and left out the $6,000 OBBBA senior deduction. The correct total is $16,100 + $2,050 + $6,000 = $24,150." +us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Filing status and AGI were right, but it used an inflation-projected $17,510 pre-OBBBA standard deduction and left out the $6,000 senior deduction. That taxed $32,926 instead of $26,287, and its bracket boundary of $12,283 was not the 2026 figure of $12,400." +us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Counted the ESI premiums as a deductible medical expense and left out the $6,000 senior deduction, which applies whether or not the filer itemizes. Actual itemizable deductions come to $17,587.20, less than the $18,150 standard deduction, so the correct total is $24,150." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"Relied on a 'surviving-spouse rate structure' that does not apply without a dependent child, and claimed the deductions eliminate the tax. Single status gives $50,437 AGI less $24,150 of deductions, leaving $26,287 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Claimed the age-77 standard deduction offsets all income, but it ignored the $18,542 of taxable Social Security on top of $31,894 of other income. AGI of $50,437 minus $24,150 of deductions leaves $26,287 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse/joint Social Security thresholds, which gave $11,542 taxable SS, and the joint $32,200 standard deduction. Without a dependent child the head files Single: taxable SS is $18,542 and deductions total $24,150." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Framework and brackets were right, but it overstated taxable Social Security at $18,592 instead of $18,542.70, which put AGI at $50,486 and taxable income at $26,336 instead of $26,287.11. The $49 excess taxed at 12% adds about $5.89." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse (joint) thresholds, standard deduction and 10% bracket, which is consistent with $11,542 taxable SS and $3,586 taxable. QSS requires a dependent child, so the head files Single with $26,287 taxable." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the $33,850 joint/QSS age-adjusted standard deduction. The head has no dependent child and files Single, with a $18,150 standard deduction plus the $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Overstated taxable income at $26,336 instead of $26,287.11 because it carried about $49 too much taxable Social Security. It then rounded, landing at $2,912 instead of $2,906.45." +us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Applied the surviving-spouse joint Social Security thresholds, which gave $11,542 taxable SS, and the $33,850 joint deduction. With no dependent child the head files Single, so taxable SS is $18,542 and taxable income is $26,287." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Claimed the senior deductions exceed taxable income, but AGI of $50,437 minus the $24,150 total of the standard, aged and senior deductions leaves $26,287 taxable at 10%/12%." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset law, with a $5,300 personal exemption and a 15% bracket, and also treated the head as QSS with a joint $23,200 10% bracket. Correct 2026 law files Single with $24,150 of standard, aged and senior deductions, 10% up to $12,400 and 12% above." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Assumed the TCJA expired, so it itemized property tax, took a personal exemption and used a 15% bracket. OBBBA made the TCJA structure permanent: the $18,150 standard deduction plus the $6,000 senior deduction applies, with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Taxable income of $26,286 was essentially correct, but it ended the 10% bracket at $12,200 instead of the 2026 single threshold of $12,400. That produced $2,910.32 instead of $1,240 + 12% × $13,887.11 = $2,906.45." +us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Assumed the TCJA sunset: it treated the SALT cap as gone, taxed at 15% above the first bracket, and took a medical deduction from the ESI premiums. Under 2026 OBBBA law the $18,150 standard deduction plus the $6,000 senior deduction applies, with 10%/12% brackets." +us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No value or reasoning was returned for federal_income_tax_before_refundable_credits. +us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Subtracted a $750 Credit for the Elderly and a Lifetime Learning credit to reach zero. The education credit has no qualifying expenses. The elderly credit's $5,000 initial amount is fully eliminated by nontaxable Social Security plus half of AGI above $7,500, so no nonrefundable credits apply against the $2,906.45." +us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Computed provisional income as $37,472 without adding half of Social Security, and dropped the $4,500 lower-tier amount, which gave only $2,951 of taxable SS. With half of SS included, provisional income is $50,520 and taxable SS is $4,500 + 0.85 × $16,520 = $18,542." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated Social Security as entirely nontaxable, even though provisional income of $50,520 far exceeds the base amount. It also used a joint/QSS $31,150 standard deduction without a dependent child and left out the $6,000 senior deduction." +us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the single aged standard deduction eliminates taxable income, but $50,437 of AGI, including $18,542 of taxable Social Security, minus $24,150 of deductions leaves $26,287 taxable." us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 77 and an asserted income exclusion calculation as sufficient for Texas aged Medicaid without identifying or satisfying an aged eligibility pathway. The person qualifies through no Medicaid category and receives no SSI, so the claimed below-threshold aged eligibility does not apply." us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were reported and that payroll tax therefore equals zero, but then submitted $1,534 instead. Its numeric output contradicts its own payroll-tax-base analysis and the required final-value contract." us,scenario_000,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so the required output was missing." -us,scenario_001,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model got AGI right ($31,402) but used a sunset-regime deduction stack — a $16,600 basic standard deduction plus $10,600 of personal exemptions — when OBBBA permanently set the 2026 joint standard deduction at $32,200 and kept the personal exemption at $0. Applying the correct $32,200 base plus $3,300 of age-65 additions (and the $6,000-per-senior OBBBA deduction it never mentioned) wipes out the $31,402 AGI entirely; its $902 of taxable income and $90.23 of tax exist only because it halved the standard deduction and resurrected exemptions repealed through 2028." -us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly premised its answer on 'the sunset of the Tax Cuts and Jobs Act in 2026,' projecting a $15,150 standard deduction and $10,100 of personal exemptions; OBBBA extended the TCJA structure permanently, so the 2026 joint standard deduction is $32,200 with personal exemptions at $0. Its $28,350 total deduction is roughly $19,000 short of the correct $47,500 stack ($32,200 + $3,300 age-65 additions + $12,000 of senior deductions), and the $3,052 of taxable income it taxed at 10% does not exist against AGI of $31,402." -us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model built a $29,600 deduction from a $19,600 standard deduction (age-65 additions included) plus $10,000 of personal exemptions, which is the pre-TCJA regime; the 2026 joint standard deduction alone is $32,200 before the $3,300 of age-65 additions and the $6,000-per-senior OBBBA deduction, and personal exemptions are zero. Its $1,802 of taxable income is exactly the gap created by understating the standard deduction by about $12,600 and adding back exemptions that do not exist in 2026." -us,scenario_001,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model made two compounding errors: it applied the 85% Social Security inclusion tier to combined income of $36,677, including $11,588 of benefits, when income between the $32,000 and $44,000 joint base amounts includes only the lesser of 50% of the excess over $32,000 ($2,338) or 50% of benefits; and it used a sunset-style $20,400 standard deduction instead of the 2026 joint $32,200 plus $3,300 of age-65 additions plus $12,000 of OBBBA senior deductions. Correct inclusion gives AGI of $31,402, which the basic standard deduction alone exceeds, so its $20,252 of taxable income and $2,025 of tax are entirely artifacts of the over-included benefits and the understated deduction." +us,scenario_001,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,402.25 correctly but assumed the TCJA had sunset. So it used a $16,600 joint standard deduction plus $3,300 in aged additions, and added $10,600 in personal exemptions. That left $902.25 of taxable income. Under the permanent TCJA structure from P.L. 119-21, exemptions are $0 and the 2026 joint standard deduction is $32,200, which by itself exceeds AGI, so taxable income and tax are $0." +us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It explicitly assumed the TCJA sunset in 2026. It deducted a projected $15,150 standard deduction, $3,100 in aged additions and $10,100 in personal exemptions, totaling $28,350, which left $3,052.25 taxed at 10%. P.L. 119-21 kept personal exemptions at $0 and set the 2026 joint standard deduction at $32,200, which is above the $31,402.25 AGI, so tax is $0." +us,scenario_001,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA-style deductions for 2026: a $19,600 standard deduction including the aged additions, plus $10,000 in personal exemptions. That left $1,802.25 of taxable income. Personal exemptions remain $0 permanently under P.L. 119-21, and the 2026 joint standard deduction of $32,200 alone wipes out the $31,402.25 AGI, before the aged addition and the $6,000-per-senior deduction." +us,scenario_001,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It reasoned 'after TCJA expiration' and deducted a $16,600 standard deduction, a $3,200 aged addition and $10,600 in personal exemptions, totaling $30,400. That left $1,002.25 taxed at 10%. The TCJA did not expire: P.L. 119-21 made the $0 exemption and the higher standard deduction permanent. The 2026 joint standard deduction is $32,200, which exceeds the $31,402.25 AGI, so tax is $0." +us,scenario_001,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted $11,588 of Social Security as taxable by using the 85% formula, even though combined income of $36,677 is below the $44,000 joint second threshold. Only the 50% tier applies, so taxable Social Security is $2,338.25 and AGI is $31,402.25, not $40,652. It then used a sunset-era $20,400 deduction built from a $12,700 base, although its stated components sum to only $16,070. The actual 2026 joint standard deduction under P.L. 119-21 is $32,200, which exceeds the correct AGI and gives $0 tax." us,scenario_001,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model named the correct SSI-related aged pathway but asserted rather than computed the SSI income test: applying the $20 general exclusion, the $65 earned-income exclusion, and the one-half earned-income disregard gives countable income of about $15,205 from Social Security plus about $14,500 from wages, roughly $2,475 per month, far above the couple SSI federal benefit rate and above Virginia's ~80% FPL aged/blind/disabled limit, which is why SSI received is 0 and medicaid_category is NONE. It compounded this by treating Medicare Savings Program status (QMB/SLMB) and medically needy spend-down as Medicaid-eligibility categories, neither of which is a qualifying pathway here, and by never noticing that household income at 2.05x FPL exceeds even the 100% FPL QMB test." us,scenario_001,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"The model invented a rule that PolicyEngine's aged category is age-only with no income test; the aged/disabled pathway is SSI-linked and requires passing both the SSI income and asset tests, which this household fails with $44,289 of combined wages and Social Security. Because SSI is 0, no category attaches and medicaid_category resolves to NONE despite the head being 71." us,scenario_001,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model supplied no rule at all and defaulted age 71 to Medicaid eligibility. Age 71 removes the head from the MAGI expansion adult group (which ends at 64) and routes eligibility to the SSI-linked aged pathway, whose income and asset tests this household fails, so the correct derivation yields no qualifying category; the model's answer is consistent with an age-alone heuristic that skips every income test." @@ -68,57 +72,64 @@ us,scenario_001,state_income_tax_before_refundable_credits,glm-5.2,llm_error,sta us,scenario_001,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $250.20 implies about $7,600 of Virginia taxable income, i.e. roughly $21,460 of total deductions against $29,064 of Virginia AGI — consistent with a single $12,000 age deduction plus the standard deduction and exemptions, not the ""two age deductions"" its explanation claims. Both spouses are 65 or older and combined AFYA of $29,064 is below the $75,000 joint phase-out, so $24,000 of age deductions applies and by itself exceeds Virginia AGI, leaving $0 of taxable income and $0 of tax." us,scenario_001,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated Virginia's age benefit as the $1,600 of ""age/blind additions"" — the two $800 additional 65+ exemptions — and never claimed the separate $12,000-per-person age deduction of § 58.1-322.02(10)-(11), which is unreduced because AFYA of $29,064 is far under the $75,000 joint threshold. Adding the missing $24,000 to its own $16,000 standard deduction and $1,860 of exemptions eliminates the $9,600 of taxable income it computed, so the tax is $0 rather than $350." us,scenario_001,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Put the entire $15,225 of Social Security into federal AGI for a $44,289 starting point when only $2,338.25 is federally taxable under the 50% tier, and then never applied Virginia's subtraction for that taxable portion — Virginia AGI is $29,064. It also discarded its own branch that used the correct $24,000 of age deductions and submitted $739.15, a figure none of its three calculations ($449.37, $33.78, and the $10,989 taxable-income branch) produces. Virginia AGI of $29,064 less $24,000 of age deductions plus the standard deduction and exemptions yields $0 taxable income and $0 tax." -us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It computed the return as a single filer — stating ""only the head's income is relevant"" — applying the single-filer $25,000/$34,000 provisional-income thresholds and a $20,550 single 65+ standard deduction instead of joint treatment, and it also carried the $24,000 disability benefits into both provisional income and includible income after asserting they were excluded. On the joint return with the disability and veterans benefits excluded, provisional income is $40,780, taxable Social Security is 50% × ($40,780 − $32,000) = $4,390, and AGI of $23,130 falls below the $35,500 MFJ standard deduction ($32,200 + two $1,650 age additions), before the $12,000 senior deduction, so tax is $0 rather than $4,427." -us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It treated the $24,000 disability benefits as taxable, which raised provisional income to $64,780, past the $44,000 MFJ threshold, and produced $23,663 of taxable Social Security instead of the $4,390 that the 50%-tier calculation yields at the true $40,780 provisional income. It then subtracted a pre-TCJA deduction stack — a $15,650 standard deduction plus two $5,050 personal exemptions, which have been repealed since 2018 — and applied a 15% bracket that no longer exists, when the actual 2026 MFJ deductions ($32,200 + $3,300 age additions + $12,000 senior deduction) exceed the $23,130 AGI and zero out the tax." -us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,other,False,"Its reasoning reached the correct conclusion — the 2026 married-both-65+ deduction exceeds AGI, so tax before refundable credits is $0 — and its explanation ends ""value = 0,"" but the submitted numeric value was $120.30. The scored answer diverges from the model's own stated derivation, so this is a contract violation between explanation and submitted value rather than a substantive tax error." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It counted the $24,000 disability benefits as taxable, driving provisional income above the $44,000 MFJ threshold and forcing the 85%-tier Social Security inclusion of $23,663 rather than the 50%-tier $4,390 that $40,780 of provisional income produces. It then reduced the resulting $66,403 by $29,100 of ""standard deduction, personal exemptions, and senior additions"" — personal exemptions are repealed — and applied a nonexistent 15% bracket; with the disability and veterans benefits excluded, AGI of $23,130 is below the $35,500 MFJ standard deduction and the tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It included the full $44,080 of Social Security in income with no provisional-income computation, when the §86 test caps inclusion at 50% of the $8,780 excess over the $32,000 MFJ threshold, or $4,390. That single error produced $62,820 of income against a $34,500 deduction; the correct AGI of $23,130 is below the $35,500 MFJ standard deduction (and further below it after the $12,000 senior deduction), so tax is $0 instead of $3,122." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $22,203 of taxable income is exactly $66,403 less $44,200, meaning it correctly applied the 2026 MFJ $32,200 standard deduction plus the $12,000 senior deduction but built AGI by adding the $24,000 of non-taxable disability benefits, which also pushed provisional income to $64,780 and swapped the $4,390 50%-tier Social Security inclusion for the $23,663 85%-tier amount. Excluding the disability benefits leaves AGI of $23,130, below even the base standard deduction, so tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its explanation states that taxable income ""reflects taxable pension, interest, disability income, taxable Social Security"" — treating the $24,000 disability benefits as taxable, which also lifts provisional income to $64,780 and adds $19,273 of extra Social Security inclusion ($23,663 instead of $4,390). The $3,429 result is consistent with an AGI near $66,400 reduced by roughly $33,700 of deductions; with the disability and veterans benefits excluded, AGI is $23,130, under the $35,500 MFJ standard deduction, and the tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It made two compounding errors: it put the $24,000 of non-taxable disability benefits into provisional income ($64,780 versus the correct $40,780), and then took a flat 85% of gross benefits ($37,468) instead of the §86 lesser-of test, which even at its own $64,780 caps inclusion at 0.85 × ($64,780 − $44,000) + $6,000 = $23,663. Correctly, only $4,390 of Social Security is includible, AGI is $23,130, and that is below the $35,500 MFJ standard deduction it approximated as $34,110, before the $12,000 senior deduction, so tax is $0." -us,scenario_002,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $24,000 disability benefits to taxable income, which is the sole driver of its $66,403 AGI and which also raised provisional income past $44,000 and produced $23,663 of taxable Social Security rather than the 50%-tier $4,390. Its $35,500 deduction figure ($32,200 base plus $3,300 of age additions) is the correct base amount and already exceeds the true $23,130 AGI, so removing the disability benefits alone drives taxable income and tax to $0." -us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After correctly dropping the disability and veterans benefits, it declared the full $44,080 of Social Security ""fully taxable since combined income exceeds the threshold"" — §86 never includes more than 85%, and at a provisional income of $40,780 (between the $32,000 and $44,000 MFJ thresholds) the includible amount is 50% × $8,780 = $4,390. Substituting that gives AGI of $23,130, below its own $32,600 deduction estimate and far below the actual $35,500 standard deduction plus $12,000 senior deduction, so tax is $0 instead of $3,149.40." +us,scenario_002,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the return as a single filer — stating ""only the head's income is relevant"" — applying the single-filer $25,000/$34,000 provisional-income thresholds and a $20,550 single 65+ standard deduction instead of joint treatment, and it also carried the $24,000 disability benefits into both provisional income and includible income after asserting they were excluded. On the joint return with the disability and veterans benefits excluded, provisional income is $40,780, taxable Social Security is 50% × ($40,780 − $32,000) = $4,390, and AGI of $23,130 falls below the $35,500 MFJ standard deduction ($32,200 + two $1,650 age additions), before the $12,000 senior deduction, so tax is $0 rather than $4,427." +us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $24,000 disability benefits as taxable, which raised provisional income to $64,780, past the $44,000 MFJ threshold, and produced $23,663 of taxable Social Security instead of the $4,390 that the 50%-tier calculation yields at the true $40,780 provisional income. It then subtracted a pre-TCJA deduction stack — a $15,650 standard deduction plus two $5,050 personal exemptions, which have been repealed since 2018 — and applied a 15% bracket that no longer exists, when the actual 2026 MFJ deductions ($32,200 + $3,300 age additions + $12,000 senior deduction) exceed the $23,130 AGI and zero out the tax." +us,scenario_002,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reached the correct conclusion — the 2026 married-both-65+ deduction exceeds AGI, so tax before refundable credits is $0 — and its explanation ends ""value = 0,"" but the submitted numeric value was $120.30. The scored answer diverges from the model's own stated derivation, so this is a contract violation between explanation and submitted value rather than a substantive tax error." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $24,000 disability benefits as taxable, driving provisional income above the $44,000 MFJ threshold and forcing the 85%-tier Social Security inclusion of $23,663 rather than the 50%-tier $4,390 that $40,780 of provisional income produces. It then reduced the resulting $66,403 by $29,100 of ""standard deduction, personal exemptions, and senior additions"" — personal exemptions are repealed — and applied a nonexistent 15% bracket; with the disability and veterans benefits excluded, AGI of $23,130 is below the $35,500 MFJ standard deduction and the tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It included the full $44,080 of Social Security in income with no provisional-income computation, when the §86 test caps inclusion at 50% of the $8,780 excess over the $32,000 MFJ threshold, or $4,390. That single error produced $62,820 of income against a $34,500 deduction; the correct AGI of $23,130 is below the $35,500 MFJ standard deduction (and further below it after the $12,000 senior deduction), so tax is $0 instead of $3,122." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $22,203 of taxable income is exactly $66,403 less $44,200, meaning it correctly applied the 2026 MFJ $32,200 standard deduction plus the $12,000 senior deduction but built AGI by adding the $24,000 of non-taxable disability benefits, which also pushed provisional income to $64,780 and swapped the $4,390 50%-tier Social Security inclusion for the $23,663 85%-tier amount. Excluding the disability benefits leaves AGI of $23,130, below even the base standard deduction, so tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation states that taxable income ""reflects taxable pension, interest, disability income, taxable Social Security"" — treating the $24,000 disability benefits as taxable, which also lifts provisional income to $64,780 and adds $19,273 of extra Social Security inclusion ($23,663 instead of $4,390). The $3,429 result is consistent with an AGI near $66,400 reduced by roughly $33,700 of deductions; with the disability and veterans benefits excluded, AGI is $23,130, under the $35,500 MFJ standard deduction, and the tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It made two compounding errors: it put the $24,000 of non-taxable disability benefits into provisional income ($64,780 versus the correct $40,780), and then took a flat 85% of gross benefits ($37,468) instead of the §86 lesser-of test, which even at its own $64,780 caps inclusion at 0.85 × ($64,780 − $44,000) + $6,000 = $23,663. Correctly, only $4,390 of Social Security is includible, AGI is $23,130, and that is below the $35,500 MFJ standard deduction it approximated as $34,110, before the $12,000 senior deduction, so tax is $0." +us,scenario_002,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $24,000 disability benefits to taxable income, which is the sole driver of its $66,403 AGI and which also raised provisional income past $44,000 and produced $23,663 of taxable Social Security rather than the 50%-tier $4,390. Its $35,500 deduction figure ($32,200 base plus $3,300 of age additions) is the correct base amount and already exceeds the true $23,130 AGI, so removing the disability benefits alone drives taxable income and tax to $0." +us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"After correctly dropping the disability and veterans benefits, it declared the full $44,080 of Social Security ""fully taxable since combined income exceeds the threshold"" — §86 never includes more than 85%, and at a provisional income of $40,780 (between the $32,000 and $44,000 MFJ thresholds) the includible amount is 50% × $8,780 = $4,390. Substituting that gives AGI of $23,130, below its own $32,600 deduction estimate and far below the actual $35,500 standard deduction plus $12,000 senior deduction, so tax is $0 instead of $3,149.40." us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified a zero taxable wage base and then contradicted that computation by inventing taxable wage-equivalent income and assigning $1,926 of Social Security and Medicare taxes. No listed income is employee compensation subject to payroll tax, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components all equal zero." us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly evaluated the spouse as a separate zero-income individual and assumed that age alone opened an aged Medicaid pathway. Washington's pathway evaluation includes the applicable household income and categorical requirements; this spouse receives no SSI, has household MAGI of 2.90 times FPL, and qualifies for no category." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own worked derivation used the correct 2026 brackets ($24,800/$100,800/$211,100) and landed at $22,379, but it submitted $27,049, a figure no step in its reasoning produces. Within that derivation it also used a $32,300 standard deduction instead of $32,200 and denied the $1,118 traditional IRA above-the-line deduction, leaving AGI at $183,291 rather than $182,172.27." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It stated ""no wages, so no 401k/IRA deductions"" and omitted the $1,118 traditional IRA above-the-line deduction, leaving AGI at $183,290.50 instead of $182,172.27, then subtracted a $500 auto-loan-interest deduction the reference does not allow. The two errors partly cancel: $1,118 × 22% = $246 of excess tax less $500 × 22% = $110, giving its $136.01 overshoot." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 85% of the excess of benefits over a $32,000 figure ($23,384) rather than the lesser-of test that caps at 85% of benefits, understating taxable benefits by $27,200 against the correct $50,583.50. It compounded this by deducting $23,926 of traditional 401(k) elective deferrals and $1,118 of IRA contributions against zero wage compensation, using a stale $27,700 base standard deduction with a $1,200 age-65 addition for a couple aged 64 and 61, and then discounting its own $14,887 result to $12,898 with unnamed credits." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning produced $22,652–$22,674, and it then submitted $19,524 ""given estimation uncertainty"" — an arbitrary haircut unconnected to any computation it performed. Its underlying derivation also used 2025-era bracket edges ($24,150/$98,200) instead of the 2026 $24,800/$100,800 edges and denied the $1,118 traditional IRA deduction." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 2025 MFJ bracket edges ($23,850 and $96,950) and a $32,300 standard deduction rather than the 2026 values ($24,800, $100,800, and $32,200), inflating the 22%-bracket base by roughly $4,000. It also concluded that the $1,118 traditional IRA contribution generates no deduction, holding AGI at $183,291 instead of $182,172.27." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It asserted that ordinary income of about $147,331 is ""taxed at 10/12/22/24% brackets"" yielding ~$23,886, but the 24% bracket for MFJ in 2026 starts at $211,100, so the entire base falls within the 22% bracket and the correct ordinary tax on that figure is $21,836.71. It also denied the $1,118 traditional IRA above-the-line deduction, leaving AGI $1,118 high." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It guessed the 2026 MFJ standard deduction at $30,000 (""indexed from $29,200 in 2024""), missing the OBBBA amount of $32,200, and inflated the bracket edges to $24,450/$99,375 instead of $24,800/$100,800. It also ruled the $1,118 traditional IRA contribution non-deductible for want of earned income, so its AGI stayed at $183,291 rather than $182,172.27." -us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly applied the $1,118 IRA deduction and the $32,200 standard deduction, then handled the qualified dividends by taxing all $150,073 at ordinary rates and subtracting $564, rather than removing the $3,760 from the ordinary base and taxing it at 15% — a double-adjustment on top of guessed bracket edges of $24,000/$97,000. It then discarded its own $22,272 figure by ""rounding to the nearest hundred"" to $22,000, throwing away another $272." -us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It deducted $23,500 of traditional 401(k) elective deferrals plus the $1,118 IRA contribution from AGI, cutting AGI to $158,673, but elective deferrals reduce W-2 wages and this household reports none — only the $1,582 capital loss and the $1,118 IRA contribution are above-the-line, giving AGI of $182,172.27. It also used a $30,600 standard deduction instead of $32,200." -us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law for 2026 — a $15,000 standard deduction, two $5,100 personal exemptions, and 10/15/25% brackets — but OBBBA made the TCJA rate and deduction structure permanent, so 2026 uses a $32,200 MFJ standard deduction, no personal exemptions, and 10/12/22% brackets. Those substitutions cut its deductions by $7,000 and raised its marginal rate from 22% to 25%, producing $6,255 of excess tax." -us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It stated ""Social Security is excluded in 2026"" and dropped all $59,510 of benefits from AGI; provisional income of $167,704 exceeds the $44,000 MFJ threshold, so 85% of benefits — $50,583 — is taxable. Removing that income and using a $30,700 standard deduction cut its taxable income to $102,007 against the correct $149,972.27." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, subtracting a $16,000 standard deduction and $10,000 of personal exemptions and applying 10/15/25% rates. OBBBA made the TCJA structure permanent: 2026 gives a $32,200 MFJ standard deduction, no personal exemptions, and a 22% top marginal rate on this income, so its taxable income of $157,290.50 and its rate schedule are both wrong." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reported ""taxable income of approximately $183,865,"" which is AGI with no standard deduction removed at all, despite claiming to account for deductions. Subtracting the $32,200 MFJ standard deduction gives $149,972.27 of taxable income, and splitting out the $3,760 of qualified dividends at 15% yields $22,154.70 rather than $31,317." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed ""the expiration of the TCJA,"" subtracting a small standard deduction plus personal exemptions to reach taxable income of about $155,490 and taxing it on the pre-TCJA 15%/25% schedule. Under OBBBA's permanent TCJA parameters the 2026 deduction is $32,200 with no exemptions, taxable income is $149,972.27, and the top rate reached is 22%." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It stated it was ""using the 2026 tax brackets (after sunset of TCJA provisions)"" with a $16,100 standard deduction and $10,100 of personal exemptions. The TCJA structure was made permanent by OBBBA, so the correct 2026 figures are a $32,200 standard deduction, zero personal exemptions, and 10/12/22% brackets — a $6,200 deduction shortfall and a 25%-versus-22% top rate account for its $6,152 overshoot." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $0 with a boilerplate restatement of the tax formula and no arithmetic. AGI of $182,172.27 less the $32,200 standard deduction leaves $149,972.27 of taxable income; no deduction or nonrefundable credit in these facts offsets it, and the liability is $22,154.70." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its stated ordinary base of $148,730.50 implies $22,548.71 of ordinary tax, which is exactly the 2025 schedule ($23,850/$96,950 edges); the 2026 schedule ($24,800/$100,800) gives $22,144.71 on that base. It also used a $30,800 standard deduction instead of $32,200 and omitted the $1,118 traditional IRA above-the-line deduction." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied ""the standard deduction and personal exemptions for 2026,"" but OBBBA leaves no personal exemptions in 2026 and sets the MFJ standard deduction at $32,200. Its $27,845 is consistent with the pre-TCJA package of a ~$16,000 deduction plus ~$10,000 of exemptions taxed on the 15%/25% schedule, roughly $5,690 above the correct liability." -us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It never applied the $1,118 traditional IRA above-the-line deduction and mis-added its own components to AGI of $183,280.50 (they sum to $183,290.50). Its $23,287 result is consistent with a ~$30,000 standard deduction combined with 2025 bracket edges, whereas the 2026 figures are $32,200 and $24,800/$100,800." -us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Its own reasoning ended at $28,867 and it submitted $49,523.45, a number no step in its work generates. The $28,867 was itself built on a fabricated ""$500 nonrefundable credit for sick spouse,"" a $30,000 standard deduction described as an age-65 amount for a couple aged 64 and 61, and an ordinary tax of $28,803 on $151,953 that overshoots the correct 2026 schedule by roughly $6,700." -us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain twice — once inside the $1,582 net capital loss and again in ordinary income — pushing AGI to $184,370.50 instead of $183,290.50 before deductions. It then used an estimated $32,600 standard deduction rather than $32,200 and omitted the $1,118 traditional IRA above-the-line deduction." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It zeroed the liability on ""age-65 deductions and itemized medical/interest deductions,"" but the head is 64 and the spouse 61, so neither the $1,650 aged standard-deduction addition nor the $6,000 senior deduction applies. Total medical of $10,900 falls below the 7.5%-of-AGI floor of $13,663, so the household takes the $32,200 standard deduction and owes $22,154.70 on $149,972.27 of taxable income." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It rested its $2,320 on itemized medical expenses and ""available retirement-account deductions,"" but the $10,900 of medical costs is below the 7.5%-of-AGI floor of $13,663 and the only above-the-line items are the $1,582 capital loss and the $1,118 IRA contribution. With the $32,200 standard deduction, taxable income is $149,972.27 and the tax is $22,154.70." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction, 2026 brackets, and 15% qualified-dividend rate, but treated the $1,118 traditional IRA contribution as non-deductible, giving taxable income of about $151,091 instead of $149,972.27. That single omitted above-the-line deduction at the 22% marginal rate is the entire $246 overstatement." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $22,400.71 is the exact result of the reference method with AGI held at $183,290.50 — it omitted the $1,118 traditional IRA above-the-line deduction that brings AGI to $182,172.27. Everything else (85% Social Security inclusion, $32,200 standard deduction, $24,800/$100,800 bracket edges, $564 of qualified-dividend tax) matches, so the $246.01 gap is $1,118 × 22%." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It reported taxable income of ""about 151091,"" which is AGI of $183,290.50 less the $32,200 standard deduction with no reduction for the $1,118 traditional IRA contribution; the correct taxable income is $149,972.27. Applying the same brackets and 15% dividend rate to the correct base gives $22,154.70, so the missing above-the-line deduction accounts for its full $246 excess." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It estimated taxable income at $151,090.50 — AGI of $183,290.50 minus the $32,200 standard deduction — omitting the $1,118 traditional IRA above-the-line deduction that reduces AGI to $182,172.27. Its bracket work and $564 preferential-rate treatment of the qualified dividends are otherwise exact, leaving a $246.01 overstatement equal to $1,118 at 22%." -us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It stated that ""with no wage income, desired retirement contributions produce no deduction"" and therefore excluded the $1,118 traditional IRA contribution from above-the-line deductions, holding AGI at $183,290.50 instead of $182,172.27. Its taxable Social Security, $32,200 standard deduction, 2026 brackets, and $564 qualified-dividend tax are all correct, so that one omission is the whole $246.01 error." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed that deductions, contributions, and the MFJ standard deduction reduce the liability to zero, but the only above-the-line items are the $1,582 capital loss and the $1,118 IRA contribution, and the $32,200 standard deduction still leaves $149,972.27 of taxable income. The 401(k) deferrals it implicitly relied on require wage compensation, of which this household has none." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied ""2026 post-TCJA-sunset rules"" — a ~$16,100 standard deduction plus ~$10,600 of personal exemptions and 10/15/25% ordinary rates. OBBBA made the TCJA parameters permanent, so 2026 has no personal exemptions, a $32,200 MFJ standard deduction, and a 22% top marginal rate here; its taxable income of ~$156,591 exceeds the correct $149,972.27 and it taxed the excess at 25%." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used ""2026 post-TCJA-sunset MFJ standard deduction and personal exemptions"" to reach taxable income of $155,391 and taxed the ordinary portion at 10/15/25%. The correct 2026 parameters are a $32,200 standard deduction, no personal exemptions, and 10/12/22% brackets breaking at $24,800/$100,800/$211,100, giving taxable income of $149,972.27 and $21,590.70 of ordinary tax." -us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used ""inflation-adjusted pre-TCJA 2026 MFJ parameters"" — a $17,103 standard deduction, $10,908 of personal exemptions, and pre-TCJA rate brackets — producing taxable income of $155,280 and $26,551 of ordinary tax. OBBBA's permanent TCJA parameters give a $32,200 standard deduction, no exemptions, and $21,590.70 of ordinary tax on $146,212.27." -us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $1,080 short-term gain separately while also subtracting the $1,582 net capital loss that already nets that gain, inflating AGI to about $184,371 instead of $183,290.50. It also omitted the $1,118 traditional IRA above-the-line deduction, so its taxable income of ~$152,171 exceeds the correct $149,972.27 by roughly $2,199." -us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It projected 2026 ""current law"" as pre-TCJA — a $16,950 standard deduction, $10,800 of personal exemptions, and 10/15/25% brackets — reaching taxable income of about $155,541. OBBBA made the TCJA structure permanent, so the correct parameters are a $32,200 standard deduction, no personal exemptions, and a 22% top rate on this income, giving $149,972.27 of taxable income." -us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It ruled the $1,118 traditional IRA contribution non-deductible for want of compensation and instead subtracted a $500 auto-loan-interest deduction the reference does not allow, then used bracket edges of $24,400/$99,150 rather than the 2026 $24,800/$100,800. Those three errors move it from the correct $22,154.70 to $22,463.71, with the bracket-edge slip alone adding $173." -us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 on the ground that income and deductions could not be pinned down without wages, but the prompt fixes unlisted wages at zero and fully specifies $94,720 of IRA distributions, $32,525 of pension, $59,510 of Social Security, and the investment items. Those yield AGI of $182,172.27, taxable income of $149,972.27 after the $32,200 standard deduction, and $22,154.70 of tax." -us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the capital loss twice — listing ""+$1,080 ST gains − $2,662 LT gains"" inside its income sum and then a separate ""− $1,582 net capital loss"" — so its taxable income of $149,508.50 sits $1,582 below what its own stated AGI of $183,290.50 less $32,200 implies. It also omitted the $1,118 traditional IRA above-the-line deduction, leaving it $102.03 under the correct $22,154.70." -us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning computed $25,531.98 and it submitted $41,228.50, a number no step in its work produces. The $25,531.98 was itself built on a $32,300 standard deduction described as a ""65+ head"" amount for a 64-year-old and an ordinary tax of $24,967.98 on $148,268.50, which the 2026 schedule taxes at $22,010.87." -us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its $21,902.90 implies taxable income of about $148,827 — roughly $1,145 below the correct $149,972.27, i.e. deductions beyond the $32,200 standard deduction, the $1,582 capital loss, and the $1,118 traditional IRA contribution that the facts support. No further deduction is available: medical costs of $10,900 fall under the 7.5%-of-AGI floor of $13,663 and neither spouse reaches 65." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"The model's own work reached about $22,379, using an incorrect $32,300 standard deduction and no $1,118 IRA deduction. It then submitted $27,049, which nothing in its reasoning supports." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $500 auto-loan interest deduction, although no facts show a qualifying new vehicle assembled in the U.S. It also left out the $1,118 traditional IRA deduction, which the frozen reference takes above the line and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The net result was taxable income of $150,590.50, against the frozen reference's $149,972.27, which gave $22,290.71, $109.94 below the corrected value, $22,400.65." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed only 85% of Social Security above $32,000 ($23,384) instead of the 85% cap of $50,583. It deducted a $23,926 401(k) deferral even though there are no wages. It also used a $28,900 standard deduction with an age-65 addition that neither spouse qualifies for, instead of the $32,200 MFJ amount, and then subtracted credits that do not exist." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It worked out about $22,652–$22,674 using 2025 bracket thresholds ($24,150/$98,200) and no IRA deduction. It then submitted $19,524 'given estimation uncertainty', a number that matches none of its calculations." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 bracket thresholds ($23,850/$96,950) instead of the 2026 MFJ thresholds ($24,800/$100,800). It also used a $32,300 standard deduction instead of $32,200 and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The 2025 thresholds overstate ordinary tax, so its $22,783 is $382.35 above the corrected value, $22,400.65 ($628.30 above the frozen reference)." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It reported ordinary tax of about $23,886 on $147,331 and cited a 24% bracket, but that income sits entirely inside the 22% bracket under the 2026 thresholds and the tax is about $21,837. This overstated the result by about $2,050. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA contribution out of above-the-line deductions because there is no compensation, while the reference subtracts it. That kept AGI at $183,290.50, where the frozen reference has $182,172.27. Every other step matches the reference, and $1,118 at the 22% marginal rate accounts for the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 standard deduction instead of the 2026 MFJ $32,200, and extrapolated bracket thresholds ($24,450/$99,375) instead of $24,800/$100,800. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so only the smaller standard deduction overstated its $153,291 of taxable income." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the $1,118 IRA deduction, as the frozen reference does, although the exclusion's corrected value disallows it because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It also overstated pre-Social Security income by $100 ($132,807 instead of $132,707) and used rough bracket thresholds of $24,000 and $97,000. It then taxed qualified dividends by subtracting the full 15% × $3,760 instead of the 7-point gap between the 22% and 15% rates, and rounded the result down to $22,000." +us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $183,291 subtracts only the $1,582 capital loss and leaves out the $1,118 traditional IRA contribution that the reference takes above the line. With everything else matching, the $1,118 at 22% explains the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It deducted a $23,500 traditional 401(k) deferral even though there are no wages to defer from; in the reference, 401(k) deferrals only reduce wages and give $0 here. It also used a $30,600 standard deduction instead of $32,200. Together these cut taxable income by about $22,000." +us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA law for 2026, with personal exemptions, a $15,000 standard deduction and 10/15/25% brackets, even though the OBBBA made the TCJA rate structure and the $32,200 MFJ standard deduction permanent. It also made an AGI arithmetic error, getting $180,611." +us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly left Social Security out of income on the claim that benefits are exempt in 2026. With provisional income far above $44,000, 85% ($50,583) is taxable. It also used a $30,700 standard deduction instead of $32,200." +us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,000 MFJ standard deduction instead of the 2026 $32,200, and 2025 bracket thresholds ($23,850/$96,950) instead of $24,800/$100,800. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)); its taxable income was $153,290.50." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $16,000 standard deduction, $10,000 in personal exemptions and 10/15/25% brackets. Under the OBBBA, 2026 keeps the $32,200 MFJ standard deduction, no exemptions and the 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It treated about $183,865, which is roughly AGI, as taxable income and never actually subtracted the $32,200 standard deduction. It also skipped the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). This overstated taxable income by more than $33,000 against the frozen reference and more than $32,000 against the corrected value." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and used a pre-TCJA standard deduction plus personal exemptions, with 10/15/25% brackets, to reach taxable income of $155,490. The 2026 law uses the $32,200 standard deduction and the 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-sunset parameters: a $16,100 standard deduction, $10,100 in personal exemptions and pre-TCJA brackets. The 2026 law keeps the $32,200 MFJ standard deduction and the 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 tax without any calculation, even though taxable income is about $150,000 after the $32,200 standard deduction. That income produces about $21,591 of ordinary tax plus $564 on qualified dividends in the frozen reference, which subtracts the $1,118 IRA deduction; the exclusion's corrected value disallows that deduction because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), for about $21,837 of ordinary tax." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,800 standard deduction instead of $32,200. Its $23,112.71 result also shows it applied 2025 bracket thresholds ($23,850/$96,950) instead of 2026's $24,800/$100,800, and it left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied personal exemptions and a pre-TCJA standard deduction and brackets for 2026, even though the OBBBA made the TCJA structure permanent: no exemptions, a $32,200 MFJ standard deduction and 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $23,287 matches a $30,000 standard deduction and 2025 bracket thresholds ($23,850/$96,950), instead of the 2026 $32,200 deduction and $24,800/$100,800 thresholds. It also made a $10 AGI arithmetic slip and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." +us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It misapplied the capital loss by using the $3,000 cap instead of the actual $1,582 net loss, and used a $30,000 standard deduction. It invented a $500 nonrefundable credit, and after getting $28,867 it submitted $49,523.45, which its reasoning does not support." +us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain twice: once as income and again inside the net $1,582 loss. That inflated AGI to $184,370.50. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It then used a $32,600 standard deduction instead of $32,200." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed age-65 deductions even though the head is 64 and the spouse is 61, and it counted itemized medical expenses. The $10,900 in medical costs is below the 7.5%-of-AGI floor (about $13,660), so those deductions are $0 and taxable income stays near $150,000." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It deducted retirement contributions and itemized medical expenses that yield little or no deduction: 401(k) deferrals need wages, and medical costs fall below the 7.5%-of-AGI floor. That shrank taxable income far below the frozen reference's $149,972 (which the exclusion's corrected value raises by disallowing the $1,118 IRA deduction, because neither spouse has compensation) and produced only $2,320." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $151,091 comes from AGI of $183,291 with no $1,118 traditional IRA deduction, which the reference subtracts above the line. Everything else matches, and the IRA amount at 22% explains the $246 difference from the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA contribution out of above-the-line deductions, while the reference subtracts it. That kept AGI at $183,290.50, where the frozen reference has $182,172.27, and put tax $246 above the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $151,091 leaves out the $1,118 traditional IRA deduction that the reference takes above the line, where the reference gets $149,972.27. That produces the $246 gap to the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It reached taxable income of $151,090.50 without subtracting the $1,118 traditional IRA contribution that the reference deducts above the line. The remaining steps match, and the gap to the frozen reference is 22% × $1,118. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly treated the traditional IRA contribution as non-deductible because there are no wages, while the reference subtracts the $1,118 above the line. That left AGI at $183,290.50, where the frozen reference has $182,172.27, and tax $246 above the frozen reference. The exclusion's corrected value, $22,400.65, also treats the contribution as non-deductible because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $151,091 leaves out the $1,118 traditional IRA deduction that the reference applies above the line. Every other step matches, and the difference from the frozen reference is $1,118 taxed at 22%. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $183,290.50 with no traditional IRA deduction, while the reference subtracts the $1,118 contribution along with the capital loss. That put tax $246 above the frozen reference at the 22% marginal rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left the $1,118 traditional IRA deduction out of AGI, getting $183,290.50 where the reference gets $182,172.27, which made ordinary taxable income $147,330.50 instead of $146,212.27. That adds $246 at the 22% rate against the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and contributions wipe out all tax, but 401(k) deferrals need wages, and the only deductions available are the $1,582 capital loss and the $32,200 standard deduction; the frozen reference also deducts the $1,118 IRA contribution, which the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). That still leaves about $150,000 of taxable income." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: a $16,100 standard deduction, $10,600 in exemptions and 10/15/25% brackets. The OBBBA kept the $32,200 MFJ standard deduction and the 10/12/22% brackets for 2026." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a post-TCJA-sunset standard deduction plus personal exemptions and 10/15/25% brackets to get taxable income of $155,391. For 2026 the law uses the $32,200 standard deduction, no exemptions and 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset and applied a $17,000 standard deduction, $10,800 in personal exemptions and pre-TCJA brackets. The 2026 law keeps the $32,200 MFJ standard deduction and the 10/12/22% brackets with no exemptions." +us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-TCJA 2026 parameters: a $17,103 standard deduction, $10,908 in exemptions and pre-TCJA brackets. These do not apply under the OBBBA, which sets the $32,200 MFJ standard deduction and the 10/12/22% brackets." +us,scenario_003,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain twice, both as income and inside the $1,582 net loss, which inflated AGI to $184,371 and taxable income to $152,171, where the frozen reference has $149,972.27. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." +us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied 'current-law' post-sunset parameters: a $16,950 standard deduction, $10,800 in exemptions and 10/15/25% brackets. The OBBBA made permanent the $32,200 MFJ standard deduction and the 10/12/22% brackets for 2026." +us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used wrong 2026 bracket thresholds ($24,400/$99,150 instead of $24,800/$100,800). It also subtracted a $500 car-loan interest deduction with no qualifying vehicle facts and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." +us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It gave up because there are no wages and reported $0. Wages are not needed here: the retirement distributions, pension, taxable Social Security and investment income produce about $150,000 of taxable income and about $22,000 of tax." +us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $1,582 net capital loss twice, once through the ST/LT netting and again as a separate deduction, which dropped taxable income to $149,508.50. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so its taxable income was $464 below the frozen reference's and the whole gap to the corrected value, $22,400.65, comes from the second $1,582 subtraction." +us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $1,582 capital loss and overstated ordinary tax at $24,968. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It then submitted $41,228.50, which does not match its own $25,531.98 total." +us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $21,902.90 implies taxable income of about $148,828, roughly $1,145 below the reference's $149,972.27. That means it subtracted a deduction beyond the $1,582 capital loss and the $32,200 standard deduction, and beyond the $1,118 IRA contribution that the frozen reference also subtracts and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c))." us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,The model replaced current-year Medicare eligibility with an invented near-term eligibility rule. Being age 64 and approaching 65 does not satisfy the age-65 threshold during a year in which age is constant. us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The reasoning correctly stated that the age-64 head is not Medicare eligible, but the model submitted value = 1, reversing its own eligibility conclusion. The required numeric encoding was 0 for “not eligible.”" us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a projection from age 64 to future age-65 eligibility. The benchmark holds age constant throughout 2026, and no disability or qualifying disease pathway is present, so the head fails every Medicare eligibility pathway." @@ -142,258 +153,290 @@ us,scenario_004,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_e us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $1,582 corresponds to roughly $34,800 of New York taxable income under the 2026 MFJ brackets (4% to $17,150, 4.5% to $23,600, 5.25% to $27,900, 5.5% above), i.e. the household's $52,540 of gross receipts less a standard-deduction-sized subtraction — so it ran the full $51,475 Social Security benefit through the NY brackets. None of that benefit is federally taxable (provisional income $26,803 is under the $32,000 MFJ base) and New York subtracts taxable Social Security in full, leaving $1,065 of NY AGI that the ~$16,050 standard deduction erases to $0." us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 implies about $64,000 of New York taxable income under the MFJ brackets — more than the household's entire $52,540 of gross receipts including Social Security — so it both taxed the $51,475 benefit and imputed unlisted wage earnings from the 40-hour work weeks that the prompt requires be treated as 0. The actual New York base is $1,065 of interest less partnership loss, fully offset by the ~$16,050 MFJ standard deduction, for $0 of tax." us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrote that New York does not tax Social Security yet built its base from $52,540 with the full $51,475 benefit included, never applying the federal provisional-income test ($26,803 against the $32,000 MFJ base) that makes the benefit 0% taxable and therefore leaves only $1,065 of federal AGI to flow to New York. Its arithmetic is also self-contradictory: $52,540 less its stated $16,850 standard deduction is $35,690, not the $18,426 it claimed, and $3,451.12 on $18,426 is an 18.7% effective rate against NY's 4%–4.5% brackets at that level (about $743). The correct result is $1,065 of NY AGI wiped out by the ~$16,050 standard deduction, giving $0." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its build stops at $88,518 of ordinary tax plus $11,688 of qualified-dividend tax and never adds the 3.8% net investment income tax on the $150,578.34 of interest and dividends net of the $3,000 capital-loss allowance, which is $5,721.98 of the $5,686.90 shortfall. It also deducted the full $46,308 of desired traditional 401(k) deferrals rather than the $41,650 allowed after the 2026 $24,500 combined elective-deferral limit prorates each worker's traditional share to $20,825." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its structure matches the reference, but three composition errors leave taxable income $559 high and the ordinary-bracket tax $178.94 over: it deducted the full $46,308 of traditional 401(k) deferrals instead of the $41,650 surviving the combined traditional-plus-Roth elective-deferral limit, added the $4,148 state tax refund that PolicyEngine excludes from gross income, and used a bare $32,200 standard deduction without the $1,069.77 OBBBA non-itemizer cash charitable deduction that brings total deductions to $33,269.77." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented roughly $110,000 of mortgage interest from the $2.85M loan balance when no interest expense is listed and unlisted numeric inputs are zero, producing $129,854 of itemized deductions in place of the $33,269.77 standard-plus-charitable deduction. Its submitted $249,914 also does not follow from its own stated ~$408,000 of taxable income, which produces about $85,000 of tax rather than $250,000." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It computed $91,219 of ordinary tax plus $11,688 of qualified-dividend tax for a total of $102,907, then submitted $91,458 — the ordinary layer alone, dropping its own qualified-dividend component. It also declared NIIT outside this line, omitting the $5,721.98 that PolicyEngine includes in federal income tax before refundable credits, and applied the repealed $10,000 SALT cap instead of the OBBBA cap." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It imputed roughly $190,000 of mortgage interest from the loan balance to build ~$60,000 of itemized deductions when no interest is listed, and taxed the $77,920 of qualified dividends at 20% though the 2026 MFJ 15% bracket extends past $580,000 of taxable income. Its own components ($92,000 ordinary plus $15,584 preferential) total $107,584, so the submitted $154,430 exceeds its own derivation by roughly $47,000." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It double-counted the $65,936 of taxable interest, reporting AGI of about $604,318 when the components it listed sum to $538,418, and then subtracted roughly $60,000 of itemized deductions built on mortgage interest that is not listed. It also blended NIIT into an 18.8% rate on qualified dividends while separately adding a net investment income tax, inflating the preferential layer." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset for 2026 and applied pre-TCJA machinery that OBBBA made permanent in the other direction: a 39.6%-topped bracket schedule, restored 2%-floor miscellaneous itemized deductions for the $13,848 of employee business expenses, and a $7,141 AMT that never arises because the 2026 MFJ exemption of roughly $140,200 does not begin phasing out until $1,000,000 of AMTI. It compounded this with about $52,500 of imputed mortgage interest on an unlisted interest expense, itemizing $68,154 instead of taking the $33,269.77 standard-plus-charitable deduction." -us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It imputed roughly $28,000 of mortgage interest that is not listed, pushing itemized deductions to $40,574 so the household itemizes, when the actual itemizable total (about $31,000 of OBBBA-capped SALT, with the $2,574 of gifts falling under the new 0.5%-of-AGI floor) stays below the $32,200 standard deduction. It then declined to run the brackets at all, rounding to a flat $115,000 rather than computing the $89,095.92 ordinary layer plus $11,688 of qualified-dividend tax and $5,721.98 of NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It overstated taxable income by $4,173 — deducting the full $46,308 of 401(k) deferrals instead of the capped $41,650, adding the $4,148 state refund, using a $30,750 standard deduction instead of $32,200, and omitting the $1,069.77 non-itemizer charitable deduction and the $2,164 IRA deduction — and used stale bracket thresholds that add roughly $1,800 more. Its NIIT base of $153,578 also fails to net the $3,000 deductible capital loss against investment income." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It replaced regular tax with a $131,247 alternative minimum tax that does not apply: with the standard deduction there are no SALT or miscellaneous preference add-backs, and the 2026 MFJ AMT exemption of roughly $140,200 phases out only above $1,000,000 of AMTI, leaving tentative minimum tax near $96,000 against $100,783.92 of regular tax. Its $465,449 of taxable income is also $38,000 below the correct $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the pre-TCJA Pease limitation to $59,513 of itemized deductions and then layered a $127,938 AMT on top. OBBBA permanently repealed Pease (replacing it with a 2/37 haircut that reaches only itemizers in the 37% bracket), the $59,513 of itemized deductions requires mortgage interest that is not listed, and the full 2026 AMT exemption leaves tentative minimum tax below the $100,783.92 regular tax." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expiration for 2026, claiming $10,400 of personal exemptions and $108,283 of itemized deductions built on uncapped state and local taxes plus mortgage interest on a $1M balance. OBBBA keeps personal exemptions at zero and caps SALT at $40,400 phasing down to about $31,000 at this MAGI, and no mortgage interest is listed, so the $32,200 standard deduction plus the $1,069.77 charitable deduction applies and taxable income is $503,494.75, not the roughly $420,000 its deductions imply." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated AGI of $460,936 omits the $77,920 of qualified dividends entirely (the listed items sum to $538,418 before the deferral and refund adjustments), and it treats the tax-exempt $70,000 private pension as part of the income base. The submitted $110,996 does not even follow from that AGI, which after the standard deduction and the 15% dividend layer produces about $81,000." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It added an alternative minimum tax that does not arise — a standard-deduction return has no SALT or miscellaneous add-backs and the 2026 MFJ exemption of about $140,200 is unphased below $1,000,000 of AMTI — and it built AGI of $525,049 by subtracting the $13,369 of employer-sponsored insurance premiums from wages a second time when the $430,000 wage figure is already the taxable wage base." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It assumed TCJA individual provisions expired in 2026 and therefore itemized a full uncapped deduction for state income and real estate taxes under a schedule topping at 33%. OBBBA keeps the $40,400 SALT cap (phased to about $31,000 here), so itemized deductions stay below the $32,200 standard deduction, and the top 2026 rate structure produces $89,095.92 of ordinary tax rather than the unshown arithmetic behind $117,920." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted a bare assertion with no computation. The correct build is $89,095.92 of bracket tax on $425,574.75 of ordinary taxable income, $11,688 on $77,920 of qualified dividends at 15%, and $5,721.98 of NIIT on $150,578.34; its $106,093 falls $412.90 short of that, consistent with about $1,290 more deduction than the $33,269.77 PolicyEngine allows." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages to $370,323 by subtracting the $13,369 of employer-sponsored insurance premiums on top of the 401(k) deferrals, when the $430,000 wage input is already the taxable wage base and only the $41,650 of allowed traditional deferrals reduces it. It then claimed personal exemptions that OBBBA keeps at zero and taxed the $77,920 of qualified dividends at 20% when the 15% bracket covers this taxable income." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Its narrative claims preferential rates and NIIT, but the submitted $88,392 equals the ordinary-bracket layer alone ($89,095.92) with nothing added for the $11,688 of qualified-dividend tax or the $5,721.98 of NIIT. It also assumed TCJA-expiration brackets and treated the health insurance premiums as a further pre-tax reduction of wages." -us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It itemized under post-TCJA-sunset rules, claiming an uncapped state income tax deduction and a 2%-floor miscellaneous deduction for the $13,848 of employee business expenses; OBBBA permanently repealed miscellaneous itemized deductions and caps SALT at $40,400 phased down to about $31,000, so the $32,200 standard deduction plus the $1,069.77 charitable deduction governs. It also used the full $46,308 of 401(k) deferrals rather than the $41,650 the elective-deferral limit permits and rounded to a flat $105,000 without bracket arithmetic." -us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2025 parameters throughout — a $30,000 standard deduction and 2025 MFJ brackets rather than the 2026 $32,200 deduction and inflation-adjusted brackets — and its $91,885.36 plus $11,688 total confirms it omitted the $5,721.98 net investment income tax altogether. It also deducted the full $46,308 of desired traditional 401(k) contributions instead of the $41,650 left after the combined elective-deferral limit." -us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It claimed a $2,000 non-itemizer charitable deduction when only the $1,070 of cash gifts qualifies (the $1,504 of non-cash gifts are excluded from the OBBBA above-the-line charitable deduction) and deducted the full $46,308 of traditional 401(k) deferrals instead of the $41,650 surviving the combined traditional-plus-Roth deferral limit, while omitting the $2,164 traditional IRA above-the-line deduction. Its NIIT base of $153,578 also fails to net the $3,000 deductible capital loss." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It performed no bracket computation and submitted $21,735, roughly the MFJ tax on $135,000 of ordinary income, for a household with $503,494.75 of taxable income reaching the 32% bracket. The correct build is $89,095.92 of ordinary tax plus $11,688 on qualified dividends at 15% plus $5,721.98 of NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $75,657 equals the 2026 MFJ bracket tax on roughly $377,000 of ordinary income, about $48,600 below the $425,574.75 of ordinary taxable income here, and includes no separate $11,688 qualified-dividend layer and no $5,721.98 NIIT. It listed deductions generically without ever computing AGI or taxable income." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Three AGI errors net to $1,653 of excess taxable income and exactly the $529.02 overage: it deducted the full $46,308 of traditional 401(k) deferrals instead of the $41,650 allowed after the combined elective-deferral limit, added the $4,148 state tax refund that PolicyEngine excludes, and dropped the $2,164 traditional IRA above-the-line deduction. Its deduction stack of $32,200 plus the $1,070 cash charitable deduction was otherwise correct." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It asserted the standard-deduction path and NIIT without any arithmetic, landing $1,970.90 low. That gap corresponds to about $6,160 of excess deduction against the 32% bracket, consistent with taking the full $46,308 of desired traditional 401(k) deferrals rather than the $41,650 the elective-deferral limit allows plus an overstated standard deduction; the correct taxable income is $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It states that itemized deductions were used, but the household's itemizable total (about $31,000 of OBBBA-capped SALT, with the $2,574 of gifts under the 0.5%-of-AGI floor and no listed mortgage interest) falls below the $32,200 standard deduction. Its $632 overage corresponds to roughly $1,975 of excess taxable income, the signature of deducting the full $46,308 of 401(k) deferrals, adding the $4,148 state refund, and dropping the $2,164 IRA deduction." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $871 overage is exactly 32% of a $2,723 taxable-income overstatement: it used the full $46,308 of traditional 401(k) deferrals rather than the capped $41,650, added the $4,148 state tax refund, and omitted both the $2,164 traditional IRA deduction and the $1,069.77 non-itemizer cash charitable deduction, giving taxable income of $506,218 instead of $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It built AGI of $525,049 by subtracting the $13,369 of employer-sponsored insurance premiums from wages on top of the 401(k) deferrals, when the $430,000 wage figure is already the taxable wage base reduced only by the $41,650 of allowed traditional deferrals. Claiming $36,959.30 of itemized deductions instead of the $33,269.77 standard-plus-charitable amount drops taxable income to about $488,000 and the ordinary layer to roughly $84,172, which with the correct $11,688 and $5,722 layers reproduces its $101,576.27." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed a section 199A qualified-business-income deduction for a household with no business income, and asserted itemized deductions under a $10,000 SALT cap without computing anything. The correct path takes the $32,200 standard deduction plus the $1,069.77 non-itemizer charitable deduction for $503,494.75 of taxable income, giving $89,095.92 plus $11,688 plus $5,721.98." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules — the Pease limitation, personal exemptions, and restored miscellaneous itemized deductions — and then let a $125,199 tentative AMT override regular tax. OBBBA made the TCJA structure permanent, so there are no personal exemptions, no Pease, and no miscellaneous deductions, and the 2026 MFJ AMT exemption of roughly $140,200 is unphased below $1,000,000 of AMTI, leaving tentative minimum tax under the $100,783.92 of regular tax." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It claimed $59,573 of post-Pease itemized deductions, which requires mortgage interest that is not listed, and added a $25,486 AMT driven by SALT and miscellaneous add-backs that do not exist on a standard-deduction return under OBBBA's permanent repeal of miscellaneous deductions. Its regular tax of $111,776 on $478,845 of taxable income also overshoots the $100,783.92 that the 2026 MFJ brackets plus the 15% dividend rate produce on $503,494.75." -us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Its ordinary tax of $106,958 on $423,928 of ordinary taxable income is roughly $18,400 above what the 2026 MFJ brackets yield on that amount (about $88,575), a bracket-schedule error. It also subtracted the $13,369 of employer health premiums from wages a second time and claimed $3,347 of 2%-floor miscellaneous itemized deductions that OBBBA permanently repealed." -us,scenario_005,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It assumed the TCJA sunset — $64,011 of uncapped SALT, a Pease haircut, $3,347 of restored miscellaneous deductions, and an $18,221 AMT — none of which survive OBBBA, which caps SALT at $40,400 phased to about $31,000, repeals Pease and miscellaneous deductions, and leaves the 2026 MFJ AMT exemption unphased below $1,000,000 of AMTI. It also cut wages by the $13,369 of employer-sponsored insurance premiums already excluded from the $430,000 figure." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so no substantive computation was submitted to evaluate. The required answer is the sum of $89,095.92 of ordinary bracket tax, $11,688 of tax on $77,920 of qualified dividends at 15%, and $5,721.98 of NIIT." -us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It reached taxable income of $506,218 instead of $503,494.75 by deducting the full $46,308 of desired traditional 401(k) deferrals rather than the $41,650 left after the 2026 combined elective-deferral limit prorates each worker's traditional share to $20,825, adding the $4,148 state refund, and omitting the $2,164 traditional IRA deduction and the $1,069.77 non-itemizer cash charitable deduction. The resulting $2,723.25 of excess taxable income at the 32% rate is the entire $871.42 overage." -us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It concluded that itemized deductions wipe out all taxable income, but with no mortgage interest listed the itemizable total is roughly $31,000 of OBBBA-capped SALT (the $2,574 of gifts falls under the 0.5%-of-AGI floor), below the $32,200 standard deduction, leaving $503,494.75 of taxable income on $541,928.34 of gross income. The $70,000 tax-exempt pension and the $46,308 of retirement contributions cannot offset income they never entered." -us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It overstated taxable income by $2,723.25 — full $46,308 of 401(k) deferrals instead of the capped $41,650, the $4,148 state refund added, and no $2,164 IRA or $1,069.77 charitable deduction — adding $871 of 32%-bracket tax. It then computed NIIT on $142,667, netting the entire $10,911 capital loss against investment income when only the $3,000 allowed loss reduces the $150,578.34 NIIT base, understating that layer by $301." -us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It imputed $163,750 of mortgage interest from the loan balance to claim a $43,118 deduction when no interest expense is listed, netted the $10,911 capital loss against qualified dividends to shrink preferential income to $67,009 when adjusted net capital gain remains the full $77,920 and only $3,000 of the loss offsets ordinary income, and omitted the $5,721.98 net investment income tax entirely." -us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a $15,700 standard deduction, less than half the 2026 MFJ amount of $32,200, and started from $528,543 of gross income after re-subtracting the $13,369 of employer health premiums already excluded from wages, arriving at $466,534 of taxable income instead of $503,494.75. Its total also omits the $5,721.98 net investment income tax and invokes an AMT that the unphased 2026 MFJ exemption forecloses." +us,scenario_004,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model correctly found federal AGI under $5,000 and used the table amount of $90 plus $15 for the second exemption, but it counted the NY household credit (Tax Law §606(b)) as refundable. That credit is nonrefundable: it only offsets NY tax, which is zero here, and it never enters state_refundable_credits. None of NY's refundable credits apply (EITC, Empire State child credit, real property tax credit), so the answer is $0, not $105." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left the $5,722 net investment income tax (3.8% × $150,578) out of this variable. Used a $32,600 standard deduction instead of $32,200 plus the $1,070 non-itemizer cash-charity deduction. Its computed $100,206 was then raised by an arbitrary 'variation' to $100,819." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Left out the OBBBA non-itemizer cash-charity deduction ($1,070 of cash gifts on top of the $32,200 standard deduction). It also used full $46,308 traditional 401(k) deferrals instead of $41,650 (prorated to fit the $24,500 combined traditional+Roth limit) and included the $4,148 state refund. Net result: taxable income is $559 too high, which adds $179 at 32%." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $110,000 of mortgage interest from the $2.85M balance even though no interest paid was listed. It then reported $249,914, which follows from no bracket computation. The standard-deduction path gives the frozen reference's $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly excluded NIIT from this line, although the variable includes the $5,722 NIIT. The final $91,458 also drops the $11,688 qualified-dividend tax from its own $102,907 regular-tax total, leaving roughly the ordinary-bracket tax alone. It also used a $31,500 standard deduction and estimated brackets instead of the 2026 values ($32,200; 32% bracket starting at $403,550)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Imputed mortgage interest from the $2.85M balance to reach about $60,000 of itemized deductions, even though no interest was listed. It taxed qualified dividends at 20% instead of 15%. Its $154,430 total also contradicts its own parts ($92,000 + $15,584 = $107,584)." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Summed its own AGI components to $604,318 instead of $538,418, an arithmetic error of about $66,000. It then subtracted about $60,000 of itemized deductions built on mortgage interest the facts never provide. Taxable income ended up at $544,000 instead of $503,495." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $1,070 non-itemizer cash-charity deduction that 2026 law adds to the $32,200 standard deduction. It also kept the full $46,308 of 401(k) deferrals rather than the $41,650 prorated under the $24,500 combined limit, and included the $4,148 refund. Taxable income was $504,054 instead of $503,495, overstating tax by $179." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired for 2026, but OBBBA made it permanent. It therefore applied inflated pre-TCJA brackets (up to 33%), a $10,000 SALT cap, and restored 2%-floor miscellaneous deductions. It also imputed $52,500 of mortgage interest at an assumed 7% rate and added a $7,141 AMT that does not arise under the 2026 AMT exemption." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Invented about $28,000 of mortgage interest from the loan balance and applied the old $10,000 SALT cap, then itemized $40,574. It asserted a rough $115,000 without a bracket computation. The correct path uses the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction. With NIIT the frozen reference totals $106,506; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted $2,574 of charitable gifts as itemized without applying the 2026 floor of 0.5% of AGI, which reduces them to zero. It also compared against the bare $32,200 standard deduction instead of $32,200 plus the $1,070 non-itemizer cash-charity deduction. Standard ($33,270) actually beats itemizing, since SALT is capped at about $30,900; this error, plus using full 401(k) deferrals, put its total $144 too high." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of the 2026 MFJ $32,200 plus the $1,070 non-itemizer cash-charity deduction. It computed NIIT on $153,578 without subtracting the allowable $3,000 capital loss, when the correct base is $150,578. It also kept full 401(k) deferrals and included the $4,148 refund." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Applied an AMT of $131,247 that exceeds regular tax. Under 2026 law the MFJ AMT exemption ($140,200) does not start phasing out until $1,000,000, so no AMT arises for a standard-deduction filer with $537k of AGI. Its $465,449 taxable income also reflects inflated itemized deductions instead of the $33,270 standard-plus-charity deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: a Pease limit on itemized deductions and a pre-TCJA AMT that raised tax to $127,938. OBBBA made TCJA permanent, so 2026 has no Pease limit and no AMT at this income. It also computed NIIT on $153,578 instead of the $150,578 base that nets the $3,000 capital loss." +us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income using pre-TCJA sunset brackets for 2026 instead of the permanent 10/12/22/24/32/35/37% OBBBA brackets. It itemized $63,508 instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration, so it allowed uncapped SALT, imputed mortgage interest on a $1M balance even though no interest was listed, and claimed $10,400 of personal exemptions. None of these exist in 2026 law. Under the standard deduction the frozen reference leaves $503,495 taxable, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its near-miss total comes from offsetting errors." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Built AGI of $460,936 that counts the tax-exempt $70,000 pension as income while omitting large parts of the $87,642 of dividends and other investment income; the frozen reference's AGI is $536,765, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its $110,996 does not follow from 2026 brackets, the 15% qualified-dividend rate and the $5,722 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Added an AMT that does not arise under 2026 law, where the $140,200 MFJ exemption phases out only above $1,000,000. It also cut AGI to about $525,049 by subtracting the $13,369 of employer insurance premiums from the listed gross wages." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA individual provisions expired in 2026 and deducted full, uncapped SALT with pre-TCJA brackets up to 33%. Under the permanent OBBBA rules, SALT is phased down to about $30,900 and the standard deduction plus the $1,070 charity deduction wins. The frozen reference's ordinary tax is then $89,096 at the 2026 brackets, on taxable income net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. The frozen reference's figure is $89,095.92 of ordinary tax on $425,575, plus $11,688 on $77,920 of qualified dividends, plus $5,721.98 NIIT; its taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)). The answer is $413 short of that, consistent with overstating deductions by about $1,290 at the 32% marginal rate." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from the listed gross wages, cutting AGI to $525,049. It then applied sunset-era itemized deductions and personal exemptions, and taxed qualified dividends at 20% instead of 15%. The 15% bracket extends to $613,700 for MFJ in 2026." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration brackets and itemized with imputed mortgage interest, even though no interest was listed. It also subtracted health premiums from wages. That drove the result to $88,392, far below the frozen reference's $106,506, which the 2026 permanent brackets, the $33,270 standard-plus-charity deduction and $5,722 NIIT produce together with a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used post-TCJA-sunset rules: itemizing state income tax without the phased-down SALT cap and restoring 2%-floor miscellaneous deductions. OBBBA permanently eliminated miscellaneous deductions, and the $33,270 standard-plus-charity deduction applies. It also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), and reported a rounded $105,000." +us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Left the $5,722 NIIT out of this variable. It also used 2025 parameters (a $30,000 standard deduction and 2025 brackets) instead of the 2026 values ($32,200 plus the $1,070 non-itemizer charity deduction)." +us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Took the full $2,000 cap for the non-itemizer charity deduction, although only the $1,070 of cash gifts qualifies and non-cash gifts are excluded. It kept full $46,308 401(k) deferrals. It disallowed the $2,164 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out. It also computed NIIT on $153,578 without netting the $3,000 capital loss." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735 with no computation. Taxable income of $503,495 under 2026 MFJ brackets yields $89,096 of ordinary tax alone, before $11,688 of qualified-dividend tax and $5,722 of NIIT. The answer omits most of the ordinary-income tax." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Reported $75,657 without showing brackets, and cited itemized medical deductions that are zero under the 7.5%-of-AGI floor. The frozen reference's total is $89,096 of ordinary tax, $11,688 of qualified-dividend tax and $5,722 of NIIT; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)). The answer is about $30,800 short, missing roughly the qualified-dividend tax, the NIIT and part of the 32%-bracket tax." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used AGI of $538,418 instead of $536,765 for three reasons. It deducted the full $46,308 of traditional 401(k) deferrals instead of $41,650 prorated under the $24,500 combined limit. It included the $4,148 state refund, which the reference excludes. It disallowed the $2,164 traditional IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out. The resulting $1,653 of taxable income above the frozen reference's, taxed at 32%, explains its $529 excess over the frozen reference; the answer is $163.42 below the corrected value." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no figures. The frozen reference reaches $503,495 of taxable income after the $32,200 standard deduction and $1,070 charity deduction, with tax of $89,096 + $11,688 + $5,722; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)). Its $104,535 is $1,971 low, consistent with understating taxable income by about $6,200 at the 32% marginal rate." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 state refund in AGI, kept full 401(k) deferrals, and itemized instead of taking the $32,200 standard deduction plus the $1,070 non-itemizer cash-charity deduction. That overstated taxable income by about $2,000, which adds $632 at 32%." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $32,200 standard deduction and left out the $1,070 non-itemizer cash-charity deduction. It also computed AGI as $538,418 by including the $4,148 refund, deducting the full $46,308 of 401(k) deferrals instead of $41,650, and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.66 above the corrected value." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer health premiums from the listed gross wages, reaching AGI of $525,049 instead of $536,765. It then itemized $36,959, but the charitable portion is zeroed by the 0.5%-of-AGI floor and SALT is phased down to about $30,900. The correct deduction is the $33,270 standard-plus-charity amount." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly included the $4,148 state refund in AGI, which the reference excludes, and deducted the full $46,308 of 401(k) deferrals instead of the prorated $41,650. It also took the plain standard deduction without the $1,070 non-itemizer cash-charity deduction. Together these overstated taxable income and pushed tax $571 above the frozen reference's total; the exclusion's corrected value, $107,198.34, which disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $121.34 above the answer." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer insurance premiums from the listed gross wages to reach AGI of $525,049. It then itemized a phased-down SALT deduction instead of taking the larger $32,200 standard deduction plus the $1,070 non-itemizer charity deduction." +us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Reduced AGI to $525,049 by subtracting the $13,369 of employer insurance premiums from the listed gross wages; the frozen reference's AGI is $536,765 on wages of $388,350, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). It then itemized $34,385 instead of the $33,270 standard-plus-charity deduction, so taxable income came out $12,831 too low." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Invoked a qualified business income deduction for a household with no business income and gave no bracket computation. Its $112,450 is $5,944 above the $89,096 + $11,688 + $5,722 total that 2026 brackets produce on $503,495 of taxable income." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: Pease, personal exemptions, miscellaneous deductions, and an AMT that exceeds regular tax. OBBBA made TCJA permanent, so none of these apply in 2026. The frozen reference's result is regular tax on $503,495 plus NIIT with no AMT; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Used 2026 sunset law, with Pease-limited itemized deductions and a $25,486 AMT driven by SALT and miscellaneous add-backs. Under permanent TCJA/OBBBA rules miscellaneous deductions do not exist, the standard deduction applies, and the $140,200 AMT exemption leaves no AMT at this income." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of the 2026 $32,200 and left out the $1,070 non-itemizer cash-charity deduction. It also computed AGI as $538,418 by including the $4,148 refund, deducting the full $46,308 of 401(k) deferrals and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Together these put taxable income $3,423 above the frozen reference's, and the answer is $332.66 above the corrected value." +us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Computed $106,958 of ordinary tax on $423,928, which reflects pre-TCJA-style rates; the 2026 brackets give about $88,600. It also subtracted $13,369 of employer premiums from wages and itemized with uncapped property tax and 2%-floor miscellaneous deductions, which OBBBA permanently repealed." +us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Applied sunset-era rules: $64,011 of SALT, miscellaneous deductions, a Pease reduction, and an $18,221 AMT. None of these apply under the permanent 2026 rules, where SALT is phased down to about $30,900, the standard-plus-charity deduction wins, and there is no AMT. It also subtracted employer premiums from wages." +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Took the $32,200 standard deduction but left out the 2026 non-itemizer deduction for the $1,070 of cash charitable gifts. It also used AGI of $538,418 by including the $4,148 refund, deducting full $46,308 401(k) deferrals instead of $41,650, and disallowing the $2,164 IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.98 above the corrected value." +us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that itemized deductions, including mortgage interest never listed in the facts, wipe out all tax. In fact no interest was paid, SALT is capped, and the standard-plus-charity deduction of $33,270 leaves $503,495 taxable in the frozen reference, which produces $106,506 of tax including NIIT; the exclusion's corrected value, $107,198.34, also disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $1,070 non-itemizer cash-charity deduction and used AGI of $538,418, including the refund and full 401(k) deferrals. It miscomputed the NIIT base as $142,667; the correct base is $150,578 of interest plus dividends less the $3,000 capital loss. These errors partially offset to $107,077." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the full $10,911 net capital loss instead of the $3,000 limit and imputed $43,118 of mortgage interest from the balance. It netted the capital loss against qualified dividends, shrinking the 15% base to $67,009 instead of $77,920. It used wrong 2026 bracket edges and left out the $5,722 NIIT." +us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Used the $15,700 single-filer standard deduction for a joint return instead of the $32,200 MFJ deduction plus the $1,070 non-itemizer charity deduction. It added AMT, which does not apply, and left out the $5,722 NIIT." us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"It used a 2026 Social Security wage base of $183,600 instead of $184,500 (spouse SS $11,383.20 rather than $11,439) and priced California SDI at 1.2% ($5,160) rather than the 1.3% uncapped rate that yields $5,590, arriving at $35,558.20 — $485.80 short. It then submitted $33,296.55, a figure $2,261.65 below its own stated total and unsupported by any step in its derivation." -us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Its federal build is exact — SS $22,599 on the $184,500 base, Medicare $6,235, Additional Medicare $1,620, totaling $30,454 — but it stopped there and never added California's mandatory employee SDI contribution. CA SDI at 1.3% on the full uncapped $430,000 wage base is $5,590, the entire $5,590 gap between its answer and $36,044." -us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It made three separate errors: the 2024 Social Security wage base of $168,600 (SS $20,906.40 instead of $22,599), a $153,164 SDI wage cap at a 1.0% rate ($3,063.28 instead of 1.3% uncapped = $5,590), and inclusion of $11.20 of employer-paid ETT, which the question excludes. It then added roughly $10,086 of California personal income tax withholding into the payroll-tax total, a state income tax that belongs in state_income_tax_before_refundable_credits, pushing its answer $5,815.60 above $36,044." -us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Two rate/parameter errors account for the entire $597.40 gap: a Social Security wage base of $181,800 instead of $184,500, costing $167.40 on the spouse's capped wages, and a CA SDI rate of 1.2% instead of 1.3%, costing $430 on the uncapped $430,000 base. Every other component — Medicare $6,235, Additional Medicare $1,620, head SS $11,160 — matches." -us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It derived the federal legs correctly ($22,599 + $6,235 + $1,620 = $30,454) and computed CA SDI at $5,160, then declared 'SDI uncertainty' and dropped the state leg entirely; CA SDI is a mandatory employee contribution of 1.3% on the full $430,000, or $5,590. It compounded that by submitting $28,586.60, which is $1,867.40 below even its own federal-only figure of $30,454." -us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It computed the federal legs exactly ($22,599 + $6,235 + $1,620 = $30,454) and then asserted CA SDI was 'already reflected' in that sum, which it is not — the $5,590 of 1.3% uncapped SDI is a separate additive component. Its submitted $30,437 is the federal-only total less an unexplained $17." -us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied a $176,100 Social Security wage base, which wrongly caps the head's $180,000 of wages as well as the spouse's, producing $21,836.40 instead of $22,599 — a $762.60 shortfall — and priced CA SDI at 1.2% ($5,160) instead of 1.3% ($5,590). Those two parameter errors are exactly the $1,192.60 gap to $36,044." -us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its derivation used a $176,100 wage base (understating SS by $762.60) and a 1.1% SDI rate (understating the state leg by $860), reaching $34,421.40, and it then submitted $27,306 — $7,115.40 below its own arithmetic with no supporting step. The correct build is $22,599 SS on the $184,500 base plus $6,235 Medicare, $1,620 Additional Medicare, and $5,590 of 1.3% uncapped CA SDI." -us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It applied the Social Security taxable maximum once to the household's combined $430,000 of wages ($181,500 × 6.2% = $11,253) instead of separately to each worker, losing $11,346 — the cap is per individual, so the head owes 6.2% on all $180,000 and the spouse owes 6.2% on $184,500. It also omitted CA SDI's $5,590 entirely, leaving a federal-only total of $19,108." -us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its CA SDI figure of $4,080 is $1,510 short of the $5,590 produced by the 1.3% rate on the full uncapped $430,000 wage base, and its Social Security of $22,407 implies a taxable maximum near $181,400 rather than $184,500, a further $192. Medicare and Additional Medicare Tax are exact." -us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied a $153,164 SDI taxable wage ceiling per worker at 1.2%, yielding $3,675.94; California eliminated the SDI wage ceiling effective 2024, so the tax is 1.3% of all $430,000 = $5,590, a $1,914.06 understatement. Its Social Security of $22,431.60 also reflects a $181,800 wage base rather than $184,500, adding the remaining $167.40 of the gap." -us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its Social Security figure of $21,242 implies a taxable maximum near $171,300 instead of $184,500, understating that leg by $1,357, and it applied a 1.1% SDI rate ($4,730) rather than 1.3% ($5,590), understating the state leg by $860. Those two rate errors are precisely the $2,217 shortfall." -us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It described only Social Security, Medicare, and Additional Medicare Tax on the $430,000 of combined wages and never applied California's mandatory employee SDI of $5,590 (1.3%, no wage cap). Its $31,398 also overshoots the correct federal-only total of $30,454 by $944, implying a Social Security wage base above the $184,500 taxable maximum." -us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It stated it computed the wage base 'after ESI deductions,' subtracting the $13,369 of employer-sponsored insurance premiums from the $430,000 of gross wages; the reference applies Social Security, Medicare, Additional Medicare Tax, and CA SDI to gross employment income of $430,000 with no premium offset. Netting the premiums out plus an under-priced SDI leg leaves it $1,786 short of $36,044." -us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It priced California SDI at 1.1% of wages ($4,730) rather than the 1.3% uncapped rate that produces $5,590, an $860 shortfall, and its remaining $28,970 of federal tax falls $1,484 below the correct $30,454 ($22,599 + $6,235 + $1,620), indicating a Social Security wage base well below $184,500." -us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It reported only Social Security and Medicare on the two wage amounts, omitting California's mandatory employee SDI contribution of $5,590 (1.3% of the uncapped $430,000). Its $26,588 also falls $3,866 below the correct federal total of $30,454, consistent with capping combined wages near $302,000 rather than applying the $184,500 taxable maximum to each worker." -us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced each worker's FICA wages by their employer-sponsored insurance premiums — head $173,111 ($180,000 − $6,889) and spouse $243,520 ($250,000 − $6,480) — while the reference taxes gross employment income of $430,000, which is why its Medicare ($6,041.15) and Additional Medicare ($1,499.68) both fall short. It compounded this with a $181,200 Social Security wage base instead of $184,500 and a 1.1% SDI rate instead of 1.3%, leaving it $1,952.95 low." -us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Taking the federal build at its correct $30,454 ($22,599 + $6,235 + $1,620), its $33,814 leaves only $3,360 for California SDI, $2,230 below the $5,590 that the 1.3% rate on the full uncapped $430,000 produces. It named SDI but priced it at a capped or sub-1.3% base." -us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It used a Social Security taxable maximum below $180,000 — $10,918.20 each, implying a $176,100 base — which wrongly caps the head's wages and understates that leg by $762.60, and it omitted California SDI entirely. CA SDI is a mandatory employee contribution of 1.3% on all $430,000 of wages, $5,590, so its $29,691.40 misses $6,352.60." -us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It imposed a $165,384 per-worker SDI wage cap at a 1.2% rate ($3,969.22); California removed the SDI taxable wage ceiling effective 2024, so the contribution is 1.3% of the entire $430,000 = $5,590, a $1,620.78 shortfall. Its $182,100 Social Security wage base instead of $184,500 accounts for the remaining $148.80." -us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Its federal computation is exact ($22,599 + $6,235 + $1,620 = $30,454), but it asserted that 'California has no mandatory employee state payroll tax.' California's State Disability Insurance is an employee-paid mandatory contribution withheld from wages, 1.3% of the uncapped $430,000 = $5,590, which is the whole of its error." -us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It stated that no Additional Medicare Tax was reached, but combined wages of $430,000 exceed the $250,000 joint-filer threshold by $180,000, producing $1,620 of surtax. It also omitted California's $5,590 SDI contribution, and its $22,295.10 falls $6,538.90 below even the correct Social Security plus Medicare of $28,834." -us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $13,905 is below the $15,064 the spouse alone owes in Social Security and Medicare, and close to the head's standalone $13,770 ($11,160 + $2,610), so it taxed effectively one earner's wages instead of the household's $430,000. The correct build applies the $184,500 cap to each worker separately ($22,599), Medicare of $6,235, Additional Medicare of $1,620, and CA SDI of $5,590." -us,scenario_005,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"It used a $183,600 Social Security wage base instead of $184,500, understating the spouse's SS by $55.80, and applied a 1.2% CA SDI rate ($5,160) instead of 1.3% ($5,590), understating the state leg by $430. Those two parameter values are the entire $485.80 gap." -us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It got the 1.3% SDI rate essentially right but overstated Social Security by $623: netting out Medicare ($6,235), Additional Medicare ($1,620), and SDI ($5,590), its SS leg is $23,222, implying a spouse wage base near $194,500 rather than the $184,500 taxable maximum that caps the spouse's SS at $11,439." -us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Every federal leg is exact ($22,599 + $6,235 + $1,620 = $30,454); the sole error is pricing California SDI at 1.2% of $430,000 ($5,160) instead of 1.3% ($5,590). The 0.1-point rate difference on the uncapped $430,000 base is exactly the $430 gap." -us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It correctly built Social Security of $22,599 on the $184,500 base, Medicare of $6,235, and Additional Medicare of $1,620, then withheld California SDI at 1.2% ($5,160) rather than the 2026 rate of 1.3% on the uncapped $430,000 wage base ($5,590). That single rate value is the entire $430 shortfall." -us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It subtracted the $13,369 of pre-tax employer-plan premiums from wages, computing every leg on a $416,631 base — SS $22,171.88, Medicare $6,041.15, Additional Medicare $1,499.68, SDI $5,416.20; the reference computes all four on gross employment income of $430,000. Its 1.3% SDI rate is right, and the premium netting is exactly the $915.09 shortfall." -us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It enumerated only OASDI, HI, and the Additional Medicare Tax and never named California SDI; with the correct federal total of $30,454, its answer leaves just $4,061 for the state leg, $1,529 below the $5,590 that a 1.3% rate on the uncapped $430,000 wage base produces." -us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% against a capped base (~$176,000 per worker) for $4,226, but California's SDI wage ceiling was repealed effective 2024, making the tax 1.3% of the full $430,000 = $5,590 — a $1,364 shortfall. Its spouse Social Security of $11,284 also reflects a $182,000 wage base rather than $184,500, adding $155." -us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It explicitly applied CA SDI 'at 1.2% up to the SDI wage limit,' producing about $4,250 on a capped base of roughly $354,000; California has no SDI wage limit for 2026 and the rate is 1.3%, so the contribution is $5,590 on all $430,000. That $1,340 is the whole gap, since its federal legs total the correct $30,454." -us,scenario_005,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It capped CA SDI at an $83,067 base per person at 1.1% for $1,827, missing $3,763 — the SDI wage ceiling is repealed and the rate is 1.3% on the full $430,000, giving $5,590. It also netted $13,369 of pre-tax health premiums out of the FICA base ($416,631 instead of $430,000), understating Medicare, the Additional Medicare Tax, and Social Security by a further $825." -us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It reduced FICA wages by pre-tax health premiums (head $173,111, spouse $243,520) instead of using gross wages of $430,000, and applied the 2024 Social Security cap of $168,600 rather than the $184,500 taxable maximum, cutting Social Security to $20,906.40 from $22,599. Capping CA SDI at 1.2% on top of that left the state leg near $4,326 against the correct $5,590." -us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value and no explanation were returned, so nothing was submitted against the required key. The absence of an answer, not a substantive computation error, is the failure." -us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its federal derivation is exact — $11,160 + $11,439 on the $184,500 base, $6,235 Medicare, $1,620 Additional Medicare — and it correctly recognized that CA SDI has no wage cap, but it used a 1.2% rate for $5,160 instead of the 1.3% rate that gives $5,590. The 0.1-point rate error on $430,000 is the entire $430 gap." -us,scenario_005,payroll_tax,minimax-m3,llm_error,other,False,"It claimed that exceeding the Social Security wage base means 'only a partial year' of tax applies and that Medicare and the Additional Medicare Tax are 'offset by deductions/exemptions,' then submitted $0. Payroll tax has no deductions, exemptions, or credits: the wage base caps only the OASDI leg, and $430,000 of wages generates $22,599 of Social Security, $6,235 of Medicare, $1,620 of Additional Medicare Tax, and $5,590 of CA SDI." -us,scenario_005,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It applied the Additional Medicare Tax to the spouse's wages above $200,000 — the employer withholding trigger — instead of to the $180,000 of combined household wages above the $250,000 joint-filer threshold, overstating that leg by $630 ($2,250 vs $1,620). It then declared 'no mandatory CA employee payroll tax modeled,' omitting the $5,590 of 1.3% uncapped SDI." -us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It capped CA SDI at a $153,164 wage base per earner at 1.1%, yielding $3,369.60 against the correct $5,590 (1.3% of the full, uncapped $430,000) — a $2,220.40 shortfall. It also used the $176,100 wage base, which wrongly caps the head's $180,000 of wages and understates Social Security by $762.60 relative to the $184,500 taxable maximum." -us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It reported only Social Security and Medicare with the Additional Medicare Tax and omitted California's mandatory employee SDI of $5,590 (1.3%, no wage cap). Its per-person figures are also too low — the head owes $13,770 ($11,160 + $2,610) and the spouse $15,064 ($11,439 + $3,625), so with the $1,620 surtax the federal total is $30,454, not the $25,474 it submitted." +us,scenario_005,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It used a $183,600 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It then submitted $33,296.55, which matches neither its own sum of $35,558.20 nor any correct component total." +us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The federal part was right ($22,599 SS + $6,235 Medicare + $1,620 Additional Medicare = $30,454), but it left out California's mandatory employee SDI. SDI is 1.3% of all $430,000 of wages, or $5,590." +us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It used the 2024 Social Security wage base of $168,600, even capping the head's $180,000. It applied SDI at 1% with the old $153,164 cap instead of 1.3% uncapped. It also wrongly added the employer ETT and roughly $10,087 of state income tax to the payroll tax total." +us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It capped the spouse's Social Security wages at $181,800 instead of the 2026 base of $184,500, which understated SS by $167. It applied CA SDI at 1.2% ($5,160) instead of 1.3% on $430,000 ($5,590)." +us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly derived $30,454 of federal employee payroll tax, then dropped CA SDI (1.3% × $430,000 = $5,590) as 'uncertain'. It submitted $28,586.60, which is below even its own federal figure." +us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Its federal components were correct ($22,599 + $6,235 + $1,620 = $30,454). It claimed CA SDI was included but added nothing for it, missing the $5,590 SDI at 1.3% of $430,000, and its stated total of $30,437 even misadds the federal parts." +us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 for both spouses, capping the head's $180,000 as well, instead of the 2026 base of $184,500 ($22,599 total SS). It also applied CA SDI at 1.2% instead of the 2026 rate of 1.3%." +us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It capped both spouses at a $176,100 Social Security base instead of $184,500, and used a 1.1% SDI rate instead of 1.3% on $430,000. It then submitted $27,306, which does not match its own $34,421.40 computation." +us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It computed Social Security for only one capped wage base ($11,253), even though each spouse owes 6.2% up to their own $184,500 cap ($11,160 + $11,439 = $22,599). It also left out California SDI of $5,590 entirely." +us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its $22,407 of Social Security implies a spouse wage base of about $181,400 instead of $184,500. Its $4,080 of SDI reflects a per-worker wage cap, when 2026 CA SDI is 1.3% of all $430,000 with no ceiling ($5,590)." +us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% on a $153,164 per-worker cap ($3,675.94), but the wage ceiling ended in 2024 and the 2026 rate is 1.3% on all $430,000 ($5,590). It also used a $181,800 Social Security base instead of $184,500." +us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a $184,200 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)." +us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its $21,242 of Social Security comes from capping both spouses at an old wage base of about $171,300, including the head's $180,000, instead of taxing the head's full $180,000 plus the spouse's $184,500 ($22,599)." +us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out California's mandatory employee SDI ($5,590 at 1.3% of $430,000). Its federal figure of $31,398 also misstates the correct $30,454 federal total." +us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted employer-sponsored insurance premiums from the FICA and SDI wage bases, although the gross wages of $180,000 and $250,000 are the payroll tax base. It also applied SDI at about 1.1% instead of 1.3% of $430,000." +us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied CA SDI at 1.1% ($4,730) instead of the 2026 rate of 1.3% ($5,590). Its federal portion is also understated because its Social Security wage base was too low, when it should tax the head's full $180,000 plus the spouse's $184,500." +us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare, left out California SDI ($5,590), and understated the federal portion. The correct federal total is $30,454 once the $1,620 Additional Medicare Tax and the $184,500 per-worker SS base are included." +us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced the head's FICA wages to $173,111 and the spouse's Medicare/SDI wages by their employer insurance premiums, although payroll tax applies to the stated gross wages. It also used a $181,200 SS base instead of $184,500 and a 1.1% SDI rate instead of 1.3%." +us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $33,814 is $2,230 short. That is consistent with SDI at the outdated 1.1% rate ($4,730 instead of $5,590 at 1.3%) combined with a Social Security base below the 2026 base of $184,500." +us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It capped both spouses at a $176,100 Social Security base ($10,918.20 each), including the head's $180,000, instead of using the 2026 base of $184,500. It also left out California SDI of $5,590 entirely." +us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% with a $165,384 per-worker cap ($3,969.22), but 2026 SDI is 1.3% with no wage ceiling ($5,590). It also used a $182,100 Social Security base instead of $184,500." +us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It stated that California has no mandatory employee payroll tax and so left out CA SDI. SDI is withheld at 1.3% of all $430,000 of wages ($5,590) on top of the correct $30,454 federal total." +us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It explicitly skipped the 0.9% Additional Medicare Tax on combined wages above the $250,000 MFJ threshold ($1,620) and left out CA SDI ($5,590). It also understated employee Social Security and Medicare below the correct $22,599 and $6,235." +us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It capped Medicare at the Social Security wage base, although Medicare's 1.45% applies to all $430,000 of wages ($6,235). Its $13,905 total also falls far short of the $22,599 in SS, the $1,620 Additional Medicare Tax, and the $5,590 CA SDI." +us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It used a $183,600 Social Security wage base instead of $184,500 and applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)." +us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It had the SDI rate right at 1.3% ($5,590), but its total is $623 too high. That leaves federal tax of $31,077 instead of $30,454, which fits a spouse Social Security base about $10,000 above the 2026 base of $184,500." +us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The federal part was correct at $30,454, but its $35,614 total reflects CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)." +us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The federal part was correct at $30,454, but its $35,614 total reflects CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)." +us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the employer insurance premiums ($6,889 and $6,480) as pretax reductions to FICA and SDI wages, shrinking the base to $416,631. The payroll tax base is the full stated gross wage of $430,000, which gives SS $22,599, Medicare $6,235, Additional Medicare $1,620, and SDI $5,590." +us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its federal components were correct ($22,599 + $6,235 + $1,620), but it computed CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)." +us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the employer insurance premiums as pretax and subtracted them from the FICA, Additional Medicare, and SDI wage bases, which understated every component. Payroll tax applies to the full $430,000 of gross wages, giving $36,044 with SDI at 1.3%." +us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only OASDI, HI, and Additional Medicare Tax, so it left out CA SDI ($5,590 at 1.3% of $430,000). Yet its $34,515 total is well above the correct federal amount of $30,454, so it also misstated the federal components." +us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% with a per-worker cap ($4,226), but 2026 SDI is 1.3% of all wages with no ceiling ($5,590). It also capped the spouse at about $182,000 for Social Security instead of $184,500." +us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied CA SDI at 1.2% up to an 'SDI wage limit', but SB 951 removed the SDI wage ceiling. The 2026 rate is 1.3% on all $430,000 ($5,590)." +us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted $13,369 of health premiums from FICA wages, although payroll tax applies to the stated gross $430,000. It also applied SDI at 1.1% on an invented $83,067 per-person cap ($1,827) instead of 1.3% uncapped ($5,590)." +us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It reduced FICA wages by the employer insurance premiums, used the obsolete $168,600 Social Security cap instead of $184,500, and applied SDI to capped wages. The correct SDI is 1.3% on all $430,000." +us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no reasoning, so there was no answer to score." +us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its federal components were correct ($11,160 + $11,439 + $6,235 + $1,620), but it applied uncapped CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)." +us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It claimed employee Social Security and Medicare were 'offset by deductions/exemptions' and answered $0, but income-tax deductions do not reduce FICA or SDI. The household owes $22,599 SS + $6,235 Medicare + $1,620 Additional Medicare + $5,590 CA SDI." +us,scenario_005,payroll_tax,ox-alpha,llm_error,household_unit_or_filing_status,False,"It computed Additional Medicare Tax on the spouse's wages above the $200,000 single-filer withholding threshold ($2,250). For joint filers it applies to combined wages above $250,000 ($1,620). It also wrongly stated that California has no mandatory employee payroll tax and left out SDI of $5,590." +us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It applied SDI at 1.1% on a $153,164 per-worker cap ($3,369.60), but the cap was removed in 2024 and the 2026 rate is 1.3% of all $430,000 ($5,590). It also capped both spouses at the 2025 Social Security base of $176,100 instead of $184,500." +us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It counted only federal Social Security, Medicare, and Additional Medicare and left out California SDI ($5,590 at 1.3% of $430,000). Its federal total of $25,474 is also below the correct $30,454." us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Invented ~$22,800 of deductible mortgage interest from the $2,850,000 balance although no interest amount is listed and unlisted numeric inputs are zero, and added $3,163 of unreimbursed employee business expenses, reaching ~$43,761 of deductions against the allowed $15,338.79. It then discarded its own bracket computation of ~$38,832 and submitted $45,856, a figure none of its stated steps produce." -us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $532,106 falls exactly $4,658.50 short of the reference because it deducted the full $46,308 of traditional 401(k) contributions; combined traditional ($23,154) and Roth ($4,086) deferrals of $27,240 per filer exceed the 2026 $24,500 elective-deferral limit, capping the deductible traditional amount at $20,825 each. It compounded that by putting ~$3,206 of unreimbursed employee business expenses into California itemized deductions, which the allowed $15,338.79 (property tax plus charity less the 6% cutback) excludes." -us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Subtracted an invented ~$23,000 ""deduction for federal income tax paid"" — California grants no deduction for federal income tax — and then stripped ~$21,621 of unnamed nonrefundable credits from its own ~$38,000 pre-credit figure, when the only California nonrefundable credit here is the phased-down $156.93 exemption credit. It also cited a 12% rate above $325,000; the rate applying to this $521,425.72 of taxable income is 9.3%." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Fabricated ~$60,000 of mortgage interest by assuming a 6% rate on the $2,850,000 balance, but no mortgage interest is listed and unlisted numeric inputs are zero. Its resulting $79,854 of California itemized deductions displaced the allowed $15,338.79 (property tax plus charity after the 6% high-income cutback), understating taxable income by roughly $68,000." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Pulled the $70,000 of tax-exempt private pension income into California income, which the $536,764.50 CA AGI excludes, and simultaneously invented ~$35,000 of mortgage interest on a balance with no listed interest amount. Its $36,767 is also inconsistent with its own stated ~$595,000 of taxable income, which under the 2026 CA MFJ schedule yields roughly $48,000." -us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Never applied the bracket schedule: it asserted ~$560,000 of taxable income by treating mortgage interest and real estate taxes as fully deductible, ignoring both the zero mortgage-interest input and the 6% high-income cutback that holds California deductions to $15,338.79 on $536,764.50 of AGI. Its $47,500 corresponds to about $587,000 of taxable income and reflects its appeal to 10.3%–11.3% rates whose brackets begin far above this household's $521,425.72, where 9.3% is the top applicable rate." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Correctly zeroed mortgage interest and excluded the state tax refund, then added $3,163 of unreimbursed employee business expenses over the 2% floor and applied no California high-income itemized cutback, producing $23,017 of deductions instead of $15,338.79. It also deducted the full $46,308 of traditional 401(k) contributions past the 2026 $24,500 per-filer deferral limit and used a $500 exemption credit rather than the phased-down $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Assumed $50,000 of deductible mortgage interest on the $2,850,000 balance although no interest amount is listed, cutting taxable income to $464,416 against the reference's $521,425.72. It then abandoned its own bracket schedule — which yields about $36,100 on that figure — for an asserted 6.9% effective rate, landing at $32,000." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used federal AGI of $538,418 as California AGI, failing to subtract the $4,148 state tax refund California excludes, and deducted the full $19,854 of property tax plus charity without the 6% high-income cutback that reduces it to $15,338.79. Its schedule arithmetic then overstates the tax on its own $518,564 of taxable income by about $1,200, and it applied none of the $156.93 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its stated $497,700 of taxable income is $23,726 below the reference, consistent with allowing roughly $39,000 of deductions — a mortgage-interest and employee-expense stack — instead of the $15,338.79 California permits after the 6% cutback on $17,280 of property tax and $2,574 of charity. It also used a flat $300 exemption credit rather than the phased-down $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $512,889 of post-limitation taxable income is $8,537 below the reference, consistent with folding unreimbursed employee business expenses into the itemized base before the 6% cutback; the allowed base is property tax plus charity only, yielding $15,338.79. It also declared that no California nonrefundable credits apply, while the exemption credit survives phaseout at $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Claimed $73,017 of California itemized deductions by including mortgage interest on a balance with no listed interest amount plus employee expenses over the 2% floor, when the allowed total is $15,338.79 after the 6% cutback on $17,280 of property tax and $2,574 of charity. Its taxable income of $461,253 is therefore $60,173 too low." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $31,215 corresponds to roughly $412,000 of CA MFJ taxable income under the 2026 schedule, about $109,000 below the reference's $521,425.72. A shortfall that size is the footprint of a fabricated mortgage-interest deduction on the $2,850,000 balance, for which no interest amount is listed and the deductible amount is zero." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $501,848 of California taxable income is $19,578 below the reference, consistent with deducting roughly $35,000 rather than the $15,338.79 that survives the 6%-of-excess-AGI itemized limitation applied to $17,280 of property tax and $2,574 of charity. No mortgage interest and no employee-expense deduction enter that base, so no larger itemized total exists here." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 is the naive $534,270 minus the $13,369 of employer-sponsored insurance premiums, which are already excluded from the listed gross wages and are not a further AGI adjustment; the correct AGI is $536,764.50. It then deducted the full $19,854 of property tax and charity without California's 6% high-income cutback to $15,338.79." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $33,816 corresponds to about $442,000 of CA MFJ taxable income, roughly $79,000 below the reference's $521,425.72. Reaching that figure requires nearly $95,000 of deductions, whereas California allows $15,338.79 here — property tax plus charity after the 6% cutback, with zero mortgage interest." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Built its itemized total from property tax, charity and ""allowable employee business expenses"" with no 6% high-income cutback, when the allowed amount is $15,338.79 and excludes the employee expenses. Its $42,150 still lands above the reference because it also kept the $4,148 CA-exempt state tax refund in AGI, applied an un-indexed rate schedule, and dropped the $156.93 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $36,918 corresponds to about $475,000 of CA MFJ taxable income, some $46,000 below the reference's $521,425.72, consistent with deducting roughly $60,000 — a fabricated mortgage-interest stack on the $2,850,000 balance — instead of the $15,338.79 allowed. Its narrative invokes pre-tax deductions and exemption credits but never applies the 6%-of-excess-AGI itemized limitation or the zero mortgage-interest input." -us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Started from the naive $534,270 CA AGI, over-deducting the full $46,308 of traditional 401(k) contributions past the 2026 $24,500 per-filer elective-deferral limit and omitting the $2,164 traditional IRA deduction the unlisted plan-coverage flag permits, and then folded unreimbursed employee expenses into the itemized base before the cutback. The reference's $536,764.50 AGI less $15,338.79 gives $521,425.72, about $6,400 above its ~$515,000." -us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Dropped the $77,920 of qualified dividends from income entirely — even its own listed components sum far above the $452,587 federal AGI it reported — and invented a $2,000 California capital-loss limit when the limit is $3,000. It then took an $11,000 standard deduction instead of the $15,338.79 of limited California itemized deductions, landing $9,109 below the reference." -us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Took the ~$11,300 California standard deduction when the limited itemized total of $15,338.79 (property tax plus charity, less the 6% cutback) is larger and controls, and applied no nonrefundable credit against the $156.93 exemption credit. Its $534,270 AGI also deducted the full $46,308 of traditional 401(k) contributions past the 2026 $24,500 per-filer elective-deferral limit and omitted the allowed $2,164 traditional IRA deduction." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Submitted a number with no computation; $20,677 corresponds to about $300,000 of CA MFJ taxable income, roughly $221,000 below the reference's $521,425.72. The required path is $536,764.50 of CA AGI less $15,338.79 of limited itemized deductions, taxed at $41,208.45 before the $156.93 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $37,099 corresponds to about $477,000 of CA MFJ taxable income, some $44,000 below the reference, consistent with claiming a mortgage-interest deduction on the $2,850,000 balance for which no interest amount is listed and the deductible amount is zero. California's allowed deductions here total $15,338.79." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Subtracted ""pretax health premiums"" from wages on top of the 401(k) deferrals, but the $13,369 of employer-sponsored insurance premiums is already excluded from the listed gross wages and is not a further adjustment. Combined with an over-large itemized stack in place of the $15,338.79 allowed, its implied taxable income of about $495,900 sits $25,500 below the reference's $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $519,427 of California taxable income is $1,999 short of the reference for a single reason: it used the naive AGI that deducts all $46,308 of traditional 401(k) contributions, when combined traditional ($23,154) and Roth ($4,086) deferrals of $27,240 per filer exceed the 2026 $24,500 elective-deferral limit and only $20,825 each is deductible, while the $2,164 traditional IRA deduction is allowed because the workplace-plan-coverage flag is unlisted. That $2,494.50 understatement of AGI is the whole miss." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $41,150 tracks the naive $534,270 CA AGI with no exemption credit applied: it deducted the full $46,308 of traditional 401(k) contributions despite the 2026 $24,500 per-filer elective-deferral limit that caps the traditional deduction at $20,825 each, omitted the $2,164 traditional IRA deduction the unlisted plan-coverage flag permits, and never subtracted the $156.93 phased-down exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Explicitly added the $8,961 of tax-exempt interest to California income, which the $536,764.50 CA AGI excludes, and still finished $885 low because it carried unreimbursed employee business expenses into the itemized base that the allowed $15,338.79 (property tax plus charity after the 6% cutback) excludes. The two errors run in opposite directions and do not cancel." -us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its state AGI of $520,901 is the naive $534,270 less the $13,369 of employer-sponsored insurance premiums, which are already excluded from the listed gross wages and are not a further adjustment; the correct AGI is $536,764.50. It also placed ""eligible employee expenses"" in its $22,294.58 itemized total, which the allowed $15,338.79 excludes, and used a $222 exemption credit instead of $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $48,720 corresponds to about $602,000 of CA MFJ taxable income, roughly $81,000 above the reference's $521,425.72 — the combined size of the $70,000 tax-exempt private pension and the $8,961 of tax-exempt interest, both of which stay outside California AGI here." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Applied the 6% high-income cutback but also allowed the unreimbursed employee business expenses over the 2% floor, producing about $512,000 of taxable income against the reference's $521,425.72; the deductible base is $17,280 of property tax plus $2,574 of charity only, yielding $15,338.79. It also carried a ~$280 exemption credit rather than the phased-down $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Set CA AGI equal to federal AGI at $538,418, failing to subtract the $4,148 state tax refund California excludes, and then over-deducted with $22,934 of itemized deductions including employee expenses and only a partial cutback, versus the allowed $15,338.79. Its taxable income of $515,484 lands $5,942 below the reference's $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $525,049 is $11,716 below the reference's $536,764.50, and its $23,201 itemized total added employee expenses after the 2% floor with no 6% high-income cutback, against the allowed $15,338.79. It also stated that no nonrefundable credits were used, dropping the $156.93 exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Added the $70,000 of tax-exempt private pension income and the $8,961 of tax-exempt interest back into CA AGI to reach $604,010, when California AGI here is $536,764.50 and excludes both. Its taxable income of $582,388 is therefore about $61,000 too high." -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for this key, so the failure is a missing submission rather than a substantive computation error. The required derivation is CA AGI of $536,764.50 less $15,338.79 of limited itemized deductions, giving $521,425.72 of taxable income taxed at $41,208.45 under the 2026 CA MFJ schedule, less the $156.93 phased-down exemption credit." -us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Correctly excluded the $4,148 state tax refund from CA AGI, but its $21,837 of post-limitation itemized deductions includes the unreimbursed employee business expenses that the allowed $15,338.79 excludes, leaving taxable income $8,993 low at $512,433. Its $258 exemption credit also exceeds the $156.93 that survives phaseout at this AGI." -us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Applied the federal $10,000 SALT cap to California, which has no SALT cap — the full $17,280 of real estate taxes enters the itemized base — and layered on mortgage interest for the $2,850,000 balance for which no interest amount is listed, concluding that taxable income was eliminated. California taxable income is $521,425.72 and the tax before refundable credits is $41,051.51, not zero." -us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Two errors ran in opposite directions: it kept the $4,148 state tax refund in AGI at $538,418 rather than $536,764.50, and it over-applied the 6% cutback to leave only $11,912 of itemized deductions against the correct $15,338.79. The net leaves taxable income of $526,500, some $5,074 above the reference's $521,425.72." -us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Its own stated $565,375 of taxable income produces about $45,300 under the 2026 CA MFJ schedule, not the $33,544 it reported — its bracket arithmetic understates the tax by roughly $12,000. It also took an $11,440 standard deduction instead of the $15,338.79 of limited itemized deductions and an $892 exemption credit instead of the phased-down $156.93." -us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its own $495,603 of taxable income yields about $38,800 under the 2026 CA MFJ rate schedule, not the $29,423 it submitted — a bracket-arithmetic shortfall of roughly $9,400. It also used the California standard deduction when the limited itemized total of $15,338.79 controls on $536,764.50 of CA AGI." +us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $22,800 of mortgage interest on the first $1M of the $2.85M balance, but no interest is listed and unlisted amounts are $0. It also counted charity and employee expenses, which cut taxable income to $490,509 instead of $521,425.72. Its own bracket math on that base came to about $38,832, and the $45,856 it submitted is an unexplained upward 'refinement'; it also set the $156.93 exemption credits to zero." +us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 traditional 401(k) request even though each spouse's combined traditional+Roth request of $27,240 exceeds the $24,500 limit, so its CA AGI of $532,106 is $4,658.50 below $536,764.50. It also itemized about $21,560 including charity and employee expenses, instead of the $17,280 real-estate-tax base limited to $15,338.79. That left taxable income about $10,880 short of $521,425.72." +us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a California deduction for federal income tax paid that does not exist, plus employer health premiums, and used an invented 12% rate above $325,000. It then took off about $21,621 of made-up 'standard deduction credit' and other nonrefundable credits. The only nonrefundable credit here is the phased-down MFJ exemption credit, subtracted from the tax on $521,425.72 of taxable income: $156.93 from $41,208.45 on the frozen reference's projected 2026 amounts, or $150 from $41,369.87 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $60,000 of mortgage interest on the $1M-limited share of the $2.85M balance although no interest is listed. That raised itemized deductions to $79,854 instead of $15,338.79 and cut taxable income to $453,116 instead of $521,425.72. It lands near the target only because its bracket math overstated tax on that base (~$38,800 vs ~$34,856 on the frozen reference's projected 2026 schedule and ~$35,017 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), and it used $560 of exemption credits instead of $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to CA income and imputed about $35,000 of unlisted mortgage interest, reaching about $595,000 of taxable income instead of $521,425.72. It then reported $36,767, far below what the California schedule gives on its own $595,000 base (about $48,050 on the frozen reference's projected 2026 schedule, and slightly more on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026). Both its income base and its use of the rate schedule are wrong." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put taxable income at about $560,000 by partly including the $70,000 tax-exempt pension and imputing mortgage interest; the correct base is $521,425.72. It also applied rates up to 11.3%, but the 10.3% MFJ bracket starts above roughly $740,000. At this income every marginal dollar is taxed at 9.3%, giving $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026." +us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 of traditional 401(k) deferrals although each spouse's $27,240 combined traditional+Roth request exceeds the $24,500 limit, so its CA AGI of $532,106 is $4,658.50 short of $536,764.50. It also itemized $22,454 including charity and employee expenses, instead of the $15,338.79 left from the $17,280 real-estate-tax base after the 6% limitation. Its taxable income of $509,652 is $11,774 below $521,425.72." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It set CA AGI at $534,270, $2,494.50 below $536,764.50, by excluding the full $46,308 traditional 401(k) request despite the $24,500 per-person combined-deferral cap, net of the $2,164 IRA deduction. It itemized $23,017 including charity and employee expenses and never applied the 6% limitation, though AGI exceeds the $504,411 threshold; the correct figure is $15,338.79. It also used an unphased $500 exemption credit instead of $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It imputed about $50,000 of mortgage interest (5% on $1M of the $2.85M balance) although no interest is listed and unlisted amounts are $0. That cut taxable income to $464,416 instead of $521,425.72. Its bracket math then understated tax even on that base (~$35,900 on the frozen reference's projected 2026 schedule and ~$36,068 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026, not $32,000), and it ignored the $156.93 exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It used about $534k of CA AGI by excluding the full $46,308 401(k) request despite the $24,500 combined-deferral cap. It itemized about $19.9k of property tax plus charity with no 6% high-income limitation instead of $15,338.79, leaving taxable income around $514k rather than $521,425.72. It also treated the exemption credits as fully phased out, when some remain after the $6-per-$2,500 reduction: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $518,564 is within about $2,900 of $521,425.72; it started from federal AGI of $538,418 without removing the $4,148 refund and deducted $19,854 of itemized deductions with no limitation. But its bracket math produced $42,151 where the frozen reference's projected 2026 MFJ schedule gives about $40,942 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,104. It also left out the $156.93 of phased-down exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It taxed only $497,700, which is $23,726 below $521,425.72. That figure matches CA AGI of $520,901, built by treating the $13,369 of employer insurance premiums as a wage exclusion, minus about $23,200 of itemized deductions with charity and employee expenses and no limitation. It also used $300 of exemption credits instead of the phased-down $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $512,889 is $8,537 below $521,425.72. That gap fits excluding the full traditional 401(k) request despite the $24,500 combined-deferral cap and deducting more than the $15,338.79 of limited real-estate-tax itemized deductions. It also said no nonrefundable credits apply at this AGI, but MFJ exemption credits survive the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference." +us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short of $536,764.50, because it excluded the full $46,308 traditional 401(k) request although each spouse's $27,240 combined request exceeds the $24,500 limit (net of the $2,164 IRA deduction). Its $18,663 of itemized deductions after limitation adds charity to the $17,280 real-estate-tax base; the correct figure is $15,338.79. Its $310 exemption credit ignores the $6-per-$2,500 phase-down, which leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference)." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It deducted mortgage interest on the $2.85M balance, for $73,017 of itemized deductions, although no mortgage interest is listed and unlisted amounts are $0. The itemized base is the $17,280 of real estate taxes limited to $15,338.79. Its mistake cut taxable income to $461,253 instead of $521,425.72, and it used a $288 exemption credit instead of $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $31,215 matches about $415,700 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $413,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $106,000 to $108,000 below $521,425.72. That fits imputing a large mortgage-interest deduction on the $2.85M balance or dropping investment income. The frozen reference taxes $521,425.72 at up to 9.3% for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $501,848 of taxable income equals federal AGI minus $13,369 of employer insurance premiums (treated as a wage exclusion) minus $23,201 of itemized deductions with charity, employee expenses and no limitation. Employer premiums are not subtracted, and the itemized base is $17,280 of real estate taxes limited to $15,338.79, giving $521,425.72. That left its tax about $856 short." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer insurance premiums as if they were pre-tax wage reductions and excludes the full $46,308 401(k) request; the correct CA AGI is $536,764.50. It also itemized $19,854 (property tax plus charity) with no 6% high-income limitation instead of $15,338.79. Its taxable income ended about $20,400 too low." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $33,816 matches about $443,600 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $441,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $78,000 to $80,000 below $521,425.72. That fits over-deducting, such as mortgage interest on the $2.85M balance. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI without removing the $4,148 state refund and itemized property tax, charity and employee expenses with no 6% limitation, giving about $515,500 of taxable income. The frozen reference's projected 2026 MFJ schedule taxes that at about $40,650 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 at about $40,820, so its $42,150 overshoots by roughly $1,300 to $1,500 through misapplied bracket thresholds. It also subtracted none of the $156.93 exemption credits." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its reasoning mentions 'pre-tax deductions' plus standard/itemized deductions, and $36,918 matches about $477,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $475,200 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $44,400 to $46,300 below $521,425.72. That fits excluding employer health premiums and claiming extra itemized deductions. The correct base is $536,764.50 of CA AGI minus $15,338.79 of limited real-estate-tax itemized deductions." +us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270 (the full $46,308 401(k) exclusion, missing the $24,500 combined-deferral cap) and itemized real estate taxes plus charity and employee expenses, reaching about $515,000 of taxable income instead of $521,425.72. It then reported a round $40,000 rather than the ~$40,450 its own base gives after the $156.93 exemption credits on the frozen reference's projected amounts (about $40,620 after $150 of credits on the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026)." +us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI and deducted the full $10,911 net capital loss instead of the $3,000 limit. It then invented a $2,000 CA capital-loss cap, producing CA AGI of $449,439 instead of $536,764.50. It also took an ~$11,000 standard deduction where $15,338.79 of limited itemized real estate taxes applies." +us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $522,970 is within roughly $1,550 of $521,425.72, but its bracket math gave $41,942 where the frozen reference's projected 2026 MFJ schedule yields about $41,352 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,513. It also claimed no nonrefundable credits, when phased-down exemption credits apply: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference. It took the standard deduction even though the $15,338.79 of limited itemized real estate taxes exceeds the $11,412 standard deduction." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It offered only a generic statement, and $20,677 matches about $302,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $300,500 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026). That drops roughly $219,000 from the correct $521,425.72 base, which is wages net of capped 401(k) deferrals, $65,936 of interest and $87,642 of dividends taxed as ordinary income, minus the $3,000 capital loss, the $4,148 refund and $15,338.79 of itemized deductions." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no concrete derivation, and $37,099 matches about $479,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $477,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $42,500 to $44,300 below $521,425.72. That fits over-deducting, such as imputed mortgage interest or excluded employer health premiums. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It cut wages by the $13,369 of employer insurance premiums as if they were pre-tax, on top of the full $46,308 traditional 401(k) request, and itemized beyond the $17,280 real-estate-tax base. Its result matches about $497,600 of taxable income instead of $521,425.72. That left its tax about $2,218 short." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of about $519,427 is $1,999 below $521,425.72, so its combination of CA AGI and itemized deductions understated the base (correct: $536,764.50 minus $15,338.79). The resulting ~$186 of missing 9.3% tax, partly offset by its bracket and credit estimates, left it $27.51 short of $41,051.51." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its answer is $98.49 too high, which equals about $1,059 of excess taxable income at 9.3%. Its 'California additions' step adds income that is not in CA AGI. CA AGI is federal AGI minus the $4,148 refund, $536,764.50, with no add-back of the $8,961 tax-exempt interest or the pension." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It added the $8,961 of tax-exempt interest to CA income, but that interest is not part of the $536,764.50 CA AGI. Its full $46,308 401(k) exclusion and its itemized deductions with charity and employee expenses more than offset that addition. Taxable income ended about $9,000 below $521,425.72 and the tax $885.51 short." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer insurance premiums from wages and excluded the full $46,308 401(k) request, giving CA AGI of $520,901 instead of $536,764.50. It itemized $22,294.58 (charity plus employee expenses above 2%, limited), which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading, where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing the $15,338.79 real-estate-tax-only figure; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference. It also used the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 exemption credit is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It kept the $4,148 state tax refund in CA income, although California subtracts it. It also took the standard deduction instead of itemizing the $17,280 of real estate taxes, which are limited to $15,338.79 and still exceed the $11,412 standard deduction. Both errors pushed taxable income above $521,425.72 and left the answer $738.49 too high." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $492,027, which is $29,399 below $521,425.72, because its estimated deductions and exclusions exceed the actual $15,338.79 of limited real-estate-tax itemized deductions taken against CA AGI of $536,764.50. Its bracket and exemption-credit math on that base is consistent with the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($38,635.79 of tax on $492,027 before credits) as well as with the frozen reference's projected schedule, so the whole miss comes from the understated income base." +us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its CA AGI of $520,901 comes from subtracting the $13,369 of employer insurance premiums from wages and excluding the full $46,308 401(k) request; the correct AGI is $536,764.50. It itemized $22,294.58 including charity and employee expenses, which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading ($17,280 of real estate taxes, $2,574 of charity and $3,429.98 of employee expenses above 2% of AGI, less the $989.40 high-income limitation, 6% of AGI above $504,411), where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing $15,338.79; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference. It also held the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 of credits is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $48,720 matches about $604,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $602,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $80,600 to $82,500 above $521,425.72. That fits adding the $70,000 tax-exempt private pension and the $8,961 tax-exempt interest to CA income. Neither is part of the $536,764.50 CA AGI." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short, by excluding the full $46,308 401(k) request despite the $24,500 combined-deferral cap. It kept charity and 2%-floor employee expenses in the itemized base before the 6% cutback, reaching about $512,000 of taxable income instead of $521,425.72. It also used about $280 of exemption credits instead of what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It itemized $22,934 (property tax, charity, and employee expenses over the 2% floor) with no 6% high-income limitation. The correct figure is $15,338.79: the $17,280 real-estate-tax base cut by 6% of AGI over $504,411. Taxable income therefore fell to $515,484 instead of $521,425.72, even though its $538,418 AGI wrongly kept the $4,148 state refund that California subtracts." +us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the full $46,308 of traditional 401(k) deferrals, giving CA AGI of $534,270 instead of $536,764.50, although each spouse's $27,240 combined request exceeds the $24,500 limit. It added $2,574 of charity to the $17,280 real-estate-tax base, for $18,100 of itemized deductions instead of $15,338.79, leaving taxable income of $516,170, $5,256 low. It also claimed $314 of exemption credits where the $6-per-$2,500 phase-down leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference)." +us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer insurance premiums from federal AGI, giving CA AGI of $525,049 instead of $536,764.50. It itemized $23,201 with charity and employee expenses and no 6% limitation instead of $15,338.79, so taxable income was $501,848 rather than $521,425.72. It also applied no exemption credits instead of $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and $8,961 of tax-exempt interest to CA AGI and subtracted employer health premiums, reaching $604,010 instead of $536,764.50. Neither tax-exempt item belongs in CA AGI. With about $21,622 of itemized deductions and no high-income limitation, its $582,388 of taxable income is about $61,000 too high." +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so there was nothing to score. The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts, giving $41,051.51; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40." +us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used CA AGI of $534,270, $2,494.50 short, because it excluded the full $46,308 traditional 401(k) request despite the $24,500 combined-deferral cap. It itemized $21,837 after limitation with charity and employee expenses included instead of $15,338.79, giving taxable income of $512,433 rather than $521,425.72. Its $258 exemption credit is also larger than what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference." +us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap, which California does not apply to property tax, imputed unlisted mortgage interest, and concluded that deductions wipe out all tax. With CA AGI of $536,764.50 and only $15,338.79 of itemized deductions, taxable income is $521,425.72. Tax is $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026." +us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI of $538,418 without subtracting the $4,148 state tax refund that California excludes. It also over-applied the high-income limitation, cutting its $19,854 itemized total to $11,912; the actual 6% reduction on AGI over $504,411 is only $1,941.21 on the $17,280 real-estate-tax base, giving $15,338.79. Its taxable income of about $526,500 is therefore about $5,074 too high." +us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It skipped the traditional 401(k) exclusion, deducted the full $10,911 net capital loss instead of the $3,000 limit, and kept the $4,148 refund. It then claimed about $33,544 of tax on $565,375 of taxable income, where the frozen reference's projected 2026 MFJ schedule gives about $45,300 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $45,457. It also subtracted an inflated $892 of exemption credits instead of $156.93." +us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the standard deduction instead of the $15,338.79 of limited itemized real estate taxes and reached $495,603 of taxable income. It then computed only $29,423 of tax on that amount, while the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $38,968: $6,403.94 through the 8% bracket plus 9.3% above $145,448 (the frozen reference's projected schedule gives about $38,800: about $6,549 through the 8% bracket plus 9.3% above roughly $148,700). The rate-schedule error is the bigger of the two." us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It abandoned its own IRC §86 analysis and declared SSDI fully non-taxable, stopping AGI at the $35,280 pension; provisional income of $44,922.50 ($35,280 + half of $19,285) exceeds the $34,000 single second-tier threshold, so $13,784.12 of benefits is taxable. It also used the 2024 standard deduction of $14,600 rather than the 2026 amount of $16,100, and the $2,559 it submitted does not follow from either bracket schedule it wrote down." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its first pass reproduced the reference exactly — taxable SS $13,784, AGI $49,064, $16,100 standard deduction, taxable income $32,964, tax $3,713 — and it then discarded that result, second-guessed the standard deduction to $16,600, and submitted $2,386, a number none of its steps produce. The $16,100 standard deduction and the $12,400 top of the 10% bracket it used first are the correct 2026 parameters." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It took the 85% ceiling ($16,392.25) as the taxable Social Security amount instead of the §86 lesser-of test, which caps taxable benefits at 0.85×($44,922.50−$34,000)+$4,500 = $13,784.12, inflating AGI to $51,672 and taxable income to $35,572. It then computed $4,021 from that inflated base and submitted $1,858, which corresponds to no step in its work." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived the reference chain correctly — taxable SS $13,784 under the §86 second tier, AGI $49,064, $16,100 standard deduction, taxable income $32,964, tax ≈$3,708 — and then wrote $2,534 into the value field, discarding its own correct result." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a flat 85% of gross benefits ($16,392.25) as taxable Social Security rather than the §86 lesser-of amount 0.85×($44,922.50−$34,000)+$4,500 = $13,784.12, over-including $2,608 of income. It compounded this by explicitly falling back to 2025 parameters — a $15,000 standard deduction and an $11,925 top of the 10% bracket — where 2026 uses $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It replaced the §86 lesser-of result of $13,784.12 with an invented ""worksheet approximation"" of $15,650 taxable Social Security, computed $3,941 from it, and then submitted $1,611, a figure unconnected to any line of its arithmetic." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It reached the correct AGI of $49,064 via the two-tier §86 formula, then subtracted a $15,400 standard deduction and used a $12,250 top of the 10% bracket; the 2026 values are $16,100 and $12,400. The $700 of extra taxable income at 12% plus the $150 bracket shift at 2 points is exactly its $87 overstatement." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"AGI of $49,064 is right, but it used a $15,300 standard deduction and a $12,200 top of the 10% bracket instead of the 2026 figures of $16,100 and $12,400. The $800 of over-taxed income at 12% plus the $200 bracket shift at 2 points produces precisely its $100 excess over $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used the 85% ceiling ($16,392.25) as taxable Social Security instead of the §86 lesser-of amount $13,784.12, and paired that with a $15,400 standard deduction rather than $16,100, overstating taxable income by $3,308 ($36,272.25 against $32,964.12)." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed taxable Social Security of $13,784 and AGI of $49,064 correctly, then applied a $15,700 standard deduction and an $11,925 top of the 10% bracket; the 2026 single standard deduction is $16,100 and the 10% bracket runs to $12,400, which together account for its $57 overstatement." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It filed this single 56-year-old as head of household and added the age-65 additional standard deduction; head-of-household status requires a qualifying dependent and none is listed, and the additional deduction requires age 65. It further subtracted unnamed nonrefundable credits, none of which a childless filer with $49,064 AGI qualifies for; the correct chain is $49,064 less the $16,100 single standard deduction taxed at 10%/12% for $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It modeled 2026 as a post-TCJA-sunset year, restoring the personal exemption and a pre-TCJA standard deduction. The 2025 reconciliation law made the TCJA structure permanent: 2026 gives a single filer a $16,100 standard deduction, no personal exemption, and a 12% bracket starting above $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"With the correct AGI of $49,064, it subtracted an $8,300 standard deduction plus a $5,150 personal exemption — pre-TCJA-sunset parameters that do not apply in 2026 — and taxed the result at pre-TCJA rates. The 2026 single standard deduction is $16,100 with no personal exemption, and the top of the 10% bracket is $12,400, giving $32,964.12 of taxable income and $3,707.70 of tax." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation and subtracted unspecified nonrefundable credits. The correct chain — $35,280 pension plus $13,784.12 of taxable Social Security, less the $16,100 standard deduction, taxed at 10% to $12,400 and 12% above — gives $3,707.70; its $3,212 corresponds to roughly $4,100 more income sheltered than the standard deduction allows, and no nonrefundable credit is available to this childless filer." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It carried the correct taxable Social Security of $13,784.13 and AGI of $49,064.13, then subtracted a standard deduction near $15,300 instead of the 2026 single amount of $16,100, leaving about $830 of extra income taxed at 12% and producing the $99 overstatement." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Its explanation derives the reference chain and states the answer as $3,707.70, but the submitted numeric value is $3,744 — it entered a number its own worksheet never produced, breaking the requirement that the explanation match the value." -us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"AGI of $49,064 and taxable Social Security of $13,784 are correct, but it used a $15,350 standard deduction and a $12,200 top of the 10% bracket rather than the 2026 values of $16,100 and $12,400, taxing an extra $750 at 12% and shifting $200 from 10% to 12%." -us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It added the $284 of tax-exempt private pension into provisional income, which under §86 admits only tax-exempt interest, pushing taxable benefits to $14,025.53 instead of $13,784.13. The dominant error is its use of 2024 parameters — a $14,600 standard deduction and an $11,600 top of the 10% bracket — where 2026 uses $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It applied the first-tier §86 rule, 0.5×(provisional income − $25,000), even though provisional income of $44,922.50 exceeds the $34,000 second-tier threshold, understating taxable benefits at $9,961.25 instead of 0.85×($44,922.50−$34,000)+$4,500 = $13,784.12. It also used a $15,750 standard deduction and a $12,000 bracket top rather than the 2026 $16,100 and $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted the standard deduction offsets the income. AGI is $49,064.12 — the $35,280 pension plus $13,784.12 of taxable Social Security — and the $16,100 single standard deduction leaves $32,964.12 taxable, producing $3,707.70; reaching zero requires about $33,000 of additional deductions this household does not have." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed unnamed nonrefundable credits zero out the liability. No nonrefundable credit applies to a childless 56-year-old with $49,064 of AGI and no dependent-care, education, or foreign-tax items, so the tax on $32,964.12 of taxable income stands at $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Every step matches the reference except the standard deduction: it used the 2025 single amount of $15,750 rather than the 2026 amount of $16,100, leaving $350 of extra income taxed at 12% — exactly the $42 by which its $3,750 exceeds $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $35,572 of taxable income is the reference $32,964 plus $2,608, the amount by which taking a flat 85% of benefits ($16,392) overstates the §86 lesser-of result of 0.85×($44,922.50−$34,000)+$4,500 = $13,784.12. That single substitution is its only deviation from the correct chain." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It added the $284 of tax-exempt private pension into provisional income; §86 adds back tax-exempt interest only, and the engine's provisional income excludes tax-exempt pension. That raised taxable benefits by $241.40 to $14,025.52 and taxable income to $33,205.52, producing $3,736.66, which it rounded to $3,737." -us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,736.66 is exactly the result of adding the $284 of tax-exempt private pension into provisional income, which lifts taxable Social Security from $13,784.12 to $14,025.52; §86 adds back tax-exempt interest only, so provisional income here is $44,922.50 and taxable income is $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $2,444 implies about $22,400 of taxable income against the $16,100 standard deduction — an AGI near $38,500, consistent with excluding roughly three-quarters of the SSDI. Provisional income of $44,922.50 exceeds the $34,000 second-tier threshold, making $13,784.12 of benefits taxable and taxable income $32,964.12." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly projected a post-TCJA-sunset 2026 — an $8,535 standard deduction, a $5,443 personal exemption, and a 15% second bracket. 2026 law retains the TCJA structure: a $16,100 single standard deduction, no personal exemption, and 12% above $12,400, so its correct AGI of $49,064 yields $32,964.12 of taxable income and $3,707.70 of tax." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Like its sibling, it applied post-TCJA-sunset parameters — an $8,496 standard deduction plus a $5,419 personal exemption and a 15% rate above the 10% bracket. The personal exemption remains repealed in 2026, the single standard deduction is $16,100, and the second bracket rate is 12%." -us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a flat 85% of benefits ($16,392) as taxable Social Security instead of the §86 lesser-of amount $13,784.12, lifting AGI to $51,672, and additionally used a $15,420 standard deduction rather than $16,100. Together these overstate taxable income by $3,288." -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so there is nothing substantive to evaluate. The required chain — $35,280 of pension plus $13,784.12 of taxable Social Security, less the $16,100 standard deduction, taxed at 10% through $12,400 and 12% above — yields $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It got AGI right at $49,064.13, but its $4,708.37 implies roughly $41,300 of taxable income — a combined deduction and exemption near $7,800 with a 15% second bracket, i.e. pre-TCJA-sunset parameters. 2026 gives a $16,100 single standard deduction, no personal exemption, and 12% above $12,400." -us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It excluded the SSDI entirely, stating AGI of about $35,280; §86 makes $13,784.12 of the $19,285 in benefits taxable because provisional income of $44,922.50 exceeds the $34,000 single threshold. Its bracket math also uses a fabricated $1,540 base, where the 2026 tax on the first $12,400 is $1,240." -us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It folded the $284 of tax-exempt private pension into provisional income — §86 adds back tax-exempt interest, not tax-exempt pension — and then misadded its own components ($35,280 + $284 + $9,643 = $45,207, not the $45,406 it used). The resulting $14,196 of taxable benefits overstates the correct $13,784.12 by $412, and 12% of that is the full $49 gap from $3,707.70." -us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used the $6,000 second-tier add-back, which applies to joint filers, instead of the $4,500 single amount, and included the $284 tax-exempt pension in combined income, producing $15,525.53 of taxable benefits rather than $13,784.12. It also used a $15,750 standard deduction where 2026 provides $16,100." -us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It omitted the SSDI from income entirely, leaving AGI at $35,280 when $13,784.12 of benefits is taxable under the §86 second tier, and used a $15,750 standard deduction with a fabricated 12.5% rate. It then submitted $4,202 while its explanation computes $2,177, so the value matches neither its own work nor the correct $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $19,285 SSDI as non-taxable, but SSDI is a Social Security benefit taxed under IRC §86, so $13,784.12 belongs in AGI. It also used the outdated 2024 $14,600 standard deduction instead of the 2026 $16,100. Its taxable income was therefore $20,680 instead of $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Its own derivation was nearly right: AGI $49,064, taxable income $32,964 and tax of about $3,713, off only because its 10% bracket ended at $12,150 instead of $12,400. It then swapped in a guessed $16,600 deduction and finally submitted $2,386, a number its reasoning never produced." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It taxed the full 85% cap ($16,392) of the SSDI. §86 taxes the lesser of that cap and 0.85×($44,922.50 − $34,000) + $4,500 = $13,784, so this overstated AGI by $2,608. It then abandoned its own $4,021 result and submitted an unsupported $1,858." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It derived the correct figures (taxable SS $13,784, AGI $49,064, taxable income $32,964, tax about $3,708 using the $12,400 10% bracket). It then submitted $2,534, a value that contradicts its own completed calculation." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed that provisional income above $34,000 makes a flat 85% of benefits ($16,392.25) taxable. It skipped the §86 lesser-of formula, which gives $13,784.12, and it wrongly added the tax-exempt pension to provisional income. It also used the 2025 pre-OBBBA $15,000 standard deduction and $11,925 bracket instead of the 2026 $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It replaced the exact §86 result of $13,784 with a made-up 'worksheet approximation' of $15,650 and used the 2025 $11,925 bracket, reaching $3,941. It then submitted $1,611, which its reasoning does not support." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it estimated the 2026 single standard deduction at $15,400 instead of the OBBBA-indexed $16,100. It also ended the 10% bracket at $12,250 instead of $12,400, which overstated taxable income by $700." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used a $15,300 standard deduction instead of the 2026 $16,100 and ended the 10% bracket at $12,200 instead of $12,400. Taxable income came out at $33,764 instead of $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It counted the full 85% cap ($16,392.25) as taxable SS. The §86 lesser-of formula, 0.85×($44,922.50 − $34,000) + $4,500, gives $13,784.12. It also used a guessed $15,400 standard deduction and a $12,225 bracket instead of $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied pre-TCJA rules for 2026: an $8,100 standard deduction plus a $5,300 personal exemption, and a 15% bracket. OBBBA (P.L. 119-21) made the TCJA structure permanent, so the correct figures are a $16,100 deduction, no exemption, and 10%/12% brackets." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used an estimated $15,700 standard deduction instead of $16,100. It also taxed only $11,925 at 10% (the 2025 bracket) instead of $12,400, which pushed the tax up to $3,765." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied a head-of-household standard deduction with the age-65-or-older add-on and subtracted nonrefundable credits. The Head is a 56-year-old single filer, so the deduction is $16,100 and no nonrefundable credit applies, which leaves $3,707.70 of tax." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly applied pre-TCJA rules for 2026, with a standard deduction plus a personal exemption. OBBBA made the $16,100 single standard deduction, the zero personal exemption and the 10%/12% brackets permanent, and those give $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied TCJA-sunset rules: an $8,300 standard deduction plus a $5,150 personal exemption and pre-TCJA brackets. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent for 2026." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its explanation subtracts nonrefundable credits, but none apply here. The credit for the elderly or disabled is fully phased out: its $5,000 base is reduced by $5,501 of nontaxable SS and by half of AGI over $7,500. Its $3,212 is $496 below the zero-credit result of $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 was correct, but its $3,807.18 is $99.48 too high. That gap matches a standard deduction of about $15,300 instead of the 2026 $16,100, which put taxable income near $33,760 instead of $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation reaches the correct $3,707.70 from AGI of $49,064.13, the single standard deduction and the 2026 brackets. It nevertheless submitted $3,744 in the value field, contradicting its own stated result." +us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used a $15,350 standard deduction instead of $16,100 and ended the 10% bracket at $12,200 instead of $12,400. Taxable income came out at $33,714 and tax at $3,801.68." +us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used 2024 parameters (a $14,600 standard deduction and an $11,600 10% bracket) instead of the 2026 $16,100 and $12,400. It also added the $284 tax-exempt pension to provisional income, which inflated taxable SS to $14,025.53 instead of $13,784.12." +us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It applied only the first-tier 50% formula, min(0.85×SS, 0.5×(PI − $25,000)) = $9,961.25. Because provisional income exceeds $34,000, the second tier applies: 0.85×(PI − $34,000) + $4,500 = $13,784.12. It also used the 2025 $15,750 deduction and a $12,000 bracket instead of $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction offsets the income and reported $0. The $16,100 deduction against $49,064.12 of AGI leaves $32,964.12 taxable, which produces $3,707.70 of tax." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that nonrefundable credits reduce the tax to $0, but no nonrefundable credit applies. The single filer has no dependents, and the elderly-or-disabled credit is fully phased out by $49,064 AGI and $5,501 of nontaxable SS. The $3,707.70 of tax on $32,964.12 of taxable income stands in full." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it used the 2025 OBBBA standard deduction of $15,750 instead of the inflation-indexed 2026 amount of $16,100. That overstated taxable income by $350 and tax by $42." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,572 implies it counted the full 85% cap ($16,392) of the SSDI as taxable. The §86 lesser-of formula caps taxable SS at $13,784.12, which gives taxable income of $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $33,206 is exactly what results from adding the $284 tax-exempt pension to provisional income, which gives taxable SS of $14,025.53. §86 modified AGI adds back only tax-exempt interest, so taxable SS is $13,784.12 and taxable income is $32,964.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,736.66 equals $1,240 + 12% × ($33,205.53 − $12,400), which means it added the $284 tax-exempt pension to provisional income and got $14,025.53 of taxable SS. The pension is not tax-exempt interest under §86(b)(2), so the correct taxable SS is $13,784.12." +us,scenario_007,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took taxable SS as the full 85% cap of $16,392, giving AGI of $51,672. It should have applied the §86 lesser-of formula, 0.85×($44,922.50 − $34,000) + $4,500 = $13,784.12, which gives AGI of $49,064.12." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $2,444 implies taxable income of about $22,400, roughly $10,500 below the correct $32,964.12. That shortfall matches leaving out most of the $13,784.12 of taxable SSDI required by §86." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it projected post-TCJA-sunset 2026 law: an $8,535 standard deduction, a $5,443 personal exemption and 10%/15% brackets. OBBBA made the $16,100 deduction and the 10%/12% brackets permanent." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it applied post-TCJA-sunset rules: an $8,496 standard deduction, a $5,419 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, with a $16,100 deduction and 10%/12% rates." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064 was correct, but it treated 2026 as post-TCJA-sunset, with a $5,450 personal exemption, an $8,550 standard deduction and a 15% bracket. OBBBA permanently extended the TCJA rates and the $16,100 single deduction." +us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It counted the full 85% cap ($16,392) as taxable SS instead of the §86 lesser-of amount, $13,784.12. It also used a guessed $15,420 standard deduction and a $12,259 bracket instead of $16,100 and $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $49,064.13 was correct, but it applied a 2026 'deduction/exemption' pair under pre-TCJA law, and its $4,708.37 matches a roughly $13,650 deduction-plus-exemption taxed with a 15% bracket. OBBBA-permanent law uses a $16,100 deduction with 10%/12% brackets, which gives $3,707.70." +us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It set AGI to the $35,280 pension alone and left out the $13,784.12 of SSDI that §86 makes taxable. It also misstated first-bracket tax as $1,540 instead of $1,240 on the first $12,400." +us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added the $284 tax-exempt pension to combined income, and it summed the components to $45,406 when they total $45,206.50. That produced taxable SS of $14,196 instead of $13,784.12, and AGI and tax came out $412 and $49 too high." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"In the second-tier §86 formula it used $6,000, the married-filing-jointly figure, instead of the single filer's min($4,500, 50% of benefits). It also added the tax-exempt pension to combined income, which gave $15,525.53 of taxable SS instead of $13,784.12. It then used the 2025 $15,750 deduction instead of $16,100." +us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left the $13,784.12 of taxable SSDI out of income and used a nonexistent 12.5% rate, computing $2,177. It then submitted $4,202, which contradicts its own arithmetic." us,scenario_007,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"It correctly identified that the head is 56 and below the age-65 threshold, then overrode that with an SSDI disability pathway that `is_medicare_eligible` does not contain — the variable tests age against the Medicare age threshold alone. It compounded the error by assuming the 24-month SSDI entitlement waiting period was satisfied, a fact the household sheet never states and which the prompt's treat-unlisted-status-as-false rule forbids inventing." -us,scenario_007,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It treated the presence of Social Security disability income as conferring Medicare eligibility outright, skipping the age test entirely. `is_medicare_eligible` turns solely on age ≥ 65, and the head is 56, so the receipt of $19,285 in SSDI leaves the variable False." -us,scenario_007,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It applied the real-world SSDI 24-month Medicare entitlement rule and treated the prompt's constant-status stipulation as proof the waiting period had run. The constancy stipulation fixes the listed facts across the year; it does not create an unlisted entitlement history, and `is_medicare_eligible` has no disability branch — a 56-year-old fails its age-65 test." -us,scenario_007,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It stated the age-65 threshold is waived after 24 months of SSDI receipt and applied that waiver here. The modeled `is_medicare_eligible` variable has no such waiver: it evaluates age against the Medicare age threshold only, and the head's age of 56 yields False." -us,scenario_007,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"It asserted that PolicyEngine treats disabled SSDI recipients as Medicare eligible, attributing a disability pathway to `is_medicare_eligible` that the variable does not implement — eligibility is the age ≥ 65 test alone. It also inferred disability status from a disability income amount, when the prompt requires unlisted status flags to be treated as false." -us,scenario_007,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It recited the SSDI 24-month waiting period as a rule that qualifies a person 'regardless of age' and then assumed the waiting period had been passed because disability income was present all year. `is_medicare_eligible` applies only the age-65 threshold, so the head at 56 is not eligible, and the assumed entitlement history is an unlisted fact the prompt bars." -us,scenario_007,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It read the Social Security disability income line as a disability-based Medicare qualification and never applied the age test. Medicare eligibility in the model is age ≥ 65 with no disability route, and the head is 56." -us,scenario_007,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It explicitly waived the age requirement — 'despite being under age 65' — on the strength of SSDI receipt. The modeled `is_medicare_eligible` variable contains no under-65 disability pathway, so the 56-year-old head fails the sole age-65 test." -us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,It equated receipt of SSDI with Medicare eligibility and produced no age analysis at all. The correct derivation compares the head's age of 56 against the Medicare age threshold of 65 and returns False; the model's answer is consistent with substituting a disability-income test for the age test. -us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It concluded eligibility from SSDI receipt alone, omitting the age-65 threshold that is the only condition `is_medicare_eligible` evaluates. At age 56 the head fails that threshold, and no disability-based branch exists to rescue eligibility." -us,scenario_007,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It invoked 'disability rules' as an alternative Medicare pathway triggered by SSDI income. No such pathway is modeled: eligibility is age ≥ 65, the head is 56, and disability status itself is an unlisted boolean the prompt requires to be false." -us,scenario_007,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It jumped from 'head receives SSDI' straight to Medicare qualification without testing age. The head's age of 56 is below the age-65 threshold that alone governs `is_medicare_eligible`, so the value is False." -us,scenario_007,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"It claimed that Medicare eligibility 'under PolicyEngine rules' includes SSDI recipients, ascribing a disability pathway to the model that does not exist. The variable applies the age-65 threshold only, so the 56-year-old head with $19,285 of disability income is not eligible." -us,scenario_007,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It correctly noted that the head is not age-eligible at 56, then discarded that conclusion by asserting that PolicyEngine bases Medicare eligibility on disability status. Medicare eligibility in the model is the age test alone, and a disability-income amount is not a disability status flag under the prompt's treat-unlisted-as-false rule." -us,scenario_007,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,It acknowledged the head is under 65 and then overrode the age test with a disability-grounds pathway it attributed to PolicyEngine. `is_medicare_eligible` implements no disability grounds; at age 56 the head fails the age-65 threshold and the value is False. -us,scenario_007,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It applied a 'disability-based Medicare rule' triggered by the presence of Social Security disability income. The modeled variable has only the age-65 condition, so a 56-year-old is not Medicare eligible however the disability income is characterized." -us,scenario_007,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"It named a 'modeled Medicare disability pathway' keyed to positive Social Security disability benefits, a pathway `is_medicare_eligible` does not contain. The variable's sole condition is age ≥ 65, which the head fails at 56." -us,scenario_007,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"It attributed an SSDI-based qualification to PolicyEngine's rules and never evaluated age. Eligibility is determined by the age-65 threshold alone; the head is 56, so the correct value is 0." -us,scenario_007,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"It recognized the head is 56 but applied the SSDI 24-month waiting-period rule to grant eligibility, assuming an entitlement duration the household facts never state. The modeled variable contains no waiting-period or disability branch — only the age-65 test, which the head fails." -us,scenario_007,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"It explicitly set aside the under-65 age because disability income was present, applying a disability-based Medicare route. `is_medicare_eligible` has no such route; the age-65 threshold alone controls and the head at 56 is not eligible." -us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"It converted the Social Security disability income line into a disability-based Medicare qualification and skipped the age test. The head's age of 56 fails the age-65 threshold that is the only condition the variable evaluates, so the value is 0." +us,scenario_007,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model correctly noted that the head is under 65, but it assumed the 24-month SSDI waiting period was met because of the ""constant full-year status"" rule. A constant status over one tax year shows at most 12 months of receipt. No listed fact shows 24 months of entitlement, and the prompt treats unlisted status inputs as false, so neither the disability pathway nor the age pathway applies." +us,scenario_007,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model said SSDI income by itself confers Medicare eligibility and skipped the 24-month entitlement requirement for disability-based Medicare. No listed fact satisfies that requirement, and at 56 the head fails the age-65 test, so the answer is not eligible." +us,scenario_007,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model named the 24-month waiting period and then treated a year-long constant status as meeting it. One constant year gives at most 12 months, and the prompt treats unlisted facts, including how long SSDI has been received, as false or 0. The 24-month test fails, and the head at 56 fails the age-65 test." +us,scenario_007,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model stated that Medicare follows after 24 months of SSDI receipt but never checked whether this household met that duration. It simply assumed it did. Nothing listed shows 24 months of entitlement and unlisted inputs default to 0 or false, so at 56 the head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine treats SSDI recipients as Medicare eligible once ""the qualifying period"" has passed, and assumed that period had passed without any listed fact to support it. The 24-month entitlement requirement is not established, and the head is under 65, so the reference returns not eligible." +us,scenario_007,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model reasoned that because SSDI is treated as constant through the year, the head had already passed the 24-month waiting period. Constancy within one tax year covers only 12 months, and the prompt forbids inferring unlisted status facts. The disability pathway is unmet, and at 56 the head fails the age-65 test." +us,scenario_007,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated receipt of SSDI income as ""indicating"" Medicare eligibility and ignored the 24-month entitlement requirement for disability-based Medicare. No listed fact satisfies that requirement, and the head is 56, below the age-65 threshold." +us,scenario_007,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model said SSDI benefits confer Medicare eligibility under age 65 without applying the 24-month entitlement wait. Duration of receipt is an unlisted input that defaults to 0 or false, so the disability pathway fails and the head at 56 is not eligible." +us,scenario_007,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model equated being an SSDI recipient with being Medicare eligible and skipped the 24-month entitlement requirement. The household facts do not establish that requirement, and the head is 56, so neither the age pathway nor the disability pathway applies." +us,scenario_007,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model said SSDI receipt qualifies the head for Medicare and never applied the 24-month waiting period. No listed fact meets that period and unlisted facts default to false, so at 56 the head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model invoked ""disability rules"" but treated current SSDI income as enough, leaving out the 24-month entitlement condition in those same rules. That condition is not established, and at 56 the head fails the age-65 test." +us,scenario_007,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model concluded that SSDI receipt qualifies the head for Medicare without testing the 24-month entitlement requirement. The facts do not establish it, and the head is under 65, so the answer is not eligible." +us,scenario_007,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model asserted that PolicyEngine grants Medicare to anyone receiving SSDI, basing this on the $19,285 of income alone. Disability-based Medicare requires 24 months of SSDI entitlement, which no listed fact establishes, and at 56 the head fails the age-65 test." +us,scenario_007,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly failed the head on the age test at 56, then asserted that PolicyEngine sets Medicare eligibility to yes whenever SSDI income is present. The disability pathway requires 24 months of SSDI entitlement, and no disability status or duration of receipt is listed, so the pathway is not met." +us,scenario_007,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model accepted that the head is under 65 but treated SSDI income as enough for disability-based Medicare. It left out the 24-month entitlement requirement, which the facts do not establish because unlisted status and duration inputs are false or 0." +us,scenario_007,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied a disability-based Medicare rule keyed only on receiving SSDI income and skipped that rule's 24-month entitlement condition. No listed fact meets that condition, and the head at 56 fails the age-65 test, so the correct answer is not eligible." +us,scenario_007,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model claimed the modeled disability pathway turns on SSDI benefits being positive. The disability pathway actually requires 24 months of SSDI entitlement, and positive current-year income does not establish that. With no such fact listed and the head at 56, the head is not Medicare eligible." +us,scenario_007,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model applied a ""modeled eligibility rule"" that grants disability-based Medicare to anyone receiving SSDI and left out the 24-month entitlement requirement. That requirement is unmet on the listed facts, and the head is under 65, so the answer is not eligible." +us,scenario_007,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model asserted that SSDI income qualifies the head for Medicare under PolicyEngine rules and skipped the 24-month entitlement requirement for the disability pathway. That requirement is not established, and the head at 56 fails the age-65 test." +us,scenario_007,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"The model cited the 24-month waiting period and then assumed it had passed with no supporting fact. The prompt says to treat unlisted facts, including how long SSDI has been received, as 0 or false. The waiting period is unmet, and the head at 56 is not Medicare eligible." +us,scenario_007,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model applied disability-based Medicare because SSDI income is present and ignored the 24-month entitlement condition that pathway requires. The listed facts do not establish that condition, so the head, who is under 65, is not eligible." +us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model treated SSDI income alone as establishing disability-based Medicare and never applied the 24-month entitlement requirement. That requirement is unmet on the listed facts, and the head at 56 fails the age-65 test." us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first recognized that no wage was listed but then asserted $1,516.30 of employee Social Security and Medicare tax on unspecified “applicable taxable wages.” The listed Social Security disability and private pension payments are not employee FICA wages, and the 50 weekly hours cannot create wage income when the unlisted wage rate and annual wages are zero." us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reconstructed the base correctly — federal AGI $49,064, Idaho subtraction of the $13,784 federally taxable Social Security, $16,100 standard deduction, $19,180 Idaho taxable income — then applied a repealed 5.695% rate instead of Idaho's 5.3% flat rate and ignored the indexed zero-rate bracket ($4,920 for a 2026 single filer). It then discarded even its own $1,092 arithmetic and reported $1,571 as an unexplained inflation 'adjustment'; 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Every step through Idaho taxable income of $19,180 is exact, and the 5.3% rate is the correct 2026 rate, but the model taxed the entire $19,180 rather than only the amount above Idaho's zero-rate threshold. Idaho Code 63-3024 imposes the flat rate on taxable income in excess of an inflation-indexed floor ($4,920 single for 2026), so the tax is 0.053 × $14,260 = $755.78, not 0.053 × $19,180 = $1,016.54." -us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wiped out the entire $35,280 private pension with a '$41,110 retirement income exclusion,' which is Kentucky's pension exclusion, not Idaho's. Idaho's retirement benefits deduction is limited to specified public annuities (civil service, firefighter, police, military) and requires age 65, or 62 with disability, so a 56-year-old's private pension gets no exclusion; the base is $35,280 less the $16,100 standard deduction, taxed above the $4,920 floor." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model derived taxable Social Security of $13,785 and Idaho taxable income of $19,180 correctly, then applied 5.695% — Idaho's 2024 rate, superseded by the 5.3% flat rate — and omitted the $4,920 zero-rate bracket. It then abandoned its own $1,092 computation for an unexplained '$1,379 conformity estimate'; the correct arithmetic is 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the 2025 standard deduction of $15,750 instead of the 2026 amount of $16,100 (taxable income $19,530 rather than $19,180), applied the superseded 5.695% rate instead of 5.3%, and used a stale $4,673 zero-bracket instead of the 2026 indexed $4,920. It then discarded its own $846 result for an unexplained $1,300." -us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model stated the correct method — Idaho taxable income of about $19,180, a 5.3% rate applying only above the roughly $4,700 exempt amount — but submitted $1,112, which its own stated method cannot produce (0.053 × ($19,180 − $4,700) = $767). With the actual 2026 indexed floor of $4,920 the answer is $755.78; the submitted figure corresponds to a 5.8% rate on the full taxable income." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a flat 85% inclusion to Social Security ($16,392) instead of the tiered formula ($13,784.12), then explicitly denied Idaho's subtraction for federally taxable Social Security, which removes that entire amount from Idaho AGI. It compounded this with a fabricated $4,950 Idaho personal exemption (repealed when the federal exemption was zeroed), a $15,000 standard deduction instead of $16,100, the superseded 5.695% rate, and a $120 grocery credit that is refundable in Idaho and excluded from this output." -us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model excluded Social Security disability entirely from federal AGI and used a $14,600 standard deduction (a 2024 figure) instead of $16,100, producing a $20,680 base rather than $19,180, then applied 5.695% instead of 5.3% and skipped the $4,920 zero-rate bracket. It reported $1,800 without deriving it, discarding even its own $1,178 figure." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model computed federal AGI of $49,064 exactly right but never applied Idaho's subtraction for the $13,784.12 of federally taxable Social Security, leaving that income in the Idaho base. It also used a $14,000 standard deduction instead of $16,100 and 5.695% instead of 5.3% with no zero-rate bracket, taxing $35,064 when the correct Idaho taxable income is $19,180." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model's structure — federal AGI less the Social Security subtraction less the standard deduction — is correct, but it used a $15,300 standard deduction instead of $16,100 and applied 5.8%, Idaho's 2023 rate, rather than the 5.3% flat rate in effect for 2026. It also taxed the full base instead of only the amount above the $4,920 indexed zero bracket." -us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model reached the right Idaho AGI of $35,280 but subtracted a $15,400 standard deduction instead of the 2026 amount of $16,100 and applied the superseded 5.695% rate rather than 5.3%. It also taxed all $19,880 rather than only the excess over Idaho's $4,920 zero-rate threshold." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model's Social Security subtraction of $13,784 from the $49,064 federal AGI is exact, but it used a $15,700 standard deduction instead of $16,100 and Idaho's 2023 rate of 5.8% instead of the 5.3% flat rate. It applied that rate to the entire $19,580 rather than only to income above the $4,920 zero-rate bracket." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, and $1,582 corresponds to taxing roughly $29,800 at 5.3% — the $35,280 Idaho AGI reduced by only about $5,400, an exemption-sized amount. The correct base is $35,280 less the $16,100 standard deduction, leaving $19,180, taxed at 5.3% only above the $4,920 zero-rate threshold, for $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly noted that Idaho excludes Social Security but stopped there; $1,246 implies taxing about $23,500 at 5.3%, i.e. reducing the $35,280 Idaho AGI by roughly $11,800 instead of the $16,100 standard deduction, with no zero-rate bracket. The correct computation is 0.053 × ($35,280 − $16,100 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model reduced the $35,280 Idaho AGI by only $13,450 — a understated standard deduction plus a personal exemption Idaho no longer has — leaving $21,830 instead of $19,180, and applied the superseded 5.695% rate rather than 5.3%. It also taxed the entire base rather than only the excess over Idaho's $4,920 indexed zero-rate threshold." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, and $3,371 corresponds to applying a flat rate near 6% to roughly the full $54,565 of gross income — the pension plus the entire $19,285 of Social Security disability — with no Social Security subtraction, no standard deduction, and no zero-rate bracket. Idaho taxable income here is $19,180 and the tax is 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model handled the Social Security subtraction exactly, reaching $35,280 of Idaho AGI, but its $1,115.08 equals 5.695% × $19,580 — a $15,700 standard deduction instead of $16,100 and Idaho's superseded 2024 rate instead of the 5.3% flat rate. It also applied the rate to the whole base rather than only above the $4,920 zero-rate threshold." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's explanation computed 5.695% × $19,180 = $1,092.30, using Idaho's superseded 2024 rate instead of the 5.3% flat rate and ignoring the $4,920 zero-rate bracket, while the value it actually submitted was $1,894 — a figure its own explanation never derives. Correct: 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model reached $35,280 of Idaho AGI correctly, then used a $15,350 standard deduction instead of $16,100 and Idaho's 2023 rate of 5.8% instead of the 5.3% rate in effect for 2026. It taxed all $19,930 rather than only the amount above the $4,920 indexed zero-rate bracket." -us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model zeroed the tax by claiming an Idaho retirement benefit deduction for disabled individuals that covers the full $35,280 private pension; Idaho Code 63-3022A limits that deduction to specified public retirement annuities (civil service, firefighter, police, military) and requires age 65, or 62 with disability, so a 56-year-old with a private pension qualifies for none of it. It also used a flat 85% Social Security inclusion instead of the tiered formula, and after the $16,100 standard deduction $19,180 of Idaho taxable income remains." -us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model computed the federally taxable Social Security as $9,961.25 instead of $13,784.12 (the tiered formula gives 0.85 × ($44,922.50 − $34,000) + $4,500), landing on $19,530 of Idaho taxable income rather than $19,180, and then applied 5.3% to the entire amount instead of only to income above Idaho's $4,920 zero-rate threshold. The $10 permanent building fund add-on is not part of the Idaho individual income tax liability at issue." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and excluded income fully offset Idaho taxable income, but the $16,100 standard deduction is far short of the $35,280 Idaho AGI that remains after subtracting the $13,784.12 of federally taxable Social Security. That leaves $19,180 of Idaho taxable income and a tax of 0.053 × ($19,180 − $4,920) = $755.78, not zero." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model returned zero without any derivation, implying the $35,280 taxable private pension is fully sheltered; only the $13,784.12 of taxable Social Security is subtracted at the Idaho level, and the $16,100 standard deduction leaves $19,180 of Idaho taxable income. Idaho's 5.3% flat rate on the excess over the $4,920 zero bracket yields $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model got the entire structure right — Social Security excluded, $16,100 standard deduction, 5.3% flat rate applied only above the zero bracket — and erred solely on the zero-bracket amount, using a round $5,000 instead of Idaho's inflation-indexed 2026 single figure of $4,920. That $80 of base at 5.3% is exactly the $4.24 shortfall between $751.54 and $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, and $1,885 corresponds to applying Idaho's flat rate to a base near $35,300 — the Idaho AGI before any standard deduction — rather than to taxable income. Subtracting the $16,100 standard deduction leaves $19,180, and the 5.3% rate applies only above the $4,920 zero-rate threshold, giving $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model derived Idaho taxable income of $19,180 exactly, excluding Social Security and applying the $16,100 standard deduction, then multiplied the whole amount by 5.3% to get about $1,017. Idaho's flat rate applies only to taxable income in excess of the indexed zero bracket ($4,920 single for 2026), so the tax is 0.053 × $14,260 = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, and $1,759.89 corresponds to taxing roughly $33,000 — federal AGI of $49,064 less the $16,100 standard deduction — meaning it never applied Idaho's subtraction for the $13,784.12 of federally taxable Social Security. With that subtraction the base is $19,180, taxed at 5.3% above the $4,920 zero bracket for $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model deducted $2,080 of health insurance premiums on top of the standard deduction, but this head's coverage is employer-sponsored, and Idaho's health-insurance-premium deduction does not reach premiums paid through such a plan; the Idaho base stays at $35,280 − $16,100 = $19,180. It also used a stale zero-rate threshold of $4,811 instead of the 2026 indexed $4,920, and the two errors together account for the $104 shortfall from $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no derivation, and $1,477 corresponds to taxing about $27,900 at 5.3% — the $35,280 Idaho AGI less roughly $7,400 — so it used neither the full $16,100 standard deduction nor the $4,920 zero-rate bracket. The correct chain is $35,280 − $16,100 = $19,180, then 0.053 × ($19,180 − $4,920) = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model correctly subtracted federally taxable Social Security to reach $35,280 of Idaho AGI but then used a 'conforming standard deduction' of $8,535 — roughly half the actual 2026 single amount of $16,100 — leaving $26,745 instead of $19,180. It compounded this with the superseded 5.695% rate rather than 5.3% and no zero-rate bracket." -us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model applied the right Social Security subtraction but deducted only about $8,477 as the conforming standard deduction instead of $16,100, taxing $26,803 rather than $19,180. It also used Idaho's superseded 5.695% rate instead of the 5.3% flat rate and applied it to the full base rather than the excess over the $4,920 zero bracket." -us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed $36,252, which is federal AGI computed with a flat 85% Social Security inclusion less a $15,420 standard deduction — it never applied Idaho's subtraction for the federally taxable Social Security, and it used the wrong inclusion formula and deduction amount. Idaho AGI is $35,280, taxable income $19,180, and the 5.3% rate applies only above the $4,920 zero bracket." -us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model taxed $32,964, which is federal AGI of $49,064 minus the $16,100 standard deduction — it used the correct rate of 5.3% but omitted Idaho's subtraction of the $13,784.12 of federally taxable Social Security. Removing that leaves $19,180, and the rate applies only above the $4,920 zero-rate threshold, for $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits and no explanation accompanied the submission, so the answer was missing rather than substantively wrong. The requested figure is $755.78: 5.3% of Idaho taxable income of $19,180 in excess of the $4,920 zero-rate threshold." -us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model understated the deduction (reducing the $35,280 pension by only about $13,600 to reach $21,680 instead of applying the $16,100 standard deduction for $19,180) and used a $2,500 zero-rate threshold instead of Idaho's inflation-indexed 2026 amount of $4,920. The two offsetting errors amount to taxing the full $19,180 at 5.3%, when only the $14,260 above the threshold is taxable." -us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model asserted that Idaho has no individual income tax, which is false — Idaho imposes a flat individual income tax of 5.3% for 2026 on taxable income above an indexed zero-rate threshold. Here that base is $19,180 after the Social Security subtraction and the $16,100 standard deduction, producing $755.78, not $0." -us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model landed on Idaho taxable income of $19,180 and the correct 5.3% rate but applied the rate to the entire amount, ignoring that Idaho taxes only income in excess of the indexed zero-rate threshold ($4,920 single for 2026). That single omission is the whole gap: 0.053 × $19,180 = $1,016.54 versus 0.053 × $14,260 = $755.78." -us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $19,285 of Social Security disability as fully taxable in federal AGI (the tiered formula includes $13,784.12) and never applied Idaho's subtraction that removes that included amount, taxing $39,965 instead of $19,180. It further used a $14,600 standard deduction instead of $16,100, a 5.8% rate instead of 5.3%, subtracted the grocery credit that is refundable in Idaho and excluded from this output, and then abandoned its own $2,198 arithmetic for an unexplained $1,707.48." -us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model stacked a $5,200 'Idaho standard deduction' on top of an already-deducted federal taxable base and then applied graduated brackets ('$430 on the first $4,300'), which Idaho eliminated when it moved to a single flat rate. Idaho's deduction is the federal $16,100 taken once, leaving $19,180, taxed at a flat 5.3% on the excess over the $4,920 zero-rate threshold." -us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly identified the Idaho Grocery Credit (Idaho Code § 63-3024A) as the state's refundable credit and correctly applied one credit for a single under-65 resident, but used the stale $120 per-person amount that applied for tax years 2023-2024. Idaho HB 231 (2025) raised the base grocery credit to $155 per person effective for tax year 2025 and forward, so the 2026 amount is $155, not $120." -us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Idaho has no refundable individual income tax credits, omitting the Idaho Grocery Credit entirely from its list of state credits. The grocery credit under Idaho Code § 63-3024A is expressly refundable — it is paid out in excess of tax liability and is even claimable by non-filers on Form 24 — and at the 2026 amount of $155 for a full-year resident under 65 it is the whole of this household's state refundable credits." -us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the refundable Idaho Grocery Credit and one qualifying person, but its stated derivation ($120 for a non-senior) contradicts its submitted value of $140, which is the pre-2025 age-65-or-older amount ($120 base plus the $20 senior add-on) and does not apply to a 56-year-old. Both figures are superseded: HB 231 (2025) set the base credit at $155 per person for tax year 2025 forward, which for 12 qualifying months yields $155." -us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model invented a refundability restriction, claiming the grocery credit is nonrefundable when claimed on the Idaho income tax return and refundable only through a separate filing. Idaho Code § 63-3024A makes the credit refundable regardless of the form used — Form 24 exists as an alternative for people with no filing requirement, not as the only path to a refund — so the full $155 base amount for a full-year resident under 65 is a state refundable credit here." -us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model recognized the Idaho grocery credit but reclassified it as a nonrefundable or ""other"" item and zeroed the refundable-credit output. The grocery credit is a refundable credit under Idaho Code § 63-3024A, is not limited by tax liability, and applies to any full-year resident regardless of children or filing status, giving $155 for this single 56-year-old resident with 12 qualifying months." -us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"After correctly identifying the grocery credit, the model fabricated a rule that the credit is nonrefundable to the extent the taxpayer has Idaho tax liability, and concluded that the $35,280 of taxable pension income absorbs the credit and leaves $0 refundable. Idaho Code § 63-3024A makes the entire grocery credit refundable and independent of liability; the model also used the superseded $120 amount rather than the $155 set by HB 231 for tax years after 2024." -us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model classified the Idaho grocery credit as nonrefundable against tax liability and reported no other applicable credit, producing $0. The grocery credit is refundable under Idaho Code § 63-3024A, has no liability limitation and no dependency on having children, and equals $155 per qualifying person for a full-year resident under 65 in 2026." -us,scenario_007,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model invented an income phaseout for the Idaho grocery credit, claiming a roughly $22,500 single-filer threshold that this household's pension and SSDI income exceeds. Idaho Code § 63-3024A contains no income phaseout — the credit is a flat per-person amount reduced only for nonresident, incarcerated, or nonqualifying months — so all 12 months qualify and the credit is the full $155." -us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly applied the refundable Idaho Grocery Credit to one person but used the $120 per-person amount that expired after tax year 2024. HB 231 (2025) increased the base credit to $155 per person for tax year 2025 forward, which is the 2026 figure for a full-year resident under 65." -us,scenario_007,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used $100 per person, the Idaho grocery credit amount in effect before the 2022 increase to $120, and thereby missed two successive statutory increases. The controlling 2026 amount under Idaho Code § 63-3024A as amended by HB 231 (2025) is $155 per person for a full-year resident under 65." -us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly treated the Idaho Grocery Credit as refundable and per-person but priced it at the obsolete $120 rate. The 2026 base amount is $155 per qualifying person under Idaho Code § 63-3024A as amended by HB 231 (2025), so the single-person household receives $155." -us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model asserted without derivation that the household qualifies for no Idaho refundable credit. Every full-year Idaho resident who is not claimed as a dependent qualifies for the refundable grocery credit under Idaho Code § 63-3024A with no income, asset, or family-composition test, giving this 56-year-old resident $155 for 12 qualifying months." -us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model reached the right credit and the right eligibility conclusion but applied the pre-2025 $120 per-person amount. Idaho HB 231 (2025) raised the grocery credit to $155 per person beginning with tax year 2025, and that amount governs 2026." -us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model identified the refundable Idaho grocery credit for an under-65 filer but used the superseded $120 per-person amount. The 2026 statutory amount is $155 per qualifying person, so the correct value is $155." -us,scenario_007,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned $0 with a bare assertion and no identification of any Idaho credit. Idaho's refundable grocery credit under Idaho Code § 63-3024A applies to every full-year resident not claimed as a dependent, and at $155 per person for 12 qualifying months it is this household's entire state refundable credit." -us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly characterized the Idaho Grocery Credit as refundable and per-resident but used $120, the amount in force for 2023-2024 only. HB 231 (2025) set the per-person credit at $155 for tax year 2025 and later, so the 2026 value is $155." -us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model applied the refundable Idaho grocery credit to the head but at the stale $120 rate. The credit is $155 per qualifying person in 2026 under Idaho Code § 63-3024A as amended in 2025. -us,scenario_007,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used $100 per person for the under-65 grocery credit, the pre-2023 amount, missing both the increase to $120 and the 2025 increase to $155. The correct 2026 per-person amount is $155, awarded in full for 12 qualifying months." -us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model claimed the Idaho grocery credit has been repealed and therefore returned $0. The credit remains in force under Idaho Code § 63-3024A and was increased — not repealed — by HB 231 (2025) to $155 per person; the model also wrongly conditioned the outcome on Idaho tax liability, which is irrelevant to a refundable credit." -us,scenario_007,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model correctly established full eligibility for the refundable Idaho grocery credit for one under-65 resident but priced it at the obsolete $100 per-person amount. The 2026 amount is $155 per person; its observation that no phaseout applies is right, which makes the full statutory $155 the answer." -us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model gave no derivation and asserted that no Idaho refundable credit is indicated by the facts. Idaho residency for the full year is by itself sufficient for the refundable grocery credit under Idaho Code § 63-3024A, which pays $155 per qualifying person in 2026 with no income or dependency condition to satisfy." -us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model reasoned that the household facts list no qualifying condition for a refundable state credit and returned $0. The single qualifying condition for Idaho's refundable grocery credit is full-year Idaho residency without being claimed as a dependent, which the state and tax-year facts establish, yielding $155 for the head." -us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model asserted that this household's income exceeds a phaseout for the Idaho grocery credit. Idaho Code § 63-3024A imposes no income limit or phaseout on the grocery credit — it is a flat per-person refundable amount prorated only for nonqualifying months — so the head receives the full $155 for 2026. -us,scenario_007,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model stated flatly that Idaho offers no refundable credits applicable to this household, overlooking the grocery credit. That credit is refundable under Idaho Code § 63-3024A, requires only full-year Idaho residency, and pays $155 per person in 2026." -us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly identified the refundable Idaho grocery credit for a full-year resident under 65 but used the long-superseded $100 per-person figure. The 2026 amount set by HB 231 (2025) is $155 per person. -us,scenario_007,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model got the credit and its refundability right but used $100, the pre-2023 per-person amount. Idaho's grocery credit is $155 per qualifying person for tax year 2026, awarded in full here for 12 qualifying months." -us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model concluded from household composition that no state refundable credit applies. The Idaho grocery credit is not composition-dependent — it accrues to each qualifying resident individually, including a childless 56-year-old single filer — and pays $155 for full-year residency in 2026." -us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so the submission was scored as missing rather than substantively wrong. The correct output is the refundable Idaho Grocery Credit of $155 for one full-year resident under 65." -us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly limited the answer to the refundable Idaho grocery credit for one qualifying under-65 person but used the $120 amount that expired after tax year 2024. The 2026 per-person amount under Idaho Code § 63-3024A as amended by HB 231 (2025) is $155. -us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model asserted that Idaho has no state income tax and therefore no refundable state credits, confusing Idaho with a no-income-tax state. Idaho levies a flat individual income tax and administers the refundable grocery credit under Idaho Code § 63-3024A, worth $155 to this full-year resident in 2026." -us,scenario_007,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model identified the single refundable Idaho grocery credit but estimated its 2026 value as ""roughly $140"" by treating the amount as inflation-indexed. The grocery credit is a fixed statutory dollar amount, not indexed; $140 was the pre-2025 age-65-plus figure, while the 2026 amount for a 56-year-old is the $155 base set by HB 231 (2025)." -us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model dismissed Idaho as having no significant refundable credits applicable here and returned $0. The refundable grocery credit under Idaho Code § 63-3024A attaches to full-year Idaho residency with no income or household test and pays $155 per person in 2026. -us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model asserted that no Idaho refundable credit applies to the listed facts. The facts state Idaho residency for the full 2026 tax year, which alone qualifies the head for the refundable grocery credit of $155 under Idaho Code § 63-3024A." +us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of Idaho taxable income. It then applied the obsolete 5.695% rate from the first dollar ($1,092) and raised that to $1,571 with an unsupported 'inflation adjustment'. The frozen reference applies 5.3% only to the $14,260 above the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It derived the frozen reference's $19,180 of taxable income but taxed all of it at 5.3%. It left out Idaho's 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920 and so taxes only $14,260, for $755.78. The exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It invented a $41,110 Idaho 'retirement income exclusion' for the private pension. Idaho's retirement benefits deduction covers only qualifying public and military retirement pay, and only at age 65 (or 62 if disabled). This head is 56 with a private pension, so the $35,280 stays taxable and gives $755.78 of tax in the frozen reference; the exclusion's corrected value, which also subtracts the $2,080 of premiums and uses the $4,811 zero-rate band that c_id_hold_2025 keeps for 2026, is $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of Idaho taxable income. It then used the outdated 5.695% rate on the full amount and raised the result to $1,379 with an arbitrary 'conformity' adjustment. The frozen reference taxes $19,180 − $4,920 = $14,260 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used the 2025 standard deduction of $15,750 instead of 2026's $16,100, and the old 5.695% rate above a $4,673 threshold, giving $846. It then raised that to $1,300 without any basis. The frozen reference's figure is 5.3% × ($19,180 − $4,920) = $755.78, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It said it would apply 5.3% above a roughly $4,700 exempt amount, but its $1,112 equals 5.8% × $19,180. That means it actually applied the superseded 5.8% rate to all taxable income with no zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $109 more at 5.3% than the frozen reference, giving $761.56 against its $755.78." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It explicitly rejected Idaho's full subtraction of federally taxable Social Security, and it overstated taxable Social Security at 85% ($16,392 instead of $13,784). It also used a $15,000 standard deduction, a $4,950 personal exemption that doesn't exist, and the 5.695% rate, and it subtracted Idaho's grocery credit, which is refundable and so doesn't belong in a before-refundable-credits figure." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used a $14,600 standard deduction instead of the 2026 amount of $16,100 and applied 5.695% to the full $20,680, giving $1,178. It then raised that to $1,800 with no stated computation. It never applied the 5.3% rate above the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 of taxable income but applied 5.3% to all of it. The frozen reference taxes only the $14,260 above the engine's projected $4,920 zero-rate band, giving $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and taxes only the income above the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It did not apply Idaho's full subtraction of the $13,784 of federally taxable Social Security, and it used a $14,000 standard deduction instead of $16,100. It then applied 5.695% to $35,064 with no zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It correctly subtracted Social Security but used a $15,300 standard deduction instead of the 2026 amount of $16,100. It then taxed all $19,980 at the repealed 5.8% rate. The frozen reference taxes $19,180 − $4,920 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, taxes $17,100 − $4,811 at 5.3%, after the $2,080 premium subtraction and with the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It used a $15,400 standard deduction instead of $16,100 and applied the outdated 5.695% rate to all $19,880. It never excluded Idaho's zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference) or used the 5.3% rate." +us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,100 standard deduction instead of the federal-conforming 2026 single deduction of $16,100, which overstated taxable income at $27,180. It then taxed all of it at 5.3% without the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It used a $15,700 standard deduction instead of $16,100 and applied the repealed 5.8% rate to all $19,580. It left out the 2026 5.3% rate and the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It treated the disability income as taxable for Idaho. Idaho subtracts all federally taxable Social Security, including SSDI, leaving only the $35,280 pension. In the frozen reference, after the $16,100 deduction and the engine's projected $4,920 zero-rate band, tax is 5.3% × $14,260 = $755.78, about half of its $1,582; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It correctly excluded Social Security, but $1,246 matches roughly $21,500–$23,500 taxed at a flat rate from the first dollar. That means it understated the $16,100 standard deduction and left out the $4,920 zero-rate band. The frozen reference's computation is 5.3% × $14,260 = $755.78, above the engine's projected $4,920 zero-rate band; the exclusion's corrected value is 5.3% × $12,289 = $651.32, after the $2,080 premium subtraction and the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It subtracted only $13,450 of standard deduction and 'personal exemption' from $35,280, reaching $21,830, when the 2026 standard deduction alone is $16,100. It then applied the outdated 5.695% rate to all of it with no zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"$3,371 is more than 6% of total gross income, which means it taxed roughly all $54,565 of income. It never applied the Social Security subtraction, the $16,100 standard deduction, the $4,920 zero-rate band, or the 5.3% rate." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It correctly reached $35,280 after the Social Security subtraction, but its $1,115.08 equals 5.695% × $19,580. That means it used a $15,700 standard deduction instead of $16,100 and the outdated 5.695% rate on all taxable income, with no $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation computes 5.695% × $19,180 = $1,092.30, using the outdated rate with no zero-rate band, but it submitted an unrelated $1,894. Neither figure applies 5.3% only to income above Idaho's zero-rate band: the frozen reference taxes the $14,260 above the engine's projected $4,920, and the exclusion's corrected value taxes the $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction." +us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It used a $15,350 standard deduction instead of $16,100 and taxed all $19,930 at the repealed 5.8% rate. It left out the 2026 5.3% rate and the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It claimed Idaho's retirement benefits deduction removes the $35,280 private pension because the head is disabled. That deduction covers only qualifying public and military retirement pay for recipients aged 65+ (or 62+ if disabled), so it doesn't apply to a 56-year-old's private pension, and $19,180 remains taxable." +us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It used the 2025 standard deduction of $15,750 instead of $16,100, taxed all $19,530 at 5.3% with no $4,920 zero-rate band, and added a $10 permanent building fund tax. That fund tax is not part of Idaho income tax before refundable credits." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed the standard deduction and exclusions fully offset income. Idaho income after the Social Security subtraction is $35,280, which exceeds the $16,100 standard deduction plus the zero-rate band. The frozen reference, with the engine's projected $4,920 band, leaves $14,260 taxed at 5.3%; the exclusion's corrected value also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026, leaving $12,289 taxed at 5.3%, for $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It returned zero tax without any computation. The $35,280 private pension remains in Idaho income. In the frozen reference, after the $16,100 standard deduction and the engine's projected $4,920 zero-rate band, $14,260 is taxed at 5.3%, for $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used a guessed $5,000 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,180 at 5.3%, giving $751.54, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"$1,885 equals 5.3% × about $35,570, which is federal AGI with 85% of the SSDI included, minus $16,100. That means it skipped Idaho's full subtraction of taxable Social Security and also left out the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It reached the frozen reference's $19,180 but applied 5.3% to the entire amount. It left out Idaho's zero-rate band: the frozen reference excludes the engine's projected $4,920, leaving $14,260 taxable, and the exclusion's corrected value, after the $2,080 premium subtraction, excludes the $4,811 that c_id_hold_2025 keeps for 2026, leaving $12,289." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"$1,759.89 equals 5.3% × about $33,200, which is federal taxable income. That means it never applied Idaho's subtraction of the federally taxable Social Security and also left out the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It subtracted the head's $2,080 of health insurance premiums from Idaho income, the Idaho subtraction (Idaho Code 63-3022P) that the engine leaves out; the frozen reference's Idaho income of $35,280 has only the Social Security subtraction. It also used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Its $651.32 is the exclusion's corrected value." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It computed taxable income of $35,572 from a federal AGI that included 85% of the SSDI, and never applied Idaho's full subtraction of taxable Social Security. It then taxed everything at 5.3% with no $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It used an approximate $4,990 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,190 at 5.3% and gave $752, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $14,369 at 5.3%, against $14,260 in the frozen reference." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"$1,477 implies roughly $26,000–$28,000 taxed at a flat rate. That means it used a TCJA-sunset standard deduction of about $8,000 instead of the permanent 2026 amount of $16,100, and left out the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It assumed an $8,535 standard deduction, as if TCJA had expired, instead of the permanent 2026 single deduction of $16,100. It then applied the outdated 5.695% rate to all $26,745 with no zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It used a standard deduction of about $8,477, based on a TCJA sunset, instead of $16,100. It then taxed all $26,803 at 5.695% instead of taxing at 5.3% only the income above the zero-rate band: $14,260 above the engine's projected $4,920 in the frozen reference, or $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction, in the exclusion's corrected value." +us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It assumed the standard deduction reverted to the post-sunset $8,550. OBBBA made the higher deduction permanent, at $16,100 for 2026. It also applied 5.3% to all $26,730 without the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"Its $36,252 taxable income comes from a federal AGI that included 85% of the SSDI, with no Idaho Social Security subtraction and a $15,420 deduction. It then applied the repealed 5.8% rate from the first dollar." +us,scenario_007,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It taxed federal taxable income of $32,964 directly and never subtracted the $13,784 of federally taxable Social Security that Idaho excludes. It also left out the $4,920 zero-rate band." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the answer is missing rather than wrong." +us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It subtracted only $13,600 of deduction/exemption from $35,280 and then used an invented $2,500 zero-rate band, and the two errors happened to land on $19,180 × 5.3%. The frozen reference subtracts the $16,100 standard deduction to reach $19,180 and then excludes the engine's projected $4,920 zero-rate band; the exclusion's corrected value also subtracts the $2,080 of premiums, for $17,100, and excludes the $4,811 band that c_id_hold_2025 keeps for 2026." +us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It stated that Idaho has no individual income tax. Idaho does: it taxes income above a zero-rate band at a flat 5.3%. With the engine's projected $4,920 band the frozen reference is $755.78; the exclusion's corrected value, with the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026 and the $2,080 premium subtraction, is $651.32." +us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It derived the frozen reference's $19,180 of Idaho taxable income but applied 5.3% to all of it. It left out the 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920." +us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It treated the full $19,285 of SSDI as taxable in federal AGI and in Idaho, instead of applying Idaho's full Social Security subtraction. It used a $14,600 deduction and the 5.8% rate, subtracted the grocery credit even though it is refundable, and then made an arbitrary cut to reach $1,707.48." +us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It started from a base that already had the $15,750 deduction applied, then subtracted a second, invented $5,200 'Idaho standard deduction'. It also applied the repealed graduated brackets instead of the 5.3% flat rate above the zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference)." +us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly picked the Idaho Grocery Credit as the refundable credit, but used the outdated $120 under-65 amount. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Idaho has no refundable individual income tax credits. That overlooks the Idaho Grocery Credit, which is refundable and worth $155 in 2026 to this under-65 resident, who received no SNAP." +us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning cites a $120 under-65 Grocery Credit, yet it submitted $140, which is the old 65-and-over amount. Neither is the 2026 under-65 amount of $155." +us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Claimed the Grocery Credit is nonrefundable when claimed on the income tax return. It is a refundable credit on the Idaho return, so the full $155 counts as a refundable state credit." +us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Recognized that the Grocery Credit works like a refundable credit, then reclassified it as a nonrefundable 'other item' and zeroed it. The credit is refundable and worth $155 in 2026 to a resident under 65." +us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Reasoned that a filer with Idaho tax liability uses the Grocery Credit up as a nonrefundable offset, leaving $0 refundable. The Grocery Credit is refundable whatever the liability, so the full amount counts. The model also used the outdated $120 figure, not the 2026 amount of $155." +us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Treated the Idaho Grocery Credit as nonrefundable against tax liability. It is a refundable credit, worth $155 to this under-65 resident in 2026." +us,scenario_007,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Correctly treated the Grocery Credit as refundable, but used the outdated $120 per-person amount. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Invented a roughly $22,500 income phaseout for the Idaho Grocery Credit. The credit has no income phaseout, so this resident receives the full $155 regardless of pension and SSDI income." +us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit for one person, but used the outdated $120 amount. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used $100, the Grocery Credit amount from before 2023, for a filer under 65. The 2026 amount is $155." +us,scenario_007,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Applied the pre-2023 $100 Grocery Credit amount. The 2026 amount for a resident under 65 is $155. +us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Identified the refundable Grocery Credit, but used the outdated $120 per-member amount. The 2026 amount for this under-65 member is $155." +us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that the household qualifies for no refundable Idaho credits. That misses the Idaho Grocery Credit: $155 in 2026 for a full-year resident under 65 without SNAP, refundable and with no income test." +us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Got eligibility right (under 65, no SNAP), but used the outdated $120 Grocery Credit amount. The 2026 amount is $155." +us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,Applied the outdated $120 Grocery Credit for a single filer under 65. The 2026 amount is $155. +us,scenario_007,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Answered $0 without reasoning, so it never applied the Idaho Grocery Credit. That credit is refundable and gives this under-65 resident, who received no SNAP, $155 in 2026." +us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Identified the refundable Grocery Credit per resident, but used the outdated $120 amount. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly found the head eligible for the refundable Grocery Credit, but used $120. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Applied the pre-2023 $100 Grocery Credit amount for a resident under 65. The 2026 amount is $155. +us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Falsely asserted that Idaho repealed the Grocery Credit. The credit remains in force and is refundable, worth $155 in 2026 to this under-65 resident, who received no SNAP." +us,scenario_007,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Correctly applied the refundable Grocery Credit with no income limit, but used the pre-2023 $100 amount. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable Idaho credit, missing the Grocery Credit. Every full-year resident without SNAP receives it, and it is worth $155 in 2026 to this filer under 65." +us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed refundable state credits require a special qualifying condition, such as having children. The Idaho Grocery Credit needs only Idaho residency and no SNAP receipt, and pays $155 in 2026 to a filer under 65." +us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Claimed the household's income is above a phaseout for the Idaho Grocery Credit. The credit has no income phaseout, so the full $155 under-65 amount applies." +us,scenario_007,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Concluded that no Idaho refundable credit applies. It overlooked the Grocery Credit, which is refundable and worth $155 in 2026 to this full-year resident under 65." +us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit for a full-year resident under 65, but used the pre-2023 $100 amount. The 2026 amount is $155." +us,scenario_007,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,Applied the outdated $100 Grocery Credit amount. The 2026 amount for a resident under 65 is $155. +us,scenario_007,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Correctly found the full-year resident eligible for the refundable Grocery Credit, but used $120. The 2026 amount for a filer under 65 is $155." +us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Concluded that no state refundable credit applies to this household composition. The Idaho Grocery Credit applies to any resident without SNAP and pays $155 in 2026 to this filer under 65. +us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_refundable_credits, so there is no answer to score against the $155 Grocery Credit." +us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly identified the refundable Grocery Credit as the only applicable Idaho refundable credit, but used the outdated $120 amount. The 2026 amount for a person under 65 is $155." +us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Asserted that Idaho has no state income tax. Idaho does levy an individual income tax, and its return includes the refundable Grocery Credit, worth $155 in 2026 to this resident under 65." +us,scenario_007,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,Guessed an inflation-adjusted Grocery Credit of about $140. The credit is not indexed; the statutory 2026 amount for a resident under 65 is $155. +us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Dismissed Idaho's refundable credits as insignificant and answered $0. It skipped the Grocery Credit, which pays this under-65 resident, who received no SNAP, $155 in 2026." +us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Concluded that no Idaho refundable credit applies. It missed the Grocery Credit, which is refundable, has no income test and pays $155 in 2026 to a resident under 65." us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only NJ's upper CHIP ceiling (""well below NJ CHIP income limit (350% FPL)"") and never tested the lower Medicaid children's threshold that CHIP sits above. At ~$30,915 for a household of eight, the 17-year-old falls in the OLDER_CHILD Medicaid category, and Medicaid eligibility precludes CHIP, so the correct answer is 0." us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model reasoned that ""very low income"" plus age under 19 establishes CHIP eligibility, inverting the actual rule: income far below the Medicaid children's limit routes the child to Medicaid, and CHIP covers only children who do not qualify for Medicaid. Child1 is Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is 0." us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model built its entire test around the 350% FPL CHIP ceiling ($205,240 for a family of eight) and concluded eligibility because $30,915 fell under it, treating the absence of listed coverage as the only other condition. It omitted the Medicaid screen that runs first: a 17-year-old at roughly 54% FPL qualifies for NJ Medicaid in the OLDER_CHILD category, which by construction makes is_chip_eligible False." @@ -460,10 +503,11 @@ us,scenario_008,child5_chip_eligible,ox-alpha,llm_error,health_coverage,False,"T us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model computed MAGI at about 59% FPL for a household of eight, checked it against NJ CHIP's 355% ceiling, and then cleared the child because ""no disqualifying health coverage is listed."" The disqualifier is computed, not listed: at 59% FPL a 5-year-old is Medicaid-eligible under the under-6 (YOUNG_CHILD) category, and Medicaid eligibility — not enrollment — bars CHIP." us,scenario_008,child5_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model ran a single test — MAGI of about $30,800 is ""within NJ CHIP income limits"" — and answered yes. Being within CHIP limits is not sufficient: the child must first fail Medicaid, and at this income a 5-year-old clears New Jersey's Medicaid threshold for the under-6 band, so Medicaid eligibility under the YOUNG_CHILD category makes the child CHIP-ineligible." us,scenario_008,child5_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated Early Head Start as serving children from birth through age 5. Because Child 5 is age 5, the child is outside Early Head Start's under-age-3 category; income and housing assistance do not cure that categorical age failure." -us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model incorrectly restricted preschool Head Start to ages 3–4 and treated age five as an automatic exclusion. The benchmark’s Head Start age rule includes this 5-year-old, who also qualifies through SNAP participation and household income below 130% of the federal poverty guideline." -us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model substituted a presumed usual preschool range for the benchmark’s Head Start age rule and incorrectly excluded the child at age five. With age five included and both the SNAP categorical and direct income pathways satisfied, the child is eligible." -us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model incorrectly treated age five as aging the child out of Head Start. The applicable age rule includes this child, and the household satisfies both SNAP categorical eligibility and the Head Start income threshold." -us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"The model incorrectly classified a 5-year-old as outside the Head Start preschool age range. Under the benchmark’s inclusive age rule, the child qualifies, with SNAP participation and income below 130% of the federal poverty guideline independently establishing the financial pathway." +us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"It limited preschool Head Start to ages 3-4 and ruled out Child 5 on the grounds that 5-year-olds are of kindergarten age. Head Start eligibility actually runs from age 3 until the age of compulsory school attendance (6 in NJ), and PolicyEngine includes age 5. Because it stopped at the wrong age cutoff, it never applied the SNAP categorical pathway, which the household's SNAP receipt satisfies." +us,scenario_008,child5_head_start_eligible,claude-sonnet-5.5,llm_error,age_disability,False,"It set the Head Start preschool age range at 3-4 and treated age 5 as 'at or above' it. The eligible range actually extends to the compulsory school age of 6, so 5-year-olds qualify. It therefore never applied the SNAP categorical eligibility or the income test, both of which Child 5 meets." +us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"It declared age 5 'past the usual Head Start preschool age range'. Under the Head Start rule that PolicyEngine encodes, children are eligible from age 3 until the age of compulsory school attendance, so a 5-year-old is inside the range. Because it excluded Child 5 on age, it skipped the SNAP categorical eligibility that makes the child eligible." +us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"It assumed that turning 5 ages a child out of preschool Head Start. The program actually covers children from age 3 until the age of compulsory school attendance (6 in NJ), and PolicyEngine counts age 5 as eligible. Its answer ignores the household's SNAP-based categorical eligibility, which settles the income side." +us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"It placed age 5 outside the Head Start preschool age range. The rule actually runs from age 3 until the age of compulsory school attendance, so a 5-year-old is age-eligible. It never evaluated the SNAP categorical pathway or the income test, both of which qualify Child 5." us,scenario_008,child5_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact instead of applying the age-based YOUNG_CHILD pathway. Child 5's age of 5 and MAGI of 0.55 times FPL satisfy that pathway, yielding eligibility." us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model wrote the correct rule — ""children under 5 years old are eligible"" — and then contradicted it, asserting ""age under 5"" for a child whose stated age is 5. WIC's child category (7 CFR 246.2) runs from the first birthday to the fifth birthday, so a child who has reached age 5 is out of category and no income test is ever reached; the model instead let the 185%-FPL income comparison and receipt of housing assistance carry the answer." us,scenario_008,child5_wic_eligible,claude-opus-5,llm_error,age_disability,False,"The model defined the WIC child category as ""1-4 completed years through age 5,"" stretching the upper bound a full year past the fifth birthday where the category actually terminates. It then compounded the error by invoking SNAP adjunctive eligibility, which only substitutes for the income test and cannot confer eligibility on someone who fails the categorical age requirement." @@ -493,43 +537,47 @@ us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categori us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 1 as within the preschool-age Head Start range. A one-year-old falls under Early Head Start rather than Head Start, and age alone does not establish income eligibility." us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact and failed to apply the age-based YOUNG_CHILD pathway. Child 6 is age 1 and a dependent, and the household's MAGI of 0.55 times FPL satisfies the New Jersey income test for that category." us,scenario_008,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It subtracted a phase-out from the 3+-child maximum using a fabricated $30,050 joint threshold, when $30,517 of earned income sits below the 2026 MFJ phase-out start and the credit stays at the full $8,231. It also phased in the refundable CTC on $30,494 (gross SE income times 0.9235) instead of $30,517.41 (SE income less the deductible half of SE tax), and then submitted $12,130 even though its own components summed to roughly $12,372." -us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It invented an EITC phase-out that cut the 3+-child credit to $1,395, when $30,517 of joint earned income leaves the 2026 credit on its $8,231 plateau. It then submitted a refundable CTC of $1,305 after deriving the 15%-of-earnings-over-$2,500 amount as about $4,245, discarding its own arithmetic; the correct refundable CTC is 15% x ($30,517.41 - $2,500) = $4,202.61." -us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its derivation reached $8,267 of EITC plus $4,202 of ACTC for a total of $12,469, then it submitted $13,167 — a $698 inflation over its own total with no supporting step. The correct components are the $8,231 plateau EITC and $4,202.61 of refundable CTC." -us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"It truncated the refundable CTC to $388 on the theory that nonrefundable use consumed most of the credit, when zero income tax liability leaves the entire earned-income phase-in refundable: 15% x ($30,517.41 - $2,500) = $4,202.61. It also added a $1,000 refundable AOTC even though no qualified education expense is listed and unlisted numeric inputs are zero." -us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It cut the 3+-child EITC to roughly $7,700 with a phase-out that does not reach a joint filer at $30,517 of earned income, where the 2026 credit remains $8,231, and it added about $355 of refundable AOTC with zero listed tuition expenses. Its $4,245 refundable CTC also used gross self-employment income rather than the $282.59-lower earned-income base of $30,517.41." -us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deliberately substituted 2025 parameters — an $8,046 maximum and a $26,511 joint phase-out start — and subtracted $868, when the 2026 3+-child maximum is $8,231 and the joint phase-out threshold exceeds this household's $30,517 of earned income, so no reduction applies. Its $4,199 ACTC also used $30,494 of earned income instead of $30,517.41 after the half-SE-tax deduction, where the correct figure is $4,202.61." -us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"It set the ACTC at 5 x $1,700 = $8,500 and declared the earned-income formula non-binding, inverting the limit: the refundable CTC equals the smaller 15% x ($30,517.41 - $2,500) = $4,202.61, with the $8,500 per-child cap unreached. It also added $1,000 of refundable AOTC with no listed tuition and applied a phase-out at about $29,600 that never reaches this household, then submitted $12,900 unconnected to its own $17,356 total." -us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its $8,021 EITC understates the 2026 3+-child plateau of $8,231, which applies in full because $30,517 of joint earned income falls below the phase-out start. Its $4,245 refundable CTC used $30,800 of earned income (wages plus gross self-employment income), omitting the $282.59 half-SE-tax deduction that sets the base at $30,517.41 and the credit at $4,202.61." -us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the repealed pre-TCJA $3,000 refundable-CTC phase-in floor instead of the $2,500 floor in force for 2026, producing $4,170 rather than $4,202.61. It also shaved the EITC to $7,787 with a phase-out that does not reach $30,517 of joint earned income, where the credit is the full $8,231." -us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its $4,124 refundable CTC is exactly 15% x ($30,494 - $3,000), applying the pre-TCJA $3,000 phase-in floor rather than $2,500 and an earned-income base that omits the half-SE-tax adjustment; the correct amount is 15% x ($30,517.41 - $2,500) = $4,202.61. Its $8,190 EITC also falls below the 2026 3+-child maximum of $8,231, which applies undiminished at this income." -us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It reduced the 3+-child EITC to $7,691 with a phase-out, when $30,517 of joint earned income is below the 2026 MFJ phase-out threshold and the credit stays at $8,231. Its $4,199 ACTC also used $30,494 of earned income rather than $30,517.41 (self-employment income net of the deductible half of SE tax), which yields $4,202.61." -us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It asserted the maximum 3+-child EITC plus 'the full Child Tax Credit' without applying the 15%-of-earnings-over-$2,500 phase-in that fixes the refundable CTC at $4,202.61. Its $11,400 falls $1,034 short of $8,231 + $4,202.61 and is consistent with a stale-year EITC maximum paired with an ad-hoc child-credit figure." -us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $4,124 refundable CTC is 15% x ($30,494 - $3,000), using the pre-TCJA $3,000 phase-in floor instead of $2,500 and an earned-income base that omits the half-SE-tax deduction; the correct amount is 15% x ($30,517.41 - $2,500) = $4,202.61. Its approximate $8,200 EITC also sits below the exact 2026 3+-child maximum of $8,231 that applies here." -us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Its $7,128 EITC applies a phase-out that does not reach $30,517 of joint earned income, where the 2026 3+-child credit is the full $8,231. Its $4,128 ACTC also uses the pre-TCJA $3,000 phase-in floor instead of $2,500, against the correct $4,202.61." -us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It supplied no derivation; the correct components are the $8,231 3+-child EITC plateau and 15% x ($30,517.41 - $2,500) = $4,202.61 of refundable CTC. Its $9,750 is $2,684 short and is consistent with subtracting a phase-out from the EITC that a joint filer with $30,517 of earned income never reaches." -us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed the refundable CTC correctly at $4,203 on the $30,517 earned-income base (wages plus self-employment income less half the SE tax), but used the 2025 3+-child EITC maximum of $8,046 instead of the 2026 figure of $8,231. That single stale parameter accounts for its entire $185 shortfall." -us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It gave no components; the correct total is the $8,231 3+-child EITC plateau plus $4,202.61 of refundable CTC from 15% x ($30,517.41 - $2,500). Its $11,940 is consistent with applying a phase-out reduction of roughly $490 to the EITC, which a joint filer at $30,517 of earned income does not incur." -us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its EITC of about $8,233 matches the $8,231 plateau, but its $4,128 refundable CTC applies the pre-TCJA $3,000 phase-in floor to $30,494 of earned income. The 2026 computation is 15% x ($30,517.41 - $2,500) = $4,202.61, with the $1,700-per-child cap for the five children under 17 non-binding." -us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no reasoning were returned for federal_refundable_credits, so nothing was scored against the $12,433.61 reference. The required derivation is the $8,231 3+-child EITC plus 15% x ($30,517.41 - $2,500) = $4,202.61 of refundable CTC." -us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used an inflated 2026 3+-child EITC maximum of $8,360 in place of $8,231, and computed the refundable CTC on $30,494 of earned income (gross SE income times 0.9235) rather than $30,517.41 (SE income less the deductible half of SE tax), giving $4,199 instead of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It declared that the facts do not establish qualifying children or a valid EITC claim, when six children ages 1 to 17 living with the married couple satisfy the relationship, age, and residency tests, five of them under 17 for the CTC. With $30,517 of earned income the household receives the full 3+-child EITC of $8,231 and a refundable CTC of $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It reported zero on the ground that no qualifying income structure was specified, when $26,800 of wages plus $4,000 of self-employment income is earned income producing the full 3+-child EITC of $8,231. The five children under 17 further generate a refundable CTC of 15% x ($30,517.41 - $2,500) = $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It identified the $8,231 plateau EITC exactly but phased in the refundable CTC on $30,800 of earned income, using gross self-employment income of $4,000. Earned income for the phase-in is net of the deductible half of the $565.18 SE tax, giving a $30,517.41 base and a refundable CTC of $4,202.61 rather than $4,245." -us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the 2025 3+-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231, which applies in full at $30,517 of joint earned income. It also computed the refundable CTC on gross self-employment income ($30,800 base), where deducting half the SE tax gives $30,517.41 and a credit of $4,202.61 rather than $4,245." -us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It named the $8,231 three-or-more-child EITC correctly but computed the earned-income-limited refundable CTC as $4,245, which is 15% x ($30,800 - $2,500) using gross self-employment income. Subtracting the deductible half of SE tax ($282.59) sets earned income at $30,517.41 and the refundable CTC at $4,202.61." -us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 is the $8,231 EITC plus $4,245, the latter being 15% x ($30,800 - $2,500) with self-employment income counted gross. The phase-in base is wages plus SE income less half the SE tax, or $30,517.41, so the refundable CTC is $4,202.61." -us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"It treated the credits as inapplicable because tax liability was already zero, which is the rule for nonrefundable credits only. The EITC and the additional CTC are refundable and pay out in full at zero liability: $8,231 plus 15% x ($30,517.41 - $2,500) = $4,202.61." -us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA child credit rules — a $3,000 phase-in floor and a $1,000-per-child cap — to get $4,124, when the 2026 refundable CTC is 15% x ($30,517.41 - $2,500) = $4,202.61 with a non-binding $1,700-per-child cap. Its $8,247 EITC also overstates the 2026 3+-child maximum of $8,231." -us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed the ACTC as 15% of earned income over $3,000, using the repealed pre-TCJA floor instead of $2,500, and on a $30,494 base rather than $30,517.41, yielding $4,124 against the correct $4,202.61. Its $8,250 EITC also exceeds the 2026 3+-child maximum of $8,231." -us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"It phased the EITC down to $7,901 on AGI of $30,632, when the 2026 joint 3+-child phase-out threshold exceeds that amount and the credit stays at $8,231. Its $4,199 ACTC also used $30,494 of earned income rather than the $30,517.41 base that yields $4,202.61." -us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used $8,271 for the 2026 3+-child EITC maximum instead of $8,231, and computed the refundable CTC on about $30,800 of earned income including gross self-employment income. Deducting half the SE tax lowers the base to $30,517.41 and the refundable CTC to $4,202.61 rather than $4,245." -us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so no substantive computation was scored. The required result is the $8,231 3+-child EITC plus a refundable CTC of 15% x ($30,517.41 - $2,500) = $4,202.61." -us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It stated the $8,231 three-or-more-child EITC exactly but phased in the refundable CTC on earnings of $30,800 that count self-employment income gross. Earned income for the phase-in is $26,800 plus $4,000 less the deductible half of the $565.18 SE tax, or $30,517.41, so the credit is $4,202.61, not $4,245." -us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a 3+-child maximum of $7,152 and a joint phase-out start of $28,120, subtracting $564, when the 2026 maximum is $8,231 and the joint phase-out threshold is above this household's $30,517 of earned income so the full credit applies. Its $4,245 ACTC also counted self-employment income gross instead of using the $30,517.41 base that gives $4,202.61." -us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used $8,262 as the 2026 3+-child EITC maximum instead of $8,231, and computed the refundable CTC as 15% x ($30,800 - $2,500) = $4,245 with self-employment income counted gross. The deductible half of SE tax reduces earned income to $30,517.41 and the refundable CTC to $4,202.61." -us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It cut the EITC to $7,144 using the 2025 maximum of $8,046 and a $26,210 joint phase-out threshold, when $30,517 of earned income leaves the 2026 credit at its full $8,231. It also added $1,000 of refundable AOTC with zero listed qualified education expenses, and used $30,494 rather than $30,517.41 of earned income for the refundable CTC of $4,202.61." -us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It limited the additional CTC to $1,600 by 'remaining tax liability room,' inverting the rule — the additional CTC is refundable precisely because liability is zero and equals 15% x ($30,517.41 - $2,500) = $4,202.61. It also phased the EITC down to $6,801 when the full $8,231 applies at this income, and its own stated components sum to $8,401 rather than the $6,586 submitted." +us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It used a joint-filer EITC phase-out start of $30,050, which is below the household's AGI, and cut the $8,231 maximum to $8,173; in 2026 AGI of about $30,632 is still on the plateau. It set ACTC earned income at 0.9235 x SE income ($30,494) instead of SE income minus the half-SE-tax deduction ($30,517), and then submitted an unexplained $12,130, below its own component sum of $12,372." +us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the EITC at about $1,395. A married couple with three or more children and about $30,600 of AGI is on the plateau and gets the full $8,231. It also dropped its own $4,245 ACTC figure to an unexplained $1,305 in the total; the correct ACTC is 15% x ($30,517 - $2,500) = $4,203." +us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its ACTC of $4,202 was correct and its components summed to $12,469, close to the reference. It then applied an unsupported 'adjusting EITC' step that added about $700, which is impossible because the EITC was already at its maximum. It also put that maximum at $8,267 instead of the 2026 figure of $8,231." +us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"It added a $1,000 refundable AOTC even though qualified education expenses are $0. It also cut the ACTC to $388 on the theory that nonrefundable credits absorb it. Tax liability is zero, so the full earned-income-limited ACTC of 15% x ($30,517 - $2,500) = $4,203 is refundable." +us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It put the EITC at about $7,700, but the household is below the 2026 joint phase-out start and gets the full $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting half the SE tax from SE income, and it added a $355 AOTC even though qualified education expenses are $0." +us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It used 2025 parameters: an $8,046 maximum and a phase-out start of $26,511, which is below the joint-filer threshold. That wrongly cut the EITC by $868 to $7,178, when the 2026 joint threshold is above its $30,632 AGI and the full $8,231 applies. Its ACTC also used 0.9235 x SE income ($4,199) instead of SE income minus half the SE tax ($4,203)." +us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It applied an EITC phase-out from about $29,600 (giving $7,856) instead of the full $8,231. It treated the 15% x (earned income - $2,500) limit as if it were per child and claimed $8,500 of ACTC, when that limit caps the total ACTC at $4,203. It added a $1,000 AOTC with zero qualified expenses, then submitted an arbitrary 'reconciled' $12,900." +us,scenario_008,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It put the EITC at $8,021 instead of the full 2026 maximum of $8,231; the household's AGI is below the joint phase-out start. It computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction, which gives $30,517 and $4,203." +us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the $3,000 ACTC earned-income threshold instead of the 2026 threshold of $2,500, on gross earnings of $30,800 ($4,170 instead of $4,203). It also put the EITC at $7,787 instead of the full $8,231 plateau amount." +us,scenario_008,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $4,124 matches 15% x ($30,494 - $3,000). That uses the $3,000 threshold instead of $2,500, and 0.9235 x SE income instead of SE income minus half the SE tax; the correct ACTC is $4,203. Its EITC of $8,190 is also below the 2026 three-child maximum of $8,231." +us,scenario_008,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,247 instead of $8,231. It measured ACTC earned income as 0.9235 x SE income ($30,494), when the correct figure is $4,000 - $282.59 of half SE tax, giving $30,517.41 and an ACTC of $4,202.61 rather than $4,199.10." +us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It cut the EITC to $7,691 even though the household's AGI of about $30,632 is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,199 used 0.9235 x SE income instead of SE income minus half the SE tax, which gives $4,203." +us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no breakdown; its round $11,400 is $1,034 short of the correct total. That total is the $8,231 maximum EITC plus ACTC of 15% x ($30,517 - $2,500) = $4,203. Its claim of the 'full Child Tax Credit' is wrong too: the refundable portion is limited by the 15% earned-income formula, not paid at the per-child amount, and its total is consistent with neither." +us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its ACTC of $4,124 equals 15% x ($30,494 - $3,000). That uses the $3,000 threshold instead of $2,500, and 0.9235 x SE income instead of SE income minus half the SE tax; the correct ACTC is $4,203. Its EITC of about $8,200 is also below the 2026 maximum of $8,231." +us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It phased the EITC down to $7,128 even though AGI is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,128 matches 15% x ($30,517 - $3,000): it used the $3,000 threshold instead of $2,500, which gives $4,203." +us,scenario_008,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no component breakdown, and its $9,750 is $2,684 below the correct total of the $8,231 maximum EITC plus a $4,203 ACTC (15% x ($30,517 - $2,500)). The figure implies it sharply understated the EITC, the ACTC, or both, rather than computing either from the plateau and the earned-income formula." +us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,203 was correct. For the EITC it used the 2025 three-child maximum of $8,046 instead of the 2026 maximum of $8,231, which understated the total by $185." +us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It gave no component breakdown, and its $11,940 is $494 below the correct total. That total is the full 2026 three-child EITC of $8,231 (AGI is below the joint phase-out start) plus ACTC of 15% x ($30,517 - $2,500) = $4,203. The shortfall is consistent with either a stale or phased-out EITC or the $3,000 ACTC threshold." +us,scenario_008,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $4,128 matches 15% x ($30,517 - $3,000), so it used the $3,000 earned-income threshold instead of $2,500, which gives $4,203. Its EITC of $8,233 is essentially the $8,231 maximum." +us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive calculation to evaluate. The correct total is the $8,231 EITC plus a $4,202.61 refundable CTC." +us,scenario_008,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,360 instead of $8,231. For ACTC earned income it used 0.9235 x SE income ($30,494) instead of SE income minus the half-SE-tax deduction ($30,517), giving $4,199 instead of $4,203." +us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It denied both credits, claiming the qualifying-child facts were not established. The six resident children aged 1-17 are all EITC qualifying children, and the five under 17 are CTC qualifying children. With $30,517 of earned income, that yields an $8,231 EITC and a $4,203 refundable CTC." +us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0, claiming no qualifying income structure. The household has $30,517 of earned income and six qualifying children, which yields the full $8,231 three-child EITC plus a refundable CTC of 15% x ($30,517 - $2,500) = $4,203." +us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct. It computed ACTC on gross earnings of $30,800, but earned income counts SE income net of the half-SE-tax deduction ($4,000 - $282.59), which gives $30,517.41. The ACTC is therefore $4,202.61, not $4,245." +us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the 2025 three-child EITC maximum of $8,046 instead of the 2026 maximum of $8,231. It also computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction ($30,517, giving $4,203)." +us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 was correct, but its ACTC of $4,245 is 15% of gross earnings of $30,800 minus $2,500. It never subtracted the $282.59 half-SE-tax deduction from SE income; the correct earned income is $30,517.41 and the correct ACTC is $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 equals the correct $8,231 EITC plus an ACTC of $4,245, which is 15% x ($30,800 - $2,500) on gross earnings. It did not reduce SE income by the half-SE-tax deduction; the correct earned income of $30,517.41 gives an ACTC of $4,202.61." +us,scenario_008,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It put the EITC at $8,162 instead of the full 2026 maximum of $8,231; AGI is below the joint phase-out start. It computed ACTC on gross earnings of $30,800 ($4,245) instead of SE income minus the half-SE-tax deduction, which gives $4,203." +us,scenario_008,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $4,203 was correct. It overstated the 2026 three-child EITC maximum as $8,342 instead of $8,231." +us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,other,False,"It treated zero net tax as a reason for $0 refundable credits, which gets refundability backwards: the EITC and the refundable CTC are paid even when there is no liability. This household gets the $8,231 EITC plus a $4,203 refundable CTC." +us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules to the CTC: a $3,000 ACTC earned-income threshold and a $1,000 per-child amount. In 2026 the threshold is $2,500, which gives an ACTC of $4,203 instead of $4,124. It also overstated the EITC maximum as $8,247 instead of $8,231." +us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed ACTC with the $3,000 earned-income threshold ($4,124) instead of the 2026 threshold of $2,500, which gives 15% x ($30,517 - $2,500) = $4,203. It also rounded the EITC maximum to $8,250 instead of $8,231." +us,scenario_008,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its EITC of $8,231 was correct. For ACTC it used the 'post-sunset' $3,000 threshold instead of the 2026 threshold of $2,500, and 0.9235 x SE income ($30,494) instead of SE income minus half the SE tax ($30,517); the ACTC is $4,202.61, not $4,124.10." +us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"It phased the EITC down to $7,901 even though AGI of $30,632 is below the 2026 joint phase-out start, so the full $8,231 applies. Its ACTC of $4,199 used 0.9235 x SE income instead of SE income minus half the SE tax, which gives $4,203." +us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting the half-SE-tax deduction; the correct earned income of $30,517 gives $4,203." +us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive calculation to evaluate. The correct total is the $8,231 EITC plus a $4,202.61 refundable CTC." +us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its EITC of $8,231 and its AOTC of $0 were correct. Its ACTC of $4,245 applies 15% to gross earnings of $30,800 minus $2,500, but earned income counts SE income net of the $282.59 half-SE-tax deduction. That makes earned income $30,517.41 and the ACTC $4,202.61." +us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used $7,152, the two-child EITC maximum, instead of the $8,231 maximum for three or more children. It then applied a phase-out from $28,120, which is below the 2026 joint threshold, reaching $6,588. Its ACTC used gross earnings of $30,800 ($4,245) instead of SE income minus half the SE tax ($4,203)." +us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It overstated the 2026 three-child EITC maximum as $8,262 instead of $8,231. It computed ACTC on gross earnings of $30,800 ($4,245) without subtracting the half-SE-tax deduction from SE income; the correct earned income of $30,517 gives $4,203." +us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It used the 2025 EITC maximum of $8,046 and a phase-out start of $26,210, which is below the joint-filer threshold, and cut the EITC to $7,144 instead of the full $8,231. It also added a $1,000 refundable AOTC even though qualified education expenses are $0, and its ACTC used 0.9235 x SE income ($4,199 instead of $4,203)." +us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It limited the ACTC to $1,600 based on 'remaining tax liability room'. The ACTC is limited by 15% of earned income over $2,500 ($4,203), not by tax liability. It also understated the EITC at $6,801 instead of the $8,231 plateau amount, and it added $6,801 + $1,600 incorrectly to get $6,586." us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required a separately provided school-meals eligibility indicator and treated housing assistance as the only relevant benefit fact. It failed to compute the income-based free tier at 55% of the federal poverty guideline and also missed categorical eligibility through the household's computed SNAP eligibility. us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted income exceeds New Jersey's guidelines, but it never scaled the FPL to the eight-person household: 138% of the eight-person 2026 FPL is roughly $76,000 while total countable MAGI is $30,915, i.e. 0.55 x FPL. Its denial is consistent with testing the household's combined $30,915 against a one- or two-person 138% FPL limit (roughly $21,600 / $29,200), which is the only way this income clears any NJ adult Medicaid threshold." us,scenario_008,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model correctly identified that NJ adult eligibility runs through the ACA expansion rule but applied the 138% FPL limit without sizing it to the eight-person household, so $30,915 of MAGI read as too high when it is 0.55 x FPL against a 138% limit of roughly $76,000. Household size drives the FPL denominator in the MAGI test, and with six children plus two adults this family sits at less than half the expansion ceiling." @@ -538,89 +586,102 @@ us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model computed the MAGI test correctly (income under the 138% FPL line for eight people, head in an eligible non-elderly adult category) and then overrode its own result with a fabricated rule that receiving housing assistance disqualifies adults from Medicaid. MAGI Medicaid has no housing-assistance bar and no asset test; Section 8 or public housing subsidies are excluded from MAGI income entirely, so the household's receipt of housing assistance leaves the ADULT expansion determination untouched." us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly transferred the infant's categorical eligibility to the Head through household membership. WIC requires the applicant personally to be a pregnant, postpartum, or breastfeeding woman, infant, or child under age 5; the 42-year-old Head meets none of those categories and is therefore ineligible despite household income below the limit." us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It asserted the NJ employee temporary disability insurance rate is ""~0% in recent years"" and set TDI to $0, when the 2026 employee TDI rate of 0.19% yields $50.92 on the $26,800 wage base. It then understated FLI at 0.09% ($24.12) instead of 0.23% ($61.64) and padded the total with employee UI (0.3825%) and workforce-development (0.0425%) contributions that this payroll_tax measure excludes, producing $128.65 of NJ tax instead of $112.56." -us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It stopped at federal FICA ($1,661.60 + $388.60 = $2,050.20) and applied no New Jersey employee contribution, despite the output definition explicitly including mandatory employee state payroll taxes. The omitted NJ temporary disability (0.19% = $50.92) and family leave insurance (0.23% = $61.64) contributions account for the entire $112.56 shortfall." -us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It first applied a 0.765% NJ unemployment-insurance rate, then abandoned that figure mid-explanation and substituted an invented ""$2.12 rounding adjustment / minimum assessment"" as the entire state portion. The NJ component is $112.56 — employee TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64) on the full $26,800 — and no unemployment-insurance contribution belongs in this measure at all." -us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It stated the 2026 NJ SDI employee rate is 0% and, hedging on FLI and UI, discarded the state component entirely to submit bare federal FICA. The NJ employee TDI rate is 0.19% ($50.92) and the FLI rate is 0.23% ($61.64), which together add the missing $112.56." -us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It declared that NJ mandatory employee UI/SDI/FLI taxes ""are not modeled here"" and set the state portion to 0, contradicting the output definition's explicit inclusion of mandatory employee state payroll taxes. NJ employee TDI (0.19% = $50.92) and FLI (0.23% = $61.64) on $26,800 supply the entire $112.56 gap." -us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It ruled NJ employee UI/TDI/FLI out of scope by fiat and reported only 7.65% of $26,800, further rounding away the $0.20 of exact FICA. NJ employee TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) are mandatory employee contributions inside this measure and add $112.56." -us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It reasoned that NJ TDI is employer-funded with a 0% employee rate and that FLI is ""treated separately,"" then excluded both. New Jersey funds temporary disability with a mandatory 0.19% employee contribution ($50.92) alongside the 0.23% FLI employee contribution ($61.64), and both are employee-side state payroll taxes totaling $112.56." -us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It computed a NJ estimate of roughly $201-$210 using a blended UI/DI/FLI rate near 0.75%-0.78%, then deleted it from the total citing wage-base cap uncertainty and submitted federal FICA alone. The $26,800 in wages sits below every NJ contribution cap, and the in-scope employee rates are TDI 0.19% ($50.92) plus FLI 0.23% ($61.64) = $112.56, with employee UI excluded." -us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed federal FICA correctly at $2,050.20 but applied no New Jersey employee contribution and then truncated its own figure to $2,050. The state component is $112.56: employee TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) on the $26,800 wage base." -us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It identified the correct two NJ components, TDI and FLI, but used stale employee rates of 0.10% and 0.09%, producing $50.92 of state tax instead of $112.56. The 2026 employee rates are 0.19% for TDI ($50.92) and 0.23% for FLI ($61.64)." -us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It attached an undifferentiated NJ state payroll figure of $154.10 — 0.575% of wages, a rate that only reaches that level by folding in the employee UI/workforce-development contribution. The in-scope NJ employee taxes are TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64), $112.56 combined, with UI/WF excluded from this measure." -us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It built the NJ portion from ""SUI and FLI"" at roughly 0.515% ($138.02), including the employee unemployment-insurance contribution this output excludes while omitting temporary disability entirely. The correct NJ portion is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64) = $112.56." -us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only the 7.65% federal employee rate to $26,800 and rounded to $2,050, ignoring the mandatory state component named in the output definition. NJ employee TDI (0.19% = $50.92) and FLI (0.23% = $61.64) add $112.56." -us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It rounded federal FICA to $2,050 and appended ""roughly $138"" of NJ SUI/FLI/TDI without decomposing the rates, a figure that only arises by including the employee unemployment-insurance contribution excluded from this measure. The exact NJ portion is $50.92 of TDI at 0.19% plus $61.64 of FLI at 0.23%, or $112.56." -us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It estimated the NJ employee component as ""UI/FLI"" at $123.80 (0.462% of wages), substituting the out-of-scope unemployment-insurance contribution for the in-scope temporary disability contribution. The NJ portion consists solely of TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) = $112.56." -us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It applied only the combined 7.65% federal employee rate to $26,800 and reported no state component. New Jersey's mandatory employee TDI (0.19% = $50.92) and FLI (0.23% = $61.64) contributions add the missing $112.56." -us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It added $138.02 of ""NJ mandatory employee SUI/FLI"" on top of correct federal FICA, importing the employee unemployment-insurance contribution this output excludes and omitting temporary disability. The NJ portion is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64) = $112.56." -us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"It bundled FLI with SUI and the workforce-development surcharge for $139.32 of NJ tax, and those UI/WF items sit outside this employee payroll-tax measure. New Jersey contributes exactly $112.56 here: TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) on $26,800." -us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It computed only the two federal components on the spouse's $26,800 and reported no New Jersey employee contribution. The mandatory NJ employee TDI (0.19% = $50.92) and FLI (0.23% = $61.64) contributions account for the full $112.56 difference." -us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for payroll_tax, so no substantive computation was submitted. The required derivation is federal FICA of $2,050.20 on the spouse's $26,800 plus NJ employee TDI of $50.92 and FLI of $61.64, totaling $2,162.76." -us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It explicitly concluded that no NJ employee state payroll tax is modeled and submitted federal FICA alone. New Jersey imposes mandatory employee temporary disability (0.19% = $50.92) and family leave insurance (0.23% = $61.64) contributions on the $26,800 in wages, adding $112.56." -us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its explanation derived the correct federal employee figures ($1,661.60 + $388.60 = $2,050.20) but it submitted $4,097 — essentially the combined 15.3% employer-plus-employee rate on $26,800 — which violates the instruction to exclude employer payroll taxes and contradicts its own stated arithmetic. It also omitted the NJ employee TDI ($50.92) and FLI ($61.64) contributions that make up the remaining $112.56 of the correct $2,162.76." -us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It named no wage base and no rate decomposition, submitting $1,525 — $525 below the 7.65% federal employee tax on $26,800 and consistent with applying the FICA rate to roughly $19,900 of wages it never identified. The correct derivation is $1,661.60 Social Security + $388.60 Medicare + $50.92 NJ TDI + $61.64 NJ FLI = $2,162.76." -us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It applied a blended NJ employee rate of 0.9425% ($252.59) that folds the UI/WF/SWF contribution in with TDI and FLI, more than doubling the state component. This measure counts only NJ employee TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64), a combined 0.42% or $112.56, and excludes employee unemployment-insurance contributions." -us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It appended $253.26 of ""NJ employee payroll contributions,"" roughly 0.945% of wages, a rate reachable only by including the out-of-scope UI/workforce-development contribution. The NJ component is $112.56: temporary disability at 0.19% ($50.92) plus family leave insurance at 0.23% ($61.64)." -us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"It estimated NJ mandatory employee contributions at $263.98, roughly 0.985% of the $26,800 wage base and more than double the in-scope rate. The correct NJ portion is the 0.19% TDI contribution ($50.92) plus the 0.23% FLI contribution ($61.64) = $112.56, with the employee UI/WF contribution excluded from this output." -us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It added $202.34 of NJ employee contributions, a 0.755% blended rate that includes the employee unemployment-insurance and workforce-development items this measure excludes. The NJ portion is exactly $112.56 — TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) on $26,800." -us,scenario_008,payroll_tax,gpt-6-astra,llm_error,state_local_rule,False,"It explicitly summed NJ unemployment, workforce, disability and family-leave contributions for $226.46, but the employee UI and workforce-development items fall outside this payroll_tax measure. Only the disability and family-leave employee contributions count: 0.19% ($50.92) plus 0.23% ($61.64) = $112.56." -us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only the 7.65% federal employee rate to the spouse's $26,800 and rounded to $2,050, omitting the state component entirely. New Jersey's mandatory employee TDI (0.19% = $50.92) and FLI (0.23% = $61.64) contributions add $112.56." -us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It selected the right two NJ components but used a stale combined TDI+FLI employee rate of 0.18% ($48.24) instead of the 2026 rates of 0.19% TDI and 0.23% FLI, which total 0.42% or $112.56. That rate error accounts for $64.32 of the shortfall, with the rest from rounding $2,098.44 down to $2,098." -us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It added the NJ employee unemployment-insurance contribution at 0.3825% on top of TDI/FLI for about $178 of state tax, inflating the state portion with an item this measure excludes. The NJ component is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64) = $112.56, and it also rounded federal FICA from $2,050.20 to $2,050." -us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It stated outright that there are no state payroll taxes and reported 7.65% of $26,800 rounded to $2,050. New Jersey imposes mandatory employee temporary disability (0.19% = $50.92) and family leave insurance (0.23% = $61.64) contributions, adding $112.56 to the federal $2,050.20." -us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"It built the NJ portion from employee UI at 0.3825% plus FLI at 0.09% for about $127, using an out-of-scope unemployment contribution, an outdated FLI rate, and no temporary disability contribution at all. The correct NJ portion is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64) = $112.56, and it also rounded the federal components to $1,662 and $389." -us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for payroll_tax, so nothing substantive was submitted for scoring. The required computation is $1,661.60 Social Security + $388.60 Medicare + $50.92 NJ TDI + $61.64 NJ FLI = $2,162.76." -us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It summed NJ employee UI/workforce, temporary disability, and family-leave contributions for $257.95, more than doubling the state component by including the UI/WF contribution this measure excludes. Only TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) count, for $112.56 on the $26,800 wage base." -us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It computed federal FICA of $2,050.20, rounded it to $2,050, and applied no New Jersey employee contribution. NJ employee TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) on $26,800 supply the missing $112.56." -us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,It asserted that no mandatory employee NJ payroll tax is modeled and submitted federal FICA alone. New Jersey's employee temporary disability contribution (0.19% = $50.92) and family leave insurance contribution (0.23% = $61.64) are mandatory employee-side state payroll taxes worth $112.56 on these wages. -us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It set both the NJ employee disability and family-leave rates to 0% and used the employee UI/WF/SWF rate of 0.3825% ($102.51) as the sole state item, precisely inverting the composition of this measure. The NJ portion consists of TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) = $112.56, and employee UI/WF contributions are excluded." -us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied only the combined 7.65% federal employee rate to the spouse's $26,800 and reported no state component. The mandatory NJ employee TDI contribution (0.19% = $50.92) and FLI contribution (0.23% = $61.64) add $112.56 to reach $2,162.76." +us,scenario_008,payroll_tax,claude-fable-5,reference_engine_defect,payroll_tax_base,False,"Added the NJ UI (0.3825%) and workforce/SWF (0.0425%) employee contributions, which New Jersey requires and the frozen reference leaves out. It also set the TDI employee rate to 0% instead of 0.19% ($50.92) and used a stale 0.09% FLI rate instead of 0.23% ($61.64), giving $128.65 of NJ tax, against the frozen reference's $112.56 and the $226.46 NJ component of the exclusion's corrected value." +us,scenario_008,payroll_tax,claude-fable-5.1,reference_engine_defect,payroll_tax_base,False,"Stopped at federal FICA ($2,050.20) and left out NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions, which add $112.56." +us,scenario_008,payroll_tax,claude-haiku-4.5,reference_engine_defect,payroll_tax_base,False,"Computed FICA of $2,050.20 correctly, then dropped its own NJ calculation and added an invented '$2.12 minimum assessment'. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), which is $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46." +us,scenario_008,payroll_tax,claude-opus-4.7,reference_engine_defect,payroll_tax_base,False,"Treated the 2026 NJ TDI employee rate as 0%, was unsure of the FLI rate, and fell back to FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64)." +us,scenario_008,payroll_tax,claude-opus-4.8,reference_engine_defect,payroll_tax_base,False,Assumed NJ employee UI/SDI/FLI contributions are 'not modeled' and set them to $0. The output counts NJ TDI ($50.92 at 0.19%) and FLI ($61.64 at 0.23%) as mandatory employee state payroll taxes. +us,scenario_008,payroll_tax,claude-opus-5,reference_engine_defect,payroll_tax_base,False,"Explicitly left NJ employee TDI and FLI out of the measure, even though the question's definition includes mandatory employee state payroll taxes. That drops $112.56 (0.19% TDI + 0.23% FLI), and the model also rounded FICA down to $2,050." +us,scenario_008,payroll_tax,claude-opus-5.5,reference_engine_defect,payroll_tax_base,False,"Applied only the 7.65% federal FICA rate and never added NJ's employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions." +us,scenario_008,payroll_tax,claude-sonnet-4.6,reference_engine_defect,payroll_tax_base,False,"Wrongly concluded that NJ TDI is employer-funded with a 0% employee rate and that FLI does not count as a payroll tax, so it reported FICA only. NJ withholds TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) from employees, and both are included." +us,scenario_008,payroll_tax,claude-sonnet-5,reference_engine_defect,payroll_tax_base,False,"Recognized that NJ has mandatory employee UI/DI/FLI taxes and estimated about $201, then set them to $0 because of 'cap uncertainty'. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46. Wages are well below the cap." +us,scenario_008,payroll_tax,claude-sonnet-5.5,reference_engine_defect,payroll_tax_base,False,Assumed NJ employee payroll taxes are 'not modeled' and reported FICA only. That omits TDI of $50.92 (0.19%) and FLI of $61.64 (0.23%). +us,scenario_008,payroll_tax,deepseek-v4-flash-0731,reference_engine_defect,payroll_tax_base,False,"Computed FICA only and rounded $2,050.20 down to $2,050. It omitted the NJ employee TDI (0.19%) and FLI (0.23%) contributions totaling $112.56." +us,scenario_008,payroll_tax,deepseek-v4-pro,reference_engine_defect,payroll_tax_base,False,"Included NJ TDI and FLI but used outdated rates: 0.10% TDI ($26.80) and 0.09% FLI ($24.12). The 2026 rates are 0.19% TDI ($50.92) and 0.23% FLI ($61.64), so NJ tax was understated by $61.64." +us,scenario_008,payroll_tax,deepseek-v4-pro-0813,reference_engine_defect,payroll_tax_base,False,"Put NJ employee payroll taxes at $154.10 (about 0.575% of wages). That is $41.54 above the frozen reference's TDI at 0.19% plus FLI at 0.23% ($112.56) and $72.36 below the $226.46 of the exclusion's corrected value, which adds the worker unemployment and workforce contributions New Jersey requires and the frozen reference leaves out." +us,scenario_008,payroll_tax,deepseek-v4.1-flash,reference_engine_defect,payroll_tax_base,False,"Reported federal FICA only and omitted NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions." +us,scenario_008,payroll_tax,gemini-3-flash-preview,reference_engine_defect,payroll_tax_base,False,"Added 0.515% for NJ SUI plus FLI ($138.02). It counted the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and left out TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44." +us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA and rounded to $2,050, omitting the NJ employee TDI and FLI contributions of $112.56." +us,scenario_008,payroll_tax,gemini-3.1-pro-preview,reference_engine_defect,payroll_tax_base,False,"Added about $138 of NJ SUI/FLI/TDI. It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and misstated the rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, and the model's $2,188 is $88.66 below the corrected value, $2,276.66." +us,scenario_008,payroll_tax,gemini-3.5-flash,reference_engine_defect,payroll_tax_base,False,"Added $123.80 of NJ UI/FLI. It counted the UI employee contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI (0.19%, $50.92). Its $2,174 is $102.66 below the exclusion's corrected value, $2,276.66." +us,scenario_008,payroll_tax,gemini-3.5-flash-lite,reference_engine_defect,payroll_tax_base,False,Applied only 7.65% federal FICA and never added NJ's employee TDI (0.19%) and FLI (0.23%) contributions of $112.56. +us,scenario_008,payroll_tax,gemini-3.6-flash,reference_engine_defect,payroll_tax_base,False,"Added $138.02 of NJ SUI/FLI. It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44." +us,scenario_008,payroll_tax,gemini-3.7-flash,reference_engine_defect,payroll_tax_base,False,"Added $139.32 of NJ FLI/SUI/WF. It included the UI and workforce development contributions, which New Jersey requires and the frozen reference leaves out, and left out TDI (0.19%, $50.92). Its $2,189.52 is $87.14 below the exclusion's corrected value, $2,276.66." +us,scenario_008,payroll_tax,gemini-3.8-flash,reference_engine_defect,payroll_tax_base,False,Reported FICA only and omitted the NJ employee TDI ($50.92) and FLI ($61.64) contributions. +us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_008,payroll_tax,glm-5.3,reference_engine_defect,payroll_tax_base,False,Explicitly treated NJ employee state payroll tax as 'not modeled' and reported FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64). +us,scenario_008,payroll_tax,gpt-5.4-mini,reference_engine_defect,payroll_tax_base,False,"Correctly worked out $1,661.60 Social Security and $388.60 Medicare, then submitted an unsupported $4,097, roughly double its own FICA sum. It also added no NJ state payroll tax; the frozen reference's total is $2,050.20 + $112.56 NJ TDI/FLI ($2,162.76), and the exclusion's corrected value adds $113.90 of NJ worker unemployment and workforce contributions, for $2,276.66." +us,scenario_008,payroll_tax,gpt-5.4-nano,reference_engine_defect,payroll_tax_base,False,"Reported $1,525, which is below even the 7.65% FICA on $26,800 ($2,050.20), so its rate arithmetic was wrong. It also added nothing for NJ TDI and FLI ($112.56)." +us,scenario_008,payroll_tax,gpt-5.5,reference_engine_defect,payroll_tax_base,False,"Applied a combined 0.9425% NJ rate ($252.59) that includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, along with an overstated FLI rate. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $252.59 exceeds by $26.13." +us,scenario_008,payroll_tax,gpt-5.6-luna,reference_engine_defect,payroll_tax_base,False,"Added $253.26 of NJ contributions (0.945%). That is 0.425% UI/WF/SWF, which New Jersey requires and the frozen reference leaves out, plus TDI 0.19% and FLI at 0.33% instead of the 2026 rate of 0.23%, so it exceeds the $226.46 of the exclusion's corrected value by $26.80." +us,scenario_008,payroll_tax,gpt-5.6-sol,reference_engine_defect,payroll_tax_base,False,"Added $263.98 of NJ contributions (0.985%). That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, and overstates FLI at 0.33% and TDI at 0.23%. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI, or $112.56; the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52." +us,scenario_008,payroll_tax,gpt-5.6-terra,reference_engine_defect,payroll_tax_base,False,"Added $202.34 of NJ contributions (0.755%). That includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, and misstates TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $202.34 falls $24.12 short of." +us,scenario_008,payroll_tax,gpt-6-astra,reference_engine_defect,payroll_tax_base,False,"Used the correct TDI (0.19%) and FLI (0.23%) rates and added the 0.425% UI/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. The frozen reference counts only TDI and FLI ($112.56); the model's $226.46 NJ addition, for $2,276.66, is the exclusion's corrected value." +us,scenario_008,payroll_tax,gpt-6-luna,reference_engine_defect,payroll_tax_base,False,"Added $263.98 of NJ contributions (0.985%). That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, with overstated TDI/FLI rates. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI ($112.56); the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52." +us,scenario_008,payroll_tax,gpt-6-sol,reference_engine_defect,payroll_tax_base,False,"Applied TDI at 0.19% and FLI at 0.23% correctly and added the 0.425% unemployment/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. Its NJ addition of $226.46, for $2,276.66, is the exclusion's corrected value; the frozen reference's is $112.56." +us,scenario_008,payroll_tax,gpt-6.1-sol,reference_engine_defect,payroll_tax_base,False,"Added $215.07 of NJ contributions: 0.3825% UI/workforce ($102.51) plus TDI 0.19% and FLI 0.23%. That 0.3825% is the worker unemployment contribution, which New Jersey requires and the frozen reference leaves out; the model did not add the 0.0425% workforce share ($11.39), so its $2,265.27 is $102.51 above the frozen reference's $2,162.76 and $11.39 below the exclusion's corrected value, $2,276.66." +us,scenario_008,payroll_tax,grok-4.3,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA (rounded to $2,050) and omitted NJ's mandatory employee TDI (0.19%) and FLI (0.23%) contributions of $112.56." +us,scenario_008,payroll_tax,grok-4.5,reference_engine_defect,payroll_tax_base,False,"Included NJ TDI and FLI but at a stale combined rate of 0.18% ($48.24). The 2026 rates are 0.19% TDI plus 0.23% FLI, or 0.42% ($112.56)." +us,scenario_008,payroll_tax,grok-4.6,reference_engine_defect,payroll_tax_base,False,"Added the NJ employee UI contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, plus about $75 of TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds UI and the 0.0425% workforce share ($113.90), for $226.46, and the model's $2,228 is $48.66 below the corrected value, $2,276.66." +us,scenario_008,payroll_tax,grok-4.7,reference_engine_defect,payroll_tax_base,False,"Included a 0.425% UI/workforce contribution, which New Jersey requires and the frozen reference leaves out, and overstated TDI (0.23% vs 0.19%) and FLI (0.33% vs 0.23%). That gives $263.98 of NJ tax, $151.42 above the frozen reference's $112.56 and $37.52 above the $226.46 of the exclusion's corrected value." +us,scenario_008,payroll_tax,grok-build-0.1,reference_engine_defect,payroll_tax_base,False,"Explicitly assumed there are no state payroll taxes and reported FICA only (rounded to $2,050). It omitted NJ TDI ($50.92) and FLI ($61.64)." +us,scenario_008,payroll_tax,inkling,reference_engine_defect,payroll_tax_base,False,"Added the NJ UI employee contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, used a stale 0.09% FLI rate instead of 0.23%, and omitted TDI at 0.19% and the 0.0425% workforce share. That gave about $127 of NJ tax, against the frozen reference's $112.56 and the $226.46 of the exclusion's corrected value; its $2,177 is $99.66 below the corrected value, $2,276.66." +us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_008,payroll_tax,kimi-k3,reference_engine_defect,payroll_tax_base,False,"Added $257.95 of NJ contributions, including the UI/workforce employee contribution, which New Jersey requires and the frozen reference leaves out, and inflated TDI/FLI rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $257.95 exceeds by $31.49." +us,scenario_008,payroll_tax,minimax-m3,reference_engine_defect,payroll_tax_base,False,"Computed FICA only (rounded to $2,050) and never added NJ's mandatory employee TDI (0.19%) and FLI (0.23%) contributions of $112.56." +us,scenario_008,payroll_tax,ox-alpha,reference_engine_defect,payroll_tax_base,False,Assumed no NJ employee payroll tax is modeled and reported FICA only. That omits TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64). +us,scenario_008,payroll_tax,qwen-3.7-max,reference_engine_defect,payroll_tax_base,False,"Used 2024-era assumptions: NJ employee DI and FLI both at 0%, plus the 0.3825% UI/WF/SWF contribution ($102.51). For 2026 the frozen reference counts TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) and leaves out UI, which New Jersey requires; the exclusion's corrected value counts UI at 0.3825% and the workforce funds at 0.0425% ($113.90), for $2,276.66, which its $2,152.71 misses by $123.95: TDI, FLI and the workforce share ($11.39)." +us,scenario_008,payroll_tax,qwen3.8-max,reference_engine_defect,payroll_tax_base,False,"Applied only 7.65% FICA to the spouse's wages and omitted NJ's mandatory employee TDI (0.19%, $50.92) and FLI (0.23%, $61.64) contributions." us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling and housing assistance as sufficient for reduced-price support, without applying the mutually exclusive free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility through SNAP/TANF, the household qualifies for free meals, which makes reduced-price eligibility false." us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract." us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $306, exactly 7.65% of $4,000, despite stating that it applied 15.3%. It effectively used only half the combined Social Security and Medicare self-employment tax rate and also failed to apply the 92.35% statutory earnings factor, instead of calculating $4,000 × 0.9235 × 0.153 = $565.18." us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract." us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated gross self-employment tax of $565.18, then incorrectly subtracted a supposed deduction from that tax liability. The deductible half of self-employment tax reduces adjusted gross income for income-tax purposes; it is not a credit against self-employment tax, so the liability remains $565.18." -us,scenario_008,snap,claude-fable-5,llm_error,other,False,"It computed a defensible $1,214/month allotment ($1,756 max minus 30% of $1,805 net) and then discarded it, submitting an unexplained $687/month — barely half its own result. It also used the FY2025 maximum allotment of $1,756 instead of the $1,789 in effect for FY2026 and a $258 standard deduction instead of $299, and asserted that housing assistance disallows the shelter deduction when shelter is zero here simply because $579.08 rent falls below half of adjusted income." -us,scenario_008,snap,claude-fable-5.1,llm_error,period_annualization,False,"It reproduced the reference derivation — gross $2,576.25, $513.33 earned-income deduction, ~$300 standard deduction, no excess shelter, no medical deduction, $1,789 maximum minus 30% of net — and then multiplied the resulting $1,260.72 monthly benefit by 12. Calendar 2026 straddles the October 2026 allotment adjustment that lifts the 8-person maximum to about $1,829, so the final three months pay roughly $1,300/month rather than $1,260." -us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used $1,164 as the 8-person maximum allotment — roughly the five-person figure, against the correct $1,789 — and omitted the 20% earned-income deduction entirely, subtracting only a $227 standard deduction to reach $2,349 of countable income. Its own stated inputs yield $459/month, not the $583 it submitted." -us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted the monthly standard deduction (~$280) a single time from annual income, leaving net income of $24,475/year ($2,040/month) instead of $1,763.92/month and inflating the 30% household contribution by about $83/month. It compounded this with the FY2025 maximum allotment of $1,756 rather than FY2026's $1,789, then submitted $13,620 instead of the $13,728 its own arithmetic produced." -us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It applied the $258 standard deduction once against annual income rather than each month, overstating net income by roughly $2,800/year, and used the FY2025 $1,756 maximum allotment instead of FY2026's $1,789. It then abandoned its own $13,728 result for an unexplained $11,352 'adjusted for income/utility estimates.'" -us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"Its net income estimate of about $1,700/month is close to the correct $1,763.92, but subtracting $1,040 from its stated $1,802 maximum implies a household contribution of $762 — about 45% of net income rather than the statutory 30% ($529.18). Applying 30% to its own figures yields roughly $1,292/month, not $1,040." -us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried a nearly correct net income of $1,804.92/month through a long derivation and then applied a maximum allotment of $1,536 for an 8-person household, understating the FY2026 figure of $1,789 by $253/month. Its claim that housing assistance eliminates the shelter deduction is beside the point: excess shelter is zero because $579.08 rent is below half of adjusted income." -us,scenario_008,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It used $1,691/month as the 8-person maximum — the FY2023 value, against FY2026's $1,789 — and an internally inconsistent net income of 'around $8,800/year,' about one-third of what its own deduction list produces. Its $958/month follows from neither figure: $1,691 minus 30% of $733/month is $1,471." -us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It set the maximum allotment at about $1,780 instead of $1,789 and overstated net income at $1,830/month against the correct $1,763.92, while citing a shelter deduction that is zero because $579.08 rent falls below half of adjusted income. It also applied one monthly figure across all twelve months, dropping the October 2026 step-up to about $1,829." -us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It reached the reference net income of $1,763/month with exactly the right deductions ($302 standard plus $513 earned-income), then used a maximum allotment of about $1,900 — $111 above the $1,789 that applies through September 2026 and $71 above the ~$1,829 that applies from October. The inflated maximum accounts for the entire $1,207 overshoot." -us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $279 standard deduction (net income $1,783.92) instead of $299 and the FY2025 maximum allotment of $1,756 instead of FY2026's $1,789, then held that single figure for all twelve months, missing the October 2026 increase to about $1,829." -us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It submitted a round $1,275/month with no computation while crediting a shelter deduction that is zero: $579.08 monthly rent falls below half of adjusted income ($881.96) and no utility expense is listed. The correct schedule is $1,260.83/month for the nine months before the October 2026 adjustment and about $1,300.83 after it." -us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation and submitted $1,366.67/month, which implies a household contribution of only $422 against net income of $1,763.92 — 24% rather than the statutory 30% ($529.18). It never applied the benefit-reduction formula to a computed net income." -us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,307/month implies a household contribution of $482 against the $1,789 maximum, short of the required 30% of $1,763.92 net income ($529.18). Reaching that contribution requires deductions beyond the $513.33 earned-income and $299 standard amounts, and both the excess shelter and excess medical deductions are zero for this household." -us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied an excess shelter deduction built on 'New Jersey's standard utility allowance,' but no utility expense is listed and the prompt sets unlisted inputs to zero; with $579.08 rent below half of adjusted income ($881.96), the shelter deduction is zero and total monthly deductions are $812.33. That phantom deduction inflated the annual benefit by about $1,550." -us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted $1,208.33/month with no derivation. Against net income of $1,763.92 and the statutory 30% contribution of $529.18, that implies a maximum allotment of $1,737 — below both the $1,789 in effect through September 2026 and the ~$1,829 that applies afterward." -us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used FY2025 parameters — a $287 standard deduction and a $1,756 maximum allotment for eight — instead of FY2026's $299 and $1,789, and held that maximum across all of calendar 2026, missing the October increase to about $1,829. Every other step, including the correct zero shelter deduction, matches the reference." -us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It submitted $1,404/month, which leaves a household contribution of only $385 against the $1,789 maximum — 22% of the $1,763.92 net income rather than the statutory 30% ($529.18). It stated no income, deduction, or allotment figures at all." -us,scenario_008,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It submitted $1,316.67/month with no numbers, implying a contribution of $472 against the $1,789 maximum instead of the required 30% of $1,763.92 net income ($529.18). Its appeal to a shelter deduction is unsupported: excess shelter is zero because $579.08 rent is below half of adjusted income." -us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted the monthly standard deduction of $198 exactly once from annual income, producing net income of $24,557/year against the correct $21,167 — a $3,390 overstatement that cost about $1,017 of benefit at the 30% rate. It also used $198 rather than the $299 large-household standard deduction and a $1,748/month maximum allotment instead of $1,789." -us,scenario_008,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a $210 standard deduction instead of the $299 that applies to this 8-person household, leaving net income of $1,853/month against the correct $1,763.92 and raising the household contribution by about $27/month. Holding a single $1,797 maximum across the year also dropped the October 2026 increase to about $1,829." -us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned zero on the assertion that a positive benefit is not supportable, when the household's $2,576.25 monthly gross is 57% of the federal poverty guideline for eight — far under both the 130% gross test and the 100% net test — and it is categorically eligible through TANF non-cash assistance. Housing assistance is excluded from SNAP countable income and does not reduce the allotment." -us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned zero for want of stated deduction inputs, when the question supplies everything the formula needs: $30,915 of gross income, household size 8, and the instruction to treat unlisted inputs as zero, giving $513.33 earned-income plus $299 standard deductions and net income of $1,763.92/month. The benefit is the $1,789 maximum allotment minus 30% of that net income." -us,scenario_008,snap,gpt-5.5,llm_error,period_annualization,False,"It derived the correct monthly benefit of about $1,260 — gross $2,576.25 less the 20% earned-income and standard deductions, times 30%, subtracted from the $1,789 8-person maximum — and then multiplied by 12. The last three months of calendar 2026 fall after the October adjustment that raises the maximum to about $1,829, worth roughly $127 more." -us,scenario_008,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,139/month requires a household contribution of $650 against the $1,789 maximum, implying net income of about $2,167/month — roughly $403 above the correct $1,763.92. It credited only about half of the $812.33 in monthly deductions ($513.33 earned-income plus $299 standard), and the shelter deduction it cites is zero because rent is below half of adjusted income." -us,scenario_008,snap,gpt-5.6-sol,llm_error,period_annualization,False,"It landed within $2/month of the correct pre-October benefit and then applied that figure to all twelve months of 2026. The October 2026 adjustment lifts the 8-person maximum from $1,789 to about $1,829, so the final quarter pays roughly $1,300/month rather than $1,262." -us,scenario_008,snap,gpt-5.6-terra,llm_error,period_annualization,False,"Its $1,284.02/month implies a maximum allotment of about $1,813 held flat across the year — an average of the two fiscal-year maxima rather than $1,789 for the nine months before the October 2026 adjustment and about $1,829 for the three months after. Averaging over-pays the nine months still on the lower schedule." -us,scenario_008,snap,gpt-6-astra,llm_error,period_annualization,False,"It matched the reference exactly on net income ($21,167/year) and the 30% contribution, then set the annual maximum at $21,468 — the $1,789 monthly figure times 12. Calendar 2026 includes three months after the October adjustment to about $1,829, and that flat annualization drops the resulting $129." -us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It treated receipt of housing assistance as disqualifying and returned zero. Housing assistance is excluded from SNAP countable income and has no bearing on eligibility; this household's $2,576.25 monthly gross is 57% of the poverty guideline for eight and it is categorically eligible through TANF non-cash assistance." -us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a maximum allotment of about $1,822.50/month for the whole year, above the $1,789 that applies through September 2026, and cut net income to $1,741.67/month by crediting an excess-medical deduction for the disabled spouse's $300 of annual OTC expenses — $25/month, below the $35/month threshold, so that deduction is zero. Correct deductions are the $513.33 earned-income and $299 standard amounts only." -us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used FY2025 parameters throughout — a $291 standard deduction (net income $1,772) and a $1,759 maximum allotment — against FY2026's $299 and $1,789, and held that maximum through December, missing the October 2026 increase to about $1,829." -us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It denied the household under the $4,250 resource limit for units containing a disabled member. This household is categorically eligible through TANF non-cash assistance under New Jersey's broad-based categorical eligibility, which waives the SNAP resource test entirely, so the $69,500 bank balance is irrelevant to eligibility." -us,scenario_008,snap,inkling,llm_error,period_annualization,False,"It reached the correct net income of about $1,765/month and a $529 contribution, then applied a single $1,806 maximum allotment for all twelve months instead of $1,789 through September 2026 and about $1,829 from October. The averaged maximum over-pays the nine months still on the lower schedule." -us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for snap, so the submission carried nothing to score against the $15,246.91 reference. The failure is a missing output rather than a substantive SNAP computation error." -us,scenario_008,snap,kimi-k3,llm_error,period_annualization,False,"It matched the reference derivation digit for digit — $2,576.25 gross, $513.33 earned-income deduction, $299 standard deduction, $1,763.92 net income, $1,789 maximum, $1,259.83 monthly benefit — and then multiplied by 12. Three months of calendar 2026 fall after the October allotment adjustment to about $1,829, which the flat annualization omits." -us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It denied eligibility on a $3,750 asset limit for households with a disabled member. Categorical eligibility through TANF non-cash assistance waives the SNAP resource test in New Jersey, so the $69,500 bank balance does not disqualify the household, which passes both the gross and net income tests at 57% and 39% of the poverty guideline." -us,scenario_008,snap,ox-alpha,llm_error,period_annualization,False,"Its monthly derivation is structurally correct — $297 standard deduction, zero excess shelter because $579 rent is below half of adjusted income, no utility allowance, categorical eligibility waiving the asset test — but it applied a single $1,795 maximum allotment for all twelve months. The correct schedule is $1,789 through September 2026 and about $1,829 from October, which the flat multiplication misses." -us,scenario_008,snap,qwen-3.7-max,llm_error,other,False,"It used a $202/month standard deduction instead of $299 and a $1,838/month maximum allotment instead of $1,789, producing $15,391 by its own arithmetic, then cut that to $14,784 on the ground that housing assistance limits the shelter deduction. The shelter deduction is already zero in its own calculation, and housing assistance neither counts as income nor reduces the allotment." -us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed the household's roughly $2,576 monthly gross exceeds 130% of the poverty guideline for eight and returned zero. The 130% gross limit for an 8-person household is about $5,866/month; this household's gross is 57% of the guideline and its net income is 39%, so it passes both the gross and net income tests." +us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a stale $258 standard deduction and the FY2025 $1,756 maximum allotment instead of $299 and $1,789, and its own formula gave $1,214/month. It then replaced that with an unexplained $687/month, blamed on housing assistance disallowing a shelter deduction that it had already set to $0, so the change had no basis in its own derivation." +us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of the FY2026 $299. It also skipped the SNAP rounding steps: rounding net income to the whole dollar ($1,764) and rounding the 30% contribution up to $530. That gave $1,260.72/month instead of $1,789 − $530 = $1,259." +us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It left out the 20% earned income deduction ($513.33), used a $227 standard deduction instead of $299, and used a $1,164 maximum allotment for eight people instead of $1,789. Its $583/month does not even follow from its own inputs." +us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted the monthly standard deduction (about $280) once from annual income ($24,755 → $24,475) instead of 12 times. That inflated monthly net income to $2,040 instead of $1,764. It also used the FY2025 $1,756 maximum instead of $1,789, then gave $13,620 against its own $13,728." +us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted a $258 standard deduction once from annual income instead of $299 each month, which inflated net income to $2,041/month instead of $1,764. It used the FY2025 $1,756 maximum instead of $1,789, then cut its own $13,728 to $11,352 with an unspecified 'income/utility' adjustment." +us,scenario_008,snap,claude-opus-5,llm_error,thresholds_rates,False,"It used an $1,802 maximum allotment instead of $1,789 and net income of about $1,700 instead of $1,764, calling on a shelter deduction that does not apply. Its $1,040/month also contradicts its own formula, since $1,802 − 0.3 × $1,700 = $1,292." +us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"It had every parameter right ($299 standard deduction, $513.33 earned income deduction, $1,789 maximum) but skipped SNAP's rounding. Net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259/month, not $1,259.82." +us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $1,536 maximum allotment for eight people instead of the FY2026 $1,789, and a $258 standard deduction instead of the $299 for six or more people. That gave $1,536 − $541 = $995/month instead of $1,789 − $530 = $1,259." +us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It invented a shelter deduction with a utility allowance and a disability-related deduction, cutting net income to about $8,800/year when the correct figure is $1,764/month ($21,168/year). No utility costs are listed and rent is below half of adjusted income. It also used a $1,691 maximum instead of $1,789, and its $958/month does not match its own formula." +us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It had net income ($1,764) and the maximum ($1,789) right, but rounded the 30% contribution ($529.20) down to $529 instead of up to the next whole dollar ($530). That gave $1,260/month instead of $1,259." +us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated net income at about $1,830/month instead of $1,764, which implies a standard deduction below the $299 for six or more people. It also used a maximum allotment of about $1,780 instead of $1,789, which left it at about $1,225/month." +us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used an $1,900 maximum allotment for eight people instead of the FY2026 $1,789, and a $302 standard deduction instead of $299. It also skipped the rounding of net income and the 30% contribution, which gave $1,371.12/month instead of $1,259." +us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 $1,756 maximum allotment instead of the FY2026 $1,789. Its net income of $1,783.92 implies a $279 standard deduction instead of $299, which gave $1,220.83/month instead of $1,259." +us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used an $1,800 maximum allotment instead of $1,789. Its net income of $1,752.92 implies a $310 standard deduction instead of $299. It also skipped the round-up of the 30% contribution to $530, which gave $1,274.12/month." +us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It adjusted for a shelter deduction that does not apply, since $579 rent is below half of the $1,764 adjusted income and no utility costs are listed. Its $1,275/month implies a contribution of about $514 instead of $530, so its net income is below the correct $1,764." +us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no computation, and $16,400 ($1,366.67/month) is a guess about $108/month above the correct $1,789 − $530 = $1,259. It never applied the 30% contribution on the $1,764 net income." +us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,other,False,"Its $1,307/month implies a 30% contribution of only about $482 against the $1,789 maximum, meaning net income of about $1,607 instead of $1,764. It understated net income after the $513.33 earned income and $299 standard deductions, and so overstated the benefit." +us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added New Jersey's standard utility allowance to rent to create an excess shelter deduction. No utility expense is listed, so shelter costs are only the $579 rent, which is below half of adjusted income. That spurious deduction lowered net income below $1,764 and pushed the benefit to $1,399.75/month instead of $1,259." +us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation, and its $1,208.33/month matches the FY2025 $1,756 maximum minus a contribution of about $548 (net income near $1,826). The correct inputs are the FY2026 $1,789 maximum and a $530 contribution on $1,764 net income." +us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $287 standard deduction instead of the FY2026 $299 for six or more people, and the FY2025 $1,756 maximum instead of $1,789. That gave $1,756 − $533 = $1,223/month instead of $1,789 − $530 = $1,259." +us,scenario_008,snap,gemini-3.7-flash,llm_error,other,False,"It gave no computation. Its $1,404/month implies a contribution of only about $385 against the $1,789 maximum, meaning net income near $1,283, roughly $480 less than the correct $1,764. That reflects deductions that do not apply, since there is no shelter or medical deduction here." +us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It cited a shelter deduction that does not apply, since $579 rent is below half of the $1,764 adjusted income and there are no utility costs. Its $1,316.67/month implies a contribution of about $472 instead of $530, so it understated net income." +us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 standard deduction once from annual income, instead of the $299 monthly deduction applied each month ($3,588/year). That overstated net income at $24,557/year instead of $21,168. It also used a $1,748 maximum instead of $1,789." +us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of the $299 for six or more people, which raised net income to $1,853 instead of $1,764. It also used a $1,797 maximum instead of $1,789, which gave $1,241/month instead of $1,259." +us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned $0 on the claim that housing assistance and earnings push the household past a benefit threshold. Gross income is only 57% of the poverty guideline, NJ categorical eligibility applies, and housing assistance does not bar SNAP. The standard formula gives $1,789 − $530 = $1,259/month." +us,scenario_008,snap,gpt-5.4-nano,llm_error,other,False,"It returned $0 on the claim that no computation inputs were given, but household size, wages, self-employment income, interest and rent were all provided. Those inputs yield $1,764 net income and a $1,259 monthly benefit." +us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It applied the right deductions and maximum but gave $1,260/month. The 30% contribution on $1,764 net income ($529.20) rounds up to $530, which yields $1,259/month, not $1,260." +us,scenario_008,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,139/month implies a contribution of about $650 against the $1,789 maximum, meaning net income near $2,167 instead of $1,764. It under-applied the $513.33 earned income and $299 standard deductions it named, despite also claiming a shelter deduction." +us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"Its $1,262/month implies a contribution of $527 instead of $530. It did not round net income to $1,764 and round the 30% contribution up to the next whole dollar against the $1,789 maximum, which gives $1,259." +us,scenario_008,snap,gpt-5.6-terra,llm_error,other,False,"Its $1,284.02/month implies a contribution of about $505 instead of $530, meaning net income near $1,683 instead of $1,764. It overstated the deductions and never rounded the contribution up to a whole dollar." +us,scenario_008,snap,gpt-6-astra,llm_error,period_annualization,False,"It computed on annual totals ($21,468 maximum less 30% of $21,167 net) and skipped SNAP's monthly rounding. Monthly net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259 × 12 = $15,108, not $15,117.90." +us,scenario_008,snap,gpt-6-luna,llm_error,other,False,"Its $1,288.17/month implies a contribution of about $501 against the $1,789 maximum, meaning net income near $1,670 instead of $1,764. It overstated the earned income and standard deductions, or the maximum, instead of reaching the $530 contribution." +us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It truncated net income to $1,763 instead of rounding $1,763.92 to $1,764, and treated the contribution as roughly $529 instead of rounding 30% up to $530. That gave $1,260/month instead of $1,259." +us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 on the claim that housing assistance eliminates SNAP. Receiving housing assistance does not disqualify the household: it is categorically eligible in NJ with gross income at 57% of the poverty guideline, and its $1,764 net income yields $1,259/month." +us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a projected maximum of about $21,870/year ($1,822.50/month) instead of $1,789. It also applied a medical deduction from over-the-counter costs, but the spouse's $25/month is below the $35 threshold. Together these gave $1,300/month instead of $1,259." +us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a $1,759 maximum allotment instead of the FY2026 $1,789. Its net income of $1,772 implies a standard deduction of about $291 instead of $299, which gave $1,227/month instead of $1,259." +us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used FY2025 parameters: a $1,756 maximum ($21,072/year) and a $291 standard deduction ($3,492/year). The FY2026 values are $1,789 and $299. It also computed annually without the monthly rounding to a $530 contribution." +us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the federal $4,250 resource limit for disabled-member households to the $69,500 bank balance and zeroed SNAP. The household is categorically eligible through New Jersey's TANF-funded broad-based categorical eligibility, which removes the asset test." +us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used an $1,806 maximum allotment instead of the FY2026 $1,789. It also took a $529 contribution instead of rounding 30% of $1,764 up to $530, which gave about $1,276/month instead of $1,259." +us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value and no explanation, so there is no usable answer to compare with the $15,108 derived from $1,789 − $530 per month." +us,scenario_008,snap,kimi-k3,llm_error,other,False,"It used the correct $299 standard deduction, $513.33 earned income deduction and $1,789 maximum, but skipped the rounding steps. Net income rounds to $1,764 and the 30% contribution rounds up to $530, so the benefit is $1,259/month, not $1,259.825." +us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied a $3,750 asset limit to the $69,500 bank balance and declared the household ineligible. It ignored that New Jersey's broad-based categorical eligibility (TANF non-cash) waives the SNAP asset test." +us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used an $1,795 maximum allotment instead of the FY2026 $1,789, and a $297 standard deduction instead of $299. That gave $1,265/month instead of $1,789 − $530 = $1,259." +us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of $299 and an $1,838 maximum instead of $1,789, and left the $115 of interest out of net income. It then cut its own $15,391 to $14,784 on an unsupported housing-assistance shelter adjustment, even though no shelter deduction applies." +us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed $2,567/month exceeds 130% of the poverty guideline for eight people. That limit is about $5,900/month, and the household's gross income is only 57% of the guideline, so it passes the gross income test and receives $1,259/month." us,scenario_008,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Declared the spouse ineligible ""under adult NJ rules"" without running New Jersey's ACA expansion income test, which covers adults 19-64 up to 138% FPL and requires no categorical hook such as pregnancy or caretaker status. Household MAGI of about $30,900 against the eight-person FPL is 0.55 x FPL, far inside that limit; the bare denial is consistent with comparing the spouse's own $26,800 of wages to a one- or two-person 138% FPL line (roughly $22,600 to $30,700) instead of household MAGI over a family size of eight." us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to 0 by treating Medicaid eligibility as something requiring an affirmative listed trigger fact, extending the prompt's ""treat unlisted inputs as false"" rule to the computed output itself. Every input the determination needs was supplied - age 41, non-dependent, New Jersey, household income - and the MAGI adult-expansion test it never ran places household MAGI at 0.55 x FPL against the 138% FPL threshold, which returns eligible." us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Ran the MAGI test correctly - about $31,420 of household income against the roughly $73,637 138%-FPL line for eight people - then reversed its own correct result with a fabricated rule that receipt of housing assistance disqualifies adults from Medicaid. Housing assistance is excluded from MAGI and disqualifies no one from any Medicaid category, and the MAGI adult category applies no resource test, so neither the housing subsidy nor the $69,500 in bank accounts affects eligibility." @@ -635,114 +696,126 @@ us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligib us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred breastfeeding or postpartum status and nutritional risk solely from the presence of a one-year-old, despite the prompt requiring unlisted statuses to be false. With neither pregnancy, breastfeeding, nor postpartum status listed, the spouse fails WIC's categorical test regardless of income." us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical qualifier. Disability and income eligibility do not establish WIC eligibility for an adult who is not pregnant, breastfeeding, or postpartum." us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Counted the spouse's blindness and disability as two separate $1,000 additions, inflating exemptions to $13,000 when NJ grants a single $1,000 addition per blind-or-disabled taxpayer, and then subtracted a further $1,250.82 tenant property-tax deduction equal to 18% of the $6,949 rent. Exemptions total $12,000 with no rent deduction, so taxable income is $18,915 rather than $16,664 and the 1.4% bracket yields $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Reached the correct $12,000 of exemptions and $18,915 base but then subtracted $1,250.82 as an 18%-of-rent tenant property-tax deduction. No rent-based deduction enters this NJ liability, so the base stays at $18,915 and taxes at 1.4% to $264.81, not $247.30." -us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Zeroed the liability by stacking a NJ standard deduction, a pension/income exclusion, and state relief from the federal American Opportunity Credit; NJ provides none of these — it has no standard deduction, the pension exclusion requires age 62 or disability-based retirement income the household does not have, and the AOC is a federal credit with no NJ counterpart. NJ gross income of $30,915 less $12,000 of exemptions leaves $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Double-counted the blind and disabled additions as $2,000 instead of the single $1,000 NJ allows, reaching $17,915 and about $251, then discarded its own arithmetic by asserting that NJ rounds small liabilities to zero. No such provision exists; the correct $18,915 base taxes at 1.4% to $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Conflated NJ's $20,000 married-joint minimum filing threshold with exemption capacity, concluding the household owes nothing. Gross income of $30,915 exceeds that threshold, so the zero-tax provision does not apply and $18,915 of taxable income after $12,000 of exemptions taxes at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the $1,500 dependent exemption rate to the two joint filers, using $3,000 of regular exemptions where NJ allows $1,000 per taxpayer for $2,000. That overstated exemptions by $1,000, giving $17,915 instead of $18,915 and $250.81 instead of the correct $264.81 at 1.4%." -us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Granted the spouse both a $1,000 blind exemption and a separate $1,000 disabled exemption, where NJ allows one $1,000 addition per qualifying taxpayer, and then abandoned its own $251 figure by invoking a low-income phase-out to zero that NJ does not have. Exemptions are $12,000, leaving $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Asserted that NJ gross income of $30,915 falls below a married-joint standard deduction; NJ has no standard deduction at all, and its base is gross income less personal and dependent exemptions. $30,915 − $12,000 = $18,915, taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Started from federal AGI $30,632, which nets out the half-of-self-employment-tax deduction NJ does not allow, instead of nj_agi $30,915, and claimed only the $9,000 of dependent exemptions while dropping the $2,000 joint exemption and the $1,000 blind/disabled addition. Those two errors pushed the base to $21,632 and across into the 1.75% bracket; the correct $18,915 stays wholly in the 1.4% bracket at $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Subtracted a $181.70 medical deduction built from the household's $800 of over-the-counter health spending in excess of 2% of gross income. Over-the-counter, non-prescription items are not qualifying medical expenses, so no medical deduction arises; taxable income remains $18,915 and the tax $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Computed the base correctly at $18,915 after $12,000 of exemptions, then subtracted a $50 NJ tenant property-tax credit. That credit does not reduce this liability, which is $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Declared a zero liability from 'low income levels and personal exemptions' without computing the base. NJ has no standard deduction, and the $12,000 of exemptions against $30,915 of gross income leaves $18,915 taxable at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Anchored on a $251 pre-credit tax, the figure produced by $13,000 of exemptions from double-counting the blind and disabled additions, then subtracted a $50 tenant property-tax credit. NJ allows one $1,000 blind-or-disabled addition for $12,000 of exemptions and no tenant credit against this output, giving $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Used $3,000 of regular exemptions by applying the $1,500 dependent rate to the two filers instead of $1,000 each, and $2,000 for blind plus disabled instead of a single $1,000, overstating exemptions by $2,000. The correct $12,000 leaves $18,915 taxable and $264.81 of tax, not $237." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Submitted zero with no derivation. A zero result is consistent only with assuming a NJ standard deduction large enough to erase $30,915 of gross income; NJ has none, and $12,000 of exemptions leaves $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Derived the correct $18,915 base and the correct $264.81 of tax, then subtracted a $50 NJ property-tax credit to arrive at $214.81. No tenant property-tax credit reduces state income tax before refundable credits here, so the answer is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Claimed $13,000 of exemptions by treating the spouse's blindness and disability as two $1,000 additions rather than the single $1,000 NJ grants per qualifying person. That understates taxable income by $1,000: $18,915 at 1.4% is $264.81, not $250.81." -us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no NJ computation was submitted. The required derivation is $30,915 of NJ gross income less $12,000 of exemptions, with the resulting $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Stacked three errors: priced the six dependents at $1,000 each rather than NJ's $1,500 (reaching only $8,000 of regular exemptions), invented a $4,000 married-joint standard deduction, and subtracted an 18%-of-rent tenant deduction of $1,251. NJ has no standard deduction and no rent deduction applies; exemptions total $12,000 and $18,915 taxes at 1.4% to $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Declined to compute and asserted that exemptions and credits erase the liability at this income. NJ's $12,000 of exemptions against $30,915 leaves $18,915 taxable, and NJ offers no standard deduction and no nonrefundable credit here, so $264.81 stands at the 1.4% rate." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Estimated zero from an unspecified 'low tax base after deductions and credits' without applying NJ's structure. NJ subtracts only exemptions — $2,000 joint, $9,000 for six dependents, $1,000 blind/disabled — from $30,915, leaving $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Correctly took $12,000 of exemptions but then subtracted a $181.70 medical deduction computed as the $800 of over-the-counter health expenses less 2% of gross income. Over-the-counter, non-prescription spending is not a qualifying medical expense, so the base is $18,915 and the tax $264.81, not $262.27." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Offset the entire liability with the NJ child tax credit. That credit is refundable and therefore excluded from state income tax before refundable credits, which remains $18,915 × 1.4% = $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Reduced the base by an over-the-counter medical deduction and then applied a $50 nonrefundable renter property-tax credit. Over-the-counter health spending is not a qualifying medical expense and no tenant credit reduces this output, so $18,915 of taxable income taxes at 1.4% to $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Took separate $1,000 exemptions for blindness and for disability, where NJ allows one $1,000 addition per qualifying taxpayer. Exemptions total $12,000, taxable income $18,915, and tax $264.81 rather than the $250.81 that a $13,000 exemption total produces." -us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Applied the correct $12,000 of exemptions but subtracted a $181.70 medical deduction derived from the $800 of over-the-counter health spending above the 2%-of-gross-income floor. Over-the-counter items do not qualify, leaving taxable income at $18,915 and tax at $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Attributed the zero to a NJ standard deduction and credits. NJ has no standard deduction and no nonrefundable credit applies to this household; $30,915 less $12,000 of exemptions leaves $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Used federal AGI $30,632 rather than nj_agi $30,915 — NJ does not allow the deduction for half of self-employment tax — and priced all eight personal exemptions at $1,000, missing the $1,500 dependent rate that gives $9,000 for the six children. The resulting $21,632 base crossed into the 1.75% bracket; the correct $18,915 stays in the 1.4% bracket at $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Derived exactly the right exemptions and base — $12,000 and $18,915 — then subtracted a $50 renter property-tax credit as the better of the credit and the 18%-of-rent deduction. Neither reduces this liability, which is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Omitted the $1,000 blind/disabled exemption, using $11,000, and then taxed the entire $19,915 base at 1.75%, although NJ's married-joint schedule taxes the first $20,000 at 1.4%. The correct base of $18,915 at 1.4% is $264.81, not $349." -us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Claimed only $10,000 of exemptions, pricing the six dependents below NJ's $1,500 rate, and then subtracted a $1,251 tenant deduction equal to 18% of the $6,949 rent. Exemptions total $12,000 with no rent deduction, so taxable income is $18,915 and the tax $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for this variable, so no NJ liability was submitted. The correct derivation subtracts $12,000 of NJ exemptions from $30,915 of gross income and taxes the $18,915 remainder at 1.4% for $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Reproduced the reference exactly — $12,000 of exemptions, $18,915 taxable, $264.81 at 1.4% — then subtracted a $50 renter property-tax credit it judged more favorable than the rent deduction. Neither the credit nor the deduction reduces this output, so the answer is $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Started from federal AGI $30,632 instead of nj_agi $30,915 and omitted the $1,000 blind/disabled exemption, using $11,000 of exemptions. The correct $12,000 against $30,915 gives $18,915 and $264.81 at 1.4%, not $275." -us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Deducted $181.70 of over-the-counter health spending above the 2%-of-gross-income floor and then subtracted a $50 NJ tenant credit derived from 18% of the $6,949 rent. Over-the-counter items are not qualifying medical expenses and no tenant credit reduces this output, leaving $18,915 taxable and $264.81 of tax." -us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Priced the six dependents at $1,000 each rather than NJ's $1,500, reaching $9,000 of exemptions and a $296–$314 tax, then discarded that result for zero by invoking a 'nonrefundable exemption reduction' NJ does not have. Exemptions are $12,000, taxable income $18,915, and tax $264.81." -us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Asserted a zero base from unspecified NJ exemptions and deductions without computing them. NJ has no standard deduction, and $30,915 of gross income less $12,000 of exemptions leaves $18,915 taxed at 1.4% for $264.81." -us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It cited the correct $8,231 maximum and a ~$30,470 married-joint phase-out threshold, then subtracted roughly $2,100 of phase-out anyway, cutting the federal EITC to $6,110 when earned income of ~$30,800 leaves the credit at or essentially at the plateau maximum. It compounded that by using a $400-per-child NJ CTC bracket instead of the $1,000 top bracket that NJ taxable income of $18,915 commands, and by omitting the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,categorical_eligibility,False,"It computed both large components exactly right — 40% × $8,231 = $3,292.40 of NJ EITC and $1,000 × 2 children under 6 = $2,000 of NJ CTC — and missed only the third refundable pathway, the $50 New Jersey property tax credit available to this rent-paying household. The entire $50 shortfall is that omitted tenant credit." -us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It asserted that New Jersey offers no refundable state income tax credits, which is false: NJ runs a refundable EITC at 40% of the federal credit, a refundable child tax credit of up to $1,000 per child under 6, and a $50 refundable property tax credit for tenants. Denying the existence of all three pathways forced a $0 answer against $5,342.40 actually due." -us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own stated components — $2,880 of NJ EITC plus about $1,000 of NJ CTC — total roughly $3,880, yet it submitted $9,492, a figure no step in its derivation produces. Underneath that, it phased the federal EITC down to ~$7,200 when the household is on the $8,231 plateau, and counted only the 1-year-old as under 6 while the 5-year-old also qualifies." -us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It nailed the NJ EITC at 40% × $8,231 = $3,292 but then applied a guessed $600-per-child NJ CTC rate for income ""near $30,500,"" producing $1,200 instead of the $2,000 the top bracket yields — NJ's schedule is keyed to NJ taxable income of $18,915, well inside the $1,000-per-child tier. It also padded the total with unexplained ""minor amounts"" rather than the actual $50 property tax credit." -us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It phased the federal EITC down to ~$7,700 when married-joint earned income of ~$30,800 sits below the phase-out start and takes the full $8,231, understating the NJ EITC by about $212. It then dismissed the NJ child tax credit as inapplicable ""at this income,"" inverting the schedule: NJ taxable income of $18,915 is the lowest bracket, which pays the maximum $1,000 per child under 6, and it also dropped the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It applied a $25,511 phase-out start — a single-filer figure — to a married-joint return, subtracting $1,114 from a credit that is unreduced because the joint threshold sits above this household's ~$30,800 of earned income; the federal EITC is $8,231, not $6,716. It then used $500 per child under 6 for the NJ CTC instead of the $1,000 top bracket set by NJ taxable income of $18,915, and omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It put the federal EITC for 3+ children at ""roughly $8,000-$9,000"" and took the NJ EITC as ~$3,800, overshooting the actual 40% × $8,231 = $3,292.40 by about $500. Its $2,000 NJ CTC was correct, but it never identified the $50 refundable NJ property tax credit, so the total is an over-estimate built on an inflated federal EITC base." -us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It treated the NJ EITC as the household's only refundable state credit, omitting the $2,000 NJ child tax credit for the two children under age 6 and the $50 tenant property tax credit — $2,050 of the $2,134 shortfall. Its EITC base of $8,021 also undershoots the 2026 maximum of $8,231 for three or more qualifying children." -us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It stopped at the NJ EITC and never applied the two other refundable NJ pathways: $1,000 per child under 6 for the 5-year-old and the 1-year-old ($2,000) and the $50 tenant property tax credit. It further reduced the federal EITC to $7,787 when this married-joint household with ~$30,800 of earned income takes the unreduced $8,231." -us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It identified the NJ EITC and the $2,000 NJ CTC correctly in structure but used an $8,190 federal EITC base instead of $8,231, and it omitted the $50 refundable NJ property tax credit for tenants. Those two gaps account for the entire $66.40 shortfall." -us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It selected the NJ CTC's $30,000–$40,000 bracket at $800 per child, but the bracket is set by NJ taxable income of $18,915, which lands in the top tier at $1,000 per child, costing $400. It also cut the federal EITC to $7,691 when the household's earned income leaves the married-joint credit at the $8,231 maximum, and it skipped the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It found no qualifying state refundable credits, missing all three New Jersey pathways this household satisfies: the 40% refundable NJ EITC on a full $8,231 federal credit, the $1,000-per-child NJ CTC for the two children under 6, and the $50 tenant property tax credit. The stated facts — NJ residence, ~$30,800 of earned income, six children, $6,949 of rent — establish every one of them." -us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its NJ EITC of ~$3,280 is essentially right, but it applied an $800-per-child NJ CTC tier when NJ taxable income of $18,915 puts the household in the $1,000-per-child top bracket, losing $400. It also omitted the $50 refundable NJ property tax credit for renters." -us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It used a federal EITC of $7,128, a phase-out reduction of over $1,100 against a credit that is unreduced at this married-joint income, understating the NJ EITC by $441. It then took $800 per child under 6 rather than the $1,000 top-bracket rate set by NJ taxable income of $18,915, and omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $1,200 with no derivation; the correct computation is 40% × $8,231 = $3,292.40 of NJ EITC plus $2,000 of NJ CTC plus a $50 tenant property tax credit. No component or combination of the New Jersey refundable schedule produces $1,200 at this income, so the figure is a guess untethered from the state's credit formulas." -us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its NJ EITC of $3,219.60 implies a federal EITC of $8,049 — the prior year's maximum rather than the 2026 figure of $8,231 for three or more qualifying children. It also used $800 per child for the NJ CTC instead of the $1,000 top bracket that NJ taxable income of $18,915 commands, and omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $4,776 total matches a $1,600 NJ CTC at $800 per child under 6 plus about $3,176 of NJ EITC, versus the correct $2,000 CTC at the $1,000 top bracket keyed to NJ taxable income of $18,915 and $3,292.40 of EITC on the full $8,231 federal credit. It also never applied the $50 refundable NJ property tax credit for tenants." -us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It assembled all three components — NJ EITC, NJ CTC, and the $50 property tax credit — but priced the child credit at $800 per child under 6 rather than $1,000, the top bracket that NJ taxable income of $18,915 qualifies for. That single rate error accounts for the entire $399.20 shortfall." -us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so the contract requiring every requested key with a numeric value went unmet. No substantive computation was produced to evaluate against the $5,342.40 total." -us,scenario_008,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It counted one child under age 6 for the NJ child tax credit when two qualify — the 1-year-old and the 5-year-old, since the test is under 6, not under 5 — and priced that child at $500 rather than the $1,000 top bracket set by NJ taxable income of $18,915, losing $1,500. Its federal EITC base of $8,360 also overstates the $8,231 maximum, and it omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It declared no refundable New Jersey credit indicated by the facts, but the stated facts trigger all three: NJ residence with ~$30,800 of earned income and six children yields a 40% refundable NJ EITC of $3,292.40, two children under 6 yield $2,000 of refundable NJ CTC, and $6,949 of rent yields the $50 property tax credit. Nothing in the schedule requires additional facts beyond those given." -us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It treated low income alone as insufficient to trigger a New Jersey refundable credit, when low income is precisely what qualifies this household: 40% of the full $8,231 federal EITC, $1,000 per child for the two children under 6 at NJ taxable income of $18,915, and a $50 tenant property tax credit. Its $0 discards $5,342.40 of credits the listed facts establish." -us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It computed the NJ EITC at 40% × $8,231 = $3,292.40 and the NJ CTC at $1,000 for each of the two children age 5 or younger, both exactly right, and missed only the $50 refundable NJ property tax credit this rent-paying household receives. That single omitted pathway is the whole $50 gap." -us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It treated the NJ EITC as the sole refundable state credit, dropping the $2,000 NJ child tax credit for the two children under 6 and the $50 tenant property tax credit. Its $8,046 federal EITC base is the prior year's maximum rather than the 2026 figure of $8,231, understating the state credit by a further $74." -us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It got the NJ EITC exactly right at $3,292.40 but applied $800 per child under 6 for the NJ CTC instead of the $1,000 top bracket that NJ taxable income of $18,915 qualifies for, losing $400. It also omitted the $50 refundable NJ property tax credit for tenants." -us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $4,892.40 equals the correct $3,292.40 NJ EITC plus a $1,600 child credit priced at $800 per child, versus the $1,000-per-child top bracket that NJ taxable income of $18,915 commands. The remaining $50 gap is the omitted refundable NJ property tax credit for renters." -us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It identified all three components correctly — $3,292.40 of NJ EITC, a child credit for the two children under six, and the $50 renter property tax credit — but valued the child credit at $800 per child instead of the $1,000 top bracket keyed to NJ taxable income of $18,915. That rate choice is the entire $400 shortfall." -us,scenario_008,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It asserted that no state refundable credits apply, ignoring New Jersey's refundable EITC at 40% of the federal credit, its refundable child tax credit of $1,000 per child under 6 at NJ taxable income of $18,915, and its $50 tenant property tax credit. The household's NJ residence, earnings, children's ages, and $6,949 of rent satisfy all three tests." -us,scenario_008,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It placed the household in the NJ CTC's $30,001–$40,000 bracket at $400 per child by using gross income, but the bracket is determined by NJ taxable income of $18,915, which sits in the top tier at $1,000 per child — a $1,200 loss on the two children under 6. Its NJ EITC of $3,299 also rests on an $8,247 federal base rather than $8,231, and it omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $500-per-child NJ child tax credit, the credit's original enacted amount, rather than the $1,000-per-child top bracket in force for this year at NJ taxable income of $18,915, forfeiting $1,000. Its NJ EITC of $3,300 rests on an $8,250 federal base instead of $8,231, and it explicitly ruled out any further NJ credit, dropping the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It concluded no state refundable credit exists beyond the NJ EITC, omitting the $2,000 NJ child tax credit for the two children under age 6 and the $50 tenant property tax credit. It also reduced the federal EITC to $7,901 when the married-joint plateau leaves it at $8,231 for this household's ~$30,800 of earned income." -us,scenario_008,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It computed only the NJ EITC, skipping the $1,000-per-child NJ child tax credit for the 5-year-old and 1-year-old and the $50 tenant property tax credit — $2,050 of the $2,034 gap, partly offset by its overstated $8,271 federal EITC base against the actual $8,231." -us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_refundable_credits, leaving the required key unpopulated. There is no substantive derivation to compare against the $5,342.40 reference." -us,scenario_008,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"It derived both major components correctly — 40% of the $8,231 federal EITC for $3,292.40 and $1,000 for each of the two children under 6 for $2,000 — and omitted only the $50 refundable NJ property tax credit for this rent-paying household. That single missing pathway is the full $50 shortfall." -us,scenario_008,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It stated that no NJ refundable credits apply on the listed facts, yet those facts satisfy three: a 40% refundable NJ EITC on the full $8,231 federal credit, $1,000 per child under 6 for two children at NJ taxable income of $18,915, and the $50 renter property tax credit. The $0 answer discards $5,342.40 of credits." -us,scenario_008,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It phased the NJ child tax credit down linearly by income above $30,000 using gross income of $30,915, but the credit is a step schedule keyed to NJ taxable income of $18,915, which pays the undiminished $1,000 per child — its $36.60 reduction should be zero. Its NJ EITC of $3,304.80 also rests on an $8,262 federal base rather than $8,231, and it omitted the $50 tenant property tax credit." -us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It claimed the federal EITC for three or more children phases out to approximately zero at ~$30,800 of married-joint earned income, when the joint phase-out has not even begun at that income and the credit stands at its $8,231 maximum. That error zeroed the NJ EITC, and it then abandoned the NJ child tax credit and the $50 tenant property tax credit rather than applying the $1,000-per-child top bracket set by NJ taxable income of $18,915." -us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It invented a New Jersey EITC phase-out threshold that eliminated the state credit; the NJ EITC has no independent income test and is simply 40% of the federal credit, $3,292.40 here. It then substituted a nonexistent $100 minimum refundable credit and omitted the $2,000 NJ child tax credit and the $50 tenant property tax credit." +us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse two separate $1,000 exemptions for blind and disabled, so exemptions came to $13,000 instead of $12,000. It then subtracted the 18%-of-rent property tax deduction ($1,251). The household takes the larger $50 refundable property tax credit instead, so no rent deduction reduces taxable income. Taxable income should be $18,915, taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it also subtracted the renter property tax deduction of 18% × $6,949 = $1,250.82. That deduction saves only $17.51 at 1.4%, less than the $50 refundable property tax credit, so the credit is taken instead and the deduction is not. Taxable income stays at $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Applied the federal American Opportunity Credit, a nonexistent NJ standard deduction and a pension exclusion to NJ tax, and concluded taxable income was zero. NJ gives no AOC and no standard deduction. The only reductions are $12,000 of exemptions, leaving $18,915 taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Computed about $251 of tax (using $13,000 of exemptions, which double-counted blind and disabled). It then rounded that to $0 on the basis of unnamed nonrefundable credits, although no NJ nonrefundable credit applies. The correct $12,000 of exemptions leaves $18,915 taxable and $264.81 of tax." +us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Said $30,900 of gross income was below the $20,000 MFJ filing threshold, which is backwards: $30,915 is above it, so the household owes tax. After $12,000 of exemptions, $18,915 is taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used $1,500 per spouse for the regular exemption ($3,000) when NJ allows $1,000 each ($2,000). That pushed total exemptions to $13,000 and taxable income down to $17,915, giving $251 instead of 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Gave the spouse separate $1,000 exemptions for blind and disabled ($13,000 total instead of $12,000). It then invented a low-income phase-in that cuts liability to zero. NJ's only low-income relief is the $20,000 MFJ gross-income threshold, which this household exceeds, so tax is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Claimed an NJ MFJ standard deduction larger than $30,915. NJ has no standard deduction. Only $12,000 of exemptions apply, leaving $18,915 taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $30,632, which subtracts half of SE tax, a deduction NJ does not allow; NJ gross income is $30,915. It also allowed only $9,000 of exemptions, leaving out the $2,000 of spouse exemptions and the $1,000 blind/disabled exemption. That wrongly pushed taxable income into the 1.75% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it took a $181.70 medical deduction built from over-the-counter health expenses. Those expenses don't qualify for NJ's medical deduction, so taxable income is $18,915 and tax is $264.81, not $262.27." +us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,age_disability,False,"Allowed only $11,000 of exemptions, leaving out the $1,000 exemption for the blind and disabled spouse. That gave $19,915 taxable instead of $18,915 and tax of $278.81 instead of $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Correctly reached $18,915 of taxable income and about $265 of tax, then subtracted a $50 tenant property tax credit it treated as nonrefundable. NJ's property tax credit is refundable, so it does not reduce tax before refundable credits, which is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Claimed personal exemptions eliminate the tax. Total NJ exemptions are only $12,000 against $30,915 of gross income, leaving $18,915 taxable at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its preliminary tax of about $251 implies $13,000 of exemptions (blind and disabled counted separately) instead of $12,000. It then subtracted the $50 property tax credit, which is refundable and does not reduce tax before refundable credits. Correct tax is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Overstated exemptions at $14,000. It used $3,000 for the MFJ personal exemptions instead of $2,000, and $2,000 for the spouse being blind and disabled instead of a single $1,000. That left $16,915 taxable instead of $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Reported $0 with no derivation. NJ gross income of $30,915 less $12,000 of exemptions leaves $18,915 taxable at 1.4%, so tax is $264.81; the $0 answer implies it wrongly assumed exemptions or credits wipe out the tax." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Correctly computed $264.81 on $18,915 of taxable income, then subtracted a $50 property tax credit. NJ's property tax credit is refundable, so it belongs below this line, and tax before refundable credits stays at $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,age_disability,False,"Gave the spouse separate $1,000 blind and disabled exemptions ($13,000 total). NJ allows a single $1,000 exemption for a person who is blind or disabled, so exemptions are $12,000 and tax is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Valued all eight people's exemptions at $1,000 ($8,000), when dependents get $1,500 each. It also invented a $4,000 MFJ standard deduction, which NJ doesn't have, and subtracted the 18%-of-rent property tax deduction, which isn't taken because the $50 refundable credit is worth more. The correct figure is $12,000 of exemptions, $18,915 taxable and $264.81 of tax." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Assumed exemptions and credits wipe out the tax without computing them. $12,000 of exemptions leaves $18,915 taxable, no nonrefundable NJ credit applies, and tax is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Assumed deductions and credits reduce the tax to zero. NJ allows only $12,000 of exemptions here and no nonrefundable credits, so $18,915 is taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction built from over-the-counter health expenses, which don't qualify for NJ's medical deduction. Taxable income is therefore $18,915, not $18,733.30, and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Used the NJ Child Tax Credit to offset tax before refundable credits. The NJ CTC is refundable, so it doesn't reduce this amount, which stays at 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Took a medical deduction built from over-the-counter health expenses, which don't qualify. It also subtracted the renter property tax credit as if it were nonrefundable. The NJ property tax credit is refundable, and no medical deduction applies, so tax before refundable credits is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"Counted blindness and disability as two separate $1,000 exemptions for the spouse, which implies $13,000 of exemptions and $250.81 of tax. NJ allows a single $1,000 blind-or-disabled exemption per person, so taxable income is $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions was correct, but it subtracted an estimated $181.70 medical deduction built from over-the-counter health expenses, which don't qualify for NJ's medical deduction. Taxable income is $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Correctly computed $264.81 on $18,915, then subtracted a $50 renter property tax credit. That credit is refundable in NJ, so it doesn't reduce tax before refundable credits, which is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Relied on an NJ standard deduction that doesn't exist, plus unspecified credits. Only $12,000 of exemptions apply, leaving $18,915 taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used federal AGI of $30,632 (after the half-SE-tax deduction) instead of NJ gross income of $30,915. It also valued the six dependent exemptions at $1,000 each instead of $1,500, for $9,000 of exemptions instead of $12,000. That pushed taxable income into the 1.75% bracket." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Correctly reached $18,915 taxable and about $265 of tax, then subtracted the $50 renter property tax credit as if it were nonrefundable. NJ's property tax credit is refundable, so tax before refundable credits is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Treated $800 of over-the-counter health expenses as deductible medical costs above 2% of gross income, a $181.70 deduction. OTC expenses don't qualify for NJ's medical deduction, so taxable income is $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Left out the spouse's $1,000 blind/disabled exemption, giving $11,000 of exemptions. It then taxed the whole $19,915 at 1.75%, although the MFJ schedule taxes the first $20,000 at 1.4%. Correct tax is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Understated exemptions at $10,000 instead of $12,000. It also subtracted a $1,251 renter property tax deduction, which isn't taken because the $50 refundable property tax credit is worth more. Taxable income is $18,915 and tax is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Correctly reached $12,000 of exemptions, $18,915 taxable and $264.81 of tax, then subtracted the $50 renter property tax credit. That credit is refundable, so it doesn't reduce tax before refundable credits, which is $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $30,632, which subtracts half of SE tax, instead of NJ gross income of $30,915. It also left out the spouse's $1,000 blind/disabled exemption ($11,000 instead of $12,000), leaving $19,632 taxable instead of $18,915." +us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Took a $181.70 medical deduction built from over-the-counter health expenses, which don't qualify, and subtracted the $50 renter property tax credit, which is refundable. Without either, tax before refundable credits is 1.4% × $18,915 = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Valued the dependent exemptions at $1,000 instead of $1,500. It then zeroed the tax with an invented nonrefundable exemption reduction for low-income households, a provision NJ doesn't have. With $12,000 of exemptions, $18,915 is taxed at 1.4% = $264.81." +us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Said taxable income is zero after exemptions and deductions. NJ exemptions total only $12,000 against $30,915 of gross income, leaving $18,915 taxable at 1.4% = $264.81." +us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the federal EITC down to about $6,110, but earned income of about $30,500 is below the 2026 MFJ phase-out start, so the full $8,231 applies and the NJ EITC is $3,292.40. It then gave a muddled NJ CTC of $800 based on gross income. The June 2026 schedule applied to taxable income under $30,000 gives $2,500 for the two children under 6. It also left out the $50 tenant property tax credit." +us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the old $1,000-per-child NJ CTC. The schedule enacted June 30, 2026 for 2026–2028 pays $2,500 for the two children under 6 at this taxable income. It also left out the $50 refundable property tax credit available to renters with $6,949 of listed rent." +us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It said New Jersey has no refundable income tax credits. In fact the NJ EITC (40% of federal, $3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit are all refundable and all apply to this household." +us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own reasoning added up to about $3,880 ($2,880 EITC plus $1,000 CTC), yet it submitted $9,492, which none of its components support. Its steps were also wrong in three ways. It phased the federal EITC below the $8,231 maximum. It counted only the age-1 child for the NJ CTC, although the age-5 child also qualifies. And it left out the June 2026 schedule's $2,500 CTC and the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The $3,292 EITC is correct, but it cut the NJ CTC to $600 per child 'near $30,500' using gross income. NJ taxable income after exemptions is about $18,900, and the 2026 schedule pays $2,500 for the two children under 6. It then padded the total with unexplained 'minor amounts' rather than the specific $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It said the NJ Child Tax Credit does not apply at this income. With NJ taxable income of about $18,900, the credit pays $2,500 for the children aged 5 and 1. It also understated the federal EITC as $7,700 instead of the full $8,231, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct, but it used the pre-2026 NJ CTC of $1,000 per child under 6. The schedule enacted June 30, 2026 pays $2,500 for the two young children at this taxable income, so its answer is $500 short." +us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used an outdated $7,830 EITC maximum and began the phase-out at about $25,511, which is not the MFJ threshold. That cut the federal EITC to $6,716, when the correct 2026 figure is the full $8,231 (NJ EITC $3,292.40). It also used a $500-per-child NJ CTC instead of the 2026 schedule's $2,500 total, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It guessed the federal EITC at $8,000–$9,000 and the NJ EITC at about $3,800, overstating the exact 40% × $8,231 = $3,292.40. It used the old $1,000-per-child CTC instead of the 2026 schedule's $2,500. It also left out the $50 property tax credit, so its total is a rough sum rather than a computed one." +us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It put the family in the $30,001–$40,000 CTC bracket ($800 per child) using gross income. The NJ CTC uses NJ taxable income, which is about $18,900 after $12,000 of exemptions, and the June 2026 schedule pays $2,500 at that level. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only the NJ EITC, on an understated federal EITC of $8,021 instead of the full $8,231. It left out the refundable NJ Child Tax Credit ($2,500 for the children aged 5 and 1) and the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a federal EITC of $7,787, when the full $8,231 applies. It left out the $2,500 refundable NJ Child Tax Credit for the two children under 6 and the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a federal EITC of $8,190 instead of the 2026 maximum of $8,231, and the pre-2026 NJ CTC of $1,000 per child instead of the June 2026 schedule's $2,500. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used the old $1,000-per-child NJ CTC instead of the 2026 schedule's $2,500 for two children under 6. It also made up a $75 renters' credit; the actual NJ property tax credit for tenants is $50. And its federal EITC base of $8,247 overstates the $8,231 maximum." +us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It understated the federal EITC as $7,691 instead of the full $8,231. It then put the NJ CTC in the $800 tier using gross income, but the tiers use NJ taxable income (about $18,900), and the June 2026 schedule pays $2,500 there. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no refundable NJ credits. This low-income MFJ family with six children receives the refundable NJ EITC ($3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It put the family in the $800-per-child NJ CTC tier using gross income, instead of the lowest taxable-income tier of the June 2026 schedule, which gives $2,500. It also left out the $50 property tax credit, and its EITC of about $3,280 is slightly below $3,292.40." +us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It phased the federal EITC down to $7,128, but this household is below the 2026 MFJ phase-out start, so the full $8,231 applies (NJ EITC $3,292.40). It also used $800 per child in CTC instead of the 2026 schedule's $2,500 total, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave $1,200 with no calculation. The correct figure is $5,842.40: a $3,292.40 NJ EITC (40% of $8,231), a $2,500 NJ CTC for the two children under 6, and a $50 property tax credit. Its $1,200 does not come from any of these." +us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It used a $1,600 NJ CTC ($800 per child, the gross-income tier) instead of the June 2026 schedule's $2,500 at NJ taxable income of about $18,900. Its EITC of $3,219.60 implies a federal EITC of $8,049, below the $8,231 maximum. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,776 fits a slightly understated NJ EITC plus a $1,600 CTC at $800 per child. That misses the June 2026 schedule's $2,500 for the two children under 6, which uses taxable income, and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are right, but it used an $800-per-child NJ CTC based on gross income. The correct amount is $2,500, from the June 2026 schedule applied to NJ taxable income of about $18,900." +us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there is nothing to parse." +us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It counted only one child under 6 for the NJ CTC, but children aged 5 and younger qualify, so both the 5-year-old and the 1-year-old count. It also used $500 instead of the 2026 schedule's $2,500 total. It overstated the federal EITC as $8,360 instead of $8,231, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no refundable NJ credit applies. This household qualifies for the refundable NJ EITC ($3,292.40 from its $8,231 federal EITC), the NJ Child Tax Credit ($2,500), and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It returned $0, saying low-income status alone creates no refundable NJ credits. The NJ EITC and the NJ CTC are exactly low-income refundable credits, and together with the $50 property tax credit they total $5,842.40 here." +us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the pre-2026 NJ CTC of $1,000 per child. The schedule enacted June 30, 2026 gives $2,500 for the two young children. It also left out the $50 refundable property tax credit for renters." +us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an understated $8,046 federal EITC. It left out the refundable $2,500 NJ Child Tax Credit for the children aged 5 and 1 and the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used $1,600 of NJ CTC ($800 per child, the gross-income tier) instead of the June 2026 schedule's $2,500 at NJ taxable income under $30,000. It also left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 equals the correct $3,292.40 EITC plus a $1,600 CTC. That CTC uses the $800-per-child gross-income tier instead of the 2026 schedule's $2,500, and the total leaves out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are correct, but it used a $1,600 NJ CTC at $800 per child, which puts the family in a tier using gross income. The correct amount is $2,500, from the June 2026 schedule applied to NJ taxable income of about $18,900." +us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It used an NJ EITC rate of 50% instead of the statutory 40%, which overstates that credit ($4,081 vs $3,292.40). It also used the old $1,000-per-child CTC instead of the 2026 schedule's $2,500, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It overstated the NJ EITC at $3,337 instead of $3,292.40. It used $800 per child in CTC instead of the June 2026 schedule's $2,500 at NJ taxable income, and it left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct, but it used the pre-2026 NJ CTC of $1,000 per child. The schedule enacted June 30, 2026 for 2026–2028 pays $2,500 for the two children under 6, so its answer is $500 short." +us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It said no state refundable credits apply. The household receives the refundable NJ EITC ($3,292.40), the NJ Child Tax Credit ($2,500), and the $50 property tax credit." +us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It picked the $400-per-child CTC tier using gross income ($30,001–$40,000), but the tiers use NJ taxable income, which is about $18,900 after exemptions, and the June 2026 schedule pays $2,500 at that level. It also left out the $50 property tax credit and slightly overstated the federal EITC as $8,247." +us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $500-per-child NJ CTC instead of the June 2026 schedule's $2,500 for the two children under 6 at this taxable income. It also said no other refundable credits apply, which leaves out the $50 property tax credit." +us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It used the old $1,000-per-child NJ CTC instead of the 2026 schedule's $2,500. It also refused the $50 property tax credit on the grounds that it would not infer rent, but the spouse's $6,949 pre-subsidy rent is listed and makes the household eligible as a tenant." +us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on a $7,901 federal EITC instead of the full $8,231, and said there are no other refundable credits. That leaves out the $2,500 NJ Child Tax Credit and the $50 property tax credit." +us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It counted only the NJ EITC, based on an overstated $8,271 federal EITC. It left out the refundable NJ Child Tax Credit ($2,500 for the children aged 5 and 1) and the $50 property tax credit." +us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_refundable_credits, so there is nothing to parse." +us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The $3,292.40 EITC is correct, but it used the old $1,000-per-child NJ CTC instead of the June 2026 schedule's $2,500 for two children under 6. It also left out the $50 property tax credit for renters." +us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no NJ refundable credits apply. The household's earned income and young children qualify it for the NJ EITC ($3,292.40) and the NJ CTC ($2,500), and its rent qualifies it for the $50 property tax credit." +us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It treated the NJ CTC as phasing out continuously on gross income above $30,000. The credit is a step schedule based on NJ taxable income, and at about $18,900 the June 2026 schedule pays $2,500. It also overstated the federal EITC as $8,262 and left out the $50 property tax credit." +us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It said the federal EITC is near $0 at about $30,800 of earned income. That is wrong: the household is below the 2026 MFJ phase-out start and gets the full $8,231, so the NJ EITC is $3,292.40. It also counted only one child under 6 and zeroed out the NJ CTC ($2,500) and the $50 property tax credit." +us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It made up an NJ EITC income phase-out and a $100 minimum credit. The NJ EITC is simply 40% of the federal EITC ($3,292.40 here), with no separate state phase-out. It also left out the $2,500 NJ CTC and the $50 property tax credit." us,scenario_008,tanf,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model submitted a bare number with no derivation; the correct derivation applies New Jersey's WFNJ eligibility standard and yields $0, because countable income of about $2,576/month exceeds the family-of-eight standard (roughly $770/month) even after the 50% earned-income disregard, and $69,500 in bank assets exceeds the $2,000 TANF resource limit. Its $5,800 figure (about $483/month) is consistent with pulling a large-family NJ grant-schedule amount and shading it down for earnings — treating TANF as a residual gap-filler for a household with six children — while never running the income eligibility gate or the asset test at all." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning reproduces the reference chain exactly — $105,900 AGI, $32,200 MFJ standard deduction, $73,700 taxable income, $2,480 + 12% × $48,900 = $8,348 — and it then wrote ""value = 3455"" in the submitted field. The $3,455 corresponds to no step in its own derivation; the model discarded a fully correct computation at the emission stage." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It reduced the spouse's wages by the $8,139 employer-sponsored insurance premium, which is not an AGI adjustment for this household — taxable wages remain the full $60,000 — and used a $30,000 standard deduction instead of the 2026 MFJ figure of $32,200. It then submitted $2,950 against its own stated ""approximately $8,600"", effectively deducting the ESI premium a second time after the bracket calculation." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It used a $32,300 standard deduction instead of $32,200 and reached $8,336, within $12 of the reference, then replaced it with $4,880 under the label ""rounding to 2026 brackets approximation"". No bracket application to $73,600 or $73,700 of taxable income produces $4,880; the correct result is $2,480 + 12% × $48,900 = $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived $8,349 from the correct $105,900 AGI with a $32,300 standard deduction, then talked itself down to ""~$8,300"", then ""roughly $8,000"", and submitted $5,732, which follows from none of its own intermediate figures. The 2026 MFJ parameters are a $32,200 standard deduction and 10% to $24,800 then 12%, giving exactly $8,348 on $73,700." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reached the correct $105,900 AGI and applied the correct 2026 brackets (10% to $24,800, then 12%) to a taxable income of about $73,300, producing $8,300, then submitted $4,989. The exact $32,200 standard deduction gives $73,700 taxable and $8,348; nothing in its reasoning yields $4,989." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected the 2026 MFJ standard deduction as $30,700 and the top of the 10% bracket as $24,450, instead of the post-OBBBA values of $32,200 and $24,800. The $1,500 deduction shortfall shifts $1,500 into the 12% band ($180) and the low bracket ceiling adds $7 more, producing $8,535 against the correct $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It asserted 85% inclusion but then used $16,150 of taxable Social Security instead of the $25,500 cap on the $30,000 benefit, understating AGI by $9,350 ($96,550 instead of $105,900). It then cut its own $7,178 result to $6,350 for ""minor credit adjustments"" even though this childless couple qualifies for no nonrefundable credits." -us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"AGI of $105,900 and the $25,500 Social Security inclusion are exactly right; the two errors are a $30,750 standard deduction in place of $32,200 and a $24,500 top of the 10% bracket in place of $24,800. Those parameter misses raise taxable income to $75,150 and the tax to $8,528 instead of $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 employer-sponsored insurance premium from wages to get a $97,761 AGI, when taxable wages remain the full $60,000 and AGI is $105,900. It compounded this with a $30,600 standard deduction and a $24,400 10%-bracket ceiling rather than the 2026 values of $32,200 and $24,800." -us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a TCJA-sunset regime to 2026 — a $16,600 standard deduction, $10,600 of personal exemptions, and a 15% second bracket — but OBBBA made the TCJA rate schedule and the repeal of personal exemptions permanent, so 2026 uses a $32,200 MFJ standard deduction, no exemptions, and a 12% second bracket. Its $105,900 AGI is correct; the regime error alone inflates taxable income to $78,700 and the tax to $10,572.50." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It stacked three errors: subtracting the $8,139 ESI premium from wages, manufacturing $19,362.68 of itemized deductions from an unlisted mortgage interest rate and unlisted state tax when the prompt requires unlisted inputs to be treated as 0 (actual itemized deductions are $3,207.96, far below the standard deduction), and applying a TCJA-sunset regime with $10,200 of personal exemptions and a 15% bracket. The correct chain is $105,900 AGI less the $32,200 standard deduction, taxed at 10%/12% for $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted the full $30,000 Social Security benefit rather than the $25,500 taxable portion and applied a $10,040 deduction that is neither the 2026 MFJ standard deduction of $32,200 nor any statutory amount; its stated $80,360 income does not even sum from the $110,400 of components it lists. The correct derivation is $105,900 AGI − $32,200 = $73,700 taxable, taxed 10% to $24,800 then 12%, for $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $8,139 ESI premium from wages for a $97,761 AGI and assumed TCJA expiration, combining a roughly $27,600 standard-deduction-plus-personal-exemption stack with a 15% second bracket. 2026 retains the permanent TCJA schedule — full $60,000 of wages in AGI, a $32,200 standard deduction, no personal exemptions, and a 12% second bracket — giving $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied ""reverted pre-TCJA"" 2026 parameters — a $16,256 standard deduction, two $5,184 personal exemptions, and a 15% second bracket — and additionally netted the spouse's $8,139 ESI premium out of wages. Under the actual 2026 law the household keeps $60,000 of wages in a $105,900 AGI, takes the $32,200 standard deduction, claims no exemptions, and pays 10%/12% on $73,700 for $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It submitted $6,715 with no derivation. The correct chain — $60,000 wages + $20,400 pension + $25,500 taxable Social Security = $105,900 AGI, less the $32,200 MFJ standard deduction, taxed 10% to $24,800 then 12% — gives $8,348; $6,715 corresponds to taxable income near $60,100, i.e. omitting roughly $13,600 of the $25,500 includable Social Security." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the spouse's $8,139 ESI premium to reach a $97,761 AGI and used a standard deduction of about $30,660 rather than $32,200; the resulting $67,103 of taxable income taxed at 10%/12% produces its $7,558.32. Taxable wages are the full $60,000 here, so AGI is $105,900, taxable income $73,700, and the tax $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It carried the correct $105,900 AGI but subtracted ""standard deduction and personal exemptions"" — personal exemptions are repealed for 2026 under the permanent OBBBA schedule, and the MFJ standard deduction is $32,200. Its $10,735 corresponds to a sunset-regime deduction stack near $27,000 with a 15% second bracket, where the actual parameters give $73,700 taxable and $2,480 + 12% × $48,900 = $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 parameters — a $30,000 MFJ standard deduction and a $23,850 top of the 10% bracket — instead of the 2026 values of $32,200 and $24,800. Its taxable income of $75,900 overstates the correct $73,700 by $2,200 ($264 of extra 12% tax) and the low bracket ceiling adds $19 more, producing $8,631." -us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Every substantive step matches the reference — the $25,500 Social Security inclusion and the $105,900 AGI — but it estimated the 2026 MFJ standard deduction at $30,750 instead of $32,200 and the 10% bracket ceiling at $24,450 instead of $24,800. Those two parameters raise taxable income to $75,150 and the tax to $8,529." -us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $33,300 standard deduction, $1,100 above the 2026 MFJ figure of $32,200, together with the 2025 bracket ceiling of $23,850 rather than $24,800; $72,600 of taxable income under those brackets gives $2,385 + $5,850 = $8,235. The correct $73,700 taxed 10% to $24,800 then 12% gives $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It declared the household below any taxable threshold and submitted $0, when the spouse's $60,000 of wages alone exceeds the $32,200 MFJ standard deduction before the $20,400 taxable pension and the $25,500 includable Social Security are added. The correct taxable income is $73,700 and the tax $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its $2,757 corresponds to taxable income near $27,100 — approximately the $60,000 of wages less the standard deduction — meaning it dropped both the $20,400 taxable pension and the $25,500 includable Social Security from AGI. The full $105,900 AGI less the $32,200 standard deduction gives $73,700 and a tax of $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Its explanation states ""$26,150 tax before credits"" while the submitted value is $3,922, and neither figure results from applying any bracket schedule to this household's income. The reference derivation is $105,900 AGI − $32,200 standard deduction = $73,700 taxable, taxed $2,480 at 10% plus 12% of $48,900, for $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed a 2026 TCJA sunset, applying a ~$17,018 standard deduction, $10,854 of personal exemptions, and a 15% second bracket; OBBBA made the TCJA rate schedule and the exemption repeal permanent, so 2026 uses a $32,200 MFJ standard deduction, no exemptions, and a 12% bracket. Its $105,900 AGI is correct, and the sunset regime alone accounts for the $10,455 versus $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied a sunset regime for 2026 — a $16,550 standard deduction, $10,600 of personal exemptions, and a 15% second bracket — producing $78,750 of taxable income. Under the permanent post-OBBBA schedule the deduction is $32,200 with no exemptions, giving $73,700 taxed at 10% to $24,800 then 12%, for $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly invoked ""pre-TCJA rules"" for 2026 with a $15,700 standard deduction, $10,100 of personal exemptions, a $22,808 top of the 10% bracket, and a 15% second bracket. 2026 uses the permanent TCJA schedule: $32,200 standard deduction, no personal exemptions, 10% to $24,800 then 12%, so $73,700 of taxable income yields $8,348 rather than $10,875." -us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its sole error is netting the spouse's $8,139 employer-sponsored insurance premium out of wages: taxable wages remain the full $60,000, so AGI is $105,900, not $97,761. With its own correct $32,200 standard deduction and 10%/12% brackets, taxable income is $73,700 and the tax $8,348, not the $7,371 it computed on $65,561." -us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The Social Security inclusion and $105,900 AGI are exact; it then used a $30,600 standard deduction instead of $32,200 and a $24,400 top of the 10% bracket instead of $24,800. Taxable income of $75,300 under those parameters gives $8,548 rather than the correct $8,348 on $73,700." -us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"With the correct $105,900 AGI, it applied a ~$30,900 standard deduction instead of $32,200 and a $24,500 10%-bracket ceiling instead of $24,800, giving $75,000 of taxable income and $8,510. The 2026 parameters give $73,700 taxable and $2,480 + 12% × $48,900 = $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $30,450 standard deduction and the 2025 bracket ceiling of $23,850 rather than the 2026 values of $32,200 and $24,800, computing $8,577, and then submitted $8,077.50 — $499.50 below its own stated total. Correct parameters give $73,700 of taxable income and $8,348." -us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It stated that 85% of the $30,000 benefit is taxable yet used $4,000 of taxable Social Security instead of $25,500, producing an $84,440 AGI that does not sum from its own listed components, and it then claimed a $1,050 CDCC for a household with no children and no care expenses. Its final $12,110.90 is larger than its own pre-credit $5,930.25 after supposedly subtracting that credit; the correct figure is $73,700 of taxable income taxed at 10%/12% for $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got every step right and reached exactly $8,348: AGI $105,900, standard deduction $32,200, taxable income $73,700, 10% to $24,800 and 12% above that. It then submitted $3,455, a number that no step of its own computation supports." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted all $30,000 of Social Security instead of the $25,500 taxable portion. It also subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages and used the pre-OBBBA $30,000 standard deduction instead of $32,200. It then submitted $2,950, which does not match its own tax estimate of about $8,600." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly derived AGI of $105,900 and the 2026 brackets, but used a $32,300 standard deduction instead of $32,200, giving $8,336 (versus $8,348). It then submitted $4,880, which has no connection to its own $8,336 computation." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It used a $32,300 standard deduction (instead of $32,200) and a 10% bracket ending at $24,150 (instead of $24,800), and still reached $8,349. It then submitted $5,732, which no step of its reasoning produces." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It overstated the 2026 MFJ standard deduction as $32,600 instead of $32,200, giving taxable income of $73,300 and tax of $8,300. It then submitted $4,989, which is disconnected from its own $8,300 result." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 $30,000 standard deduction to $30,700 instead of using the 2026 MFJ amount of $32,200. It also projected the 10% bracket to end at $24,450 instead of $24,800. That overstated taxable income ($75,200 versus $73,700) and gave $8,535." +us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at $16,150 instead of the $25,500 cap (85% of $30,000), which understated AGI at $96,550. It also used a $32,600 standard deduction instead of $32,200. It then cut its own $7,178 to $6,350 for 'credit adjustments', even though this childless household has no nonrefundable credits." +us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"AGI of $105,900 was correct, but it used a $30,750 standard deduction instead of the 2026 MFJ $32,200. It also ended the 10% bracket at $24,500 instead of $24,800. That overstated taxable income at $75,150 and gave $8,528." +us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages, which cut AGI to $97,761 instead of $105,900. It also used a $30,600 standard deduction instead of $32,200 and a $24,400 top for the 10% bracket instead of $24,800." +us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA individual provisions expired in 2026 and applied a $16,600 pre-TCJA standard deduction, $10,600 of personal exemptions and a 15% bracket. The OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no exemptions and a 12% bracket up to $100,800." +us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules for 2026: a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket. The correct rules are the permanent TCJA/OBBBA $32,200 MFJ standard deduction and the 12% bracket, which give $8,348 on $73,700 of taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 ESI premium from wages and invented $14,640 of mortgage interest and $1,865.76 of state tax to itemize, even though no interest or tax paid was listed. Its $2,856.92 medical deduction is also wrong, because the medical costs are far below the 7.5%-of-AGI floor. On top of that it assumed TCJA had expired, adding personal exemptions and a 15% bracket instead of the $32,200 standard deduction and the 12% bracket." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $30,000 of Social Security instead of the $25,500 taxable portion and used a $10,040 standard deduction instead of $32,200. Its arithmetic was also wrong: the listed items net to $100,360, not the $80,360 it reported." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the spouse's $8,139 ESI premium from gross wages, putting AGI at $97,761 instead of $105,900. It also assumed TCJA had expired, using about $27,600 of standard deduction plus personal exemptions and a 15% bracket. The correct 2026 figures are the $32,200 MFJ standard deduction and the 10%/12% brackets." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reduced wages by the spouse's $8,139 ESI premium (AGI $97,761 instead of $105,900). It also applied a reverted pre-TCJA $16,256 standard deduction, $10,368 of personal exemptions and a 15% bracket, instead of the permanent 2026 $32,200 standard deduction with no exemptions and a 12% bracket." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Under the 2026 MFJ brackets, its $6,715 corresponds to taxable income of about $60,100, roughly $13,600 below the correct $73,700 ($105,900 AGI including $25,500 of taxable Social Security, minus the $32,200 standard deduction), so it understated taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $8,139 ESI premium as a pre-tax exclusion from the $60,000 gross wages, which understated AGI at $97,761. Its $7,558.32 is also higher than the $7,371.32 that its own AGI gives with the $32,200 deduction, so it also used a standard deduction smaller than $32,200." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"AGI of $105,900 was correct, but it subtracted personal exemptions and a pre-TCJA standard deduction as if TCJA had expired in 2026. The correct rule is the $32,200 MFJ standard deduction with no exemptions, taxed at 10%/12%, which gives $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 figures: a $30,000 standard deduction and a 10% bracket ending at $23,850. The 2026 figures are a $32,200 standard deduction and a 10% bracket to $24,800. That overstated taxable income at $75,900 and gave $8,631." +us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,750 instead of $32,200 and ended the 10% bracket at $24,450 instead of $24,800. Taxable income came out at $75,150 instead of $73,700, giving $8,529." +us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It overstated the 2026 MFJ standard deduction at $33,300 instead of $32,200 and used the 2025 10% bracket top of $23,850 instead of $24,800. That gave taxable income of $72,600 and tax of $8,235." +us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the household falls below the taxable threshold without computing anything. $80,400 of wages and pension plus $25,500 of taxable Social Security gives $105,900 of AGI, which leaves $73,700 taxable after the $32,200 standard deduction and produces $8,348 of tax." +us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It cited reductions from itemized health deductions. But about $1,600 of qualifying medical costs is far below the 7.5%-of-AGI floor ($7,943), and itemized deductions of $3,208 lose to the $32,200 standard deduction. Its $2,757 implies taxable income of only about $27,100, instead of $73,700." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It stated $26,150 of tax before credits and then submitted $3,922. Neither figure follows from $73,700 of taxable income ($105,900 AGI minus the $32,200 standard deduction) at 10% to $24,800 and 12% above, which gives $8,348." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA sunset and used a $17,018 standard deduction, $10,854 of personal exemptions and a 15% bracket. The OBBBA made the TCJA rules permanent, so the correct inputs are the $32,200 standard deduction, no exemptions and 12% up to $100,800." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law for 2026: a $16,550 standard deduction, $10,600 of personal exemptions and a 15% bracket. It should have used the permanent 2026 MFJ standard deduction of $32,200 and the 10%/12% brackets, which put taxable income at $73,700 instead of $78,750." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed TCJA had expired and used a $16,600 standard deduction, $10,600 of personal exemptions and a 15% bracket. The OBBBA extended the TCJA structure permanently, so 2026 has a $32,200 MFJ standard deduction, no exemptions and a 12% bracket up to $100,800." +us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA rules for 2026: a $15,700 standard deduction, $10,100 of exemptions and a 15% bracket. It should have used the permanent $32,200 MFJ standard deduction with no exemptions and the 10%/12% brackets, which give $8,348 on $73,700 of taxable income." +us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly used the $32,200 standard deduction and the 10%/12% brackets. However, it subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 gross wages, which understated AGI at $97,761 instead of $105,900 and gave $7,371." +us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction instead of the 2026 MFJ $32,200 and ended the 10% bracket at $24,400 instead of $24,800. That put taxable income at $75,300 instead of $73,700 and tax at $8,548." +us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a $30,900 standard deduction instead of $32,200 and ended the 10% bracket at $24,500 instead of $24,800. Taxable income came out at $75,000 instead of $73,700, giving $8,510." +us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a projected $30,450 standard deduction instead of $32,200 and the 2025 10% bracket top of $23,850 instead of $24,800, reaching $8,577. It then submitted $8,077.50, which drops $500 from its own total." +us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at $4,000 instead of the $25,500 cap (85% of $30,000), which understated AGI at $84,440. It also invented a $1,050 child and dependent care credit, even though the household has no dependents or care expenses. Its final $12,110.90 is higher than its own $5,930.25 tax, so it did not subtract that credit." us,scenario_009,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_009,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model improperly inferred SSDI and disability-based Medicare eligibility from Social Security retirement income despite the instruction that unlisted statuses are false. At age 57 with no disability, ESRD, or ALS stated, the Head does not satisfy either the age pathway or an under-65 Medicare pathway." us,scenario_009,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly concluded that the 57-year-old Head is not Medicare eligible, but it submitted value = 1, which denotes eligible. This is an answer-value inversion: its stated conclusion required value = 0." @@ -757,31 +830,32 @@ us,scenario_009,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model us,scenario_009,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_009,spouse_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model correctly concluded that the 57-year-old spouse was not Medicare eligible but then submitted value = 1, reversing the prompt’s binary encoding. “Not eligible” required value = 0." us,scenario_009,spouse_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own derivation reproduced the reference chain exactly — federal AGI $105,900, less $25,500 of taxable Social Security and the $25,500 NC MFJ standard deduction, $54,900 x 3.99% = $2,190.51 — and it then submitted $2,726, a figure none of its steps produces. The error is at the submission step: it discarded a fully correct North Carolina computation." -us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It zeroed the liability by asserting that available federal nonrefundable credits offset North Carolina tax; NC computes liability independently from NC taxable income, and this childless couple qualifies for no NC nonrefundable credit at all. It also used a $25,000 standard deduction instead of NC's $25,500 MFJ amount and never applied the NC subtraction for the $25,500 of Social Security in federal AGI, so its own base of $85,400 was wrong before the phantom credits erased it; $54,900 x 3.99% = $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct NC taxable income of $54,900 but applied a 4.25% rate; North Carolina's flat rate for tax year 2026 is 3.99%, giving $2,190.51. It then submitted $2,891, which matches neither its own 4.25% result ($2,333) nor the 4.5% alternative it named." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed the reference chain correctly to $54,900 x 3.99% = $2,191, then applied an unexplained second 'adjustment for taxable SS included then deducted' and submitted $2,974. North Carolina subtracts the $25,500 of federally taxable Social Security exactly once; the extra adjustment added roughly $19,600 of phantom taxable income to an already-correct base." -us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived the correct NC taxable income of $54,900 and the correct 3.99% rate for $2,190, then overrode that with an invented claim that PolicyEngine applies the rate to $71,600 of 'pre-deduction-adjusted income' and submitted $2,856. NC taxable income is federal AGI $105,900 less the $25,500 Social Security subtraction and the $25,500 MFJ standard deduction, so no $71,600 base exists." -us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It cut spouse wages to $51,861 by treating the $8,139 employer-sponsored insurance premium as a Section 125 pre-tax reduction, while the stated $60,000 is the gross wage input that enters AGI unreduced, producing $105,900 rather than $97,761. It then replaced the $25,500 NC standard deduction with a fabricated $25,620 mortgage-interest itemized deduction built from a guessed 7% rate on the $366,000 balance, when mortgage interest is an unlisted input and therefore 0; the two errors dropped taxable income from $54,900 to $46,641." -us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It subtracted the full $30,000 gross Social Security benefit instead of the $25,500 actually included in federal AGI, applied a 4.25% rate rather than NC's 3.99% for 2026, and then added a guessed mortgage-interest itemized deduction. Its own arithmetic yielded $2,142 and then $1,912, and it submitted $3,200 — a number no step in its reasoning produces." -us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Every step matched the reference except the standard deduction: it used $26,000 for married filing jointly where North Carolina's is $25,500, leaving taxable income of $54,400 instead of $54,900. The $500 deduction overstatement is the entire error, worth $19.95 at the 3.99% rate." -us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made two deduction errors: federal AGI of $97,761 reflects netting the $8,139 ESI premium out of the $60,000 gross wages, which PolicyEngine includes in full for AGI of $105,900, and it used a $26,500 NC standard deduction instead of $25,500. Its $45,761 base is $9,139 below the correct $54,900." -us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It substituted the federal MFJ standard deduction of $30,000 for North Carolina's $25,500 and applied a 4.25% rate instead of NC's 3.99% for 2026. The two errors partly offset — $50,400 x 4.25% = $2,142 against the correct $54,900 x 3.99% = $2,190.51 — but neither parameter is a North Carolina 2026 value." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It had NC's 3.99% rate and $25,500 MFJ standard deduction right but reduced spouse wages by the $8,139 employer-sponsored insurance premium, giving NC income of $72,261 instead of $80,400. The stated $60,000 is gross wages and enters AGI without any premium offset, so taxable income is $54,900 and the tax is $2,190.51, not $1,865.76." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied a 4.5% rate when North Carolina's 2026 flat rate is 3.99%, and to an unsupported $60,800 base. The correct base is federal AGI $105,900 less the $25,500 Social Security subtraction and the $25,500 MFJ standard deduction, or $54,900, taxed at 3.99% for $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its rate (3.99%) and standard deduction ($25,500) were correct, but it started from federal AGI of $97,761 — a figure reachable only by netting the $8,139 ESI premium out of the $60,000 gross wages. AGI includes the full $60,000, making it $105,900 and NC taxable income $54,900 rather than $46,761." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It compounded two deduction errors: the $97,761 AGI comes from subtracting the $8,139 ESI premium from gross wages, which PolicyEngine does not do, and the $29,250 standard deduction is not North Carolina's MFJ figure of $25,500. Its $43,011 base understates the correct $54,900 by $11,889." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted $3,212 with no derivation; the figure corresponds to the 3.99% rate applied to roughly $80,500 — federal AGI net of only one $25,500 subtraction. North Carolina allows both the subtraction for the $25,500 of taxable Social Security and the separate $25,500 MFJ standard deduction, giving a $54,900 base and $2,190.51 of tax." -us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It applied the correct 3.99% rate and $25,500 MFJ standard deduction to a wage base reduced by the $8,139 employer-sponsored insurance premium, reporting federal AGI of $97,761 instead of $105,900. Gross wages of $60,000 enter AGI unreduced, so NC taxable income is $54,900 and the tax is $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It performed no North Carolina computation and defaulted to zero on the assertion that nothing supported a positive liability. The household has $54,900 of NC taxable income — federal AGI $105,900 less the $25,500 Social Security subtraction and the $25,500 MFJ standard deduction — which NC's 3.99% flat rate taxes at $2,190.51, and no NC nonrefundable credit reaches this childless couple." -us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted zero as an approximation without deriving NC taxable income or naming a deduction, rate, or credit. The full derivation is federal AGI $105,900 less the $25,500 taxable Social Security subtraction and the $25,500 MFJ standard deduction, taxed at North Carolina's 3.99% flat rate, for $2,190.51; nothing in this household offsets that liability." -us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a 4.5% rate where North Carolina's tax year 2026 rate is 3.99%, and its $2,340 implies a $52,000 base rather than the correct $54,900. The correct base is federal AGI $105,900 less $25,500 of federally taxable Social Security and the $25,500 MFJ standard deduction, taxed at 3.99% for $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It got AGI of $105,900, the $25,500 Social Security subtraction, and the 3.99% rate all correct, then used a $21,000 NC standard deduction for joint filers instead of $25,500. The $4,500 deduction shortfall raised taxable income to $59,400 from $54,900 and added $179.55 of tax, the whole of its error." -us,scenario_009,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the correct 3.99% rate and $25,500 MFJ standard deduction but built its base from a $97,761 federal AGI, which reflects removing the $8,139 employer-sponsored insurance premium from the $60,000 gross wages. With wages included in full, AGI is $105,900 and NC taxable income is $54,900, giving $2,190.51 rather than $1,866." -us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so nothing substantive was submitted for grading. The required derivation is federal AGI $105,900 less the $25,500 Social Security subtraction and the $25,500 NC MFJ standard deduction, with the $54,900 remainder taxed at North Carolina's 3.99% flat rate for $2,190.51." -us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It never applied North Carolina's subtraction for Social Security benefits included in federal AGI, taxing the whole $105,900 including the $25,500 of taxable benefits. That omission alone adds $1,017 of tax at 3.99%, and it compounded it with a $26,200 standard deduction and a 4.0% rate in place of NC's $25,500 and 3.99%." -us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"After several restarts it converged on the exact reference base — federal AGI $105,900, less the $25,500 Social Security subtraction and the $25,500 MFJ standard deduction, for $54,900 — and then taxed it at 4.5%. North Carolina's flat rate for tax year 2026 is 3.99%, so the liability is $2,190.51, not $2,470.50; the rate is the sole error in its final chain." -us,scenario_009,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It manufactured four North Carolina provisions that do not exist — a $6,000 retirement exclusion, an $1,800 state EITC, a $500 state child tax credit, and a $2,371.05 'retirement exclusion credit' — to drive the liability to zero; NC repealed its EITC after tax year 2013, offers no state child tax credit, and grants no private-pension exclusion to a 57-year-old. Its starting base of $84,440 and $25,750 standard deduction are also wrong: the computation is $54,900 x 3.99% = $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It worked out the correct chain: AGI of $105,900, minus $25,500 Social Security, minus the $25,500 standard deduction, equals $54,900, and 3.99% of that is $2,190.51. It then submitted 2,726, a number that contradicts its own derivation and has no supporting step." +us,scenario_009,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It never subtracted the taxable Social Security that North Carolina excludes, and it used a $25,000 standard deduction instead of $25,500. It then made up unnamed 'federal nonrefundable credits' that wipe out the North Carolina tax. North Carolina has no such credits for this childless couple, so the real liability is $54,900 × 3.99% = $2,190.51, not $0." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income. It then applied 4.25% (the 2025 rate) instead of the 2026 flat rate of 3.99%. It also submitted 2,891, which does not match its own 4.25% figure of $2,333." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It correctly computed $54,900 × 3.99% = $2,191. It then invented an 'adjustment for taxable SS included then deducted' that raised the answer to 2,974. This counts the Social Security subtraction twice, even though that subtraction is already built into the $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It reached $54,900 of taxable income and $2,190 of tax, then threw that out. Instead it applied 3.99% to a made-up $71,600 base that skips the standard deduction, which gives $2,856." +us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It cut the spouse's wages by the $8,139 employer-sponsored insurance premium, which lowered AGI to $97,761; the reference keeps the full $60,000 of gross wages in AGI. It also assumed about $25,620 of mortgage interest from the loan balance and itemized with it, even though no mortgage interest was listed. That gave $46,641 × 3.99% = $1,861 instead of $54,900 × 3.99%." +us,scenario_009,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used 4.25% instead of the 2026 rate of 3.99%. It subtracted the full $30,000 of Social Security instead of the $25,500 actually included in federal AGI, and it floated an itemized deduction built on assumed mortgage interest. It then submitted 3,200, which none of its intermediate figures ($2,142, $1,912) support." +us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $26,000 married-filing-jointly standard deduction instead of North Carolina's $25,500. That gives $54,400 × 3.99% = $2,171 instead of $54,900 × 3.99% = $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $97,761, which takes the $8,139 employer insurance premium out of wages; the reference AGI is $105,900. It also used a $26,500 standard deduction instead of $25,500, which gave $45,761 of taxable income instead of $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction (close to the federal amount) instead of North Carolina's $25,500 for married filing jointly. It also used a 4.25% rate instead of the 2026 rate of 3.99%, which gives 4.25% × $50,400 instead of 3.99% × $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income but taxed it at 4.25% instead of the 2026 flat rate of 3.99%. That produces $2,333.25 instead of $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $8,139 employer-sponsored insurance premium from the $60,000 of gross wages, which cut North Carolina income to $72,261 instead of $80,400. Its taxable income was therefore $46,761 instead of $54,900, even though its rate (3.99%) and standard deduction ($25,500) were right." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied a 4.5% rate instead of the 2026 rate of 3.99%. It also overstated taxable income at about $60,800 instead of $54,900 (AGI of $105,900 minus $25,500 Social Security minus the $25,500 standard deduction)." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a federal AGI of $97,761, which treats the spouse's $8,139 insurance premium as a pre-tax cut to wages. The reference AGI includes the full $60,000 of wages and comes to $105,900. That left taxable income at $46,761 instead of $54,900 under the correct 3.99% rate." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $97,761, which removes the $8,139 employer premium from wages. It also used a $29,250 standard deduction instead of North Carolina's $25,500 for married filing jointly, which gave $43,011 of taxable income instead of $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,212 matches 3.99% applied to about $80,400 (wages plus pension) with no $25,500 standard deduction. The correct taxable income is $54,900, which gives $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It took federal AGI as $97,761 by netting the spouse's $8,139 employer insurance premium out of wages. That understates North Carolina income by $8,139 and gives $46,761 of taxable income instead of $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed zero North Carolina tax without computing a taxable income. The couple has $80,400 of North Carolina income (wages plus pension, with Social Security excluded). After the $25,500 standard deduction that leaves $54,900, taxed at 3.99%." +us,scenario_009,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It estimated zero tax without any computation. It missed that taxable income is $54,900 after the Social Security subtraction and the $25,500 standard deduction, and that no nonrefundable credit offsets the 3.99% tax." +us,scenario_009,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied a 4.5% flat rate instead of the 2026 rate of 3.99%. Its $2,340 implies about $52,000 of taxable income instead of the correct $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly reached $80,400 of North Carolina income but used a $21,000 standard deduction instead of $25,500. That gives $59,400 × 3.99% = $2,370 instead of $54,900 × 3.99%." +us,scenario_009,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its North Carolina base of $72,261 comes from subtracting the spouse's $8,139 employer insurance premium from wages. The reference base is $80,400 (the full $60,000 of wages plus the $20,400 pension), so its taxable income was $46,761 instead of $54,900." +us,scenario_009,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for state_income_tax_before_refundable_credits, so it missed the $2,190.51 result (3.99% of $54,900)." +us,scenario_009,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It never applied North Carolina's subtraction for the $25,500 of Social Security included in federal AGI. It also used a $26,200 standard deduction and a 4.0% rate. That taxed $79,700 instead of $54,900 at 3.99%." +us,scenario_009,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It correctly reached $54,900 of North Carolina taxable income. It then applied 4.5% instead of the scheduled 2026 flat rate of 3.99%, which gave $2,470.50 instead of $2,190.51." +us,scenario_009,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It made up North Carolina credits and exclusions that do not exist for this household: a $6,000 retirement exclusion, a state earned income credit, a child tax credit for a childless couple, and a 'retirement exclusion credit'. It used them to drive the tax to $0. The only adjustments are the Social Security subtraction and the $25,500 standard deduction, which leave $54,900 taxed at 3.99%." us,scenario_009,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_009,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_012,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and income limits and omitted the requirement that a CHIP child be ineligible for Medicaid. Child 1 qualifies for Medicaid under the OLDER_CHILD category, which precludes CHIP eligibility regardless of being below the CHIP income ceiling." @@ -798,42 +872,45 @@ us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the WIC income limit without first enforcing the program's categorical age restriction. A 10-year-old is outside WIC's under-five child category, so income below the limit does not confer eligibility." us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model explicitly misidentified age 10 as within WIC's age limit. WIC child eligibility ends at age five, making this 10-year-old categorically ineligible even though household income satisfies the financial threshold." us,scenario_012,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived both components correctly in its own text — the $4,427 one-child EITC maximum plus the $1,700 refundable CTC cap, explicitly writing ""EITC $4,427 + ACTC $1,700 = $6,127"" — then added a phantom $966 of ""recovery/other credits"" and submitted $7,093. Its stated components sum to the reference; the submitted value is the reference plus an invented credit that no fact in the household supports." -us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted the pregnant 18-year-old spouse as a qualifying child and claimed three qualifying children under head-of-household status; a spouse on a joint return is never a qualifying child and an unborn child is not a qualifying child for federal EITC or CTC, leaving exactly one qualifying child. It then submitted a bare $3,995 three-child EITC figure and omitted the $1,700 refundable CTC entirely." -us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model got the structure right — full plateau EITC plus the $1,700 refundable CTC cap — but replaced the published 2026 one-child EITC maximum of $4,427 with an ad-hoc inflation guess of $4,472 (its $6,172 minus $1,700). The $45 overshoot comes entirely from inventing an inflation adjustment instead of using the statutory indexed maximum." -us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model first computed the correct plateau EITC (~$4,400) plus the $1,700 refundable CTC cap, then discarded that for a ""refined 2026 EITC max of ~$4,731,"" which exceeds the actual 2026 one-child maximum of $4,427 by $304. The entire $304 error is the substituted EITC ceiling; the ACTC leg was right." -us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used $4,367 for the one-child 2026 EITC maximum instead of $4,427 and applied an $1,800 per-child refundable CTC cap instead of the $1,700 cap in force for 2026. Both legs of its $6,167 are off — the EITC understated by $60 and the ACTC overstated by $100." -us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After correctly finding zero tax liability and no EITC phaseout at $22,000, the model froze the EITC at the 2025 maximum of $4,328 rather than the 2026 indexed $4,427, and capped the ACTC at the full $2,000 CTC amount instead of the $1,700 refundable ceiling. Its $6,328 is exactly $4,328 + $2,000, so both the stale EITC parameter and the ignored refundable cap contribute." -us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model placed $22,000 of earnings inside the EITC phaseout range and cut the credit to roughly $3,200, but the joint-filer phaseout for one child does not begin until above $31,000, so the household sits on the plateau at the full $4,427. It then abandoned its own arithmetic and submitted $7,200 built on an unexplained ""~$5,500"" EITC, above the one-child maximum." -us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied the correct structure — full plateau EITC below the joint phaseout start plus the $1,700 refundable CTC — but used $4,404 as the 2026 one-child EITC maximum instead of $4,427. The entire $23 shortfall is that misremembered indexed ceiling." -us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the pre-2018 additional child tax credit rules — a $1,000 refundable cap with a $3,000 earned-income floor — when the 2026 refundable cap is $1,700 with a $2,500 floor, costing $700. It also used $4,355 rather than the $4,427 one-child EITC maximum." -us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model paired a $4,442 EITC maximum (the 2026 figure is $4,427) with a $1,000 additional child tax credit, the repealed pre-TCJA refundable cap; the 2026 refundable ceiling is $1,700 per child and 15% of ($22,000 − $2,500) = $2,925 clears it. The $700 ACTC understatement drives nearly all of the $685 total error." -us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable Child Tax Credit, which is the pre-TCJA cap; for 2026 the refundable portion is capped at $1,700 per child and the earned-income formula 15% × ($22,000 − $2,500) = $2,925 exceeds that cap. Its EITC of $4,423 also falls $4 short of the 2026 one-child maximum of $4,427." -us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used $5,571 for the one-child EITC, $1,144 above the 2026 one-child maximum of $4,427 and closer to the two-child ceiling, and then applied the repealed $1,000 refundable CTC cap in place of the 2026 $1,700 cap. The two errors partially offset to land at $6,571." -us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a $1,000 refundable Child Tax Credit, the pre-TCJA cap, rather than the $1,700 per-child refundable ceiling that governs 2026 and that the 15% × ($22,000 − $2,500) = $2,925 earned-income formula fully unlocks. Its $4,500 EITC estimate is also $73 above the 2026 one-child maximum of $4,427." -us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model capped the refundable CTC at $1,000, the pre-TCJA amount, instead of the $1,700 per-child refundable cap applicable in 2026, and set the one-child EITC at $4,320 rather than $4,427. The $700 ACTC shortfall accounts for most of the $807 miss." -us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model named EITC and CTC but computed neither: the correct derivation is the $4,427 plateau one-child EITC plus the $1,700 refundable CTC for $6,127, while its $4,050 matches no component or sum. The number is consistent with a single half-remembered one-child EITC figure with no refundable child credit added at all." -us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got the structure exactly right — plateau EITC plus the $1,700 refundable CTC — but used the 2025 one-child EITC maximum of $4,328 instead of the 2026 indexed maximum of $4,427. The entire $99 error is the un-indexed EITC ceiling." -us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly identified the full plateau EITC plus a $1,700 additional child tax credit but carried the 2025 one-child EITC maximum of $4,328 into 2026, where the indexed maximum is $4,427. Its $6,028 misses the reference by exactly that $99 of inflation indexing." -us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model combined two stale parameters: the 2025 one-child EITC maximum of $4,328 rather than the 2026 figure of $4,427, and the pre-TCJA $1,000 refundable CTC cap rather than the $1,700 cap that applies for 2026. Its $5,328 is $99 + $700 below the reference." -us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model phased the one-child EITC down to $2,591 at $22,000 of earnings, but a joint filer with one child is still on the $4,427 plateau until the phaseout begins above $31,000. It then zeroed the refundable CTC on the reasoning that there is no tax liability, inverting the rule — the additional child tax credit exists precisely for zero-liability filers and pays $1,700 here." -us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 per-child 2026 Child Tax Credit as refundable, but the refundable portion is separately capped at $1,700 per child even when 15% × ($22,000 − $2,500) = $2,925 exceeds it. Its EITC of $4,420 is also $7 short of the $4,427 one-child maximum, giving $6,620 against $6,127." -us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model denied the EITC outright on a fabricated rule that ""a spouse age 18"" blocks eligibility; the age-25 floor applies only to childless claimants, and with a qualifying 10-year-old the couple takes the full $4,427 one-child EITC regardless of the spouse's age. It then submitted $2,524 of refundable child credit, which itself exceeds the $1,700 per-child refundable cap." -us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model declared refundable credits to be zero on the ground that filing status and earned-income details were unspecified, but the prompt supplies a married couple with $22,000 of wages and one 10-year-old, which fully determines a $4,427 EITC and a $1,700 refundable CTC. It treated a complete fact pattern as under-specified and zeroed a $6,127 answer." -us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated both parameters: $4,526 for the 2026 one-child EITC maximum, which is $4,427, and an $1,800 refundable CTC cap, which is $1,700 for 2026. Its $6,326 is the sum of a $99 EITC overshoot and a $100 ACTC overshoot." -us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model nailed the EITC leg at the correct $4,427 one-child maximum but applied an $1,800 per-child refundable CTC cap; the 2026 refundable ceiling is $1,700. The full $100 error is that single misremembered cap." -us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model ran a phaseout against $22,000 of earnings to reach $2,479, but a joint filer with one qualifying child stays at the $4,427 plateau until the phaseout threshold above $31,000. It also omitted the $1,700 refundable child tax credit entirely, submitting an EITC-only figure." -us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a ""post-sunset"" ACTC of $1,000 with a $3,000 earned-income floor, but the TCJA child credit structure was made permanent for 2026 with a $1,700 refundable cap and a $2,500 floor, so the ACTC is $1,700. Its EITC of $4,318 is also $109 below the 2026 one-child maximum of $4,427." -us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and capped the additional child tax credit at $1,000; no sunset applies in 2026 and the refundable cap is $1,700, which 15% × ($22,000 − $2,500) = $2,925 fully unlocks. Its $4,436 EITC also exceeds the $4,427 one-child maximum by $9." -us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,000 refundable child tax credit, the pre-TCJA cap, instead of the $1,700 per-child refundable ceiling that governs 2026, giving up $700. Its projected EITC of $4,436 also overshoots the actual 2026 one-child maximum of $4,427." -us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so the contract-required key never reached the grader. This is a missing submission rather than a substantive computation error." -us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied the correct structure — plateau one-child EITC plus min($1,700, 15% of earnings above $2,500) = $1,700 — but set the 2026 one-child EITC maximum at $4,401 instead of $4,427. The entire $26 gap is that indexed ceiling." -us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that low earnings with one child produce ""small refundable credits"" and then submitted zero, contradicting its own statement. At $22,000 of wages the couple receives the full $4,427 plateau EITC plus a $1,700 refundable CTC, so a zero answer forfeits the entire $6,127." -us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model reasoned the case correctly end to end — earnings past the phase-in end, below the joint phaseout start, plus a $1,700 refundable CTC because 15% × ($22,000 − $2,500) clears the cap — but rounded the 2026 one-child EITC maximum to $4,425 instead of $4,427. The $2 miss is entirely that parameter." -us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model counted the spouse's pregnancy as a second qualifying child, doubling the refundable CTC to $3,400 and switching to the two-child EITC schedule; federal law recognizes no unborn child for either credit, leaving one qualifying child, a $1,700 ACTC, and a $4,427 EITC. Its $7,830 two-child EITC also exceeds the actual 2026 two-child maximum, compounding the error to $11,230." -us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model set the one-child EITC at $6,936, far above the 2026 one-child maximum of $4,427 and nearer the two-child ceiling, and then zeroed the ACTC by claiming the $2,000 CTC was consumed offsetting regular tax. Tax liability is zero because the $32,200 joint standard deduction exceeds $22,000 of AGI, so the full $1,700 refundable CTC is paid out." -us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored SNAP receipt and therefore missed categorical eligibility for free school meals. It also failed to apply the direct income pathway: the household is at 81% of the federal poverty guideline, below the 130% free-meal threshold." -us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly placed $22,000 above 130% of the applicable federal poverty guideline; the engine's household-size calculation puts it at 81%. It also omitted the independent SNAP categorical-eligibility pathway, which qualifies the child for free meals regardless of that income comparison." +us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived EITC $4,427 + ACTC $1,700 = $6,127. It then added an unexplained $966, reaching $7,093, even after stating that no other refundable credits apply. The extra amount has no basis in any credit this household qualifies for." +us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model left out the refundable CTC entirely, claiming it was limited by an income phase-out. The CTC phase-out starts at $400,000 for MFJ, so the full $1,700 ACTC applies at $22,000. It also counted the pregnant spouse as a qualifying child and used head-of-household status. It then plugged in a stale $3,995 EITC maximum instead of the 2026 one-child maximum of $4,427." +us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model got ACTC right at $1,700 but started from the 2025 one-child EITC maximum of $4,328. It then applied an arbitrary 'inflation adjustment' to $6,172, which implies an EITC of $4,472 instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed ACTC at $1,700 correctly and first estimated the EITC near $4,400. It then replaced that with a 'refined' 2026 one-child maximum of $4,731, which overstates the actual $4,427 maximum by $304." +us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an $1,800 per-child refundable CTC cap instead of the 2026 cap of $1,700. It also put the EITC at $4,367 by placing $22,000 near the phase-out. At $22,000 the family is on the plateau and gets the full $4,427 maximum." +us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the pre-OBBBA $2,000 CTC and treated all of it as refundable. It never applied the $1,700 ACTC refundable cap. For the EITC it fell back on the 2025 one-child maximum of $4,328 instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used stale one-child EITC parameters ($11,610 phase-in end, $3,995 plateau) and wrongly put $22,000 in the phase-out range, reaching about $3,200. It then dropped its own work and substituted an unsupported $5,500 EITC. At $22,000 the full 2026 one-child maximum of $4,427 applies, plus $1,700 ACTC." +us,scenario_012,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model had the structure right: full one-child EITC plus $1,700 ACTC. But it used $4,404 as the 2026 one-child EITC maximum instead of $4,427, which leaves the total $23 short." +us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the TCJA-sunset ACTC rules: a $1,000 cap and a $3,000 earnings threshold. OBBBA made the $2,200 CTC permanent with a $1,700 refundable cap and a $2,500 threshold. It also understated the 2026 one-child EITC maximum as $4,355 instead of $4,427." +us,scenario_012,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model capped the refundable CTC at the pre-TCJA $1,000 instead of the 2026 ACTC cap of $1,700, which alone loses $700. It also put the 2026 one-child EITC maximum at $4,442 instead of $4,427." +us,scenario_012,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model set the ACTC at $1,749, which exceeds the 2026 per-child refundable cap of $1,700. It also overstated the one-child EITC maximum as $4,454 instead of $4,427." +us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC, the TCJA-sunset figure. The 2026 ACTC cap under OBBBA is $1,700, and 15% of earnings above $2,500 ($2,925) exceeds it. Its EITC of $4,423 is also slightly below the $4,427 one-child maximum." +us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model overstated the EITC at $5,571, well above the 2026 one-child maximum of $4,427. It also understated the refundable CTC at the sunset-law $1,000 instead of the $1,700 ACTC cap." +us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC instead of the 2026 ACTC cap of $1,700. It also rounded the EITC to $4,500 instead of the exact one-child maximum of $4,427." +us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $1,000 for the refundable CTC instead of the 2026 ACTC cap of $1,700. It also used an EITC of $4,320, below even the 2025 one-child maximum, instead of the 2026 maximum of $4,427." +us,scenario_012,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no computation. The correct derivation is the full one-child EITC of $4,427 (earnings are on the plateau) plus the $1,700 ACTC. Its $4,050 is below the EITC maximum alone, so it either left out the ACTC entirely or badly understated both credits." +us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got ACTC right at $1,700 but used the 2025 one-child EITC maximum of $4,328 instead of the 2026 inflation-adjusted $4,427. That leaves the total $99 short." +us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model got the $1,700 ACTC right but used the 2025 one-child EITC maximum of $4,328 instead of the 2026 maximum of $4,427. That understates the total by $99." +us,scenario_012,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 maximum of $4,427. It also used the TCJA-sunset $1,000 ACTC instead of the 2026 refundable cap of $1,700." +us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model set the refundable CTC to $0 because there was no tax liability, which has refundability backwards. With zero liability the full $1,700 ACTC is paid out, since 15% × ($22,000 − $2,500) exceeds the cap. It also put the EITC at $2,591 even though $22,000 is on the one-child plateau, where the full $4,427 maximum applies." +us,scenario_012,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the full $2,200 CTC as refundable and never applied the 2026 per-child refundable cap of $1,700. It also used $4,420 for the EITC instead of the $4,427 one-child maximum." +us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"The model denied the EITC because the spouse is 18. The age-25 minimum applies only to the childless EITC; a joint filer with a qualifying child faces no age test, so the full $4,427 applies. Its $2,524 refundable CTC also exceeds the 2026 per-child ACTC cap of $1,700." +us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0, claiming filing status and earned income were not specified. The facts give $22,000 in wages, a married couple filing jointly (filing is assumed), and a 10-year-old qualifying child. Those facts yield EITC $4,427 plus ACTC $1,700." +us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated the 2026 one-child EITC maximum as $4,526 instead of $4,427. It also used $1,800 for the refundable CTC, above the 2026 ACTC cap of $1,700." +us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the correct 2026 one-child EITC maximum of $4,427. Its only error is putting the refundable CTC at $1,800 instead of the 2026 ACTC cap of $1,700, which overshoots by $100." +us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model applied phase-out reductions and got a $2,479 EITC. But $22,000 is past the one-child phase-in end and below the MFJ phase-out start, so the full $4,427 maximum applies. It also left out the $1,700 ACTC entirely." +us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the post-sunset ACTC: a $1,000 cap above $3,000 of earnings. OBBBA made the TCJA CTC permanent, so the 2026 refundable cap is $1,700 above $2,500. It also understated the one-child EITC maximum as $4,318 instead of $4,427." +us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and capped the ACTC at $1,000. Under OBBBA the 2026 refundable CTC cap is $1,700. It also set the EITC at $4,436 instead of the 2026 one-child maximum of $4,427." +us,scenario_012,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model's structure is correct: plateau EITC plus the $1,700 ACTC cap. But it used $4,437 as the 2026 one-child EITC maximum instead of $4,427, which overshoots by $10." +us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied the pre-TCJA ACTC ($1,000 cap, 15% above $3,000) instead of the 2026 OBBBA refundable cap of $1,700. It also inflated the 2025 EITC to $4,436 instead of using the 2026 one-child maximum of $4,427." +us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_refundable_credits, so there was no answer to score. The correct derivation is EITC $4,427 plus ACTC $1,700." +us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied the ACTC correctly as min($1,700, 15% × earnings above $2,500) = $1,700. But it put the 2026 one-child EITC maximum at $4,401 instead of $4,427, leaving the total $26 short." +us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model claimed the EITC and refundable CTC 'partially offset' each other and returned $0. Both credits are positive and additive for this household: the full one-child EITC of $4,427 on the plateau plus the $1,700 ACTC, since tax liability is zero." +us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the plateau EITC and the $1,700 ACTC cap correctly. It used $4,425 for the 2026 one-child EITC maximum instead of $4,427, a $2 shortfall." +us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model counted the unborn child as a second qualifying child. That doubled the ACTC to $3,400 and switched to the two-child EITC maximum of $7,830. An unborn child is not a qualifying child for either credit, so the correct amounts are the one-child EITC of $4,427 and a single $1,700 ACTC." +us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an EITC of $6,936, a multi-child figure far above the 2026 one-child maximum of $4,427. It also claimed the CTC was used against regular tax. That is wrong: $22,000 is below the MFJ standard deduction, so tax liability is zero and $1,700 is paid as ACTC." +us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts did not support eligibility and never ran either qualifying test. At $22,000 for a family of three, income is 81% of the poverty guideline, under the 130% free-meal limit. The household's SNAP receipt of about $407.90 a month separately makes the child categorically eligible, so the correct tier is FREE (1), not 0." +us,scenario_012,free_school_meals_eligible,grok-4.7,llm_error,age_disability,False,"The model applied the rule that unlisted booleans are false to K-12 enrollment and set it to false. PolicyEngine does not take enrollment as an input here; it derives it from age, so a 10-year-old counts as a K-12 student. With enrollment in place, SNAP categorical eligibility and income at 81% of the poverty guideline both put the child in the FREE tier." +us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model claimed income was above 130% of the poverty guideline, but $22,000 for a three-person household is only 81% of the guideline, well below the roughly $35,000 limit for that size. It also missed that SNAP receipt of about $407.90 a month makes the child categorically eligible for free meals whatever the income." us,scenario_012,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely stated that Mississippi adopted ACA Medicaid expansion and applied the 138% FPL expansion-adult threshold. Mississippi has no such pathway for this head, whose Medicaid category is NONE." us,scenario_012,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly rejected Medicaid expansion but treated $22,000, equal to 0.81 FPL under the engine’s MAGI calculation, as low enough for Mississippi’s much narrower parent/caretaker pathway. The head does not meet that pathway’s income requirements, so merely living with a minor child does not establish eligibility." us,scenario_012,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model claimed that income at 0.81 FPL falls below a parent/caretaker limit it identified as approximately 27% FPL, reversing the threshold comparison. It also invented pregnancy-spouse and MAGI-disregard routes that do not place the head in any Medicaid category." @@ -846,45 +923,51 @@ us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without first assigning the household to the free-meals tier. The household's 0.81 FPG ratio and categorical eligibility produce FREE status, which makes reduced-price eligibility false." us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that income below 130% FPG places the child in the free-meal range and that reduced-price support applies only from 130% through 185%, but then submitted 1 contrary to its own derivation. FREE-tier classification, independently reinforced by categorical eligibility, makes the reduced-price output 0." -us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It invented a ~$400 Standard Utility Allowance on the theory that the $411 SPM energy subsidy establishes a deductible utility expense, pushing shelter costs to $850 and manufacturing a $222 excess-shelter deduction; with no utility expense input, shelter is the $450 rent, which falls below half of the $1,257.67 post-deduction income, so the excess shelter deduction is $0. It then compounded this by using a $1,023 three-person maximum allotment instead of the FY2026 $785 and finally discarding its own $713 result for an unexplained $378/month." -us,scenario_012,snap,claude-fable-5.1,llm_error,period_annualization,False,"Its monthly derivation is right — $209 standard deduction, 20% earned-income deduction, no excess shelter because $450 rent is under half of $1,257.67 net-before-shelter, and $785 − 30% of net = $407.90 — but it multiplied that single monthly figure by twelve. Calendar-year 2026 spans two federal fiscal years: October through December 2026 use FY2027 standard deductions and poverty guidelines, which raise those three months' allotments and lift the annual total from $4,892 to $4,952.09." -us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the 20% earned-income deduction entirely, taking net income as $1,833 − $256 = $1,577 rather than $1,833.33 − $366.67 − $209 = $1,257.67, and used a $694 three-person maximum allotment instead of the FY2026 $785. It then subtracted the $411 LIHEAP energy subsidy as if it were countable income, when energy assistance is excluded from SNAP income, driving the monthly benefit down to $84." -us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"Its derivation was essentially correct — $1,242.66 net income, $785 maximum allotment, no excess shelter because $484 in shelter is below half of adjusted income, yielding about $412/month and $4,946/year — and it then abandoned that result for an unexplained '$601/mo × 12 = $7,212' with no supporting arithmetic. The $601 monthly figure corresponds to no deduction sequence: $785 minus 30% of $1,257.67 is $407.90." -us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"It correctly found no excess shelter deduction ($450 rent below the $631 half-of-net threshold) and computed $785 − 30% × $1,262 = $406/month, or $4,872/year, then discarded that for an unexplained '$514/mo' rounded to a 'typical MS award.' The correct monthly allotment is $407.90; nothing in the rules produces $514." -us,scenario_012,snap,claude-opus-5,llm_error,other,False,"Its $378/month figure is the household's own 30%-of-net-income contribution ($377.30 on net income of $1,257.67), not the allotment; it reported the contribution instead of subtracting it from the $785 FY2026 three-person maximum to get $407.90. It correctly recognized that the $5,400 rent generates no excess shelter deduction but then never applied the maximum allotment step." -us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It counted the spouse's unborn child as a fourth household member — SNAP household size is the three people living together, and unlike WIC, pregnancy adds no member — so it used the four-person $973 maximum allotment and four-person deduction schedule instead of the three-person $785 and $209. It compounded this by fabricating a $414/month Mississippi Standard Utility Allowance from the $411 energy subsidy, creating a $233 excess-shelter deduction where the $450 rent alone yields $0." -us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It treated the pregnancy as adding a member, sizing the SNAP unit at four and applying a ~$994 maximum allotment instead of the three-person FY2026 $785. It also added an unstated utility allowance to the $450 rent to manufacture a shelter deduction; with no utility expense input, shelter costs never reach half of the $1,257.67 post-deduction income and the excess shelter deduction is $0, leaving $407.90/month." -us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,period_annualization,False,"It got the structure right — no excess shelter deduction, benefit = maximum allotment minus 30% of net income — but used a $786 maximum and $211 standard deduction instead of the FY2026 $785 and $209, rounded the monthly allotment down to a whole $409, and held FY2026 parameters across all twelve calendar months. The October–December 2026 months are governed by FY2027 standard deductions and poverty guidelines, which raise the annual total to $4,952.09." -us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It invented a $500/month Standard Utility Allowance and added it to the $450 rent to create a $316.67 excess-shelter deduction; the household reports no utility expense, so shelter is $450, which is below half of the $1,257.67 income after the earned-income and standard deductions, making the excess shelter deduction $0. That fabricated deduction cut net income from $1,257.67 to $950 and inflated the monthly allotment from $407.90 to $500." -us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly found no excess shelter deduction but used the FY2025 three-person maximum allotment of $768 instead of the FY2026 $785, and a $204 standard deduction instead of $209, giving $389.20/month where the correct figure is $407.90. Its answer is exactly twelve times that stale-parameter monthly benefit." -us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It asserted that the $411 SPM energy subsidy triggers a Standard Utility Allowance, and its $482/month figure is consistent with roughly a $250/month excess-shelter deduction built on that allowance. No utility expense is listed, so deductible shelter is the $450 rent alone, which is below half of the $1,257.67 income after the 20% earned-income and $209 standard deductions, making the excess shelter deduction $0 and the allotment $785 − $377 = $407.90." -us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $446/month figure is consistent with reducing net income to about $1,130 via a roughly $128/month shelter deduction; the correct net income is $1,257.67 with no excess shelter deduction, since the $450 rent is below the $628.83 half-of-income threshold and no utility expense is listed. Subtracting 30% of $1,257.67 from the FY2026 three-person maximum of $785 gives $407.90/month, not $446." -us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly applied an excess-shelter deduction 'based on rent and the standard utility allowance'; its $497.50/month result implies roughly a $300/month shelter deduction. The household lists no utility expense, so shelter is the $450 rent, under half of the $1,257.67 post-deduction income, and the excess shelter deduction is $0, leaving $407.90/month." -us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction of roughly $219/month to reach $457, treating the $411 energy subsidy as establishing utility costs. With only $450/month rent and no utility expense input, shelter never exceeds half of the $1,257.67 income after the 20% earned-income and $209 standard deductions, so the excess shelter deduction is $0 and the monthly allotment is $407.90." -us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $120.83/month result is consistent with applying the 30% contribution rate to income that never received the 20% earned-income deduction ($366.67) or the $209 standard deduction. Applying both gives net income of $1,257.67, a $377 contribution, and $785 − $377 = $407.90/month against the FY2026 three-person maximum." -us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It claimed an excess-shelter deduction of roughly $228/month to arrive at $453. Deductible shelter here is the $450 rent only — the $411 energy subsidy creates no utility expense — and $450 is below half of the $1,257.67 income remaining after the earned-income and standard deductions, so the excess shelter deduction is $0 and the allotment is $407.90/month." -us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It listed 'shelter costs' as a deduction and landed at $471.83/month, implying about a $281/month excess-shelter allowance. With $450 rent and no utility expense, shelter falls short of the $628.83 half-of-income threshold, the excess shelter deduction is $0, and the monthly allotment is $785 − 30% × $1,257.67 = $407.90." -us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its structure and its finding of no excess shelter deduction are right, but it used a $9,216 annual maximum ($768/month, the FY2025 three-person figure) instead of the FY2026 $785, and a $2,400 annual standard deduction ($200/month) instead of $209. Correcting both gives $407.90/month rather than the $388/month its arithmetic implies." -us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"It added a fourth member for the spouse's pregnancy — SNAP counts only the three people in the household, with no unborn-child member — and therefore used a $975 four-person maximum allotment and $238 standard deduction instead of $785 and $209. It also fabricated a $360/month Standard Utility Allowance from the $411 energy subsidy, producing a $195.67 excess-shelter deduction where the $450 rent alone yields $0." -us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It treated the $411 energy subsidy as conferring a utility allowance that pushes shelter costs above half of net income, cutting net income from $1,257.67 to about $1,044. The household lists no utility expense, so shelter is the $450 rent, below the $628.83 half-of-income threshold; the excess shelter deduction is $0 and the allotment is $785 − $377 = $407.90/month, not $467." -us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It credited the household with 'housing/utility costs' deductions to reach $497/month, roughly a $300/month excess-shelter allowance. Only $450/month rent is listed and no utility expense, so shelter stays under half of the $1,257.67 income after the 20% earned-income and $209 standard deductions, the excess shelter deduction is $0, and the monthly allotment is $407.90." -us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It declared the household ineligible for want of 'SNAP eligibility flags,' but the listed facts fully determine eligibility: gross monthly income of $1,833.33 is 83% of the three-person poverty guideline (well under the 130% gross test), net income of $1,257.67 is 57% of it, and countable assets are $2. The household qualifies and receives $407.90/month, not $0." -us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It stated outright that 'the energy subsidy qualifies the unit for the heating/cooling utility allowance,' using that to cut net income to about $1,000 and raise the allotment to $483/month. The $411 SPM energy subsidy is not a utility expense input; deductible shelter is the $450 rent, which is below half of the $1,257.67 post-deduction income, so the excess shelter deduction is $0 and the allotment is $407.90." -us,scenario_012,snap,gpt-5.6-luna,llm_error,period_annualization,False,"It correctly refused to grant a utility allowance and computed a monthly benefit of $408, then multiplied one FY2026 monthly figure by twelve. The precise monthly allotment is $407.90, and October–December 2026 fall in FY2027, where updated standard deductions and poverty guidelines raise those months, bringing the calendar-year total to $4,952.09." -us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied a 'heating/cooling utility deduction' on top of rent to reach $478/month, implying about a $250/month excess-shelter allowance. No utility expense is listed and the $411 energy subsidy does not create one, so shelter is the $450 rent, below the $628.83 half-of-income threshold; the excess shelter deduction is $0 and the allotment is $407.90/month." -us,scenario_012,snap,gpt-5.6-terra,llm_error,period_annualization,False,"It reached the right monthly structure — three-person unit, 20% earned-income and standard deductions, no shelter deduction — landing at $408/month, then annualized by a flat twelve. The exact FY2026 monthly allotment is $407.90, and the October–December 2026 months use FY2027 standard deductions and poverty guidelines, which lift the year to $4,952.09." -us,scenario_012,snap,gpt-6-astra,llm_error,period_annualization,False,"It correctly rejected the utility allowance and computed exactly $407.70/month ($9,420 annual maximum less 30% of $15,092 net income), then multiplied by twelve to get $4,892.40. Calendar-year 2026 crosses the federal fiscal-year boundary: October through December 2026 apply FY2027 standard deductions and poverty guidelines, raising the annual benefit to $4,952.09." -us,scenario_012,snap,grok-4.3,llm_error,thresholds_rates,False,"Its $300/month figure is consistent with subtracting the correct $377 contribution from a maximum allotment near $677 — an obsolete pre-FY2023 three-person level — rather than the FY2026 $785. With the correct maximum and net income of $1,257.67, the monthly allotment is $407.90." -us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"Its deduction logic is right, including correctly finding zero excess shelter because $450 rent is below half of adjusted income, but it used a $766 maximum allotment and a $198 standard deduction instead of the FY2026 $785 and $209. Those two stale parameters cut the monthly allotment from $407.90 to $385." -us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied an 'excess-shelter deduction that includes the LIHEAP-triggered SUA,' reaching $456/month. Receipt of the $411 energy subsidy does not supply a utility expense here; deductible shelter is the $450 rent alone, which is below half of the $1,257.67 income after the earned-income and standard deductions, so the excess shelter deduction is $0 and the allotment is $407.90/month." -us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"It mixed periods: it subtracted the $210 standard deduction as an annual amount from annual income instead of $210 per month ($2,520/year), leaving net income of $17,390 rather than about $15,092. It also compared the $5,811 annual shelter figure against half of that inflated annual income and used a $9,422 annual maximum; with the standard deduction applied monthly, the benefit is $785 − 30% × $1,257.67 = $407.90/month." -us,scenario_012,snap,inkling,llm_error,period_annualization,False,"Its derivation reached the correct $4,896 (about $407.90/month), then it rounded the monthly allotment down to $407 and multiplied by twelve. The exact monthly benefit is $407.90 for the FY2026 months, and October through December 2026 use FY2027 standard deductions and poverty guidelines that raise those months to a $4,952.09 annual total." -us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for snap, so the required output key was absent from the submission. The substantive answer is $4,952.09, from a $407.90 monthly allotment ($785 maximum less 30% of $1,257.67 net income) with the October–December FY2027 parameter update." -us,scenario_012,snap,kimi-k3,llm_error,period_annualization,False,"Every monthly step is correct — $366.67 earned-income deduction, $209 standard deduction, no shelter deduction because $450 rent is under half of $1,257.67, and $785 − $377.30 = $407.70 — but it annualized by multiplying that one FY2026 month by twelve. The October–December 2026 months are computed with FY2027 standard deductions and poverty guidelines, raising the year from $4,892.40 to $4,952.09." -us,scenario_012,snap,minimax-m3,llm_error,other,False,"It abandoned the calculation mid-sentence ('need to compute') and submitted a placeholder $0 for a household that plainly qualifies: gross income at 83% of the three-person poverty guideline, $2 in assets, and all members meeting work and immigration tests. The correct figure is $785 minus 30% of $1,257.67 net income, or $407.90/month." -us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added a '$470 LIHEAP-funded standard utility allowance' to the $450 rent, generating a roughly $290 excess-shelter deduction that cut net income from $1,257.67 to about $985. The $411 SPM energy subsidy supplies no deductible utility expense; with rent alone, shelter is below the $628.83 half-of-income threshold, the excess shelter deduction is $0, and the allotment is $407.90/month." -us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the $411 energy subsidy as countable unearned income — energy assistance is excluded from SNAP income — and used a $768 maximum allotment and $198 standard deduction instead of the FY2026 $785 and $209. It then abandoned its own $4,525 result for an unexplained $426/month; the correct monthly allotment is $785 − 30% × $1,257.67 = $407.90." -us,scenario_012,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied the net income test to undeducted gross income, comparing $22,000 against the $21,080 three-person poverty line and declaring the household ineligible. The net income test is applied after the 20% earned-income deduction ($4,400) and the standard deduction, leaving net income of about $15,092 — 57% of the poverty guideline — so the household qualifies and receives $407.90/month." +us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model gave the energy subsidy a ~$400 utility allowance, but LIHEAP receipt does not trigger one for a household with no elderly or disabled member. It also used a $1,023 maximum allotment. Its final $378/month is exactly the household's 30% contribution, not the $785 − $378 = $407 allotment." +us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The deductions ($209 standard, $367 earned income) and the no-shelter conclusion are right. However, the model left the benefit at $407.70 instead of applying SNAP rounding: net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407/month, or $4,884." +us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left out the 20% earned income deduction entirely. It also used an outdated $256 standard deduction and a $694 maximum allotment. It then treated the excluded $411 energy subsidy as reducing net income, which cut the benefit to $84/month instead of $407." +us,scenario_012,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached ~$412/month using a $224 standard deduction instead of $209, then dropped that result for an unsupported $601/month. The correct figures are $785 minus 30% of $1,258 (rounded up to $378), which gives $407/month." +us,scenario_012,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly found no shelter deduction and got about $406/month, using a $204 standard deduction instead of $209. It then raised the result to $514/month by 're-estimating with shelter', even though it had already shown rent was below half of net income." +us,scenario_012,snap,claude-opus-5,llm_error,other,False,"The model reported $378/month as the SNAP benefit. That is the household's expected contribution (30% of $1,258 net income, rounded up), not the allotment, which is $785 − $378 = $407/month." +us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as qualifying for a ~$395 utility allowance and took a $216 excess shelter deduction. Under the 2026 rule, LIHEAP receipt gives a utility allowance only to households with an elderly or disabled member, so only the $450 rent counts, and it is below half of the $1,258 net income." +us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child as a fourth SNAP member (SNAP counts only born members) and used the four-person $973 maximum. It also added a LIHEAP-triggered utility allowance the household does not qualify for. Its submitted $5,136 does not even match its own $7,968 derivation." +us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model treated the pregnant spouse's unborn child as a SNAP member, making a household of four with a ~$994 maximum. It also added a utility allowance from the energy subsidy. The correct unit is three people with a $785 maximum and no shelter deduction." +us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"The model's structure is correct (net $1,258, no shelter deduction), but it rounded the 30% contribution down to $377 instead of up to $378. That gave $408/month instead of $407, and $4,890 is not even 12 × $408." +us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $211 standard deduction and a $786 maximum allotment instead of the FY2026 three-person figures of $209 and $785. It also skipped rounding the 30% contribution up, giving $409/month instead of $407." +us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added a $500 utility allowance to rent and took a $316.67 excess shelter deduction. This household has no elderly or disabled member, so its LIHEAP subsidy does not trigger a utility allowance and there is no shelter deduction. The model also used a $200 standard deduction instead of $209." +us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,The model used the FY2025 standard deduction ($204) and maximum allotment ($768) instead of the FY2026 figures ($209 and $785). It also left the benefit unrounded at $389.20 instead of computing $785 − $378 = $407/month. +us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"The model counted the unborn child and used a four-person SNAP household and maximum allotment. SNAP counts only the three born members, which gives a $785 maximum and $407/month." +us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy triggers a Standard Utility Allowance and took an excess shelter deduction, giving $482/month. For a household with no elderly or disabled member, LIHEAP receipt does not confer a utility allowance, so the $450 rent alone produces no shelter deduction." +us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $446/month implies net income of about $1,130, roughly $128 below the correct $1,258. That matches an excess shelter deduction the household is not entitled to, since the $450 rent is below half of net income and no utility allowance applies. The correct figure is $785 − $378 = $407/month." +us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model included a Standard Utility Allowance in shelter costs and took an excess shelter deduction. Under the 2026 rule, a household with no elderly or disabled member gets no utility allowance from LIHEAP receipt, so there is no shelter deduction and the benefit is $407/month." +us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction, giving $457/month. Shelter costs are only the $450 rent, which is below half of the $1,258 net income, because the energy subsidy does not trigger a utility allowance for this household." +us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"The model's $1,450/year (about $121/month) implies countable net income of about $2,213, which is above the household's $1,833 gross income. This is not a coherent use of the SNAP formula. The correct figure is $785 − 30% of $1,258 net income = $407/month." +us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model took an excess shelter deduction, giving $453/month. The household's only shelter cost is $450 rent, which is below half of net income, and the LIHEAP subsidy does not give a utility allowance to a household with no elderly or disabled member." +us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model counted shelter costs as a deduction, giving about $472/month. Rent of $450 is below half of the $1,258 net income and no utility allowance applies, so there is no shelter deduction and the benefit is $407/month." +us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $200/month standard deduction and the FY2025 $768 maximum allotment ($9,216/year) instead of the FY2026 figures of $209 and $785. It also computed on annual totals without the monthly whole-dollar rounding." +us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added a SNAP member for the pregnancy and used the four-person $975 maximum and a $238 standard deduction. It also added a LIHEAP-triggered utility allowance. The correct unit is three people with a $785 maximum, a $209 standard deduction, and no shelter deduction." +us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model gave the $411 energy subsidy a utility allowance and took an excess shelter deduction, lowering net income to about $1,044. Without a utility allowance (no elderly or disabled member), net income stays at $1,258 and the benefit is $407/month." +us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model credited 'housing/utility costs' as deductions, giving about $497/month. The household has no utility allowance, and its $450 rent is below half of net income, so there is no shelter deduction and the benefit is $407/month." +us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned $0, saying SNAP eligibility flags were missing. Wages, household composition, and assets are enough to compute SNAP, and the household passes the gross and net income tests with a $407/month allotment." +us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model said the energy subsidy qualifies the household for the heating/cooling utility allowance and applied an excess shelter deduction. Under the 2026 rule, that pathway applies only to households with an elderly or disabled member, so net income is $1,258 and the benefit is $407/month, not $483." +us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"The model correctly applied no utility allowance and no shelter deduction, but rounded $785 − $377.40 to $408. SNAP rounds 30% of net income up to $378, which gives $407/month, or $4,884." +us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included a heating/cooling utility deduction from the energy subsidy and a rent-based shelter deduction, giving $478/month. The household has no elderly or disabled member, so it gets no utility allowance, and rent alone is below half of net income." +us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"The model got $408/month by rounding $785 − $377.40 to the nearest dollar. SNAP rounds the 30% contribution up to $378, which gives $407/month." +us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"The model correctly ruled out the utility allowance and shelter deduction, but computed on annual totals ($9,420 − 30% × $15,092). SNAP is computed monthly: net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407, giving $4,884 for the year." +us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $406.20/month implies net income of $1,262.67, which means it used the FY2025 $204 standard deduction instead of $209. It also skipped rounding the 30% contribution up. The correct result is $407/month." +us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model applied a shelter deduction, giving $504/month. Rent of $450 is below half of the $1,258 net income and no utility allowance applies, so there is no shelter deduction." +us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model added an estimated $433 heating utility allowance 'triggered by the energy subsidy'. For a household with no elderly or disabled member, LIHEAP receipt no longer confers a utility allowance, so there is no excess shelter deduction and the benefit is $407/month." +us,scenario_012,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $300/month matches $785 − 30% × ($1,833 − $209), which leaves out the 20% earned income deduction of $366.67. With that deduction, net income is $1,258 and the benefit is $407/month." +us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used outdated parameters: a $198 standard deduction and a $766 maximum allotment (FY2024-era figures) instead of FY2026's $209 and $785. It also annualized $385 as $4,625 instead of $4,620." +us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model included a 'LIHEAP-triggered SUA' in an excess shelter deduction, giving $456/month. That trigger applies only to households with an elderly or disabled member, so this household has no shelter deduction and receives $407/month." +us,scenario_012,snap,grok-4.7,llm_error,other,False,"The model used the correct $209 and $785 parameters but computed on annual totals without SNAP's monthly rounding. Net income rounds to $1,258, 30% rounds up to $378, and the allotment is $407 × 12 = $4,884." +us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted the $210 monthly standard deduction once from annual income instead of 12 times, which overstated net income by about $2,300 a year. It also added the energy subsidy to shelter costs and used a $9,422 annual maximum instead of 12 × $785." +us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so nothing was scored against the $4,884 reference." +us,scenario_012,snap,kimi-k3,llm_error,other,False,"The deductions and the no-shelter conclusion are correct, but the model left the benefit at $407.70. SNAP rounds net income to $1,258 and 30% of it up to $378, which gives $407/month, or $4,884." +us,scenario_012,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model stopped before doing any calculation and output $0. The household passes the gross and net income tests, with net income of $1,258 and an allotment of $785 − $378 = $407/month." +us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model added a 'LIHEAP-funded standard utility allowance' of about $470 and a ~$290 excess shelter deduction. With no elderly or disabled member, LIHEAP receipt does not confer a utility allowance, so net income is $1,258, not ~$985." +us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model counted the $411 energy subsidy as unearned income, but energy assistance is excluded from SNAP income. It also used an outdated $768 maximum and a $198 standard deduction, then changed its own $4,525 result to $426/month without justification." +us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model set net income equal to gross earnings and left out the $209 standard deduction and the 20% earned income deduction. It then compared that against an outdated $21,080 poverty line. Correct net income is $1,258/month, well under the $2,220.83 net income limit." us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It omitted the prior Medicaid screen: the 18-year-old spouse qualifies for Medicaid under the OLDER_CHILD category, which makes her ineligible for CHIP." us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age once income requirements are met. It failed to apply the rule excluding anyone already Medicaid-eligible; this spouse is Medicaid-eligible as an OLDER_CHILD. us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified low-income pregnancy coverage as CHIP-funded coverage without first testing Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP, irrespective of household size adjustments for the unborn child or pregnancy income limits." @@ -895,217 +978,286 @@ us,scenario_012,spouse_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibil us,scenario_012,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to classify the 18-year-old spouse under Mississippi's OLDER_CHILD Medicaid category and therefore never applied that category's MAGI income test. At 0.67 times FPL, the spouse satisfies the income condition and is eligible." us,scenario_012,spouse_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the spouse's explicitly listed pregnancy, which places her in a WIC categorical group, and failed to apply the WIC income limit to the household's $22,000 income. Those facts establish her eligibility; the presence of a young child is not needed for her pregnancy-based pathway." us,scenario_012,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_012,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $4,600 MFJ standard deduction and omitted Mississippi's personal exemption entirely ($12,000 for married filing jointly plus $1,500 for the dependent child), leaving $17,400 instead of the correct $3,900 of taxable income. It then applied the 4% rate to the whole base, ignoring that Mississippi's flat rate reaches only taxable income above the $10,000 zero-rate amount; with $3,900 of taxable income the liability is $0, not $696." -us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model built the Mississippi base correctly — $22,000 less the $4,600 MFJ standard deduction and $13,500 of exemptions ($12,000 joint personal plus $1,500 dependent) = $3,900 — then taxed the entire $3,900 at 4%. Mississippi's post-HB 531 structure taxes only taxable income exceeding $10,000, so $3,900 falls entirely in the 0% band and the tax is $0, not $156." -us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model reached the correct $3,900 Mississippi taxable income but applied a rate to the full amount rather than only to income above the $10,000 zero-rate threshold, which exempts all $3,900. It compounded this with a 3.9% rate that is not the 2026 Mississippi rate (the HB 531 phase-down sets 4.0% for 2026), producing $152.10 where the correct liability is $0." -us,scenario_012,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly computed Mississippi taxable income of $3,900 ($22,000 less the $4,600 MFJ standard deduction, $12,000 joint personal exemption, and $1,500 dependent exemption) and then multiplied the entire base by the 4% 2026 rate. Mississippi applies that rate only to taxable income over $10,000, so the $3,900 base is fully within the zero-rate amount and produces $0, not $156." -us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $6,000 personal exemption for a joint return (Mississippi's married-filing-jointly exemption is $12,000) and dropped the $1,500 dependent exemption for the 10-year-old, inflating taxable income to $11,400 instead of $3,900. It also used the 2025 rate of 4.4% rather than the 2026 rate of 4.0% and taxed the full base instead of only the portion above the $10,000 zero-rate amount, yielding $502 where the correct answer is $0." -us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model substituted $2,300 per household member as the ""personal exemption,"" which is Mississippi's single-filer standard deduction, not its exemption; the actual exemptions are $12,000 for the joint filers plus $1,500 for the dependent child, so taxable income is $3,900 rather than $10,500. It then taxed the entire base at 4% instead of taxing only income above the $10,000 zero-rate amount, producing $420 where the liability is $0." -us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own derivation twice landed on a Mississippi base at or below zero and stated the tax is $0, then discarded that result and submitted $340 as a hedge against ""potential lower deduction amounts."" It also used a repealed 5% rate and a fabricated $6,000 dependent exemption (Mississippi's dependent exemption is $1,500); the correct base of $3,900 sits below the $10,000 zero-rate amount, so the answer it abandoned — $0 — was the right one." -us,scenario_012,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied Mississippi's repealed graduated schedule with the brackets inverted — 5% on the first $10,000 and 3% above it — when 2026 law imposes a single 4.0% rate that applies only to taxable income exceeding $10,000. It also invented a $7,400 standard deduction and a $1,500 spouse exemption in place of the $4,600 MFJ standard deduction and $12,000 joint personal exemption, and subtracted a $970 ""family credit"" that does not exist in Mississippi law; the correct $3,900 base is entirely untaxed, giving $0 rather than $510." +us,scenario_012,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted only the $4,600 MFJ standard deduction and left out the $12,000 MFJ personal exemption and the $1,500 dependent exemption, which reduce taxable income to $3,900. It then applied 4% to all $17,400 and ignored Mississippi's 0% bracket on the first $10,000 of taxable income. $3,900 falls entirely inside that bracket, so the tax is $0." +us,scenario_012,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It got taxable income right at $3,900 but applied the old 3% low bracket to it. Current Mississippi law taxes the first $10,000 of taxable income at 0% and charges the flat rate only above $10,000, so $3,900 produces $0 tax." +us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed $3,900 of taxable income correctly and then applied the 4.0% rate from the first dollar. Mississippi's first $10,000 of taxable income is taxed at 0%, so none of the $3,900 is taxed and the liability is $0." +us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It derived the $3,900 taxable income correctly but taxed all of it at a flat 3.9%. That rate is also not Mississippi's 2026 rate of 4.0%. Its main error is missing the 0% bracket on the first $10,000 of taxable income, which makes the tax $0 at any rate." +us,scenario_012,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly reduced AGI to $3,900 of taxable income and then applied the 4% rate to the whole amount. Mississippi exempts the first $10,000 of taxable income with a 0% bracket, so the tax is $0." +us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $6,000 personal exemption for a joint couple instead of Mississippi's $12,000 MFJ exemption and dropped the $1,500 dependent exemption for the child, which overstated taxable income at $11,400 instead of $3,900. It also used the 2025 rate of 4.4% instead of 4.0% and taxed from the first dollar, ignoring the 0% bracket on the first $10,000. Together these turned a $0 liability into $502." +us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It replaced Mississippi's exemptions ($12,000 for the MFJ couple plus $1,500 per dependent, $13,500 in total) with a made-up $2,300 per person ($6,900 in total), which overstated taxable income at $10,500 instead of $3,900. It then applied 4% to all of it without the 0% bracket on the first $10,000, so it reported $420 instead of $0." +us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Its own reasoning reached $0 because deductions and exemptions exceeded the $22,000 of income. It then dropped that result over stated uncertainty and submitted $340, which is 5% of an assumed $6,800 of taxable income with no derivation. The correct $4,600 standard deduction, $12,000 personal exemption, and $1,500 dependent exemption (not the $6,000 it used) leave $3,900. That amount falls inside Mississippi's 0% bracket on the first $10,000, so the tax is $0." +us,scenario_012,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It used a tax structure that is not Mississippi's: a $7,400 standard deduction instead of the $4,600 MFJ amount, a $1,500 spouse exemption instead of the $12,000 MFJ personal exemption, graduated 5% and 3% brackets, and a $970 family credit that Mississippi does not have. Mississippi's actual rules give $3,900 of taxable income, taxed at 0% because it is under the $10,000 zero bracket, so the liability is $0." us,scenario_013,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"GLM-5.3 included roughly 11.4% of the $23,736 Social Security benefit in income and reported AGI of $8,270, but provisional income here is $11,868 (half of benefits) + $6,720 pension + $16 dividends + $72 capital gain = $18,676, below the $25,000 single base amount, so $0 of Social Security is taxable and AGI is $6,808; it compounded this by reading the $16 as qualified dividends and double-counting the $72 as both non-qualified dividends and non-Sch-D capital gains. It then submitted -$3 for a quantity that is floored at zero — nonrefundable credits, including the $250 it invented, cannot reduce liability below zero — directly contradicting its own statement that taxable income after the standard deduction was effectively 0." -us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age and disability as establishing an SSI-related or medically needy Arizona Medicaid pathway without identifying or applying an actual qualifying category. The head receives no SSI and qualifies through none of the available pathways, so assigning eligibility from an asserted aged/disabled income standard was incorrect." -us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model improperly applied ALTCS's 300%-of-SSI income ceiling without establishing eligibility for the institutional long-term-care pathway, then ignored that pathway's $2,000 resource limit despite $58,700 in bank assets. It also invented a large Social Security exclusion and medical-expense spend-down to force income below a regular aged/disabled limit; those calculations do not place the head in any Arizona Medicaid category." +us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model asserted that income was ""near SSI/FPL-based aged-disabled standards"" but never compared it to a limit. The head's countable income is about $30,5xx: $23,736 of Social Security plus the $6,720 pension and $88 of dividends and capital gains, less the $20 general disregard. That is about 1.9x FPL, well above the 100% FPL aged/disabled income standard and the SSI benefit rate. SSI is 0, so no SSI-linked pathway applies, and at age 80 the head is outside the MAGI adult group." +us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model said Social Security is ""mostly excluded via SSI methodology"" and cut it to $10,196 countable. SSI methodology counts Social Security in full as unearned income, minus only the $20 general disregard, so countable income is about $30,5xx, or 1.9x FPL, above the aged/disabled standard. The model also applied the ALTCS 300%-of-SSI limit, which requires an institutional level-of-care need this household does not have. It then deducted health premiums and OTC costs as a spend-down, which does not create eligibility here, and it ignored its own finding that the $58,700 in assets exceeds the $2,000 limit." +us,scenario_013,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model tested AGI of $6,808 against 138% FPL. MAGI adds back the $23,736 of nontaxable Social Security, which puts income at about $30,544, or 1.91x FPL. The 138% FPL adult expansion group also covers only ages 19-64, so it cannot apply to an 80-year-old. The aged/disabled pathway counts total income including Social Security, and that income is far above its limit." +us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied only the regular elderly/disabled tests: the 100% FPL net income limit and the ~$4,500 asset limit. It never applied Arizona's March 2026 expansion of categorical eligibility to 200% FPL ($2,608.33/mo), which covers its ~$2,545 of income and waives both tests. As a categorically eligible one-person household, it gets the $24 minimum for March–December, or $240." +us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Denied eligibility because net income (~$2,322) was above the 100% FPL net limit. It missed that Arizona's categorical eligibility at 200% FPL, in effect from March 2026, covers the $2,539 of gross income and treats the net income test as met. The categorically eligible single-person household receives the $24 minimum allotment for 10 months, or $240." +us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test and a $2,500 asset limit. Elderly/disabled households are exempt from the gross test, and Arizona's categorical eligibility at 200% FPL from March 2026 ($2,608.33/mo) waives the asset test for this $2,539/mo household. It therefore missed the $24/month minimum for March–December ($240)." +us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Correctly skipped the gross test for an elderly/disabled household, but then treated the 100% FPL net income limit as binding. Arizona's categorical eligibility at 200% FPL from March 2026 treats that test as met. The benefit formula comes out below zero, so the categorically eligible one-person household gets the $24 minimum for 10 months, or $240." +us,scenario_013,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Denied SNAP solely because the $58,700 in the bank exceeds the ~$4,500 elderly/disabled resource limit. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) removes the asset test for this $2,539/mo household. That leaves $24/month minimum benefits for March–December ($240)." +us,scenario_013,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Treated the regular SNAP asset limit as disqualifying. Under Arizona's categorical eligibility at 200% FPL ($2,608.33/mo from March 2026), the $2,539/mo household is categorically eligible and the asset test does not apply. The one-person household is therefore owed the $24 minimum allotment for 10 months ($240)." +us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Ended at the 100% FPL net income test (net ~$2,330 vs ~$1,304), without recognizing that Arizona's categorical eligibility at 200% FPL from March 2026 treats the net income test as met. It also missed that categorically eligible one-person households receive the $24 minimum even when the formula is negative, which gives $240 for March–December." +us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Denied eligibility using the regular net income test and the $4,500 asset limit. It also misread the $298 annual premium as a monthly amount. It never applied Arizona's categorical eligibility at 200% FPL (from March 2026), which covers $2,539/mo of income and waives both tests, leaving the $24 minimum for 10 months ($240)." +us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Invoked a 200% FPL categorical eligibility limit but set it at ~$2,510/mo instead of $2,608.33 (200% of the FY2026 $1,304.17 guideline). It also counted the $72 of capital gains, putting gross income at $2,545 instead of $2,539.33. With the correct figures the household passes, the asset test is waived, and it receives the $24 minimum for March–December ($240)." +us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Correctly found that $298 minus 30% of ~$2,330 net income is below zero, but stopped there. A one-person household made categorically eligible by Arizona's 200% FPL expansion (from March 2026) receives the $24 minimum allotment regardless, which totals $240 over March–December." +us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Applied the regular elderly/disabled asset limit to the $58,700 bank balance. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) covers this $2,539/mo household and waives the resource test. The result is the $24/month minimum for 10 months ($240)." +us,scenario_013,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Used the $4,250 elderly/disabled asset limit as disqualifying. From March 2026, Arizona's categorical eligibility at 200% FPL ($2,608.33/mo) covers $2,539.33 of gross income and eliminates the asset test. The household therefore receives the $24 one-person minimum for March–December ($240)." +us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied both the regular net income limit and the elderly/disabled resource limit. Arizona's categorical eligibility at 200% FPL from March 2026 treats both as met for this $2,539/mo household. A categorically eligible one-person household receives the $24 minimum, or $240 over 10 months." +us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Denied SNAP on the regular net income limit and asset limit, missing Arizona's categorical eligibility at 200% FPL effective March 2026, which covers $2,539/mo of gross income and waives both tests. The benefit formula is negative, so the one-person household gets the $24 minimum for 10 months ($240)." +us,scenario_013,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 185% FPL categorical eligibility limit (~$2,321/mo, itself miscomputed) for Arizona. From March 2026 the limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income (capital gains excluded) passes. Categorical eligibility then yields the $24 minimum for March–December ($240)." +us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Applied the regular one-person resource limit and net income test, ignoring Arizona's categorical eligibility at 200% FPL from March 2026, which covers the $2,539/mo household and waives both. The categorically eligible single-person household receives the $24 minimum allotment for 10 months ($240)." +us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Treated the elderly net income limit as disqualifying. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met for gross income of $2,539/mo. The one-person household therefore gets the $24 minimum benefit for March–December ($240)." +us,scenario_013,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Measured income against a 185% FPL categorical eligibility threshold and then applied the $4,250 asset limit. Arizona's limit is 200% FPL from March 2026 ($2,608.33/mo), and the $2,539.33 of countable income passes it, so the asset test is waived. The result is the $24 minimum for 10 months ($240)." +us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Applied the regular income and asset thresholds without recognizing Arizona's categorical eligibility at 200% FPL from March 2026, which covers $2,539/mo of gross income and treats the income and asset tests as met. The one-person household receives the $24 minimum for March–December ($240)." +us,scenario_013,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Claimed gross income exceeds Arizona's threshold, but $2,539.33/mo is below Arizona's 200% FPL categorical eligibility limit ($2,608.33/mo from March 2026, $2,660 from October). Categorical eligibility then gives the $24 one-person minimum for 10 months ($240)." +us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Relied on the net income limit and a zero formula benefit. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 treats the net income test as met, and that categorically eligible one-person households receive the $24 minimum even when the formula is negative, which gives $240 for March–December." +us,scenario_013,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used a 185% FPL categorical eligibility limit for Arizona and then applied the federal elderly asset limit. From March 2026 Arizona's limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income passes, so the asset test is waived. The household receives the $24 minimum for 10 months ($240)." +us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit for elderly/disabled households as the sole test. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met for this $2,539/mo household. The one-person household receives the $24 minimum for March–December ($240)." +us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"Applied gross and net income limits to a senior household, although elderly households face no gross test. It also counted $72 of capital gains as income. It missed Arizona's categorical eligibility at 200% FPL from March 2026, which covers $2,539.33/mo and deems the tests met, yielding the $24 minimum for 10 months ($240)." +us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Declared Social Security plus pension income too high under regular SNAP rules. Arizona's categorical eligibility at 200% FPL ($2,608.33/mo from March 2026) covers this $2,539/mo household. The categorically eligible one-person household receives the $24 minimum for March–December ($240)." +us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Gave no substantive computation and defaulted to zero. The correct derivation compares $2,539.33/mo of gross income with Arizona's 200% FPL categorical eligibility limit ($2,608.33 from March 2026), which confers eligibility, and pays the $24 one-person minimum for 10 months ($240)." +us,scenario_013,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Denied SNAP because $58,700 exceeds the elderly/disabled resource limit. Arizona's categorical eligibility at 200% FPL from March 2026 waives the asset test for this $2,539/mo household, leaving the $24 minimum allotment for March–December ($240)." +us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Concluded the allotment formula yields zero but omitted the minimum benefit. Under Arizona's categorical eligibility at 200% FPL from March 2026, the one-person household is categorically eligible and receives the $24 minimum each month, or $240 for March–December." +us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Stopped at the benefit formula producing no positive allotment. A one-person household that is categorically eligible through Arizona's 200% FPL expansion (from March 2026) receives the $24 minimum allotment regardless, totaling $240." +us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the regular net income limit, missing that Arizona's categorical eligibility at 200% FPL from March 2026 deems the net income test met for $2,539/mo of gross income. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)." +us,scenario_013,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Asserted that income exceeds Arizona's categorical eligibility limit, which means it used the older 185% FPL cap. From March 2026 the limit is 200% FPL ($2,608.33/mo), which the $2,539.33 of countable income passes, so the asset test is waived and the $24 minimum is paid for March–December ($240)." +us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Concluded no positive allotment is produced after deductions and omitted the $24 minimum benefit. That minimum applies to one-person households made categorically eligible by Arizona's 200% FPL expansion from March 2026, totaling $240." +us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Treated a negative formula benefit as zero SNAP. A one-person household categorically eligible under Arizona's 200% FPL limit (from March 2026) receives the $24 minimum allotment each month, or $240 for March–December." +us,scenario_013,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"Claimed income exceeds Arizona's categorical eligibility limit, applying the pre-2026 185% FPL cap. From March 2026 the cap is 200% FPL ($2,608.33/mo), which $2,539.33 of countable income passes. That waives the asset test and yields the $24 minimum for 10 months ($240)." +us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"Applied the regular income and asset limits. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 covers this $2,539/mo household and treats both tests as met. The household gets the $24 one-person minimum for March–December ($240)." +us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"Denied SNAP on the 100% FPL net income test, which Arizona's categorical eligibility at 200% FPL (from March 2026) deems met for $2,539/mo of gross income. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)." +us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"Applied the regular net income limit and the elderly/disabled resource limit. It missed Arizona's categorical eligibility at 200% FPL from March 2026, which waives both for this $2,539/mo household. That produces the $24 minimum for March–December ($240)." +us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the $298 maximum, but missed that one-person households categorically eligible under Arizona's 200% FPL expansion (from March 2026) receive the $24 minimum allotment anyway. That totals $240 for 10 months." +us,scenario_013,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Treated the $4,250 elderly/disabled resource limit as disqualifying. Arizona's categorical eligibility (gross income ≤200% FPL from March 2026) covers the $2,539/mo household and removes the asset test, leaving the $24 minimum for March–December ($240)." +us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"Denied SNAP because net income (~$2,310) exceeds the 100% FPL limit, without applying Arizona's categorical eligibility at 200% FPL from March 2026, which deems that test met. The categorically eligible one-person household receives the $24 minimum for 10 months ($240)." +us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Applied the federal elderly/disabled resource limit and the 100% FPL net income test. It missed that Arizona's categorical eligibility at 200% FPL from March 2026 covers $2,539/mo of gross income and treats both tests as met, producing the $24 minimum for March–December ($240)." +us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"Ended at the one-person net income limit, missing that Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met. It also omitted the $24 minimum allotment that categorically eligible one-person households receive, which totals $240 over 10 months." +us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"Applied the regular income and asset thresholds for an elderly household. Arizona's categorical eligibility at 200% FPL from March 2026 covers this $2,539/mo household and waives those tests, yielding the $24 minimum for March–December ($240)." +us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly exempted the household from the gross test but denied it on the 100% FPL net income test. Arizona's categorical eligibility at 200% FPL from March 2026 deems that test met, and the one-person household receives the $24 minimum for 10 months ($240)." +us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test, which does not apply to elderly/disabled households. It also missed Arizona's categorical eligibility at 200% FPL from March 2026 ($2,608.33/mo), which the $2,539/mo household passes, leaving the $24 minimum for March–December ($240)." +us,scenario_013,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Invented a rule that households made up entirely of people aged 60+ are ineligible for SNAP; no such exclusion exists, and elderly households get more favorable treatment. Through Arizona's categorical eligibility at 200% FPL from March 2026, this household is eligible for the $24 one-person minimum for 10 months ($240)." us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Arizona’s 2.5% rate to $88 of dividends and capital gains without subtracting the Arizona standard deduction, which reduces taxable income to zero. It also violated its own computation by submitting $744.10 after its explanation calculated $2.20." -us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted flatly that ""Arizona's tax code does not provide refundable credits at the state level for individual income tax purposes,"" which is false: A.R.S. § 43-1072's increased excise tax credit is refundable and pays $25 per resident person when federal AGI is under $12,500 for a single filer. With AGI of $6,736 and one person in the unit, the credit is $25." -us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model enumerated only the family income tax credit and the property tax credit and omitted the increased excise tax credit entirely, then imposed those credits' prerequisites — dependents, rent paid, or property tax paid — on the whole category. The excise tax credit conditions only on Arizona residency and federal AGI at or below $12,500 (single), both met at $6,736, yielding $25 for the taxpayer alone." -us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model named the increased excise tax credit but then applied Form 140PTC's rent/property-tax requirement to it and concluded ""no refundable credit conditions are met."" The excise tax credit has no rent, property tax, or dependent condition — low income is the qualifying condition, and the model's own finding that Arizona taxable income is very low is exactly what triggers the $25 credit." -us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model conditioned the credit on having dependents and on having tax liability to offset, concluding zero for ""a single filer with no dependents and no tax liability."" The increased excise tax credit counts the taxpayer himself as one person at $25 and is refundable, so it pays out in full against zero Arizona liability." -us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly identified the increased excise tax credit and the $25 per-person rate, then multiplied it by three claimed exemptions (personal, age 65+, and disability) to reach $75. A.R.S. § 43-1072 multiplies $25 by the number of resident persons on the return — taxpayer, spouse, and dependents — not by age or disability exemptions, so a one-person tax unit receives exactly $25; it also used $25,000 as the single-filer AGI cap when the statutory single/MFS cap is $12,500." -us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model claimed Arizona's credits are ""nonrefundable or require different qualifying conditions,"" citing the property tax credit and a dependent-based family income tax credit threshold. The increased excise tax credit is statutorily refundable and payable to a childless single filer whose federal AGI ($6,736) is under the $12,500 cap, which is worth $25 here." -us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model made the credit contingent on state tax liability and dependents — ""zero state tax liability and no dependents."" The increased excise tax credit is refundable, so zero liability is irrelevant, and the taxpayer counts as one person for the $25 per-person amount without any dependent." -us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated dependents as a prerequisite for every Arizona refundable credit. The increased excise tax credit pays $25 for the taxpayer himself with no dependents required, once federal AGI ($6,736) falls under the $12,500 single-filer cap." -us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model required ""dependents, donations, or qualifying property tax or rent,"" which are the conditions of the nonrefundable family income tax credit, the QCO charitable credit, and Form 140PTC respectively. It never reached the increased excise tax credit, whose only tests — Arizona residency and federal AGI at or below $12,500 single — are satisfied, producing $25." -us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model gave a bare denial with no rule cited. The correct derivation applies A.R.S. § 43-1072: federal AGI of $6,736 is under the $12,500 single-filer cap, so the refundable increased excise tax credit pays $25 for the one person in the unit; the model's zero is consistent with treating Arizona as having no refundable individual income tax credits at all." -us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model stated there are ""no state-specific refundable tax credits,"" contradicting Arizona's increased excise tax credit, which is refundable and paid at $25 per resident person. The single filer's federal AGI of $6,736 clears the $12,500 cap, so the correct amount is $25." -us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model asserted non-qualification without naming a program or test. The household qualifies on the only two tests the increased excise tax credit imposes — Arizona residency and federal AGI ($6,736) at or below $12,500 for a single filer — so the answer of zero is consistent with omitting that credit from its inventory of Arizona programs." -us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model denied any Arizona refundable credit without analysis. The increased excise tax credit is refundable, requires neither dependents nor earnings nor tax liability, and pays $25 for this one-person Arizona unit with $6,736 of federal AGI." -us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model limited its search to ""major refundable individual income tax credits"" and so discarded the small $25 increased excise tax credit, which is the only refundable Arizona credit in play. Its size does not affect eligibility: residency plus federal AGI of $6,736 under the $12,500 single cap yields the full $25." -us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model claimed ""household income exceeds limits for Arizona refundable tax credits."" The controlling figure is federal AGI, which is $6,736 because none of the $23,736 in Social Security is taxable at this income level; that is far below the $12,500 single-filer cap, and the model's zero is consistent with comparing gross Social Security plus pension (about $30,500) to the cap." -us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model declared that Arizona has no refundable individual income tax credit available to this household. The increased excise tax credit under A.R.S. § 43-1072 is refundable and available to any Arizona resident filer with federal AGI at or below $12,500 (single), paying $25 for the taxpayer here." -us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model gave a one-line denial that no refundable Arizona credit applies. Applying the statute, the $6,736 federal AGI is under the $12,500 single-filer ceiling and the one-person unit earns one $25 unit of the refundable increased excise tax credit." -us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said no Arizona refundable credit is ""triggered by the facts,"" but the triggering facts are present: Arizona residency and federal AGI of $6,736, which is below the § 43-1072 single-filer cap of $12,500. The credit is $25 per resident person, so $25 here." -us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model looked for dependents or earned income as triggers, which are conditions of earnings-based credits, not of Arizona's increased excise tax credit. That credit turns solely on residency and federal AGI at or below $12,500 for a single filer, both met, producing $25." -us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model concluded no qualifying Arizona refundable credit is supported by the facts. The facts given — Arizona residence, single filer, federal AGI $6,736 — are precisely the qualifying facts for the refundable increased excise tax credit, which pays $25 per person in the unit." -us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model required ""qualifying property tax, rent, dependents, or other credit-generating facts,"" importing the Form 140PTC and family income tax credit conditions. The increased excise tax credit imposes none of those and grants $25 to this resident single filer with $6,736 of federal AGI." -us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model concluded that a ""one-person elderly household"" indicates no refundable credit eligibility. Household size and age are not disqualifiers under A.R.S. § 43-1072; the one person in the unit is exactly one $25 credit unit, and the $6,736 federal AGI is under the $12,500 single cap." -us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model evaluated only the ""Working Poor Tax Credit"" — the nonrefundable QCO charitable contribution credit, which turns on donations rather than income sources. It never considered the refundable increased excise tax credit, which pays $25 to this Arizona resident with $6,736 of federal AGI whether the income is earned or unearned." -us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model checked only the Arizona property tax credit and concluded zero because no property tax or rent is listed. The increased excise tax credit is a separate refundable credit with no property tax or rent requirement, paying $25 per resident person when federal AGI is under $12,500 for a single filer." -us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model cited ""household composition and income level"" as disqualifying, but both point the other way: a one-person Arizona unit is one $25 credit unit under the increased excise tax credit, and the low income ($6,736 federal AGI) is what satisfies the $12,500 cap rather than defeating it." -us,scenario_013,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model reasoned from ""zero taxable income and no qualifying dependents"" to zero credit. Zero taxable income does not zero a refundable credit, and the increased excise tax credit counts the taxpayer alone as one person, so the correct amount is $25." -us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model searched for a refundable Arizona EITC or property tax credit and required earned income or qualifying expenses. Arizona's refundable credit here is the increased excise tax credit, which has no earnings test and no expense test — only residency and federal AGI at or below $12,500 single, met at $6,736 — so it pays $25." -us,scenario_013,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly identified the increased excise tax credit but disqualified the household by comparing $30,544 of gross income to a $25,000 limit. The test uses federal AGI, which is $6,736 because no Social Security is taxable at this income level, and the single-filer cap is $12,500 (the $25,000 figure applies to joint filers and heads of household), so the household clears the test and receives $25." -us,scenario_013,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model made dependents the gating condition for Arizona refundable credits. The increased excise tax credit allows $25 for each resident person on the return beginning with the taxpayer, so a childless single filer with $6,736 of federal AGI receives $25." -us,scenario_013,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model evaluated only Arizona's Form 140PTC property tax refund and rejected it on its roughly $3,750 single-senior income limit, using an inflated $30,544 income figure that counts gross Social Security. The applicable program is the separate refundable increased excise tax credit, whose test is federal AGI of $6,736 against a $12,500 single-filer cap, producing $25." -us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model looked for an ""Arizona Earned Income Tax Credit or family tax credit"" and required qualifying children or earned income. Arizona has no state EITC; its refundable credit is the increased excise tax credit, which requires neither children nor earnings and pays $25 to this resident single filer with $6,736 of federal AGI." -us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model reasoned that ""there is no Arizona tax liability to offset,"" applying nonrefundable-credit logic. The increased excise tax credit is refundable and is paid in full even with zero liability, giving $25 for the one person in this Arizona tax unit with $6,736 of federal AGI." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Subtracted the spouse's $14,717 employer-sponsored insurance premium from the $88,928 of wages, but that input never reduces PolicyEngine's employment income, so AGI is $88,927.65 rather than $74,211. Its $32,200 standard deduction and 10%/12% bracket ceilings were the correct 2026 values, and applying them to the correct base yields $2,480 + 12% × $31,927.65 = $6,311.32. It then submitted $5,647, a third figure its own arithmetic ($4,545) never produced." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a $28,000 MFJ standard deduction instead of the 2026 amount of $32,200, and then taxed the whole $60,928 at a flat 12%, skipping the 10% rate on the first $24,800. It then moved its own $7,311.36 to $6,819 with no computation; the correct schedule on $56,727.65 gives $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Set the 2026 MFJ 10%-bracket ceiling at $24,150 instead of $24,800, producing $6,324 on the correct $56,728 taxable income, and then discarded that result for an unexplained $5,235. With the correct ceiling the tax is $2,480 + 12% × $31,927.65 = $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning reached the reference itself — wages of $88,928 as the only taxable income, a ~$32,300 standard deduction, and 'tax on $56,628 ≈ $6,311' — and then it submitted $7,466. The submitted value is disconnected from every step it wrote; the tax rules it applied were the correct 2026 ones." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Applied the exact 2026 parameters — $32,200 MFJ standard deduction, 10% to $24,800, 12% above — and computed $2,480 + $3,831 = $6,311, then submitted $6,712 as an 'estimated bracket' adjustment. The abandoned computation was the correct answer of $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Carried 2025 parameters into 2026: a $30,000 MFJ standard deduction and a $23,850 ceiling on the 10% bracket, instead of $32,200 and $24,800. That overstates taxable income by $2,200 and yields $6,594.36 rather than $6,311.32; its exclusion of veterans benefits and rejection of itemizing were correct." -us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Estimated a $32,600 standard deduction and a $24,000 10%-bracket ceiling, derived $6,279, and then submitted $8,079 as an unexplained adjustment. The 2026 values are $32,200 and $24,800, which give taxable income of $56,727.65 and tax of $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Used a $30,600 MFJ standard deduction and a $24,400 ceiling on the 10% bracket; the 2026 figures are $32,200 and $24,800. Taxable income is $56,727.65, not $58,328, so the tax is $2,480 + 12% × $31,927.65 = $6,311.32, not $6,511." -us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Assumed the TCJA individual provisions lapse for 2026 and rebuilt the return on a $15,400 standard deduction, $10,000 of personal exemptions, and a 15% second bracket; 2026 law provides a $32,200 MFJ standard deduction, no personal exemptions, and a 12% second bracket. It compounded this by subtracting the $14,717 ESI premium from wages, which does not reduce employment income, so its $48,811 taxable income replaces the correct $56,727.65 and its $6,190.90 replaces $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Built taxable income on a TCJA-sunset structure — a $16,600 standard deduction plus $10,600 of personal exemptions and a 15% bracket — none of which applies in 2026, where the MFJ standard deduction is $32,200, exemptions are zero, and the second bracket is 12%. It also removed the $14,717 ESI premium from the $88,928 of wages, which PolicyEngine does not do, landing at $47,011 taxable instead of $56,727.65." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset rules: a $15,850 standard deduction, $10,100 of personal exemptions, and a 15% rate above $23,200. For 2026 the MFJ standard deduction is $32,200 with no personal exemptions and a 12% second bracket, and the $14,717 ESI premium it netted out of wages does not reduce employment income, so taxable income is $56,727.65 and the tax is $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Submitted $13,915 with no derivation beyond naming wages and the standard deduction. The correct computation is $88,927.65 − $32,200 = $56,727.65 taxed at 10%/12% for $6,311.32; its value is consistent with taxing the full $88,928 of wages with no standard deduction applied at single-filer rather than joint rates." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Assumed TCJA expiration and combined a standard deduction with personal exemptions totaling about $27,500 against pre-TCJA brackets; 2026 grants a $32,200 MFJ standard deduction, no personal exemptions, and a 12% bracket above $24,800. It also cut wages by the $14,717 ESI premium, which does not reduce employment income, giving $46,711 of taxable income instead of $56,727.65." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a $16,000 sunset-style standard deduction and a 10% bracket ending at $16,800 with 15% above — a single-filer-sized bracket applied to a joint return — while 2026 gives $32,200 and 10% to $24,800 then 12%. Its $14,717 ESI subtraction from wages is not a PolicyEngine adjustment either, so taxable income is $56,727.65 and tax is $6,311.32, not $7,891.65." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Asserted that the spouse's wages minus the standard deduction leave zero tax, treating the $32,200 MFJ standard deduction as absorbing all $88,927.65 of wages. It leaves $56,727.65 of taxable income, which produces $2,480 + 12% × $31,927.65 = $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Reduced wages by the $14,717 ESI premium to a $74,211 AGI, then subtracted a standard deduction plus personal exemptions that 2026 law does not provide. The correct base is the full $88,927.65 less the $32,200 MFJ standard deduction — $56,727.65 taxable — taxed at $6,311.32 rather than $6,069.15." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Computed a $47,511 taxable income from a $16,100 standard deduction plus $10,600 of personal exemptions on a $74,211 AGI, then applied a 15% rate above $23,200. For 2026 the deduction is $32,200 with no exemptions, the ESI premium does not reduce the $88,927.65 of wages, and the second rate is 12%, giving $56,727.65 taxable and $6,311.32 of tax." -us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the 2025 MFJ standard deduction of $30,000 and the 2025 bracket break at $23,850 rather than the 2026 values of $32,200 and $24,800. Its structure was otherwise right — wages only, veterans benefits excluded — so the $2,200 of extra taxable income and lower bracket break turn $6,311.32 into $6,594.36." -us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied a $30,000 MFJ standard deduction and a $24,000 ceiling on the 10% bracket; the 2026 amounts are $32,200 and $24,800. Correcting both gives taxable income of $56,727.65 and tax of $2,480 + 12% × $31,927.65 = $6,311.32 instead of $6,591.36." -us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Answered a different computation entirely: it asserted both filers are 65 or older when they are 41 and 38, applied a 5.3% rate to $25,050, added Form 8814 child income for a household with no children, and then wiped the result out with a Kentucky family size tax credit and elderly credit in a West Virginia case. The federal calculation is $88,927.65 less the $32,200 standard deduction taxed at 10%/12% for $6,311.32, with no nonrefundable credit available." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Gave no bracket or deduction figures, only the claim that taxable income after the standard deduction produces $10,458. That value corresponds to running the MFJ schedule over roughly $91,000 of taxable income — the wage total with essentially no standard deduction — while the correct base is $88,927.65 − $32,200 = $56,727.65, taxed at $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Claimed nonrefundable credits and the standard deduction fully offset the liability, but the household has no dependents and qualifies for no nonrefundable credit — no CTC, no CDCC, no education credit. The $32,200 MFJ standard deduction leaves $56,727.65 of taxable income, producing $6,311.32 with nothing to offset it." -us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used 'scheduled 2026 post-TCJA parameters' — a pre-TCJA standard deduction plus two personal exemptions — to reach $61,828 of taxable income under pre-TCJA rates. 2026 provides a $32,200 MFJ standard deduction, no personal exemptions, and a 12% bracket above $24,800, giving $56,727.65 taxable and $6,311.32 of tax rather than $8,062." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Concluded there is no taxable income after deductions and exemptions, but the $32,200 MFJ standard deduction offsets only part of the spouse's $88,927.65 in wages, leaving $56,727.65 taxable. Its two supporting points are correct and irrelevant to the base — veterans benefits are excluded and child support paid is nondeductible — so the tax is $6,311.32, not zero." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed the 2026 TCJA sunset and used a $17,018 standard deduction with $10,854 of personal exemptions and a 15% rate above about $24,991. In force for 2026 are a $32,200 MFJ standard deduction, no personal exemptions, and 12% above $24,800, so taxable income is $56,727.65 and tax is $6,311.32 rather than $7,909." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Treated current law as restoring personal exemptions and the pre-TCJA standard deduction for 2026, producing $61,928 of taxable income under 10/15 percent brackets. The 2026 return uses a $32,200 MFJ standard deduction with no exemptions and a 12% bracket, giving $56,727.65 taxable and $6,311.32 of tax." -us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Netted the $14,717 employer-sponsored insurance premium out of the spouse's wages to reach a $74,211 AGI; that input does not reduce PolicyEngine's employment income, which stays at $88,927.65. Combined with a $30,840 standard deduction instead of $32,200, it computed $43,371 of taxable income against the correct $56,727.65, giving $4,714 rather than $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Applied pre-TCJA 2026 rules with a ~$16,700 MFJ standard deduction, two ~$5,300 personal exemptions, and a 15% rate above about $24,550. For 2026 the standard deduction is $32,200, personal exemptions are zero, and the rate above $24,800 is 12%, so taxable income is $56,727.65 and the tax is $6,311.32, not $8,017." -us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so nothing was submitted against the $6,311.32 reference. The miss is a missing output rather than a substantive tax error." -us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction plus one personal exemption reduce the spouse's wages to near zero; 2026 provides no personal exemption and a $32,200 MFJ standard deduction against $88,927.65 of wages. That leaves $56,727.65 of taxable income and $2,480 + 12% × $31,927.65 = $6,311.32." -us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Included the spouse's $18,480 of veterans benefits in AGI even though VA benefits are excluded from gross income for both filers, and then invented two children to claim a $4,000 child tax credit for a household with no dependents. Its own arithmetic ended at $4,616.80 while it submitted −$17,600; the correct figure is $6,311.32 on $56,727.65 of taxable income with no nonrefundable credits." -us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Zeroed taxable income against the standard deduction and then invoked CTC/ODC that no listed dependent supports. The $32,200 MFJ standard deduction leaves $56,727.65 of the $88,927.65 in wages taxable, and with no dependents there is no nonrefundable credit, so the tax is $6,311.32." +us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Arizona has no refundable individual income tax credits at all. That ignores the refundable Credit for Increased Excise Taxes, which pays $25 per tax-unit member when a single filer's AGI is at or below $12,500. This filer's AGI is $6,736, so the credit is $25." +us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Considered only the Family Income Tax Credit and the Property Tax Credit and concluded that no other refundable credit applies. It left out the refundable Increased Excise Tax Credit, which needs no dependents, rent or property tax: $25 for a one-person unit with AGI of $6,736, below the $12,500 single limit." +us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Named the increased excise tax credit but lumped it with credits that require rent or property-tax payments, and so concluded that no conditions were met. The excise credit's only test is income (AGI at or below $12,500 single). The very low income the model itself identified ($6,736 AGI) qualifies the filer for $25." +us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"Claimed the increased excise tax credit yields nothing for a single filer with no dependents and no tax liability. The credit is refundable and pays $25 for the taxpayer alone whenever AGI is at or below $12,500. At $6,736 AGI the filer gets $25 despite having zero liability." +us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Correctly identified the refundable excise tax credit and a qualifying AGI of about $6,800, but then counted $25 per exemption: personal, age 65+ and disability, giving $75. The credit is $25 per person in the tax unit (taxpayer, spouse, dependents), and age and disability exemptions do not add persons, so a one-person unit gets $25." +us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Looked only for the family income tax credit and the property tax credit and concluded that nothing refundable applies. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member for single filers with AGI at or below $12,500, which this $6,736-AGI filer receives." +us,scenario_013,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Checked only the property tax credit (no rent or property tax listed) and the family credit (no dependents). It skipped the refundable Increased Excise Tax Credit, which pays $25 to a single filer at $6,736 AGI with no expense or dependent requirement." +us,scenario_013,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Treated zero state tax liability and no dependents as ruling out refundable credits. The Increased Excise Tax Credit is refundable regardless of liability and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single limit." +us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Assumed Arizona's refundable credits require dependents. The Increased Excise Tax Credit pays $25 per tax-unit member, including a lone taxpayer, when AGI is at or below $12,500, so this $6,736-AGI filer gets $25." +us,scenario_013,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Required dependents, donations or property-tax/rent facts for any refundable credit. It missed the income-tested Increased Excise Tax Credit, $25 for a one-person unit with AGI at or below $12,500, which applies at this filer's $6,736 AGI." +us,scenario_013,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Framed Arizona's refundable credits around an EITC that requires earned income, but Arizona has no EITC. The applicable refundable credit is the Increased Excise Tax Credit, which needs no earnings and pays $25 to a single filer with AGI of $6,736." +us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies without naming any credit test. It overlooked the refundable Increased Excise Tax Credit, $25 per tax-unit member when a single filer's AGI is at or below $12,500, which this $6,736-AGI filer meets." +us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Stated that the household has no state refundable credits, which misses Arizona's refundable Increased Excise Tax Credit. That credit pays $25 for a one-person tax unit with AGI of $6,736, below the $12,500 single threshold." +us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Asserted that no state refundable credit applies and never evaluated Arizona's Increased Excise Tax Credit. That credit is refundable and pays $25 per tax-unit member at or below $12,500 AGI for single filers, so this $6,736-AGI filer gets $25." +us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies without evaluating the Increased Excise Tax Credit. That credit has only an income test, AGI at or below $12,500 single, which this filer's $6,736 AGI meets, yielding $25." +us,scenario_013,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Claimed Arizona has no major refundable credits this head qualifies for. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member for single filers with AGI at or below $12,500, which applies at $6,736 AGI." +us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Asserted that household income exceeds the limits for Arizona's refundable credits. The Increased Excise Tax Credit tests AGI, which is $6,736 because Social Security is excluded, against a $12,500 single limit, so the filer qualifies for $25." +us,scenario_013,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Stated that Arizona has no refundable credits available to this household. It never applied the refundable Increased Excise Tax Credit, which pays $25 to a single filer with AGI of $6,736, below the $12,500 limit." +us,scenario_013,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Concluded that no Arizona refundable credit applies and overlooked the Increased Excise Tax Credit. That credit gives $25 per tax-unit member to single filers with AGI at or below $12,500, and this filer's AGI is $6,736." +us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no triggering facts for an Arizona refundable credit. Low AGI alone ($6,736, below the $12,500 single limit) triggers the refundable Increased Excise Tax Credit of $25 per tax-unit member." +us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed Arizona refundable credits hinge on qualifying dependents or earned income. The Increased Excise Tax Credit requires neither and pays $25 to a lone taxpayer with AGI at or below $12,500; here AGI is $6,736." +us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Concluded that the facts support no Arizona refundable credit. The reported pension and investment income give AGI of $6,736, which qualifies the single filer for the refundable $25 Increased Excise Tax Credit (limit $12,500)." +us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Required property tax, rent, dependents or other credit-generating facts. It missed that the refundable Increased Excise Tax Credit is purely income-tested and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single limit." +us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Found no refundable credit eligibility for the one-person elderly household. The refundable Increased Excise Tax Credit pays $25 per tax-unit member when a single filer's AGI is at or below $12,500, and this filer's AGI is $6,736." +us,scenario_013,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Concluded that the facts support no refundable Arizona credit and never evaluated the Increased Excise Tax Credit. That credit's only test is AGI at or below $12,500 for single filers, which $6,736 meets, yielding $25." +us,scenario_013,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Required earnings, dependents or expenses to establish a refundable Arizona credit. The Increased Excise Tax Credit needs none of these and pays $25 to a single taxpayer with AGI of $6,736, below the $12,500 limit." +us,scenario_013,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Looked only at the Working Poor Tax Credit, which is a nonrefundable credit for charitable contributions. It missed the refundable Increased Excise Tax Credit, $25 per tax-unit member at AGI at or below $12,500 single, which this $6,736-AGI filer receives." +us,scenario_013,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Limited its analysis to the property-tax credit, which requires property taxes or rent. It skipped the refundable Increased Excise Tax Credit, which needs no expenses and pays $25 to a single filer with AGI of $6,736, below the $12,500 limit." +us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded that household composition and income level rule out state refundable credits. A one-person unit with $6,736 AGI is exactly who the refundable Increased Excise Tax Credit targets: $25 at or below the $12,500 single limit." +us,scenario_013,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated zero taxable income and no dependents as ruling out refundable credits. The Increased Excise Tax Credit is refundable even with no liability and pays $25 for the taxpayer alone at AGI of $6,736, below the $12,500 single threshold." +us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Searched for a refundable Arizona EITC or property tax credit, but Arizona has no EITC and the filer lists no property tax. It missed the refundable Increased Excise Tax Credit, which pays $25 to a single filer with AGI of $6,736 and needs no earned income." +us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Tested the excise tax credit against total household income of $30,544, which includes all $23,736 of Social Security, and a $25,000 single limit. The credit tests AGI, which excludes the non-taxable Social Security and is $6,736, against a $12,500 single limit, so the filer qualifies for $25." +us,scenario_013,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Treated the absence of dependents as ruling out Arizona refundable credits. The Increased Excise Tax Credit pays $25 for the taxpayer alone when AGI is at or below $12,500, and this filer's AGI is $6,736." +us,scenario_013,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Evaluated only the property tax refund credit, testing $30,544 of gross income (including non-taxable Social Security) against its low senior limit. It never considered the refundable Increased Excise Tax Credit, which tests AGI of $6,736 against $12,500 and pays $25." +us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Required qualifying children or earned income and cited a nonexistent Arizona EITC. The refundable Increased Excise Tax Credit requires neither and pays $25 to a single filer with AGI of $6,736, below the $12,500 limit." +us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Reasoned that with no Arizona tax liability there is nothing for a credit to offset. A refundable credit does not depend on liability, and the Increased Excise Tax Credit pays $25 to this single filer with $6,736 AGI, below the $12,500 limit." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the correct 2026 parameters ($32,200 deduction, 10% to $24,800), but it subtracted the $14,717 ESI premium from the stated gross wages as a pre-tax exclusion. That gave AGI of $74,211 instead of $88,927.65 and tax of $4,545. It then submitted $5,647, which matches neither its own figure nor the correct $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,000 MFJ standard deduction instead of the 2026 $32,200. It then taxed all $60,928 at a flat 12%, skipping the 10% bracket on the first $24,800. Finally it replaced that result with an unexplained $6,819. The correct $56,727.65 of taxable income yields $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the $32,200 deduction and $56,728 taxable income right, but it put the top of the 10% bracket at $24,150 instead of $24,800 and computed $6,324. It then dropped that figure for an unexplained $5,235 that no bracket schedule produces from $56,728 of taxable income." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own calculation (AGI $88,928, deduction about $32,300, 10%/12% brackets) came to $6,311, which matches the correct answer. It then submitted $7,466, a number that contradicts its own stated working." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It did the calculation exactly right: $32,200 deduction, $56,728 taxable income, and $2,480 + $3,831 = $6,311. It then submitted $6,712, citing unspecified 'estimated bracket thresholds', which overrides its own correct figure." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It carried forward pre-OBBBA 2025 parameters: a $30,000 MFJ standard deduction instead of $32,200, and a 10% bracket ending at $23,850 instead of $24,800. That overstated taxable income at $58,928 and produced $6,594 instead of $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It guessed a $32,600 deduction and a 10% bracket ending at $24,000, and computed about $6,279. It then submitted an unexplained $8,079, which is inconsistent with any 10%/12% calculation on roughly $56,000 of taxable income. The correct $32,200 deduction and $24,800 bracket yield $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 MFJ standard deduction instead of $32,200 and a 10% bracket ending at $24,400 instead of $24,800. That gave taxable income of $58,328 and tax of $6,511 instead of $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, so it used a $15,400 deduction, $10,000 of personal exemptions, and a 15% bracket. OBBBA made the TCJA structure permanent: the 2026 deduction is $32,200, there are no exemptions, and the second bracket is 12%. It also wrongly subtracted the $14,717 ESI premium from the stated gross wages." +us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a $16,600 deduction, $10,600 of personal exemptions, and a 15% bracket. The actual 2026 law has a $32,200 deduction, no exemptions, and a 12% bracket. It also subtracted the $14,717 ESI premium from the $88,928 of gross wages, so it reached $5,829.15 instead of $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the 2025 OBBBA MFJ standard deduction of $31,500 instead of the 2026 inflation-adjusted $32,200. That overstated taxable income by $700 ($57,428 vs $56,727.65) and tax by about $84 ($6,395.36 vs $6,311.32)." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed tax under a TCJA sunset: a $15,850 deduction, $10,100 of personal exemptions, and a 15% second bracket. It also cut AGI to $74,211 by subtracting the $14,717 ESI premium. The correct figures are a $32,200 deduction, no exemptions, the 10%/12% brackets, and AGI of $88,927.65." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no working, and its $13,915 is more than double the correct $2,480 + 12% x $31,927.65 = $6,311.32. The number also exceeds the $10,175 tax on the full $88,928 with no deduction at all, so it applied rates far above the 2026 10%/12% brackets to the spouse's wages." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying about $27,500 of deduction plus personal exemptions and pre-TCJA brackets. It also subtracted the $14,717 ESI premium from gross wages. The correct inputs are AGI of $88,927.65, the permanent $32,200 deduction, and the 10%/12% brackets." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed a TCJA sunset with a $16,000 deduction and a 15% rate on everything above $16,800. The 2026 schedule is actually a $32,200 deduction with 10% to $24,800 and 12% above it. It also subtracted the $14,717 ESI premium from the stated gross wages." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out the spouse's $88,928 of wages. The 2026 MFJ standard deduction is only $32,200, which leaves $56,727.65 of taxable income and $6,311.32 of tax." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium to get AGI of $74,211 and then applied TCJA-sunset personal exemptions and a pre-TCJA deduction. The 2026 rules give no exemptions, a $32,200 deduction, and 10%/12% brackets on AGI of $88,927.65, for $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters ($16,100 deduction, $10,600 of exemptions, 10%/15% brackets) to an AGI of $74,211 reduced by the ESI premium. Its own inputs actually give $5,966.65, not the $6,031.65 it submitted. The correct figures are a $32,200 deduction, no exemptions, 10%/12% brackets, and AGI of $88,927.65." +us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 figures, a $30,000 deduction and a 10% bracket ending at $23,850, instead of the 2026 $32,200 deduction and $24,800 bracket. That gave $58,928 of taxable income and $6,594.36 instead of $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 MFJ standard deduction instead of the 2026 $32,200 and a 10% bracket ending at $24,000 instead of $24,800. That overstated taxable income at $58,928 and produced $6,591.36." +us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It hallucinated a different household: both spouses aged 65 or older (they are 41 and 38), Form 8814 child income, and Kentucky senior and family-size credits. It computed a 5.3% state-style tax instead of the federal 10%/12% tax on $56,727.65 of taxable income, and it zeroed the result with state credits." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It showed no working. Its $10,458 is roughly the tax on the full $88,928 of wages with no deduction ($10,175 at 2026 brackets), so it effectively skipped the $32,200 MFJ standard deduction. The correct $56,727.65 of taxable income yields $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that nonrefundable credits and the standard deduction fully offset the tax. The household has no dependents or qualifying expenses, so no nonrefundable credits apply, and the $32,200 deduction leaves $56,727.65 taxable, which produces $6,311.32 of tax." +us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used 'scheduled 2026 post-TCJA parameters' with two personal exemptions and a pre-TCJA deduction, which gave $61,828 of taxable income. OBBBA made the $32,200 deduction, zero exemptions, and 10%/12% brackets permanent, so taxable income is $56,727.65 and tax is $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction offsets all of the $88,928 in wages. The 2026 MFJ standard deduction is $32,200, which leaves $56,727.65 of taxable income and $6,311.32 of tax." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a $17,018 deduction, $10,854 of personal exemptions, and a 15% bracket. Under OBBBA the 2026 deduction is $32,200, there are no exemptions, and the second bracket is 12%, giving $6,311.32 instead of $7,909." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset restores personal exemptions and the pre-TCJA deduction, which gave $61,928 of taxable income taxed at 10%/15%. OBBBA made the $32,200 deduction and 10%/12% brackets permanent, so taxable income is $56,727.65 and tax is $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It stated that the TCJA deduction increase and exemption suspension expire in 2026 and used a $16,600 deduction, $10,600 of exemptions, and a 15% bracket. OBBBA extended the TCJA permanently, so the correct figures are a $32,200 deduction, no exemptions, and a 12% bracket, for $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium from the stated gross wages to get AGI of $74,211 instead of $88,927.65. It also used a $30,840 deduction and a $24,520 top for the 10% bracket instead of $32,200 and $24,800, which gave $4,714 instead of $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 rules: a deduction of about $16,700, two exemptions of about $5,300 each, and a 15% bracket. The correct 2026 figures are a $32,200 deduction with no exemptions and 10%/12% brackets, so taxable income is $56,727.65 and tax is $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction plus a personal exemption reduce the spouse's $88,928 of wages to near zero. There are no personal exemptions in 2026, and the $32,200 deduction leaves $56,727.65 taxable, which produces $6,311.32." +us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It added the spouse's $18,480 of veterans benefits to AGI even though they are federally excluded. It used a $32,600 deduction and invented two children to claim a $4,000 CTC, although no dependents are listed. It then submitted -17,600, a negative value that its own $4,616.80 does not support." +us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction reduces taxable income to zero and that CTC/ODC credits remove what remains. The $32,200 deduction leaves $56,727.65 taxable, and with no dependents listed there is no CTC or ODC, so tax is $6,311.32." us,scenario_014,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract. us,scenario_014,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded refundable CTC despite the household having no qualifying children and awarded EITC despite approximately $183,584 of joint income, which exceeds the childless EITC phase-out limit. Those two credits both equal zero, so their asserted $3,067 total has no eligible credit pathway." us,scenario_014,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the head's lack of earned income as though Medicaid were tested on the head's income alone, omitting the spouse's wages and other amounts included in household tax-unit MAGI. The resulting MAGI is 4.11 times FPL, above the West Virginia adult limit, and the head has no pregnancy, disability, dependent-child, SSI, or other categorical eligibility pathway." -us,scenario_014,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the federal employee share correctly on the full $88,928 of gross wages ($5,513.54 Social Security + $1,289.56 Medicare) and then added $6.21 for an invented ""0.2% state tax"" for West Virginia. West Virginia imposes no mandatory employee-side payroll tax — its unemployment insurance is entirely employer-financed — so nothing is added to the federal total of $6,802.96. It also cited a stale $168,600 Social Security wage base instead of the 2026 base, which changes nothing only because these wages fall below both." -us,scenario_014,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the spouse's $14,717 employer-sponsored insurance premium from wages to build a $74,211 FICA base. PolicyEngine applies the 6.2% Social Security and 1.45% Medicare employee taxes to gross employment income of $88,928 with no premium offset, and the listed premium is the total policy cost — predominantly employer-paid and already outside wages — not an employee cafeteria-plan salary reduction that can be netted out. That single exclusion is the entire $1,125.82 shortfall ($14,717 × 7.65% = $1,125.85)." -us,scenario_014,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It wrote the base as ""$88,928 - $14,717 = $74,211,"" treating the employer-sponsored insurance premium as a pre-tax exclusion from FICA wages. The payroll tax base is the full $88,928 of gross wages: only an employee's own §125 salary reduction leaves the wage base, and the $14,717 shown is the total ESI premium, the employer portion of which is already excluded and cannot be removed a second time. Its rates and its no-Additional-Medicare-Tax conclusion were right; only the base was wrong, costing $1,125.82." -us,scenario_014,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It applied the correct 7.65% combined employee rate but to ""taxable FICA wages"" of $74,211 after deducting the $14,717 ESI premium. The premium never reduces the Social Security or Medicare wage base here; 7.65% belongs on the full $88,928 of gross wages, yielding $6,802.96 rather than $5,677.14." -us,scenario_014,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It stated as a rule that ""employee payroll tax (FICA) is 7.65% applied to wages after subtracting pre-tax health insurance premiums"" and netted $14,717 off $88,928. That rule does not apply to this input: the ESI premium is the total policy cost, not an employee pre-tax deferral, and PolicyEngine computes employee Social Security and Medicare on gross employment income, so the base stays $88,928 and the tax is $6,802.96." -us,scenario_014,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It defined the spouse's ""taxable wages"" as $74,211, gross wages minus the pre-tax ESI premium, then split that base into $4,601.08 of Social Security and $1,076.06 of Medicare. Both components must be computed on the full $88,928 — $5,513.51 and $1,289.45 — because the ESI premium input feeds coverage and medical-expense logic, not the payroll tax base." -us,scenario_014,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced gross wages by the $14,717 ESI premium to reach ""FICA taxable wages"" of $74,211 before applying 6.2% and 1.45%. The employer-paid premium is already outside wages and the employee has no stated §125 salary reduction, so the base is the full $88,928 and the employee payroll tax is $6,802.96, $1,125.82 more than it reported." -us,scenario_014,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It carved $14,717 of ""pre-tax health insurance premiums"" out of the spouse's $88,928 before applying the 6.2% and 1.45% employee rates. Neither PolicyEngine's employee_social_security_tax nor employee_medicare_tax reduces the wage base for that input, so the correct components are $5,513.51 and $1,289.45 on the full wages, totaling $6,802.96." -us,scenario_014,payroll_tax,glm-5.3,llm_error,other,False,"Its derivation is exactly right — 6.2% and 1.45% of the full $88,928 with veterans benefits excluded as non-wages, no Additional Medicare Tax below the $250,000 MFJ threshold, and no West Virginia employee payroll tax — and its explanation even ends ""value = 6802.99."" The submitted numeric field was 6.00, so the graded answer contradicts the model's own stated total by three orders of magnitude. The break is in the submission contract, not in the payroll tax reasoning." -us,scenario_014,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its explanation named the right scope — 6.2% plus 1.45% on the spouse's $88,928, no Additional Medicare Tax, no WV employee payroll tax — but the submitted $13,591.63 is essentially the combined 15.3% employer-plus-employee rate on those wages ($13,605.98) rather than the 7.65% employee share. The question explicitly excludes employer payroll taxes, so the answer is $6,802.96, half what it reported." -us,scenario_014,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It recited the 6.2% and 1.45% employee rates but never performed the multiplication: 7.65% of the spouse's $88,928 is $6,802.96, while its round $12,450 corresponds to 7.65% of roughly a $163,000 base. That base is reachable only by pulling the head's $76,176 of veterans benefits into wages — VA compensation is not FICA wages — or by folding in the employer share the question excludes." -us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It got everything else right — head has no wages, no Additional Medicare Tax, no West Virginia employee payroll tax — but computed 7.65% on $74,211 after subtracting the $14,717 ESI premium. The employee payroll tax base is the full $88,928 of gross wages, so the total is $6,802.96 rather than $5,677." -us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value and no explanation were returned, so the model took no position on the wage base or the rates. The required computation is 6.2% plus 1.45% of the spouse's $88,928 in gross wages, with no ESI-premium offset and no state component, giving $6,802.96." -us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"Its explanation performs the reference computation exactly — $5,513.54 Social Security plus $1,289.46 Medicare on the spouse's full $88,928, head has no wages, total ""≈ $6,803"" — but the numeric field carries 6581. The $222 gap corresponds to no step anywhere in its reasoning; the modeling was right and the submitted value was not the one it derived." -us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"It computed the employee share correctly on the full, unreduced $88,928 — $5,513.54 plus $1,289.46, no Additional Medicare Tax — which sums to $6,802.99, then submitted 8075.02. That value sits $1,272.06 above its own stated components and is not produced by any rate or base it wrote down, so the answer failed at submission rather than at the payroll tax rules." -us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It built the base correctly ($88,928 wages, veterans benefits excluded, less the $4,000 two-person exemption = $84,928) but applied the 2025 West Virginia MFJ schedule (2.22/2.96/3.33/4.44/4.82%) instead of the 2026 schedule, which is 5% lower (2.11/2.81/3.16/4.22/4.58%) and produces $3,092.19. It then abandoned its own $3,256 arithmetic and reported $3,374 as a 'rounding' adjustment, moving the answer further from the correct figure." -us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The base was exact ($84,928 after the $4,000 exemption), but the model applied a 4% reduction to the 2025 rates (2.13/2.84/3.20/4.26/4.63%) rather than the enacted 5% reduction that sets the 2026 West Virginia MFJ rates at 2.11/2.81/3.16/4.22/4.58%. Using the correct 2026 rates on $84,927.65 gives $3,092.19, not $3,125.17." -us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 employer-sponsored insurance premium from the stated gross wages, but PolicyEngine's AGI equals the full $88,927.65 of wages; it also skipped West Virginia's $4,000 personal exemption and replaced the graduated bracket schedule with an assumed ~4.5% flat effective rate. The correct path is $88,927.65 AGI − $4,000 exemption = $84,927.65 run through the 2026 MFJ brackets (2.11/2.81/3.16/4.22/4.58%) = $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the right base and computed $3,255 using the 2025 rate schedule (2.22/2.96/3.33/4.44/4.82%), missing the 5% trigger reduction that sets 2026 rates at 2.11/2.81/3.16/4.22/4.58%. It then shaved its own total down to $2,877 for a family tax credit it had just stated does not apply at this income." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It mixed schedule vintages (2.2/2.96/3.33% from 2025 with a 4.07% fourth bracket and a 4.625% top rate from no West Virginia schedule) to get $3,130, then applied an unspecified additional 9% haircut to land on $2,849. The 2026 MFJ rates are 2.11/2.81/3.16/4.22/4.58% and on $84,927.65 yield $3,092.19 with no further adjustment." -us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It identified the correct taxable income of $84,928 but never ran the bracket arithmetic, asserting ~$1,655 'after nonrefundable credits' — an effective rate of 1.95%, below even West Virginia's lowest 2026 bracket rate of 2.11%. The 2026 MFJ schedule (2.11/2.81/3.16/4.22/4.58%) on $84,927.65 gives $3,092.19, and no West Virginia nonrefundable credit reduces it." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2023 post-HB-2526 rates (2.36/3.15/3.54/4.72%) and collapsed the top two brackets into a single 4.72% rate above $40,000, missing the 2025 4% and 2026 5% trigger cuts that set the 2026 schedule at 2.11/2.81/3.16/4.22/4.58%. It then discarded its own $3,360 computation and reported $1,919 by applying an additional ~43% cut and credits it had explicitly ruled out." -us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The base was correct at $84,928, but the model estimated the liability at 'approximately $3,200' using a top rate 'around 4.82%' — the 2025 rate. The 2026 West Virginia MFJ rates are 5% lower (2.11/2.81/3.16/4.22/4.58%) and produce exactly $3,092.19 rather than an eyeballed figure." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a $4,000 West Virginia MFJ standard deduction and $6,000-per-person exemptions; West Virginia allows no standard deduction and a $2,000 personal exemption per person, so the base is $88,927.65 − $4,000 = $84,927.65, not $72,928. It compounded this with the stale 2023 rate schedule (2.36/3.15/3.54/4.50/5.16%) instead of the 2026 rates of 2.11/2.81/3.16/4.22/4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from $74,211 by subtracting the $14,717 employer insurance premium from gross wages and then subtracted a fabricated $15,400 West Virginia standard deduction; West Virginia grants no standard deduction and PolicyEngine's AGI is the full $88,927.65. Its bracket schedule (2.50/3.13/5.12% with a three-band structure) matches no West Virginia year — the 2026 MFJ schedule is 2.11/2.81/3.16/4.22/4.58% across five bands, giving $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It stacked two base errors: subtracting the $14,717 pre-tax insurance premium to reach $74,211 (PolicyEngine's AGI is the full $88,927.65) and then subtracting a nonexistent $4,000 West Virginia standard deduction on top of the $4,000 personal exemption. Its rate schedule (2.18/2.90/3.26/4.35/4.72%) is a 2% cut off the 2025 rates rather than the enacted 5% cut to 2.11/2.81/3.16/4.22/4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $14,717 employer-sponsored insurance premium to reach AGI of $74,211, but PolicyEngine's West Virginia AGI is the full $88,927.65 of gross wages, making taxable income $84,927.65 rather than $70,211. It also used the 2023 schedule topping out at 5.12% instead of the 2026 rates of 2.11/2.81/3.16/4.22/4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation and reported $4,125, which corresponds to a flat ~4.6% rate applied to the whole wage base with no personal exemption. The correct computation subtracts the $4,000 two-person exemption from $88,927.65 and applies the graduated 2026 West Virginia MFJ rates (2.11/2.81/3.16/4.22/4.58%), yielding $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used $74,211 as AGI, having netted the $14,717 employer insurance premium out of the $88,928 of gross wages that PolicyEngine taxes in full, so its taxable income of $70,211 understates the correct $84,927.65 by $14,717. Applying the 2026 West Virginia MFJ rates (2.11/2.81/3.16/4.22/4.58%) to the correct base gives $3,092.19, not $2,706." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 employer-sponsored insurance premium from gross wages to get AGI of $74,211; PolicyEngine's West Virginia AGI equals the stated $88,927.65 of wages, so taxable income after the $4,000 exemption is $84,927.65. Its $2,708 also reflects the pre-2025 rate schedule rather than the 2026 rates of 2.11/2.81/3.16/4.22/4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that 'standard deductions and exemptions' zero out the liability, but West Virginia grants no standard deduction and its personal exemption is only $2,000 per person, leaving $84,927.65 of taxable income on $88,927.65 of wage AGI. That base carries $3,092.19 of tax under the 2026 MFJ brackets." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It set AGI at $74,211 by removing the $14,717 employer insurance premium from the gross wages PolicyEngine taxes in full, so its $70,211 taxable income is $14,717 short of the correct $84,927.65. Its rates also predate the 2026 5% trigger cut that sets the schedule at 2.11/2.81/3.16/4.22/4.58%, together producing $2,706.30 instead of $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It carried the same $74,211 AGI produced by subtracting the $14,717 pre-tax insurance premium from gross wages, understating taxable income by that amount relative to the correct $84,927.65. Applying the 2026 West Virginia MFJ schedule (2.11/2.81/3.16/4.22/4.58%) to the full wage base gives $3,092.19, not $2,977.17." -us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its base and exemption were exact ($84,928), but it applied the 2023 post-HB-2526 schedule (2.36/3.15/3.54/4.72/5.12%), missing both the 4% cut effective 2025 and the 5% cut effective 2026 that set the rates at 2.11/2.81/3.16/4.22/4.58%. Those 2026 rates on $84,927.65 give $3,092.19 rather than $3,459.81." -us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It assumed an $8,000 West Virginia standard deduction that does not exist and then applied the 2025 top marginal rate of 4.82% as a flat rate to the entire $76,928 base, ignoring the graduated bracket structure. The correct computation runs $84,927.65 through the five 2026 MFJ brackets at 2.11/2.81/3.16/4.22/4.58%, giving $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It computed a Kentucky liability — $25,050 of taxable income at 5.3% offset by Kentucky's family size tax credit — for a household whose stated state is West Virginia, and West Virginia has no family size tax credit. The West Virginia computation is $88,927.65 AGI − $4,000 personal exemption = $84,927.65 taxed at the 2026 MFJ rates for $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It supplied no bracket arithmetic and reported $2,331, an effective rate of 2.74% on the correct $84,927.65 base — below West Virginia's third 2026 bracket rate of 3.16%. Running that base through the 2026 MFJ schedule (2.11% to $10k, 2.81% to $25k, 3.16% to $40k, 4.22% to $60k, 4.58% above) produces $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and nonrefundable items fully eliminate the liability, but West Virginia allows no standard deduction, the personal exemption is $2,000 per person, and no nonrefundable credit applies to a joint filer with $88,928 of wages. Taxable income is $84,927.65 and the 2026 MFJ brackets yield $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its base, exemption, and bracket widths were exact, but it used the 2025 West Virginia MFJ rates (2.22/2.96/3.33/4.44/4.82%) and missed the 5% reduction effective for 2026, which sets them at 2.11/2.81/3.16/4.22/4.58%. That single rate-vintage error is the entire $162.84 gap between its $3,255.03 and the correct $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It computed the correct $84,928 base but applied a rate schedule about 1% below the 2025 rates rather than the 5% reduction that sets the 2026 West Virginia MFJ rates at 2.11/2.81/3.16/4.22/4.58%. Those rates on $84,927.65 give $3,092.19, $125 less than its $3,217.53." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It derived $84,928 of taxable income correctly and then applied the 2025 rate schedule (2.22/2.96/3.33/4.44/4.82%), which yields exactly its $3,255.03. The 2026 schedule is 5% lower — 2.11/2.81/3.16/4.22/4.58% — and produces $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $3,255 is the exact output of the 2025 West Virginia MFJ rate schedule (2.22/2.96/3.33/4.44/4.82%) applied to the correct $84,928 base, so it used the prior year's rates for a 2026 filing. The 2026 rates of 2.11/2.81/3.16/4.22/4.58% give $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It stated the 2025 bracket rates (2.22/2.96/3.33/4.44/4.82%) explicitly and applied them to the correct $84,928 base, missing the 5% trigger reduction effective January 1, 2026. The 2026 West Virginia MFJ rates of 2.11/2.81/3.16/4.22/4.58% yield $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It reported zero on the premise of a West Virginia standard deduction, which the state does not provide; the only subtraction is the $2,000-per-person exemption, leaving $84,927.65 of taxable income on $88,927.65 of wage AGI. That base produces $3,092.19 under the 2026 MFJ brackets." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly treated the 2024 schedule (2.36/3.15/3.54/4.72/5.12%) as 'the baseline for 2026,' ignoring the 4% cut effective 2025 and the 5% cut effective 2026 that set the rates at 2.11/2.81/3.16/4.22/4.58%. Its base was right, so the stale rates alone account for the $368 overstatement against $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted roughly $20,600 of 'deductions/exemptions' from the $88,928 base, but West Virginia's only subtraction here is the $4,000 personal exemption ($2,000 × 2), leaving $84,927.65. It also cited the 2.36–5.12% schedule rather than the 2026 rates of 2.11/2.81/3.16/4.22/4.58%, which produce $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It started from an ESI-reduced AGI of $74,211 and then subtracted the federal standard deduction of $30,840, but West Virginia neither excludes employer premiums from the stated gross wages nor allows the federal standard deduction — only the $4,000 personal exemption applies, giving $84,927.65. It compounded this with a 2/3/4/5% rate schedule that matches no West Virginia year; the 2026 MFJ rates are 2.11/2.81/3.16/4.22/4.58%." -us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached the correct ~$84,900 base after exemptions but applied 'approximate 2026 brackets' that sit near the 2023 post-HB-2526 schedule, overstating the tax by $330. West Virginia's 2026 MFJ rates are 2.11/2.81/3.16/4.22/4.58%, yielding $3,092.19 on $84,927.65." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It made three separate errors: subtracting the $14,717 employer premium from the gross wages PolicyEngine taxes in full, adding a nonexistent $8,000 West Virginia standard deduction, and treating West Virginia as a 4.5% flat tax. West Virginia remains graduated in 2026 at 2.11/2.81/3.16/4.22/4.58%, and on $84,927.65 the tax is $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its base of $84,928 was exact, but it used the 2023 bracket schedule — $2,183.50 of cumulative tax through $60,000 plus 5.12% above — which predates the 4% cut effective 2025 and the 5% cut effective 2026. The 2026 rates of 2.11/2.81/3.16/4.22/4.58% give $3,092.19 rather than $3,459.81." -us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed West Virginia's personal exemption reduces taxable wages to zero, but the exemption is $2,000 per person — $4,000 for this two-person tax unit — against $88,927.65 of wage AGI, leaving $84,927.65 taxable. The 2026 MFJ brackets on that base produce $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It got the base and exemption exactly right but applied the 2025 top rate of 4.82% (plus lower-bracket amounts of 222/438/552/854 that match no single West Virginia schedule), missing the 5% reduction that sets the 2026 rates at 2.11/2.81/3.16/4.22/4.58%. Those rates on $84,927.65 give $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the $14,717 employer premium out of the gross wages PolicyEngine taxes in full, then applied a regressive invented schedule whose rates fall from 4.80% to 2.88% as income rises — West Virginia's brackets are progressive at 2.11/2.81/3.16/4.22/4.58% for 2026 — and finally trimmed 4% for a 'family tax credit' that West Virginia does not provide. The correct base is $84,927.65 and the tax is $3,092.19." -us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the federal standard deduction to the West Virginia base and assumed nonrefundable state credits erase the remainder; West Virginia allows no standard deduction and no nonrefundable credit reaches a joint filer with $88,928 of wages. After the $4,000 personal exemption the base is $84,927.65 and the 2026 MFJ brackets produce $3,092.19." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own arithmetic — AGI $42,909, the $16,100 standard deduction, $26,809 of taxable income, $1,240 at 10% plus $1,729 at 12% — lands on the reference $2,969, and it then submitted $3,103, a figure no step of its derivation produces. The failure is the reporting step, not the tax computation." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $960 of workers' compensation to gross income, which IRC §104(a)(1) excludes, deducted the full $4,510 long-term loss instead of the $3,000 §1211(b) limit, and applied the 2024 single standard deduction of $14,600 rather than 2026's $16,100. It then reported $3,509 after computing $3,094.28, so the submitted number matches neither the statute nor its own arithmetic." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It reached the reference taxable income of $26,809 using the correct $16,100 standard deduction, then set the 10% bracket ceiling at $12,150 instead of 2026's $12,400 to get $2,974, and submitted $3,268 — a value none of its steps yields. The decisive error is abandoning its own derivation at the reporting step." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It declared the $45,000 gross wage figure already net of the traditional 401(k) deferral and so left the $1,041.93 elective deferral in income, producing AGI of $43,951 instead of $42,909.38. Its $16,100 standard deduction and $12,400/12% bracket were right, so the entire $125 overstatement is 12% of the un-deducted deferral." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It deducted the entire $4,510 long-term capital loss while asserting it fell under the cap — §1211(b) limits the deduction to $3,000 — skipped the $1,041.93 traditional 401(k) deferral, and used a $16,700 standard deduction instead of $16,100. Its own text lands near $2,841 and it submitted $4,172, a number produced by no step of its derivation." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After correctly reaching AGI of $42,909 it chose 2025 parameters as a proxy — a $15,000 standard deduction and an $11,925 top of the 10% bracket — instead of 2026's $16,100 and $12,400. That parameter-year substitution accounts for the whole $142 overstatement." -us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a $15,750 standard deduction rather than $16,100, subtracted a $135 saver's credit even though AGI of $42,909 sits well above the single-filer AGI ceiling for that credit, and then submitted $1,750 in place of its own $2,876. The submitted figure comes from an unexplained downward adjustment rather than any rule." -us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized on the strength of $13,800 of mortgage interest it invented by applying a 6% rate to the $230,000 balance; only a balance is given and the prompt directs that unlisted numeric inputs are 0, so there is no interest deduction. Stripping that fabricated interest drops itemized deductions below the $16,100 standard deduction, leaving taxable income of $26,809.38 and tax of $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It excluded the $2,080 of employee health insurance premiums from wages as a §125 pre-tax exclusion, cutting AGI to $40,829 when the taxable base is $45,000 reduced only by the $1,041.93 401(k) deferral. It compounded that with a $15,000 standard deduction and an $11,600 top of the 10% bracket instead of 2026's $16,100 and $12,400." -us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It priced 2026 under a TCJA sunset: an $8,250 standard deduction, a $5,250 personal exemption, and a 15% second bracket. The TCJA rate schedule and the repeal of personal exemptions govern 2026, where a single filer gets a $16,100 standard deduction and a 12% bracket above $12,400." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It explicitly assumed TCJA expiration and applied an $8,300 standard deduction plus a $5,050 personal exemption with a 15% second bracket. 2026 grants a $16,100 single standard deduction, no personal exemption, and 12% above $12,400, so its $13,350 of deductions understated the true $16,100 and its 15% rate overtaxed the $14,409 above the 10% bracket." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It stated AGI of 'approximately $44,000', which requires ignoring both the $1,041.93 401(k) deferral and the $3,000 capital loss deduction, and gave no deduction or bracket arithmetic at all. The correct chain — $42,909.38 AGI less the $16,100 standard deduction, taxed at 10% to $12,400 and 12% above — yields $2,969.13; $1,541 corresponds to no deduction or credit available to this filer." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It removed the $2,080 employee premium from wages to reach AGI of $40,829 and then applied a sunsetted 2026 regime combining a standard deduction with a personal exemption and a 15% rate. 2026 taxes the full $45,000 less the $1,041.93 deferral, allows a $16,100 standard deduction with no personal exemption, and taxes the excess over $12,400 at 12%." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its AGI of $42,909 matches the reference, but it applied a $15,000 standard deduction and an $11,600 top of the 10% bracket — 2024/2025 figures — instead of 2026's $16,100 and $12,400. Using the correct parameters on its own AGI gives $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It named no parameters, and $3,046 is what the correct $42,909.38 AGI produces under a standard deduction near $15,450 — a 2025-era figure — rather than 2026's $16,100. With $16,100 and the $12,400 top of the 10% bracket, the liability is $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the $2,080 of health insurance premiums as an exclusion from wages, cutting AGI to $40,829 when the taxable base is $45,000 less only the $1,041.93 401(k) deferral, and paired that with a $15,700 standard deduction instead of $16,100. Both adjustments push taxable income $2,080 below the reference $26,809.38 before its bracket arithmetic even begins." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It reached the correct $42,909 AGI and then subtracted a personal exemption alongside the standard deduction 'under 2026 law'. Personal exemptions are repealed for 2026, the single standard deduction is $16,100, and the second bracket is 12% to $50,400 — the phantom exemption plus the implied 15% rate produces the $820 overstatement." -us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its AGI of $42,909 is correct, but $3,111 is exactly the 2025 schedule applied to it — a $15,000 standard deduction and an $11,925 top of the 10% bracket. The 2026 parameters of $16,100 and $12,400 give $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It derived the reference AGI of $42,909 and then estimated a $15,400 standard deduction and a $12,247 ceiling on the 10% bracket instead of the 2026 statutory $16,100 and $12,400. The $700 of extra taxable income at 12% plus the shifted bracket boundary is the entire $87 overstatement." -us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It had the AGI and the $12,400/12% bracket exactly right but used a $15,750 standard deduction instead of 2026's $16,100. The $350 shortfall taxed at 12% is precisely the $42 by which it overshot $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It asserted without computation that the standard deduction plus unspecified nonrefundable credits reduce regular tax to zero. AGI of $42,909.38 exceeds the $16,100 standard deduction by $26,809.38, and a single filer with no dependents, no care expenses, and no education expenses has no nonrefundable credit — the saver's credit is out of reach at this AGI — leaving $2,969.13 of tax." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It claimed a deduction for the $6,500 of child support paid; child support is not deductible by the payor under any provision. Its $5,199 exceeds even the tax on the full undeducted AGI of $42,909.38 ($4,901), so the number follows from no deduction, bracket, or credit path in the code." -us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"Its explanation states the reference chain exactly — $42,909 AGI, a $16,100 standard deduction, $26,809 taxable, 10% on $12,400 plus 12% on $14,409 — which totals $2,969.13, and it submitted $3,211.08. The tax reasoning is correct; the reported number does not come from it." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It netted $1,351 of retirement contributions from wages, folding in the $184 Roth 401(k) and $76 Roth IRA amounts, which are after-tax and reduce nothing, and applied the full $4,510 capital loss rather than the $3,000 §1211(b) limit. Only the $1,041.93 traditional 401(k) deferral and the $48.69 traditional IRA deduction reduce income, and the resulting $26,809.38 of taxable income taxes to $2,969.13 under the $12,400/12% schedule." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It reached the correct $42,909 AGI and then applied 'pre-TCJA rules' for 2026 — an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. 2026 provides a $16,100 single standard deduction, no personal exemption, and 12% above $12,400." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its AGI of $42,909 and its rejection of itemizing are both right, but it used a $15,750 standard deduction rather than the 2026 single amount of $16,100. That $350 difference at the 12% marginal rate is the $42 by which $3,011 exceeds $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It declared that 2026 reverts to pre-2018 rules and inflated pre-TCJA amounts to a combined $14,131 standard deduction plus personal exemption with a 15% second bracket. The TCJA schedule governs 2026: a $16,100 single standard deduction, no personal exemption, and 12% above $12,400." -us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized $22,018 built on $13,800 of mortgage interest imputed from the $230,000 balance and an imputed Indiana income tax as SALT; the prompt supplies only a balance and directs that unlisted numeric inputs are 0, so neither deduction exists. With the $16,100 standard deduction, taxable income is $26,809.38 and tax is $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so no tax computation was submitted against the $2,969.13 reference. The failure is the missing output itself, not a substantive tax error." -us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It concluded that the standard deduction and 'any applicable credits' leave approximately zero tax without ever computing taxable income. Subtracting the $16,100 standard deduction from AGI of $42,909.38 leaves $26,809.38 taxable, and no nonrefundable credit is available to a single filer with no dependents, so the liability is $2,969.13." -us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It added the $960 of workers' compensation to AGI, which IRC §104(a)(1) excludes from gross income, and used a $15,750 standard deduction with a $12,150 top of the 10% bracket instead of $16,100 and $12,400, computing $3,137.16 — then submitted $2,104, a number its own derivation never produces." -us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It applied the married-filing-jointly schedule (10% up to $23,850) to a single filer with no spouse, never subtracted the $1,041.93 401(k) deferral or the $48.69 IRA deduction, and took a $500 credit for other dependents in a household with no dependents. Its final $4,072.21 is two of its own intermediate figures added together rather than a tax liability." +us,scenario_014,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got the right Social Security figure on the spouse's $88,928. It then miscomputed Medicare as $1,289.56 instead of $1,289.46. It also added a made-up $6.21 '0.2% state tax' for West Virginia, right after saying WV has no mandatory employee state payroll tax. That extra state tax and the Medicare slip raised the total from $6,802.99 to $6,809.31." +us,scenario_014,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the spouse's $14,717 in employer-sponsored insurance premiums from her wages as if they were pre-tax cafeteria-plan deductions. It then applied FICA to $74,211, but the facts give no salary-reduction arrangement, so the full $88,928 is the base. The correct 7.65% of $88,928 is $6,802.99, not $5,677." +us,scenario_014,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model subtracted $14,717 of premiums from the $88,928 FICA base as pre-tax ESI, which the facts do not support. Social Security and Medicare apply to the spouse's full gross wages. That makes the total $5,513.51 + $1,289.45 rather than $5,677.14." +us,scenario_014,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model shrank the spouse's FICA wages to $74,211 by subtracting the $14,717 in employer-sponsored insurance premiums, a Section 125 exclusion the household facts never mention. Applying 7.65% to the full $88,928 gives about $6,803." +us,scenario_014,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $14,717 in ESI premiums as pre-tax salary reductions and applied 7.65% to $74,211. The listed $88,928 is the spouse's gross wage, and all of it is subject to employee Social Security and Medicare tax, which gives $6,802.96." +us,scenario_014,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model computed Social Security ($4,601.08) and Medicare ($1,076.06) on $74,211 after subtracting ESI premiums, but no pre-tax cafeteria plan is stated. The correct base is the spouse's full $88,928, which gives $5,513.51 and $1,289.45." +us,scenario_014,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted $14,717 in premiums from the spouse's $88,928 gross wages to reach $74,211 of 'FICA taxable wages'. The facts do not say those premiums are withheld pre-tax, so FICA applies to the full $88,928, for a total of $6,802.96." +us,scenario_014,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model excluded $14,717 of 'pre-tax health insurance premiums' from the FICA base without any stated salary-reduction plan. That understated the employee Social Security and Medicare base by $14,717, and the correct total is $6,802.96 rather than $5,677.14." +us,scenario_014,payroll_tax,glm-5.3,llm_error,other,False,"The model's explanation computes the right components: 6.2% × $88,928 = $5,513.54 plus 1.45% × $88,928 = $1,289.46, ending 'value = 6802.99'. The numeric value field it submitted was 6, however, so the recorded answer does not match its own stated total." +us,scenario_014,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said it would apply 6.2% + 1.45% to the spouse's $88,928, which gives $6,802.99, but it submitted $13,591.63. That is roughly double, consistent with applying the combined 15.3% employee-plus-employer FICA rate (about $13,606) instead of the 7.65% employee share only." +us,scenario_014,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied FICA to 'the spouses' wages' as a joint base of about $162,700 ($12,450 / 0.0765). Only the spouse has wages ($88,928); the head's income is non-wage veterans benefits. Taxing only the spouse's $88,928 at 7.65% gives about $6,803." +us,scenario_014,payroll_tax,gpt-6-luna,llm_error,other,False,"The model correctly applied 7.65% to the spouse's $88,928 wages with no Additional Medicare Tax and no WV employee payroll tax. It then reported $6,808.99 instead of the $6,802.99 that product actually gives, a $6 arithmetic error." +us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $14,717 ESI premiums from the spouse's wages as pre-tax and applied 7.65% to $74,211. The household facts establish no salary-reduction plan, so the full $88,928 is the FICA base and the total is $6,802.96." +us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there is no answer to grade. The correct derivation is 7.65% of the spouse's $88,928 wages, about $6,803." +us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model's explanation computes $5,513.54 Social Security plus $1,289.46 Medicare, about $6,803, on the spouse's $88,928 wages. It then submitted 'value = 6581', a number that contradicts its own sum." +us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct components, $5,513.54 Social Security and $1,289.46 Medicare on $88,928 with no Additional Medicare Tax. It then submitted $8,075.02 instead of their $6,802.99 sum, adding about $1,272 that its own explanation never supports." +us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the right $84,928 base but applied the 2025 rates (2.22–4.82%) instead of the 2026 schedule (2.11/2.81/3.16/4.22/4.58%). It then raised its own ~$3,256 result to $3,374 with no basis." +us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rates (2.13/2.84/3.20/4.26/4.63%) reflect about a 4% cut from 2025. The 2026 cut is 5% (2.11/2.81/3.16/4.22/4.58%), so it overstated tax by about $33." +us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium from gross wages and skipped the $4,000 personal exemptions. It then applied a guessed ~4.5% average rate instead of the graduated 2026 MFJ brackets on $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got $3,255 using the 2025 rates instead of the 5%-lower 2026 schedule. It then subtracted about $378 of nonexistent nonrefundable credits to reach $2,877." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The $84,928 base is right, but it mixed 2025 and made-up rates (4.07%, 4.625%) to get about $3,130. It then applied an unexplained extra cut down to $2,849 instead of the 2026 rates of 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The $84,928 base is right, but $1,655 is only about a 1.95% average rate. The 2026 brackets produce $3,092.19, so it greatly understated the rates and assumed nonrefundable credits that do not exist." +us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22/2.96/3.33/4.44/4.82%) to the correct $84,928 and missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2023 21.25%-cut rates with a collapsed four-bracket structure to get $3,360. It then cut the answer to $1,919 for credits it admitted do not apply, instead of using the 2026 five-bracket schedule." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The $84,928 base is right, but it guessed an approximate $3,200 from vague 2.36–5.12% rates. It never applied the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,It assumed the 2023–24 rates (2.36/3.15/3.54/4.72/5.12%) still apply in 2026. That misses both the 2025 cut and the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%. +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a $4,000 WV standard deduction and $6,000 exemptions per person. WV has no standard deduction and allows $2,000 per exemption, so taxable income is $84,927.65. It also used a made-up rate schedule." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium, getting AGI of $74,211 instead of $88,928, and invented a $15,400 WV standard deduction instead of $4,000 in exemptions. It also used wrong rates (2.50/3.13/5.12%)." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $74,211 by subtracting the ESI premium and added a $4,000 standard deduction that WV does not have. Its rates (2.18–4.72%) reflect only about a 2% cut instead of the 2026 5% cut." +us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $8,000 when WV allows only the $4,000 of exemptions, leaving $80,928 instead of $84,927.65. Its rates (1.69–4.29%) are far below the 2026 schedule of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium, getting a $70,211 base instead of $84,927.65. It also applied the outdated 2.36–5.12% rates instead of the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. $4,125 matches a flat ~4.6% rate on gross wages with no $4,000 exemption and no graduated brackets, but the 2026 brackets on $84,927.65 yield $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $70,211 base after wrongly subtracting the ESI premium from gross wages. It also applied the old 2.36–5.12% rates instead of the 5%-reduced 2026 rates on $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It subtracted the $14,717 ESI premium, getting a $70,211 base instead of $84,927.65. It then used the outdated 2.36–5.12% schedule instead of the 2026 rates of 2.11–4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It assumed standard deductions and exemptions wipe out the tax. WV has no standard deduction and allows only $4,000 of exemptions, leaving $84,927.65 taxable and $3,092.19 of tax." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It reduced the base to $70,211 by subtracting the ESI premium from gross wages. It also applied the 2023–24 rates (2.36–5.12%) instead of the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used a $70,211 base after subtracting the ESI premium, instead of $84,927.65. It also applied an outdated, overstated rate schedule instead of the 2026 5%-reduced brackets." +us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $84,928 base is right, but it applied the 2023–24 rates (2.36/3.15/3.54/4.72/5.12%). It missed the 2025 and 2026 cuts, which bring the rates to 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It invented an $8,000 WV standard deduction and taxed all of the remaining $76,928 at a flat 4.82% top rate. It should have used the graduated 2026 MFJ brackets on $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It computed Kentucky tax on an invented $25,050 of income with a family size credit, even though the household is in West Virginia. WV tax on $84,927.65 at 2026 rates is $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation. $2,331 is about a 2.7% average rate on $84,928, well below what the 2026 brackets (2.11–4.58%) produce on that base, which is $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and nonrefundable items fully eliminate WV tax. WV allows only $4,000 of exemptions and no standard deduction, and no nonrefundable credit applies at this income." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22/2.96/3.33/4.44/4.82%) to the correct $84,928 and missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rate schedule overstates the 2026 rates by about 4%, giving $3,217.53. The 5%-reduced schedule (2.11–4.58%) gives $3,092.19." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It carried the 2025 rates into 2026, getting $3,255.03. It missed the 5% cut to 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"$3,255 is the 2025 schedule applied to $84,928. It missed the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22–4.82%) instead of the 2026 schedule (2.11/2.81/3.16/4.22/4.58%) to the correct $84,928 base." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The $84,928 base is right, but its rate schedule reflects only about a 3% cut from 2025 instead of the 5% 2026 cut, so it overstated tax by about $100." +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,It used the 2025 joint rates of 2.22/2.96/3.33/4.44/4.82% instead of the 5%-lower 2026 rates of 2.11/2.81/3.16/4.22/4.58%. +us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The $84,928 base is right, but $2,952.20 matches about a 9–10% cut from the 2025 rates. The 2026 cut is 5% (2.11/2.81/3.16/4.22/4.58%), so it understated tax." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It assumed a standard deduction wipes out WV tax. WV has no standard deduction, only $2,000 per exemption, so $84,927.65 stays taxable." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,It assumed the 2024 rates (2.36–5.12%) remain the baseline for 2026. It missed the 2025 cut and the 2026 5% cut to 2.11/2.81/3.16/4.22/4.58%. +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted an invented ~$20,600 of deductions when WV allows only $4,000 of exemptions. It also used the outdated 2.36–5.12% rates." +us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,It carried rates of 2.27/3.02/3.40/4.53/4.92% into 2026. These are higher than both the 2025 schedule and the 2026 schedule (2.11/2.81/3.16/4.22/4.58%). +us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium to get AGI of $74,211, took off the $30,840 federal standard deduction, which WV does not allow, and used made-up 2/3/4/5% rates. The correct figures are $4,000 of exemptions and the 2026 WV rate schedule." +us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ~$84,900 base is right, but $3,422 matches the pre-2025 rate schedule. It missed the reductions to 2.11/2.81/3.16/4.22/4.58% in 2026." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium from gross wages and invented an $8,000 WV standard deduction. It then applied a nonexistent flat 4.5% rate instead of the graduated 2026 brackets on $84,927.65." +us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the 2024 rates, $2,183.50 plus 5.12% over $60k, to $84,928. It missed the cuts to a 4.58% top rate and to 2.11/2.81/3.16/4.22% below it." +us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the personal exemption reduces taxable wages to zero. WV exemptions total only $4,000 for two people, leaving $84,927.65 taxable." +us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The $84,928 base is right, but it used inconsistent bracket amounts and a 4.82% top rate from the 2025 schedule. The 2026 rates are 2.11/2.81/3.16/4.22/4.58%." +us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It subtracted the ESI premium, getting a $70,211 base, and applied an invented, inverted rate schedule that declines as income rises. It also subtracted a nonexistent 4% family tax credit instead of applying the 2026 progressive brackets." +us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the federal standard deduction and assumed nonrefundable credits eliminate the tax. WV allows neither; the only offset is the $4,000 of exemptions, leaving $3,092.19 of tax." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It derived the correct AGI of $42,909, taxable income of $26,809, and tax of $1,240 + $1,729 = $2,969. It also correctly ruled out the Saver's Credit. It then submitted $3,103 with no computation behind the $134 increase, overriding its own correct result." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used 2024 parameters (a $14,600 standard deduction and a $11,600 top for the 10% bracket) instead of the 2026 values of $16,100 and $12,400. It also included the $960 of nontaxable workers' compensation and deducted the full $4,510 capital loss instead of the $3,000 cap. It then submitted $3,509, which does not match its own $3,094 computation." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It correctly reached AGI of $42,909 and taxable income of $26,809. It then used a $12,150 top for the 10% bracket instead of the 2026 value of $12,400, which gave $2,974. It submitted $3,268 anyway, with no step supporting the extra $294." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said the $1,042 traditional 401(k) deferral was already excluded from wages, but then built AGI from the full $45,000. That gave AGI of $43,951 instead of $42,909 and taxable income of $27,851 instead of $26,809. The $3,094 result is exactly $125 (12% of $1,042) above the correct $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,510 capital loss, wrongly saying it was under the limit, instead of capping the deduction at $3,000. It never excluded the $1,042 traditional 401(k) deferral, and it used a $16,700 standard deduction instead of $16,100. It then submitted $4,172, which contradicts its own $2,841 computation." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"It correctly found taxable income of $26,809 and the $16,100 standard deduction. It reported only the 12% slice (12% × $14,409 = $1,729.08) and dropped the $1,240 of tax on the first $12,400 at 10%, so the correct $2,969.13 was never summed." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the correct AGI of $42,909, then chose 2025 parameters as a proxy: a $15,000 standard deduction and a $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400. That inflated taxable income to $27,909 and tax to $3,111." +us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It used a $15,750 standard deduction instead of the 2026 value of $16,100, which gave $3,011. It then subtracted a $135 Saver's Credit even though AGI of $42,909 is above the single Saver's Credit income limit. On top of that it cut the result to $1,750 with no identifiable credit or deduction behind the reduction." +us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized using $13,800 of mortgage interest invented as 6% of the loan balance, although no mortgage interest was given and unlisted amounts are 0. It also double-counted the $2,080 premium to reach $10,200 of medical expenses. Legitimate itemized deductions (medical above 7.5% of AGI plus Indiana tax) fall well short of the $16,100 standard deduction, so taxable income is $26,809, not $20,891." +us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It treated the $2,080 health insurance premium as a pre-tax wage reduction, but the premium is not a payroll exclusion here, so AGI stays at $42,909 rather than $40,829. It also used an outdated $15,000 standard deduction and $11,600 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400." +us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law for 2026: an $8,250 standard deduction, a $5,250 personal exemption, and a 15% second bracket. The TCJA structure is permanent, so 2026 uses a $16,100 standard deduction, no personal exemption, and 10%/12% brackets. That puts taxable income at $26,809, not $29,409." +us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health insurance premium as a pre-tax wage exclusion, which lowered AGI to $40,829 instead of $42,909. It also used a $15,750 standard deduction instead of $16,100. The resulting $25,079 of taxable income is below the correct $26,809." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and used an $8,300 standard deduction, a $5,050 personal exemption, and a 15% bracket. The 2026 schedule actually has a $16,100 standard deduction, no exemption, and 10%/12% brackets with the 10% bracket ending at $12,400. That inflated taxable income to $29,559 and tax to $3,853.85." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It started from AGI of about $44,000, which omits the $1,042 401(k) exclusion and $49 IRA deduction, and then subtracted nonrefundable credits even though none apply. With the $16,100 standard deduction, the correct taxable income of $26,809 produces $2,969.13. The $1,541 answer can only come from an unsupported credit or deduction." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied a pre-2018 standard deduction plus personal exemption with 15% brackets, instead of the 2026 $16,100 standard deduction and 10%/12% schedule. It also wrongly excluded the $2,080 health insurance premium from wages." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used a $15,000 standard deduction and a $11,600 top for the 10% bracket. The 2026 values are $16,100 and $12,400, which give taxable income of $26,809 and tax of $2,969.13 rather than $3,117." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its computation lists only wages, interest and the $3,000 capital-loss limit. It omitted the $1,042 traditional 401(k) exclusion and the $49 IRA deduction that bring AGI down to $42,909. It never showed the 2026 figures of a $16,100 standard deduction, $26,809 taxable income, and $2,969.13 tax." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $2,080 health insurance premium as a pre-tax wage exclusion, dropping AGI to $40,829 from the correct $42,909. It then used a $15,700 standard deduction and a 2025 bracket top of $11,925 instead of the 2026 values of $16,100 and $12,400." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then applied a standard deduction plus personal exemption ""under 2026 law"" as if the TCJA had sunset. The 2026 schedule has no personal exemption, a $16,100 standard deduction, and 10%/12% brackets, which give $2,969.13 rather than $3,788.85." +us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, but its $3,111 matches the 2025 parameters: a $15,000 standard deduction and a $11,925 top for the 10% bracket (taxable income of $27,909). The 2026 values are $16,100 and $12,400, which give $26,809 of taxable income." +us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used an estimated $15,400 standard deduction and a $12,247 top for the 10% bracket. The 2026 values are $16,100 and $12,400, so taxable income was overstated at $27,509." +us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It had the correct AGI and the correct 10% bracket top of $12,400, but used the 2025 standard deduction of $15,750 instead of the 2026 value of $16,100. That overstated taxable income by $350 ($27,159) and tax by $42." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It assumed nonrefundable credits wipe out the tax, but none are available. There are no dependents, the Saver's Credit is gone because AGI of $42,909 exceeds the single limit, and the elderly/disabled credit's $5,000 base is fully eliminated by the reduction of half of AGI over $7,500. The $26,809 of taxable income therefore carries $2,969.13 of tax." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated child support paid as deductible, which it is not. Its $5,199 is higher than the tax on the entire $42,909 AGI with no standard deduction at all (about $4,901). So its number implies it never subtracted the $16,100 standard deduction that brings taxable income down to $26,809." +us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"It correctly found taxable income of $26,809, taxed $12,400 at 10% ($1,240) and $14,409 at 12% ($1,729.08), and applied no credits. It then added those two pieces wrong, reporting $3,211.08 instead of $2,969.08." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It deducted all $1,351 of retirement contributions, including the $184 Roth 401(k) and $76 Roth IRA contributions, which are after-tax. It also subtracted the full $4,510 capital loss instead of the $3,000 cap. Its $3,200 does not follow from the correct $26,809 taxable income under the 2026 $16,100 standard deduction and 10%/12% brackets." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then applied pre-TCJA rules for 2026: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket. The 2026 law uses a $16,100 standard deduction with no exemption and 10%/12% brackets, which give $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, but its taxable income of $27,159 implies the 2025 standard deduction of $15,750 rather than the 2026 value of $16,100. That overstated taxable income by $350 and gave $3,011 instead of $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It had the correct AGI of $42,909, then used an estimated $15,400 standard deduction and a $12,250 top for the 10% bracket. The 2026 values are $16,100 and $12,400, so taxable income came out at $27,509 instead of $26,809." +us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset and inflated pre-2018 law, using about $14,131 of standard deduction plus exemption and a 15% bracket above $12,670. The 2026 schedule keeps the $16,100 standard deduction and 10%/12% brackets, so taxable income is $26,809 and tax is $2,969.13." +us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized with about $13,800 of mortgage interest invented from the $230,000 loan balance, although no mortgage interest was listed and unlisted amounts are 0. It also double-counted the $2,080 premium in its medical deduction. The genuine itemizable amounts fall short of the $16,100 standard deduction, so taxable income is $26,809, not $20,891." +us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $2,969.13 reference." +us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It assumed the standard deduction plus credits tied to the disability would zero out the tax. The elderly/disabled credit's $5,000 base is fully eliminated by the reduction of half of AGI over $7,500, and no other nonrefundable credit applies, so $26,809 of taxable income carries $2,969.13 of tax." +us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $960 of workers' compensation to AGI even though that income is excluded, and it omitted the $49 IRA deduction. It also used a $15,750 standard deduction and a $12,150 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. It then submitted $2,104, which contradicts its own $3,137.16 computation." +us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It computed tax on MFJ brackets for a single filer with no spouse, and subtracted a $500 credit for other dependents and an invented child-related credit even though there are no dependents. It also never excluded the $1,042 401(k) deferral or the $49 IRA deduction. The correct single computation is $26,809 of taxable income and $2,969.13 of tax." us,scenario_015,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no parseable value for federal_refundable_credits, violating the required output contract." us,scenario_015,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $1,000 refundable Child Tax Credit despite the household having no qualifying child, and it awarded $1,735.80 of EITC even though this childless adult's approximately $43,450 income exceeds the applicable childless EITC range. Both components are $0, so their sum is $0." us,scenario_015,head_chip_eligible,gemini-3.5-flash-lite,llm_error,other,False,"The model applied the governing rule correctly — CHIP's categorical scope is limited to children under 19 (plus pregnant women at state option), so a 36-year-old head fails the age gate before any income screen — and its explanation ends ""Head is 36 years old, so CHIP eligibility is false. value = 0"". Its submitted numeric value for head_chip_eligible was nevertheless 1, contradicting its own explanation and violating the prompt's rule that a 1 be submitted only when the explanation says eligible. The defect is in the emitted binary, not in the derivation: the stated reasoning matches the reference's chip_category = 3 / is_chip_eligible = False." us,scenario_015,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented qualification through Indiana’s Aged, Blind, and Disabled or working-disabled buy-in pathways from the disability and work facts without applying their actual categorical and financial requirements. The engine assigns category NONE: the head receives no SSI, qualifies through no disability-related pathway, and has MAGI at 2.69 times FPL, so the asserted buy-in eligibility does not exist." us,scenario_015,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated Indiana Medicaid expansion and the disability flag as sufficient for eligibility without identifying or testing a qualifying category. At age 36 with MAGI at 2.69 times FPL and no qualifying disability-related pathway, the head is assigned Medicaid category NONE." -us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model stated the rule as ""65+ OR disabled,"" turning PolicyEngine's Medicare eligibility test — a comparison of age against the 65 age threshold — into a disjunction that does not exist. Under-65 Medicare entitlement requires 24 months of SSDI entitlement, ESRD, or ALS (42 U.S.C. §426(b)), none of which follows from the `is disabled` demographic flag; this head is 36, works 28 hours a week for $45,000 in wages, and receives no Social Security disability benefits, so the age test is the only test and it fails." -us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model asserted that ""under the modeled Medicare eligibility rule, disability qualifies an adult under 65 for Medicare,"" substituting the household's `is disabled` status for the statutory disability pathway, which is entitlement to Social Security disability insurance for 24 months (or ESRD/ALS). The head has $0 of Social Security disability income — the $960 workers' compensation is not SSDI and confers no Medicare entitlement — so at age 36 the age-65 threshold governs and eligibility is False." -us,scenario_015,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model invoked ""PolicyEngine's disability-based Medicare eligibility treatment"" solely because the head is listed as disabled; Medicare eligibility here is determined by age against the 65 threshold, and the disability route to Medicare requires 24 months of SSDI entitlement rather than a disability indicator. With the head aged 36, employed at $45,000 in wages, and receiving no disability insurance benefits, the correct evaluation returns False." -us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $2,080 of health insurance premiums as a pre-tax cafeteria-plan deduction from FICA wages. The facts only report premium spending, so Social Security and Medicare taxes apply to the full $45,000 wage amount, not $42,920." -us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $2,080 health premium from FICA wages without any fact establishing that it was paid through a pre-tax employer plan. The correct payroll-tax base is the full $45,000, producing $2,790 of Social Security tax and $652.50 of Medicare tax." -us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the listed health insurance premiums as pre-tax deductions from Social Security and Medicare wages. No pre-tax payroll treatment is specified, so both FICA components use the full $45,000 wage base." -us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model derived the correct components—$2,790 of Social Security tax and $652.50 of Medicare tax—but added them incorrectly. Their sum is $3,442.50, not $3,445.50." -us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stated Social Security amount of $2,790 and rounded Medicare amount of $653 already sum to $3,443, not $3,419. Using the exact Medicare calculation of $652.50 yields total payroll tax of $3,442.50." -us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than completing the FICA calculation. -us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly calculated $3,442.50 of employee FICA and correctly stated that Indiana adds no mandatory employee state payroll tax, then replaced that total with $3,495 based on nonexistent county-tax adjustments. County income taxes do not belong in this payroll-tax output." -us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model improperly deducted the $1,042 traditional 401(k) contribution from the FICA wage base; traditional 401(k) deferrals remain subject to Social Security and Medicare taxes. Its submitted $3,425.67 also contradicts its own stated component total of $3,362.79, while applying both rates to the full $45,000 yields $3,442.50." -us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Derived the answer exactly right in its reasoning — federal AGI $42,909, less the $1,000 exemption, at the 2026 flat rate of 2.95% = $1,236 — and then discarded that result, submitting $1,234 as a hand adjustment 'to reflect minor rounding in exemption computation.' Indiana's exemption is a flat $1,000 with no proration or rounding step, so the figure it had already computed was the answer." -us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Applied 3.15%, the rate Indiana levied in 2023, instead of the 2026 flat 2.95%, and built its base from gross income rather than AGI — including the $960 of workers' compensation, which IRC 104(a)(1) excludes from gross income, deducting the full $4,510 capital loss instead of the $3,000 annual limit, and never subtracting the $1,000 exemption. It then shaved its own $1,334 result to $1,255 for 'disability exemptions and nonrefundable credits' that Indiana does not provide to a single, non-elderly, childless wage earner." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Reached the correct federal AGI of $42,909 but added a $500 disabled exemption Indiana does not grant — the additional $1,000 exemption requires age 65 or blindness, and IC 6-3-2-9's disability deduction requires disability retirement income this wage earner has none of — and applied 3.05%, Indiana's 2024 rate, rather than 2026's 2.95%. It then submitted $1,352, a number inconsistent with both figures ($1,263 and $1,304) its own reasoning produced." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Failed to remove the $1,042 traditional 401(k) deferral from wages, leaving AGI at $43,951 instead of $42,909.38, then abandoned its own correct 2.95% computation of $1,267 with an unsupported jump to $1,392 'reflecting flat rate near 3.05%.' Both the pre-tax deferral exclusion and the 2.95% 2026 rate were stated correctly in its reasoning before it overrode them." -us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Applied roughly 2.85% instead of Indiana's 2026 statutory 2.95%, and built AGI of $42,441 by deducting the entire $4,510 net long-term capital loss rather than the $3,000 annual capital-loss limitation while omitting the $1,042 traditional 401(k) deferral. The correct base is $41,909.38 at 2.95%, giving $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Got federal AGI ($42,909), the $1,000 exemption, and the 2.95% 2026 rate all correct, then stripped a further $3,080 from the base with two subtractions Indiana does not allow: a $1,000 disabled exemption (Indiana's second $1,000 exemption is for age 65+ or blindness) and a $2,080 deduction for 'other health insurance premiums,' which appears nowhere in Indiana's deduction schedule for a wage earner. Taxing $38,829 instead of $41,909.38 is the entire $90.87 shortfall." -us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Claimed a $1,000 'developmentally disabled' exemption for the filer — Indiana's additional exemptions run to age 65+, blindness, and dependent children, not a disabled adult filer — and used the 2024 rate of 3.05% instead of 2026's 2.95%, producing $1,248. It then inflated that to $1,350 'adjusting for county/local considerations,' which this output's definition explicitly excludes from state income tax." -us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from $40,829 rather than the correct federal AGI of $42,909.38 — exactly $2,080 low, the amount of health insurance premiums, which are already reflected in the stated wage figure and are not a separate above-the-line or Indiana subtraction — and applied 3.05%, Indiana's 2024 rate, instead of 2026's 2.95%." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Invented two Indiana subtractions that do not exist: a $3,000 deduction for the $6,500 of child support paid (child support is nondeductible for both federal and Indiana purposes) and a $1,500 disabled exemption (Indiana's disability provision, IC 6-3-2-9, requires disability retirement income this wage earner does not have). It also used 3.0% rather than the 2026 rate of 2.95%, taxing $37,409 instead of $41,909.38." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Applied 2.79%, a rate Indiana has never levied, to an unstated base; $1,256 is 2.79% of about $45,020, i.e. gross wages with none of the $1,042 401(k) deferral, $49 IRA deduction, $3,000 capital-loss offset, $2,000 interest, or the $1,000 exemption applied. The correct computation is $41,909.38 at the 2026 flat rate of 2.95% = $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Used the correct $1,000 exemption and the correct 2.95% 2026 rate but started from $40,829 instead of $42,909.38, understating federal AGI by exactly the $2,080 of health insurance premiums, which are already excluded from the stated employer-plan wages and generate no additional Indiana deduction. That single $2,080 base error accounts for the whole $61.33 shortfall." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Every step was correct — $42,909 federal AGI, the $1,000 Indiana exemption, $41,909 of Indiana taxable income — except the rate: it used 3.05%, Indiana's 2024 rate, rather than the 2.95% that the HEA 1001-2022 schedule sets for 2026. That 0.10-point error is the entire $41.89 overstatement." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Applied 3.05% (Indiana's 2024 rate) directly to adjusted gross income without subtracting the $1,000 exemption, and then folded county local income tax into the figure, which this output's definition excludes and routes to local_income_tax instead. Indiana's 2026 rate of 2.95% on $41,909.38 gives $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Understated federal AGI by the $2,080 of health insurance premiums, using $40,829 instead of $42,909.38 — those premiums are already reflected in the stated wages and are not an Indiana subtraction — and applied 3.0% instead of the 2026 flat rate of 2.95%." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Computed the base exactly right ($42,909 federal AGI less the $1,000 exemption = $41,909) and then applied a rounded 3.0% rate; Indiana's 2026 flat rate is 2.95% under the HEA 1001-2022 schedule, and that 0.05-point difference is the entire $20.94 overstatement." -us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Identified the correct Indiana base — federal AGI $42,909 less the $1,000 exemption = $41,909 — but taxed it at 3.0% instead of Indiana's 2026 statutory 2.95%. Applying 2.95% to that same base yields $1,236.33, so the rate is the sole error." -us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Omitted both the $1,042 traditional 401(k) deferral and the $49 IRA deduction while deducting the full $4,510 capital loss instead of the $3,000 annual limitation, landing on $42,490 of AGI rather than $42,909.38, and then applied 3.0% instead of the 2026 rate of 2.95%." -us,scenario_015,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Built the base exactly right — federal AGI $42,909 with the capital loss correctly capped at $3,000, less the $1,000 individual exemption — but applied 2.90%; Indiana's 2026 flat rate is 2.95%, and that 0.05-point understatement is the entire $21 gap." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Applied a federal-style standard deduction to the Indiana base: $1,018 corresponds to roughly $34,500 of taxable income at 2.95%, about $7,400 below the correct $41,909.38. Indiana grants no standard deduction — the only reduction from federal AGI of $42,909.38 is the $1,000 exemption — so the tax is $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that unspecified deductions, contributions, and credits reduce Indiana liability to zero. Indiana has no standard deduction and no nonrefundable credit available to this filer — the unified tax credit for the elderly requires age 65+, the renter's deduction requires rent, and the state EIC keys off a federal EITC of $0 for a childless filer at $42,909 of AGI — so the full $41,909.38 × 2.95% = $1,236.33 stands." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Derived the base perfectly — $42,909 federal AGI less the one $1,000 state exemption = $41,909 — then applied 3.025%, a rate that appears nowhere in Indiana's schedule; HEA 1001-2022 sets 2.95% for 2026. That 0.075-point invention is the entire $31 overstatement." -us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Subtracted the $1,000 exemption twice, once as 'Indiana's $1,000 deduction' and again as the 'personal exemption,' taking the base to $40,909 instead of $41,909.38. Its 2.95% rate was correct, so the duplicated exemption is the whole $29.51 shortfall." -us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Gave no derivation and submitted a round $1,400, which is about 3.1% of unadjusted gross wages — it applied neither the $1,000 exemption nor any of the AGI adjustments ($1,042 401(k) deferral, $49 IRA deduction, $2,000 interest, $3,000 capital-loss limit), and used a rate above Indiana's 2026 flat 2.95%. The correct computation is $41,909.38 × 2.95% = $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Base was exact — federal AGI $42,909 less the $1,000 personal exemption = $41,909, with no Indiana nonrefundable credits — but it rounded the rate to a flat 3%. Indiana's 2026 statutory rate is 2.95%, which yields $1,236.33 rather than $1,257." -us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Correctly identified Indiana's structure — federal AGI $42,909 less the $1,000 exemption, with no standard or itemized deduction and no additions or subtractions — then applied 2.85%. Indiana's 2026 flat rate is 2.95%, and that 0.10-point understatement is the entire $42 gap." -us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing was submitted for a key the contract required in the outputs object. The substantive computation is $42,909.38 federal AGI less the $1,000 Indiana exemption at 2.95% = $1,236.33." -us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Named 3.05% as Indiana's rate — the 2024 rate, superseded by 2.95% for 2026 — and then submitted an approximate $1,230 that corresponds to no base it identified: 3.05% of the correct $41,909.38 is $1,278.24 and 2.95% is $1,236.33. It never carried out the arithmetic on a stated Indiana taxable income." -us,scenario_015,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Set the computation up exactly right — federal AGI $42,909 less the $1,000 personal exemption = $41,909, no applicable nonrefundable credits — but applied 2.97%. Indiana's 2026 flat rate is 2.95%, and that 0.02-point overstatement produces the entire $8.37 error." -us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Included the $960 of workers' compensation in AGI, which IRC 104(a)(1) excludes from gross income, deducted the full $4,510 capital loss instead of the $3,000 annual limit, and never subtracted Indiana's $1,000 exemption at all, taxing $42,359 instead of $41,909.38. It compounded that with a 3.05% rate rather than Indiana's 2026 flat 2.95%." -us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Applied Indiana's rate to federal taxable income after the $15,000 standard deduction ($30,860) — Indiana starts from federal adjusted gross income of $42,909.38 and allows no standard deduction, only the $1,000 exemption — and then rounded its own $941 result up to $950 with no basis." +us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model said Medicare eligibility applies to anyone who is 65+ or disabled. That is wrong: under-65 disability eligibility requires 24 months of SSDI entitlement, or ESRD or ALS status. The head is 36 and receives no Social Security disability benefits, so the disability pathway doesn't apply and the answer should be 0." +us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model decided that being disabled qualifies an adult under 65 for Medicare. That skips the requirement of 24 months of Social Security Disability Insurance entitlement. The 36-year-old head has no SSDI income and no ESRD or ALS status, so they are not Medicare eligible." +us,scenario_015,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model based eligibility only on the head being listed as disabled. Disability-based Medicare requires 24 months of SSDI benefit receipt, and a disability flag alone doesn't meet that. The head is 36 and gets no Social Security disability benefits, so the answer is not eligible (0)." +us,scenario_015,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the head's disabled status as enough for Medicare. Under-65 eligibility requires 24 months of SSDI entitlement, or ESRD or ALS status. The head is 36 and has none of these, so Medicare eligibility is 0." +us,scenario_015,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted the $2,080 health insurance premiums from wages as a pre-tax cafeteria-plan exclusion and applied 7.65% to $42,920. The FICA base here is the full $45,000 of gross wages, since the listed premiums are not a Section 125 salary reduction. The correct total is $2,790 + $652.50 = $3,442.50." +us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It lowered FICA wages to $42,920 by treating the $2,080 health insurance premiums as a pre-tax payroll deduction. Employee Social Security and Medicare tax apply to the full $45,000 of gross wages, which gives $3,442.50 rather than $3,283." +us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $2,080 employer-sponsored insurance premiums were pre-tax and cut FICA wages to $42,920. The 6.2% and 1.45% employee rates apply to the full $45,000 of gross wages, giving $2,790 + $652.50 = $3,442.50." +us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It defined FICA taxable wages as $45,000 minus the $2,080 health premiums, treating those premiums as a Section 125 exclusion. The correct base is the full $45,000 of gross wages, so the 7.65% employee FICA total is $3,442.50, not $3,283.38." +us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly computed Social Security of $2,790 and Medicare of $652.50 on $45,000 of wages, but reported their sum as $3,445.50 instead of $3,442.50. The error is purely in the final addition." +us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It found the correct components, Social Security of $2,790 and Medicare of $653 (rounded from $652.50), but summed them to $3,419 instead of about $3,443. The correct total is $3,442.50, so the miss is an addition error." +us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value and no explanation for payroll_tax. The correct derivation is 7.65% of $45,000 in wages: $2,790 Social Security plus $652.50 Medicare, or $3,442.50." +us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It correctly reached $3,442.50 from $2,790 Social Security and $652.50 Medicare, then raised it to $3,495 for made-up 'Indiana county tax adjustments'. Indiana county taxes are local income taxes, not employee payroll taxes, and Indiana has no mandatory employee state payroll tax. Nothing should be added to the FICA total." +us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It subtracted the $1,042 traditional 401(k) deferral from the FICA base. Under IRC 3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base stays at the full $45,000. It then submitted $3,425.67, which does not match its own computed total of $3,362.79. The correct amount is $3,442.50." +us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly built Indiana taxable income of $41,909 (federal AGI $42,909 less the $1,000 exemption) and applied the 2026 2.95% rate to get about $1,236. It then cut the answer to $1,234 for an invented 'rounding in exemption computation' that has no basis, since the exemption is a flat $1,000." +us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,510 capital loss instead of the $3,000 cap, wrongly included $960 of nontaxable workers' compensation, and skipped the $1,000 Indiana exemption. It used a 3.15% rate instead of the 2026 rate of 2.95%, then made an arbitrary cut for 'disability exemptions' to reach $1,255 instead of $41,909 x 2.95% = $1,236." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It invented a $500 exemption for a disabled filer under 65 (Indiana's additional exemptions are for age 65+ and blindness only) and used a 3.05% rate instead of the 2026 rate of 2.95%. It then submitted $1,352, which does not match its own $1,263 result." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It failed to subtract the $1,042 traditional 401(k) deferral from wages, which put federal AGI at $43,951 instead of $42,909. It then pushed its own 2.95% result ($1,267) up to $1,392 by switching to a 3.05% rate, and even at 3.05% its base would not produce that figure." +us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a 2.85% rate instead of the 2026 rate of 2.95%. It also put federal AGI at $42,441, about $468 too low, instead of $42,909. Applying 2.95% to $41,909 gives $1,236." +us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but cut Indiana taxable income to $38,829 with two deductions Indiana does not allow. One was a $1,000 'disability exemption' (Indiana's additional exemptions cover only age 65+ and blindness). The other was a $2,080 deduction for health insurance premiums. Only the $1,000 personal exemption applies, which leaves $41,909." +us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 3.05% rate instead of the 2026 rate of 2.95% and subtracted an extra $1,000 exemption for being disabled, which Indiana does not give a disabled filer under 65. It then inflated its own $1,248 to $1,350 for 'county/local considerations', even though the requested output excludes local tax." +us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,age_disability,False,"It used the correct 2.95% rate and $42,909 AGI but subtracted $2,000 of exemptions by adding a $1,000 exemption for being disabled. Indiana's additional $1,000 exemptions apply only to filers aged 65+ or blind, so taxable income is $41,909, not $40,909." +us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $40,829, which subtracts the $2,080 of health insurance premiums; those are not an above-the-line deduction. It also applied 3.05% instead of the 2026 rate of 2.95%. The correct figure is $41,909 x 2.95% = $1,236." +us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used an AGI of $40,829, reduced by the $2,080 of premiums that are not deductible. It then invented a $1,000 Indiana standard deduction and a $1,000 disabled exemption, neither of which exists, so it taxed $37,829 instead of $41,909 at 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a $3,000 Indiana deduction for child support paid and a $1,500 exemption for being disabled; Indiana allows neither. It also used a 3.0% rate instead of the 2026 rate of 2.95%, taxing $37,409 instead of $41,909." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied a made-up 2.79% rate. Its $1,256 implies a base of about $45,000, meaning it taxed roughly gross wages without the AGI adjustments (401(k), IRA, capped capital loss) or the $1,000 exemption. The correct figure is $41,909 x 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate and $1,000 exemption but started from a federal AGI of $40,829, which wrongly subtracts the $2,080 of health insurance premiums. The correct AGI is $42,909." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It built the correct Indiana taxable income of $41,909 but applied the 2024 rate of 3.05% instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $1,373 equals 3.05% of the $45,000 gross wages. It skipped the AGI adjustments (401(k), IRA, capped capital loss, interest) and the $1,000 exemption, used the 2024 rate instead of 2.95%, and wrongly mixed county tax into a state-only output." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $40,829, which wrongly subtracts the $2,080 of health insurance premiums, and it applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the 2025 rate of 3.0% instead of the 2026 rate of 2.95%. It also built federal AGI as $42,490 by deducting the full $4,510 capital loss instead of the $3,000 cap and by omitting the $1,042 traditional 401(k) and $49 IRA deductions." +us,scenario_015,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied 2.90%, the rate scheduled for 2027, instead of the 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It applied a flat rate after 'standard deductions and exemptions', but Indiana has no standard deduction. Its $1,018 implies a base of about $34,500, far below the correct $41,909 (federal AGI less only the $1,000 exemption), taxed at 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits wipe out Indiana tax. In fact, with $42,909 of AGI, only a $1,000 exemption and no nonrefundable credits available, the 2.95% flat tax leaves $1,236 owed." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a made-up 3.025% rate instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but subtracted an invented extra $1,000 'Indiana deduction' on top of the $1,000 personal exemption, which put taxable income at $40,909 instead of $41,909." +us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied a flat rate to roughly the $45,000 of wages after an invented 'standard deduction'. Its $1,400 implies a base above $45,000 at 3.05%, so it skipped the federal AGI adjustments and the $1,000 exemption, and it did not use 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied the 2025 rate of 3.0% instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a 2.85% rate instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied the 2024 rate of 3.05% instead of the 2026 rate of 2.95%. Its $1,230 implies a base of about $40,300, which means it subtracted more than the single $1,000 exemption from the $42,909 AGI." +us,scenario_015,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It built the correct taxable income of $41,909 but applied a made-up 2.97% rate instead of the scheduled 2026 rate of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrongly included $960 of nontaxable workers' compensation, deducted the full $4,510 capital loss instead of the $3,000 cap, and skipped the $1,000 Indiana personal exemption. It then applied the 2024 rate of 3.05% instead of 2.95%." +us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the rate to federal taxable income ($30,860, after the federal standard deduction) rather than to Indiana AGI less the $1,000 exemption ($41,909). It also used 3.05% instead of 2.95% and rounded its $941 up to $950." us,scenario_015,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_016,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly derived that itemizing the medical expenses against the 7.5%-of-AGI floor ($2,410) drives taxable income to zero and stated ""tax = 0,"" then discarded that result on an invented benchmark convention that a wage earner's self-paid premiums are pre-tax and therefore not itemizable. Having reverted to the standard deduction it computed $1,677 ($12,400 at 10% plus $3,638 at 12%), yet submitted $1,523, a figure that matches neither the itemized nor the standard-deduction branch of its own work." us,scenario_016,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never compared itemized deductions against the standard deduction, missing the roughly $44,590 net medical expense deduction ($47,000 of premiums, other medical, and OTC expenses less the $2,410 7.5%-of-AGI floor) that exceeds AGI and zeroes taxable income. It compounded this by deducting the full $3,000 of student loan interest instead of the $2,500 statutory cap, using a $15,000 standard deduction, fabricating $600 of mortgage interest from a $20,000 balance, and awarding a $575 Saver's Credit even though no retirement contributions are listed and unlisted inputs are zero." @@ -1136,40 +1288,43 @@ us,scenario_016,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wron us,scenario_016,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_016,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_016,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It reconstructed AGI ($61,592), the $16,100 standard deduction, and the 2026 bracket edges ($12,400/$50,400) exactly right, then subtracted $1,625 of auto loan interest under the OBBBA vehicle-loan interest deduction. That deduction reaches only interest on a loan originated after December 31, 2024 for a new, US-final-assembly personal-use vehicle, and every qualifying-vehicle input is unlisted and therefore zero, so PolicyEngine allowed none of it; the $1,625 of phantom deduction at 12% is the entire $195.05 shortfall from $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It stacked two deductions that do not exist for this filer onto a correct $45,492 taxable income: the $1,625 auto loan interest deduction (which requires a post-2024 loan on a new US-assembled vehicle, all unlisted inputs) and a self-invented $6,760 overtime deduction built from 50 usual weekly hours times a $13 half-time premium. The prompt states gross wages already include overtime and qualified overtime compensation is an unlisted input equal to zero, so the OBBBA overtime deduction is $0; removing both phantom deductions restores $8,385 of taxable income at 12% and returns $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 single standard deduction of $14,600 instead of the 2026 amount of $16,100, and applied a fabricated schedule that taxes the first $46,225 at 12% with no 10% bracket at all, producing $5,716 in its own explanation; the true 2026 schedule taxes the first $12,400 at 10% and the balance to $50,400 at 12%. It then submitted $4,903, a number its own stated derivation never produces, so the answer is disconnected from the $5,716 it computed." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its derivation is exactly the reference chain — AGI $61,592, standard deduction $16,100, taxable income $45,492, $1,240 + 12% × $33,092 = $5,211 — and it then discarded that result, asserting that 'rounded brackets' give approximately $6,754. No rounding of the $12,400/$50,400 bracket edges moves the tax by $1,543; the model overwrote a correct computation with an unanchored number." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the reference exactly — $1,240 on the first $12,400 plus 12% of $33,092 ($3,971), which sums to $5,211.05 — and then reported $6,033 as the total 'after applying bracket structure.' Its own two bracket components add to $5,211, so the submitted figure contradicts the arithmetic it wrote down; there is no 22% exposure because taxable income of $45,492 sits below the $50,400 bracket top." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It claimed a $200 Saver's Credit by inventing a 10%-rate AGI ceiling of about $64,000 for single filers; the credit's top single AGI band ends near $40,500 in 2026, so at AGI $61,592 the rate is zero and no nonrefundable credit reduces the tax. It compounded this by carrying 2025 parameters forward — the $15,000 standard deduction and the $11,925/$48,475 bracket edges instead of $16,100 and $12,400/$50,400 — which inflated pre-credit tax to $5,352.54 and partially masked the credit error." -us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It reached taxable income of $45,492 correctly, then applied fabricated bracket edges (10% to $12,150, 12% to $49,400 rather than $12,400 and $50,400) for $5,216, and finally submitted $4,913 — $303 lower — after stating that the Saver's Credit is phased out. Having correctly ruled out the only candidate credit, it subtracted an amount it never justified; with the true $12,400 bracket top and no credits the answer is $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the single $9,000 employer-plan premium twice (as 'health insurance premiums excluding Medicare Part B' plus 'other health insurance premiums') to reach $18,000, added $1,500 of over-the-counter costs, and got a $15,380.60 medical deduction that pushed itemized deductions to $16,535.40. The medical out-of-pocket total nets to roughly $6,400 after the 7.5% AGI floor, so with AZ income tax the itemized total stays far below the $16,100 standard deduction PolicyEngine applied; it also used a $15,400 standard deduction and a $12,075 10%-bracket top instead of $16,100 and $12,400." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law to 2026 — a $7,700 standard deduction, a $4,950 personal exemption, and a 15% second bracket — but OBBBA made the TCJA rate schedule and enlarged standard deduction permanent, so 2026 uses a $16,100 standard deduction, no personal exemption, and a 12% second bracket topping out at $50,400. It also cut $9,000 of employer health premiums out of wages, which PolicyEngine did not do: AGI is $61,592.09, not $52,592." -us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its AGI of $61,592 and its no-credit conclusion are correct, but it used a $15,650 standard deduction and a $12,450 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. The $450 of extra taxable income taxed at 12% plus the shifted bracket edge account for the full $52.99 overstatement above $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset took effect in 2026 and restored an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA made the TCJA structure permanent: 2026 gives a $16,100 standard deduction, no personal exemption, and 10%/12% rates through $50,400, so taxable income is $45,492.09 and the tax is $5,211.05 rather than the $6,703 its reverted schedule produces." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It dropped both the $50 of taxable interest and the $289 deductible IRA contribution from AGI, used the 2024 standard deduction of $14,600 instead of the 2026 $16,100, and then applied a flat 10% rate to the whole $47,231 of taxable income — its $4,724 is essentially 10% × $47,231. The graduated 2026 schedule charges $1,240 on the first $12,400 and 12% on the rest, which on the correct $45,492.09 gives $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It built taxable income from a combined $13,790 standard deduction plus personal exemption on 'projected pre-TCJA' brackets, applying rates of 15% and above. In 2026 the personal exemption remains repealed and the single standard deduction is $16,100 with 10%/12% rates through $50,400; the correct chain from its own AGI of $61,592 is $45,492.09 taxable and $5,211.05 in tax." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reverted 2026 to pre-TCJA parameters — an $8,300 standard deduction, a $5,300 personal exemption, and a 15% second bracket — and additionally excluded $9,000 of health insurance premiums from wages, dropping AGI to $52,592. PolicyEngine's AGI is $61,592.09 with no premium exclusion, the 2026 standard deduction is $16,100 with no personal exemption, and the second bracket rate is 12%." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no parameters, and its $5,304 is what the correct AGI of $61,592.09 yields on the true 2026 brackets with a standard deduction of about $15,325 — a pre-OBBBA-sized amount — rather than the $16,100 that applies for 2026. The $775 of excess taxable income taxed at 12% is the whole $92.95 overstatement above $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its AGI of $61,592 is right, but it used a $15,700 standard deduction and a $12,275 ceiling on the 10% bracket; the 2026 values are $16,100 and $12,400. Substituting the correct parameters turns its $45,892 taxable income into $45,492.09 and its $5,261.54 into $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It computed AGI correctly at $61,592 and then applied 'TCJA provisions sunset' law, subtracting a combined standard deduction plus personal exemption of $13,450 and taxing the result under 10%/15% brackets. The personal exemption stays repealed for 2026 and the standard deduction is $16,100 with a 12% second bracket, so taxable income is $45,492.09 and the tax is $5,211.05, not $6,601.30." -us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It carried 2025 parameters into 2026: a $15,000 standard deduction and bracket edges of $11,925 and $48,475, instead of the 2026 $16,100 deduction and $12,400/$50,400 edges. The $1,100 of excess taxable income at 12% plus the lower 10%-bracket ceiling account for the entire $141.49 gap between its $5,352.54 and $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI of $61,592 and its correct rejection of the Saver's Credit leave only the parameters wrong: it estimated a $15,400 standard deduction and a $12,260 top for the 10% bracket, where 2026 uses $16,100 and $12,400. Those two substitutions move taxable income from its $46,192 to $45,492.09 and the tax from $5,297.84 to $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $15,000 single standard deduction and a roughly $12,300 10%-bracket ceiling for 2026 rather than the actual $16,100 and $12,400. The $1,100 of overstated taxable income taxed at 12% is the bulk of the $134 by which its $5,345 exceeds $5,211.05; every other step, including AGI of $61,592 and the absence of credits, is right." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It reached AGI of about $61,592 and then subtracted 'the nonrefundable child tax credit for one qualifying child' from a single-person household with no dependents listed. With no children, the CTC is $0 and no other nonrefundable credit applies to this filer, so the tax stands at $1,240 + 12% × $33,092.09 = $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It performed no bracket computation and simply asserted that nonrefundable credits wipe out the liability. A childless 53-year-old single filer with $45,492.09 of taxable income has no CTC, no CDCC, no education credit, and no Saver's Credit (AGI $61,592 sits far above the roughly $40,500 single ceiling), so nothing offsets the $5,211.05 of bracket tax." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied a $6,760 'qualified overtime deduction' on top of the correct $16,100 standard deduction, reducing taxable income from $45,492.09 to $38,732. The OBBBA overtime deduction covers only the FLSA half-time premium reported as qualified overtime compensation, an unlisted input equal to zero, and the prompt states gross wages already include overtime; removing it restores $6,760 at 12% and yields $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It got AGI of $61,592 and the $16,100 standard deduction exactly right, then taxed the $45,492.09 using the 2025 10%-bracket ceiling of $11,925 instead of 2026's $12,400: $1,192.50 + 12% × $33,567 = $5,220.54. Moving the bracket edge to $12,400 gives $1,240 + 12% × $33,092.09 = $5,211.05, the full $9.49 difference." -us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its AGI, $16,100 standard deduction, and bracket arithmetic are all correct; it then deducted an estimated $6,760 overtime premium (10 hours × $13 × 52 weeks) that no input supports. Qualified overtime compensation is unlisted and therefore zero, and the prompt states the $68,005 of wages already includes overtime pay, so taxable income is $45,492.09 and the tax is $5,211.05 rather than $4,399.84." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It never ran the bracket arithmetic, describing '~$61k taxable income' when $61,592 is AGI and taxable income after the $16,100 standard deduction is $45,492.09, then submitted a one-significant-figure $5,200. The actual schedule gives $1,240 on the first $12,400 plus 12% of $33,092.09, or $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It double-counted the $9,000 employer premium to claim $18,500 of medical costs and a $13,881 medical deduction, itemized at $15,035, and then also subtracted a $5,300 personal exemption and taxed the balance at 10%/15%. The personal exemption remains repealed for 2026, rates are 10%/12% through $50,400, and even its own itemized total falls below the $16,100 standard deduction PolicyEngine applied, giving $45,492.09 taxable and $5,211.05 of tax." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It derived AGI of $61,592 correctly and then used a $15,400 standard deduction with a $12,250 10%-bracket ceiling, where 2026 sets $16,100 and $12,400. The $700 of overstated taxable income at 12% plus the shifted bracket edge produce its $87 excess over $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Its AGI of $61,592 is exact, but it assumed a $15,375 single standard deduction for 2026 instead of $16,100, leaving taxable income of $46,217 rather than $45,492.09. That $725 of extra income at the 12% rate, plus a 10%-bracket ceiling set below the actual $12,400, accounts for the $91 by which its $5,302 exceeds $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized at $16,461 by counting the $9,000 employer-plan premium twice and including $1,500 of over-the-counter health costs, producing a $15,381 medical deduction that narrowly beat the $16,100 standard deduction. The medical out-of-pocket base nets to roughly $6,400 after the 7.5% AGI floor, so with Arizona income tax the itemized total is far under $16,100 and PolicyEngine took the standard deduction, leaving taxable income of $45,492.09 and tax of $5,211.05 instead of $5,168." -us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the submission never engaged the computation. The required chain is AGI $61,592.09 less the $16,100 standard deduction, taxed at 10% to $12,400 and 12% above, giving $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted zero with no derivation at all. Zero requires either no taxable income or a full credit offset, and neither exists here: AGI of $61,592.09 less the $16,100 standard deduction leaves $45,492.09 of taxable income, and a childless single filer at that AGI qualifies for no CTC, CDCC, or Saver's Credit, so the liability is $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It omitted the $289 deductible traditional IRA contribution, leaving AGI at $61,881 instead of $61,592, and used a $15,750 standard deduction with a $12,550 10%-bracket ceiling rather than the 2026 $16,100 and $12,400. Its rejection of the Saver's Credit is right; correcting the IRA deduction and the two parameters moves taxable income from $46,131 to $45,492.09 and the tax from $5,285 to $5,211.05." -us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $32,942 is $45,492.09 less a second deduction of roughly $12,550 — it subtracted retirement contributions and a standard deduction that were already reflected in AGI, deducting the same amounts twice. It then removed $303 of Saver's Credit, which is zero for a single filer whose AGI of $61,592 exceeds the roughly $40,500 top of the credit's income range, so the correct result is $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model got AGI ($61,592), the $16,100 standard deduction and the 2026 brackets right, then also subtracted a $1,625 OBBBA car-loan interest deduction. That deduction requires a qualifying new, US-assembled vehicle, and the household lists no such vehicle. The extra deduction cut taxable income to $43,867 instead of $45,492 and the tax to $5,016 instead of $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Beyond the correct $16,100 standard deduction, the model subtracted a $1,625 car-loan interest deduction and a $6,760 overtime premium deduction that it inferred from 50 weekly hours. No qualified overtime compensation is listed, so the overtime deduction is $0. Wages of $68,005 are close to straight-time pay at $26 × 50 × 52, so no time-and-a-half premium is embedded. Neither deduction applies, so taxable income is $45,492, not $37,107." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It also used a bogus bracket structure: 12% on everything up to $46,225 with no 10% bracket, then 22% above that, when the 2026 12% bracket runs to $50,400. It computed $5,716, then submitted an unrelated $4,903." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed the correct liability, $1,240 + 12% × $33,092 = $5,211 on taxable income of $45,492. It then threw that result away for an unexplained 'adjusting with rounded brackets' figure of $6,754, which no bracket computation on $45,492 produces." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model set up the correct computation, $1,240 on the first $12,400 plus $3,971 at 12% on $33,092, which totals $5,211. It then misadded the two pieces and reported $6,033." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed FLSA non-exempt status and took a $6,760 overtime premium deduction (0.5 × $26 × 10 hours × 52 weeks). It also assumed the loan qualified and took a $1,625 car-loan interest deduction. Unlisted status facts are false and unlisted amounts are zero, so neither deduction applies. The standard deduction alone gives taxable income of $45,492, not $37,107." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model subtracted a $200 saver's credit at a 10% rate using a $64,000 AGI ceiling. The single-filer ceiling is about $40,000, so AGI of $61,592 gets no credit. It also used the 2025 pre-OBBBA $15,000 standard deduction and $11,925 bracket top instead of 2026's $16,100 and $12,400." +us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model reached the correct taxable income of $45,492 but used a $12,150 top for the 10% bracket instead of $12,400, giving $5,216. It then cut about $303 more to reach $4,913, even though its own reasoning says the saver's credit phases out at this income and no other nonrefundable credit applies." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model counted $18,000 of health premiums by adding both $9,000 premium lines, but they describe the same $9,000 premium. That inflated medical expenses to $20,000 and itemized deductions to $16,535, so the model itemized. With the premium counted once, itemized deductions fall far below the $16,100 standard deduction. It also used a $12,075 top for the 10% bracket instead of $12,400." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026, applying a $7,700 standard deduction, a $4,950 personal exemption and a 15% bracket. OBBBA made the TCJA structure permanent, with a $16,100 standard deduction and 10% and 12% brackets. It also subtracted the $9,000 health premium from AGI as a pre-tax payroll deduction, which the facts do not list, giving AGI of $52,592 instead of $61,592." +us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model got AGI right at $61,592 but used a $15,650 standard deduction and a $12,450 top for the 10% bracket. The 2026 values are $16,100 and $12,400. As a result it overstated taxable income by $450, at $45,942 instead of $45,492." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and the 15% and 25% brackets. OBBBA made the TCJA brackets permanent and set the 2026 single standard deduction at $16,100, with no personal exemption. Its taxable income of $47,992 and 25%-bracket tax are both wrong as a result." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model left out the $289 IRA deduction and the $50 of interest, and used the 2024 standard deduction of $14,600 to reach taxable income of $47,231. It then applied a flat 10% ($4,724) to all of it instead of taxing the amount above $12,400 at 12%." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used projected pre-TCJA 2026 law, with a combined standard deduction and personal exemption of $13,790 and pre-TCJA 15% and 25% brackets. The OBBBA-extended structure applies instead: a $16,100 standard deduction, no exemption, and 10% and 12% brackets on $45,492 of taxable income." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied TCJA-sunset law, with an $8,300 standard deduction, a $5,300 exemption and a 15% bracket starting at $11,000, instead of the OBBBA 2026 $16,100 standard deduction and 10% and 12% brackets. It also subtracted the $9,000 health premium from AGI as a pre-tax deduction, which the facts do not list, giving AGI of $52,592 instead of $61,592." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no figures. Its $5,304 is about $93 above the correct $5,211.05, which fits pre-2026 parameters: a standard deduction around $15,000 to $15,400 and an older, lower 10% bracket top. The correct 2026 values are a $16,100 standard deduction and a $12,400 bracket top, applied to AGI of $61,592." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got AGI right at $61,592 but used a $15,700 standard deduction and a $12,275 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $45,892 instead of $45,492." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed TCJA provisions expired in 2026 and subtracted a pre-TCJA standard deduction plus a personal exemption, reaching taxable income of $48,142 taxed at 10% and 15%. OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no exemption, and 10% and 12% brackets." +us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied 2025 pre-OBBBA parameters: a $15,000 standard deduction and a $11,925 top for the 10% bracket. The 2026 values are $16,100 and $12,400. That gave taxable income of $46,592 and tax of $5,352.54." +us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model estimated the 2026 standard deduction at $15,400 and the 10% bracket top at $12,260. The actual values are $16,100 and $12,400. That overstated taxable income by $700, at $46,192 instead of $45,492." +us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of the 2026 OBBBA amount of $16,100, giving taxable income of $46,592. It also used about $12,300 for the 10% bracket top instead of $12,400." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model subtracted a nonrefundable child tax credit for 'one qualifying child', but the household is a single adult with no dependents, so no CTC is available. The tax before refundable credits is the full $5,211.05 on $45,492 of taxable income." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model asserted zero liability without computing any tax. $68,005 of wages leaves $45,492 of taxable income after the 401(k), IRA and $16,100 standard deduction, which produces $5,211.05 of tax at the 10% and 12% rates, with no credits to offset it." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction, inferred from 50 weekly hours at $26. No qualified overtime compensation is listed, and wages of $68,005 are close to straight-time pay for all hours, so the deduction is $0. Taxable income is $45,492, not $38,732." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model got AGI and the $16,100 standard deduction right, but its $5,220.54 implies a $11,925 top for the 10% bracket, the 2025 pre-OBBBA figure. The 2026 top is $12,400, which gives $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model took an estimated $6,760 qualified overtime premium deduction (10 hours × $13 × 52). No qualified overtime compensation is listed, so the deduction is $0, and wages of $68,005 are close to straight-time pay at $26 × 50 × 52. Taxable income is $45,492, not $38,732." +us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 qualified overtime deduction that none of the listed inputs support. Unlisted amounts are $0, and the stated wages already reflect straight-time pay for 50 hours. That lowered taxable income to $38,732 instead of $45,492." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model gave a rounded guess of about $5,200 without doing the bracket computation. Exact 2026 parameters (a $16,100 standard deduction, a $12,400 10% bracket top and 12% above it) on $61,592 of AGI give $5,211.05." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied TCJA-sunset law: a $5,300 personal exemption and pre-TCJA 10% and 15% brackets. It also itemized $15,035 using $18,500 of medical expenses, which counts the $9,000 premium twice. Under OBBBA's permanent structure the $16,100 standard deduction exceeds the correctly counted itemized total, and no exemption exists." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and a $12,250 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. That gave taxable income of $46,192 instead of $45,492." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got the $16,100 standard deduction and $45,492 of taxable income right, but capped the 10% bracket at $12,200 instead of the 2026 top of $12,400. That overstated tax by $4 ($5,215.04 instead of $5,211.05)." +us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 standard deduction instead of the 2026 OBBBA amount of $16,100, giving taxable income of $46,217 instead of $45,492. The tax is overstated as a result." +us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model itemized about $15,381 of medical deductions, which assumes $20,000 of medical costs, adding both $9,000 premium lines as separate premiums. With the $9,000 premium counted once, medical deductions plus state tax fall well below the $16,100 standard deduction, which leaves taxable income at $45,492, not $45,132." +us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so its answer is missing, not a substantive miscalculation." +us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model gave $0 with no reasoning, which implies it treated the filer as having no tax liability. AGI of $61,592 less the $16,100 standard deduction leaves $45,492 of taxable income, which produces $5,211.05 of tax, and no nonrefundable credits apply." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the 2025 OBBBA standard deduction of $15,750 and a $12,550 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. It also left out the deductible $289 traditional IRA contribution, putting AGI at $61,881 instead of $61,592." +us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted extra 'qualifying retirement contributions' to reach taxable income of $32,942, $12,550 below the correct $45,492. Only the traditional 401(k) and $289 IRA reduce AGI, and the Roth contributions do not. It also subtracted a $303 saver's credit, but AGI of $61,592 is well above the single-filer eligibility ceiling." us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies." us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38." @@ -1179,85 +1334,99 @@ us,scenario_018,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated us,scenario_018,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract." us,scenario_018,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits federal employee FICA on $68,005 of wages. The correct computation includes $4,216.31 of Social Security tax and $986.07 of Medicare tax, totaling $5,202.38." us,scenario_018,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its own derivation reached the exact reference — $61,592 AGI less the $15,750 Arizona standard deduction, times 2.5%, equals $1,146 — and it then abandoned that by subtracting the $1,625 auto-loan interest from Arizona taxable income; the OBBBA car-loan interest deduction is a federal below-the-line deduction that never enters Arizona taxable income, which begins at federal AGI. It then submitted 1,503, a figure matching neither its correct $1,146 nor its own erroneous $1,105 recomputation." -us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"AGI ($61,592) and the 2.5% flat rate were correct, but it used the 2026 federal single standard deduction of $16,100 instead of Arizona's $15,750, overstating the deduction by $350 and understating taxable income to $45,492 rather than $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied Arizona's pre-2023 graduated brackets (2.55% / 3.34%), which SB 1828 replaced with a single 2.5% flat rate effective tax year 2023, and used a $14,600 standard deduction rather than $15,750. It then discarded its own $1,304.40 result for $2,175 on the strength of a nonexistent Arizona health-insurance-premium adjustment; premiums enter Arizona only through itemized deductions, which at $10,786.71 lose to the standard deduction." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It derived the correct $61,592 AGI and applied the correct 2.5% flat rate, but guessed a $15,000 Arizona standard deduction instead of the $15,750 in force for 2026, leaving $750 of excess taxable income and $18.75 of excess tax." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a $14,600 standard deduction rather than $15,750, producing $1,175, then raised the submitted answer to $1,471 to account for 'a slightly higher standard deduction and indexing' — an adjustment that runs backwards, since a larger deduction lowers Arizona tax and the correct $15,750 yields $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It substituted the $16,100 federal standard deduction for Arizona's $15,750 and concluded in its own text that the tax is $1,137, then submitted 1,481 — a number its stated computation never produces at any step." -us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It correctly built AGI of $61,592 and applied Arizona's 2.5% flat rate, but settled on a $14,600 Arizona standard deduction instead of $15,750, adding $1,150 of taxable income and $28.75 of tax." -us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It computed $1,175 using a $14,600 standard deduction rather than the $15,750 Arizona amount, then inflated the submitted value to $1,450 for 'HSA-like premium considerations'; Arizona has no such addition to income, and health premiums reach the return only as itemized deductions, which total $10,786.71 and lose to the standard deduction." -us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"AGI and the 2.5% flat rate were exact; it used a $15,400 standard deduction instead of Arizona's $15,750, so its taxable income of $46,192 exceeds the correct $45,842.09 by $350." -us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Two compounding errors: it reduced AGI from $61,592 to $52,592 by subtracting $9,000 of employer-plan health premiums, which are already excluded from the $68,005 W-2 wage figure and are not an above-the-line adjustment, and it used a $7,700 standard deduction in place of Arizona's $15,750." -us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reproduced the correct $61,592 AGI and 2.5% flat rate but used a $15,650 Arizona standard deduction instead of $15,750, leaving taxable income $100 high and tax $2.50 high." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The $61,592 state AGI and the 2.5% flat rate were right; the projected $14,600 standard deduction is $1,150 below Arizona's actual $15,750, which is what pushes its $46,992 taxable income above the correct $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It offered no computation at all. The correct derivation is $61,592.09 AGI less the $15,750 Arizona standard deduction, or $45,842.09, times the 2.5% flat rate, giving $1,146.05; the submitted $1,400 implies $56,000 of Arizona taxable income, i.e. a deduction of roughly $5,600 that corresponds to no Arizona parameter." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied the correct 2.5% flat rate to the correct AGI but subtracted a $13,790 standard deduction — an obsolete pre-2022 federal figure — rather than Arizona's $15,750, inflating taxable income to $47,802 against the correct $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from $52,592 instead of the correct $61,592 AGI, a $9,000 reduction for employer-sponsored health premiums that are already excluded from the $68,005 wage amount, and compounded it with a $14,600 standard deduction in place of Arizona's $15,750." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"No derivation was given, and $1,530 is 2.5% of $61,200 — the flat rate applied to essentially the entire $61,592 AGI with no deduction subtracted. Arizona's $15,750 single standard deduction brings the base to $45,842.09 and the tax to $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Every step was right except the deduction parameter: it used $15,700 rather than Arizona's $15,750, so its $45,892 taxable income overstates the correct $45,842.09 by $50 and the tax by $1.25." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied the correct 2.5% flat rate but to $46,092 of taxable income, which implies a $15,500 standard deduction against Arizona's actual $15,750; the correct base is $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It carried the correct $61,592 Arizona AGI and 2.5% rate but subtracted a $15,000 standard deduction instead of $15,750, adding $750 to taxable income and $18.75 to tax." -us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used a $15,400 standard deduction rather than Arizona's $15,750 and computed $1,154.80, then wrote that result as '$1,1548.0' and submitted 11,548 — an order-of-magnitude transcription error ten times its own arithmetic and ten times the $1,146.05 reference." -us,scenario_018,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"AGI and the 2.5% flat rate were correct; the single defect is a $15,000 Arizona standard deduction in place of the $15,750 that applies for 2026, which raises taxable income from $45,842.09 to $46,592." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It asserted $1,941 without any stated computation. Arizona's 2.5% flat rate on the correct base of $45,842.09 (AGI $61,592.09 less the $15,750 standard deduction) gives $1,146.05; $1,941 implies $77,640 of taxable income, more than the household's entire $68,055 of gross income, so it corresponds to no Arizona rate-and-base combination." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It declared the Arizona tax zero as a 'simplified policy-quantity estimate'. Arizona's 2.5% flat rate applies to $45,842.09 of taxable income after the $15,750 standard deduction, and no filing threshold, zero bracket, or nonrefundable credit zeroes out a single filer at $61,592 AGI — the family tax credit cuts off near $20,000 of income." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $18,500 of medical and health-insurance deductions by counting the $9,000 employer-plan premiums, which are paid pre-tax and are not deductible again, alongside the $9,000 of other premiums. Arizona itemized deductions here total $10,786.71 ($9,500 medical plus $1,286.71 of state and local taxes), so the $15,750 standard deduction controls and the base is $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It substituted the 2026 federal single standard deduction of $16,100 for Arizona's $15,750, producing $45,492 of taxable income instead of $45,842.09; the AGI and the 2.5% flat rate were otherwise exact." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It began from the correct $61,592 federal AGI and applied the correct 2.5% rate, but deducted $16,100 — the federal standard deduction — rather than Arizona's $15,750, understating tax by $8.75." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $45,492 Arizona taxable income reflects a $16,100 standard deduction; Arizona's 2026 amount is $15,750, which leaves $45,842.09 and $1,146.05 of tax at the 2.5% flat rate." -us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Only the deduction parameter is wrong: it subtracted the $16,100 federal single standard deduction from the correct $61,592 AGI instead of Arizona's $15,750, so its base of $45,492 falls $350 short of the correct $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied 'progressive rates averaging under 3%' to roughly $61,000. Arizona repealed its graduated brackets effective tax year 2023 in favor of a 2.5% flat rate, and the base is $45,842.09 after the $15,750 standard deduction, not AGI." -us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It matched the reference on AGI and rate but projected a $15,450 Arizona standard deduction rather than the actual $15,750, giving $46,142 of taxable income against the correct $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction in place of Arizona's $15,750, so its $46,192 base exceeds the correct $45,842.09 by $350; AGI construction and the 2.5% flat rate were correct." -us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The one error is a $15,375 Arizona standard deduction instead of $15,750, which leaves $46,217 of taxable income rather than $45,842.09 at the correct 2.5% flat rate." -us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It stacked a $16,100 federal standard deduction on top of a phantom $2,300 personal exemption. Arizona eliminated personal exemptions when it moved to the federal standard deduction, so the only subtraction from the $61,592.09 AGI is the $15,750 standard deduction, leaving $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for this output, so there is no substantive computation to assess. The required derivation is $61,592.09 of Arizona AGI less the $15,750 standard deduction, giving $45,842.09 taxed at the 2.5% flat rate for $1,146.05." -us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly identified the flat 2.5% rate, the $61,592 AGI, and the absence of nonrefundable credits, but used a $15,000 Arizona single standard deduction instead of $15,750, raising taxable income to $46,592 from the correct $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted zero with no derivation. A single Arizona filer with $61,592.09 of AGI has $45,842.09 of taxable income after the $15,750 standard deduction and owes $1,146.05 at the 2.5% flat rate; no exemption, filing threshold, or nonrefundable credit reduces that to zero." -us,scenario_018,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It correctly ruled out the family tax credit on income grounds and applied the 2.5% flat rate to the right AGI, but subtracted the $16,100 federal standard deduction rather than Arizona's $15,750, so its base of $45,492 falls short of $45,842.09." -us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated the $6,174 traditional 401(k) and $289 traditional IRA contributions as already reflected in the stated gross wages and took no above-the-line adjustments, taxing $68,055 instead of the $61,592.09 AGI, and compounded that with a $14,600 standard deduction rather than Arizona's $15,750." -us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its stated figures contradict its own arithmetic — $61,592 less a $14,508 deduction is $47,084, not the $45,486 it claims — and $832 is 1.81% of that base rather than Arizona's 2.5% flat rate. Applying 2.5% to the correct $45,842.09 base after the $15,750 standard deduction gives $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning correctly reached $61,592 − $15,750 = $45,842 × 2.5% = $1,146. It then subtracted the $1,625 auto loan interest as a federal car-loan deduction, which Arizona taxable income does not include here. It finally submitted $1,503, a number that none of its own arithmetic supports." +us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the federal 2026 single standard deduction of $16,100 instead of Arizona's $15,750. That understated taxable income ($45,492 vs $45,842.09) and gave $1,137.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied repealed graduated brackets (2.55%/3.34%) instead of Arizona's flat 2.5% rate. It also used a $14,600 standard deduction instead of $15,750 and misadded AGI as $61,567. It then submitted $2,175, which does not match even its own $1,304 bracket computation." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It guessed an Arizona standard deduction of $15,000 instead of $15,750. That gave taxable income of $46,592 and tax of $1,165 instead of $45,842.09 and $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used a $14,600 standard deduction instead of $15,750 and computed $1,175. It then inflated that to $1,471 through an unexplained 'indexing' adjustment that moves in the wrong direction for a larger deduction." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of Arizona's $15,750 and reached $1,137. It then submitted $1,481, which contradicts its own stated final computation." +us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the federal 2026 standard deduction of $16,100 instead of Arizona's $15,750. Taxable income came out $350 too low ($45,492) and tax was $1,137.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It projected the Arizona single standard deduction at $14,600 instead of $15,750. That overstated taxable income at $46,992 and gave $1,175 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $14,600 standard deduction instead of $15,750 and reached $1,175. It then added an invented 'HSA-like premium' adjustment to submit $1,450, although no such addition to Arizona taxable income exists." +us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the federal $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30 instead of $45,842.09 and $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $15,400 Arizona standard deduction instead of $15,750. Taxable income came out at $46,192 and tax at $1,154.80 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut AGI to $52,592 by excluding the $9,000 health premiums as pre-tax, even though AGI is $61,592.09. It then used a $7,700 standard deduction instead of $15,750. The two errors partly offset and gave $1,122.30." +us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $15,650 Arizona standard deduction instead of $15,750. That overstated taxable income by $100 ($45,942) and gave $1,148.55 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of $15,750 and also subtracted a $2,100 personal exemption. Arizona has no taxpayer personal exemption, so its taxable income of $43,392 is too low and gave $1,084.80." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It projected a $14,600 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,992 and tax of $1,174.80 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. The correct figure is 2.5% × ($61,592.09 − $15,750) = $1,146.05, while its $1,400 matches 2.5% of about $56,000 of taxable income. That means it applied far less than the $15,750 standard deduction, or simply rounded a guess." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a stale $13,790 standard deduction instead of Arizona's 2026 $15,750. That overstated taxable income at $47,802 and gave $1,195." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $52,592, excluding $9,000 of health premiums that are not an above-the-line adjustment (AGI is $61,592.09). It then used a $14,600 standard deduction instead of $15,750, which gave $949.80." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,530 equals 2.5% of about $61,200, roughly the full $61,592 AGI, so it skipped the $15,750 standard deduction that brings taxable income down to $45,842.09." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,700 standard deduction instead of Arizona's $15,750. Taxable income came out $50 too high ($45,892) and tax was $1,147.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its taxable income of $46,092 implies a $15,500 standard deduction instead of $15,750. That gave $1,152.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona standard deduction instead of $15,750. That gave taxable income of $46,592 and tax of $1,164.80." +us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used a $15,400 standard deduction instead of $15,750 and computed 2.5% × $46,192 = $1,154.80. It then misplaced the decimal ('$1,1548.0') and submitted $11,548, a tenfold error." +us,scenario_018,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona standard deduction instead of $15,750. That overstated taxable income at $46,592 and gave $1,165 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $1,941 is more than 2.5% of total gross income ($68,055 × 2.5% = $1,701), so it applied a rate above Arizona's flat 2.5%. The correct figure is 2.5% on $45,842.09 of taxable income after the $15,750 standard deduction." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It set Arizona tax to $0 without any computation. It ignored the $45,842.09 of taxable income that remains after the $15,750 standard deduction and is taxed at the flat 2.5% rate for $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $18,500 of medical and health-insurance deductions by counting the two $9,000 premium fields as separate premiums. Arizona itemized deductions are only $10,786.71 ($9,500 medical plus $1,286.71 in state and local taxes), so the $15,750 standard deduction applies, not its $18,500." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the federal 2026 standard deduction of $16,100 instead of Arizona's $15,750. That understated taxable income at $45,492 and gave $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction instead of Arizona's $15,750 from $61,592 of AGI. That gave $1,137.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its taxable income of $45,492 reflects the federal $16,100 standard deduction instead of Arizona's $15,750. That understated tax at $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used the federal 2026 single standard deduction of $16,100 instead of Arizona's $15,750. That gave taxable income of $45,492 instead of $45,842.09 and tax of $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $45,492 of taxable income comes from the federal $16,100 standard deduction instead of Arizona's $15,750. That gave $1,137.30 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It treated $16,100 as the Arizona standard deduction, but the Arizona 2026 single amount is $15,750. Its taxable income was $45,492 and its tax $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It applied 'progressive rates averaging under 3%' to about $61,000, even though Arizona has used a flat 2.5% rate since 2023. It also never subtracted the $15,750 standard deduction, which brings taxable income to $45,842.09." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It projected a $15,450 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,142 and tax of $1,154." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction instead of $15,750. That overstated taxable income at $46,192 and gave $1,155 instead of $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It conformed Arizona to the federal 2026 $16,100 standard deduction, but Arizona's 2026 amount is $15,750. Its taxable income was $45,492 and its tax $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $15,375 standard deduction instead of Arizona's $15,750. That gave taxable income of $46,217 and tax of $1,155." +us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used the federal $16,100 standard deduction instead of $15,750 and also subtracted a $2,300 personal exemption. Arizona has no taxpayer personal exemption, so its taxable income of $43,192 is too low and gave $1,080." +us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is nothing to score against $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used a $15,000 Arizona single standard deduction instead of $15,750. That gave taxable income of $46,592 and tax of $1,164.80." +us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with no reasoning. That ignores the $45,842.09 of Arizona taxable income ($61,592.09 AGI minus the $15,750 standard deduction), which the flat 2.5% rate taxes at $1,146.05." +us,scenario_018,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the federal 2026 $16,100 standard deduction instead of Arizona's $15,750. That gave taxable income of $45,492 and tax of $1,137.30." +us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left the $6,174 traditional 401(k) and the $289 traditional IRA in income, using AGI of $68,055 instead of $61,592.09. It also used a $14,600 standard deduction instead of $15,750, which inflated taxable income to $53,455 and gave $1,336." +us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $14,508 standard deduction instead of $15,750. It then applied an effective rate of about 1.83% ($832 on $45,486) instead of Arizona's flat 2.5% rate, which on $45,842.09 gives $1,146.05." us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted only the $13,834 of real estate taxes as SALT and therefore took the $16,100 standard deduction; the SALT deduction is $17,090.34, adding $3,256.34 of state and local sales tax deductible in lieu of a Texas income tax and far below the $40,400 2026 cap, so the filer itemizes to $283,059.84 of taxable income. Every other step it took — the alimony adjustment, the fully phased-out overtime deduction, the $222.64 NIIT — matches the reference, and the $990.34 of forgone deduction at the 35% rate is the entire $347 gap." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Treated $13,834 of real estate taxes as the whole SALT deduction and so took the $16,100 standard deduction, missing the $3,256.34 state and local sales tax component that lifts SALT to $17,090.34 and makes itemizing the larger deduction. It compounded this by claiming the $196 non-itemizer cash charitable deduction, which an itemizer cannot take and whose $196 of gifts falls below the 0.5% AGI floor anyway." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $138,375 of unreimbursed employee business expenses in reaching a $161,775 AGI; that miscellaneous itemized deduction is repealed for 2026, and AGI is $300,150.19 after only the $5,708.82 alimony adjustment. It then abandoned its own $27,268 bracket result and reported $66,661, a number no step in its stated derivation produces." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Added the $42,857 FLSA overtime premium to the $300,000 of wages as separate income, double-counting a component already inside the stated wage total, and applied the repealed $10,000 SALT cap instead of the $40,400 2026 cap that leaves all $17,090.34 deductible. It also reported $70,813 while its own bracket arithmetic on a $326,907 taxable income produced $82,743." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Denied the $5,708.82 above-the-line alimony deduction and invented $23,855 of mortgage interest by applying a 6.5% rate to the $367,000 balance, when the prompt sets unlisted amounts to 0 and SALT of $17,090.34 is the only deductible item. It also capped SALT at the repealed $10,000, used 2025 brackets, and reported $71,866 against its own $64,068 computation." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Guessed an itemized total of $30,000-$40,000 instead of computing the deduction set, which is $17,090.34 of SALT alone — the $196 of charitable gifts falls below the 0.5% AGI floor and the $138,375 of employee business expenses is repealed. Its reported $72,700 also exceeds the $63,000-$64,000 its own reasoning produced; the correct taxable income of $283,059.84 yields $67,834.07 of regular tax plus $222.64 of NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Denied the $5,708.82 alimony deduction, applied the repealed $10,000 SALT cap, and projected a $15,400 standard deduction and bracket set from 2025 rather than using the enacted 2026 $16,100 figure and rate schedule. The filer itemizes $17,090.34 of SALT against a $300,150.19 AGI, and it further omitted the $222.64 net investment income tax." -us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added $900 of Additional Medicare Tax to the income tax figure — a payroll tax excluded from this output — and used the 2025 $15,000 standard deduction and brackets in place of the 2026 $16,100 and rate schedule, never testing the $17,090.34 SALT itemized deduction that beats the standard deduction. It then reported $76,500 against its own $70,467 total." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Assumed TCJA expiration and deducted $132,258 of unreimbursed employee business expenses above a 2% AGI floor plus a personal exemption under pre-TCJA brackets. Miscellaneous itemized deductions and personal exemptions remain repealed in 2026, leaving $17,090.34 of SALT as the only itemized deduction and $283,059.84 of taxable income taxed at 10/12/22/24/32/35%." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA law throughout — $132,426 of miscellaneous itemized employee expenses, a $5,455 personal exemption, and the 10/15/25/28% rate schedule — none of which exist in 2026, and additionally subtracted the $8,389 ESI premium from the stated $300,000 of employment income. The correct computation is $300,150.19 of AGI less $17,090.34 of SALT, taxed under the 2026 single brackets." -us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Claimed a $25,000 overtime deduction; the 2026 deduction caps at $12,500 for a single filer and is reduced $100 per $1,000 of MAGI above $150,000, so a $300,150.19 MAGI eliminates it entirely. It also dropped the $5,708.82 alimony adjustment and used a $15,350 standard deduction rather than the $17,090.34 SALT itemized deduction." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Built regular taxable income on a TCJA sunset — subtracting the $8,389 ESI premium, $146,455.60 of itemized deductions including the repealed employee business expenses, and a $5,050 personal exemption — then used AMT to recover a number near the right magnitude. AMT does not bind: taxable income is $283,059.84 after the $17,090.34 SALT deduction, and regular tax of $67,834.07 plus $222.64 of NIIT is the answer." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Applied the repealed $10,000 SALT cap to the $13,834 of real estate taxes and denied the $5,708.82 alimony deduction, so it taxed $305,859 of AGI less a standard deduction instead of $300,150.19 less $17,090.34 of itemized SALT. Its $68,630 also falls below the $70,178.70 that its own described inputs produce under the 2026 single schedule." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA individual provisions expire and allowed the $138,375 of unreimbursed employee business expenses as a 2%-floor miscellaneous itemized deduction, then used the resulting tentative minimum tax as the liability. The 2026 rules keep that deduction repealed, so no AMT arises and regular tax on $283,059.84 of taxable income plus $222.64 of NIIT governs." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premium to reach a $291,761 AGI and claimed $146,570 of itemized deductions built on the repealed employee business expenses, then applied AMT as a limiter. The deductible total is $17,090.34 of SALT against a $300,150.19 AGI, giving $283,059.84 of taxable income and $67,834.07 of regular tax before the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its stated inputs — $305,859 of income less a standard deduction — produce $70,178.70 under the 2026 single rate schedule, while its reported $50,568 corresponds to taxable income near $231,000, roughly $60,000 of deduction it never identifies. The correct path deducts $5,708.82 of alimony and itemizes $17,090.34 of SALT to reach $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premium from wages and itemized the $138,375 of unreimbursed employee business expenses above a 2% AGI floor along with personal exemptions, all of which are repealed for 2026. The only itemized deduction is $17,090.34 of SALT, leaving $283,059.84 of taxable income rather than the roughly $145,000 its deduction stack implies." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Assumed a 2026 TCJA sunset and deducted the ESI premium from wages, the $138,375 of employee business expenses above a 2% AGI floor, and a personal exemption. Miscellaneous itemized deductions and personal exemptions do not exist in 2026, so taxable income is $283,059.84 after the $17,090.34 SALT deduction, not the roughly $146,000 its number implies." -us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Denied the $5,708.82 above-the-line alimony deduction and applied the 2025 $15,000 standard deduction with 2025 brackets rather than the 2026 $16,100 figure and OBBBA schedule, never testing the $17,090.34 SALT itemized deduction that exceeds it. It also omitted the $222.64 net investment income tax." -us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Deducted $132,372 of unreimbursed employee business expenses as a 2%-floor miscellaneous itemized deduction plus a personal exemption under pre-TCJA brackets, on the belief the TCJA expired in 2026. Both are permanently repealed; with only $17,090.34 of SALT deductible, taxable income is $283,059.84 and the 2026 rate schedule applies." -us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Denied the $5,708.82 above-the-line alimony deduction and counted only $13,834 of real estate taxes as SALT, so its $14,030 itemized total lost to the $16,100 standard deduction. The actual SALT deduction of $17,090.34 includes $3,256.34 of state and local sales tax deductible in lieu of a Texas income tax and wins; those two omissions overstate taxable income by exactly $6,699.16." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that mortgage interest, SALT, medical, and unreimbursed employee business expense deductions offset all federal taxable income. Mortgage interest is 0 because only a balance is listed, the $440 of medical costs falls under the 7.5% AGI floor, employee business expenses are repealed, and $17,090.34 of SALT is the sole itemized deduction, leaving $283,059.84 of taxable income and $68,056.71 of tax." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Gave no derivation, and its $31,472 matches the TCJA-sunset path of deducting the $138,375 of unreimbursed employee business expenses above a 2% AGI floor plus a personal exemption at pre-TCJA rates. Those deductions are repealed for 2026; taxable income is $283,059.84 after the $17,090.34 SALT deduction." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Took the $16,100 standard deduction because it treated the $13,834 of real estate taxes as the entire SALT deduction, omitting the $3,256.34 of state and local sales tax that raises SALT to $17,090.34 and makes itemizing larger. Its AGI, phased-out overtime deduction, qualified-dividend rate, and $222.64 NIIT all match the reference; the $990.34 of lost deduction at 35% is the whole $347 shortfall." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 employer-sponsored insurance premium from income to reach $291,761 — the listed $300,000 is the taxable employment income and the ESI premium is not a further adjustment — and took the $16,100 standard deduction rather than the $17,090.34 SALT itemized deduction. Its $65,244.40 is exactly the 2026 tax on the resulting $275,661 of taxable income, and it omitted the $222.64 NIIT." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Counted only $13,834 of real estate taxes as SALT and so took the $16,100 standard deduction plus a $196 non-itemizer charitable deduction, missing the $3,256.34 of state and local sales tax that lifts the itemized SALT deduction to $17,090.34. It also omitted the $222.64 net investment income tax that the reference adds." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Left the $5,708.82 alimony deduction out of AGI and applied the $16,100 standard deduction instead of the $17,090.34 SALT itemized deduction, taxing $289,759 rather than $283,059.84. It also omitted the $222.64 net investment income tax." -us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction because it never credited the $3,256.34 of state and local sales tax that, with $13,834 of real estate taxes, makes SALT $17,090.34 and itemizing the larger deduction. It also omitted the $222.64 net investment income tax on the $5,859 of investment income." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Reported a round $65,000 without applying the rate schedule at all; the standard-deduction path it sketches yields $68,403.34 and the correct itemized path yields $67,834.07 of regular tax plus $222.64 of NIIT. It never computed the $17,090.34 SALT deduction that makes itemizing the better choice against the $16,100 standard deduction." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Deducted $132,372 of employee business expenses after a 2% AGI floor and a $5,427 personal exemption under pre-TCJA brackets inflated by a 1.34 factor, treating the TCJA as expired. Those provisions are repealed for 2026: the deduction set is $17,090.34 of SALT, taxable income is $283,059.84, and the 2026 single schedule applies." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Assumed TCJA sunset restored the miscellaneous itemized deduction for the $138,375 of employee expenses and uncapped real estate taxes, then treated $70,254 of AMT as the binding liability. Those deductions stay repealed in 2026, so no AMT arises on $283,059.84 of taxable income and regular tax of $67,834.07 plus $222.64 of NIIT governs." -us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premium from wages and itemized $146,456 built on the $138,375 of employee business expenses after a 2% AGI floor, then applied pre-TCJA 10/15/25/28% brackets. AGI is $300,150.19 after the alimony adjustment, only $17,090.34 of SALT is deductible, and the 2026 schedule tops out at 35% on $283,059.84 of taxable income." -us,scenario_020,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 ESI premium from wages, dropped the $5,708.82 alimony deduction, and applied a $196 above-the-line charitable deduction, landing on a $297,274 AGI instead of $300,150.19. It then took the $16,100 standard deduction rather than the $17,090.34 SALT itemized deduction, understating taxable income relative to $283,059.84 by the net of those errors." -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for federal_income_tax_before_refundable_credits, so no substantive computation exists to evaluate. The required answer is $68,056.71 — $67,834.07 of regular tax on $283,059.84 of taxable income plus $222.64 of net investment income tax." -us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Treated $13,834 of real estate taxes as the whole itemized amount and so took the $16,100 standard deduction plus a $196 non-itemizer charitable deduction; the SALT deduction is $17,090.34 once the $3,256.34 of state and local sales tax is included, so the filer itemizes to $283,059.84 of taxable income. Its AGI, phase-out of the overtime deduction, and $222.64 NIIT are otherwise correct." -us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Returned 0 with no derivation. A single Texas filer with $305,859 of income, a $5,708.82 alimony adjustment, and $17,090.34 of SALT deductions has $283,059.84 of taxable income and owes $67,834.07 of regular tax plus $222.64 of NIIT; zero is reachable only by wiping out the entire wage base." -us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Identified the $40,400 2026 SALT cap correctly but counted only $13,834 of SALT, missing the $3,256.34 of state and local sales tax that lifts the itemized total to $17,090.34 above the $16,100 standard deduction, and it denied the $5,708.82 alimony deduction. It taxed $289,759 instead of $283,059.84 and omitted the $222.64 net investment income tax." -us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Added the $42,857 FLSA overtime premium on top of $300,000 of wages, double-counting a component already inside the stated wage total, applied the repealed $10,000 SALT cap, and used a $15,750 standard deduction instead of the $17,090.34 itemized SALT. Its own arithmetic produced $85,241.65 while it submitted $68,513.50." -us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"Applied married-filing-jointly brackets, a qualified business income deduction, and a child tax credit to a single filer with no business income and no dependents. Taxable income is $283,059.84 under the single schedule after the $5,708.82 alimony adjustment and $17,090.34 of SALT, with no nonrefundable credits available." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"It counted only the $13,834 real estate tax as SALT and so used the $16,100 standard deduction, for taxable income of $284,050. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax raises itemized SALT to $17,090.34, and the extra ~$990 of taxable income at 35% accounts for its ~$347 gap to that reference. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% is nearly all of its $69.27 gap to the $68,334.73 the conventions give with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 nonitemizer charitable deduction, for taxable income of $283,854. Under the release's sales tax convention (c_irs_sales_tax_2025) that is the household's choice, because the IRS 2025 table leaves SALT below the standard deduction; the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84, and the $794 difference at 35% explains its $278 gap to that reference. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"It subtracted the $138,375 of unreimbursed employee business expenses above the line, but those are nondeductible miscellaneous itemized deductions under current law. It also used a $15,000 standard deduction. It then submitted $66,661, a figure disconnected from its own $27,268 bracket computation, instead of taxing $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), and adding NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 of wages, although gross wages already include overtime. It capped SALT at $10,000 instead of the $40,400 2026 cap and ignored the sales tax table deduction; neither changes its $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but it left out the $196 nonitemizer charitable deduction. It also omitted NIIT and then submitted $70,813, which does not match its own $82,743 computation." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"It disallowed the alimony deduction and invented $23,855 of mortgage interest from the loan balance, even though no interest was listed. It capped SALT at $10,000 and used 2025 brackets. It then submitted $71,866, which does not match its own ~$64,291 sum." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"It guessed itemized deductions of $30,000–40,000 with no basis. Itemized deductions are SALT alone: $13,834 of real estate tax plus the sales tax table amount, which the frozen reference projects for $17,090.34 in all; the release's sales tax convention (c_irs_sales_tax_2025) takes the IRS 2025 table, whose $1,595 for this filer leaves SALT below the $16,100 standard deduction. Its $72,700 answer is an unsupported estimate rather than tax on $283,854.19 of taxable income under the convention ($283,059.84 in the frozen reference) plus $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 nonitemizer charity deduction, for taxable income of $283,854. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction even with Texas's general sales tax deduction added to the $13,834 of property tax; only the frozen reference's projected sales tax brings SALT to $17,090.34 and lowers taxable income to $283,059.84. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction and capped SALT at $10,000 instead of $40,400. It used a projected $15,400 standard deduction and stale bracket projections instead of the $16,100 deduction and actual 2026 brackets. It also left out the $222.64 NIIT, and its $70,398 reflects all of these errors." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"It used the 2025 brackets and $15,000 standard deduction instead of 2026 values and capped SALT at $10,000 instead of $40,400. It also added $900 of Additional Medicare Tax, which is a payroll tax and not part of income tax. It then inflated its own $70,467 to an arbitrary $76,500." +us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"It treated alimony as nondeductible, which left AGI at $305,859 instead of $300,150. It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the better choice), but not the $196 nonitemizer charitable deduction. Its taxable income of $289,759 is $5,904.81 above the convention's $283,854.19 and $6,699.16 above the frozen reference's $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"It applied pre-TCJA sunset law, deducting $132,258 of unreimbursed employee expenses above a 2% floor and claiming a personal exemption. OBBBA permanently eliminated both, so the correct figure is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"It applied pre-TCJA rules, including miscellaneous itemized deductions for employee expenses, a personal exemption, and 15/25/28% brackets, even though OBBBA made the TCJA structure permanent. It also subtracted the ESI premiums from the listed gross wages and dropped the alimony deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"It took a $25,000 overtime deduction without applying the single-filer $12,500 cap and the $100-per-$1,000 phaseout above $150,000 MAGI. That phaseout eliminates the deduction entirely at $300k. It also dropped the alimony deduction and omitted the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"It deducted $132,258 of unreimbursed employee expenses as miscellaneous itemized deductions and claimed a $5,450 personal exemption under pre-TCJA brackets, although OBBBA permanently suspended both. It also omitted the alimony deduction and NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"It assumed TCJA had expired, allowed miscellaneous itemized employee expenses and a personal exemption, and then substituted an AMT computation. It also subtracted the ESI premiums from AGI. Under current law, regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or on $283,059.84 in the frozen reference, which itemizes a projected sales tax." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"It used AGI of $305,859, ignoring the $5,709 alimony deduction, and capped SALT at $10,000 instead of the 2026 $40,400 cap. Not itemizing is still the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction under the $40,400 cap; only the frozen reference's projected sales tax lifts real estate plus sales tax to $17,090.34, above it." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"It assumed the TCJA individual provisions expire in 2026, deducted employee business expenses over the 2% floor, and then applied AMT. OBBBA made the suspension of miscellaneous itemized deductions permanent, so the answer is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"It subtracted the ESI premiums from AGI and itemized $146,570, including the permanently suspended miscellaneous employee expenses. It then relied on an AMT computation instead of regular tax on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), or $283,059.84 in the frozen reference, which itemizes a projected sales tax, plus NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"It skipped the alimony deduction, used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax), and left out NIIT. Its $50,568 is far below even the bracket tax on its own taxable income, which shows a computation error." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"It applied pre-TCJA law: uncapped SALT, miscellaneous employee expenses above the 2% floor, and personal exemptions. It also subtracted the ESI premiums from wages. Current law permanently disallows the employee expenses and exemptions, so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax)." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"It assumed the TCJA sunset and itemized the unreimbursed employee expenses with a personal exemption. It also subtracted the ESI premiums and dropped alimony. Under permanent OBBBA law those employee expenses are nondeductible, and taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax)." +us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"It omitted the $5,709 alimony deduction and used the 2025 $15,000 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT, property plus a projected sales tax table amount). It also left out the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"It treated 2026 as a TCJA-sunset year, deducted $132,372 of employee business expenses as miscellaneous itemized deductions, and took a personal exemption. OBBBA permanently suspends both. It also omitted NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"It declared the alimony expense nondeductible, leaving AGI at $305,859 instead of $300,150. It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax lifts itemized SALT to $17,090.34, above $16,100), but not the $196 nonitemizer charitable deduction." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"It treated the $138,375 of unreimbursed employee expenses and unlisted mortgage interest as deductions that wipe out all tax. Employee expenses are nondeductible under current law, and no interest was given. Correct itemized deductions are only SALT, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction leaves $68,112.09 of regular tax plus NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its ~$31,472 answer matches pre-TCJA treatment that deducts the unreimbursed employee business expenses as miscellaneous itemized deductions. Those are permanently disallowed. Itemized deductions are SALT alone, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction gives tax of $68,112.09 plus $222.64 NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834.07 plus NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"It used the $16,100 standard deduction, for taxable income of $284,050, because it compared only the $13,834 of property tax against it. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% matches its $68.46 gap to the $68,334.73 the conventions give with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from the listed gross wages, which understates income. It used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax), but not the $196 nonitemizer charitable deduction, and it omitted the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT (property plus a projected sales tax). It also left out the $222.64 net investment income tax, which the frozen reference includes in this total; its $68,112.00 is $0.09 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"Its $70,179 matches tax on $289,759 of taxable income. That figure ignores the $5,709 alimony deduction and uses the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. It also omits the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"It used the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference's projected sales tax makes itemized SALT of $17,090.34 larger. It also left out the $222.64 NIIT, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"It computed tax on taxable income of $289,759, which drops the $5,709 above-the-line alimony deduction. It also takes the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of real estate plus a projected sales tax." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"It started from $305,859 of gross income with no alimony deduction and used the $16,100 standard deduction. The alimony deduction lowers AGI to $300,150, and the nonitemizer charitable deduction adds $196 to the standard deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's itemized SALT of $17,090.34 (including a projected sales tax table amount) exceeds it." +us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. It also omitted the $222.64 net investment income tax, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"It gave a rounded guess of ~$65k with no computation. It did not deduct alimony, add the $196 nonitemizer charitable deduction to its standard deduction, tax the resulting $283,854.19 at 2026 brackets, or add the $222.64 NIIT; its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax and taxes $283,059.84." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"It applied pre-TCJA law, deducting $132,372 of miscellaneous employee expenses and a personal exemption under reverted brackets. OBBBA permanently suspended both, so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or $283,059.84 in the frozen reference, which itemizes a projected sales tax, not $148,321." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"It assumed the TCJA sunset, deducted the employee expenses and uncapped SALT, and then fell back to AMT. Under current permanent law the employee expenses are nondeductible and regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025) or $283,059.84 in the frozen reference, which itemizes a projected sales tax." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"It used the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the larger deduction), but it left out the $196 nonitemizer charitable deduction, for taxable income of $284,050 instead of the convention's $283,854.19. It also omitted the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,It applied pre-TCJA brackets and deducted the unreimbursed employee expenses above a 2% floor. It also subtracted the ESI premiums from wages and dropped alimony. OBBBA permanently disallows miscellaneous itemized deductions. +us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"It subtracted the $8,389 of ESI premiums from gross wages, took the $196 of charity above the line rather than as the nonitemizer charitable deduction, which lowers taxable income by the same $196, and skipped the $5,709 alimony deduction. Its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. Offsetting errors landed it at $67,479." +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was no answer to score." +us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"It treated the itemized amount as $13,834 without the sales tax and took the standard deduction plus the $196 nonitemizer charity deduction, which is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025): the IRS 2025 table leaves SALT below the standard deduction either way. Only the frozen reference's projected sales tax raises itemized SALT to $17,090.34 and lowers taxable income to $283,059.84; its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"It submitted $0 with no reasoning. The correct derivation under the release's sales tax convention (c_irs_sales_tax_2025) is AGI of $300,150 after alimony, less the $16,100 standard deduction and the $196 nonitemizer charitable deduction, for taxable income of $283,854.19, tax of $68,112.09 and NIIT of $222.64; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $283,059.84 and $67,834.07. A zero is consistent only with treating the $138,375 of employee expenses as fully deductible." +us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"It declared alimony nondeductible, for AGI of $305,859, and took the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's projected sales tax, added to property tax, makes itemizing larger. It also omitted the $222.64 NIIT." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"It added the $42,857 overtime premium on top of the $300,000 of gross wages, capped SALT at $10,000 and used outdated brackets. It then submitted $68,513.50, which contradicts its own computed $85,241.65." +us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"It applied married-filing-jointly brackets, a qualified business income deduction and a child tax credit to a single, childless wage earner with no business income. None of those apply." us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_020,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model derived the reference exactly ($11,439 + $4,350 + $900 = $16,689) and then overrode it by deducting the $8,389 employer-sponsored insurance premium from Medicare wages, dropping to $16,491.86; PolicyEngine's payroll tax base is the stated gross wage of $300,000, unreduced by ESI premiums. It then submitted $15,200, a figure matching neither its correct derivation nor its own revised $16,491.86." us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500, added the $42,857 FLSA overtime premium on top of the $300,000 gross wage even though the prompt states gross wages already include overtime, and further deducted the $8,389 ESI premium from FICA wages. After cycling through $15,703.20, $16,710.34, and $16,513.19 it submitted $23,119.50, which none of its own calculations produce." @@ -1302,396 +1471,454 @@ us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract. us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits." us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Reached the single schedule, AGI of $109,149, the $3,951 phased senior deduction and the $910 QBI deduction correctly, then built SALT from the $8,829 real estate tax alone under an obsolete $10,000 cap; the 2026 cap is $40,400 and SALT also carries CA income tax and SDI, totaling $14,807.14. It further deducted the full $15,393 of gifts rather than $14,847.67 after the 0.5%-of-AGI charitable floor, leaving taxable income $5,433 too high." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Filed on the qualifying-surviving-spouse joint schedule with a $33,850 joint standard deduction plus a $2,000 non-itemizer charitable deduction; the filer uses the single schedule and itemizes $29,654.81, which far exceeds the $18,150 single standard deduction. It also took the senior deduction at the full $6,000 instead of the $3,951.05 remaining after the 6% phase-down on MAGI above $75,000." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Put the entire $20,735 of Social Security into gross income instead of the 85% maximum inclusion of $17,624.75, then subtracted the $8,351 of unreimbursed employee business expenses as an above-the-line adjustment even though §67(g) permanently disallows miscellaneous itemized deductions. It compounded that with head-of-household status and a $27,900 standard deduction in place of single filing with $34,515.86 of deductions." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"Applied the qualifying-surviving-spouse joint schedule and a $33,950 joint standard deduction, rejecting itemization because it counted only the $8,829 property tax as SALT; with CA income tax, SALT is $14,807.14 and itemized deductions reach $29,654.81 against the $18,150 single standard deduction. It also omitted the $3,951.05 senior deduction entirely." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Imputed roughly $12,771 of mortgage interest from the $212,853 loan balance, though unlisted numeric inputs are zero and the reference itemizes no mortgage interest. It also used joint surviving-spouse brackets and the unphased $6,000 senior deduction rather than $3,951.05, and its submitted $6,919 does not match the $7,350 its own arithmetic produced." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Limited SALT to the $8,829 property tax and deducted gross charitable gifts without the 0.5%-of-AGI floor, understating deductions by $5,433, and used the senior deduction at $6,000 instead of the $3,951.05 left after the 6% phase-down. Its own stated taxable income of about $78,017 produces $11,876 on the 2026 single schedule, not the $8,547 submitted." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Assumed TCJA sunset in 2026 and rebuilt the return on restored personal exemptions (~$10,600), miscellaneous itemized deductions above a 2% floor ($6,168), pre-TCJA 15%/25% rates, and imputed mortgage interest — every one of which OBBBA's permanent extension forecloses. Its submitted $13,013 also matches none of its own computations, which ranged from $5,920 to $7,205." -us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Chose a $32,600 surviving-spouse standard deduction over itemizing because it capped SALT at $10,000 and left out CA income tax; on the single schedule the $29,654.81 itemized total beats the $18,150 standard deduction, and the $3,951.05 senior deduction and $910 QBI come on top. Its submitted $6,800 also contradicts the roughly $8,700 its own explanation derived." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"Took a $32,500 surviving-spouse standard deduction over itemized deductions it put at $24,222, having counted only the $8,829 property tax as SALT and applied no 0.5% charitable floor; the correct itemized total is $29,654.81 against an $18,150 single standard deduction. It also omitted the $3,951.05 senior deduction and the $910 QBI deduction and used 2025 brackets." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied a repealed regime: a $5,300 personal exemption, unreimbursed employee expenses above a 2% floor, and pre-TCJA 10/15/25% rates. For 2026 personal exemptions are zero, §67(g) still suspends miscellaneous itemized deductions, and the single schedule runs 10% to $12,400, 12% to $50,400, then 22%." -us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Deducted a $5,200 personal exemption and $6,168 of miscellaneous employee expenses above a 2% floor, both permanently unavailable in 2026, and omitted the $3,951.05 senior deduction, the $910 QBI deduction, and the CA income tax that lifts SALT to $14,807.14. Its explanation states $12,372 while the submitted value is $13,047.69." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Subtracted a $5,000 personal exemption and unreimbursed employee expenses above a 2% floor and applied 10/15/25% rates — the pre-TCJA regime, all repealed for 2026. It also left the CA income tax out of SALT and skipped the $3,951.05 senior deduction, so its $30,390 of itemized deductions falls short of the $34,515.86 total." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $11,335 equals the reference computation with the $910 §199A QBI deduction on the $4,550 of rental income omitted: taxable income of $75,543 instead of $74,633.30 yields $11,331 at the 22% marginal rate. Its explanation also claims a mortgage interest deduction, which is zero because no interest amount is listed." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied pre-TCJA law to 2026 — personal exemptions plus roughly $33,297 of sunset-era itemized deductions including suspended employee business expenses — when personal exemptions remain zero and §67(g) still bars those expenses. It also omitted the $3,951.05 senior deduction and the $910 QBI deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a $5,050 personal exemption and $6,168 of unreimbursed employee expenses above a 2% floor under post-sunset rates, none of which exist in 2026, and priced the CA income tax inside SALT at $2,172 against the actual $5,978. It also skipped the $3,951.05 senior deduction, the $910 QBI deduction, and the 0.5% charitable floor." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; its $13,446 corresponds to taxable income near $85,000, which is AGI less roughly $24,000 — property tax plus gross charitable gifts and nothing else. The reference deducts $34,515.86: SALT of $14,807.14 including CA income tax, charity of $14,847.67 after the 0.5% floor, the $3,951.05 senior deduction, and $910 of QBI." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Padded SALT up to an obsolete $10,000 cap when the 2026 cap is $40,400 and actual SALT is $14,807.14, deducted gross charity with no 0.5%-of-AGI floor, and took no senior deduction, leaving taxable income of $82,846 against the correct $74,633.30. Its own stated taxable income also yields $12,938 on the 2026 single schedule, not the $13,140 submitted." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $9,260 equals the reference computation with about $8,400 of extra deductions — the size of the $8,351 of unreimbursed employee business expenses, which §67(g) permanently disallows. The correct stack is $29,654.81 itemized plus the $3,951.05 senior deduction and $910 QBI, giving $74,633.30 of taxable income and $11,131.33 of tax." -us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Included unreimbursed employee business expenses in itemized deductions, a category permanently suspended for 2026, while omitting the $3,951.05 senior deduction, the $910 QBI deduction, and the CA income tax that brings SALT to $14,807.14. Those offsetting errors leave taxable income near $84,000 against the correct $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Reached single filing status and the $910 QBI deduction but took no senior deduction, counted only the $8,829 property tax as SALT, deducted gross charity with no 0.5%-of-AGI floor, and priced brackets and the standard deduction at 2025 levels. Correct 2026 treatment is SALT $14,807.14, charity $14,847.67, and a $3,951.05 senior deduction, for $74,633.30 of taxable income." -us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,351 of unreimbursed employee expenses as an adjustment, added the $371 of tax-exempt pension into AGI, and took a $17,000 standard deduction instead of $29,654.81 of itemized deductions plus the $3,951.05 senior deduction and $910 QBI. It then subtracted $5,850 of assumed withholding, reporting a balance due rather than the liability the output asks for." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted there is no taxable wage income and that deductions offset the tax to zero; the return carries $52,635 of wages, $30,989.41 of taxable pension, and $17,624.75 of taxable Social Security. After the $34,515.86 of deductions, $74,633.30 of taxable income remains and the 2026 single schedule produces $11,131.33, with no nonrefundable credit available to this filer." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Submitted $1,237 with no derivation, roughly a ninth of the liability. AGI is $109,149.16, deductions total $34,515.86 (SALT $14,807.14, charity $14,847.67 after the 0.5% floor, senior $3,951.05, QBI $910), and the 2026 single schedule on $74,633.30 gives $11,131.33 with no nonrefundable credits." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used the surviving-spouse joint rate schedule and subtracted a personal exemption, which is zero for 2026, while stopping at about $30,390 of itemized deductions with no $3,951.05 senior deduction and no $910 QBI. The single schedule on $74,633.30 of taxable income gives $11,131.33." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Named the right components but overstated deductions by about $4,000: its $10,247 requires taxable income of $70,614 against the reference $74,633.30. The senior deduction is worth exactly $3,951.05 after the 6%-of-excess-MAGI phase-down and stacks once on the $29,654.81 itemized total plus $910 of QBI." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Applied the surviving-spouse joint rate schedule together with the age-adjusted joint standard deduction; this filer uses the single schedule and itemizes $29,654.81, which beats the $18,150 single standard deduction. Those two substitutions account for the entire $3,420 shortfall against $11,131.33." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Ran the surviving-spouse joint bracket schedule over taxable income built from property tax and gross charity alone; the single schedule governs (10% to $12,400, 12% to $50,400, 22% above) and SALT includes $5,978 of CA income tax. Its $9,760 matches joint-rate treatment of roughly $84,900 of taxable income rather than single-rate treatment of $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Took the $33,850 age-adjusted joint standard deduction on the surviving-spouse joint schedule; the correct treatment is the single schedule with $29,654.81 of itemized deductions, which exceeds the $18,150 single standard deduction. Its $3,951.08 senior deduction and $910 QBI were right, so filing status and the standard-versus-itemized choice carry the full $3,175 shortfall." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"Applied a flat $30,000 qualifying-surviving-spouse standard deduction on joint brackets, skipping itemization, the $3,951.05 senior deduction, and the $910 QBI deduction. The single schedule with $34,515.86 of total deductions gives $74,633.30 of taxable income and $11,131.33 of tax." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed TCJA sunset — a $5,419 personal exemption and $6,168 of miscellaneous employee expenses above a 2% floor — both permanently unavailable in 2026, and capped SALT at $10,932 instead of the uncapped $14,807.14. It also took neither the $3,951.05 senior deduction nor the $910 QBI deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Got the single schedule, the $3,951 senior deduction and the $910 QBI right but estimated the CA income tax inside SALT at $1,974 against the actual $5,978, and deducted gross charitable gifts instead of $14,847.67 after the 0.5%-of-AGI floor. Net of those two errors its taxable income of $78,092 exceeds the correct $74,633.30 by $3,459." -us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Taxed the full $20,735 of Social Security rather than the $17,624.75 that the 85% maximum inclusion allows, restored miscellaneous employee expenses above a 2% floor, and applied pre-TCJA 10/15/25% brackets, none of which apply in 2026. It also omitted the $3,951.05 senior deduction and the $910 QBI deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Applied the 0.5% charitable floor correctly but put SALT at $10,853 by pricing the CA income tax far below the $5,978 inside the $14,807.14 total, subtracted an unlisted $901 of rental depreciation from AGI, and dropped the $910 QBI deduction. Those errors leave taxable income of $78,538 against the correct $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so nothing substantive reached the scorer against the $11,131.33 reference. The failure is contractual — the required output key was absent — not a computation error." -us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Priced the CA income tax inside SALT at $2,700 against the actual $5,978, deducted gross charitable gifts with no 0.5%-of-AGI floor, and explicitly declined the $910 QBI deduction on the $4,550 of rental income. Its single schedule and $3,951.08 senior deduction were right, so taxable income landed $3,642 above the correct $74,633.30." -us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed taxable income is too low to generate any liability; with $109,149.16 of AGI against $34,515.86 of deductions, taxable income is $74,633.30. The 2026 single schedule produces $11,131.33 before refundable credits, and no nonrefundable credit is available to this filer." -us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"Stacked a $32,200 joint standard deduction, a $1,650 age addition, and an unphased $6,000 senior deduction on the surviving-spouse joint schedule, and denied QBI on the rental income. Correct treatment is the single schedule with $29,654.81 itemized, the senior deduction phased down to $3,951.05, and $910 of QBI." -us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Double-counted the taxable Social Security, reporting AGI of $126,773.50 when its own listed components sum to $109,148.75, and imputed $12,771 of mortgage interest from the loan balance although no interest amount is listed. It also used the joint surviving-spouse schedule instead of the single one and took no senior or QBI deduction." -us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Swept non-deductible items — over-the-counter health costs, unreimbursed employee business expenses, sub-floor medical expenses, and imputed mortgage interest — into itemized deductions to drive taxable income negative. Allowed itemized deductions are $29,654.81, and after the $3,951.05 senior deduction and $910 QBI, $74,633.30 of taxable income yields $11,131.33 with no nonrefundable credits." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model filed single and correctly applied the $3,951 senior deduction and $910 QBI deduction. Its SALT deduction included only the $8,829 of property tax, leaving out the roughly $6,059 of California income tax that counts under the $40,400 cap. It also skipped the 2026 0.5%-of-AGI charitable floor, so it itemized $24,222 instead of $29,735.52 and got taxable income of $80,066 instead of $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"The model used qualifying surviving spouse status (joint brackets and the $33,850 standard deduction), but that status requires a dependent child, so single filing applies. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 of MAGI, and it added a non-itemizer charitable deduction when itemizing $29,735.52 beats the $18,150 single standard deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included 100% of Social Security instead of the 85% taxable portion. It deducted the $8,351 of unreimbursed employee expenses above the line, although they are not deductible, and it used head-of-household status and brackets instead of single. It never itemized or took the senior and QBI deductions, and its submitted $11,316 does not even match the $10,564 it computed." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the head as a qualifying surviving spouse with MFJ brackets and a $33,950 standard deduction, but with no dependent child the correct status is single, with 22% applying above $50,400. It also omitted the phased $3,951 senior deduction, and it never counted CA income tax in SALT, which would have made itemizing ($29,735.52) the better choice." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used qualifying surviving spouse (MFJ) brackets even though no dependent child exists, so single status applies. It imputed $12,771 of mortgage interest from a balance when no interest paid was given, and it took the full $6,000 senior deduction without the phase-down. Its final $6,919 does not match its own $7,350 computation." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model itemized only $24,222, leaving the roughly $6,059 of CA income tax out of SALT and skipping the 0.5%-of-AGI charitable floor. It then took an unphased $6,000 senior deduction. On its own $78,017 of taxable income, the single brackets give about $11,876, so its $8,547 is an arithmetic error." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"The model applied qualifying surviving spouse joint brackets and the $33,850 standard deduction, but that status requires a dependent child and single status applies. Under single status, itemizing $29,735.52 (SALT including CA income tax, plus charity after the 0.5% floor) beats the $18,150 standard deduction, and the tax falls in the 22% bracket." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model assumed TCJA expired in 2026, bringing back personal exemptions, 2% miscellaneous deductions and 15%/25% brackets, but OBBBA made the TCJA rate and deduction structure permanent. It also used surviving spouse (MFJ) status without a qualifying child and imputed mortgage interest. Its submitted $13,013 matches none of its own intermediate results (about $5,920–$7,314)." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used MFJ/surviving spouse treatment with a $32,600 standard deduction, but with no dependent child the head files single. It skipped the senior and QBI deductions and left CA income tax out of SALT, and it applied the old $10,000 SALT cap instead of the 2026 $40,400 cap. Its final $6,800 is also below its own $8,700 estimate." +us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"The model filed as surviving spouse with joint brackets and the $33,850 standard deduction, but that status requires a dependent child, so single applies. It also took the full $6,000 senior deduction without the 6% phase-down above $75,000 of MAGI (which leaves $3,951.05) and omitted the $910 QBI deduction on rental income." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"The model used surviving spouse joint treatment (a $32,500 standard deduction and the 2025 MFJ $23,850 10% bracket), but the head has no dependent child and files single. It also omitted the phased $3,951 senior deduction, the $910 QBI deduction and the CA income tax component of SALT." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied pre-TCJA law for 2026: a $5,300 personal exemption, 2%-floor unreimbursed employee expenses and 15%/25% brackets. OBBBA made the TCJA brackets permanent, kept miscellaneous itemized deductions suspended, and kept personal exemptions at zero. It also estimated CA income tax at only $2,019 instead of about $6,059 and skipped the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used expired pre-TCJA rules: a $5,200 personal exemption, $6,168 of 2%-floor employee expenses and old 15%/25% single brackets, instead of 2026 OBBBA law, which has no exemptions, no miscellaneous itemized deductions, and 10/12/22% brackets. It skipped the senior and QBI deductions, and its submitted $13,047.69 contradicts its own stated $12,372." +us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model itemized only property tax and charity ($24,222), leaving out the roughly $6,059 of CA income tax and the 0.5% charitable floor. It omitted both the $3,951.05 phased senior deduction and the $910 QBI deduction, so it taxed $84,926.75 instead of $74,552.59 using bracket widths that do not match the 2026 single schedule." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA 2026 law: a $5,000 personal exemption, a deduction for unreimbursed employee expenses above a 2% floor, and 10%/15%/25% brackets. Under OBBBA, exemptions stay at zero, miscellaneous itemized deductions stay suspended, and the 10/12/22% brackets apply. It also left CA income tax out of SALT and skipped the $3,951 senior deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model said it itemized mortgage interest, but no interest paid is given, only a loan balance. It never mentioned the CA income tax SALT component, the 0.5% charitable floor, the $3,951.05 phased senior deduction or the $910 QBI deduction. Its $11,335 does not follow from the correct $74,552.59 of taxable income taxed at single rates." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used pre-TCJA itemized deductions (about $33,297, including 2%-floor employee expenses) plus personal exemptions, even though OBBBA permanently suspended both for 2026. It also left out the senior and QBI deductions that reduce taxable income to $74,552.59 under the 10/12/22% single brackets." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model computed tax under TCJA-expiration rules: a $5,050 personal exemption, $6,168 of 2%-floor employee expenses and post-sunset rates. OBBBA kept the TCJA brackets, zero exemptions and the suspension of miscellaneous deductions. It also estimated CA income tax at $2,172 instead of about $6,059 and omitted the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $13,446 corresponds to about $85,150 of single-filer taxable income, which is AGI minus only property tax and charity. That means it left out the CA income tax SALT component, the $3,951.05 phased senior deduction and the $910 QBI deduction, which together bring taxable income down to $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied the old $10,000 SALT cap instead of the 2026 OBBBA cap of $40,400, under which the full $14,887.85 of property plus CA income tax is deductible, and it skipped the 0.5% charitable floor. It also omitted the $3,951.05 phased senior deduction, leaving taxable income at $82,846 instead of $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model's $9,260 is consistent with joint/surviving-spouse brackets (a 12% top bracket) applied to about $81,300 of taxable income. The head has no dependent child and files single, where income above $50,400 is taxed at 22%. It also never mentioned the CA income tax SALT component or the phased senior deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model itemized unreimbursed employee business expenses, which are permanently nondeductible under OBBBA. It omitted the $3,951.05 phased senior deduction and the $910 QBI deduction, and it did not capture the roughly $6,059 of CA income tax in SALT, so it overstated tax at $13,200." +us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model filed single but used 2025 brackets and a $15,000 standard deduction. It left the roughly $6,059 of CA income tax out of SALT and skipped the 0.5% charitable floor, and it omitted the OBBBA senior deduction ($6,000 phased down to $3,951.05). As a result it taxed $84,016.75 instead of $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"The model subtracted $5,850 of federal withholding that does not appear in the facts, and withholding is never subtracted from tax before refundable credits in any case. It also treated the $8,351 of unreimbursed employee expenses as an above-the-line adjustment and added tax-exempt pension income to AGI. It used the standard deduction instead of the $29,735.52 of itemized deductions and omitted the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed deductions and credits wipe out all tax, but AGI is $109,149.16 and total deductions are only $34,596.57. That leaves $74,552.59 of taxable income, and no nonrefundable credit offsets it: the elderly credit is phased out by AGI and Social Security. Its assertion that there is no taxable wage income ignores the $52,635 of wages." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model's $1,237 implies only about $12,400 of taxable income, which means it deducted roughly $96,700 from $109,149.16 of AGI. The correct deductions are $29,735.52 itemized, $910 QBI and $3,951.05 senior, leaving $74,552.59 taxable in the 22% bracket, with no nonrefundable credits available." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint brackets, but with no dependent child the head files single. It also applied a pre-TCJA personal exemption, which OBBBA made permanently zero, and omitted the phased $3,951 senior deduction and the $910 QBI deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Under single rates, the model's $10,247 implies about $70,600 of taxable income, which means it claimed roughly $3,900 more than the correct $34,596.57 of total deductions. The correct stack is $29,735.52 itemized (with charity reduced by the 0.5% AGI floor), $910 QBI and the senior deduction phased down to $3,951.05, which leaves $74,552.59 taxable." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied surviving-spouse joint-rate treatment and a joint-level age-adjusted standard deduction, but that status requires a dependent child. Single status taxes income above $50,400 at 22%, and itemizing $29,735.52 beats the $18,150 single standard deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used the surviving-spouse (joint) tax schedule, but the head has no dependent child and files single. It also never mentioned the senior deduction or the QBI deduction, which reduce taxable income to $74,552.59 under the single brackets." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"The model's senior deduction ($3,951.08) and QBI deduction ($910) were correct, but it used qualifying surviving spouse status, which requires a dependent child, taking the $33,850 joint standard deduction and joint brackets. As a single filer it itemizes $29,735.52, and taxable income above $50,400 is taxed at 22%." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Under single rates, the model's $9,651 implies about $67,900 of taxable income, roughly $6,600 below the correct $74,552.59, so its itemized and senior/QBI deductions total well above the correct $34,596.57. It overstated itemized deductions beyond the $14,887.85 of SALT plus $14,847.67 of charity left after the 0.5% floor." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model filed single and phased the senior deduction correctly to $3,951, but it estimated CA income tax at only about $2,234 instead of about $6,059, which put SALT near $11,063 instead of $14,887.85. It also left out the $910 QBI deduction on rental income, so it taxed $78,742 instead of $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint brackets and the $33,850 joint age-adjusted standard deduction, but that status requires a dependent child and single status applies. As a single filer the head itemizes $29,735.52, and the 22% rate applies above $50,400, which gives $11,113.57 instead of $7,956.52." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model applied a $30,000 QSS standard deduction and QSS brackets, but the head has no dependent child and files single. It also omitted the itemized deductions ($29,735.52), the phased $3,951.05 senior deduction and the $910 QBI deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed TCJA sunset, deducting 2%-floor employee expenses and a $5,419 personal exemption under pre-TCJA single brackets. OBBBA permanently kept those deductions at zero and kept the 10/12/22% schedule. It also underestimated CA income tax at about $2,103 and omitted the senior and QBI deductions." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model's structure was correct: single status, itemizing, a $3,951 senior deduction and a $910 QBI deduction. However, it estimated CA income tax at only $1,974 instead of about $6,059 and skipped the 0.5% AGI charitable floor, which put taxable income at $78,092 instead of $74,552.59. It also used bracket amounts that do not match the 2026 single schedule." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated CA income tax at only $1,867 instead of about $6,059, which put SALT at $10,696 instead of $14,887.85. It dropped the $910 QBI deduction on rental income and used 2026 bracket amounts that are wrong ($12,225/$49,675 instead of $12,400/$50,400)." +us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model included 100% of Social Security in AGI instead of the 85% taxable portion, and it applied pre-TCJA law, with 2%-floor employee expenses and 15%/25% brackets, that OBBBA permanently displaced. It also skipped the senior and QBI deductions and underestimated CA income tax at $2,254." +us,scenario_022,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted about $901 of rental depreciation that is not in the facts, which lowered AGI and threw off the senior-deduction phase-down, and it estimated CA income tax at about $2,024 instead of about $6,059. It also omitted the $910 QBI deduction and used wrong 2026 single bracket thresholds." +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model's structure was correct (single status, the $40,400 SALT cap, a $3,951.08 senior deduction and correct brackets), but it estimated CA income tax at $2,700 instead of about $6,059 and skipped the 0.5% AGI charitable floor. It also declined the $910 QBI deduction on rental income, which gave $78,275.67 of taxable income instead of $74,552.59." +us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed taxable income is too low to owe tax, but AGI is $109,149.16. Even the most generous deductions ($29,735.52 itemized, $910 QBI and a $3,951.05 senior deduction) leave $74,552.59 taxable, which puts the head in the 22% bracket." +us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model filed as surviving spouse with MFJ rates and the $33,850 joint standard deduction, although no dependent child exists and single status applies. It took the full $6,000 senior deduction, wrongly treating $150,000 as the phase-out start instead of $75,000 of MAGI, and it denied the $910 QBI deduction on rental income." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model added the $17,624.75 of taxable Social Security twice, which inflated AGI to $126,773.50 instead of $109,149.16. It also imputed $12,771 of mortgage interest from the loan balance and applied qualifying surviving spouse brackets even though there is no dependent child." +us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted non-deductible items (employee business expenses, health premiums below the 7.5% floor, and invented mortgage interest) as itemized deductions and concluded taxable income was negative. The allowable itemized deductions are only $29,735.52, which leaves $74,552.59 of taxable income, and no nonrefundable credit offsets that tax." us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies." us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Applied California SDI at the stale 1.2% rate rather than the 1.3% employee contribution rate in effect for 2026, producing $631.62 of state tax instead of $684.26. It then compounded the rate error by submitting $4,557.61, a figure matching neither its own computed $4,658.20 nor any rate it oscillated between in its scratch work." -us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Computed only the two federal FICA components and omitted the mandatory California SDI employee contribution entirely, leaving out $684.26 of the $4,710.83 total. It also cited the $250,000 joint Additional Medicare Tax threshold for a surviving-spouse filer whose wages are nowhere near any threshold, showing it never reached the state-tax step the prompt asks for." -us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that 'no mandatory California state payroll taxes apply,' dropping the CA SDI employee contribution of 1.3% x $52,635 = $684.26 that PolicyEngine includes. It further mis-multiplied Medicare as $762.71 instead of $763.21 and then submitted $4,028.27 against its own stated total of $4,026.08." -us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly identified CA SDI as mandatory and uncapped but used the 2025 rate of 1.2% instead of the 2026 rate of 1.3%, computing $631.62 of state tax where the correct figure is $684.26. That single stale parameter accounts for the entire $52.63 shortfall from $4,710.83." -us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Recognized that CA SDI exists but deliberately excluded it, reading the prompt's enumeration as limited to Social Security and Medicare. The prompt explicitly requests mandatory employee state payroll taxes, and California SDI is a mandatory employee-side withholding at 1.3% of all wages, so dropping its $684.26 left the answer short by exactly that amount." -us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Claimed California has no mandatory employee income-based payroll tax, omitting the CA SDI employee contribution of 1.3% x $52,635 = $684.26. It also truncated its own $4,026.58 federal subtotal to $4,026 when submitting." -us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Applied California SDI at the 1.1% rate that took effect when the wage ceiling was removed in 2024, rather than the 1.3% rate for 2026, computing $578.99 instead of $684.26. Its structure was otherwise exactly right, so the stale rate is the whole $105.26 gap." -us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Characterized CA SDI as voluntary or employer-paid and excluded it; it is a mandatory employee withholding of 1.3% on all wages, worth $684.26 here. It then submitted $3,264.37 — essentially the Social Security component alone plus a dollar — while its own text states a total of $4,026.58." -us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Explicitly stated 'no state payroll tax included,' omitting the mandatory CA SDI employee contribution of $684.26 that the requested output covers. Its Medicare figure of $763.24 is also mis-multiplied; 1.45% of $52,635 is $763.21." -us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"Included CA SDI but priced it at the pre-2026 rate of 1.1%, yielding $579 instead of the $684.26 that 1.3% of $52,635 produces. Its whole-dollar rounding of the federal components adds further drift from $4,710.83." -us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used the 2025 CA SDI employee rate of 1.2% rather than the 1.3% rate applicable in 2026, computing $631.62 of state tax instead of $684.26. Everything else in its derivation matches the reference exactly." -us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Correctly noted that the CA SDI wage cap is removed but applied the older 1.1% rate instead of the 1.3% rate for 2026, computing $578.99 rather than $684.26. That stale rate is the sole source of the $105.26 shortfall." -us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Summed only the 6.2% Social Security and 1.45% Medicare components and never added the mandatory California SDI employee contribution of 1.3% x $52,635 = $684.26. The $4,027 submitted is the rounded federal FICA subtotal alone." -us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,Applied CA SDI at 1.1% for $579 instead of the 2026 employee rate of 1.3% for $684.26. Its identification of SDI as a mandatory employee tax was right; only the rate parameter is out of date. -us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a 1.1% CA SDI rate, producing $579 of state tax where the 2026 rate of 1.3% yields $684.26. The federal components were correct, so the stale SDI rate is the entire error." -us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Limited the payroll tax to federal Social Security and Medicare, omitting the mandatory California SDI employee contribution of $684.26. Its $4,027 is simply 7.65% of $52,635 rounded." -us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied CA SDI at 0.9%, the rate from the pre-2024 capped regime, computing $473.72 instead of the $684.26 that the 2026 rate of 1.3% on uncapped wages produces. That rate is more than two revisions out of date." -us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $4,500.29 less the correct federal FICA of $4,026.58 leaves $473.71, i.e. CA SDI computed at the obsolete 0.9% rate rather than the 1.3% employee rate for 2026 that yields $684.26. It also submitted a value one cent below the $4,500.30 its own implied arithmetic gives." -us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Stopped after the two federal components and never added California's mandatory employee SDI withholding of 1.3% x $52,635 = $684.26. Its $4,026.58 is exactly the federal-only subtotal." -us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,Correctly treated CA SDI as a mandatory uncapped employee payroll tax but applied the 2025 rate of 1.2% for $631.62 rather than the 2026 rate of 1.3% for $684.26. The $52.63 gap is entirely that rate. -us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Omitted the mandatory CA SDI employee contribution of $684.26, reporting only federal FICA. It then invented a 'PolicyEngine statutory rounding to the nearest whole dollar' to justify submitting $4,026 against its own $4,027 subtotal; no such rounding rule exists and the reference is reported to the cent." -us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Asserted there is 'no state employee payroll tax in CA,' dropping the mandatory SDI employee contribution of 1.3% on all wages, worth $684.26 here. Its $4,030 is a loose rounding of the federal-only $4,026.58." -us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Omitted the mandatory CA SDI employee contribution of $684.26 and then failed to execute even the federal calculation it described: 7.65% of $52,635 is $4,026.58, not the $3,308 it submitted. The correct derivation is $3,263.37 + $763.21 + $684.26 = $4,710.83." -us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated California SDI at 1.2% rather than the 1.3% employee rate for 2026, producing $631.62 of state tax instead of $684.26. Its treatment of SDI as mandatory and uncapped was correct; only the rate is stale." -us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied only the combined 7.65% federal employee FICA rate and omitted California's mandatory SDI employee contribution of 1.3% x $52,635 = $684.26. Its $4,027 is the rounded federal-only total." -us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Summed 6.2% and 1.45% on wages and never added the mandatory California SDI employee withholding of $684.26 that the requested output includes. The $4,027 submitted is federal FICA alone." -us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,Applied CA SDI at the 2025 rate of 1.2% for $632 instead of the 2026 employee rate of 1.3% for $684.26. Its structural treatment matched the reference; only the rate parameter was out of date. -us,scenario_022,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate, yielding $631.62 rather than the $684.26 produced by the 1.3% rate in effect for 2026. It correctly identified SDI as the only mandatory state employee payroll tax, so the stale rate is the entire $52.63 shortfall." -us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"Projected the CA SDI rate forward at 1.2% instead of the 1.3% employee rate applicable in 2026, computing $632 rather than $684.26. It also invoked a 2026 SDI wage base that does not exist, since California removed the SDI taxable wage ceiling." -us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,"Applied California SDI at 'about 1.2%' rather than the 1.3% employee contribution rate for 2026, producing $632 instead of $684.26. Rounding its federal components to whole dollars adds a small further drift from $4,710.83." -us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key was absent from the submitted outputs object. This is a contract failure rather than a substantive miscalculation." -us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"Correctly treated CA SDI as mandatory and uncapped but applied the 2025 rate of 1.2% for $631.62 instead of the 2026 rate of 1.3% for $684.26. It also cited a $184,500 Social Security wage base that does not bind at this wage level." -us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Computed only Social Security and Medicare and omitted the mandatory California SDI employee contribution of 1.3% x $52,635 = $684.26. Its $4,026 also truncates its own $4,026.58 federal subtotal." -us,scenario_022,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,Applied the mandatory CA SDI employee contribution at 1.2% for $632 rather than at the 2026 rate of 1.3% for $684.26. Every other component matched the reference. -us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% for $578.99 instead of the 2026 employee rate of 1.3% for $684.26, and additionally imposed a fictitious ~$153,164 SDI wage cap that California eliminated. The stale rate accounts for the full $105.26 shortfall." -us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Stated that 'no mandatory employee state payroll taxes apply' in California, omitting the SDI employee contribution of 1.3% on all wages, which is $684.26 here. Its $4,026.58 is precisely the federal-only FICA total." +us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). It then submitted $4,557.61, which does not match its own stated sum of $4,658.20 or any other figure in its reasoning." +us,scenario_022,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal OASDI and Medicare ($4,026.58). It left out California's mandatory employee SDI contribution of 1.3% x $52,635 = $684.26, which the output definition explicitly includes as a mandatory employee state payroll tax." +us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax, so it omitted the $684.26 employee SDI contribution (1.3% of wages). It also miscomputed Medicare as $762.71 instead of $763.21 and submitted $4,028.27, which does not match its own stated total of $4,026.08." +us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly included uncapped CA SDI but used a 1.2% rate ($631.62) instead of the 2026 employee SDI rate of 1.3% ($684.26), which leaves the total $52.64 short." +us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Recognized CA SDI but excluded it on the claim that the instructions cover only Social Security and Medicare. The output definition explicitly includes mandatory employee state payroll taxes, so the 1.3% SDI contribution of $684.26 belongs in the total." +us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and omitted the 1.3% CA SDI contribution ($684.26). It then truncated its own FICA total of $4,026.58 to $4,026." +us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Included uncapped CA SDI but applied the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26), which leaves the total $105.27 short." +us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said CA SDI is not a mandatory employee tax and omitted the 1.3% SDI contribution ($684.26). It then submitted $3,264.37, which is roughly the Social Security piece alone and contradicts its own FICA total of $4,026.58." +us,scenario_022,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Stated that no state payroll tax applies, so it omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also slightly miscomputed Medicare as $763.24." +us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"Included CA SDI but at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26). It also rounded each component to whole dollars, reaching $4,605." +us,scenario_022,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 employee rate of 1.3% ($684.26), which leaves the total $52.63 short." +us,scenario_022,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($631.62) instead of the 2026 employee rate of 1.3% ($684.26), which leaves the total $52.63 short." +us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,Correctly treated CA SDI as uncapped but used the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Summed only Social Security and Medicare, then rounded to $4,027. It omitted California's mandatory 1.3% employee SDI contribution ($684.26), which the output definition includes." +us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Included CA SDI at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Included CA SDI at the 2024 rate of 1.1% ($579) instead of the 2026 rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare ($4,027). It omitted California's mandatory 1.3% employee SDI contribution of $684.26." +us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Included CA SDI but at the 2023 rate of 0.9% ($473.72) instead of the 2026 rate of 1.3% ($684.26), which leaves the total $210.53 short." +us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $4,500.29 equals FICA of $4,026.58 plus CA SDI at the 2023 rate of 0.9% (about $473.71). It needed the 2026 SDI rate of 1.3%, which gives $684.26." +us,scenario_022,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare ($4,026.58). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)." +us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,Correctly treated CA SDI as a mandatory state payroll tax but used 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Counted only Social Security and Medicare and omitted California's mandatory 1.3% employee SDI contribution ($684.26). It then rounded to $4,026 by citing a whole-dollar rounding rule that does not apply to payroll tax." +us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Stated that California has no employee state payroll tax, which omits the mandatory 1.3% SDI contribution ($684.26). Its $4,030 also misstates the FICA sum, which is $4,026.58." +us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also mis-added its own FICA components: 6.2% plus 1.45% of $52,635 is $4,026.58, not $3,308." +us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% instead of the 2026 employee rate of 1.3%, which undercounts SDI by $52.63 ($631.62 vs. $684.26)." +us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied only the 7.65% combined FICA rate ($4,027 rounded). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)." +us,scenario_022,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% instead of the 2026 employee rate of 1.3%, and rounded the total to $4,658 instead of $4,710.83." +us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare ($4,027). It omitted California's mandatory 1.3% employee SDI contribution ($684.26)." +us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Correctly included mandatory CA SDI but at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26), then rounded to $4,658." +us,scenario_022,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,Correctly included mandatory CA SDI but at 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,"Used a projected CA SDI rate of 1.2% ($632) instead of the 2026 rate of 1.3% ($684.26). It also assumed a wage base, although CA SDI has had no wage cap since 2024." +us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,"Approximated CA SDI at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for payroll_tax, so there is no substantive computation to score against $4,710.83." +us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,Correctly treated CA SDI as mandatory and uncapped but applied 1.2% ($631.62) instead of the 2026 rate of 1.3% ($684.26). +us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Summed only Social Security and Medicare and omitted California's mandatory 1.3% employee SDI contribution ($684.26). It also truncated $4,026.58 to $4,026." +us,scenario_022,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"Included mandatory CA SDI but at 1.2% ($632) instead of the 2026 employee rate of 1.3% ($684.26), and rounded the components to whole dollars." +us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"Applied CA SDI at the 2024 rate of 1.1% ($578.99) instead of the 2026 rate of 1.3% ($684.26). It also cited a wage cap near $153,164, although CA SDI has no wage cap." +us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Stated that no mandatory employee state payroll taxes apply, which omits California's 1.3% employee SDI contribution of $684.26." us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Applied the joint-width surviving-spouse rate schedule (1% to ~16,900, 2% to ~40,000, 4% to ~63,200) when qualifying-surviving-spouse status requires a dependent child under IRC §2(a) and none is listed, so CA's single schedule applies and yields $2,752.58 on $67,847.75. It also loaded $6,521 of unreimbursed employee business expenses into CA itemized deductions, which the reference's $23,676.67 (charitable $14,847.67 + real estate taxes $8,829) excludes. It then derived roughly $980 in its own reasoning and submitted $3,162 with no computation supporting that number." -us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Ran the joint/surviving-spouse schedule (1% to $22,157, 2% to $52,528, 4% above) instead of California's single schedule, which by itself drops the bracket tax from $2,752.58 to $1,419.95 on essentially the same base. It also claimed three exemption credits totaling $459, where a single filer age 77 gets exactly two — one personal and one age-65 — worth $312.93 combined." -us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"Never applied California's Social Security subtraction, starting instead from a fabricated $134,529 of federal taxable income and a married-filing-separately $27,550 standard deduction rather than CA AGI of $91,524.41 less $23,676.67 of CA itemized deductions. It then invented a $5,320 nonrefundable credit from the Senior Property Tax Postponement program, which is a property-tax deferral loan administered by the State Controller, not a credit against California personal income tax; the only nonrefundable credits here are $312.93 of personal and age-65 exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"Used MFJ/surviving-spouse brackets and the ~$11,080 MFJ standard deduction; with no dependent child in the household, IRC §2(a) surviving-spouse status is unavailable and CA's single schedule produces $2,752.58 on $67,847.75 of taxable income rather than the ~$1,400 it computed. It also deducted mortgage interest that is not listed (unlisted inputs are 0) and doubled the exemption credit to $420 against the actual single-filer total of $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Applied joint-width surviving-spouse brackets and padded CA itemized deductions to ~$30,000 with an invented ~$7,000 of mortgage interest and a medical deduction, when no interest is listed and the $5,832 of medical costs falls below the 7.5%-of-AGI floor; the reference's itemized total is $23,676.67. Its own arithmetic ($1,228 gross less a $280 exemption credit = $948) contradicts the $1,856 it submitted." -us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Got the structure right — single schedule, CA AGI $91,524.41 after excluding Social Security, charitable plus real estate taxes only — but understated the rate-schedule tax, reporting about $2,393 where California's 2026 single brackets produce $2,752.58 on the reference's taxable income. Its taxable income was also $545 low because it deducted the full $15,393 of contributions instead of $14,847.67 after the 0.5%-of-AGI charitable floor, and its $298 of exemption credits uses stale amounts against the 2026 indexed $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Correctly excluded Social Security and correctly dropped employee business expenses, then applied the MFJ/surviving-spouse brackets (1% to $20,824, 2% to $49,368, 4% above) for a $1,496.48 gross tax; the single schedule that governs a filer with no dependent child yields $2,752.58 on $67,847.75. It compounded this by stacking a $444 MFJ personal exemption credit plus a $144 senior credit against the actual $312.93 of single personal-plus-age-65 credits." -us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Selected qualifying-widow(er) (joint-width) brackets and then abandoned the schedule for an assumed '~2.8% effective average' rate, landing on $1,900 where the single schedule on $67,847.75 gives $2,752.58 before credits. It also asserted the senior exemption credit is 'embedded in the bracket tables'; California grants it as a separate $156.47 nonrefundable exemption credit stacked on the personal one, $312.93 in total." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Carried the $17,624.75 of federally taxable Social Security straight into the California base, computing CA taxable income of $84,927 from federal AGI of $109,149 less $24,222 of itemized deductions. California excludes Social Security benefits entirely (R&TC §17087), the reference's $17,624.75 subtraction to CA AGI of $91,524.41, so its gross tax of ~$4,030 overshoots the correct $2,752.58." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Added $6,168 of unreimbursed employee business expenses net of the 2% floor to reach $30,390 of CA itemized deductions; the reference allows only charitable $14,847.67 plus real estate taxes $8,829 = $23,676.67, giving taxable income of $67,847.75 rather than $61,134. It also subtracted a single $167 personal exemption credit, omitting the age-65 exemption credit that brings the nonrefundable total to $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Built the same $30,390 itemized total on unreimbursed employee business expenses, understating taxable income by $6,714 against the reference's $67,847.75, and claimed one $153 exemption credit instead of the personal-plus-age-65 pair worth $312.93. Its explanation derives $2,135 while the submitted value is $1,964.52, so the number filed matches neither its own arithmetic nor the correct chain." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Deducted $30,742.52, which includes roughly $6,521 of unreimbursed employee business expenses on top of charitable and real estate taxes; the reference's CA itemized deductions stop at $23,676.67, so its taxable income of $60,781.48 is about $7,066 below the correct $67,847.75. At California's 8% marginal rate that gap accounts for the entire shortfall between its $2,252.48 gross tax and the reference's $2,752.58." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"Supplied no derivation; $1,175 is what the joint-width surviving-spouse schedule yields on roughly $67,300 of taxable income (about $1,479) after roughly $300 of exemption credits. With no dependent child in the household, IRC §2(a) surviving-spouse status is unavailable, so the single schedule governs: $2,752.58 of bracket tax less $312.93 of personal and age-65 exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Used CA itemized deductions of $30,743, which include the unreimbursed employee business expenses net of the 2% floor; the reference's itemized total is $23,676.67 (charitable $14,847.67 after the 0.5%-of-AGI floor plus real estate taxes $8,829). That put taxable income near $60,781 instead of $67,847.75, roughly $565 of tax at the 8% marginal rate." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Deducted $30,743 including unreimbursed employee business expenses, producing taxable income of $60,781 where the reference's $23,676.67 of itemized deductions leaves $67,847.75. Its $2,415 of gross tax therefore falls $337 short of $2,752.58, and its $288 of exemption credits understates the indexed personal-plus-age-65 total of $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Submitted a bare number with no derivation. $3,211 is consistent with California taxable income near $76,700 — CA AGI of $91,524.41 less the charitable deduction alone — meaning it dropped the $8,829 of real estate taxes from the $23,676.67 itemized total that produces taxable income of $67,847.75 and tax of $2,439.65 after the $312.93 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Explicitly built $30,743 of CA itemized deductions from 'charity, real estate taxes, and employee business expenses'; the reference excludes the employee business expenses, capping itemized deductions at $23,676.67 and leaving taxable income of $67,847.75 rather than about $61,000. The roughly $7,000 of extra deduction at the 8% marginal rate is the bulk of its $405 shortfall." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,age_disability,False,"Gave no figures; $2,652 overshoots the reference by $212 and is consistent with subtracting only the single $156.47 personal exemption credit from a bracket tax near $2,808. California grants a filer age 65 or over a second exemption credit of the same amount, and the reference subtracts both ($312.93) from $2,752.58." -us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Gave no figures; $2,050 sits squarely in the band produced by deducting about $30,700 — charitable, real estate taxes, and unreimbursed employee business expenses — for taxable income near $61,000. California's itemized deductions here total $23,676.67, leaving taxable income of $67,847.75 and bracket tax of $2,752.58 before the $312.93 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Fabricated the income base — $24,556 of rental income against the $4,550 listed, $11,326 of mortgage interest that is not listed, and $1,000 of taxable Social Security — and then applied a federal standard deduction to a California return instead of the $23,676.67 of CA itemized deductions. Its $11,626 exceeds the California tax on the entire $91,524.41 of CA AGI with no deductions at all (about $4,900), so it is unreachable under CA's rate schedule." -us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Left Social Security in the California base, using CA AGI of $102,919 plus a phantom $1,750 tax-exempt-pension add-back instead of subtracting $17,624.75 to reach $91,524.41. It then claimed a nonexistent $1,549 California 'senior additional standard deduction' — California grants seniors an extra $156.47 exemption credit, not a larger standard deduction — and took a $7,089 standard deduction over the $23,676.67 of itemized deductions that actually govern." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted the liability is fully offset without computing anything. CA AGI is $91,524.41 and the largest deduction available is the $23,676.67 itemized total, leaving $67,847.75 of taxable income that reaches California's 8% bracket for $2,752.58 of tax; the personal and age-65 exemption credits total $312.93 and reduce that to $2,439.65, not zero." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"Provided no derivation; $1,036 is what the joint-width surviving-spouse schedule produces on roughly $67,300 of taxable income (about $1,479) less a doubled MFJ personal exemption credit plus senior credit (about $447). Absent a dependent child, IRC §2(a) status does not apply and the single schedule governs, giving $2,752.58 less $312.93 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Explicitly used the 'joint/surviving-spouse' schedule for a $1,136 gross tax and then removed $486 of exemption credits; the single schedule on $67,847.75 yields $2,752.58 and a single filer age 77 gets exactly two exemption credits worth $312.93. Its $60,781 taxable income also reflects the roughly $6,500 of unreimbursed employee business expenses the reference's $23,676.67 itemized total excludes." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $2,893 matches the California tax on the correct $67,847.75 of taxable income computed at unindexed prior-year bracket thresholds (about $2,888) with no exemption credit applied at all. The 2026 indexed single schedule produces $2,752.58, and the personal plus age-65 exemption credits remove a further $312.93 — the credit it says it applied is absent from its number." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Applied 'qualifying-surviving-spouse rates' — the joint-width schedule — and allowed 'eligible employee expenses' in California itemized deductions. The reference uses the single schedule on taxable income of $67,847.75, with itemized deductions of $23,676.67 that contain no employee business expenses; both errors push the same direction and drive its $657 against $2,439.65." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse (joint-width) rates instead of California's single schedule, which halves the bracket tax from $2,752.58 to roughly $1,400 on the same taxable income, and then subtracted only the personal exemption credit. A single filer age 77 also receives the age-65 exemption credit, for $312.93 of nonrefundable credits in total." -us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Ran the surviving-spouse schedule for $1,159.12 of tax on $60,781.48, compounding two errors: joint-width brackets in place of the single schedule that applies with no dependent child, and $6,521 of unreimbursed employee business expenses inside an itemized total the reference caps at $23,676.67. It further claimed $306 of personal plus $153 of senior credits where the single filer's exemption credits total $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Declined to compute and submitted zero, though the chain is fully determined by the listed facts. Federal AGI of $109,149.16 less the $17,624.75 Social Security subtraction gives CA AGI of $91,524.41; less $23,676.67 of itemized deductions leaves $67,847.75 of taxable income, taxed at $2,752.58 and reduced by $312.93 of personal and age-65 exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Included roughly $6,521 of unreimbursed employee business expenses in a $30,743 itemized total, reporting taxable income of $60,781 where the reference's $23,676.67 of deductions leaves $67,847.75. It also applied a single $153 exemption credit rather than the personal plus age-65 pair totaling $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Added $6,168 of unreimbursed employee business expenses net of the 2% floor to reach $30,390 of California itemized deductions; the reference allows charitable $14,847.67 plus real estate taxes $8,829 only, so taxable income is $67,847.75 rather than $61,134 and the bracket tax is $2,752.58 rather than about $2,280." -us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Deducted $6,521 of miscellaneous employee expenses above the 2% floor on top of charitable and property taxes ($30,743 against the reference's $23,676.67), understating taxable income by about $7,000. It then declared that 'no nonrefundable credits apply' — California gives this filer a $156.47 personal exemption credit plus a $156.47 age-65 exemption credit, $312.93 in total, which the reference does subtract." -us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Subtracted too much Social Security, reaching CA AGI of about $90,623 where the $17,624.75 subtraction from federal AGI gives $91,524.41, and then added employee business expenses to reach $30,761 of itemized deductions against the reference's $23,676.67 — together understating taxable income by roughly $8,000. Its single ~$154 exemption credit also omits the age-65 exemption credit that lifts the total to $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so nothing substantive reached the submission. The required chain is CA AGI of $91,524.41 after the $17,624.75 Social Security subtraction, less $23,676.67 of itemized deductions, giving $2,752.58 of bracket tax less $312.93 of exemption credits." -us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,age_disability,False,"Reached the right CA AGI and the right itemized composition but deducted the full $15,393 of contributions instead of $14,847.67 after the 0.5%-of-AGI charitable floor, and then overstated the rate-schedule tax at about $2,844 where the 2026 single brackets give $2,752.58 on $67,847.75. It subtracted only a $144 personal exemption credit, omitting the age-65 exemption credit that makes the nonrefundable total $312.93." -us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed California taxable income falls below the taxing threshold after deductions. CA AGI is $91,524.41 and the maximum deduction available is the $23,676.67 itemized total, leaving $67,847.75 of taxable income that reaches the 8% bracket for $2,752.58; the $312.93 of personal and age-65 exemption credits leave $2,439.65 owing." -us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Stated outright that 'CA taxes federally taxable Social Security' and used federal AGI of $109,149 as the California base; California excludes Social Security benefits entirely, which is the reference's $17,624.75 subtraction to CA AGI of $91,524.41. It compounded this with MFJ brackets and $306 of two-person exemption credits, where the single schedule and $312.93 of personal-plus-age-65 credits govern a filer with no dependent child." -us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Discarded the itemized deductions for a fabricated $10,726 standard deduction — California's 2026 single standard deduction is $5,706 and the $23,676.67 itemized total governs — and then applied an invented rate schedule starting at 4% and reaching 9.3% by $80,798, when California's single schedule starts at 1% and reaches 9.3% only above roughly $375,000. Its own steps total $6,183.26 less $639.40 = $5,543.86, not the $1,543.86 it submitted." -us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted that deductions plus a 'senior exemption allowance' drive the tax to zero. California's senior benefit is a $156.47 nonrefundable exemption credit, which with the personal credit totals $312.93 against $2,752.58 of bracket tax on $67,847.75 of taxable income — CA AGI of $91,524.41 less the $23,676.67 of charitable and real estate tax deductions." +us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"It filed as a qualifying surviving spouse with joint brackets, but there is no dependent child, so the correct status is single. It also deducted $6,521 of employee business expenses, which PolicyEngine does not allow in California itemized deductions. Its own joint-bracket arithmetic reached about $980, and it then reported an unsupported $3,162 instead of the single-schedule result: $2,752.58 minus $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It used the joint brackets and took two personal credits plus a senior credit ($459). This filer has no dependent, so the correct status is single. The single schedule on $67,847.75 of taxable income gives $2,752.58, less $312.93 of credits. It also deducted the full $15,393 of charitable gifts instead of $14,847.67." +us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It built taxable income of $106,979 from a made-up federal-style standard deduction and never excluded Social Security or itemized. California taxable income is $91,524.41 AGI minus $23,676.67 of itemized deductions, or $67,847.75. It also subtracted an invented $5,320 Senior Property Tax Postponement 'credit'. That program defers property tax; it is not a nonrefundable income tax credit." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as married filing jointly, using joint brackets and joint-plus-senior credits of about $420. With no dependent child, the filer is single. Single-bracket tax on $67,847.75 is $2,752.58, and the correct credits are $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It used joint brackets for a surviving spouse with no dependent, who is actually a single filer. It also made up about $7,000 of mortgage interest that is not in the facts. Its reported $1,856 does not match its own arithmetic: a $1,228 pre-credit tax minus credits cannot produce $1,856." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"Its single-filer setup and $24,222 of itemized deductions were close to correct. However, it put tax on $67,302 at $2,393, while the 2026 single brackets produce about $2,709 at that income. It also missed the reduction of the charitable deduction to $14,847.67, which puts taxable income at $67,847.75, gross tax at $2,752.58, and credits at $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It applied joint brackets and two personal credits plus a senior credit (about $471). The surviving spouse has no dependent child, so the filer is single. The correct result is single-bracket tax of $2,752.58 less one personal and one senior credit ($312.93)." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It used married-filing-jointly brackets, with the 2% bracket running to $49,368, and a joint $444 personal credit plus a senior credit. With no dependent child, the correct status is single. The single schedule on $67,847.75 of taxable income gives $2,752.58 before credits of $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It used qualifying-surviving-spouse brackets, but no dependent child is present, so the filer is single. It guessed a flat 2.8% effective rate instead of running the brackets. It also claimed the senior exemption credit was built into the tables, so it never subtracted the $312.93 of personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It applied joint/qualifying-surviving-spouse brackets and got about $1,426 of tax on $67,300. The filer has no dependent and is single, and single-bracket tax on $67,847.75 is $2,752.58. The correct result subtracts $312.93 of personal and senior credits from that figure." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It used federal AGI of $109,149 as California AGI and never subtracted the $17,624.75 of taxable Social Security that California excludes. As a result, taxable income was $84,927 instead of $67,847.75, pushing tax into the 9.3% bracket." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It deducted $6,168 of unreimbursed employee business expenses, which are not part of California itemized deductions here. That lowered taxable income to $61,134 instead of $67,847.75. It also subtracted only one $167 personal credit and omitted the senior exemption credit; the correct credits total $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It included unreimbursed employee business expenses and claimed $30,390 of itemized deductions, but the correct total is $23,676.67 (charity plus real estate taxes). That understated taxable income at $61,134. It applied only one $153 exemption credit, omitting the senior credit. Its submitted $1,964.52 also contradicts its own stated $2,135." +us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It deducted the full $15,393 of charitable gifts instead of the allowed $14,847.67, so its taxable income was $67,302 instead of $67,847.75. Its bracket arithmetic then overstated tax at $2,780.91, when the correct figure on the higher base is $2,752.58. It also subtracted $298 of credits instead of $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"It claimed $30,742.52 of itemized deductions by including $6,521 of employee business expenses above a 2% floor. California itemized deductions here are only charity ($14,847.67) and real estate taxes ($8,829), for $23,676.67. It therefore understated taxable income at $60,781.48 instead of $67,847.75." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It used a standard deduction, even though $23,676.67 of itemized deductions far exceeds the $5,706 single standard deduction. Its $1,175 is also far below the single-bracket tax at any taxable income consistent with the facts. The correct path is $67,847.75 of taxable income, $2,752.58 of tax, and $312.93 of credits." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It included about $6,521 of unreimbursed employee business expenses and claimed $30,743 of itemized deductions instead of $23,676.67. That understated taxable income by about $7,000, so its single-bracket tax came out below the correct $2,752.58." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It deducted employee business expenses and reached $30,743 of itemized deductions, for $60,781 of taxable income. The correct figures are $23,676.67 and $67,847.75. Its bracket tax of $2,415 on the understated base, minus $288 of credits (not $312.93), produced the wrong total." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no derivation. Its $3,211 exceeds even the correct $2,752.58 gross tax on $67,847.75 of taxable income, which means it both undercounted deductions and failed to subtract the $312.93 of personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"Its $30,743 of itemized deductions explicitly includes employee business expenses. California itemized deductions here are charity plus real estate taxes, $23,676.67. That understated taxable income by about $7,000, and at the 8% marginal rate that is roughly $400 too little tax." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"The correct result is $2,752.58 of single-bracket tax on $67,847.75, less $312.93 of personal and senior credits. Its $2,652 is consistent with gross tax near $2,752 but only about $100 of credits instead of the full $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"Its $2,050 matches the result of deducting about $7,000 of employee business expenses on top of charity and property tax. That understates taxable income below the correct $67,847.75, which is AGI of $91,524.41 minus $23,676.67 of itemized deductions." +us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It invented $24,556 of rental income, when the facts list $4,550, and a mortgage-interest figure. It also used a federal standard deduction instead of California itemized deductions. Its $11,626 comes from a fabricated income base, not California AGI of $91,524.41 minus $23,676.67." +us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It added a nonexistent $1,750 pension adjustment, never subtracted the $17,624.75 of taxable Social Security, and used a $7,089 standard deduction even though itemized deductions of $23,676.67 are larger. It then said the senior exemption credit does not apply, dropping the $312.93 of personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It claimed deductions and age-related provisions eliminate California tax. In fact, $91,524.41 of California AGI minus $23,676.67 of itemized deductions leaves $67,847.75 of taxable income, taxed at $2,752.58. Only $312.93 of credits offset that." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It gave no specific derivation. Its $1,036 is consistent with joint/surviving-spouse brackets and joint credits, not the single schedule that applies to a surviving spouse with no dependent. The single schedule gives $2,752.58 on $67,847.75, less $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It used joint/surviving-spouse brackets ($1,136 of tax) with joint-plus-senior credits of $486, but a surviving spouse with no dependent files single. It also deducted employee business expenses and reached $60,781 of taxable income instead of $67,847.75." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It subtracted only the personal exemption credit and omitted the senior exemption credit that a 77-year-old filer receives, when the combined credit is $312.93. Its $2,893 exceeds the $2,752.58 gross tax on the correct $67,847.75 base, so it also overstated the bracket tax." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It explicitly applied qualifying-surviving-spouse rates, but that status requires a dependent child, so the filer is single. It also deducted employee business expenses, which are not in the correct $23,676.67 of itemized deductions." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"It applied the surviving-spouse (joint) tax schedule. With no dependent child, the filer is single, and the single schedule on $67,847.75 gives $2,752.58 before $312.93 of credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It used the surviving-spouse schedule and a doubled $306 personal credit plus a $153 senior credit. With no dependent, the filer is single and gets $312.93 in total credits. It also deducted employee business expenses, reaching $30,742.52 of itemized deductions instead of $23,676.67." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It deducted allowable employee expenses in California itemized deductions, but only charity and real estate taxes count ($23,676.67). That understated taxable income below $67,847.75. It also mentions only the personal exemption credit and omits the senior credit." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It deducted employee business expenses and reduced taxable income to about $60,800. The correct taxable income is $67,847.75 because itemized deductions are only $23,676.67 of charity and property tax. It also subtracted only the personal exemption credit, not the combined $312.93 personal and senior credits." +us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It used surviving-spouse brackets and $474 of joint-plus-senior credits, but with no dependent child the filer is single and the credits are $312.93. It also included employee expenses, reaching $30,742.52 of itemized deductions instead of $23,676.67." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It declined to compute and defaulted to $0. The facts yield $67,847.75 of California taxable income ($91,524.41 AGI minus $23,676.67 of itemized deductions), taxed at $2,752.58 on the single schedule before $312.93 of credits." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It included employee business expenses and reached $30,743 of itemized deductions and $60,781 of taxable income, instead of $23,676.67 and $67,847.75. It also subtracted only the $153 personal credit and omitted the senior exemption credit that makes the total $312.93." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"It added $6,168 of unreimbursed employee expenses to itemized deductions, reaching $30,390 and $61,134 of taxable income. The correct figures are $23,676.67 of itemized deductions and $67,847.75 of taxable income, with no employee-expense deduction." +us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It deducted $6,521 of employee expenses above a 2% floor and used the full $15,393 of charity, reaching $30,743 of itemized deductions. The correct deductions are $23,676.67: charity of $14,847.67 plus real estate taxes of $8,829. Its taxable income of $60,781 is therefore about $7,066 too low." +us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It deducted $6,521 of miscellaneous employee expenses, understating taxable income at $60,781 instead of $67,847.75. It also stated that no nonrefundable credits apply, omitting the $312.93 of personal and senior exemption credits." +us,scenario_022,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It included miscellaneous employee expenses above the 2% floor, reaching about $30,761 of itemized deductions instead of $23,676.67. It also misstated California AGI as $90,623 instead of $91,524.41. Its taxable income of $59,862 is about $8,000 too low, and it subtracted only the personal credit without the senior credit." +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It subtracted only a $144 personal exemption credit and omitted the senior exemption credit a 77-year-old receives; together they total $312.93. It also used the full $15,393 of charity, giving $67,302 of taxable income, and overstated the bracket tax at $2,844." +us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It asserted income falls below the tax threshold after deductions. California AGI of $91,524.41 minus $23,676.67 of itemized deductions leaves $67,847.75 of taxable income and $2,752.58 of tax before $312.93 of credits." +us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It wrongly said California taxes federally taxable Social Security, so it never subtracted $17,624.75 and used $84,927 of taxable income. It also applied joint brackets and two personal credits, when a surviving spouse with no dependent files single." +us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"It used a $10,726 standard deduction instead of itemizing, when the $23,676.67 of charity plus real estate taxes is larger. It then invented a rate schedule starting at 4% and subtracted $639 of credits. The correct path is $67,847.75 of taxable income on the single brackets, less $312.93 of credits." +us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It claimed deductions and senior credits reduce tax to zero. Itemized deductions of $23,676.67 leave $67,847.75 of taxable income and $2,752.58 of tax, and senior and personal credits remove only $312.93 of that." us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Its income chain matched the reference exactly ($22,535 AGI, $16,100 standard deduction, $6,435 taxable, ~$644 at 10%), then it erased the liability with two credits that do not exist here: an AOTC despite no qualified tuition figure being listed (unlisted numerics are 0, so the credit base and the credit are $0), and a $1,000 Saver's Credit. IRC 25B(d)(2) reduces Saver's Credit contributions by testing-period distributions, and the $8,000 403(b) distribution exceeds the $3,599 of total retirement contributions, zeroing that credit; with no nonrefundable credits the answer is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It reproduced the reference computation through $643.50 of tentative tax and correctly set AOTC to $0 for lack of tuition, but then applied a $1,000 nonrefundable Saver's Credit. It skipped the Form 8880 / IRC 25B(d)(2) offset, under which the $8,000 taxable 403(b) distribution received in the testing period exceeds the $3,599 of 401(k)/IRA/Roth contributions and reduces the Saver's Credit contribution base to $0, leaving the full $643.43 standing." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $8,000 taxable 403(b) distribution from gross income entirely and used a $14,600 standard deduction (the 2024 single figure) instead of the 2026 $16,100, producing negative taxable income. Including the distribution gives AGI of $22,534.34 and, after the $16,100 standard deduction, $6,434.34 of taxable income taxed at 10% for $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It derived the reference's AGI, standard deduction, taxable income and ~$644 tentative tax correctly, then wiped the liability with a $1,500 nonrefundable AOTC portion. No qualified tuition or fees amount appears in the household facts, and the instruction to treat unlisted numeric inputs as 0 makes the AOTC expense base $0, so the credit is $0 and tax before refundable credits stays at $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its AGI, $16,100 standard deduction and ~$644 tax matched the reference, but it then assumed 'qualified 1098-T expenses' and applied the 60% nonrefundable share of a maximum $2,500 AOTC. The 1098-T and enrollment flags are eligibility switches, not expenses; with qualified education expenses of $0 the AOTC is $0, leaving $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It matched the reference through the $6,435 taxable income and ~$644 tax, then applied an AOTC nonrefundable portion of up to $1,500. The credit equals 100% of the first $2,000 plus 25% of the next $2,000 of qualified tuition, and no tuition amount is listed, so the credit is $0 and the tax before refundable credits is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It built the correct AGI of $22,535 and correctly set the AOTC to $0 for lack of listed tuition, but used a standard deduction of $15,350 derived by indexing the pre-OBBBA $15,000 figure. The 2026 single standard deduction is $16,100, which yields $6,434.34 of taxable income and $643.43 of tax rather than its $718.50." -us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It asserted that the ~$16,100 standard deduction exceeds income after adjustments when AGI is $22,534.34, exceeding the deduction by $6,434.34, and it also mis-added gross income as $26,443 instead of $25,443. It then covered the 'residual' with an AOTC nonrefundable portion that is $0 because no qualified tuition expense is listed, so $643.43 remains." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 employer-sponsored insurance premium from wages as an additional pre-tax exclusion on top of the stated gross wage, cutting AGI to $14,446. The listed gross wages of $17,443 are already the taxable W-2 wage figure and only the $2,778.48 traditional 401(k) deferral and $129.83 IRA deduction reduce it, giving AGI of $22,534.34; it also used a $15,750 standard deduction rather than the 2026 $16,100." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made two compounding parameter errors: it subtracted the $8,089 employer health premium from wages, which is not an above-the-line deduction on top of the stated gross wage, dropping AGI to $14,446 instead of $22,534.34, and it used an $8,350 standard deduction from a pre-OBBBA TCJA-sunset schedule instead of the 2026 single amount of $16,100. The correct chain is $22,534.34 − $16,100 = $6,434.34 taxed at 10% = $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed AGI of $22,535 correctly and correctly zeroed both the AOTC and the Saver's Credit, then applied an $8,300 standard deduction plus a $5,300 personal exemption on the assumption that TCJA lapsed after 2025. Personal exemptions remain $0 for 2026 and the single standard deduction is $16,100, so taxable income is $6,434.34 and the tax is $643.43, not $893.50." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It used roughly $14,000 of standard deduction plus personal exemptions instead of the $16,100 single standard deduction with no exemptions, then zeroed the resulting $853.50 with a 50%-rate $1,000 Saver's Credit. IRC 25B(d)(2) nets testing-period distributions against contributions, and the $8,000 403(b) distribution exceeds the $3,599 of contributions, so the Saver's Credit is $0 and $643.43 of tax remains." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and a taxable income of zero is reachable here only by dropping the $8,000 taxable 403(b) distribution: wages of $17,442.65 less the $2,778.48 401(k) deferral and $129.83 IRA deduction is $14,534, which falls under the $16,100 standard deduction. The distribution is includible gross income, so AGI is $22,534.34, taxable income is $6,434.34, and the tax is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It claimed the standard deduction plus a nonrefundable Saver's Credit completely offset the liability, but AGI of $22,534.34 leaves $6,434.34 taxable after the $16,100 deduction, and the Saver's Credit is $0 here. The $8,000 testing-period 403(b) distribution exceeds the $3,599 of retirement contributions, so IRC 25B(d)(2) reduces the credit's contribution base to zero and $643.43 stands." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 employer-sponsored insurance premium from wages to reach an AGI of $14,446, when the listed gross wage is already the taxable wage and only the $2,778.48 401(k) deferral and $129.83 IRA deduction reduce it to an AGI of $22,534.34. It then offset its residual with a Saver's Credit that IRC 25B(d)(2) zeroes because the $8,000 403(b) distribution exceeds total contributions of $3,599." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, and $0 requires either zero taxable income or at least $643.43 of nonrefundable credit; neither holds. AGI of $22,534.34 less the $16,100 standard deduction leaves $6,434.34 taxed at 10%, the AOTC base is $0 with no tuition listed, and the Saver's Credit is zeroed by the $8,000 testing-period 403(b) distribution, so the answer is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Its stated $553.50 implies a $753.50 pre-credit tax from a $15,000 standard deduction and a $200 Saver's Credit. The 2026 single standard deduction is $16,100, giving $6,434.34 of taxable income and $643.43 of tax, and the Saver's Credit is $0 because the $8,000 403(b) testing-period distribution exceeds the $3,599 of retirement contributions under IRC 25B(d)(2)." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It asserted that pre-tax deductions plus the standard deduction produce zero taxable income, which only follows if the $8,000 taxable 403(b) distribution is left out of gross income. With the distribution included, AGI is $22,534.34, the $16,100 standard deduction leaves $6,434.34 of taxable income, and the 10% bracket yields $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It got everything right except the standard deduction: AGI of $22,535, AOTC of $0 for lack of tuition, and Saver's Credit of $0 under the Form 8880 distribution offset. Its $754 implies a deduction of about $15,000 (the 2025 single amount); the 2026 figure is $16,100, which leaves $6,434.34 taxable and $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI of $22,535, full IRA deduction, $0 AOTC and $0 Saver's Credit (correctly citing the $8,000 distribution against $3,599 of contributions) all match the reference; the sole error is a $15,400 estimated standard deduction. The 2026 single standard deduction is $16,100, so taxable income is $6,434.34 and the tax is $643.43, not $713.50." -us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It denied that the $2,778.48 traditional 401(k) deferral reduces taxable wages, invented a $500 exclusion against the 403(b) distribution, and used a $15,000 standard deduction, and it then submitted −997 while its own text concluded the liability was $0. Elective 401(k) deferrals are excluded from wages, no exclusion applies to the $8,000 distribution, the 2026 single standard deduction is $16,100, and no nonrefundable credit applies, giving $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It claimed the standard deduction plus the head's AOTC eliminate the tax, but AGI of $22,534.34 exceeds the $16,100 standard deduction by $6,434.34 and the AOTC is $0 because no qualified tuition or fees amount is listed. The enrollment, 1098-T and EIN flags are eligibility conditions, not expenses, so the tax before refundable credits is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It applied an AOTC nonrefundable portion to reach a $121 residual, a figure no bracket computation on this household produces. The AOTC expense base is $0 because no tuition amount is listed, so no nonrefundable credit reduces the $643.43 computed on $6,434.34 of taxable income at the 10% rate." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted wages fall under the standard deduction threshold, but wages of $17,442.65 already exceed the $16,100 single standard deduction before adding the $8,000 taxable 403(b) distribution. After the $2,778.48 401(k) deferral and $129.83 IRA deduction, AGI is $22,534.34 and taxable income is $6,434.34, giving $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It computed the reference AGI of $22,535 and correctly set the AOTC to $0, then applied a projected TCJA-sunset schedule of roughly $8,500 of standard deduction plus a $5,300 personal exemption. For 2026 the single standard deduction is $16,100 and personal exemptions are $0, so taxable income is $6,434.34 and the tax is $643.43, not $860." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It used a standard deduction plus personal exemption to get roughly $8,900 of taxable income, then zeroed the resulting tax with a $1,000 nonrefundable Saver's Credit. Personal exemptions are $0 and the single standard deduction is $16,100, and the Saver's Credit is $0 because the $8,000 testing-period 403(b) distribution exceeds the $3,599 of retirement contributions under IRC 25B(d)(2), leaving $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired after 2025 and used an ~$8,500 standard deduction with a reverted rate schedule that pushed part of the income into a 15% bracket. The 2026 single standard deduction is $16,100 and the entire $6,434.34 of taxable income falls in the 10% bracket, so the tax is $643.43, not $1,485." -us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used the correct $16,100 standard deduction but subtracted the $8,089 employer health premium from income alongside the 401(k) and IRA amounts, cutting AGI to $14,446. The employer-sponsored insurance premium input does not reduce the stated gross wage of $17,442.65; only the $2,778.48 deferral and $129.83 IRA deduction do, leaving AGI of $22,534.34, taxable income of $6,434.34, and $643.43 of tax." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation was submitted for the required key. The graded answer is absent rather than incorrect." -us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation was submitted for the required key. The graded answer is absent rather than incorrect." -us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It reached $713.50 with a $15,400 standard deduction instead of the 2026 single amount of $16,100, then eliminated the entire liability with an AOTC nonrefundable portion of $1,500. The AOTC is $0 because no qualified tuition expense is listed, and with the correct deduction the tax before refundable credits is $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It matched the reference exactly through AGI $22,535, the $16,100 standard deduction, $6,435 of taxable income and ~$644 of tax, and correctly set the AOTC to $0; the single error is applying a 50%-rate $1,000 Saver's Credit on $2,908 of contributions. IRC 25B(d)(2) reduces the contribution base by testing-period distributions, and the $8,000 403(b) distribution exceeds the $3,599 of contributions, so the credit is $0 and $643.43 stands." -us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It used a $15,750 standard deduction rather than the 2026 single amount of $16,100 and then wiped out its ~$678.50 tax with the nonrefundable AOTC. The AOTC is $0 because no qualified tuition expense is listed, so the correct result is $6,434.34 of taxable income taxed at 10% for $643.43." -us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It declared income too low for federal income tax after the standard deduction, ignoring that the $8,000 taxable 403(b) distribution lifts AGI to $22,534.34, which is $6,434.34 above the $16,100 standard deduction. Its fallback AOTC is also $0 because no qualified tuition expense is listed, so the tax before refundable credits is $643.43." -us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model explicitly found that qualified tuition expenses were zero and that the AOTC therefore computed to zero, then overrode that result by inventing default education expenses. PolicyEngine does not impute qualified expenses from the enrollment flags, so the refundable AOTC is zero." -us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model awarded an AOTC solely from enrollment and documentation flags despite zero listed qualified education expenses. It also treated the entire $2,500 maximum AOTC as refundable, although only 40%, capped at $1,000, can be refundable even when sufficient expenses exist." -us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented roughly $400 of refundable AOTC without any qualified education expenses and added a residual EITC despite recognizing that the childless credit phases out at this AGI. Zero expenses produce zero AOTC, and AGI including the taxable 403(b) distribution eliminates the EITC." -us,scenario_023,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model assumed maximum qualified education expenses even though the prompt requires every unlisted numeric input to be zero. The AOTC eligibility flags do not create an expense base, so its asserted $1,000 refundable AOTC is zero." -us,scenario_023,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model invented maximum AOTC expenses and a residual childless EITC. Qualified education expenses are zero, and AGI including the taxable 403(b) distribution is beyond the childless EITC phaseout endpoint, so neither component contributes a refundable credit." -us,scenario_023,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied only the EITC phase-in calculation to an incorrectly reduced earned-income figure and ignored the separate AGI limitation. Employer health premiums and desired retirement contributions do not justify its $6,576 earned-income base, and the taxable 403(b) distribution raises AGI enough to phase the childless EITC to zero." -us,scenario_023,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model phased the childless EITC using wages alone and failed to apply the rule that the credit is computed using the lesser result under earned income and AGI. The taxable $8,000 403(b) distribution enters AGI and pushes the credit past its phaseout endpoint." -us,scenario_023,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model awarded the AOTC without any listed qualified education expenses and incorrectly called the full $2,500 credit refundable. With a zero expense base the AOTC is zero; even with maximum expenses, its refundable portion is limited to $1,000." -us,scenario_023,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,The model placed the taxpayer inside the childless EITC phaseout range without including the taxable 403(b) distribution in AGI. Applying the EITC's AGI phaseout test eliminates the credit entirely. -us,scenario_023,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model awarded a childless EITC without applying the AGI-side phaseout calculation. The taxable 403(b) distribution raises AGI beyond the phaseout endpoint, yielding zero EITC." -us,scenario_023,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,The model treated AOTC eligibility flags as sufficient for a refundable credit even though qualified education expenses are unlisted and therefore zero. Any EITC component is also eliminated by the AGI phaseout after including the taxable 403(b) distribution. -us,scenario_023,federal_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model inferred the maximum refundable AOTC from enrollment and documentation facts while omitting the required qualified-expense base. Because unlisted tuition and related qualified expenses equal zero, the refundable AOTC equals zero." -us,scenario_023,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model used an incorrect childless EITC phaseout starting near $19,000 instead of the applicable single-filer threshold and consequently left $380 of credit. It also treated the desired traditional 401(k) amount as reducing wages without establishing an actual contribution; the proper AGI phaseout calculation yields zero." -us,scenario_023,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model correctly denied the AOTC for lack of qualified expenses but computed EITC from a purported net earned-income amount alone. It failed to run the separate AGI phaseout test, under which the taxable 403(b) distribution eliminates the childless EITC." -us,scenario_023,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value for federal_refundable_credits. It therefore failed the required structured-output contract rather than completing the refundable-credit calculation. -us,scenario_023,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no value for federal_refundable_credits. It therefore failed the required structured-output contract rather than completing the refundable-credit calculation. -us,scenario_023,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented sufficient education expenses for a $1,000 refundable AOTC and used an incorrect childless EITC phaseout threshold near $19,104. Qualified expenses are zero, and the correct AGI phaseout including the taxable 403(b) distribution reduces EITC to zero." -us,scenario_023,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model invented maximum qualified expenses for a $1,000 refundable AOTC and calculated EITC from wages without applying the higher AGI that includes the taxable 403(b) distribution. It then submitted $3,925 even though its own stated component total was $1,112, an additional unsupported output-value error." -us,scenario_023,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model claimed an Additional Child Tax Credit for a child who does not exist in the one-person household. With no qualifying child, the refundable CTC component is zero." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly found the tentative tax of about $644, then wiped it out with a $1,000 Saver's Credit and an AOTC. Under Form 8880, the $8,000 403(b) distribution is subtracted from eligible contributions, which brings the Saver's Credit to zero. The AOTC is zero because there are no qualified tuition expenses. The tax is therefore $643.43, not $0." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,other,False,"It computed the $643.50 tentative tax correctly but applied a $1,000 Saver's Credit. Form 8880 subtracts the $8,000 taxable 403(b) distribution in the testing period from the $3,599 of contributions, so eligible contributions and the credit are both zero." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution out of gross income and used the outdated $14,600 standard deduction instead of $16,100, which produced negative taxable income. The correct AGI of $22,534 minus $16,100 leaves $6,434 of taxable income and $643 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It computed the $644 tax correctly but offset it with a $1,500 nonrefundable AOTC. The AOTC is a percentage of qualified tuition and related expenses, and none are listed, so the credit is zero no matter what the enrollment and 1098-T flags say." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It assumed qualified 1098-T expenses and applied the AOTC's nonrefundable portion against the $644 tax. No tuition amount is listed, so qualified expenses are $0 and the AOTC is $0, which leaves $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"After correctly reaching about $644 of tax, it applied up to $1,500 of nonrefundable AOTC. With no qualified education expenses listed, the AOTC is zero and the full $643.43 remains." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,other,False,"It explicitly assumed that the 403(b) distributions do not reduce eligible Saver's Credit contributions. That is backwards: Form 8880 requires subtracting testing-period distributions, and the $8,000 distribution reduces the $3,599 of contributions to zero, so the credit is $0 and the tax is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,350 by inflating the pre-OBBBA $15,000 figure, instead of using the enacted $16,100. That raised taxable income to $7,185 and the tax to $719 instead of $643.43. Its AGI and credit treatment were correct." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It claimed that taxable income after the $16,100 deduction is near zero and applied the AOTC to eliminate what remained. In fact, taxable income is $6,434 and the AOTC is zero because no qualified tuition expenses are listed." +us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"It computed the $644 tentative tax correctly but applied a 50% Saver's Credit on $2,000 of contributions. It missed the Form 8880 rule that subtracts the $8,000 403(b) distribution from contributions, which leaves no eligible contributions." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 of employer-sponsored insurance premiums from wages as a pre-tax exclusion, which cut AGI to $14,446, below the standard deduction. The only pre-tax reduction to wages here is the 401(k) deferral, so AGI is $22,534 and taxable income is $6,434." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It excluded the $8,089 of ESI premiums from wages and also used a pre-TCJA standard deduction of $8,350 instead of the $16,100 OBBBA amount. The two errors partly cancelled and produced $609.60, while the correct figure is 10% of $6,434." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA had expired and applied an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made the higher deduction permanent at $16,100 for 2026, with a personal exemption of zero. Its AGI and zero credits were correct, but taxable income should be $6,434, not $8,935." +us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It used the 2025 standard deduction of $15,750 instead of $16,100. It then applied a $1,000 Saver's Credit without subtracting the $8,000 403(b) distribution from contributions as Form 8880 requires. That subtraction makes the credit zero and leaves $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,other,False,"It used an estimated $14,000 for the standard deduction plus a personal exemption instead of the $16,100 standard deduction with no exemption. It then zeroed the tax with a $1,000 Saver's Credit, but the $8,000 403(b) distribution offsets all contributions on Form 8880, so the credit is $0." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted that taxable income is zero after deductions. AGI of $22,534 exceeds the $16,100 standard deduction by $6,434, which produces $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It used the nonrefundable Saver's Credit to offset the remaining tax. Form 8880 reduces contributions by testing-period distributions, and the $8,000 403(b) distribution exceeds the $3,599 of contributions, so the credit is zero." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reached an AGI of $14,446 by excluding the $8,089 of ESI premiums from wages, which is not a reduction to AGI here. It then applied a Saver's Credit that the $8,000 403(b) distribution eliminates on Form 8880. The correct AGI of $22,534 yields $643.43 of tax with no credits." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. Its $0 answer is consistent with applying a nonrefundable credit that does not apply here: the Saver's Credit is zero after the $8,000 distribution offset, and the AOTC is zero without tuition. The $643.43 tax on $6,434 of taxable income therefore stands." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its pre-credit tax of $753.50 implies the stale $15,000 standard deduction instead of $16,100. It then subtracted a $200 Saver's Credit, which the $8,000 403(b) distribution reduces to zero under Form 8880." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It claimed that taxable income is zero after the standard deduction. AGI is $22,534, including the $8,000 taxable 403(b) distribution, and the $16,100 standard deduction leaves $6,434 taxable at 10%." +us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly zeroed the AOTC and Saver's Credit, but its $754 result implies a $15,000 standard deduction (taxable income of $7,535) instead of the 2026 OBBBA amount of $16,100." +us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI and credit treatment were correct, including netting the distribution against contributions for the Saver's Credit. However, it estimated the 2026 standard deduction at $15,400 instead of $16,100, which overstated taxable income by $700." +us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"Its reasoning is self-contradictory. It left the traditional 401(k) deferral in AGI, applied an invented $500 exclusion to the 403(b) distribution, used a $15,000 deduction, concluded $0, and then submitted -$997. Tax before refundable credits cannot be negative, and the correct figure is $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It relied on the AOTC to eliminate the remaining tax. The AOTC is computed from qualified tuition expenses, none are listed, and so the credit is zero and $643.43 remains." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied a partial AOTC offset against the tax on the basis of the enrollment flags alone. With no qualified education expenses, the AOTC is zero, and the tax on $6,434 of taxable income is the full $643.43, not $121." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It offset the tax with the Saver's Credit on the listed contributions. It ignored the Form 8880 requirement to subtract the $8,000 403(b) distribution, which reduces eligible contributions and the credit to zero." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"It used the Saver's Credit to eliminate the tax. The $8,000 taxable 403(b) distribution exceeds total retirement contributions, so eligible Form 8880 contributions are zero and the credit is $0." +us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $8,089 of employer-plan premiums as pre-tax, which cut AGI to $14,446, below the standard deduction. Wages are reduced only by the 401(k) deferral, so AGI is $22,534 and $6,434 is taxable." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated wages alone as falling under the standard deduction and ignored the $8,000 taxable 403(b) distribution. With that distribution included, AGI is $22,534, which exceeds the $16,100 deduction by $6,434." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a TCJA-sunset deduction of about $8,500 plus a $5,300 personal exemption. OBBBA made the TCJA structure permanent, so the 2026 single standard deduction is $16,100 and the personal exemption is zero. Taxable income is $6,434, not about $8,600." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,other,False,"It used a standard deduction plus a personal exemption, which put taxable income at about $8,900. It then offset the tax with a $1,000 Saver's Credit even though the $8,000 403(b) distribution nets contributions to zero on Form 8880. The correct result is $16,100 of deduction, no exemption, no credit, and $643.43 of tax." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly zeroed the Saver's Credit and AOTC but assumed post-TCJA law, with an $8,500 standard deduction and a $5,450 personal exemption. Under OBBBA, 2026 uses a $16,100 standard deduction and no exemption." +us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired, used an $8,500 standard deduction, and applied a 15% bracket. The 2026 law uses a $16,100 deduction and the 10% bracket, which gives 10% of $6,434, or $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 of employer health premiums from wages to reach an AGI of $14,446. Those premiums do not reduce AGI here, so AGI is $22,534 and $6,434 remains taxable after the $16,100 deduction." +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score against $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no usable value for federal_income_tax_before_refundable_credits, so there was no answer to score against $643.43." +us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used a $15,400 standard deduction instead of $16,100 and then applied a $1,500 nonrefundable AOTC. The AOTC is zero because no qualified tuition expenses are listed, so the tax is not eliminated." +us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"It computed the $644 tentative tax correctly but applied a $1,000 Saver's Credit on the full $2,908 of contributions. Form 8880 subtracts the $8,000 403(b) distribution first, which leaves zero eligible contributions." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It used the $15,750 2025 standard deduction instead of $16,100. It then applied the AOTC's nonrefundable portion, but the AOTC is zero with no qualified tuition expenses, so $643.43 of tax remains." +us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It claimed that income is too low to leave tax after the standard deduction and that the AOTC covers any remainder. In fact, $6,434 of taxable income remains, and the AOTC is zero because there are no qualified education expenses." +us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly phased the EITC out to $0 and even noted that zero listed tuition gives a $0 AOTC. It then reversed itself by assuming 'typical qualified expenses default' of at least $4,000 and submitted a $1,000 refundable AOTC. With no qualified expenses, the AOTC is $0." +us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the full $2,500 AOTC as refundable. The refundable share is capped at 40% ($1,000). More importantly, the credit base is $0 because no qualified tuition expenses are listed. It also left the $8,000 403(b) distribution out of AGI." +us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model made up a refundable AOTC of about $400 plus a small EITC to reach $522. No qualified education expenses are listed, so the AOTC is $0. Its own AGI of $22,535 is above the roughly $19,540 point where the 2026 childless EITC reaches zero, so the EITC is also $0." +us,scenario_023,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly zeroed the EITC but explicitly 'assumes maximum qualified expenses' to claim the $1,000 refundable AOTC. Unlisted tuition is $0, so the AOTC credit base and the refundable portion are both $0." +us,scenario_023,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model assumed a $1,000 refundable AOTC with no qualified expenses listed, and added a leftover EITC of about $400 even though its own AGI of $22,535 is past the childless phaseout completion point. The total it submitted ($2,400) does not even match the parts it listed." +us,scenario_023,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model shrank earned income to $6,576 by subtracting health premiums from wages, then applied only the 7.65% phase-in. It never ran the phaseout test on the greater of AGI or earned income. AGI including the $8,000 403(b) distribution is about $22,535, above the roughly $19,540 completion point, so the EITC is $0." +us,scenario_023,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model phased out the childless EITC using only $17,443 of wages. The phaseout uses the greater of AGI or earned income, and AGI including the $8,000 taxable 403(b) distribution is about $22,535, beyond the roughly $19,540 completion point. That drives the EITC to $0, not $137." +us,scenario_023,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model declared the full $2,500 AOTC refundable. Only 40% is refundable, and no qualified tuition expenses are listed, so the credit base is $0 and the refundable AOTC is $0." +us,scenario_023,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model placed the head in the middle of the childless EITC phaseout, which only works on a wage-only income measure. AGI including the $8,000 403(b) distribution is about $22,535, above the roughly $19,540 point where the 2026 childless EITC is fully phased out, so the credit is $0." +us,scenario_023,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The $503 figure is a 7.65% phase-in on about $6,576 of reduced 'earned income', with no phaseout applied. The childless EITC phases out on the greater of AGI or earned income, and AGI of about $22,535 (including the $8,000 403(b) distribution) exceeds the roughly $19,540 completion point, giving $0." +us,scenario_023,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model claimed $1,000 of combined AOTC and EITC. The refundable AOTC is $0 because no qualified tuition expenses are listed. The EITC is $0 because AGI of about $22,535, including the $8,000 403(b) distribution, exceeds the childless phaseout completion point." +us,scenario_023,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model treated the AOTC eligibility flags as enough for the maximum $1,000 refundable credit. The credit amount comes from qualified tuition expenses, which are $0 here, so the refundable AOTC is $0." +us,scenario_023,federal_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"The model correctly excluded the AOTC, but its $274 phaseout reduction implies an income measure of only about $14,446. That leaves out the $8,000 taxable 403(b) distribution. On the full AGI of about $22,535, the $664 credit phases out completely by about $19,540, so the EITC is $0." +us,scenario_023,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used about $19,000 as the start of the childless EITC phaseout, but that is roughly the completion point. The 2026 single childless phaseout starts near $10,860. With its own AGI of $22,535 above completion, the EITC is $0, not $380." +us,scenario_023,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model correctly excluded the AOTC but computed only a 7.65% phase-in on about $6,576 of reduced earned income and skipped the phaseout test. Phased out on the greater of AGI or earned income, with AGI about $22,535 including the $8,000 403(b) distribution, the childless EITC is $0." +us,scenario_023,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for federal_refundable_credits, so its answer could not be scored." +us,scenario_023,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for federal_refundable_credits, so its answer could not be scored." +us,scenario_023,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model added a $1,000 refundable AOTC with no qualified tuition expenses listed, so the credit base is $0. It also treated about $19,104 (roughly the childless EITC completion point) as the phaseout start and got $387. With AGI of $22,535 above completion, the EITC is $0." +us,scenario_023,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model assumed a $1,000 refundable AOTC with $0 qualified expenses. It phased the EITC out on wages alone instead of the larger AGI that includes the 403(b) distribution, which zeroes it out. It then submitted $3,925, which does not match its own $1,112 subtotal." +us,scenario_023,federal_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model invented a child and claimed an Additional Child Tax Credit. The household is a single 28-year-old with no dependents, so no refundable CTC is available, and the childless EITC is fully phased out at an AGI of about $22,535." us,scenario_023,free_school_meals_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded the $8,000 taxable 403(b) distribution and compared wages alone with the federal income cutoff; the trace’s applicable income is $25,442.65, or 1.59 times the poverty guideline. It also equated income eligibility with positive annual support even though this adult-only household has no school-meal recipient." us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly identified the adult-only household and the absence of a school-aged child, then incorrectly asserted that household-level income qualification produces free-meal support anyway. California’s universal and categorical pathways establish the FREE tier, but without a student receiving meals PolicyEngine returns no positive annual support." us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, violating the required submission contract." us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California CHIP as covering disabled adults up to an extended 266% FPL threshold. It failed to apply the categorical exclusion for a person already Medicaid-eligible through the WORKING_DISABLED_BUY_IN pathway, which makes this head ineligible for CHIP regardless of the CHIP income thresholds." us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable output for head_chip_eligible. The required result was 0 because Medicaid eligibility under WORKING_DISABLED_BUY_IN categorically excludes the head from CHIP. -us,scenario_023,head_medicaid_eligible,claude-fable-5.1,llm_error,categorical_eligibility,False,"It computed MAGI correctly at $22,535 (141% FPL) and correctly ruled out both the 138% FPL adult expansion group and the SSI-linked route, but treated those two as the complete set of pathways. The head is disabled and employed ($14/hr, 24 hrs/week, $17,443 in wages), which places them in the Medicaid buy-in for working disabled category — California's 250% Working Disabled Program — whose income ceiling (~$39,900 for one person) sits far above the $22,535 the model used to deny eligibility." -us,scenario_023,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"It tested only the Medi-Cal adult expansion limit (138% FPL) against a loosely bounded income of $22,500-$25,400 and stopped there. The disability flag combined with ongoing employment triggers the working-disabled buy-in category, a non-MAGI group with a 250% FPL ceiling, so the narrow overshoot of the 138% line (actual MAGI $22,535 = 141% FPL against a ~$22,055 cutoff) does not end the analysis." -us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It got the MAGI arithmetic exactly right ($22,535) and correctly rejected the SSI-linked pathway on SSI = 0, but equated ""disability pathway"" with ""SSI receipt."" The working-disabled buy-in group conditions on being disabled and employed rather than on SSI payability, and its 250% FPL income limit accommodates $22,535 with room to spare." -us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Its entire test was MAGI $22,535 versus the 138% FPL expansion threshold, with no consideration of any non-MAGI category. It ignored the ""is disabled"" fact paired with $17,443 of wages, which qualifies the head under the working-disabled buy-in category whose income limit is 250% FPL." -us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It checked the 138% FPL expansion limit and then dismissed the non-MAGI side as ""SSI pathways,"" collapsing every disability route into SSI receipt. The buy-in for working disabled is a separate non-MAGI group requiring disability plus earnings — both present here — with a 250% FPL ceiling that $22,535 clears." -us,scenario_023,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"Its AGI derivation is exact ($17,443 - $2,778 401(k) + $8,000 403(b) - $130 IRA = $22,535) and its rejection of both the 138% FPL expansion group and the SSI/SSP-linked route is right, but it enumerated only those two categories. The head's disability plus employment opens the working-disabled buy-in group, where the ceiling is 250% FPL (~$39,900) and earned income receives the $20/$65-and-half disregards, leaving countable income far under the limit." -us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It offered only the bare assertion that ""adult Medicaid eligibility in California is not met,"" a conclusion consistent with running MAGI of $22,535 (141% FPL) against the 138% FPL adult expansion limit alone. The correct derivation routes the disabled, employed head into the working-disabled buy-in category, whose 250% FPL income ceiling makes the expansion overshoot irrelevant." -us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It saw the disability flag and explicitly discarded it as lacking ""details,"" then fell back on the MAGI expansion test alone; it also treated wages as the only income, ignoring the $8,000 taxable 403(b) distribution. Disability plus employment is by itself the full trigger for the working-disabled buy-in category, which admits the head at 141% FPL against a 250% FPL limit." -us,scenario_023,head_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"It measured wages plus the 403(b) distribution net of pre-tax retirement deductions against ""the adult Medi-Cal income limit"" — the 138% FPL expansion ceiling — and stopped. The head's disability and employment place them in the working-disabled buy-in group, a non-MAGI category whose 250% FPL limit the $22,535 figure satisfies." -us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It asserted that ""no separate qualifying Medicaid category is established by the listed facts,"" but the listed facts establish one directly: the head is disabled and works 24 hours a week for $17,443 in wages, which is the working-disabled buy-in category. That group's income ceiling is 250% FPL, so MAGI at 141% FPL qualifies despite exceeding the 138% expansion threshold." -us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It correctly computed MAGI of $22,535 above the expansion limit and correctly noted that disability alone does not create SSI-linked eligibility, but it never tested the pathway that disability plus earnings does create. The working-disabled buy-in category requires employment rather than SSI receipt and carries a 250% FPL income limit, which the head clears at 141% FPL." -us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Its sole test was MAGI including the $8,000 403(b) distribution against the California adult expansion limit. It never applied the working-disabled buy-in category, which the head's disability and $17,443 of wages trigger and whose 250% FPL ceiling leaves 141% FPL comfortably eligible." -us,scenario_023,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,"It ran two tests — 138% FPL expansion (~$22,200) and the aged/blind/disabled non-MAGI group against SSI-related limits — and both correctly failed, but the working-disabled buy-in is a third, distinct group. It carries a 250% FPL ceiling precisely because the enrollee is working, so the earnings that sank the ABD test are what qualify the head here." -us,scenario_023,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"It checked MAGI $22,535 against ~138% FPL and then rejected the non-MAGI ABD pathway on income, missing that the buy-in for working disabled is a separate category with a 250% FPL limit rather than the ABD/SSI-related standard. The head's employment is the qualifying fact for that group, not a disqualifier." -us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for head_medicaid_eligible, so the answer failed the output contract rather than the substance. The correct answer is 1: the head is disabled and employed, qualifying under the working-disabled buy-in category with its 250% FPL income limit." -us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The submission contained no head_medicaid_eligible entry, so this is a contract failure and not a computational error. The correct answer is 1 via the working-disabled buy-in category, which the head's disability plus $17,443 of wages triggers at 141% FPL against a 250% FPL ceiling." -us,scenario_023,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"It applied the 138% FPL expansion limit to MAGI of $22,535 and then checked only for SSI receipt. Medicaid's working-disabled buy-in category needs neither SSI nor income under 138% FPL — it needs disability and employment, both stated, and admits income up to 250% FPL." -us,scenario_023,head_medicaid_eligible,ox-alpha,llm_error,categorical_eligibility,False,"It labeled the head a ""non-disabled"" adult despite the explicit disability fact, and overstated MAGI as $25,443 by omitting the $2,778 traditional 401(k) and $130 traditional IRA deductions (correct MAGI is $22,535). Both the expansion and ABD tests it ran are beside the point: disability plus employment qualifies the head under the working-disabled buy-in group, whose 250% FPL ceiling clears even the inflated $25,443 figure." -us,scenario_023,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"Its only stated basis is that household income exceeds ""the California Medicaid limit,"" a conclusion consistent with testing MAGI of $22,535 against the single 138% FPL expansion threshold. It never identified the working-disabled buy-in category that the head's disability and 24-hour-a-week employment establish, which permits income up to 250% FPL." -us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as a proxy for SSDI-based Medicare entitlement. Disability alone does not confer Medicare eligibility; the facts provide neither the required disability-benefit entitlement pathway nor end-stage renal disease, and age 28 does not satisfy age-based eligibility." -us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model incorrectly made disability by itself sufficient for under-65 Medicare eligibility. The head has no listed qualifying disability-benefit entitlement or end-stage renal disease and is under 65, so the correct eligibility value is 0." -us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable head_medicare_eligible output. The required derivation yields 0 because the 28-year-old head has no listed Medicare-qualifying entitlement or medical criterion. +us,scenario_023,head_medicaid_eligible,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Correctly found that MAGI of $22,535 exceeds 138% FPL, then checked only whether the head receives SSI. It never applied the working-disabled buy-in category that covers an employed disabled adult at 1.41x FPL, so it answered not eligible." +us,scenario_023,head_medicaid_eligible,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Treated the 138% FPL adult-expansion limit as the only Medi-Cal test and ruled the head out because income is about $22,500. It ignored the stated disability and employment, which place the head in the working-disabled buy-in category with a much higher income limit." +us,scenario_023,head_medicaid_eligible,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Claimed that no other Medi-Cal pathway qualifies once MAGI of $22,535 exceeds 138% FPL. That is wrong: the head is disabled and working, and falls in the working-disabled buy-in category, whose limit easily covers income at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Used only the 138% FPL MAGI expansion test for a single adult and stopped when MAGI of about $22,535 exceeded it. It never considered the working-disabled buy-in category available to this employed disabled head." +us,scenario_023,head_medicaid_eligible,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Checked the 138% FPL expansion test and the SSI-linked pathway, rejecting both. It skipped the working-disabled buy-in category, which does not require SSI receipt and covers an employed disabled adult at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Applied only the 138% FPL Medicaid expansion threshold to MAGI of $22,535. It never evaluated the disability-based working-disabled buy-in category that makes this working disabled head eligible." +us,scenario_023,head_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Rejected the 138% FPL expansion group and the SSI non-MAGI pathway, then concluded the head is ineligible. It never tested the working-disabled buy-in category, which covers disabled workers with income well above SSI levels and above 138% FPL." +us,scenario_023,head_medicaid_eligible,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Computed MAGI of $22,535 correctly and compared it with 138% FPL. It then treated SSI-level countable income as the only disability test. It missed the working-disabled buy-in category, which uses a far higher income ceiling for employed disabled adults, so the head qualifies at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Applied only an adult income limit and concluded the head is ineligible. It never recognized that the head, who is disabled and employed, qualifies under the working-disabled buy-in category, which has a much higher income limit than the adult expansion group." +us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Noted that the head is disabled but did not treat disability plus employment as grounds for Medicaid, and answered not eligible. The engine places this head in the working-disabled buy-in category, which grants eligibility at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Judged eligibility only against the adult Medi-Cal (138% FPL) income limit using wages plus the 403(b) distribution. It never applied the working-disabled buy-in category, which covers this employed disabled head despite income above the adult limit." +us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Said the facts establish no separate qualifying category after MAGI exceeded the adult-expansion threshold. The listed disability together with 24 hours a week of paid work establishes exactly that category: the working-disabled buy-in, which makes the head eligible." +us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Correctly found that MAGI of about $22,535 exceeds the expansion limit, and correctly noted that disability alone does not create SSI-linked eligibility. It then missed the working-disabled buy-in category, which grants Medicaid to disabled adults who are employed without requiring SSI." +us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Compared the head's MAGI only against California's adult-expansion limit and answered not eligible. It ignored the working-disabled buy-in category that applies to this disabled, employed head at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Stopped after MAGI exceeded the adult-expansion limit, saying no separate coverage pathway is specified. The stated disability and employment qualify the head for the working-disabled buy-in category, which the model never evaluated." +us,scenario_023,head_medicaid_eligible,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,Relied on the adult MAGI limit and the absence of SSI receipt. It missed that the working-disabled buy-in category needs neither and covers an employed disabled adult with MAGI at 1.41x FPL. +us,scenario_023,head_medicaid_eligible,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Tested MAGI against 138% FPL and countable income against SSI-related aged/blind/disabled limits, and rejected both. It never applied the working-disabled buy-in category, whose income ceiling sits far above SSI limits and covers this working disabled head." +us,scenario_023,head_medicaid_eligible,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Limited the analysis to the 138% FPL expansion limit and SSI recipiency. It did not recognize the working-disabled buy-in category, under which the engine finds this employed disabled head eligible." +us,scenario_023,head_medicaid_eligible,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Rejected the 138% FPL MAGI group and the SSI-level aged/blind/disabled pathway on income. It missed the working-disabled buy-in category, which has a much higher income limit for employed disabled adults and makes the head eligible." +us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for head_medicaid_eligible, so no answer could be scored. The correct derivation places the working disabled head in the working-disabled buy-in category, which yields eligible (1)." +us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for head_medicaid_eligible, so no answer could be scored. The correct derivation places the working disabled head in the working-disabled buy-in category, which yields eligible (1)." +us,scenario_023,head_medicaid_eligible,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Applied only the 138% FPL adult limit and the absence of SSI receipt. It never considered the working-disabled buy-in category that covers this disabled, employed head at 1.41x FPL." +us,scenario_023,head_medicaid_eligible,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Overstated MAGI at $25,443 by not subtracting the $2,778 traditional 401(k) deferral and the $130 traditional IRA contribution. It then tested only the 138% FPL expansion group and an SSI-level aged/blind/disabled income test. It missed the working-disabled buy-in category, which makes this employed disabled head eligible." +us,scenario_023,head_medicaid_eligible,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Concluded that household income is above 'the California Medicaid limit', treating the adult-expansion income cap as the only test. It ignored the working-disabled buy-in category, which the head qualifies for as a disabled adult working 24 hours a week." +us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the `is_disabled` flag as a stand-in for SSDI-based Medicare eligibility. Medicare for someone under 65 requires 24 months of SSDI entitlement or ESRD, and this head has $0 in Social Security disability benefits and no ESRD. At age 28 with no SSDI history, the head is ineligible, so the correct value is 0." +us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model said disability alone makes someone under 65 eligible for Medicare. It skipped the requirement of 24 months of SSDI entitlement or an ESRD diagnosis, and the head meets neither because they have no SSDI income. The head is 28 and not eligible, so the correct value is 0." +us,scenario_023,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied a disability-based Medicare rule using only the `is_disabled` flag. It ignored that the disability pathway requires 24 months of SSDI entitlement or ESRD, and the head has no Social Security disability income. At age 28 without SSDI or ESRD, the head is ineligible, so the correct value is 0." +us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no value and no reasoning for `head_medicare_eligible`, so there was nothing to score. The correct answer is 0 because the head is 28 and has no SSDI entitlement and no ESRD." us,scenario_023,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Correctly held the full $17,443 as the FICA base but applied a CA SDI rate of 1.13%, which is not a California rate for any year; the 2026 employee SDI rate is 1.3% with no wage ceiling, giving $226.75 rather than its $197.11. It also submitted 1531.40 while its own stated arithmetic produced 1531.50." -us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Stopped at federal FICA and omitted California's mandatory employee State Disability Insurance contribution entirely, dropping the $226.75 (1.3% of $17,442.65 with no wage cap) that the prompt's 'mandatory employee state payroll taxes' clause explicitly requests. Its two federal components are otherwise correct." -us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that 'California does not have a mandatory state payroll income tax on wages,' which erases the employee SDI contribution — a mandatory 1.3% withholding on all wages in 2026, worth $226.75 here. It compounded this by submitting 1335.85 while its own computation produced 1334.38." -us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Used the 2025 CA SDI rate of 1.2% instead of the 2026 rate of 1.3%, computing $209.32 of SDI where the correct figure is $226.75. Its federal Social Security and Medicare components and its use of the unreduced $17,443 wage base are both correct, so the entire $17.41 shortfall is the stale state rate." -us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Applied a 0.9% CA SDI rate — a pre-2022 rate — instead of the 2026 rate of 1.3%, understating SDI by $69.76, and then submitted 1467.32 against its own stated total of 1490.38, adding a further unexplained $23 drop. The federal FICA components were computed correctly on the full wage base." -us,scenario_023,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Applied the 2025 CA SDI rate of 1.2% rather than the 2026 rate of 1.3%, producing $209.32 of SDI instead of $226.75. It then submitted 1546 while its own reasoning stated a total of approximately $1,544, so the answer does not even match its own arithmetic." -us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly rejected a Section 125 reduction of the wage base and kept the full $17,443, but locked in a CA SDI rate of 1.1% — the 2024 rate — instead of the 2026 rate of 1.3%, computing $191.87 of SDI against the correct $226.75. The $34.86 rate shortfall is the whole error." -us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Claimed California has no mandatory employee-side state payroll tax and that CA SDI was structurally repealed; SDI remains a mandatory employee contribution, assessed at 1.3% of all wages with no taxable ceiling in 2026, worth $226.75 here. Removing that component is the entire gap between its $1,334.39 and the reference." -us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $8,089 employer-sponsored insurance premium from the FICA base to get $9,354, but that figure is the employer-side premium and is not a listed employee cafeteria-plan payroll reduction of the stated $17,443 gross wages, which are already the payroll base. It then also omitted the mandatory CA SDI contribution of $226.75." -us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Reduced the wage base to $9,354 by deducting the $8,089 employer-sponsored insurance premium, which does not reduce stated gross wages for FICA or SDI purposes — the base is the full $17,442.65. It compounded this with the 2025 SDI rate of 1.2% rather than the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Applied CA SDI at 1.1%, the 2024 rate, instead of the 2026 rate of 1.3%, computing $191.87 rather than $226.75. Both federal components and the unreduced $17,443 base are correct, so the $34.86 state rate error accounts for the full miss." -us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Estimated CA SDI at 1.1%, the 2024 rate, rather than the 2026 rate of 1.3%, producing $191.87 instead of $226.75 on the correct $17,443 base. The federal FICA components are right and the entire $34.86 shortfall is the stale SDI rate." -us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Computed only federal Social Security and Medicare and omitted California's mandatory employee SDI contribution of $226.75 (1.3% of wages, no cap), which the requested output explicitly includes. Its Medicare figure of $252.97 is also off — 1.45% of $17,443 is $252.92." -us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Excluded the $8,089 employer-sponsored insurance premium from the wage base, computing FICA and SDI on $9,354 instead of the full $17,442.65; stated gross wages are already the payroll base and that premium is not an employee pre-tax payroll deduction here. Its implied SDI rate of about 1.1% is also the 2024 rate rather than 2026's 1.3%." -us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Explicitly netted the $8,089 employer-sponsored insurance premium out of gross wages to reach a $9,354 FICA and SDI base, cutting the correct $17,442.65 base nearly in half; that premium does not reduce the stated gross wage figure for payroll tax. It also used the 2024 SDI rate of 1.1% instead of the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Kept the correct $17,443 base and included CA SDI but overshot the total by $34, implying a combined rate near 9.14% and an SDI rate of roughly 1.49%; the 2026 employee SDI rate is 1.3%, giving $226.75 and a total of $1,561.12. It named no rate, so the $1,595 is an unanchored round-number estimate rather than a computed sum." -us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Applied CA SDI at 1.1%, the 2024 rate, instead of the 2026 rate of 1.3%, yielding $191.87 rather than $226.75. Federal Social Security and Medicare were computed correctly on the unreduced wage base, so the stale state rate is the sole source of the $34.86 gap." -us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 is 7.65% plus 1% of $9,354 — it stripped the $8,089 employer-sponsored insurance premium out of the wage base, which does not reduce the stated $17,442.65 of gross wages for FICA or SDI, and then applied a 1% SDI rate instead of the 2026 rate of 1.3%. Both federal components should have been computed on the full wage figure." -us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Used a 1.1% CA SDI rate — the 2024 rate — rather than the 2026 rate of 1.3%, computing $191.87 instead of $226.75 on the correct $17,443 base. Its federal FICA components are exact, so the entire miss is the outdated state rate." -us,scenario_023,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"Correctly kept both traditional and Roth 401(k) contributions in the FICA base but applied the 2025 CA SDI rate of 1.2% instead of the 2026 rate of 1.3%, computing $209.32 rather than $226.75. The $17.41 rate shortfall is the entire error." -us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Stated that 'California has no mandatory employee state payroll tax,' dropping the $226.75 employee SDI contribution assessed at 1.3% of all wages in 2026. It then submitted 1329 while its own reasoning computed 1334.39, so the answer contradicts its own arithmetic on top of the missing state component." -us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Asserted there is 'no mandatory state payroll tax in California,' omitting the employee SDI contribution of $226.75 (1.3% of wages, no ceiling, 2026). Its submitted 1375.966 does not even match its own FICA-only method, which yields $1,334.39 on $17,443." -us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Omitted the mandatory CA SDI contribution of $226.75 and then botched the federal arithmetic: it wrote 6.2% of $17,443 as $1,070.1 when it is $1,081.44, and its own two stated terms sum to $1,322.9, not the $1,208 it submitted. The correct derivation is $1,081.44 + $252.92 + $226.75 = $1,561.12." -us,scenario_023,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Used an estimated CA SDI rate of 1.2%, the 2025 rate, rather than the 2026 rate of 1.3%, so its combined 8.85% should have been 8.95% of $17,442.65. That single rate choice produces $209.32 of SDI instead of $226.75 and accounts for the whole $17.41 miss." -us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Computed only 7.65% of wages for Social Security and Medicare and omitted California's mandatory employee SDI contribution, which adds 1.3% of the full $17,442.65, or $226.75, in 2026. The requested output explicitly includes mandatory employee state payroll taxes." -us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Reported federal Social Security and Medicare only, leaving out the mandatory California employee SDI withholding of $226.75 at 1.3% of uncapped wages for 2026. Its federal figures are correct, so the omitted state component is the entire $227 gap." -us,scenario_023,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"Correctly held retirement deferrals in the FICA base but projected the CA SDI rate at 1.2%, the 2025 rate, instead of the 2026 rate of 1.3%, giving $209.32 rather than $226.75. Its rounding to $1,544 from $1,543.71 is immaterial next to the $17.41 rate error." -us,scenario_023,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Applied the CA SDI rate of 1.2% — the 2025 rate — rather than 1.3% for 2026, computing $209.32 of SDI against the correct $226.75. Federal FICA at 7.65% of the full $17,443 was handled correctly." -us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Deducted the $2,778 traditional 401(k) contribution from the FICA and SDI base to reach $14,665; elective deferrals are wages for Social Security and Medicare under IRC 3121(a)(5)(D) and for CA SDI, so the base stays at $17,442.65. It also used the 2024 SDI rate of 1.1% instead of the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Treated the $8,089 employer-sponsored insurance premium as a pre-tax reduction of wages, computing FICA and SDI on $9,354 rather than the full $17,442.65 of stated gross wages. It also used the 2025 SDI rate of 1.2% rather than the 2026 rate of 1.3%." -us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for payroll_tax and no explanation was produced, so the submission carries no substantive computation to evaluate. The required derivation is Social Security $1,081.44 plus Medicare $252.92 plus CA employee SDI $226.75 on $17,442.65 of wages, totaling $1,561.12." -us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model produced no payroll_tax value and no reasoning, leaving nothing substantive to assess. The correct answer is 6.2% plus 1.45% federal FICA plus 1.3% California employee SDI on the full $17,442.65 wage base, or $1,561.12." -us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare and omitted the mandatory California employee SDI contribution of $226.75, which is 1.3% of all wages in 2026 with no taxable wage ceiling. Its federal components are correct to the penny, so the missing state payroll tax is the entire error." -us,scenario_023,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Correctly kept 401(k) and 403(b) deferrals in the FICA base but applied the CA employee SDI rate of 1.2%, the 2025 rate, instead of the 2026 rate of 1.3%, producing $209 rather than $226.75. That single stale rate is the whole $17 shortfall." -us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Twice computed a total of $1,526.26 including CA SDI, then submitted the FICA-only $1,334.39, discarding the state disability component after correctly identifying it as a mandatory employee payroll tax. Its SDI rate of 1.1% was the 2024 rate anyway; the 2026 rate is 1.3% with no ceiling, giving $226.75." -us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its stated method — Social Security plus Medicare on wages — yields $1,334.39 on $17,443, but it submitted $261.65, which is roughly 1.5% of wages rather than the 7.65% combined FICA rate. It also omitted the mandatory California employee SDI contribution of $226.75, so the correct total is $1,561.12." +us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,Correctly included CA SDI on the full wage base but used an invented 1.13% rate ($197.11) instead of the 2026 rate of 1.3% ($226.75). Its submitted 1531.4 also doesn't match its own sum of 1531.50. +us,scenario_023,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Computed only federal FICA ($1,334.39) and left out California's mandatory employee SDI contribution, which is 1.3% of all wages in 2026 ($226.75)." +us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory employee payroll tax, so it dropped the 1.3% SDI contribution ($226.75). It then inflated its own $1,334.38 FICA total to 1335.85 with no basis." +us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75), which understates the total by about $17." +us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"Used the outdated 2023 CA SDI rate of 0.9% ($156.99) instead of the 2026 rate of 1.3% ($226.75). It also submitted 1467.32, which differs from its own computed sum of 1490.38." +us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"Applied the 2025 CA SDI rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75), then rounded its $1,544 total up to 1546." +us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly kept the full $17,443 wage base but applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75)." +us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Wrongly assumed California has no mandatory employee payroll tax and left out CA SDI, which is 1.3% of all wages in 2026 ($226.75). It reported FICA alone." +us,scenario_023,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Explicitly excluded CA SDI, even though the requested payroll_tax output covers mandatory employee state payroll taxes. That dropped the 1.3% SDI contribution ($226.75)." +us,scenario_023,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $8,089 employer-sponsored insurance premiums from the stated gross wages. The FICA base is the full $17,442.65. It also said there is no state payroll tax, which dropped the 1.3% CA SDI." +us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Shrank the wage base to $9,354 by subtracting the $8,089 employer-sponsored insurance premiums, which do not reduce stated gross wages for FICA or SDI. It also used the 1.2% SDI rate instead of the 2026 rate of 1.3%." +us,scenario_023,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used a combined 8.85% rate, which includes a 1.2% CA SDI rate. The correct combined rate is 8.95%, because the 2026 SDI rate is 1.3%." +us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,Estimated CA SDI at the 2024 rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Reported only federal FICA and left out CA SDI at 1.3% ($226.75). It also miscalculated Medicare as $252.97 instead of $252.92. +us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Excluded the $8,089 employer-sponsored insurance premiums from the wage base. Its $818 equals 8.75% (7.65% FICA plus 1.1% SDI) of $9,354. The correct figure applies 7.65% plus the 2026 SDI rate of 1.3% to the full $17,442.65." +us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Explicitly subtracted $8,089 of employer-sponsored insurance premiums to get a $9,354 base. Stated gross wages are the payroll tax base. It also used the outdated 1.1% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Correctly used the full $17,443 wage base but overstated CA SDI. Its $1,595 total implies about $261 of SDI (roughly 1.5%), while the 2026 rate of 1.3% gives $226.75." +us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $809.12 equals 8.65% of $9,354. That means it subtracted the $8,089 employer-sponsored insurance premiums from the wage base and used a 1.0% SDI rate. The correct calculation uses the full $17,442.65 and a 1.3% SDI rate." +us,scenario_023,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,Applied the 2024 CA SDI rate of 1.1% ($191.87) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,Correctly included CA SDI on full wages but used the 2025 rate of 1.2% ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory employee payroll tax, which dropped the 1.3% SDI ($226.75). It then changed its own $1,334.39 FICA sum to an unexplained 1,329." +us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said California has no mandatory state payroll tax and so left out CA SDI at 1.3% ($226.75). Its 1375.966 doesn't match its stated 7.65% of $17,443, which is $1,334.39." +us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out CA SDI at 1.3% ($226.75) and also got the FICA arithmetic wrong. It computed 6.2% of $17,443 as $1,070.10 (should be $1,081.47) and added the parts to 1,208 (should be about $1,323)." +us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at the 2025 rate of 1.2% instead of the 2026 rate of 1.3%, for a combined 8.85% instead of 8.95% of wages." +us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,Computed only 7.65% FICA and left out California's mandatory employee SDI contribution of 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,gpt-6-luna,llm_error,thresholds_rates,False,Estimated CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Used the correct 2026 SDI rate of 1.3% but applied it, along with FICA, to $9,354. That base comes from wrongly subtracting the $8,089 employer-sponsored insurance premiums from the $17,442.65 of gross wages." +us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,Computed only Social Security and Medicare and left out CA SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Carried the 2025 CA SDI rate of 1.2% ($209.32) forward to 2026, but the 2026 employee rate is 1.3% ($226.75)." +us,scenario_023,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,Applied the 1.2% CA SDI rate ($209.32) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"Explicitly carried forward the 2025 CA SDI rate of 1.2% ($209.32), but the 2026 employee rate is 1.3% ($226.75)." +us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted the $2,778 traditional 401(k) deferral from the FICA and SDI base. Elective deferrals stay in FICA wages, so the base is the full $17,442.65. It also used the outdated 1.1% SDI rate instead of 1.3%." +us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Treated the $8,089 employer-sponsored insurance premiums as pre-tax deductions from the $17,443 gross wages, which gave a $9,354 base. It also used 1.2% for SDI instead of the 2026 rate of 1.3%." +us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no payroll_tax value and no explanation, so there was nothing to score." +us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"Returned no payroll_tax value and no explanation, so there was nothing to score." +us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,Summed only employee Social Security and Medicare and left out California's mandatory employee SDI at 1.3% of wages ($226.75). +us,scenario_023,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,Applied CA SDI at 1.2% ($209) instead of the 2026 rate of 1.3% ($226.75). +us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Its reasoning worked out a total of $1,526.26 that included CA SDI, though at the outdated 1.1% rate. It then submitted the FICA-only figure of 1334.39, which drops the mandatory SDI contribution entirely." +us,scenario_023,payroll_tax,qwen3.8-max,llm_error,other,False,"Its $261.65 equals exactly 1.5% of $17,443, so it never applied the 6.2% Social Security and 1.45% Medicare rates that it said it was computing. The correct figure is $1,334.36 of FICA plus $226.75 of CA SDI." us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It read the head's `is_disabled` flag as USDA elderly-or-disabled status, so it granted an excess medical deduction and an uncapped excess shelter deduction of $1,218/month instead of holding the shelter deduction to the FY2026 $744/month cap, driving net income to $345 rather than $876.53. USDA elderly-or-disabled status requires age 60+ or receipt of a qualifying disability benefit (SSI, SSDI, VA), none of which this household has. It then submitted $3,936 after its own arithmetic ended at $194 x 12 = $2,328." -us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It treated the disability flag as removing the shelter cap and added a utility allowance on top of $2,000 rent, zeroing net income and paying the full $298 maximum allotment for 12 months. The shelter deduction is capped at $744/month because no member is age 60+ or receives a qualifying disability benefit, leaving net income of $876.53, a $262.80 expected contribution, and a $35.20 monthly benefit that rises to $48.18 for October-December 2026." -us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It applied the 130% FPL gross income test ($1,695.42/month) to gross income of $2,120.25 and ignored California's broad-based categorical eligibility through TANF non-cash assistance, which lifts the gross limit to 200% FPL ($2,608.33/month) and waives the asset test. Its claim that $24,000 of rent disqualifies the household inverts the rule: rent produces the $744/month capped shelter deduction that brings net income down to $876.53, only 67% of the $1,304.17 monthly poverty guideline." -us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It explicitly declared the shelter deduction uncapped for a 'disabled household' and took $14,837 annually plus a $1,180 medical deduction, when the head's disability flag does not confer USDA elderly-or-disabled status and the shelter deduction is capped at $744/month. It then reported $3,768 while its own derivation produced $2,529, neither of which matches the $35.20/month benefit that a $876.53 net income yields." -us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It applied both disability-contingent deductions the household does not qualify for - a $130/month excess medical deduction and an uncapped $1,167/month shelter deduction - producing net income of $498 instead of $876.53 under the $744 shelter cap. It then discarded its own $1,788 result and submitted $2,549 by asserting a $212/month benefit with no derivation." -us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It misstated California's BBCE gross limit as roughly $2,510 and concluded that $2,120.25 gross fails it, when the 200% FPL limit is $2,608.33/month and the household passes. It also asserted that net income exceeds the net limit after the shelter cap, but the capped $744 shelter deduction leaves net income of $876.53, which is 67% of the $1,304.17 poverty guideline and passes the 100% net test, giving $298 - $262.80 = $35.20/month." -us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Its long derivation hinges on the statement 'since the head IS disabled, there is no cap on the shelter deduction,' taking $1,223.90 in excess shelter plus a $73.33 medical deduction and arriving at net income of $328.30 instead of $876.53. Because no member is age 60+ or receives a qualifying disability benefit, the shelter deduction stops at the $744/month cap and no medical deduction applies; it compounded this with the FY2025 $292 allotment rather than $298." -us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It failed the household on the 130% FPL gross test and never applied California's broad-based categorical eligibility through TANF non-cash assistance, which sets the gross screen at 200% FPL ($2,608.33/month) for gross income of $2,120.25. Its assertion that net income remains too high 'given high rent' reverses the effect of rent: the $744 capped shelter deduction plus the $209 standard and 20% earned-income deductions produce net income of $876.53, well under the $1,304.17 net limit." -us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It took the excess shelter deduction 'uncapped because head is disabled,' reaching net income of about $445.80 rather than the $876.53 that results from holding the deduction to the FY2026 $744/month cap. The disability flag alone does not create USDA elderly-or-disabled status, so the cap applies and the benefit is $298 - $262.80 = $35.20/month." -us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It claimed a $747.42 medical deduction and a $1,560.94 excess shelter deduction on disability grounds, zeroing net income and paying the $298 maximum allotment for 12 months. Neither deduction is available: USDA elderly-or-disabled status requires age 60+ or a qualifying disability benefit, so shelter is capped at $744 and net income is $876.53, producing $35.20/month rising to $48.18 in October-December." -us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It applied a $98.33 excess medical deduction and an uncapped $1,238.90 excess shelter deduction, giving net income of $283.30 instead of $876.53 under the $744 shelter cap. Both deductions are gated on USDA elderly-or-disabled status the household lacks, and its $299 allotment also misses the $298 FY2026 figure and the October 2026 increase to $304.68." -us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It stacked a $98.33 medical deduction and an uncapped shelter deduction of rent plus a $595 utility allowance on the strength of the head's disability flag, zeroing net income and paying a $291 monthly allotment. The shelter deduction is capped at $744/month absent USDA elderly-or-disabled status, so net income is $876.53 and the benefit is $35.20/month at the correct $298 FY2026 allotment." -us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $3,492 is exactly 12 x $291, the full one-person allotment, which follows only from zero net income - the result of an uncapped excess shelter deduction against $2,000 monthly rent. With the shelter deduction capped at $744 because no member is age 60+ or receives a qualifying disability benefit, net income is $876.53, the expected contribution is $262.80, and the benefit is $35.20/month rising to $48.18 for October-December 2026." -us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It stated outright that 'uncapped excess shelter deductions (due to disability) and medical deductions reduce net countable income to zero,' paying 12 months of a $291 maximum allotment. The head's disability flag does not confer USDA elderly-or-disabled status, so the shelter deduction stops at $744/month, no medical deduction applies, and net income of $876.53 leaves a $35.20 monthly benefit." -us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It cited 'uncapped shelter costs and medical deductions for a disabled head' to reach net monthly income near $300; its $2,413 implies a $201 monthly benefit and net income of roughly $323. Holding the shelter deduction to the $744 cap and dropping the medical deduction gives net income of $876.53 and a $35.20 monthly benefit that rises to $48.18 in October-December 2026." -us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It returned zero on an unexplained assertion that income or resources exceed the threshold, a result consistent only with applying the 130% FPL gross test ($1,695.42/month) to $2,120.25 gross while ignoring California's TANF-funded broad-based categorical eligibility at 200% FPL. Assets of $130 clear every applicable limit, and the $744 capped shelter deduction leaves net income of $876.53, well under the $1,304.17 net limit, for a $35.20 monthly benefit." -us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It applied 'uncapped shelter expenses for a disabled individual'; its $1,980 implies a $165 monthly benefit and net income near $443. The FY2026 $744 shelter cap applies because no member is age 60+ or receives a qualifying disability benefit, so net income is $876.53 and the benefit is $298 - $262.80 = $35.20/month." -us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It declared the household a 'disabled household with uncapped excess shelter expense deductions,' zeroed net income, and paid 12 months of the $298 maximum allotment. The uncapped shelter deduction requires USDA elderly-or-disabled status this household lacks; with the $744 cap, net income of $876.53 produces $35.20/month and $48.18 for October-December, totaling $461.34." -us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It applied an uncapped excess shelter deduction on $2,000 monthly rent plus medical deductions to reach $2,529, an implied $211 monthly benefit against net income near $290. The shelter deduction is capped at $744/month and no medical deduction is allowed, so net income is $876.53 and the monthly benefit is $35.20." -us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It took a $1,180 medical deduction and a $14,836.80 'uncapped excess shelter deduction' on the basis of the head's disability, producing annual net income of $3,489.60 instead of the $10,518 that the $744/month shelter cap yields. It also used the FY2025 $292 allotment ($3,504/year) rather than $298 rising to $304.68 in October 2026." -us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It omitted the excess shelter deduction entirely, subtracting only the 20% earned income deduction and a standard deduction to reach annual net income of $21,873 and concluding the benefit is $0, then submitted $871 as an unreasoned guess that contradicts its own explanation. The $24,000 rent generates the capped $744/month shelter deduction, which brings monthly net income to $876.53 and the benefit to $35.20/month rising to $48.18 for October-December." -us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"It returned zero on the bare assertion that income and assets are not low enough, a conclusion consistent only with applying the 130% FPL gross test to $2,120.25 monthly gross and skipping California's broad-based categorical eligibility through TANF non-cash assistance, which sets the gross screen at 200% FPL ($2,608.33) and waives the $130 asset test. After the 20% earned income, $209 standard, and $744 capped shelter deductions, net income is $876.53 and the benefit is $35.20/month." -us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It asserted without derivation that the inputs do not support benefit receipt, a zero consistent with failing the household on the 130% FPL gross test rather than California's 200% FPL BBCE limit through TANF non-cash assistance. The household clears that limit at $2,120.25 gross, and the $744 capped shelter deduction leaves net income of $876.53, producing $35.20/month and $461.34 for the year." -us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It stated that the 'medical-expense and uncapped excess-shelter deductions' reduce countable net income to zero for a 'disabled California SNAP household,' paying 12 months of the $298 maximum allotment. Both deductions require USDA elderly-or-disabled status, which needs age 60+ or receipt of a qualifying disability benefit; with the $744 shelter cap, net income is $876.53 and the benefit is $35.20/month, rising to $48.18 in October-December." -us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It invoked 'the medical and uncapped shelter deductions' available to a disabled household and paid the full $298 monthly allotment. The head's disability flag does not create USDA elderly-or-disabled status, so shelter is capped at $744/month, net income is $876.53, and the expected contribution of $262.80 leaves a $35.20 monthly benefit." -us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,It applied an 'uncapped disability shelter deduction' plus a medical deduction to reach an implied net income near $320 and a $202 monthly benefit. Neither deduction applies without USDA elderly-or-disabled status; the $744 capped shelter deduction leaves net income of $876.53 and a $35.20 monthly benefit. -us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Its zero is consistent with allowing only the 20% earned-income and $209 standard deductions against $2,120.25 gross, leaving net income of $1,620.53 whose 30% contribution of $486 exceeds the $298 allotment. The $24,000 rent produces a $744/month capped excess shelter deduction, so net income is $876.53, the contribution is $262.80, and the benefit is $35.20/month rising to $48.18 for October-December." -us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It applied a California $150 standard medical deduction and 'uncapped excess shelter deductions for disability,' reaching net monthly income of $205.80 and a $236.26 benefit. Both rest on USDA elderly-or-disabled status the household lacks, so the shelter deduction stops at $744, net income is $876.53, and the benefit is $35.20/month." -us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It asserted that both income and assets exceed SNAP limits, but $130 in bank assets is far below any applicable resource limit and California's TANF-funded broad-based categorical eligibility waives the asset test and raises the gross screen to 200% FPL ($2,608.33), which $2,120.25 clears. After the $744 capped shelter deduction, net income of $876.53 is 67% of the poverty guideline and yields $35.20/month." -us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It used 'disabled single HH uses net-income test only' to justify a $1,180 excess medical deduction and an uncapped $14,837 shelter deduction, producing annual net income of $3,490 rather than the $10,518 implied by the $744/month cap. It also used the FY2025 $292 allotment; the correct benefit is $35.20/month at $298, rising to $48.18 in October-December 2026." -us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It correctly cleared the 200% FPL BBCE gross test but then applied excess medical and 'uncapped excess-shelter' deductions on the $24,000 rent to zero out net income and pay the maximum allotment. The uncapped shelter deduction requires age 60+ or receipt of a qualifying disability benefit; with the $744 cap net income is $876.53, and the allotment is $298 (not the FY2025 $292) less a $262.80 contribution." -us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"It waived the gross income test and took a $1,180 medical deduction plus a $17,128 'no cap for disabled' shelter deduction, zeroing net income and paying an assumed $300 monthly maximum allotment. The FY2026 $744 monthly shelter cap applies because the household contains no USDA elderly-or-disabled member, leaving net income of $876.53 and a $35.20 monthly benefit at the actual $298 allotment." -us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"It applied a medical deduction 'for the disabled' and an uncapped shelter deduction against the $24,000 rent to reach net income near $300 and a $210 monthly benefit. Capping the shelter deduction at $744 and dropping the medical deduction gives net income of $876.53, an expected contribution of $262.80, and a $35.20 monthly benefit that rises to $48.18 in October-December." -us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"It stated the household 'contains a disabled member, exempting it from the SNAP gross income test' and then applied an uncapped excess shelter deduction plus a medical deduction to zero out net income. The disability flag does not confer USDA elderly-or-disabled status; the gross test is cleared instead through California's 200% FPL BBCE, the shelter deduction is capped at $744, and net income of $876.53 leaves $35.20/month against the $298 FY2026 allotment it also understated as $292." -us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for snap, so no substantive computation exists to evaluate. The required derivation applies the 20% earned income, $209 standard, and $744 capped shelter deductions to $2,120.25 monthly gross for net income of $876.53, yielding $298 - $262.80 = $35.20/month and $461.34 for the year." -us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It computed gross income of $22,535 by subtracting the $2,778 traditional 401(k) and $130 traditional IRA contributions from the $25,443 total, when SNAP counts gross earned income before retirement contributions, and then failed the household on the 130% FPL limit while ignoring California's broad-based categorical eligibility through TANF non-cash assistance, which raises the gross screen to 200% FPL ($2,608.33/month). Correct gross of $2,120.25 clears that screen, and net income of $876.53 after the $744 capped shelter deduction yields $35.20/month." -us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It correctly applied the 200% FPL BBCE test and the capped $744 excess shelter deduction, but used the FY2025 $204 standard deduction instead of the FY2026 $209, leaving net income of about $882 rather than $876.53, and then held the $298 allotment flat across all 12 months. PolicyEngine's benefit is $35.20 for the first nine months and $48.18 for October-December 2026, when the allotment rises to $304.68 and the contribution falls to $256.50, totaling $461.34." -us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It excluded the $8,000 taxable 403(b) distribution from countable unearned income, using gross of only $1,454/month, omitted the excess shelter deduction on $24,000 of rent entirely, and then paid a flat $258 monthly allotment while stating net income of $944 - abandoning the 30% expected contribution its own net figure requires. Counting the distribution and applying the $744 capped shelter deduction gives net income of $876.53 and a benefit of $298 - $262.80 = $35.20/month." -us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It returned zero on an unexplained claim that income and resources are too high, consistent with applying the 130% FPL gross test to $2,120.25 monthly gross and the federal resource limit to $130 in assets, both of which California's TANF-funded broad-based categorical eligibility displaces with a 200% FPL gross screen and no asset test. Net income after the 20% earned income, $209 standard, and $744 capped shelter deductions is $876.53, which passes the 100% FPG net test and produces $35.20/month." +us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"The model treated the head's disabled flag as making this a SNAP elderly/disabled household, but the head receives no SSI or SSDI, so the model should not have applied a medical deduction or an uncapped shelter deduction. The correct $744 shelter cap leaves net income of about $876/month. The model also submitted $3,936, which does not match its own computed $2,328." +us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled and used an uncapped shelter deduction (rent plus SUA) plus a medical deduction to bring net income to $0 and award the $298 maximum. The head receives no disability-based benefit, so the shelter deduction is capped at $744, and net income of about $876 gives only $35.20/month." +us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model applied the federal 130% FPL gross income test and a $2,500 asset limit. It missed that California's broad-based categorical eligibility (TANF non-cash) raises the gross limit to 200% FPL, which $2,120/month passes. It also never applied the $744 capped shelter deduction, which brings net income to about $876, below the $1,304 net limit." +us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model gave the head elderly/disabled SNAP treatment, deducting medical costs over $35 and taking an uncapped $14,837/year shelter deduction. The head receives no SSI or SSDI, so the $744/month cap applies and no medical deduction is allowed. The model then submitted $3,768, which does not match its own computed $2,529." +us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled, taking an uncapped $1,167/month shelter deduction and a medical deduction. It also used $121 instead of $290.72 for the 20% earned-income deduction. Without disability-benefit receipt, the shelter deduction is capped at $744, net income is about $876, and the benefit is $35.20/month, not the $2,549/year it submitted." +us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"The model put California's 200% FPL gross limit at about $2,510/month and concluded that $2,120 fails it. The actual limit for one person is about $2,608, so the household passes. With the $744 shelter cap, net income of about $876 still produces a positive $35.20/month benefit." +us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model assumed the disabled flag removes the shelter cap and allows a medical deduction, which brought net income to $0 and gave the maximum $298/month. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, which the head does not, so the $744 cap applies and net income is about $876." +us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"The model applied the elderly/disabled rules because the head is disabled: it took a $73/month medical deduction and an uncapped $1,224 shelter deduction. The head receives no disability-based benefit, so there is no medical deduction and the shelter deduction is capped at $744. That makes net income about $876 and the benefit $35.20/month, not $194." +us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model used the 130% FPL gross income test and found the household ineligible. California broad-based categorical eligibility uses a 200% FPL gross screen, which $2,120/month passes, and after the $744 capped shelter deduction net income of about $876 yields a positive benefit." +us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled, applying a medical deduction and an uncapped shelter deduction to reach about $346 net income. Without receipt of a disability-based benefit, the shelter deduction is capped at $744, net income is about $876, and the monthly benefit is $35.20." +us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model took an uncapped excess shelter deduction because the head is disabled, which reduced net income to about $446. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies, net income is about $876, and the frozen reference's annual benefit is $461, not $1,995. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model treated the head as SNAP-disabled. It took a $747/month medical deduction, which overstates listed medical costs by counting employer premiums, and an uncapped $1,561 shelter deduction to reach $0 net income. The head gets no disability benefit, so no medical deduction is allowed and the shelter deduction is capped at $744, leaving about $876 net income." +us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied elderly/disabled treatment, with a $98 excess medical deduction and an uncapped $1,239 shelter deduction, to reach $283 net income. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and no medical deduction applies, which gives about $876 net income and a $35.20/month benefit." +us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model waived the gross test and used the medical and uncapped shelter deductions because the head is disabled, reaching $344 net income. SNAP disabled status requires disability-benefit receipt, so the $744 shelter cap applies, net income is about $876, and the benefit is $35.20/month." +us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model applied a medical deduction and an uncapped shelter deduction (rent plus SUA) because the head is disabled, which reduced net income to $0 and gave the maximum. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is about $876. The model also used an outdated $291 maximum instead of $298." +us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"The $3,492 answer equals a $291/month maximum allotment for 12 months, which implies zero net income from an uncapped disabled-household shelter deduction. The head is not SNAP-disabled, since no SSI or SSDI is received, so the $744 shelter cap leaves about $876 net income and a $35.20/month benefit." +us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model claimed uncapped shelter and medical deductions because of disability and awarded the maximum. SNAP disabled status requires receiving a disability-based benefit, which the head does not, so the shelter deduction is capped at $744, there is no medical deduction, and net income is about $876." +us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model assumed uncapped shelter and medical deductions for a disabled head, which put net income at about $300/month. The head receives no SSI or SSDI, so the $744 shelter cap applies and net income is about $876. That gives the frozen reference's $35.20/month for January–September, not about $200. The frozen reference's higher October–December amount uses the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"The model concluded that income exceeds the SNAP threshold. California's 200% FPL broad-based categorical eligibility screen admits $2,120/month gross, and after the earned-income, standard and $744 capped shelter deductions, net income of about $876 passes the net test with a positive benefit." +us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model applied an uncapped shelter deduction because the head is disabled. The head is not SNAP-disabled without receiving a disability benefit, so the shelter deduction is capped at $744, net income is about $876, and the frozen reference's annual benefit is $461, not $1,980. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated the household as disabled with an uncapped shelter deduction, which reduced net income to $0 and gave the $298 maximum. Without SSI or SSDI receipt, the $744 cap applies and about $876 net income leaves $35.20/month." +us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"The model correctly found the student exemption through 24 weekly work hours but wrongly applied uncapped shelter and medical deductions for disability. The head receives no disability-based benefit, so the $744 shelter cap applies and net income is about $876." +us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"The model exempted the household from the gross test and applied the $35 medical threshold and an uncapped $14,837/year shelter deduction because the head is disabled. SNAP disabled status requires receipt of a disability benefit, so the shelter deduction is capped at $744/month, no medical deduction applies, and net income is about $876/month." +us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"The model left out the excess shelter deduction entirely, even though rent is $2,000/month, and took 30% of annual net income but compared it against a monthly maximum allotment, which drove its benefit to $0. It then submitted an unsupported $871. The capped $744 shelter deduction brings net income to about $876/month and the frozen reference's benefit to $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model asserted without calculation that income and assets are too high. Assets of $130 are far below any limit, gross income passes California's 200% FPL screen, and the $744 capped shelter deduction reduces net income to about $876, which yields $35.20/month." +us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model returned $0 on the premise that eligibility was not supported, and never ran the SNAP test. The household is categorically eligible through California's TANF non-cash broad-based categorical eligibility, passes the net income test at about $876 against $1,304, and receives $461 for the year in the frozen reference, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"The model applied medical and uncapped shelter deductions as a disabled household to reach $0 net income and the $298 maximum. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and net income is about $876, giving $35.20/month." +us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model assumed the disabled head unlocks the medical deduction and an uncapped shelter deduction, producing the maximum allotment. SNAP disabled status requires receiving a disability-based benefit, so the $744 cap applies and the frozen reference's benefit is $35.20/month for January–September 2026, with a higher October–December amount from the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model correctly applied the student work exemption but treated the head as SNAP-disabled, with a medical deduction and an uncapped shelter deduction. Without disability-benefit receipt, the shelter deduction is capped at $744 and net income is about $876, giving the frozen reference's $461/year instead of $2,424. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model concluded that earnings plus the 403(b) distribution leave no benefit, which ignores the excess shelter deduction on $2,000/month rent. Even capped at $744, that deduction brings net income to about $876, and 30% of that ($262.80) is below the $298 maximum." +us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model applied California's standard medical deduction and an uncapped shelter deduction for disability, reaching $205.80 net income. Both depend on elderly/disabled status, which requires receiving a disability benefit the head does not get, so only the $744 capped shelter deduction applies and net income is about $876." +us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model treated the disabled head as qualifying for medical and uncapped shelter deductions, which gave the maximum $298/month. The head receives no SSI or SSDI, so the $744 shelter cap applies and net income of about $876 yields $35.20/month." +us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model used an uncapped shelter deduction because of disability to bring countable income to zero. SNAP disabled status requires disability-benefit receipt, so the shelter deduction is capped at $744, leaving about $876 net income and a $35.20/month benefit." +us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied an $880 disability medical deduction and an uncapped shelter deduction to reach $3,850/year net income. The head is not SNAP-disabled without SSI or SSDI receipt, so the $744/month cap applies, net income is about $876/month, and the frozen reference's annual benefit is $461, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"The model claimed income and assets exceed limits after the housing deduction. Assets are only $130, gross income passes California's 200% FPL broad-based categorical eligibility screen, and net income after the $744 capped shelter deduction is about $876, below the $1,304 net limit." +us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"The model applied the net-income-only test with an excess medical deduction and an uncapped $14,837 shelter deduction because the head is disabled. SNAP disabled status requires receiving a disability benefit, so the shelter deduction is capped at $744/month, no medical deduction applies, and the frozen reference's benefit is $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"The model correctly passed the 200% FPL broad-based categorical eligibility gross test but used excess medical and uncapped shelter deductions for disability to reach $0 net income. The head receives no SSI or SSDI, so the $744 cap leaves about $876 net income. The model also used an outdated $292 maximum instead of $298." +us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"The model applied an excess medical deduction and an uncapped shelter deduction because the head is disabled, reaching $283 net income. The head is not SNAP-disabled without disability-benefit receipt, so the shelter deduction is capped at $744 and net income is about $876, which gives $35.20/month." +us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model left the $8,000 403(b) distribution out of income and treated the head as SNAP-disabled, with a medical deduction and an uncapped shelter deduction, to reach $0 net income. Counting the $666.67/month distribution and capping shelter at $744 gives about $876 net income and $35.20/month." +us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"The model applied a medical deduction and an uncapped shelter deduction for a disabled head to reach about $300 net income. SNAP disabled status requires receiving SSI, SSDI or a similar benefit, so the $744 cap applies and net income is about $876, which gives the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." +us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model exempted the household from the gross test and used medical and uncapped shelter deductions because the head is disabled, which produced $0 net income. The head receives no disability-based benefit, so the shelter deduction is capped at $744 and net income is about $876. The model also used an outdated $292 maximum." +us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was no answer to score against the $461.34 reference." +us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"The model used an unexplained $22,535 gross income and applied the federal 130% FPL gross limit. California's broad-based categorical eligibility raises the gross screen to 200% FPL, which the actual $25,443 ($2,120/month) passes, and the net income of about $876 yields a positive benefit." +us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"The model correctly capped the shelter deduction at $744 and applied the FY2026 $298 maximum to all 12 months, as the release's SNAP convention (c_snap_hold_fy2026) does, but it used the outdated $204 standard deduction instead of $209. The frozen reference's $48.18/month for October–December ($304.68 maximum, $256.50 contribution) uses the engine's projected FY2027 schedule, which is how it reaches $461.34; with the convention's FY2026 schedule all year, the current engine gives $408 ($34 × 12, which also reflects the SNAP rounding fixes) against its $402." +us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model left out the $8,000 403(b) distribution and the excess shelter deduction, used a $258 maximum allotment, and then awarded the full maximum despite computing $944 net income. The correct figures are $2,120.25 gross, the $744 capped shelter deduction and about $876 net income, which give $35.20/month." +us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"The model asserted without calculation that income and resources are too high. Assets of $130 are trivial, gross income passes California's 200% FPL broad-based categorical eligibility screen, and the $744 capped shelter deduction brings net income to about $876, below the $1,304 net limit, which yields the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408)." us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable." us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." -us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reconstructed the entire chain correctly — AGI ~$22,535, standard deduction ~$5,700, taxable income ~$16,800, exemption credit ~$154 plus the $60 renter's credit — but understated the bracket tax as ""roughly $200"" instead of $223.27 and then declared the residual ""effectively 0"" rather than performing the subtraction. The exact arithmetic is $223.27 - $156.47 - $60 = $6.80; the model discarded a real positive liability as rounding noise." -us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $5,880 standard deduction instead of the 2026 value of $5,706 and a $153 exemption credit instead of $156.47, shrinking the taxable base to $16,655 and the tax to $222. Correct parameters give $16,828.34 taxable, $223.27 of bracket tax, and $216.47 of nonrefundable credits, leaving $6.80 rather than $9." -us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built California income from gross wages of $17,443 alone, omitting the $8,000 taxable 403(b) distribution and the $2,778 401(k) and $130 IRA adjustments that produce the $22,534.34 AGI, and used a stale $5,202 standard deduction. It then zeroed the tax with CalEITC, which is a refundable credit reported in state_refundable_credits and never reduces tax before refundable credits; the actual nonrefundable set is the $156.47 exemption credit plus the $60 renter's credit, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It computed AGI correctly at ~$22,535 but then reported tax on ~$17,000 of taxable income as ""~$170"" when the 1%/2% brackets yield $223.27, and it applied only a ~$149 exemption credit while omitting the $60 nonrefundable renter's credit. Even its own $170 figure less $149 leaves a positive balance, which it rounded to $0 instead of finishing at $223.27 - $216.47 = $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its own numbers were internally inconsistent with its answer: $233 of bracket tax less a $149 exemption credit leaves $84, yet it concluded ""effectively $0."" It also never applied the $60 nonrefundable renter's credit; the correct sequence is $223.27 of tax less $156.47 exemption and $60 renter credits, which stops at $6.80 rather than zero." -us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a ~$5,800 standard deduction rather than $5,706 and estimated the bracket tax at ~$200 instead of $223.27, then asserted that a ~$150 exemption credit ""fully offset"" it even though its own figures leave ~$50 outstanding. The $156.47 exemption credit and $60 renter's credit total $216.47, which falls $6.80 short of the tax." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It doubled the exemption credit by adding ""an additional exemption credit of ~$144 for a blind/disabled person,"" reaching $288 of credits and flooring the tax at $0. California's additional exemption credit is for blindness or age 65 and over, not for disability generally, so this 28-year-old disabled filer receives one $156.47 personal exemption credit; with the $60 renter's credit, $223.27 of tax nets to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It derived the correct $22,535 AGI and ~$16,995 taxable income but then estimated the bracket tax as ""roughly $250-$300"" without computing it, and asserted the exemption and renter credits offset it entirely. The 1%/2% brackets on $16,828.34 give $223.27, and the $156.47 exemption plus $60 renter credits total only $216.47, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It carried an AGI of $14,446, excluding the $8,000 taxable 403(b) distribution, which California taxes in full as ordinary income; the correct AGI is $22,534.34. That omission cut taxable income to $8,906 and the tax to $89, small enough for the exemption credit to erase, whereas the real $223.27 of tax exceeds the $216.47 of nonrefundable credits by $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446, dropping the fully taxable $8,000 403(b) distribution from California income, and paired it with a stale $5,363 standard deduction instead of the 2026 figure of $5,706. Taxable income is $16,828.34, not $9,083, so the bracket tax of $223.27 survives the $156.47 exemption and $60 renter credits at $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its structure was exactly right — bracket tax less the personal exemption and $60 renter's credit — but it inflated the standard deduction to $5,876 (versus $5,706), set the 1% bracket ceiling at $11,044 instead of roughly $11,330, and used a $158 exemption credit instead of $156.47. Those parameter shifts produce $222.74 of tax less $218 of credits ($4.74) rather than $223.27 less $216.47 ($6.80)." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a ~$5,800 standard deduction rather than $5,706 and rounded the bracket tax up to ~$230 while rounding the exemption credit to $155, producing $15 instead of $6.80. The exact figures are $16,828.34 taxable income, $223.27 of tax, and $216.47 of nonrefundable credits." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It claimed ""California standard deduction for single filers exceeds AGI,"" which is false here: the standard deduction is $5,706 against an AGI of $22,534.34 built from $17,443 of wages net of the $2,778 401(k) deferral plus the $8,000 taxable 403(b) distribution less the $130 IRA deduction. Taxable income of $16,828.34 generates $223.27 of tax, and the $216.47 of nonrefundable credits leaves $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It asserted the personal exemption credit fully offsets the tax without computing either quantity; the credit is $156.47 against $223.27 of bracket tax on $16,828.34 of taxable income, so it covers only part of the liability. Adding the $60 nonrefundable renter's credit still leaves $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $8,846 of taxable income implies California income of about $14,550 — the $8,000 taxable 403(b) distribution was left out of state gross income even though California taxes such distributions in full. With the distribution included, taxable income is $16,828.34 and the $223.27 of tax exceeds the $156.47 exemption and $60 renter credits by $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The answer restates the output name with no derivation. The correct computation — $22,534.34 AGI less the $5,706 standard deduction, $223.27 of tax across the 1% and 2% brackets, less $156.47 of exemption credit and $60 of renter's credit — yields $6.80; the submitted $0 is consistent only with assuming the nonrefundable credits exceed the bracket tax, which they do not." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"It reported essentially the pre-credit tax ($218 against the actual $223.27) and applied no nonrefundable credits at all, contrary to the requested output, which is state tax after nonrefundable credits. Subtracting the $156.47 personal exemption credit and the $60 nonrefundable renter's credit brings the liability to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It asserted that the standard deduction and personal exemption credits drive the tax to zero without computing the bracket tax. On $16,828.34 of taxable income the tax is $223.27, and the only nonrefundable credits available — the $156.47 exemption credit and the $60 renter's credit — total $216.47, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the right structure (1%/2% brackets, personal exemption credit, $60 renter's credit) but overstated the gross tax by about $10, which is what a standard deduction below $5,706 or a 1% bracket ceiling below roughly $11,330 produces. The exact values give $223.27 of tax less $216.47 of credits, or $6.80 rather than $17." -us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a stale $5,363 standard deduction and treated California's personal exemption as a $138 deduction from income, when it is a $156.47 nonrefundable credit against tax, and then concluded ""no nonrefundable state credits apply."" Applying the $5,706 standard deduction, the $156.47 exemption credit, and the $60 renter's credit to $223.27 of bracket tax gives $6.80, not $238.34." -us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It kept the full $25,443 as California AGI, never subtracting the $2,778 elective 401(k) deferral or the $130 traditional IRA deduction that bring AGI to $22,534.34, and it applied only the $60 renter's credit while omitting the $156.47 personal exemption credit. Both corrections together move $220 to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It asserted the tax is ""wiped out"" by the standard deduction and credits without computing either. The $5,706 standard deduction leaves $16,828.34 taxable and $223.27 of tax, which the $156.47 exemption credit and $60 renter's credit reduce to $6.80, not to zero." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It applied the American Opportunity Credit against California tax; that is a federal education credit and California does not conform to it, so it never appears in state_income_tax_before_refundable_credits. California's nonrefundable credits here are the $156.47 personal exemption credit and the $60 renter's credit, which reduce $223.27 of bracket tax to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Its $200 is 2.5% of the $8,000 403(b) distribution — it zeroed the regular tax and substituted California's additional tax on early distributions, which is not part of this output. The regular computation stands on its own: $16,828.34 of taxable income produces $223.27 of tax, less the $156.47 exemption credit and $60 renter's credit, for $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It stated the personal exemption and renter credits fully offset the tax without computing the offset. Those two credits total $216.47 against $223.27 of bracket tax on $16,828.34 of taxable income, so $6.80 remains." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It identified every component correctly — projected single standard deduction, personal exemption credit, $60 nonrefundable renter's credit — but estimated the result at $4 instead of computing it. The exact 2026 parameters ($5,706 standard deduction, $223.27 of bracket tax, $156.47 exemption credit) leave $6.80; its $4 corresponds to a standard deduction or exemption credit about $150 too generous." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It claimed the nonrefundable personal-exemption and renter credits fully offset the tax without quantifying either side. The credits total $216.47 while the bracket tax on $16,828.34 of taxable income is $223.27, leaving $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used the correct $22,535 AGI and the correct credit set but estimated rather than computed, landing on $4. With the $5,706 standard deduction, $223.27 of bracket tax, the $156.47 exemption credit, and the $60 renter's credit, the liability is $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and low income produce zero tax before credits, but the $5,706 standard deduction against $22,534.34 of AGI leaves $16,828.34 of taxable income and $223.27 of tax before credits. Even after the $156.47 exemption credit and $60 renter's credit, $6.80 remains." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $5,690 standard deduction and a $158 exemption credit, and its ~$227 bracket tax implies a 1% ceiling below the roughly $11,330 that produces $223.27 on $16,828.34 of taxable income. Those small parameter errors turn the true $223.27 - $216.47 = $6.80 into $9." -us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized about $7,499 of medical expenses, which double-counts the $8,089 of employer-sponsored premiums already excluded from wages pre-tax and counts $500 of over-the-counter health spending that is not a deductible medical expense; the remaining premiums and $500 of other medical costs fall below the 7.5%-of-AGI floor of about $1,690, so the $5,706 standard deduction applies. Taxable income is $16,828.34 with $223.27 of tax, and the $156.47 exemption plus $60 renter credits leave $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,"It applied the $60 nonrefundable renter's credit but omitted California's $156.47 personal exemption credit entirely, subtracting only $60 from a $229 tax. Including the exemption credit against the correct $223.27 of bracket tax gives $6.80, not $169." -us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It carried an AGI of ~$14,446, which excludes the $8,000 taxable 403(b) distribution that California taxes in full; the correct AGI is $22,534.34. Taxable income is therefore $16,828.34 rather than $8,720, and the resulting $223.27 of tax exceeds the $156.47 exemption and $60 renter credits by $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the submission failed the output contract rather than the tax computation. The required derivation is $22,534.34 AGI less the $5,706 standard deduction, $223.27 of bracket tax, less $156.47 of exemption credit and $60 of renter's credit, for $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the submission failed the output contract rather than the tax computation. The required derivation is $22,534.34 AGI less the $5,706 standard deduction, $223.27 of bracket tax, less $156.47 of exemption credit and $60 of renter's credit, for $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It added the $2,778 elective 401(k) deferral and the $130 traditional IRA deduction back into California AGI, but California conforms to the federal exclusion of elective deferrals and to the traditional IRA deduction, so AGI is $22,534.34, not $25,443. It also applied no nonrefundable credits; the $156.47 exemption credit and $60 renter's credit reduce the correct $223.27 of tax to $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used the 2025-vintage $5,540 standard deduction, an $11,080 1% bracket ceiling, and a $149 exemption credit instead of the 2026 values of $5,706, roughly $11,330, and $156.47. Those stale parameters inflate the tax to $229 and shrink the credits to $209, giving $20 where the correct figures give $223.27 - $216.47 = $6.80." -us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed $288 of exemption credits by adding a second $144 credit for the head being disabled; California's extra exemption credit applies to blindness or age 65 and over, not disability, so a single 28-year-old filer gets one $156.47 credit. It also skipped the $2,778 401(k) and $130 IRA adjustments and used a ~$5,994 standard deduction; with $22,534.34 of AGI, $5,706 of standard deduction, $223.27 of tax, $156.47 of exemption credit, and $60 of renter's credit, the answer is $6.80, not a floored zero." -us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It asserted zero tax after the standard deduction and nonrefundable credits without computing any component. The $5,706 standard deduction leaves $16,828.34 of taxable income and $223.27 of tax, against which the $156.47 exemption credit and $60 renter's credit total $216.47, leaving $6.80." -us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It subtracted the $2,778 traditional 401(k) deferral from wages to get $14,665, while PolicyEngine's CalEITC adjusted earnings are the full $17,442.65 of wages, and it then reported $402 — above the zero-dependent CalEITC maximum of under $300 for any income. Anchoring on a childless earnings ceiling of ""~$32k"" while returning more than the plateau maximum means it never applied the schedule it described; at $17,442.65 the credit sits roughly halfway down the phase-out at $148.31." -us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It had the right structure — childless maximum near $300 declining to zero around $33,000 — but started the decline far above the actual phase-in end near $4,200, so at $17,442.65 of earnings it retained about 58% of the maximum instead of the roughly 50% that yields $148.31. The zero-dependent CalEITC phase-out runs continuously from the end of the phase-in plateau to the indexed ceiling, not from some mid-range threshold." -us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It asserted that CalEITC requires a qualifying dependent or filing status the household lacks. CalEITC pays childless filers age 18 and over on California earned income up to the indexed ceiling near $33,000, so this 28-year-old with $17,442.65 of wages qualifies and receives $148.31." -us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It computed a childless CalEITC of roughly $44 from a phase-out ending at $30,950, then discarded that and submitted $358 — a figure above the zero-dependent CalEITC maximum of under $300 and explicitly justified by a federal American Opportunity Credit ""interaction"" that has no effect on a California refundable credit. The correct phase-out on adjusted earnings of $17,442.65 leaves about half the maximum, $148.31." -us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its reasoning states the childless CalEITC at this earnings level is small, then submits $1,208 — more than four times the zero-dependent maximum of under $300 and in the range of the one-qualifying-child CalEITC, which requires a child this household does not have. The zero-dependent schedule caps the credit below $300 before any phase-out, and the phase-out at $17,442.65 of adjusted earnings brings it to $148.31." -us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It placed the childless CalEITC phase-out end below $17,443, importing the federal childless EITC ceiling near $19,000 rather than California's indexed CalEITC ceiling near $33,000. CalEITC runs on its own schedule with a far longer phase-out, leaving $148.31 at adjusted earnings of $17,442.65." -us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It treated the $8,000 taxable 403(b) distribution as investment income exceeding the ~$3,650 disqualified-income limit and zeroed the credit. Disqualified income under IRC §32(i), which CalEITC incorporates, is interest, dividends, tax-exempt interest, net capital gain, net rent and royalty income, and passive activity income — retirement plan distributions are none of these, so the filer stays eligible and earns $148.31 on $17,442.65 of California earned income." -us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It identified the correct credit and eligibility but returned $285 — the zero-dependent CalEITC plateau maximum — without applying any phase-out. Adjusted earnings of $17,442.65 sit about halfway between the phase-in end near $4,200 and the indexed ceiling near $33,000, cutting the credit roughly in half to $148.31." -us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It computed the credit from a fabricated earned income of $6,576 and AGI of $14,446, neither of which is derivable from the stated $17,443 of wages plus $8,000 of 403(b) distributions. Placing the filer on the phase-in plateau produced a near-maximum $300; using PolicyEngine's adjusted earnings of $17,442.65 puts the filer midway down the phase-out at $148.31." -us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It applied the pre-2018 rule that CalEITC equals 85% of the federal EITC and supplied a federal childless credit of $503, when the federal childless EITC is $0 at $17,443 of earnings and roughly $22,500 of AGI (the federal childless phase-out ends near $19,000). Since 2018 CalEITC is computed on California's own phase-in, plateau, and phase-out schedule to an indexed ceiling near $33,000, giving $148.31 here." -us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It denied CalEITC outright with no derivation. A childless CA filer age 18 or older with $17,442.65 of California earned income is under the indexed CalEITC ceiling near $33,000 and receives a phased-out credit of $148.31." -us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It phased the credit out against AGI of $22,535, which includes the $8,000 taxable 403(b) distribution, instead of against California earned income. PolicyEngine's CalEITC phase-out runs on adjusted earnings of $17,442.65 — wages only, with retirement distributions excluded — so the credit is $148.31, not the $100 that the inflated income base produces." -us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It ruled the household out on income with no threshold cited. California earned income of $17,442.65 is well under the indexed CalEITC ceiling near $33,000, and the childless schedule pays $148.31 at that earnings level." -us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It named CalEITC correctly but returned $180 with no schedule applied, retaining well over half of the zero-dependent maximum. From the phase-in end near $4,200 to the indexed ceiling near $33,000, adjusted earnings of $17,442.65 cut the credit to $148.31." -us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It asserted zero California refundable credits without evaluating CalEITC. The childless CalEITC covers filers 18 and over with California earned income below the indexed ceiling near $33,000, paying $148.31 on $17,442.65 of wages." -us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It claimed the head's income exceeds the CalEITC eligibility limit. The CalEITC ceiling for 2026 is the indexed version of the $30,000 statutory cap, near $33,000, which exceeds both the $17,442.65 of earned income and the roughly $22,500 AGI, so the credit is $148.31 rather than zero." -us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It returned $195 as a bare CalEITC figure, retaining about two-thirds of the zero-dependent maximum. Adjusted earnings of $17,442.65 fall roughly halfway between the phase-in end near $4,200 and the indexed ceiling near $33,000, leaving $148.31." -us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It applied too steep a reduction, keeping only about a third of the zero-dependent maximum at $92. The linear decline from the phase-in end near $4,200 to the indexed ceiling near $33,000 leaves about half the maximum at $17,442.65 of adjusted earnings, or $148.31." -us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It measured the phase-out against AGI of $22,535 and placed the zero point below that figure. CalEITC phases out on California earned income of $17,442.65, not on AGI inflated by the $8,000 403(b) distribution, and reaches zero only at the indexed ceiling near $33,000, so the credit is $148.31." -us,scenario_023,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It conditioned CalEITC on a qualifying child or an unmet ""childless age test"" and reported $0 with no computation. California's childless CalEITC requires only that the filer be 18 or older — this head is 28 — so the schedule applies and yields $148.31 on $17,442.65 of earned income." -us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no facts indicating a California refundable credit, overlooking that $17,443 of wages is itself CalEITC-qualifying earned income. A childless filer 18 or older under the indexed ceiling near $33,000 receives $148.31." -us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It concluded no California refundable credits are indicated, ignoring the CalEITC pathway triggered by $17,442.65 of California earned income. The childless CalEITC schedule pays $148.31 at that earnings level." -us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used the right credit and the right region of the schedule but over-reduced it to $114, consistent with phasing out on the roughly $22,500 AGI that includes the $8,000 403(b) distribution rather than on California earned income. The phase-out base is adjusted earnings of $17,442.65, which leaves $148.31." -us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It returned $194 for the childless CalEITC, retaining about two-thirds of the zero-dependent maximum. The decline from the phase-in end near $4,200 to the indexed ceiling near $33,000 removes about half the maximum at $17,442.65 of adjusted earnings, leaving $148.31." -us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It estimated $160 by under-applying the phase-out rate on the zero-dependent schedule. At adjusted earnings of $17,442.65, the linear reduction from the plateau to the indexed ceiling near $33,000 gives $148.31." -us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It concluded the listed facts generate no California refundable credit. The $17,442.65 of California earned income places this childless filer inside the CalEITC phase-out range that ends near $33,000, producing $148.31." -us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It nearly exhausted the credit at $25, implying a zero point just above $18,000 — the federal childless EITC ceiling rather than California's indexed CalEITC ceiling near $33,000. On California's own schedule, $17,442.65 of adjusted earnings leaves about half the zero-dependent maximum, $148.31." -us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It denied any qualifying state refundable credit without analysis. CalEITC applies to childless California filers 18 and over with earned income below the indexed ceiling near $33,000, and pays $148.31 on $17,442.65 of wages." -us,scenario_023,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It made CalEITC contingent on federal EITC qualification, zeroing it because income exceeds the federal childless phase-out end near $19,000. CalEITC is a stand-alone California credit with its own phase-in rate, plateau, and phase-out to an indexed ceiling near $33,000, so a $0 federal EITC still leaves $148.31 of CalEITC." -us,scenario_023,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It reasoned ""federal EITC is $0 so CalEITC is $0,"" applying the repealed percentage-of-federal structure. Since 2018 CalEITC is computed directly on California earned income against its own schedule, which pays $148.31 at $17,442.65 even with no federal EITC; its treatment of the renter's credit as nonrefundable is right but does not rescue the total." -us,scenario_023,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It built the credit from federal-style 2024 parameters — a $255 maximum, a $17,640 phase-out start, and a 3.25% rate — and applied them to AGI of $22,535. CalEITC uses California's own zero-dependent maximum of under $300 with a phase-out measured on adjusted earnings of $17,442.65 running to the indexed ceiling near $33,000, which produces $148.31 rather than $96." -us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It returned roughly the zero-dependent plateau maximum at $300, treating $17,443 as ""low earned income"" still on the phase-in flat. The plateau ends near $4,200, so $17,442.65 of adjusted earnings sits about halfway down the phase-out and yields $148.31." -us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_refundable_credits, so the submission failed the output contract rather than the tax analysis. The requested value is the childless CalEITC of $148.31 on $17,442.65 of California earned income." -us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The response contained no state_refundable_credits entry, leaving the required key unanswered rather than substantively wrong. The correct entry is $148.31 of CalEITC computed on adjusted earnings of $17,442.65." -us,scenario_023,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It stated that CalEITC is nonrefundable for filers without children. CalEITC is fully refundable for every eligible filer, childless filers included, so the $148.31 computed on $17,442.65 of California earned income counts in full as a state refundable credit." -us,scenario_023,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It netted the $2,778 elective 401(k) deferral out of wages to reach $14,665 and then declared that figure to ""far exceed"" the childless phase-out ceiling. PolicyEngine's CalEITC adjusted earnings are the undeducted $17,442.65, and California's ceiling is the indexed cap near $33,000 — above both figures — leaving $148.31." -us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It added a $60 California renter's credit to its CalEITC estimate, but the renter's credit is nonrefundable and cannot appear in state refundable credits; it also inflated its own $200 CalEITC estimate to $251 without recomputation. The refundable total is CalEITC alone, $148.31 on adjusted earnings of $17,442.65." -us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It asserted that no California refundable credit applies, bypassing CalEITC entirely. A childless filer 28 years old with $17,442.65 of California earned income is inside the CalEITC phase-out range and receives $148.31." +us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found AGI of $22,535, a standard deduction of about $5,700, tax of about $200, and both the ~$154 exemption credit and the $60 renter's credit. It then called the tax 'effectively 0' instead of subtracting: $225.78 − $153 − $60 = $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a projected 2026 standard deduction of about $5,880 instead of the $5,706 published for 2025. That understated taxable income and tax (about $222 instead of $225.78) and produced $9 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ignored the $8,000 taxable 403(b) distribution and the 401(k)/IRA adjustments, and used a $5,202 standard deduction. It also subtracted CalEITC as if it were nonrefundable, but CalEITC is refundable and does not reduce this pre-refundable-credit amount." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model understated the tax at about $170 (the correct figure is $225.78, using $5,706 and a 1% bracket ending at $11,079) and never applied the $60 renter's credit. It then reported $0 even though its own $170 − $149 left a positive balance." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed about $233 in gross tax and a ~$149 exemption credit, and admitted the tax was 'not fully eliminated'. It then asserted $0 anyway, left out the $60 renter's credit, and never did the subtraction ($225.78 − $153 − $60 = $12.78)." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model claimed about $200 of tax was 'fully offset' by a ~$150 exemption credit, which is an arithmetic contradiction. It also left out the renter's credit. The correct result is $225.78 − $213 = $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model applied inflated 2026 projections: a ~$5,880 standard deduction and a ~$158 exemption credit. The published $5,706 and $153 apply. That left about $1 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"The model added a second ~$144 exemption credit for disability. California's extra exemption credit is for blindness, not disability, so the head gets only the single $153 credit. The doubled credit wiped out the tax, and the model also used outdated parameters ($5,540 deduction, $10,756 bracket)." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model estimated $250–$300 of tax, then claimed a ~$149 exemption credit exceeded it and cited nonexistent 'disability-related' credits. Only the $153 exemption credit and the $60 renter's credit apply, and together they leave $12.78 of the $225.78 tax." +us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model correctly reached $226 of tax and the $153 exemption credit but left out the $60 nonrefundable renter's credit. The household pays $24,000 in rent and is under the income limit, so the credit applies. The model got $73 and rounded to $75 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution. That cut taxable income to $8,906 and the tax below the exemption credit. The correct AGI is $22,534, which gives $225.78 of tax and leaves $12.78 after credits." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution, so its $90.83 tax was wiped out by the exemption credit. Including the distribution gives $16,828 taxable and $225.78 of tax, and the $213 in credits leave $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used inflated 2026 projections: a $5,876 standard deduction, a 1% bracket ending at $11,044, and a $158 exemption credit. The published $5,706, $11,079, and $153 apply. That understated the residual as $4.74 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's AGI of $22,665 skipped the $130 traditional IRA deduction. It also used an outdated $5,540 standard deduction and a 1% bracket ending at $10,756. That overstated tax at $234.94 and produced $20.94 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a rounded ~$5,800 deduction, ~$230 of tax, and a ~$155 exemption credit instead of the exact $5,706, $225.78, and $153. That gave $15 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model claimed the single standard deduction exceeds AGI, but the $5,706 deduction is far below the $22,534 CA AGI. That leaves $16,828 of taxable income and $225.78 of tax before credits." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model asserted that the personal exemption credit alone fully offsets the tax. The $153 credit covers only part of the $225.78 tax, and even with the $60 renter's credit, $12.78 remains." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $8,846 shows it left the $8,000 taxable 403(b) distribution out of CA AGI. The correct taxable income is $16,828, which gives $225.78 of tax and $12.78 after the $213 in credits." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $0 matches leaving out the $8,000 403(b) distribution or assuming the credits exceed the tax. The correct figures are $225.78 of tax on $16,828 taxable, less $213 in credits, for $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model's $218 is roughly the tax before credits ($225.78). It never subtracted the $153 personal exemption credit or the $60 nonrefundable renter's credit, which bring the tax down to $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model claimed the standard deduction and exemption credits eliminate the tax. The $5,706 deduction leaves $16,828 taxable and $225.78 of tax, which is more than the $153 exemption and $60 renter's credits combined, so $12.78 remains." +us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the right structure (1%/2% brackets, exemption credit, $60 renter's credit) with approximate parameters. It got $17 instead of the $12.78 that the published $5,706 deduction, $11,079 bracket, and $153 credit produce." +us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated California's personal exemption as a $138 deduction from income, but it is a $153 credit against tax. It also said no credits apply, omitting both that credit and the $60 renter's credit, and used outdated parameters ($5,363, $10,234). The result was $238.34." +us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $2,778 traditional 401(k) deferral and the $130 IRA deduction, putting AGI at $25,443 instead of $22,534. It also never applied the $153 personal exemption credit, subtracting only the $60 renter's credit." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model asserted the deduction and credits wipe out the tax. The $225.78 tax on $16,828 of taxable income is more than the $153 exemption and $60 renter's credits combined, leaving $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model's $72 matches $225.78 minus only the $153 exemption credit, so it left out the $60 nonrefundable renter's credit. It also cited an American Opportunity credit, which California does not have." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model added a 2.5% California early-distribution additional tax on the $8,000 403(b) distribution. That penalty is not part of this output, and nothing in the facts makes the distribution early. The model also wrongly claimed the regular tax was eliminated, when $225.78 − $153 − $60 = $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model claimed the exemption and renter's credits fully offset the tax. Those credits total $213, which is less than the $225.78 tax, leaving $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a 'projected' 2026 standard deduction that was inflated above the published $5,706. That understated tax and gave $4 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model claimed the exemption and renter's credits fully offset the tax. The $153 + $60 = $213 in credits is less than the $225.78 tax, leaving $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model applied an 'indexed' 2026 standard deduction above the published $5,706. That shrank taxable income and tax, giving $4 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model used the right structure but estimated parameters and rounded to a whole dollar, giving $13. The exact calculation with $5,706, $11,079, and $153 is $225.78 − $213 = $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,The model used estimated 2026 parameters above the published 2025 values: a higher standard deduction and bracket threshold. That understated tax and left $3 instead of $12.78. +us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $14,446, leaving out the $8,000 taxable 403(b) distribution, so the exemption credit appeared to wipe out the tax. The correct CA AGI is $22,534, which gives $225.78 of tax and $12.78 after credits." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed zero tax even before credits. The $5,706 standard deduction leaves $16,828 of taxable income and $225.78 of tax, and the $213 in credits reduce that only to $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used inflated 2026 projections: a $5,690 standard deduction, inflated brackets, and a $158 exemption credit. The published $5,706, $11,079, and $153 apply. It got $9 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model itemized a $7,499 medical deduction by counting employer-sponsored premiums (paid pre-tax) and OTC expenses. The deductible medical costs ($800) fall below the 7.5%-of-AGI floor (about $1,690), so the $5,706 standard deduction applies and the tax is $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used inflated 2026 estimates: a $5,843 deduction, a 1% bracket ending at $11,345, and a $157 exemption credit. The published $5,706, $11,079, and $153 apply. It got $3 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model applied only the $60 renter's credit and left out the $153 personal exemption credit that every single filer receives. That left $169 instead of $12.78. +us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an AGI of about $14,446, leaving out the $8,000 taxable 403(b) distribution, so the exemption credit appeared to wipe out an $87 tax. Including it gives $225.78 of tax and $12.78 after credits." +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score against $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so there was nothing to score against $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added back the traditional 401(k) and IRA amounts, but California conforms to the 401(k) exclusion and the IRA deduction, so CA AGI is $22,534. It also said no credits apply, omitting the $153 exemption and $60 renter's credits." +us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used an outdated $5,540 standard deduction and a $149 exemption credit instead of the published $5,706 and $153. That overstated taxable income and understated the credit, giving $20 instead of $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model added a second $144 exemption credit for disability, but California's extra exemption credit applies only to blindness. It also skipped the 401(k)/IRA adjustments and wrongly treated the renter's credit as refundable. Together these zeroed out a tax that is really $12.78." +us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted the standard deduction and nonrefundable credits eliminate the tax. The $225.78 tax on $16,828 of taxable income is more than the $213 in exemption and renter's credits, leaving $12.78." +us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It cut earned income to $14,665 by subtracting the 401(k) deferral, even though CalEITC earned income is the full $17,443 of wages. It also never ran the phase-out on the $22,534 AGI. The childless schedule at that AGI gives $95.15, not $402." +us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It phased the credit out using only earned income of $17,443. It ignored that CalEITC's phase-out uses the greater of earned income and AGI, and AGI here is $22,534 because of the $8,000 in 403(b) distributions. Applying the schedule at $22,534 lowers the credit from about $180 to $95.15." +us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It said a filer with no qualifying dependents cannot get CalEITC. In fact, CalEITC covers filers without children who are 18 or older and have earned income under roughly $32,000, and this 28-year-old with $17,443 in wages qualifies for $95.15." +us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Its own reasoning estimated the childless CalEITC at $0–$50. It then replaced that with an unsupported $358 that 'reflects interactions', even though no other California refundable credit applies. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It reported $1,208, several times the childless CalEITC maximum of about $300, even while calling the credit 'small'. It never computed the phase-out at AGI $22,534, which gives $95.15." +us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It claimed the childless CalEITC is fully phased out below this income. The childless income limit is roughly $32,000, so at AGI $22,534 the filer is still in the phase-out range and gets $95.15." +us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"It correctly phased the credit out using AGI of about $22.5k. But it misapplied the second-segment phase-out rate, landing at $115 instead of $95.15." +us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It treated the $8,000 in taxable 403(b) distributions as disqualifying investment income. That test covers only interest, dividends, rents, royalties, capital gains, and passive income, and pension or retirement distributions are not included. The filer stays eligible and gets $95.15." +us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It gave roughly the maximum childless CalEITC ($285) and applied no phase-out. At the $22,534 AGI that governs the phase-out, the credit drops to $95.15." +us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It placed the filer on the phase-out slope using only earned income of $17,443. It ignored the rule that the greater of AGI ($22,534) and earned income sets the phase-out, which cuts the credit to $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It used made-up figures of $6,576 earned income and $14,446 AGI, when the real figures are $17,443 and $22,534. That put the filer near the plateau with about $300 of credit, instead of deep in the phase-out where the credit is $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It invented a rule that CalEITC equals 85% of the federal EITC. CalEITC has its own schedule, and the federal childless EITC is fully phased out at AGI $22,534 anyway. The correct CalEITC is $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It concluded that no CalEITC applies. A childless filer aged 28 with $17,443 in wages and $22,534 AGI is under the roughly $32,000 childless limit and gets $95.15." +us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It used one invented straight-line phase-out from $15,000 to $33,000, when CalEITC actually phases out in two segments, and it left the $130 IRA deduction out of AGI. That produced $172.25 instead of $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It correctly phased the credit out using the higher AGI of $22,535. But it rounded to $100 instead of computing the exact value from the schedule, which is $95.15." +us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It said income rules out any California refundable credit. The childless CalEITC runs to roughly $32,000, so this filer still gets $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It phased the credit out only partially, getting $148. Running the full two-segment schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Its $180 matches a phase-out computed from earned income of $17,443 alone. It skipped the greater-of-AGI-or-earned-income rule, and with AGI at $22,534 the credit falls to $95.15." +us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 and never recognized that a childless filer aged 28 with wages qualifies for CalEITC. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It said the filer's income exceeds the CalEITC limits. The childless limit is roughly $32,000, well above both the $17,443 in earnings and the $22,534 AGI, so the credit is $95.15." +us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $195 matches a phase-out based on wages alone. Applying the phase-out at the higher $22,534 AGI cuts the credit to $95.15." +us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It correctly phased the credit out but approximated the schedule, getting $92. Evaluating the exact childless CalEITC schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It claimed the childless CalEITC disappears before AGI $22,535. The childless income limit is roughly $32,000, so the credit is $95.15 at this AGI." +us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It said the filer fails the childless age test. For CalEITC, the only age requirement for a childless filer is being at least 18, which this 28-year-old meets, so the credit is $95.15." +us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no California refundable credit, missing that the filer's $17,443 in wages makes them eligible for the childless CalEITC. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It found no California refundable credit, missing that the filer's $17,443 in wages makes them eligible for the childless CalEITC. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It correctly placed the filer in the childless phase-out range but misestimated the second-segment slope. It got $114 instead of the $95.15 the schedule gives at AGI $22,534." +us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It based the credit explicitly on $17,443 of earned income. It skipped the rule that the greater of AGI ($22,534) and earned income drives the phase-out, which lowers the credit to $95.15." +us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It computed the credit 'at $17,443 of earned income'. It ignored that the higher AGI of $22,534, raised by the 403(b) distributions, sets the phase-out, and that gives $95.15." +us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It said the listed facts generate no California refundable credit, missing that the childless CalEITC covers this 28-year-old wage earner. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It correctly found a phased-out childless CalEITC but pushed the phase-out too far, getting $25. The schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly phased the credit out using the higher AGI but misestimated the phase-out slope, getting $106. The exact value at AGI $22,534 is $95.15." +us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It said a childless adult at this income gets no CalEITC. The childless limit is roughly $32,000, so the filer is still in the phase-out range at AGI $22,534 and gets $95.15." +us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It applied a phase-out that was too steep, getting $28. The childless CalEITC schedule at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It found no qualifying refundable credits, missing that the childless CalEITC applies to this 28-year-old wage earner. The phase-out at AGI $22,534 gives $95.15." +us,scenario_023,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"It made CalEITC depend on federal EITC eligibility. CalEITC is separate from the federal credit and extends to roughly $32,000 for childless filers, so it pays $95.15 even though the federal childless EITC is zero at this AGI." +us,scenario_023,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It reasoned that a $0 federal EITC means a $0 CalEITC. CalEITC has its own schedule with a childless limit of roughly $32,000, which gives $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly phased the credit out at AGI $22,535 but used the wrong phase-out rate, getting $120 instead of the $95.15 the schedule gives." +us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It gave roughly the maximum childless CalEITC ($300) and applied no phase-out. At the governing AGI of $22,534, the credit is $95.15." +us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no parseable value for state_refundable_credits, so the $95.15 CalEITC was never reported." +us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no parseable value for state_refundable_credits, so the $95.15 CalEITC was never reported." +us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed CalEITC is nonrefundable for filers without children. CalEITC is fully refundable whether or not the filer has children, so this filer's $95.15 counts as a state refundable credit." +us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It said $14,665 of earned income is above the childless phase-out ceiling. The ceiling is roughly $32,000, and earned income is actually $17,443, so the filer gets $95.15 at AGI $22,534." +us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It counted California's $60 renter's credit as refundable, but that credit is nonrefundable. It also inflated CalEITC to $251, contradicting its own $200 figure, without phasing it out at AGI $22,534, where the only refundable credit is $95.15." +us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It said no California refundable credit applies, missing the childless CalEITC this 28-year-old wage earner qualifies for. The phase-out at AGI $22,534 gives $95.15." us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It reached $7,019 using a $32,300 standard deduction and the correct $24,800 10%-bracket ceiling, then discarded that derivation and submitted $3,325, a figure that appears nowhere in its own arithmetic. Its only substantive parameter error was overstating the 2026 MFJ standard deduction by $100 ($32,300 rather than $32,200), worth about $12; the remaining $3,706 gap is a bare substitution of an unsupported number for its computed total." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $29,900 MFJ standard deduction instead of the 2026 OBBBA-indexed $32,200, then abandoned bracket arithmetic entirely: $8,640 on its own $65,025 taxable base is an effective 13.3% flat rate, whereas the 10%/12% schedule on that base gives $7,307. With the correct deduction, $94,925.30 − $32,200 = $62,725.30 and $2,480 + 0.12 × $37,925.30 = $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It placed the 2026 MFJ 10% bracket ceiling at $24,000 instead of $24,800 and the standard deduction at $32,300 instead of $32,200, shifting $800 of income from the 12% band into the 10% band while cutting $100 more from taxable income. Those two parameter slips are the entire $3.96 error; its structure (no itemizing, no nonrefundable credits) was otherwise the reference's." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reproduced the reference derivation exactly — $32,200 standard deduction, $62,725 taxable, $2,480 at 10% on $24,800 plus $4,551 at 12% on $37,925, total $7,031 — and then submitted $6,934, calling it a rounding of that same figure. The $97 shortfall is an unexplained overwrite of a correct total, not a computation or parameter error." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After explicitly considering and then rejecting inflation-indexed 2026 figures, it settled on 2025 parameters — a $30,000 standard deduction and a $23,850 10%-bracket ceiling — missing the 2026 MFJ values of $32,200 and $24,800. Its lengthy medical-expense analysis was moot because itemized deductions total only $2,354.43 against the standard deduction." -us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed the reference value precisely — $32,200 standard deduction, $62,725 taxable, 10% on $24,800 plus 12% on $37,925 = $7,031 — then overrode it with an unexplained 'adjusting for standard deduction estimate and rounding' to $4,700. Nothing in its stated brackets, deduction, or income supports a $2,331 reduction." -us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction and a $24,150 10%-bracket ceiling instead of the 2026 values of $32,200 and $24,800, leaving $1,600 of extra income in the 12% band and pushing $650 from 10% to 12%. That accounts for the full $205 overstatement; its itemize-versus-standard call and zero-credit conclusion matched the reference." -us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied 2025's $30,000 standard deduction and $23,850 10%-bracket ceiling to a 2026 return. The 2026 MFJ figures are $32,200 and $24,800, which on $94,925.30 of AGI give $62,725.30 taxable and $7,031.04 of tax rather than $7,314." -us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset took effect in 2026 — a $16,700 standard deduction, two $5,350 personal exemptions, and a 15% second bracket. OBBBA made the TCJA rate and deduction structure permanent, so 2026 has no personal exemptions, a $32,200 MFJ standard deduction, and a 12% second bracket above $24,800." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It cut AGI to $73,717 by subtracting $21,208 of employer-sponsored insurance premiums that do not reduce the $62,725.29 of employment income or the $32,200 pension, then itemized on imputed figures — $14,640 of mortgage interest inferred from a $366,000 balance and $1,006 of SALT — despite the instruction to treat unlisted numeric inputs as zero, and finally subtracted $10,100 of personal exemptions that no longer exist in 2026. AGI is $94,925.30 and the $32,200 standard deduction beats the $2,354.43 of allowable itemized deductions." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $22,400 standard deduction, which is neither the 2026 MFJ amount of $32,200 nor any sunset figure, and then reported $5,236 — a number the 10%/12% schedule does not produce on its own $72,525 taxable base, which would yield $8,207. The correct deduction gives $62,725.30 taxable and $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It priced the return off a 'post-TCJA expiration baseline' with personal exemptions restored. OBBBA extended the TCJA regime permanently, so 2026 MFJ takes a $32,200 standard deduction, no exemptions, and a 12% second bracket above $24,800, producing $7,031.04 rather than $9,112." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the sunset schedule — a $16,100 standard deduction, $10,100 of personal exemptions, and 15% above $23,200 — none of which are in force for 2026. Under the permanent TCJA parameters, taxable income is $62,725.30 (not $68,725) and the second bracket is 12%, giving $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted a $0 liability with no derivation for a household with $94,925.30 of AGI and only a $32,200 standard deduction to offset it. Taxable income is $62,725.30, tax is $7,031.04, and no nonrefundable credit exists for this childless couple with no care, education, or elderly-credit qualification." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It stated the correct $94,925 AGI and then reported $8,993, which the 2026 MFJ schedule cannot produce from a $32,200 standard deduction. The figure is consistent with a sunset-era base of roughly $68,000 of taxable income taxed at 15% above the bottom bracket — i.e. a ~$16,600 standard deduction plus personal exemptions — rather than the $32,200 deduction and 12% bracket actually in force." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It stated outright that it computed 2026 tax 'under sunset rules with standard deduction and personal exemptions.' OBBBA made the TCJA structure permanent: 2026 MFJ has a $32,200 standard deduction, zero personal exemptions, and brackets of 10% to $24,800 then 12%, yielding $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 MFJ standard deduction of $30,000 and the 2025 bracket break at $23,850 for a 2026 return, where the indexed values are $32,200 and $24,800. Substituting them lowers taxable income from $64,925 to $62,725.30 and tax from $7,314 to $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It guessed a $30,900 standard deduction and a $24,600 10%-bracket ceiling instead of the 2026 values of $32,200 and $24,800. The $1,300 of extra taxable income at 12% plus the $200 shifted from 10% to 12% produce the entire $160 overstatement; its no-itemize, no-credit structure was correct." -us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It had the $24,800 bracket break right but used a $30,600 standard deduction instead of $32,200, computed $7,223, and then submitted $6,005.50 — a value absent from its own arithmetic and $1,217 below the total it stated. With the correct deduction, taxable income is $62,725.30 and tax is $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It concluded without computation that the facts 'do not create a positive federal income tax after nonrefundable credits,' ignoring $94,925.30 of AGI against a $32,200 standard deduction. Taxable income of $62,725.30 produces $7,031.04, and having no nonrefundable credits leaves that amount intact rather than eliminating it." -us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It submitted $2,460 with no derivation; that amount is what taxing only the bottom bracket produces (roughly 10% of the $24,800 10% band) while dropping the $37,925.30 of taxable income sitting in the 12% band, which alone adds $4,551.04. The full 2026 MFJ computation on $62,725.30 of taxable income is $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It claimed 'no taxable income after deductions/credits' for a couple with $62,725.29 of wages and $32,200 of taxable pension. The $32,200 MFJ standard deduction offsets barely a third of the $94,925.30 AGI, leaving $62,725.30 taxable and $7,031.04 of tax with no credits available to zero it." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It built the return on a TCJA-sunset baseline: a $16,688 standard deduction, $10,644 of restored personal exemptions, and a 15% rate above the bottom bracket. 2026 keeps the permanent TCJA parameters — $32,200 standard deduction, no exemptions, 12% second bracket — so taxable income is $62,725.30 and tax is $7,031.04, not $8,914." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the correct $24,800 10%-bracket ceiling but paired it with sunset parameters — a $16,600 standard deduction, two $5,300 personal exemptions, and a 15% second rate. The 2026 MFJ deduction is $32,200 with no exemptions and a 12% second rate, cutting the tax from $8,919 to $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted' pre-TCJA parameters — a $16,800 standard deduction, $10,800 of personal exemptions, and 15% above $23,837 — which do not apply in 2026 because OBBBA made the TCJA structure permanent. The correct $32,200 deduction and 10%/12% brackets breaking at $24,800 give $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so nothing was submitted to compare against the $7,031.04 reference. This is a missing-output failure rather than a substantive tax error." -us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It derived taxable income of roughly $63,425 and a bracket tax of '$5,500-$6,500,' then submitted $0 on the ground that no CTC, CDCC, or other nonrefundable credit applies. Nonrefundable credits subtract from tax; having none leaves the full liability standing, and the correct $32,200 deduction gives $62,725.30 taxable and $7,031.04." -us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $31,450 standard deduction and a $24,350 10%-bracket ceiling rather than the 2026 MFJ values of $32,200 and $24,800. The $750 of extra taxable income at 12% and the $450 shifted from 10% to 12% account for the $99 overstatement." -us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a $9,300 standard deduction for a joint return instead of $32,200, invented a $1,525 medical deduction (allowable medical expenses of $7,350 barely clear the 7.5% floor of $7,119.40), and then subtracted a '$3,346.75 senior bonus deduction' from tax as if it were a credit for the 61-year-old head. The OBBBA senior deduction requires age 65 or older and reduces taxable income, not liability; neither spouse qualifies, and the correct computation is $62,725.30 taxable for $7,031.04 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached about $7,019, using a $32,300 standard deduction that is $100 too high (the 2026 MFJ figure is $32,200). It then submitted $3,325, which matches none of its computation. The submitted value contradicts the bracket math it had just done." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $29,900 MFJ standard deduction instead of the 2026 OBBBA amount of $32,200, which overstated taxable income at $65,025. Its $8,640 is not even consistent with that taxable income under 10%/12% brackets (about $7,370). That means it also misapplied the rate schedule." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $32,300 standard deduction instead of $32,200. It also ended the 10% bracket at $24,000 instead of the 2026 MFJ threshold of $24,800. The two parameter errors mostly cancel and give $7,035 rather than $2,480 + 12% × $37,925.30 = $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It applied the correct $32,200 standard deduction and the $24,800 10% bracket and reached about $7,031. It then submitted $6,934 through an unexplained 'rounded with bracket estimates' adjustment that has no basis in the tax computation." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deliberately fell back to the pre-OBBBA 2025 parameters: a $30,000 MFJ standard deduction and a 10% bracket ending at $23,850. The 2026 law sets $32,200 and $24,800. Those figures inflated taxable income to $64,925 and the tax to $7,314." +us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed the correct figure: $62,725 of taxable income after the $32,200 deduction, and tax of $2,480 + $4,551 = $7,031. It then submitted $4,700 through an unexplained 'adjusting for standard deduction estimate' step that discards its own correct result." +us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 MFJ standard deduction and ended the 10% bracket at $24,150. The 2026 values are $32,200 and $24,800. This overstated taxable income at $64,325 and gave $7,236." +us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 MFJ standard deduction of $30,000 and the 2025 10% bracket top of $23,850. The 2026 figures are $32,200 and $24,800. This overstated taxable income by $2,200 and gave $7,314." +us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA expired after 2025 and applied a pre-TCJA $16,700 standard deduction, personal exemptions and a 15% second bracket. The OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no personal exemptions and a 12% bracket. That produced $8,906.25." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from AGI even though the head has no wages to exclude them from. It imputed $14,640 of mortgage interest from the loan balance, although unlisted amounts are zero. It also itemized SALT and applied expired personal exemptions and a 15% bracket. The correct approach is the $32,200 standard deduction against $94,925.30 of AGI under 10%/12% brackets." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $22,400 standard deduction instead of the 2026 MFJ amount of $32,200, which overstated taxable income at $72,525. Its $5,236 is also inconsistent with that taxable income under the brackets (about $8,200), so it misapplied the rate schedule too." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly applied a TCJA-expiration baseline, with the pre-TCJA standard deduction, personal exemptions and the 15% bracket. The OBBBA made the $32,200 standard deduction and 12% bracket permanent for 2026. That inflated the tax to $9,112." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: a $16,100 standard deduction, $10,100 of personal exemptions and a 15% bracket above $23,200. The 2026 OBBBA regime has a $32,200 deduction, no exemptions and a 12% bracket above $24,800. That produced $9,148.75 instead of $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted that the standard deduction and 'applicable credits' reduce tax to $0. The $32,200 standard deduction leaves $62,725.30 of taxable income, and this childless couple under 65 has no nonrefundable credits to offset the resulting $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no parameters, but $8,993 is about $1,960 above the $7,031.04 that the $32,200 deduction and 10%/12% brackets produce. That matches the TCJA-sunset computations other models made, with a pre-TCJA deduction, personal exemptions and a 15% bracket, rather than 2026 OBBBA law." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It explicitly computed the tax 'under sunset rules' with personal exemptions. It did not use the OBBBA-permanent 2026 structure: a $32,200 MFJ standard deduction, no exemptions and a 12% bracket above $24,800. The result was $7,306.75." +us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 MFJ standard deduction of $30,000 and the 2025 10% bracket top of $23,850 instead of the 2026 figures of $32,200 and $24,800. That overstated taxable income at $64,925 and gave $7,314." +us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,900 and the 10% bracket top at $24,600. The actual figures are $32,200 and $24,800, so taxable income came out at $64,025 and the tax at $7,191." +us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It used a $30,600 standard deduction instead of $32,200, which gave $64,325 of taxable income and $7,223 of tax, and it correctly concluded that no nonrefundable credits apply. It then submitted $6,005.50, which contradicts its own final figure of $7,223." +us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It claimed the standard deduction and unspecified credits eliminate all tax. The $32,200 MFJ standard deduction leaves $62,725.30 taxable, and no nonrefundable credit applies to this childless couple under 65, so the tax is $7,031.04, not $0." +us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $2,460 equals 10% of about $24,600, which is only the 10%-bracket tax. It left out the 12% tax on the $37,925.30 of taxable income above the $24,800 bracket threshold, a further $4,551.04." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that no taxable income remains after deductions. The $32,200 MFJ standard deduction against $94,925.30 of AGI leaves $62,725.30 taxable, which produces $7,031.04 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-expiration law: a $16,688 standard deduction, $10,644 of personal exemptions and a 15% second bracket. The OBBBA made the $32,200 standard deduction and 12% bracket permanent for 2026, so it overstated taxable income at $67,593 and the tax at $8,914." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the TCJA-sunset regime: a $16,600 standard deduction, two $5,300 exemptions and a 15% bracket above $24,800. The OBBBA-permanent 2026 regime has a $32,200 deduction, no exemptions and a 12% bracket, so it produced $8,919." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated the 2026 MFJ standard deduction at $30,700 and the 10% bracket top at $24,450. The actual figures are $32,200 and $24,800, which gave $64,225 of taxable income and $7,218 of tax." +us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted' pre-TCJA law: a $16,800 standard deduction, $10,800 of personal exemptions and a 15% bracket above $23,837. The OBBBA-permanent 2026 schedule has a $32,200 deduction, no exemptions and 12% above $24,800, so it produced $8,907." +us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against $7,031.04." +us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It estimated $5,500–$6,500 of tax on $63,425 of taxable income, using a $31,500 deduction instead of $32,200, and acknowledged that no nonrefundable credits apply. It then submitted $0 anyway. That discards a positive liability that no credit offsets." +us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $31,450 MFJ standard deduction and a $24,350 10% bracket top instead of the 2026 values of $32,200 and $24,800. That overstated taxable income at $63,475 and gave $7,130." +us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"It subtracted a $3,346.75 'senior bonus' from the tax for the 61-year-old head. The OBBBA senior deduction requires age 65, and it is a deduction, not a tax credit. It also used a $9,300 standard deduction instead of $32,200, a 10% bracket top of $29,550 instead of $24,800, and an itemized medical deduction the standard deduction would displace." us,scenario_025,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_025,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model correctly stated that the 61-year-old head does not meet Medicare eligibility, but then submitted value = 1, which denotes Yes. It inverted its own eligibility conclusion when encoding the numeric output." us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the correct FICA components ($3,888.95 + $909.51) and then added a fabricated ""Ohio state payroll tax (employee share)"" of ~$621 by applying a 0.99% Ohio income-tax rate to wages — Ohio has no mandatory employee-side payroll tax, and state income tax belongs to state_income_tax_before_refundable_credits, not payroll_tax. It then compounded that with an invented ""$13.22 OASDI contribution cap adjustment"" and a final unexplained $4,000 jump from its own $5,432.76 subtotal to the submitted $9,432.76." @@ -1701,173 +1928,204 @@ us,scenario_025,payroll_tax,glm-5.3,llm_error,other,False,"It derived the answer us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its explanation names the right base and rates (6.2% plus 1.45% on the spouse's $62,725, no Additional Medicare Tax), which yields $4,798.48, but the submitted $9,571.50 is roughly double that — consistent with applying the combined employer-plus-employee 15.3% FICA rate instead of the 7.65% employee share. The prompt explicitly excludes employer payroll taxes, so only the employee 7.65% belongs in this output." us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It stated the tax was ""computed on annual wages for both spouses,"" but the head has no gross wages listed and unlisted numeric inputs are zero, so the base is the spouse's $62,725 alone. Its $9,106 corresponds to 7.65% of roughly $119,000 — nearly twice the actual base — from imputing wages to the wageless head and/or sweeping in the spouse's $32,200 private pension, which is not FICA-taxable; the correct product is $62,725 × 7.65% = $4,798.48." us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the submission carries no substantive computation to evaluate. The required derivation is 6.2% plus 1.45% on the spouse's $62,725 of gross wages — the only FICA wages in the household — for $4,798.48." -us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed $1,776 of tax and $1,576 after the $200 retirement credit, then submitted $875 — a figure no step of its own derivation produces. Its underlying path also used $2,150 exemptions instead of Ohio's $1,900-per-person tier for OAGI above $80,000 ($3,800 total) and treated the bracket as a flat 2.75% of the excess over $26,050, omitting the $332.00 base that makes tax on $91,125.30 equal $2,121.57." -us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It reproduced the reference exactly through Ohio AGI $94,925, the two $1,900 exemptions, taxable income $91,125, and the $200 retirement income credit, but computed the bracket as a pure 2.75% of the $65,075 excess over $26,050 ($1,789.56). Ohio's 2026 schedule for that bracket is $332.00 plus 2.75% of the excess, so pre-credit tax is $2,121.57; the omitted base is the entire $332.01 shortfall." -us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It started Ohio taxable income from federal taxable income by subtracting a ~$28,000 MFJ standard deduction, when Ohio starts from federal AGI and allows only the $1,900-per-person exemption, then applied a repealed pre-2023 graduated schedule topping out at 5.75% to produce a 6.8% effective rate. Ohio's 2026 schedule taxes the first $26,050 at 0% and the excess at $332.00 plus 2.75%, and it also ignored the $200 retirement income credit earned by the $32,200 pension." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It computed $1,792 from $1,850 exemptions and a bare 2.75% of the excess over $26,050, then padded the result with an unexplained 'add modest amount' to reach $1,936. The rule misses behind that number are Ohio's $1,900-per-person exemption ($3,800), the $332.00 bracket base that makes pre-credit tax $2,121.57, and the $200 retirement income credit it denied by asserting no qualifying credits apply." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It carried the correct $1,900 exemptions and $91,125 taxable base, computed $1,790 of tax, announced it was subtracting the retirement and joint filing credits, and then submitted a larger number, $2,426, that no step of its derivation yields. The correct schedule adds a $332.00 base to 2.75% of the $65,075.30 excess for $2,121.57, less the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted $2,500 per person ($5,000) instead of Ohio's $1,900 high-income tier ($3,800), and its own stated components — a $360 base plus 2.75% of $63,875 — sum to $2,116.56, not the $1,866 it submitted. On the correct $91,125.30 base the 2026 schedule gives $332.00 plus 2.75% of $65,075.30 = $2,121.57, less the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It deducted $2,400-per-person exemptions ($4,800) rather than Ohio's $1,900 tier for OAGI above $80,000, and computed the bracket as a flat 2.75% of the excess over $26,050, omitting the $332.00 base in Ohio's 2026 schedule. Its $200 retirement income credit was correct, so the $359.51 shortfall is that missing base plus $27.50 of over-claimed exemptions." -us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It subtracted roughly $500 of credits built from a $250 retirement income credit, when Ohio's retirement credit maxes at $200 for retirement income of $8,000 or more, plus a 'senior credit' for a 61- and a 57-year-old, when Ohio's $50 senior citizen credit requires age 65. It also used $2,350 exemptions instead of $1,900 each and omitted the $332.00 base in the $26,050–$100,000 bracket." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It taxed the first $26,500 at 2.5% when Ohio's first $26,050 of taxable income is taxed at 0%, and applied 3.5% to the remaining $66,425 when the 2.75% bracket runs to $100,000. It also used $1,000 personal exemptions instead of $1,900 each and skipped the $200 nonrefundable retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It subtracted a single $2,400 exemption rather than two $1,900 exemptions and invented intermediate rates of 3.226% and 3.688% that do not exist in Ohio's schedule, where the whole excess over $26,050 up to $100,000 is taxed at 2.75% on top of a $332.00 base. It also omitted the $200 retirement income credit for the $32,200 pension." -us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used $2,800-per-person exemptions ($5,600) instead of Ohio's $1,900 tier for OAGI above $80,000 ($3,800), computed a bare 2.75% of the excess over $26,050 without the schedule's $332.00 base, and applied no credits. The correct chain is $91,125.30 taxable, $2,121.57 of tax, less the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 of employer-sponsored insurance premiums from gross income to reach a $73,717 AGI, but the head has no wages from which pre-tax premiums are withheld and Ohio AGI is the full $94,925.30. It then claimed a $1,821.22 Ohio medical deduction by counting employer-plan premiums toward the 7.5%-of-FAGI test; those premiums are excluded, leaving $850 of unreimbursed expenses against a $7,119 floor and a zero deduction, and it also used $2,400 exemptions and dropped the $332.00 bracket base." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $1,695 equals 2.75% of the full $68,875 excess of Ohio AGI over $26,050 less the $200 retirement credit, so it never subtracted the two $1,900 personal exemptions and never applied the $332.00 base in Ohio's 2026 bracket. The correct chain is $94,925.30 − $3,800 = $91,125.30 taxable, $2,121.57 of tax, $1,921.57 after the credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a flat 2.75% to the excess over $26,050 after $4,800 of exemptions and stopped there. Ohio's 2026 bracket adds a $332.00 base, its exemption at this OAGI is $1,900 per person ($3,800), and the $32,200 pension earns a $200 nonrefundable retirement income credit; those three omissions produce the $159.57 gap." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used $2,700-per-person exemptions ($5,400) instead of Ohio's $1,900 tier and computed 2.75% of the excess over $26,050 with no base amount, when the 2026 bracket is $332.00 plus 2.75% of the excess. It also omitted the $200 retirement income credit, leaving it $176.01 short of $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $1,280.50 with no derivation; that figure corresponds to 2.75% of the excess over $26,050 on roughly $72,600 of taxable income, implying about $22,300 of deductions Ohio does not grant. Ohio taxable income here is $94,925.30 less two $1,900 exemptions, or $91,125.30, taxed at $332.00 plus 2.75% of the excess for $2,121.57 before the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It matched the reference through Ohio AGI $94,925, $3,800 of exemptions, $91,125 taxable, and the $200 retirement income credit, but priced the bracket as a bare 2.75% of the $65,075 excess over $26,050. Ohio's 2026 schedule adds a $332.00 base to that bracket, so pre-credit tax is $2,121.57 and the answer is $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied 2.75% to the entire $68,875 excess of Ohio AGI over $26,050 and subtracted $200, never deducting the two $1,900 personal exemptions that reduce the base to $91,125.30. It also omitted the $332.00 base amount in Ohio's 2026 bracket, which together account for the $227.51 shortfall." -us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It asserted that Ohio personal exemptions are $0 for MAGI over $80,000; the over-$80,000 tier is $1,900 per person, so this couple deducts $3,800 and taxable income is $91,125.30, not $94,925.30. It also omitted the $332.00 bracket base and the $200 retirement income credit for the $32,200 pension, though its rejection of the joint filing credit was correct." -us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted a $30,000 federal MFJ standard deduction on top of $4,800 of Ohio exemptions, when Ohio starts from federal AGI and grants no standard deduction, then taxed the entire $60,125 at 2.75% and so also ignored its own $26,050 zero-rate bracket. Ohio taxable income is $91,125.30, taxed at $332.00 plus 2.75% of the excess over $26,050 for $2,121.57 before the $200 retirement credit." -us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied Ohio's repealed pre-2023 graduated schedule (2.85% through 4.396%) to reach $3,717, when the 2026 schedule is 0% on the first $26,050 and $332.00 plus 2.75% on the excess. It then subtracted a $432 joint filing credit, which requires each spouse to have at least $500 of qualifying Ohio income and the head has none, plus a $40 exemption credit limited to OAGI of $30,000 or less, and the $1,260.80 submitted matches neither its own $3,245 conclusion nor any Ohio computation." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It answered $0 on the claim that a 'low-income/family structure' offsets the liability, but $94,925.30 of Ohio AGI is more than three times the $26,050 zero-rate ceiling and the only nonrefundable credit available is the $200 retirement income credit. Taxable income of $91,125.30 produces $2,121.57 of tax, $1,921.57 after that credit." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It applied an 'OH standard deduction' that does not exist — Ohio's only subtraction here is the $1,900-per-person exemption ($3,800) — leaving a taxable base roughly $20,000 too low for the $1,340 it reported. The correct base of $91,125.30 yields $332.00 plus 2.75% of $65,075.30 = $2,121.57, less the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It manufactured a $230.63 Ohio medical deduction by counting the head's $6,500 of employer-plan health insurance premiums toward the 7.5%-of-FAGI test; employer-sponsored premiums are excluded from that deduction, so only $850 of unreimbursed expenses is tested against the $7,119 floor and the deduction is zero. It also used the legacy $360.69 bracket base instead of the 2026 $332.00, and those two errors produce the $22.34 overstatement." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $2,400-per-person exemptions ($4,800) instead of Ohio's $1,900 tier, computed the bracket as a bare 2.75% of the excess over $26,050 without the $332.00 base, and declared no nonrefundable credits. The $32,200 of taxable pension earns the $200 retirement income credit, which is exactly the nonrefundable credit the question asks to subtract." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It ran the correct structure — $3,800 of exemptions, a base-plus-2.75% schedule, and the $200 retirement income credit — but used the legacy $360.69 base amount rather than the 2026 value of $332.00, producing $2,150.25 of pre-credit tax instead of $2,121.57. That single stale bracket parameter is the entire $28.68 overstatement." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It excluded the $32,200 private pension from Ohio taxable income; Ohio taxes private pension distributions in full and offsets them only with the capped retirement income credit, which is $200 at this pension level and OAGI, not the $25 it used. It compounded this with $2,150 exemptions instead of $1,900 and a joint filing credit that requires each spouse to have at least $500 of qualifying Ohio income, which the head does not have." -us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It claimed a $21,388.63 Ohio medical deduction built from the head's $21,208 of employer-sponsored premiums plus $6,500 of other premiums, but employer-plan premiums are excluded from Ohio's unreimbursed medical deduction and the head has no wages from which to pay them, so OAGI remains $94,925.30. Its $332.00 base and $200 retirement credit were right, so that phantom deduction plus $4,300 rather than $3,800 of exemptions is the whole $601.94 shortfall." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that Ohio taxable income falls to zero, but the only subtraction from the $94,925.30 of Ohio AGI is the two $1,900 personal exemptions, leaving $91,125.30 taxable. That base produces $2,121.57 of tax under the $332.00-plus-2.75% bracket, reduced only by the $200 nonrefundable retirement income credit to $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It deducted $4,800 of exemptions instead of the $1,900-per-person tier ($3,800) and computed a flat 2.75% of the excess over $26,050, omitting the $332.00 base in Ohio's 2026 bracket. It then claimed zero nonrefundable credits, dropping the $200 retirement income credit earned by the $32,200 pension." -us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It subtracted a $79.48 joint filing credit, but Ohio's joint filing credit requires each spouse to have at least $500 of qualifying Ohio adjusted gross income and the head has none, so no joint filing credit is allowed. It also computed the bracket as a bare 2.75% of the $65,075 excess, missing the $332.00 base that makes pre-credit tax $2,121.57 and the final figure $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 3.5% to income between roughly $46,100 and $89,625, but Ohio's higher rate applies only above $100,000, so the entire excess over $26,050 sits in the 2.75% bracket on top of the $332.00 base. It also used $2,650-per-person exemptions instead of $1,900 and omitted the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used $4,800 of exemptions rather than Ohio's $1,900-per-person tier ($3,800) and treated the bracket as a flat 2.75% of the excess over $26,050, omitting the $332.00 base. It also concluded that no nonrefundable credits apply, missing the $200 retirement income credit generated by the $32,200 taxable pension." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used $2,500-per-person exemptions ($5,000) instead of Ohio's $1,900 tier for OAGI above $80,000 and computed a bare 2.75% of the excess over $26,050, omitting the $332.00 base in the 2026 schedule. It applied no credits, so it also missed the $200 retirement income credit that brings $2,121.57 down to $1,921.57." -us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It correctly identified the $1,900 high-income exemption tier, the $91,125 taxable base, and the $200 retirement income credit, but computed the bracket as 2.75% of the $65,075 excess with no base amount. Ohio's 2026 schedule for the $26,050–$100,000 bracket is $332.00 plus 2.75% of the excess, giving $2,121.57 before credits and $1,921.57 after." -us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed exemptions plus 'senior/aged-based adjustments' leave taxable income near zero, but Ohio's exemptions total $3,800 against $94,925.30 of OAGI and Ohio has no age-based income subtraction; its only age-linked provision, the $50 senior citizen credit, requires age 65 and neither spouse is 65. The remaining $91,125.30 is taxed to $2,121.57 before the $200 retirement income credit." -us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It stated that no Ohio exemptions reduce AGI, but Ohio allows $1,900 per personal exemption at this OAGI level, so $3,800 comes off before rates. It also denied every nonrefundable credit, missing the $200 retirement income credit for the $32,200 pension, and omitted the $332.00 base in the $26,050–$100,000 bracket." -us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It gave no bracket arithmetic, and $2,651 is about 2.79% of the full $94,925 of Ohio AGI, meaning it taxed income inside the $26,050 zero-rate band and skipped both the $3,800 of personal exemptions and the $200 retirement income credit. Ohio's schedule applies $332.00 plus 2.75% to the $65,075.30 of taxable income above $26,050, giving $2,121.57 before that credit." -us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It subtracted a $9,300 'MFJ standard deduction' that Ohio does not provide, then taxed the first $26,050 at 2.75% when that band is 0% and the balance at 3.5% when 2.75% applies below $100,000. It also claimed a $25 senior citizen credit, though Ohio's senior credit is $50 and requires age 65 while the head is 61, and it never applied the $200 retirement income credit." -us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model applied only the CHIP income ceiling and failed to determine Medicaid eligibility first. Child 1 qualifies for Medicaid under the OLDER_CHILD category, which makes the child ineligible for CHIP regardless of being below the asserted CHIP limit." -us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below 211% FPL as sufficient for CHIP eligibility and omitted the requirement that the child not qualify for Medicaid. Child 1 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false." -us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model incorrectly used 133% FPL as Child 1's Medicaid cutoff and therefore placed the child into a supposed gap between Medicaid and CHIP thresholds. The applicable OLDER_CHILD Medicaid pathway covers Child 1, and that Medicaid eligibility precludes CHIP." -us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_chip_eligible, violating the required output contract. The required result is 0 because Child 1 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP." -us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It applied North Carolina's pre-2023 older-child Medicaid limit of 133% FPL (138% with the disregard); NC folded NC Health Choice into Medicaid in April 2023, raising the ages 6–18 limit to roughly 216% FPL (211% plus the five-percentage-point disregard). Its own MAGI of ~$82,300 is 213% FPL, inside that limit, so the child qualifies under the OLDER_CHILD category." -us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It computed the income position correctly at 213% FPL but compared it to the obsolete 133% FPL ceiling for NC children ages 6–18. Since North Carolina merged NC Health Choice into Medicaid in 2023, the older-child limit is about 216% FPL, and 213% clears it." -us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a five-person 2026 poverty guideline of $27,465 — roughly the three-person figure; the correct base is about $38,600, making the household's $82,301 MAGI 213% FPL rather than the 310% its arithmetic implies. It also compared gross wages of $85,209 instead of MAGI net of the $2,908 traditional 401(k)/IRA contributions, and wrongly treated employer-sponsored insurance as disqualifying, which is a CHIP exclusion and not part of the Medicaid MAGI test." -us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"It applied a 138% FPL ceiling that NC has not used for children ages 6–18 since the April 2023 NC Health Choice-to-Medicaid transition raised that limit to about 216% FPL. It also compared gross wages of $85,209 rather than MAGI of $82,301 (213% FPL), which sits under the real limit." -us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"It used the stale 138% FPL older-child limit instead of NC's post-2023 limit of about 216% FPL, and even miscomputed its own threshold, calling 138% of a $37,650 five-person guideline ""~$45,000"" when it is $51,957. The correct comparison is $82,301 MAGI = 213% FPL against 216%, which the child clears." -us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It named NC's post-transition older-child limit correctly at 211% FPL but placed the household at 260% FPL, an income-to-poverty ratio consistent with adding the $7,746 FLSA overtime premium on top of the $85,209 wages that already include it and using a household size below five. MAGI is $85,209 less $2,908 of traditional 401(k)/IRA contributions = $82,301, or 213% of the five-person guideline, under the 216% limit that includes the five-percentage-point disregard." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"It invented an other-coverage bar for Medicaid: PolicyEngine's MAGI Medicaid test turns only on the person's category (OLDER_CHILD for age 11) and income relative to the state limit, with no exclusion for holding employer-sponsored insurance — that exclusion belongs to CHIP. Ignoring the ESI flag, the household's 213% FPL MAGI is inside NC's ~216% older-child limit." -us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It cited the right 211% FPL children's limit but converted it to ""about $67,000 for a family of 5,"" a dollar figure built on a roughly $31,800 poverty base — the four-person guideline, not the five-person ~$38,600 — and then compared gross income ""near $93,000"" that double-counts the $7,746 overtime premium already inside the $85,209 wage figure. MAGI is $82,301, or 213% FPL, below NC's 216% older-child limit." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It added the $7,746 FLSA overtime premium on top of the $85,209 wages that already include it, reaching ""~93k,"" and never subtracted the $2,908 of traditional 401(k)/IRA contributions that MAGI excludes. Actual MAGI is $82,301 = 213% FPL, under NC's older-child limit of about 216% FPL (211% plus the five-point disregard) rather than the 210%/$77k line it drew." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It computed MAGI exactly right at $82,301 but tested it against 133% FPL, the limit North Carolina used for children ages 6–18 before NC Health Choice was folded into Medicaid in April 2023. Against the current ~216% FPL older-child limit, $82,301 (213% FPL) qualifies." -us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It derived the correct $82,301 MAGI for a household of five but asserted it exceeds NC's child Medicaid limit; at 213% FPL it falls under North Carolina's post-2023 older-child ceiling of about 216% FPL. Its conclusion is consistent with applying the pre-transition 133%/138% FPL limit." -us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It set NC's ages 6–18 limit at 147% FPL when the post-2023 older-child limit is about 216% FPL, and it deflated MAGI to $61,093 by subtracting the head's $21,208 employer-sponsored insurance premium, which is the employer contribution and not a wage exclusion. Correct MAGI is $82,301 = 213% FPL, inside the limit — and even its own $61,093 would qualify under the right threshold." -us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation, only that income exceeds the limit. The correct derivation is MAGI of $82,301 (wages $85,209 less $2,908 of traditional 401(k)/IRA contributions) = 213% FPL for a household of five against NC's older-child limit of about 216% FPL; a ""not eligible"" answer is consistent with applying the pre-2023 133%/138% FPL ceiling." -us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It reproduced the pre-2023 North Carolina structure exactly — Medicaid to 133% FPL with a separate CHIP program above it — but NC eliminated NC Health Choice in April 2023 and covers children ages 6–18 up to about 216% FPL through Medicaid itself. Its own $82,301 MAGI, 213% FPL, therefore lands in Medicaid's OLDER_CHILD category, not in a separate CHIP tier." -us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It used the obsolete 133% FPL children's threshold and computed MAGI as $61,093 by improperly deducting the head's $21,208 employer-sponsored insurance premium from wages. Correct MAGI is $82,301 = 213% FPL, below NC's current ~216% FPL older-child limit." -us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It asserted ineligibility with no derivation. The computation is MAGI $82,301 = 213% FPL for five people against North Carolina's post-2023 older-child Medicaid limit of about 216% FPL, so the 11-year-old qualifies; a zero is consistent with applying the pre-transition 133% FPL limit." -us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It claimed household income exceeds NC's children's Medicaid limit without computing either side. MAGI is $82,301, or 213% FPL for a household of five, and North Carolina's limit for children ages 6–18 is about 216% FPL since NC Health Choice merged into Medicaid in 2023 — the child is inside the limit." -us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave a bare income-too-high assertion with no threshold or income figure. The correct test puts MAGI of $82,301 at 213% FPL against NC's ~216% FPL older-child ceiling, so the answer is eligible; the zero is consistent with carrying the pre-2023 133% FPL limit for ages 6–18." -us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It stated only that income exceeds North Carolina's children's Medicaid limits. Running the test, MAGI of $82,301 equals 213% FPL for five people and NC's post-2023 limit for children ages 6–18 is about 216% FPL, which the household clears; its answer matches the superseded 133%/138% FPL threshold." -us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"It treated employer-sponsored insurance as an automatic Medicaid disqualifier; PolicyEngine's MAGI Medicaid rules contain no other-coverage bar, which is a CHIP-only requirement. The child's eligibility rests on the OLDER_CHILD category for ages 6–18 and MAGI of 213% FPL against NC's ~216% limit." -us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,state_local_rule,False,"It compared gross wages of $85,209 against a bare 211% FPL line, missing both the $2,908 of traditional 401(k)/IRA contributions that MAGI excludes and the five-percentage-point disregard that lifts NC's older-child limit to about 216% FPL. MAGI is $82,301 = 213% FPL, which clears 216%." -us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"It invoked an unnamed ""benchmark Medicaid threshold"" without computing MAGI or the FPL ratio. The actual test is $82,301 MAGI = 213% FPL for a five-person household against North Carolina's post-2023 limit of about 216% FPL for children ages 6–18, which the child meets." -us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It never applied a category, reasoning that no ""special-status facts"" were present, when the OLDER_CHILD MAGI pathway for ages 6–18 requires nothing beyond age and income. That pathway puts the household's $82,301 MAGI at 213% FPL, under NC's ~216% FPL older-child limit, so the child is eligible." -us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It asserted the household is over the school-age children's limit for a five-person unit without computing it; MAGI is $82,301, or 213% of the ~$38,600 five-person poverty guideline, against North Carolina's ~216% FPL older-child limit. Its zero is consistent with the pre-2023 133%/138% FPL ceiling that ended when NC Health Choice became Medicaid." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It relied on an ""estimated"" North Carolina child Medicaid threshold rather than the actual one. NC covers children ages 6–18 up to about 216% FPL through Medicaid after the 2023 NC Health Choice transition, and this household's MAGI of $82,301 is 213% FPL, so the child is eligible." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It claimed MAGI exceeds North Carolina's limit for an 11-year-old without stating either figure. MAGI is $85,209 of wages less $2,908 of traditional 401(k)/IRA contributions = $82,301, which is 213% FPL for five people and under NC's ~216% FPL older-child limit." -us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"It gave only an unquantified ""above the limit"" conclusion. The correct comparison is 213% FPL MAGI against North Carolina's post-2023 older-child Medicaid limit of about 216% FPL, so the child qualifies; a zero matches the superseded 133% FPL threshold for ages 6–18." -us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,state_local_rule,False,"Both of its grounds fail: the household's MAGI of $82,301 is 213% FPL, inside North Carolina's ~216% FPL limit for children ages 6–18, and employer-sponsored insurance is not a bar to Medicaid MAGI eligibility, only to CHIP. Its income conclusion tracks the pre-2023 133%/138% FPL limit that NC replaced when NC Health Choice folded into Medicaid." -us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,state_local_rule,False,"It applied the 133% FPL limit that North Carolina used for children ages 6–18 before the April 2023 NC Health Choice-to-Medicaid transition raised it to about 216% FPL. At 213% FPL, the household's MAGI is within the current limit." -us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It stripped the $7,746 FLSA overtime premium out of wages to reach $74,555 — the overtime premium is part of gross wages and the OBBBA overtime deduction is below-the-line, so it does not reduce MAGI — and then measured against the obsolete 133% FPL limit. MAGI is $82,301 = 213% FPL against NC's ~216% older-child ceiling, and even its own $74,555 (193% FPL) would have qualified under the correct threshold." -us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Its ""~240% FPL"" comes from stacking the $7,746 overtime premium on top of the $85,209 wages that already include it; MAGI net of the $2,908 traditional 401(k)/IRA contributions is $82,301, or 213% FPL. It also used the pre-2023 133% FPL base rather than NC's current ~216% FPL limit for children ages 6–18." -us,scenario_026,child1_medicaid_eligible,inkling,llm_error,state_local_rule,False,"It used the 133% FPL/~$50,000 limit that North Carolina retired in April 2023 when NC Health Choice became Medicaid, lifting the ages 6–18 limit to about 216% FPL. It also imported CHIP's uninsured requirement into the analysis; Medicaid's MAGI test has no such condition, and the household's 213% FPL MAGI qualifies." -us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for child1_medicaid_eligible, so no substantive reasoning was applied to the older-child Medicaid test. The graded answer is absent rather than wrong." -us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It applied the 133%/138% FPL limit North Carolina used for children ages 6–18 before NC Health Choice merged into Medicaid in 2023, when the current limit is about 216% FPL. It also inflated the ratio to 226% by comparing gross wages of $85,209 against a 2025 five-person guideline instead of MAGI of $82,301 against the 2026 guideline, which yields 213% FPL." -us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It asserted income is ""well above NC Medicaid for Children limits"" without computing either quantity; MAGI of $82,301 is 213% FPL for five people and North Carolina's older-child limit is about 216% FPL. Its answer is consistent with the pre-2023 133% FPL ceiling for ages 6–18." -us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,state_local_rule,False,"It correctly assigned the child to NC's ages 6–18 group but pegged that group's limit at 133% FPL instead of the roughly 216% FPL North Carolina adopted when NC Health Choice folded into Medicaid in April 2023. Even its own income figure of 198% FPL is under the real limit, and the exact MAGI of $82,301 is 213% FPL." -us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"It concluded income was within the children's limit and then overrode that with a nonexistent rule that employer-sponsored insurance disqualifies a child from Medicaid; the other-coverage bar applies to CHIP, not to PolicyEngine's Medicaid MAGI test. With the OLDER_CHILD category and MAGI at 213% FPL against NC's ~216% limit, the child is eligible." -us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,state_local_rule,False,"It offered only ""household income is too high"" with no threshold or MAGI figure. MAGI is $82,301 — wages of $85,209 less $2,908 of traditional 401(k)/IRA contributions — or 213% FPL for five people, under North Carolina's post-2023 older-child Medicaid limit of about 216% FPL." -us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below 216% FPL as sufficient for CHIP and omitted the prerequisite that the child not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes the child ineligible for CHIP." -us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model applied the NC Health Choice income threshold as a standalone eligibility test and never checked Medicaid eligibility first. Child 2's OLDER_CHILD Medicaid eligibility excludes the child from CHIP. -us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility from placing MAGI between 133% and 211% FPL, but failed to apply Medicaid precedence. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP." -us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer for child2_chip_eligible. The required output is 0 because Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that Medicaid eligibility excludes CHIP eligibility." -us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It identified the correct NC pathway (~211% FPL for ages 6-18) but botched the denominator, dividing $82,301 by a stale ~$37,600-$37,900 five-person guideline to get 219% FPL; the 2026 guideline puts that same MAGI at 2.13x FPL. It also omitted the 5-percentage-point MAGI disregard that raises NC's applied older-child limit above 2.13, so a ratio that clears the threshold was scored as exceeding it." -us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It computed the income ratio correctly at 213% FPL but tested it against the federal 133% minimum for ages 6-18, missing that North Carolina covers older children under Medicaid at 211% FPL after the April 2023 NC Health Choice transition folded that CHIP population into Medicaid. With the 5-percentage-point disregard the applied older-child limit exceeds 2.13, so the child qualifies." -us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It applied an employer-coverage screen that does not exist in PolicyEngine's MAGI Medicaid test, which turns only on category and income, and asserted 'two wage earners' when the Head has zero wages. Household MAGI is $82,301, or 2.13x FPL, inside NC's older-child limit of 211% FPL plus the 5-point disregard." -us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2024 five-person poverty guideline of $36,580 instead of the 2026 figure, so 211% priced out at $77,184 against its $82,431 income; on the 2026 guideline the correct $82,301 MAGI is 2.13x FPL and clears NC's older-child limit. It then discarded the correct 211% pathway for the federal 133% floor, which NC does not use for ages 6-18." -us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It left income at ~$85,000 without subtracting the $2,778 traditional 401(k) and $130 traditional IRA, and divided by the 2024 five-person guideline of $36,580 rather than the 2026 one, producing a ratio above 211%. The actual MAGI of $82,301 is 2.13x the 2026 five-person guideline, within NC's older-child Medicaid limit." -us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"Its ~260% FPL figure comes from dividing undeducted gross wages by a four-person poverty guideline (~$32,800); the five-person 2026 guideline puts the $82,301 MAGI at 2.13x FPL. NC's older-child limit for ages 6-18 is 211% FPL plus the 5-point disregard, which 2.13 clears." -us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,It invented a rule that employer-sponsored coverage disqualifies a child from Medicaid; PolicyEngine's MAGI eligibility test contains no insured/uninsured or ESI-access screen. The child qualifies on the OLDER_CHILD income test at 2.13x FPL against NC's 211%-plus limit. -us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It named a limit its own income figure clears — 216% FPL against 213% — then contradicted itself by pricing 216% at '~$67,000 for a family of 5', a dollar amount derived from a three-person guideline; 216% of the 2026 five-person guideline is roughly $83,000, above the $82,301 MAGI. It also added overtime on top of gross wages, which the prompt states already includes overtime pay." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a flat 210% children's limit with no 5-percentage-point MAGI disregard and never computed the household ratio. MAGI of $82,301 is 2.13x FPL, inside NC's applied older-child limit of 211% plus the disregard." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but compared it to 133% FPL ($51,205), the federal minimum for ages 6-18. North Carolina covers that age group under Medicaid at 211% FPL since the 2023 NC Health Choice transition, and 2.13x FPL is under that limit." -us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It reached the correct MAGI of $82,301 and then asserted it exceeds NC's child income limits without naming one; at 2.13x the 2026 five-person guideline it falls under NC's older-child Medicaid threshold of 211% FPL plus the 5-point disregard. The verdict tracks the federal 133% floor, which NC does not apply to ages 6-18." -us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the federal 133% floor rather than NC's 211% older-child limit, and separately subtracted the $21,208 employer-sponsored insurance premium from income to reach $61,093 when PolicyEngine's Medicaid MAGI nets out only the traditional 401(k) and IRA contributions, leaving $82,301. Under the correct NC limit even its own $61,093 figure (1.58x FPL) qualifies, so the threshold error alone decided the answer." -us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no threshold or income figure. The correct derivation nets $85,209 of wages against $2,778 of traditional 401(k) and $130 of traditional IRA contributions to $82,301, or 2.13x FPL, inside NC's older-child Medicaid limit; the 'above the threshold' verdict matches only the federal 133% floor that NC does not use for ages 6-18." -us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It computed MAGI correctly at $82,301 but routed the child to CHIP under a 133% Medicaid ceiling. North Carolina folded its NC Health Choice CHIP population for ages 6-18 into Medicaid in April 2023, so the 211%-plus limit the model reserved for CHIP is the Medicaid limit, and 2.13x FPL clears it." -us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It tested income against the federal 133% FPL floor instead of NC's 211% older-child Medicaid limit, and it built $61,093 by subtracting the $21,208 employer-sponsored insurance premium, which Medicaid MAGI does not deduct; MAGI is $82,301, or 2.13x FPL. Under NC's actual limit even its own $61,093 figure would qualify." -us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It asserted ineligibility with no rule, threshold, or income figure. The correct derivation puts household MAGI at $82,301 and 2.13x the 2026 five-person poverty guideline, under NC's older-child limit of 211% FPL plus the 5-point disregard, so the answer tracks the federal 133% floor that NC replaced for ages 6-18." -us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It asserted income exceeds NC's Medicaid limit without computing either side. MAGI is $82,301, or 2.13x FPL, and NC's older-child limit for an 11-year-old is 211% FPL plus the 5-percentage-point disregard, so the child is inside the limit; the answer is consistent with applying the federal 133% floor." -us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave a bare income-exceeds-limit assertion with no threshold. The OLDER_CHILD test compares 2.13x FPL ($82,301 MAGI over the 2026 five-person guideline) against NC's 211%-plus limit and passes; the ineligible verdict matches the federal 133-138% floor that North Carolina does not use for ages 6-18." -us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It stated only that household income exceeds North Carolina's child Medicaid limits. NC's limit for ages 6-18 is 211% FPL plus the 5-point MAGI disregard, and the household's $82,301 MAGI is 2.13x FPL, so the child qualifies; the answer is consistent with the federal 133% floor rather than NC's post-2023 older-child threshold." -us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,state_local_rule,False,"It kept the pre-2023 North Carolina structure — Medicaid at 133% FPL with NC Health Choice as a separate CHIP program at 210% — after NC moved children 6-18 into Medicaid in April 2023 at 211% FPL, and added an employer-coverage bar that PolicyEngine's MAGI test does not contain. It also compared undeducted gross wages of $85,209 to an inflated ~$40,000 five-person guideline; the actual $82,301 MAGI is 2.13x FPL, within NC's older-child limit." -us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"It compared gross wages of $85,209 to the 211% limit without subtracting the $2,778 traditional 401(k) and $130 traditional IRA contributions or applying the 5-percentage-point MAGI disregard. MAGI is $82,301, or 2.13x FPL, inside NC's older-child Medicaid limit." -us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It asserted earnings are above 'the benchmark Medicaid threshold' without computing MAGI or naming a limit. MAGI is $82,301 = 2.13x the 2026 five-person guideline, and NC covers ages 6-18 under Medicaid at 211% FPL plus the 5-point disregard, so the child qualifies; the answer is consistent with the federal 133% floor." -us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It concluded from 'household income level implied by wages and no special-status facts' that the child fails, but the OLDER_CHILD MAGI category needs no disability, SSI, or other special status — only age 6-18 and income under NC's 211%-plus limit. The household's $82,301 MAGI is 2.13x FPL and satisfies that test." -us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It called $85,209 'well above the NC child Medicaid income limit' without naming the limit or netting the $2,778 traditional 401(k) and $130 traditional IRA. MAGI is $82,301, or 2.13x FPL, and NC's older-child limit is 211% FPL plus the 5-point disregard, so the child qualifies; the verdict matches the federal 133% floor NC does not use for ages 6-18." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It relied on an 'estimated' North Carolina child Medicaid threshold rather than the actual one. NC covers ages 6-18 under Medicaid at 211% FPL plus the 5-percentage-point disregard after the 2023 NC Health Choice transition, and the household's $82,301 MAGI is 2.13x FPL, so the child qualifies." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It asserted MAGI exceeds North Carolina's limit for an 11-year-old without computing either quantity. MAGI is $82,301 — wages less the traditional 401(k) and IRA contributions — which is 2.13x the 2026 five-person guideline and inside NC's 211%-plus older-child limit; the ineligible answer is consistent with applying the federal 133% floor." -us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It gave a bare assertion that income is above NC's child Medicaid limit. The OLDER_CHILD test puts $82,301 of MAGI at 2.13x FPL against NC's 211% limit plus the 5-point disregard and passes; the answer tracks the federal 133-138% floor that North Carolina replaced for ages 6-18 in 2023." -us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"It leaned on employer-sponsored insurance as disqualifying, a screen PolicyEngine's MAGI Medicaid test does not apply, and paired it with an uncomputed income claim. MAGI is $82,301, or 2.13x FPL, inside NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard." -us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It applied the federal 133% FPL minimum for children 6-18; North Carolina covers that group under Medicaid at 211% FPL since NC Health Choice moved into Medicaid in April 2023. The household's $82,301 MAGI is 2.13x FPL and clears that limit." -us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It deducted the $7,746 FLSA overtime premium to reach $74,555, but the qualified-overtime deduction reduces taxable income rather than AGI or Medicaid MAGI, leaving MAGI at $82,301. It then tested against the federal 133% floor instead of NC's 211% older-child limit, under which even its own $74,555 (1.93x FPL) qualifies." -us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Its ~240% FPL figure divides undeducted gross wages by a poverty guideline for a smaller household; the 2026 five-person guideline puts the $82,301 MAGI at 2.13x FPL. It then used the 133% federal base rather than NC's older-child limit of 211% plus the 5-point disregard, which 2.13 clears." -us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It capped ages 6-18 at the federal 133% FPL floor and added an employer-coverage bar that PolicyEngine's MAGI test does not apply. North Carolina covers older children under Medicaid at 211% FPL after the 2023 NC Health Choice transition, and the household's 2.13x FPL is inside that limit." -us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for child2_medicaid_eligible, so no determination was returned. The correct derivation nets wages to $82,301 of MAGI, or 2.13x the 2026 five-person poverty guideline, inside NC's OLDER_CHILD Medicaid limit of 211% FPL plus the 5-point disregard." -us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It divided undeducted gross wages of $85,209 by the 2025 five-person guideline (~$37,650) to get 226% and compared that to 133-138% FPL. The 2026 derivation is $82,301 of MAGI at 2.13x FPL against NC's older-child limit of 211% plus the 5-percentage-point disregard, which the household clears." -us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It asserted income is 'well above NC Medicaid for Children limits' without computing MAGI or the limit. MAGI is $82,301, or 2.13x FPL, and NC's limit for ages 6-18 is 211% FPL plus the 5-point disregard, so the child qualifies; the answer is consistent with the federal 133% floor." -us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed the child in the correct 6-18 group but capped that group at 133% FPL; North Carolina covers it at 211% FPL plus the 5-point disregard after the 2023 NC Health Choice transition. Even the model's own 198% FPL income figure clears that limit, and the actual ratio is 2.13x FPL on $82,301 of MAGI." -us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used both a 133% FPL limit and a five-person guideline of ~$33,700, when NC covers ages 6-18 under Medicaid at 211% FPL and the 2026 five-person guideline puts the $82,301 MAGI at 2.13x FPL. It also treated employer-sponsored insurance as disqualifying, a screen PolicyEngine's MAGI Medicaid test does not contain." -us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It stated only that household income is too high, with no threshold or MAGI computation. The household's MAGI of $82,301 is 2.13x the 2026 five-person poverty guideline and sits under NC's older-child Medicaid limit of 211% FPL plus the 5-percentage-point disregard, so the answer is consistent with applying the federal 133% floor NC replaced for ages 6-18." -us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated the asserted 133%–211% FPL CHIP income band as sufficient and omitted the prerequisite that a child must not qualify for Medicaid. Child 3 qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP." -us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model applied only its asserted 216% FPL CHIP ceiling and failed to test Medicaid eligibility first. Child 3's Medicaid eligibility under the OLDER_CHILD category bars CHIP eligibility even when income is below the cited CHIP limit. -us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model concluded that income below 211% FPL established CHIP eligibility, omitting CHIP's exclusion of children who already qualify for Medicaid. Child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible." -us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model relied on age, lack of Medicare eligibility, and placement within a CHIP income range but never applied the controlling Medicaid-ineligibility condition. Child 3 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false." -us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,The model treated being age 9 with MAGI between 133% and 211% FPL as sufficient for CHIP and skipped the Medicaid eligibility screen. The child's OLDER_CHILD Medicaid eligibility automatically excludes the child from CHIP. -us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model explicitly asserted that Child 3 was not Medicaid-eligible, reversing the decisive eligibility result. Child 3 is Medicaid-eligible under the OLDER_CHILD category, and that status disqualifies the child from CHIP regardless of the model's household-size and CHIP-threshold calculation." -us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Applied the ACA-minimum 138% FPL limit to the 6-18 age group and routed anything above it to CHIP, missing that North Carolina folded NC Health Choice into Medicaid in April 2023, so the older-child Medicaid limit is 211% FPL (216% after the 5-percentage-point MAGI disregard). Its own MAGI of $82,301 is 213% of the five-person 2026 guideline of about $38,600, inside that limit. Its stated 197% ratio also understates the arithmetic it described: $82,301 against the $37,650 five-person guideline it cited is 219%." -us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Computed the income ratio correctly at 213% FPL, then tested it against 133% FPL — the federal floor for ages 6-18 — instead of North Carolina's 211% older-child limit, which becomes 216% after the 5-percentage-point MAGI disregard. At 213% the child clears the state's actual older-child threshold." -us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a NC child limit in the right range (207% FPL) but converted it against a roughly three-person poverty guideline (~$27,200), producing a $56,232 cutoff instead of the ~$81,500-$83,500 range that 211%-216% of the ~$38,600 five-person 2026 guideline yields. Its own MAGI of about $82,400 is 213% FPL and falls inside NC's older-child limit." -us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Cited NC's ~211% older-child limit but converted it against a four-person poverty guideline (~$66,000) rather than the five-person guideline, where 211% is ~$81,500 and the 5-point MAGI disregard raises the cutoff to ~$83,500. It also compared gross wages of $85,209 instead of MAGI of $82,301 (wages less the $2,778 traditional 401(k) and $130 traditional IRA), which is 213% FPL and eligible." -us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Identified the 211% FPL NC Health Choice level but classified it as CHIP rather than Medicaid — NC merged NCHC into Medicaid in April 2023 — and compared gross wages of $85,209 to its own $83,200 cutoff instead of MAGI of $82,301 after the $2,778 traditional 401(k) and $130 traditional IRA deductions. That MAGI is 213% FPL, under both its own cutoff and the 216% effective limit that includes the 5-percentage-point disregard." -us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"Reached ~260% FPL by dividing gross wages of $85,209 by a four-person poverty guideline (~$32,800). MAGI is $82,301 against the ~$38,600 five-person 2026 guideline, i.e. 213% FPL, which sits inside North Carolina's 211%-plus-5-point older-child Medicaid limit." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"Invented an employer-coverage bar for Medicaid: ESI disqualifies CHIP, not Medicaid, and PolicyEngine applies no insurance test to the child Medicaid categories, so the child's ESI is irrelevant to the OLDER_CHILD determination. It then compared gross wages of $85,209 to 210% of a $36,580 guideline, while MAGI is $82,301 — 213% of the ~$38,600 five-person 2026 guideline — inside NC's 216% effective older-child limit." -us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"Added the $7,746 FLSA overtime premium on top of the $85,209 gross wage total to reach 'over $92,000', double-counting overtime the prompt states is already included in the annual gross figure, and then applied the 138% ACA floor to ages 6-18. MAGI is $82,301 (213% FPL) against North Carolina's 211% older-child limit plus the 5-percentage-point disregard." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Compared gross wages of $85,209 (220% FPL) to a bare 210% children's limit. The MAGI test uses $82,301 after the $2,778 traditional 401(k) and $130 traditional IRA deductions — 213% FPL — and NC's older-child cutoff is 211% plus the 5-percentage-point disregard, i.e. 216%." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Computed MAGI exactly at $82,301 but tested it against 133% FPL ($51,205), the federal minimum coverage level for ages 6-18, rather than North Carolina's 211% older-child limit (216% with the MAGI disregard). At 213% FPL the child qualifies under the OLDER_CHILD category." -us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Carried the correct $82,301 MAGI but asserted it exceeds NC's child limits for a household of five. North Carolina covers ages 6-18 under Medicaid to 211% FPL plus the 5-percentage-point disregard since NC Health Choice merged into Medicaid, and $82,301 is 213% of the ~$38,600 five-person 2026 guideline." -us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Deducted the $21,208 employer-sponsored insurance premium from wages to reach a $61,093 MAGI; ESI premiums are not a MAGI deduction and the correct figure is $82,301. It then applied a 133% FPL limit for ages 6-18, while NC's older-child limit is 211% plus the 5-point disregard — under which even its own $61,093 (158% FPL) qualifies." -us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave a bare 'income above the threshold' assertion with no FPL ratio or limit computed. The correct test puts MAGI at $82,301, 213% of the ~$38,600 five-person 2026 guideline, against North Carolina's 211% older-child limit plus the 5-percentage-point disregard; the zero is consistent with the 133-138% ACA floor rather than NC's post-NCHC-merger threshold." -us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Had the exact $82,301 MAGI but capped older-child Medicaid at 133% FPL and shifted the child to CHIP. North Carolina folded NC Health Choice into Medicaid in 2023, so ages 6-18 are covered under Medicaid to 211% FPL (216% with the disregard), and 213% is inside that band." -us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Netted the $21,208 ESI premium out of wages to get a $61,093 income figure — ESI premiums do not reduce MAGI, which is $82,301 — and then applied a 133% FPL children's threshold. NC's older-child limit is 211% plus the 5-point disregard, so both $61,093 (158% FPL) and the correct $82,301 (213% FPL) qualify." -us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Asserted ineligibility with no rule, ratio, or threshold stated. The OLDER_CHILD category covers ages 6-18 in NC to 211% FPL plus the 5-percentage-point disregard, and MAGI of $82,301 is 213% of the ~$38,600 five-person 2026 guideline, so the zero tracks the 133% federal floor instead of North Carolina's own limit." -us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Claimed household income exceeds the NC Medicaid limit for children without naming or computing one. MAGI is $82,301 — 213% of the five-person 2026 guideline — against North Carolina's 211% older-child limit plus the 5-point MAGI disregard, so the answer is consistent with applying the 133% ACA minimum for ages 6-18." -us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Offered a bare income-exceeds-limits conclusion with no threshold or FPL ratio. North Carolina covers ages 6-18 under Medicaid to 211% FPL plus a 5-percentage-point disregard, and MAGI of $82,301 is 213% of the ~$38,600 five-person guideline; the zero matches the 133-138% federal floor, not NC's level." -us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Stated only that income exceeds North Carolina's Medicaid limits for children. The applicable OLDER_CHILD limit is 211% FPL plus the 5-percentage-point MAGI disregard, and household MAGI of $82,301 is 213% of the five-person 2026 guideline, so the zero reflects the 133% ACA minimum rather than the state's post-merger threshold." -us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"Computed the ratio correctly at roughly 213% FPL, then treated the ~210% level as a CHIP-only ceiling and cited ESI as a disqualifier. NC Health Choice became Medicaid in April 2023, making 211% FPL the older-child Medicaid limit — 216% after the 5-percentage-point disregard — and Medicaid applies no employer-coverage test." -us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"Compared gross wages of $85,209 to the 211% level rather than MAGI of $82,301 (wages less the $2,778 traditional 401(k) and $130 traditional IRA), and omitted the 5-percentage-point MAGI disregard that lifts NC's older-child cutoff to 216% FPL. At 213% FPL the child is under that cutoff." -us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Invoked an unnamed 'benchmark Medicaid threshold' without computing an FPL ratio or a state limit. North Carolina's older-child limit for ages 6-18 is 211% FPL plus the 5-percentage-point disregard, and MAGI of $82,301 is 213% of the ~$38,600 five-person 2026 guideline, so the zero corresponds to the 133% ACA floor." -us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Inferred ineligibility from the wage level alone, with no FPL guideline, MAGI figure, or state threshold. At age 9 the OLDER_CHILD category applies, NC's limit is 211% FPL plus the 5-point disregard, and MAGI of $82,301 is 213% of the five-person guideline — inside the limit." -us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Compared 'about $85,209 before limited pretax deductions' to an unnamed NC child limit; the test uses MAGI of $82,301 after the $2,778 traditional 401(k) and $130 traditional IRA deductions, which is 213% of the ~$38,600 five-person 2026 guideline and inside NC's 211%-plus-5-point older-child limit. Its observation that employer coverage does not change the result is correct, but the income test it relied on passes rather than fails." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Estimated the North Carolina child Medicaid threshold instead of applying it: the state covers ages 6-18 to 211% FPL, 216% with the 5-percentage-point MAGI disregard, following the NC Health Choice merger into Medicaid. MAGI of $82,301 is 213% of the five-person 2026 guideline, so the zero is consistent with the 133-138% federal floor." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Asserted MAGI exceeds North Carolina's limit for a 9-year-old without stating either number. MAGI is $82,301 — 213% of the ~$38,600 five-person 2026 guideline — against the OLDER_CHILD limit of 211% FPL plus the 5-percentage-point disregard, so the answer tracks the 133% ACA minimum instead of NC's level." -us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Gave a bare 'income above the North Carolina child Medicaid limit' conclusion with no threshold computed. NC covers ages 6-18 to 211% FPL plus the 5-point MAGI disregard, and household MAGI of $82,301 is 213% of the five-person 2026 guideline, which qualifies under OLDER_CHILD." -us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"Paired an unnamed income threshold with an ESI bar; employer-sponsored coverage disqualifies CHIP, not Medicaid, and no insurance test applies to the child Medicaid categories. MAGI of $82,301 is 213% of the five-person 2026 guideline, under NC's 211% older-child limit plus the 5-percentage-point disregard." -us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Applied the 133% FPL ACA floor as North Carolina's limit for ages 6-18. The state's older-child Medicaid limit is 211% FPL — 216% with the 5-percentage-point disregard — after NC Health Choice folded into Medicaid, and household MAGI of $82,301 is 213% of the five-person 2026 guideline." -us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Subtracted the $7,746 FLSA overtime premium from wages to reach $74,555; the qualified-overtime deduction is below-the-line and does not reduce AGI or MAGI, which is $82,301 after only the $2,778 traditional 401(k) and $130 traditional IRA. It then applied a 133% FPL limit, while NC's older-child limit is 211% plus the 5-point disregard, so both $74,555 (193% FPL) and the correct $82,301 (213% FPL) qualify." -us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Placed the household at ~240% FPL by dividing gross wages by an undersized poverty guideline; MAGI of $82,301 against the ~$38,600 five-person 2026 guideline is 213% FPL. It also used 133% as NC's base for ages 6-18, when the state's older-child limit is 211% plus the 5-percentage-point MAGI disregard." -us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Used the 133% FPL federal floor for ages 6-18 and added an employer-coverage bar. North Carolina's older-child Medicaid limit is 211% FPL plus the 5-percentage-point disregard, MAGI of $82,301 is 213% of the five-person guideline, and ESI disqualifies CHIP rather than Medicaid." -us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child3_medicaid_eligible, so the required key is absent from its outputs. No older-child Medicaid income test was performed at all, making this a contract failure rather than a substantive rule error." -us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Divided gross wages of $85,209 by the 2025 five-person guideline to get ~226% FPL and compared it to a 133-138% limit. The MAGI figure is $82,301 against the ~$38,600 2026 five-person guideline — 213% — and North Carolina's older-child limit is 211% plus the 5-percentage-point disregard, i.e. 216%." -us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Declared income 'well above NC Medicaid for Children limits' without naming a limit or computing an FPL ratio. The limit for ages 6-18 is 211% FPL plus the 5-percentage-point disregard and MAGI of $82,301 is 213% of the five-person 2026 guideline, so the zero matches the 133% ACA floor rather than North Carolina's threshold." -us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"Applied a 133% FPL cap to NC's 6-18 group and placed income at ~198% FPL. North Carolina covers that group to 211% FPL plus the 5-percentage-point MAGI disregard, and MAGI of $82,301 is 213% of the ~$38,600 five-person 2026 guideline, so both its own ratio and the correct one fall inside the limit." -us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"Used a $33,700 five-person poverty guideline — the 2026 figure is about $38,600 — together with the 133% ACA floor to build a $44,821 cutoff, and compared gross wages rather than MAGI. NC's older-child limit is 211% FPL plus the 5-percentage-point disregard, and MAGI of $82,301 is 213%, which qualifies; the children's ESI is irrelevant to Medicaid." -us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Stated only that household income is too high for child Medicaid, with no guideline, MAGI, or state limit. MAGI is $82,301, 213% of the five-person 2026 guideline, against North Carolina's OLDER_CHILD limit of 211% FPL plus the 5-percentage-point disregard, so the zero is consistent with the 133% federal minimum for ages 6-18." +us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used $2,150 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It applied 2.75% above $26,050 with no $332 base amount. It then made an unexplained cut from about $1,576 down to $875, when the correct figure is $2,121.57 minus the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The exemptions ($3,800), taxable income ($91,125) and $200 retirement credit were right. The model left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess. Pre-credit tax was therefore $1,789.56 instead of $2,121.57." +us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted a federal standard deduction, but Ohio starts from federal AGI and has no standard deduction. It also used an outdated 0.501%–5.75% bracket schedule instead of the $332 + 2.75% bracket. It skipped the $1,900 exemptions and the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used $1,850 exemptions instead of $1,900. It added an arbitrary amount of about $144 instead of the $332 base. It also said no nonrefundable credits applied, which misses the $200 retirement income credit on the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model got $91,125 of taxable income but computed only 2.75% × $65,075 = $1,790, leaving out the $332 base amount. It then gave $2,426, which is higher than its own pre-credit tax even though it said credits were subtracted. It also mentioned a joint filing credit that does not apply because the head has no qualifying income." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used $2,500 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It used a $360 base amount instead of $332. The credits it subtracted do not match the single $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model assumed the 2026 schedule has no base amount. It left out the $332 fixed tax at the start of the 2.75% bracket, so pre-credit tax was $1,789.56 instead of $2,121.57, and the answer after the $200 credit was $1,589.56 instead of $1,921.57." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It also left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess before subtracting the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used $2,350 exemptions and left out the $332 base amount. It subtracted about $500 in credits, including a senior credit that requires age 65 (the head is 61), instead of the single $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Taxable income ($91,125) and the $200 credit were right. The model applied only 2.75% above $26,050 and left out the $332 base amount, which understates the answer by $332." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used $1,000 exemptions instead of $1,900. It invented a 2.5%/3.5% schedule with no $26,050 zero bracket, when the correct bracket is $332 + 2.75% over $26,050. It also skipped the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model took only one $2,400 exemption instead of two $1,900 exemptions. It used an outdated multi-bracket schedule (3.226%/3.688%) instead of $332 + 2.75% over $26,050. It also said no credits applied, missing the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used $2,800 exemptions instead of $1,900. It left out the $332 base amount and skipped the $200 retirement income credit for the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used $2,400 exemptions and applied 3.5% above $46,100 instead of $332 + 2.75% on everything up to $100,000. It took a 5% joint filing credit, which requires both spouses to have qualifying income, and missed the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the head's $21,208 employer insurance premiums from wages as if pre-tax, but the head has no wages and the stated wages are gross. It took an Ohio medical deduction for health premiums, which is not allowed when employer-subsidized coverage is available, and the other medical costs are below 7.5% of AGI. It also used $2,400 exemptions and left out the $332 base amount." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model showed no work. $1,695 matches 2.75% × ($94,925 − $26,050) − $200, which skips the $3,800 exemptions and the $332 base amount. The correct result is $332 + 2.75% × $65,075 − $200 = $1,921.57." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied only 2.75% above $26,050 with no $332 base amount, and its $1,762 implies $4,800 of exemptions instead of $3,800. It did not subtract the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $2,700 exemptions instead of the $1,900 amount for Ohio MAGI above $80,000. It left out the $332 base amount and skipped the $200 retirement income credit for the pension." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model showed no derivation. $1,280.50 is below even the 2.75%-only amount on $91,125 of taxable income, so the answer includes neither the $332 base amount nor the correct taxable income. The correct result is $2,121.57 minus the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The exemptions ($3,800) and $200 retirement credit were right. The model applied only 2.75% above $26,050 and left out the $332 base amount, which understates the tax by $332." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied 2.75% to the full $94,925 above $26,050 without subtracting the two $1,900 exemptions. It also left out the $332 base amount, while correctly subtracting the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model claimed personal exemptions are $0 above $80,000 of Ohio MAGI, when they are $1,900 each at this income. It left out the $332 base amount and skipped the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted a $30,000 federal standard deduction, which Ohio does not allow, and used $2,400 exemptions. It applied 2.75% to all of taxable income, ignoring both the $26,050 zero bracket and the $332 base amount, and skipped the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used an outdated pre-reform schedule (2.85%–4.396% brackets about $5,600 wide) and a joint filing credit that is unavailable because the head has no qualifying income. Its submitted $1,260.80 does not match its own computed $3,245, and neither matches $332 + 2.75% × $65,075 − $200." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated the household as low-income with no liability. Ohio taxable income of $91,125 is far above the $26,050 zero bracket and produces $2,121.57 of tax before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction, but Ohio has none. It should have subtracted only the two $1,900 exemptions and applied $332 + 2.75% over $26,050 before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $360.69 base amount (an earlier year's schedule) instead of $332. It also took a $230.63 medical deduction that counted $6,500 of health premiums, which are not deductible because both spouses have employer-subsidized coverage." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of $1,900 and left out the $332 base amount. It said no nonrefundable credits applied, missing the $200 retirement income credit on the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Taxable income ($91,125) and the $200 credit were right. The model used the older $360.69 base amount instead of $332, which overstates the answer by $28.69." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model excluded the $32,200 private pension from Ohio income. Ohio taxes private pensions and offers only the $200 retirement income credit. The model also used $2,150 exemptions and applied a joint filing credit, which the household does not qualify for because the head has no qualifying income." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model deducted $21,388.63 of medical costs, counting the head's $21,208 employer insurance premiums and the $6,500 of other premiums. Ohio allows no premium deduction when employer-subsidized coverage is available, and the $850 of other expenses is below 7.5% of AGI. It also used $2,150 exemptions instead of $1,900." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied only 2.75% above the zero bracket and left out the $332 base amount. Its answer is the 2.75%-only figure less the $200 credit, instead of $2,121.57 − $200." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The exemptions and retirement credit were right. The model computed only 2.75% of the excess over $26,050 and left out the $332 base amount in the bracket." +us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The $332 base amount, $3,800 exemptions and $200 credit were right. The model wrongly subtracted $6,500 of health premiums, which Ohio does not allow as a deduction when the taxpayer can get employer-subsidized coverage (both spouses have employer insurance)." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model claimed Ohio deductions and credits reduce tax to zero. With $91,125 of taxable income, well above the $26,050 zero bracket, tax is $2,121.57 before the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used $2,400 exemptions instead of $1,900 and left out the $332 base amount. It also claimed zero nonrefundable credits, missing the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model left out the $332 base amount. It also took a 5% joint filing credit, which requires each spouse to have qualifying income, while the head has none." +us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Exemptions, the $200 credit, and denial of the joint filing credit were right. The model left out the $332 base amount in the $26,050–$100,000 bracket, taxing only 2.75% of the excess." +us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used $2,650 exemptions and applied 3.5% above $46,100, when the correct bracket is $332 + 2.75% over $26,050 up to $100,000. It also skipped the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used $4,800 of exemptions instead of $3,800 and left out the $332 base amount. It said no nonrefundable credits applied, missing the $200 retirement income credit." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model used $2,500 exemptions instead of $1,900 and left out the $332 base amount. It also skipped the $200 retirement income credit for the pension." +us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The $1,900 exemptions and $200 credit were right. The model computed only 2.75% of the excess over $26,050 and left out the $332 base amount, so pre-credit tax was $1,789.56 instead of $2,121.57." +us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model invented senior or age-based adjustments that zero out the tax. Neither spouse is 65, and Ohio has no age exclusion that removes $91,125 of taxable income from the $332 + 2.75% bracket." +us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied no personal exemptions, when two $1,900 exemptions apply, and left out the $332 base amount. It missed the $200 retirement income credit on the $32,200 pension." +us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model gave no derivation. $2,651 is higher than even the correct $2,121.57 pre-credit tax, which fits an outdated multi-bracket schedule rather than $332 + 2.75% over $26,050, and it did not subtract the $200 retirement credit." +us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $9,300 Ohio standard deduction and taxed the first $26,050 at 2.75%, though that bracket is 0%. It applied 3.5% above that instead of $332 + 2.75%, and its bracket arithmetic is internally wrong. It claimed a senior credit that requires age 65 and missed the $200 retirement credit." +us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It computed MAGI of $82,301, below the 216% FPL limit (211% plus the 5% disregard), but assigned that band to CHIP. In NC that band is Medicaid coverage for older children (the OLDER_CHILD category), so the child is Medicaid-eligible and therefore excluded from CHIP." +us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It understated household income at $61,093; wages minus traditional 401(k) and IRA contributions give about $82,301. It also labeled the 211% FPL threshold as NC Health Choice/CHIP, but that program was folded into Medicaid. An 11-year-old under that limit is Medicaid-eligible (OLDER_CHILD), which rules out CHIP." +us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It set NC Medicaid for an 11-year-old at 133% FPL and treated NC Health Choice (211% FPL) as a separate CHIP tier. NC now covers children aged 6-18 up to 211% FPL through Medicaid (OLDER_CHILD), so this child is Medicaid-eligible and not CHIP-eligible." +us,scenario_026,child1_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed the child above a lower Medicaid limit and under a separate NC Health Choice CHIP limit of 211% FPL. It missed that NC's 211% coverage for school-age children is Medicaid (OLDER_CHILD), and Medicaid eligibility excludes the child from CHIP." +us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child1_chip_eligible. The correct derivation finds the 11-year-old Medicaid-eligible under NC's OLDER_CHILD category, which makes CHIP eligibility 0." +us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Used a 133% FPL limit (138% with the disregard) for NC children ages 6–18. That is the pre-2023 limit, from before NC Health Choice merged into Medicaid and raised the limit to 211% FPL (216% with the disregard). The model's own MAGI of about $82,300 is 2.13x FPL, which falls under that limit." +us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Got the income ratio right (~213% FPL) but compared it to the outdated 133% FPL limit for NC children ages 6–18. NC Medicaid now covers this group up to 211% FPL (216% with the 5% disregard), so 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Used a poverty line of $27,465 for five people, far below the 2026 guideline of about $38,640. It also applied a made-up 200% limit and compared gross wages instead of MAGI net of 401(k)/IRA contributions. It then treated employer-sponsored insurance as disqualifying, but Medicaid has no uninsured requirement. MAGI of $82,301 is 2.13x FPL, under NC's 216% effective child limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied a 138% FPL limit (~$51,957) to an 11-year-old. After the 2023 merger of NC Health Choice into Medicaid, NC's limit for ages 6–18 is 211% FPL (216% with the disregard). MAGI of $82,301 (2.13x FPL) is under that limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Used a 138% FPL limit and got its dollar value wrong (~$45,000). It also compared gross wages instead of MAGI. NC's Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the correct 211% FPL limit for ages 6–18 but put income at ~260% FPL. MAGI is only the spouse's $85,209 in wages minus $2,778 traditional 401(k) and $130 traditional IRA contributions, or $82,301. That is 2.13x FPL, under the 211%/216% limit." +us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Put income at ~219% FPL by dividing MAGI by the 2025 poverty guideline ($37,650) instead of the 2026 guideline. It also left out the 5% MAGI disregard. Under 2026 parameters, MAGI of $82,301 is 2.13x FPL, below NC's 211% child limit (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Only CHIP requires a child to be uninsured; Medicaid does not. Child 1 meets the OLDER_CHILD income test with MAGI at 2.13x FPL, under NC's 211% FPL limit (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Put income at ~$93,000 by adding the $7,746 overtime premium on top of wages that already include it, and did not subtract the traditional 401(k)/IRA contributions. It also priced 211% FPL at only ~$67,000 instead of about $81,500 (or ~$83,500 with the disregard). Correct MAGI is $82,301 (2.13x FPL), which is eligible." +us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Compared MAGI of about $82k to a 133% FPL limit. That limit is obsolete: NC Medicaid covers children ages 6–18 up to 211% FPL (216% with the disregard), and $82,301 is 2.13x FPL." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put income at ~$93k by adding the overtime premium on top of wages that already include it, and ignored the pre-tax 401(k)/IRA deductions. It also priced the 210% limit at only ~$77k and left out the 5% disregard. Correct MAGI is $82,301 (2.13x FPL), below the 216% effective limit." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but compared it to 133% FPL ($51,205). That is the pre-2023 NC Medicaid limit for ages 6–18; the current limit is 211% FPL (216% with the disregard), so 2.13x FPL qualifies." +us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but still found it over the child limit, which means it used a threshold below 2.13x FPL. NC Medicaid covers children ages 6–18 up to 211% FPL (216% with the 5% disregard)." +us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Compared gross wages of $85,209 to a bare 210% limit. It skipped the traditional 401(k) and IRA deductions that bring MAGI down to $82,301, and it left out the 5% disregard that raises the effective limit to 216%. At 2.13x FPL, Child 1 qualifies." +us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied a made-up 147% FPL limit for ages 6–18. It also subtracted the head's $21,208 ESI premium from MAGI, getting $61,093. NC's limit is 211% FPL (216% with the disregard), so even the correct MAGI of $82,301 (2.13x FPL) qualifies." +us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was over the limit without giving a threshold. MAGI is $82,301 (2.13x FPL), which is under NC's 211% FPL Medicaid limit for ages 6–18 (216% with the disregard). A 'No' only follows from the pre-2023 133% limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied a 133% FPL Medicaid limit and sent the child to CHIP instead. NC Health Choice was folded into Medicaid in 2023, so Medicaid itself now covers children ages 6–18 up to 211% FPL (216% with the disregard). MAGI of $82,301 (2.13x FPL) qualifies." +us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Compared income to the obsolete 133% FPL limit, and understated MAGI at $61,093 by subtracting ESI premiums. NC's child Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under it." +us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Said Child 1 was ineligible without showing any income test. MAGI is $82,301 (2.13x FPL), under NC's 211% FPL limit for ages 6–18 (216% with the disregard). A 'No' matches the outdated 133% limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income was over the NC limit without naming it. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for children ages 6–18 (216% with the disregard), so the model used an outdated lower threshold." +us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income was over the limit without naming a threshold. Correct MAGI is $82,301 (2.13x FPL), inside NC's 211% FPL OLDER_CHILD limit (216% with the disregard). A 'No' reflects the pre-2023 133% FPL limit." +us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said income was over NC's child limit. Since NC Health Choice merged into Medicaid, the limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) falls under it." +us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"Treated Child 1's employer-sponsored insurance as disqualifying for Medicaid. Only CHIP has an uninsured requirement; Medicaid does not. Child 1 qualifies in the OLDER_CHILD category with MAGI at 2.13x FPL, under NC's 211% limit (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used the correct 211% limit but tested gross wages of $85,209. It skipped the $2,778 traditional 401(k) and $130 traditional IRA deductions that bring MAGI to $82,301, and it left out the 5% disregard (216% effective limit). At 2.13x FPL, Child 1 is eligible." +us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings were over the Medicaid threshold without naming it. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for school-age children (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Ruled Child 1 ineligible based on income level alone, with no threshold. MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL Medicaid limit for ages 6–18 (216% with the disregard), so the OLDER_CHILD income test is met." +us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income was over the school-age child limit for five people. NC's Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used an estimated NC child threshold below the real one. The actual limit for ages 6–18 is 211% FPL (216% with the 5% disregard), and MAGI of $82,301 (2.13x FPL) falls under it." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeded NC's limit for an 11-year-old. That limit is 211% FPL (216% with the disregard) since NC Health Choice merged into Medicaid, and MAGI of $82,301 is 2.13x FPL, below it." +us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income was over NC's child Medicaid limit. MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL limit for ages 6–18 (216% with the disregard), so the model used a threshold that is too low." +us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said income exceeded NC's age-specific limit. For ages 6–18 that limit is 211% FPL (216% with the disregard), and household MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeded the child Medicaid limit. NC covers children ages 6–18 in Medicaid up to 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under that." +us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Got MAGI right at $82,301 but still found it over NC's child limit for five people, which means it used a threshold below 2.13x FPL. The OLDER_CHILD limit is 211% FPL (216% with the 5% disregard)." +us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"Said income was over the limit and treated employer-sponsored insurance as disqualifying. Medicaid has no uninsured requirement, and MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL child limit (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,Applied a 133% FPL limit to children ages 6–18. That limit ended when NC Health Choice merged into Medicaid in 2023; the current limit is 211% FPL (216% with the disregard). MAGI of 2.13x FPL qualifies. +us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Applied the obsolete 133% FPL (~$51,000) limit for ages 6–18, and understated MAGI at $74,555. NC's current limit is 211% FPL (216% with the disregard), so even the correct MAGI of $82,301 (2.13x FPL) qualifies." +us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Said NC Medicaid for ages 6–18 stops at about 138% FPL. Since 2023 it covers this group up to 211% FPL (216% with the disregard), and MAGI of $82,301 (2.13x FPL) is under that." +us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Overstated income at ~240% FPL (MAGI is $82,301, or 2.13x FPL) and compared it to the outdated 133% FPL limit. NC's current Medicaid limit for ages 6–18 is 211% FPL (216% with the disregard)." +us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Capped Medicaid for ages 6–18 at 133% FPL. That is outdated: NC Health Choice's former 211% FPL range is now part of Medicaid (216% with the disregard). MAGI of $82,301 (2.13x FPL) qualifies, and Medicaid, unlike CHIP, does not require the child to be uninsured." +us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for child1_medicaid_eligible. The expected answer is 1: MAGI of $82,301 (2.13x FPL) is under NC's 211% FPL Medicaid limit for ages 6–18." +us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (138% with the disregard) for ages 6–18 and overstated income at ~226% FPL. NC's current limit is 211% FPL (216% with the disregard), and MAGI of $82,301 is 2.13x FPL, which qualifies." +us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income was well above NC's Medicaid for Children limit. MAGI of $82,301 is 2.13x FPL, under the current 211% FPL limit for ages 6–18 (216% with the disregard), so the model used a limit that is too low." +us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"Put children ages 6–18 under a 133% FPL limit. NC Medicaid covers this group up to 211% FPL (216% with the disregard), so even the model's own ~198% FPL estimate qualifies, as does the correct 2.13x FPL." +us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Correctly found income under the child limit, then ruled Child 1 out because of employer-sponsored insurance. Medicaid has no uninsured requirement (only CHIP does), so the OLDER_CHILD income test alone makes Child 1 eligible." +us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Said income was too high without naming a threshold. MAGI of $82,301 is 2.13x FPL, below NC's 211% FPL Medicaid limit for ages 6–18 (216% with the 5% disregard)." +us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It compared MAGI of $82,301 to a 216% FPL limit and labeled that the NC CHIP limit. That band is the NC Medicaid limit for children aged 6-18. The child qualifies for Medicaid under the older-child category, and Medicaid eligibility excludes CHIP, so the answer is 0, not 1." +us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It found income of $61,093 below 211% FPL and treated that as NC Health Choice (CHIP) eligibility. At that income an 11-year-old in NC is Medicaid-eligible under the older-child category, since NC Health Choice is now part of Medicaid. A Medicaid-eligible child is not CHIP-eligible." +us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It used the federal minimum of 133% FPL as the Medicaid ceiling for school-age children and put the 133-211% band in CHIP. NC Medicaid's older-child category covers an 11-year-old across that band, so at MAGI of $74,555 the child is Medicaid-eligible. Medicaid eligibility excludes CHIP." +us,scenario_026,child2_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed household MAGI in a CHIP band above Medicaid and at or below 211% FPL, assuming NC Medicaid for an 11-year-old stops at a lower limit. The engine's NC Medicaid older-child category covers this child at this income. Because the child is Medicaid-eligible, the child is not CHIP-eligible." +us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_chip_eligible, so there is no substantive answer to grade. The correct answer is 0: the child is Medicaid-eligible under NC's older-child category, which excludes CHIP." +us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model got MAGI right at about $82,301 but divided by a 2025-level FPL (about $37,600), which gave 219% instead of the engine's 2.13× the 2026 FPL. It then compared that against a bare 211% limit without the 5-point MAGI disregard, even though NC's older-child Medicaid threshold covers 213% FPL." +us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model correctly put income at about 213% FPL but used the pre-2023 133% Medicaid limit for ages 6-18. NC merged Health Choice into Medicaid, raising the older-child limit to 211% FPL (about 216% with the disregard), so 213% qualifies." +us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model said the household has two earners, but Head has no wages; the spouse's wages are the only income. It also treated employer-sponsored coverage as a bar to Medicaid, which it is not. MAGI of about $82,300 is 2.13× FPL, within NC's older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model first used a 211% limit with an understated FPL of $36,580, then settled on a 133% FPL limit for ages 6-18. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and the household's 2.13× FPL MAGI falls under it." +us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model split coverage into 138% Medicaid and 211% CHIP, the structure NC dropped when Health Choice moved into Medicaid. It also used gross wages of about $85,000 instead of MAGI net of the 401(k) and IRA deductions, and an understated FPL of $36,580. Correct MAGI is 2.13× FPL, within NC's older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated household income at about 260% FPL. The only income is the spouse's wages net of the traditional 401(k) and IRA deductions (MAGI about $82,300), which is 2.13× the 2026 FPL for five and within NC's older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model put income at 219% FPL using a too-low FPL, when the engine's figure against the 2026 guideline is 2.13×. It then treated that as over the child limit, ignoring that NC's post-merger older-child Medicaid limit (211% plus the 5-point disregard) covers 213%." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model ruled Child 2 out because the child has employer-sponsored insurance, but Medicaid has no uninsured requirement and ESI does not disqualify a child. It never ran the income test: MAGI of 2.13× FPL is within NC's older-child limit." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model named the right roughly 216% FPL limit but turned it into about $67,000 using a far-too-low FPL. 216% of the 2026 FPL for five is about $83,500, above MAGI of about $82,300 (which excludes the 401(k) and IRA deductions, not adds overtime on top), so the child is eligible." +us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL Medicaid limit for school-age children. Since the Health Choice merger, NC Medicaid covers ages 6-18 up to 211% FPL plus the disregard, so the household's roughly $82,000 MAGI (2.13× FPL) qualifies." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,The model applied a flat 210% FPL limit without the 5-point MAGI disregard. NC's older-child Medicaid threshold in PolicyEngine covers the household's 2.13× FPL MAGI. +us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but tested it against a 133% FPL limit. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and $82,301 is 2.13× the 2026 FPL for five, which qualifies." +us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but said it exceeded NC's child limit. That MAGI is 2.13× the 2026 FPL for five, within the older-child Medicaid limit of 211% plus the 5-point disregard." +us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model compared gross wages of $85,209 against a bare 210% FPL limit. It skipped the 401(k) and IRA deductions that bring MAGI to about $82,300 (2.13× FPL) and left out the 5-point disregard that puts NC's older-child limit above that level." +us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit for ages 6-18, and it also subtracted ESI premiums nobody pays through payroll, getting a MAGI of $61,093 instead of about $82,300. NC's older-child Medicaid limit covers the correct MAGI of 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income was over the threshold without citing a limit. The correct MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the 5-point disregard. Its answer fits the outdated 133% Medicaid limit." +us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but applied a 133% Medicaid limit and sent the child to CHIP. NC ended separate Health Choice CHIP in 2023 and now covers ages 6-18 in Medicaid up to 211% FPL plus the disregard, which includes 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model subtracted insurance premiums that are not pre-tax, getting $61,093 instead of MAGI of about $82,300, and tested against the obsolete 133% FPL limit. NC's older-child Medicaid limit of 211% plus the disregard covers the household's 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. MAGI (spouse wages minus the 401(k) and IRA deductions, about $82,300) is 2.13× FPL, within NC's older-child Medicaid limit, so answering 'not eligible' means it applied a lower child threshold such as the old 133%." +us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeded the limit without citing one. MAGI of 2.13× FPL is under NC's post-Health Choice older-child limit of 211% FPL plus the 5-point disregard, so it used an outdated or too-low threshold." +us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model said income exceeded the limit without citing one. Household MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the disregard, so it applied a threshold that is too low." +us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model said income exceeded NC's child limits without citing a number. NC Medicaid covers ages 6-18 up to 211% FPL plus the 5-point disregard, and the household's MAGI is 2.13× FPL, so the child qualifies." +us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model got income right at about 213% FPL but used the pre-2023 split of 133% Medicaid and 210% Health Choice CHIP, and it also counted ESI against eligibility. NC now covers ages 6-18 in Medicaid up to 211% FPL plus the disregard (about 216%), and ESI does not disqualify." +us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"The model compared gross wages of $85,209 to a bare 211% FPL. It skipped the 401(k) and IRA deductions that bring MAGI to 2.13× FPL and left out the 5-point MAGI disregard that puts NC's older-child limit above that level." +us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model said earnings were above the Medicaid threshold without computing a ratio. Household MAGI is 2.13× FPL, under NC's older-child Medicaid limit (211% FPL plus the 5-point disregard)." +us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model denied eligibility based on a vague sense that income was too high, without an FPL comparison. MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model said income of about $85,209 was 'well above' NC's child limit. After the 401(k) and IRA deductions, MAGI is 2.13× FPL, and NC's older-child Medicaid limit (211% plus the 5-point disregard) covers that." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an 'estimated' NC child threshold set too low. The actual older-child Medicaid limit is 211% FPL plus the 5-point disregard, which covers the household's 2.13× FPL MAGI." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model said MAGI exceeded NC's limit for an 11-year-old. MAGI is 2.13× FPL, and NC's post-2023 older-child Medicaid limit of 211% plus the disregard includes that." +us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model said income was above the NC child limit without quantifying it. Household MAGI is 2.13× FPL, below NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard." +us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied a child income limit below 213% FPL. Since the 2023 Health Choice merger, NC Medicaid for ages 6-18 extends to 211% FPL plus the disregard, which covers the household's 2.13× FPL MAGI." +us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeded the child Medicaid limit. Household MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit." +us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but set NC's child limit too low. $82,301 is 2.13× the 2026 FPL for five, within the older-child Medicaid limit of 211% plus the 5-point disregard." +us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model said income was over the NC threshold and counted ESI against eligibility. ESI does not bar Medicaid, and MAGI of 2.13× FPL is within NC's older-child limit (211% plus the disregard)." +us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for children 6-18, which NC abandoned when Health Choice moved into Medicaid. The current limit of 211% plus the disregard covers the household's 2.13× FPL MAGI." +us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model understated MAGI at $74,555 instead of about $82,300 and tested it against the obsolete 133% FPL limit. NC's older-child Medicaid limit of 211% plus the 5-point disregard covers the household's actual 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model said MAGI was above NC's limit for ages 6-18. MAGI is 2.13× FPL, and NC's post-merger older-child Medicaid limit (211% FPL plus the disregard) includes it." +us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model overstated income at about 240% FPL instead of 2.13× and used the obsolete 133% child limit. NC's older-child Medicaid limit of 211% plus the disregard covers the correct ratio. +us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit for ages 6-18 and treated ESI as disqualifying. NC Medicaid covers these children up to 211% FPL plus the disregard regardless of ESI, and household MAGI is 2.13× FPL." +us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for child2_medicaid_eligible, so there was no usable answer to score." +us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133% FPL limit (138% with the disregard) for ages 6-18 and overstated income at about 226% FPL. The correct MAGI is 2.13× FPL, within NC's post-merger older-child limit of 211% plus the disregard." +us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model said income was 'well above' NC's children's limit without computing it. Household MAGI is 2.13× FPL, within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard." +us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed 11-year-olds in a 133% FPL Medicaid group, the limit NC replaced with 211% FPL plus the disregard when Health Choice moved into Medicaid. Its own estimate of about 198% FPL, and the engine's 2.13×, both fall under the actual limit." +us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL limit and an understated size-5 FPL of $33,700, and it counted ESI against eligibility. NC's older-child Medicaid limit is 211% FPL plus the disregard, and household MAGI is 2.13× the 2026 FPL, so the child qualifies." +us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model said income was too high without a threshold comparison. MAGI of about $82,300 is 2.13× the 2026 FPL for five, within NC's older-child Medicaid limit of 211% plus the 5-point disregard." +us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133%-211% FPL CHIP band for NC children aged 6-18, which assumes Medicaid for older children stops at 133% FPL. At its own 174% FPL figure, Child 3 falls under North Carolina's OLDER_CHILD Medicaid category. Medicaid eligibility excludes CHIP, so the answer is 0." +us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It checked only whether $82,301 MAGI was under a 216% FPL 'CHIP limit' and never tested Medicaid first. Child 3 is Medicaid-eligible under the NC OLDER_CHILD category, and that Medicaid eligibility bars CHIP no matter where income falls against the CHIP ceiling." +us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It concluded CHIP eligibility because $61,093 is under 211% FPL, treating that ceiling as a CHIP limit rather than checking Medicaid. At that income Child 3 qualifies for NC Medicaid under the OLDER_CHILD category, and Medicaid-eligible children are excluded from CHIP." +us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"It checked only age, Medicare status and a CHIP income range and skipped the Medicaid screen that comes before CHIP. Child 3 meets North Carolina's OLDER_CHILD Medicaid category, which makes the child ineligible for CHIP." +us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"It treated 133% FPL as the top of Medicaid for a 9-year-old and put its $74,555 MAGI in a 133%-211% CHIP band. North Carolina's OLDER_CHILD Medicaid category covers Child 3 at this income, and Medicaid eligibility excludes CHIP." +us,scenario_026,child3_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"It asserted that household MAGI was 'above Medicaid' and at or below 211% FPL, which puts the child in a separate CHIP tier. In fact Child 3 meets North Carolina's OLDER_CHILD Medicaid category, so the child is Medicaid-eligible and therefore not CHIP-eligible." +us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"It stated outright that Child 3 is 'not Medicaid-eligible', and it measured income against a 6-person FPL even though the household has 5 people. Child 3 is Medicaid-eligible under North Carolina's OLDER_CHILD category, which bars CHIP, so 216% NC Health Choice limit is irrelevant." +us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found MAGI of about $82,301, which is about 2x FPL. It then used a 138% FPL Medicaid limit for ages 6-18 and moved the 138-211% band into CHIP. That ignores the 2023 merger of NC Health Choice into Medicaid, which puts the older-child Medicaid limit at 211% FPL (216% with the disregard). At 2.13x FPL, Child 3 qualifies." +us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the income ratio right at about 213% FPL but compared it with a 133% child limit. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the 5% disregard), so 213% is eligible." +us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It named a 207% FPL limit but converted it to only $56,232 for a family of five, which is about 145% FPL. The correct figure is roughly $80,000, and the effective 216% older-child limit is about $83,500. MAGI of about $82,300 (2.13x FPL) is under that limit, so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used roughly the right 211% FPL limit but converted it to only about $66,000 for five people; the correct figure is about $81,600, or about $83,500 at the effective 216%. It also tested gross wages of $85,209 instead of MAGI of $82,301 after the traditional 401(k)/IRA deductions. MAGI is 2.13x FPL, which is under the limit." +us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used 133% FPL as the Medicaid limit for ages 6-18 and treated the 211% tier as CHIP, even though NC moved that band into Medicaid in 2023. It also tested gross wages of $85,209 instead of MAGI of $82,301 and left out the 5% disregard. MAGI is 2.13x FPL, under the 216% effective limit." +us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put household income at about 260% FPL; the correct figure is 2.13x FPL, based on $82,301 in MAGI after subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also assumed a lower child limit, but NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the disregard)." +us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It overstated income at about 219% FPL; MAGI is $82,301, or 2.13x the 2026 FPL for five, once pre-tax traditional 401(k)/IRA contributions are subtracted from wages. It then treated that as above the child limit. NC's older-child Medicaid limit is 216% FPL with the 5% disregard, so 2.13x is eligible." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It treated Child 3's employer-sponsored coverage as disqualifying for Medicaid, but ESI does not block Medicaid eligibility. It also used a stale $36,580 FPL and tested gross wages of $85,209 against 210%. The correct test is MAGI of $82,301 (2.13x the 2026 FPL) against NC's older-child limit of 211% FPL, 216% with the disregard." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a 138% FPL limit for ages 6-18, even though NC has covered that group in Medicaid up to 211% FPL (216% with the disregard) since the 2023 NC Health Choice merger. It also double-counted the $7,746 overtime premium, which is already inside the $85,209 wages, and got income above $92,000. Actual MAGI is $82,301." +us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its income figure of about $82,000 was correct, but it tested that against a 133% FPL child limit. NC's older-child Medicaid limit is 211% FPL (216% effective), and $82,301 is 2.13x FPL, so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a 210% FPL limit with no 5% MAGI disregard and never worked out the household's actual ratio. MAGI of $82,301 is 2.13x FPL, which is under the effective older-child limit of 216% (211% plus 5 points)." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but compared it with 133% FPL ($51,205). NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% with the disregard), and 2.13x FPL falls under that." +us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but applied too low a child limit. $82,301 is 2.13x the 2026 FPL for five, which is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)." +us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It tested gross wages of $85,209 against 210% FPL without subtracting $2,908 in traditional 401(k)/IRA contributions and without the 5% disregard. MAGI of $82,301 is 2.13x FPL, under the effective 216% older-child limit." +us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18; NC's older-child Medicaid category runs to 211% FPL (216% with the disregard). It also got MAGI of $61,093 by wrongly subtracting the head's $21,208 ESI premiums, though even the correct $82,301 (2.13x FPL) passes the real limit." +us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said income exceeded the limit without giving a figure. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% effective), so a 'no' implies a pre-2023 133%/138% limit." +us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but used a 133% FPL Medicaid limit and moved the child into CHIP. Since NC Health Choice merged into Medicaid in 2023, children 6-18 up to 211% FPL (216% effective) are in Medicaid, and 2.13x FPL qualifies." +us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It compared income with a 133% FPL child limit; NC's older-child Medicaid limit is 211% FPL (216% with the disregard). It also got income of $61,093 by subtracting the head's ESI premiums, though the correct MAGI of $82,301 (2.13x FPL) also qualifies." +us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard), so a 'no' implies the outdated 133%/138% child threshold." +us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's limit without giving numbers. MAGI of $82,301 is 2.13x FPL, which is below NC's older-child Medicaid limit of 211% FPL (216% effective) for ages 6-18." +us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave a generic over-income conclusion. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the disregard), so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's child limits without a figure. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% effective), and household MAGI is 2.13x FPL, so the child qualifies." +us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It correctly found about 213% FPL, but it capped Medicaid at 133%, labeled the 210% tier as CHIP, and treated 213% as over it. NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, or 216%, so 213% is eligible. Child 3's ESI coverage does not bar Medicaid." +us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It had the 211% FPL limit right but compared it with gross wages of $85,209 instead of MAGI of $82,301 and left out the 5% disregard. MAGI is 2.13x FPL, under the effective 216% older-child limit." +us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave a generic over-threshold conclusion. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% with the disregard), so the school-age child is eligible." +us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It ruled Child 3 ineligible from the wage level without testing against NC's actual child limit. Household MAGI of $82,301 is 2.13x FPL, which is inside NC's older-child Medicaid limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It called earnings of about $85,209 'well above' the NC child limit. After the traditional 401(k)/IRA deductions, MAGI is $82,301, or 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% with the disregard)." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used an estimated threshold that was too low. NC's older-child Medicaid category covers ages 6-18 up to 211% FPL (216% effective), and household MAGI of $82,301 is 2.13x FPL, so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It concluded that MAGI exceeded the limit for a 9-year-old, but MAGI of $82,301 is 2.13x FPL. That is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)." +us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was above the NC child limit without figures. MAGI is 2.13x FPL, which is below NC's post-2023 older-child Medicaid limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It applied too low an age-specific limit. For ages 6-18, NC's older-child Medicaid category reaches 211% FPL (216% with the disregard), and household MAGI is 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It said income exceeded the child Medicaid limit, but household MAGI of $82,301 is 2.13x FPL. That is under NC's older-child limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It correctly computed MAGI of $82,301 but misjudged the limit. $82,301 is 2.13x the 2026 FPL for five, under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)." +us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It treated Child 3's ESI as a bar to Medicaid, but employer coverage does not block Medicaid eligibility. It also applied too low an income limit: MAGI of $82,301 is 2.13x FPL, under NC's older-child limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It used 133% FPL as NC's Medicaid limit for ages 6-18. Since the 2023 NC Health Choice merger, the older-child category covers up to 211% FPL (216% with the disregard), and MAGI of 2.13x FPL qualifies." +us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18; NC's older-child Medicaid limit is 211% FPL (216% effective). It also understated MAGI at $74,555; the correct figure is $82,301, which is still eligible at 2.13x FPL." +us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It said MAGI exceeded NC's limit for ages 6-18, but that limit is 211% FPL (216% with the 5% disregard) and household MAGI is 2.13x FPL, so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It used a 133% base limit and overstated income at about 240% FPL. Actual MAGI is $82,301, or 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6-18 and counted employer coverage against eligibility. NC's older-child Medicaid category covers up to 211% FPL (216% with the disregard), MAGI is 2.13x FPL, and ESI does not bar Medicaid." +us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for child3_medicaid_eligible, so there is nothing to score. The correct answer is eligible: MAGI of 2.13x FPL is under NC's older-child Medicaid limit." +us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133% (138% with disregard) child limit and overstated MAGI at about 226% FPL. Actual MAGI is 2.13x FPL, and NC's older-child Medicaid limit is 211% FPL (216% with the disregard), so Child 3 is eligible." +us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income was 'well above' NC's child limits. MAGI of $82,301 is 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% effective)." +us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed Child 3 in NC's 6-18 group but capped that group at 133% FPL. After the 2023 CHIP merger, the limit is 211% FPL (216% with the disregard), so even its own estimate of about 198% FPL, and the actual 2.13x, qualify." +us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 133% FPL limit, an FPL for five that was too low ($33,700; the 2026 figure is about $38,680), and gross wages instead of MAGI. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% FPL (216% effective), and the children's ESI does not bar Medicaid." +us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called income too high without applying NC's actual limit. MAGI of $82,301 is 2.13x FPL, which is under NC's older-child Medicaid limit of 211% FPL (216% with the 5% disregard)." us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It capped the nonrefundable child tax credit at $1,500 for three children; the 2026 CTC is $2,200 per child ($6,600 total) and its nonrefundable portion is applied against the full pre-credit liability before any refundable ACTC is computed, so the $4,586.51 of tax is entirely absorbed. It also used a $29,500 MFJ standard deduction instead of $32,200 and never took the $7,746.26 deduction for the qualified FLSA overtime premium. Its own stated arithmetic ($6,313 less $1,500 = $4,813) does not equal the $5,813 it submitted." us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its derivation reached the correct conclusion — the nonrefundable CTC absorbs the entire tentative tax, 'Result near $0' — and it then submitted $4,107, a figure no step in its own work produces. It also omitted the $7,746.26 qualified-overtime deduction that brings taxable income down to $42,354.26 and tentative tax to $4,586.51." us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It invented a ~$605 ceiling on the nonrefundable CTC by treating the $1,700-per-child refundable ACTC as consumed first; the nonrefundable credit is applied against the whole liability first and the refundable portion is the residual, so $6,600 of credit for three children eliminates the entire tax. It also missed the $7,746.26 qualified-overtime deduction, which lowers taxable income from its stated $50,001 to $42,354.26." @@ -1890,40 +2148,45 @@ us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error, us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of the $85,209 gross wages that already include it, then applied pre-TCJA parameters — a $16,200 standard deduction, $26,000 of personal exemptions, a 15% bracket, and a $1,000-per-child CTC. For 2026 the premium is instead deducted, the standard deduction is $32,200 with no exemptions, and the $6,600 credit for three children offsets the full $4,586.51 tentative tax." us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation reached the grader. The correct derivation yields $42,354.26 of taxable income after the $32,200 standard deduction and $7,746.26 overtime deduction, $4,586.51 of tentative tax, and $0 after the $6,600 nonrefundable child tax credit." us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a fabricated $3,700 MFJ standard deduction in place of $32,200 and a fabricated $3,700 total child tax credit in place of $6,600, and it never subtracted the $2,778 traditional 401(k), the $130 IRA deduction, or the $7,746.26 overtime deduction. Its explanation states the credit fully offsets the tax while it submitted $5,931.15, contradicting itself." -us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly used the OBBBA $2,200-per-child credit ($6,600 total) but omitted the OBBBA qualified-overtime deduction for the $7,746 FLSA overtime premium, leaving taxable income near $50,100 instead of $42,355. It then declared the credit fully absorbed even though its own ~$5,500 tax estimate falls $1,100 short of the $6,600 credit; the correct pre-credit tax of $4,586.51 leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It never computed the refundable ACTC as the CTC remaining after pre-credit tax, instead asserting a flat 'approximately $1,000 per child' refundable amount. The refundable portion is the $6,600 CTC less the $4,586.51 of tax it offsets = $2,013.49, bounded only by the 3 x $1,700 = $5,100 cap and 15% of earnings over $2,500 ($12,406)." -us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $2,000-per-child credit ($6,000 rather than $6,600), the 2025 $31,500 standard deduction rather than 2026's $32,200, and omitted the $7,746 qualified-overtime deduction, pushing pre-credit tax to ~$5,600. Its own arithmetic still leaves $400 unabsorbed, and the correct chain ($82,301 AGI, $42,355 taxable income, $4,586.51 tax) leaves $2,013.49 of refundable CTC." -us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied a $2,000-per-child credit ($6,000) instead of the 2026 OBBBA $2,200 ($6,600) and omitted the qualified-overtime deduction for the $7,746 FLSA overtime premium, then asserted the credit was entirely nonrefundable. With the overtime deduction and $32,200 standard deduction, taxable income is $42,355 and tax $4,586.51, so $2,013.49 of the $6,600 credit is refundable." -us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the right method and the right $6,600 credit but omitted the OBBBA qualified-overtime deduction for the $7,746 overtime premium, so its ~$5,450 pre-credit tax overstates the true $4,586.51 by roughly $863. Deducting the overtime premium alongside the $32,200 standard deduction yields taxable income of $42,355 and a refundable remainder of $2,013.49." -us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Three compounding errors: a $2,000-per-child credit ($6,000 instead of $6,600), a ~$30,000 standard deduction instead of the 2026 MFJ $32,200, and no OBBBA qualified-overtime deduction for the $7,746 premium, producing $5,815 of tax against a true $4,586.51. Correcting all three gives $6,600 - $4,586.51 = $2,013.49 of refundable CTC." -us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 FLSA overtime premium on top of the $85,209 wages to reach ~$89,000 of AGI, though the prompt states gross wages already include overtime and OBBBA makes the qualified overtime premium a deduction, not extra income. Combined with a $2,000-per-child credit, it declared the CTC fully absorbed; deducting the $7,746 gives taxable income of $42,355, tax of $4,586.51, and $2,013.49 of refundable CTC." -us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $6,000 CTC instead of the 2026 OBBBA $6,600 and omitted the $7,746 qualified-overtime deduction, then asserted the credit was fully used against tax. Pre-credit tax on $42,355 of taxable income is $4,586.51, well below $6,600, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed AGI exactly right at $82,301 but stopped there: it omitted the OBBBA qualified-overtime deduction of $7,746 and used a $2,000-per-child credit, concluding the nonrefundable CTC swallowed the whole liability. With the $32,200 standard deduction plus the overtime deduction, tax is $4,586.51 and the $6,600 credit leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a $3,000 total CTC, i.e. the $1,000-per-child TCJA-sunset amount, which does not govern 2026 because OBBBA made the $2,200-per-child credit with a $1,700 refundable cap permanent. The correct $6,600 credit exceeds the $4,586.51 of pre-credit tax by $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It ran the pre-TCJA parameters it believed return in 2026 — $1,000 per child ($3,000 total), a $1,000 refundable cap, and a 15%-of-earnings-over-$3,000 formula — when 2026 law is $2,200 per child, $1,700 refundable, and a $2,500 earnings floor. Its $2,965.85 pre-credit tax also omits the $7,746 qualified-overtime deduction; the correct figures are $6,600 of credit against $4,586.51 of tax, leaving $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It asserted that no refundable credits apply 'at this income level,' applying an income disqualification the Child Tax Credit does not have — its phase-out begins at $400,000 of MAGI for joint filers, far above this household's $82,301. The $6,600 CTC exceeds the $4,586.51 of pre-credit tax, so $2,013.49 is refundable." -us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed AGI correctly at $82,301 but used a $3,000 total Child Tax Credit ($1,000 per child) rather than the 2026 OBBBA $6,600, so the credit appeared fully absorbed by tax. It also omitted the $7,746 qualified-overtime deduction; with both corrected, tax is $4,586.51 and $2,013.49 of the credit is refundable." -us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It claimed a $930 EITC computed at an invented AGI of $61,093; actual AGI is $82,301, above the 2026 MFJ three-child phase-out end near $68,700, so EITC is $0. Its $926 ACTC also understates the true remainder, which is $6,600 of credit less $4,586.51 of pre-credit tax = $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It submitted a bare zero with no derivation, a value consistent with treating the entire Child Tax Credit as nonrefundable and the household as income-disqualified. The $6,600 CTC (3 x $2,200) exceeds the $4,586.51 of pre-credit tax computed on $42,355 of taxable income, so $2,013.49 is refundable." -us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"It returned zero with a restatement of the variable name and no computation, a value consistent with assuming income at this level bars any refundable credit. The CTC has no phase-out below $400,000 for joint filers, and the $6,600 credit exceeds the $4,586.51 pre-credit tax by $2,013.49." -us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It asserted the nonrefundable CTC covers the whole tax liability without computing either side, omitting the OBBBA qualified-overtime deduction of $7,746 that drops taxable income to $42,355. Tax is then $4,586.51 against a $6,600 credit, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It declared the CTC fully absorbed against pre-credit tax without deducting the $7,746 qualified overtime premium or using the $2,200-per-child 2026 amount. With the $32,200 standard deduction plus the $7,746 overtime deduction, tax is $4,586.51 and the $6,600 credit leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It executed the ACTC formula correctly but on two wrong inputs: a $6,000 credit instead of the 2026 $6,600, and $5,676.12 of pre-credit tax that omits the OBBBA qualified-overtime deduction for the $7,746 premium. Correct taxable income is $42,355 and tax $4,586.51, so the refundable remainder is $2,013.49 rather than $323.88." -us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It reported the refundable ceiling — 3 x $1,700 = $5,100 — as the credit actually paid, skipping the step where the $6,600 CTC first offsets the $4,586.51 of pre-credit tax. Only the unused $2,013.49 is refundable; the $5,100 cap is not binding here." -us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It invoked a nonexistent age-6 condition on the refundable child credit and an EITC-style income cutoff for the CTC; the CTC qualifying-child test is age under 17 and its phase-out starts at $400,000 for joint filers. It never compared the $6,600 credit to the $4,586.51 of pre-credit tax, which leaves $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It assumed away refundable-credit eligibility 'given the household's income level' without computing the Child Tax Credit at all. The $6,600 CTC for three qualifying children exceeds the $4,586.51 pre-credit tax, and the $2,013.49 excess is refundable under the additional child tax credit." -us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a $1,000-per-child, wholly nonrefundable 'post-2025' child credit; 2026 law under OBBBA is $2,200 per child with up to $1,700 per child refundable. The $6,600 credit exceeds the $4,586.51 of pre-credit tax, producing $2,013.49 of refundable CTC." -us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It reached $2,058.76 by deducting the $378 of auto loan interest in addition to the $32,200 standard deduction and the $7,746 overtime deduction, giving taxable income of $41,977 and tax of $4,541.24. The OBBBA vehicle-loan interest deduction requires a qualifying new, US-final-assembly vehicle financed after 2024 — conditions unlisted here and therefore false — so taxable income is $42,355, tax is $4,586.51, and the refundable remainder is $2,013.49, exactly $45.27 below its answer." -us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It asserted no EITC or refundable CTC qualification 'at this income level' without computing either. EITC is indeed $0, but the CTC does not phase out until $400,000 for joint filers, and its $6,600 total exceeds the $4,586.51 of pre-credit tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 total CTC — the $1,000-per-child sunset amount — instead of the 2026 OBBBA $2,200 per child, so the credit appeared smaller than its computed tax. At $6,600 of credit against $4,586.51 of tax (after the $7,746 qualified-overtime deduction), $2,013.49 is refundable." -us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It had the structure right — $6,600 CTC, $0 EITC, refundable remainder under the earned-income formula — but computed $4,671 of pre-credit tax, about $84.51 above the true $4,586.51, implying taxable income roughly $704 too high. The correct chain is $82,301 AGI less the $32,200 standard deduction and the $7,746 qualified-overtime deduction, giving $42,355 taxable and $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium to wages to reach $90,047 of earned income instead of subtracting it as the OBBBA qualified-overtime deduction, and used a $3,000 total CTC. Deducting the premium gives $42,355 of taxable income and $4,586.51 of tax against a $6,600 credit, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the answer failed the output contract rather than the tax computation. The required value is the $6,600 CTC less the $4,586.51 of pre-credit tax, $2,013.49." -us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the right $6,600 credit and the right ACTC formula but computed $5,873 of pre-credit tax on roughly $53,075 of taxable income — it omitted the $7,746 OBBBA qualified-overtime deduction and understated the $32,200 standard deduction. Correct taxable income is $42,355, tax $4,586.51, and the refundable remainder $2,013.49." -us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It extended the EITC income disqualification to the additional child tax credit, stating both are zero without computing the CTC against tax. The CTC phase-out begins at $400,000 for joint filers, and the $6,600 credit exceeds the $4,586.51 pre-credit tax, leaving $2,013.49 refundable." -us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the correct $6,600 credit and ACTC limits but computed $5,586.60 of pre-credit tax, omitting the OBBBA qualified-overtime deduction for the $7,746 premium and using the 2025 $31,500 standard deduction rather than the 2026 $32,200. With both fixed, taxable income is $42,355, tax is $4,586.51, and the refundable remainder is $2,013.49." -us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $6,000 credit ($2,000 per child) instead of the 2026 $6,600 and a $5,479 pre-credit tax that omits the $7,746 qualified-overtime deduction. Correct taxable income is $42,355 and tax $4,586.51, so the additional child tax credit is $2,013.49, not $521." -us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It stated a $4,300 refundable ACTC with no derivation; the figure matches neither the $5,100 refundable cap, the $12,406 earned-income limit, nor the credit remaining after tax. The additional child tax credit here is the $6,600 CTC less the $4,586.51 of tax it offsets = $2,013.49." +us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It estimated pre-credit tax at about $5,500, which is less than the $6,600 CTC, but still said the full credit was used nonrefundably and paid $0 ACTC. Even by its own numbers, about $1,100 was unused and refundable. It also left out the $7,746 qualified overtime deduction, which lowers tax to $4,586.51 and leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It made up a refundable amount of about $1,000 per child and never computed tax liability. ACTC is the CTC left over after it offsets tax: $6,600 minus $4,586.51 of pre-credit tax, which is $2,013.49, within the $1,700-per-child cap." +us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) and the 2025 $31,500 standard deduction. It then called a $6,000 credit 'fully absorbed' by about $5,600 of tax, even though $400 was left over. With the $2,200-per-child CTC, the $32,200 deduction and the $7,746 overtime deduction, tax is $4,586.51 and $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It said tax was high enough to absorb a $6,000 CTC without computing it, and it used $2,000 per child instead of the 2026 $2,200. After the $32,200 standard deduction and the $7,746 qualified overtime deduction, pre-credit tax is only $4,586.51, so $2,013.49 of the $6,600 CTC is refundable." +us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It correctly used the $6,600 CTC and the credit ordering, but it put pre-credit tax at about $5,450 because it left out the OBBBA deduction for the $7,746 FLSA overtime premium. With that deduction, taxable income is about $42,355, tax is $4,586.51, and refundable ACTC is $2,013.49 rather than $1,150." +us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used pre-OBBBA and 2025 values: a $2,000-per-child CTC ($6,000), a roughly $30,000 standard deduction and a $23,850 10% bracket. It also left out the $7,746 qualified overtime deduction, which gave tax of $5,815 and ACTC of $185. The 2026 values ($2,200 per child, $32,200 deduction, $24,800 bracket) plus the overtime deduction give tax of $4,586.51 and ACTC of $2,013.49." +us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime on top of $85,209 in wages that already include it, so it put AGI at about $89,000. It should instead have taken the overtime premium as the OBBBA qualified overtime deduction. It also used a $6,000 CTC and asserted that tax exceeded it, when the correct pre-credit tax of $4,586.51 leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 pre-credit tax is the correct computation with the $378 of auto-loan interest also deducted: taxable income of $41,977 instead of $42,355. The OBBBA passenger-vehicle loan interest deduction requires a new, US-assembled vehicle, and no such fact is listed, so the deduction is unavailable. Tax is $4,586.51 and ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC and assumed it was fully used against tax. The 2026 CTC is $6,600, and pre-credit tax after the $32,200 standard deduction and $7,746 overtime deduction is only $4,586.51, which leaves $2,013.49 of refundable ACTC." +us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It asserted that the nonrefundable CTC fully offsets tax with nothing left over, which reverses the actual relationship. With AGI of $82,301, the $32,200 standard deduction and the $7,746 qualified overtime deduction, tax is $4,586.51, below the $6,600 CTC, so $2,013.49 is refundable ACTC." +us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, the TCJA-sunset amount), but OBBBA sets the 2026 CTC at $2,200 per child ($6,600). That total exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable." +us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its $5,516.12 tax is the correct 2026 computation without the $7,746 qualified overtime deduction, and it used a $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600). The overtime deduction lowers tax to $4,586.51, and the $6,600 credit leaves $2,013.49 refundable instead of $483.88." +us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used a $1,000-per-child CTC with the pre-TCJA refundability formula. OBBBA made the CTC $2,200 per child, refundable up to $1,700, and kept the TCJA rates and standard deduction, so the unused $2,013.49 above the $4,586.51 tax is refundable." +us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no computation and simply said no refundable credits apply. The $6,600 CTC exceeds the $4,586.51 pre-credit tax (after the $32,200 standard deduction and $7,746 overtime deduction), so $2,013.49 is refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed AGI correctly at $82,301 but used a $3,000 CTC (the $1,000-per-child sunset amount) instead of the OBBBA $2,200 per child. The $6,600 CTC exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $61,093 AGI is the true $82,301 minus the Head's $21,208 of employer-sponsored insurance premiums, which it wrongly subtracted. That produced a spurious $930 EITC. With the actual $82,301 AGI, EITC is zero, and ACTC is the $2,013.49 of CTC left after $4,586.51 of tax." +us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation for its $0 answer. The correct pre-credit tax is $4,586.51 (AGI of $82,301, minus the $32,200 standard deduction and $7,746 overtime deduction), which leaves $2,013.49 of the $6,600 CTC as refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its explanation only restates the variable and gives no computation. Its $0 answer misses that the $6,600 CTC exceeds the $4,586.51 pre-credit tax, which leaves $2,013.49 of refundable ACTC." +us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It correctly noted that the nonrefundable CTC covers the tax, but it did not refund the unused remainder as ACTC. Of the $6,600 credit, $4,586.51 offsets tax and the remaining $2,013.49 is refundable within the $1,700-per-child cap." +us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,other,False,"It said the nonrefundable CTC was fully absorbed by pre-credit tax, but the $4,586.51 tax is smaller than the $6,600 CTC. The unused $2,013.49 is refundable as ACTC." +us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC ($2,000 per child) and a $5,676.12 pre-credit tax that leaves out the $7,746 qualified overtime deduction. With the $6,600 CTC and tax of $4,586.51, ACTC is $2,013.49 instead of $323.88." +us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,other,False,"It treated the $1,700-per-child refundable cap ($5,100) as the ACTC amount. The cap only limits the refundable portion. ACTC is the CTC left after offsetting tax, which is $6,600 minus $4,586.51, or $2,013.49." +us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its reasoning about the children being over age 6 is irrelevant to the CTC, and it never compared the credit with tax liability. The $6,600 CTC exceeds the $4,586.51 pre-credit tax, which leaves $2,013.49 of refundable ACTC." +us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed without computation that no refundable credits apply at this income. The three children qualify for a $6,600 CTC, and the pre-credit tax is only $4,586.51, so $2,013.49 is refundable ACTC." +us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the TCJA-sunset rules: a $1,000-per-child CTC treated as nonrefundable. OBBBA sets the 2026 CTC at $2,200 per child with up to $1,700 refundable, so the $2,013.49 of credit above the $4,586.51 tax is refunded." +us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $2,058.76 means pre-credit tax of $4,541.24, which is the correct computation with the $378 of auto-loan interest also deducted. The OBBBA vehicle-loan interest deduction requires a new, US-assembled vehicle, and no such fact is listed, so tax is $4,586.51 and ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It put regular tax at about $5,532, which leaves out the OBBBA deduction for the $7,746 FLSA overtime premium. With that deduction, tax is $4,586.51 and the unused CTC is $2,013.49, not $1,068." +us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,541.24 tax is the correct computation with the $378 of auto-loan interest also deducted (taxable income $41,977 instead of $42,355). The OBBBA vehicle-loan interest deduction requires a new, US-assembled vehicle that the facts do not establish. Tax is $4,586.51 and ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,other,False,"It declared no refundable CTC at this income without computing anything. The CTC has no phase-out near $82,301 AGI, and the $6,600 credit exceeds the $4,586.51 pre-credit tax, so $2,013.49 is refundable." +us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 CTC (the $1,000-per-child sunset amount) instead of the OBBBA $2,200 per child. A $6,600 CTC exceeds the $4,586.51 pre-credit tax by $2,013.49, which is refundable ACTC." +us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC and ordering, but overstated pre-credit tax at $4,671. Wages minus the $2,778 401(k), the $130 IRA, the $32,200 standard deduction and the $7,746 overtime deduction give taxable income of about $42,355 and tax of $4,586.51, so ACTC is $2,013.49 rather than $1,929." +us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a $200 Saver's Credit before the CTC, but AGI of $82,301 is above the 2026 MFJ Saver's Credit limit of $80,500, so that credit is zero. All $4,586.51 of pre-credit tax is offset by the CTC, leaving $2,013.49 refundable, not $2,205." +us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,000 CTC instead of the OBBBA $6,600, and it added overtime on top of wages that already include it, which put tax at $5,937. It should have deducted the $7,746 overtime premium, giving tax of $4,586.51 and leaving $2,013.49 of refundable ACTC." +us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits. The required output of $2,013.49 in refundable ACTC was never submitted." +us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly used the $6,600 CTC and the earned-income test, but put pre-credit tax at $5,873, leaving out the OBBBA deduction for the $7,746 overtime premium. With that deduction and the $32,200 standard deduction, tax is $4,586.51 and ACTC is $2,013.49." +us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,other,False,"It said additional CTC is $0 without computing tax liability. The $6,600 CTC exceeds the $4,586.51 pre-credit tax, so $2,013.49 is refundable ACTC." +us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,other,False,"Its $5,586.60 pre-credit tax is exactly $1,000 more than the $4,586.60 that taxable income of $42,355 produces at the 10%/12% rates, which is an arithmetic slip in the bracket computation. The correct tax leaves about $2,013.49 of refundable ACTC rather than $1,013.40." +us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) and a $5,479 tax that leaves out the $7,746 qualified overtime deduction. The 2026 $6,600 CTC less $4,586.51 of tax leaves $2,013.49 refundable." +us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It gave an unexplained $4,300 ACTC based only on the number of children and never subtracted the part of the CTC used against tax. ACTC is the $6,600 CTC minus the $4,586.51 of pre-credit tax, or $2,013.49." us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price." us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -1941,36 +2204,41 @@ us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,t us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Applied the $500-per-child tier of North Carolina's AGI-graduated child deduction — the amount reserved for joint filers with AGI between $120,000 and $140,000 — instead of the $1,500-per-child tier that governs AGI of $82,301, understating the deduction by $3,000. It then submitted $2,882 while its own worksheet produced $2,206.51, so the reported figure tracks neither its arithmetic nor NC's schedule." -us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started from federal taxable income after the federal $29,200 standard deduction; North Carolina starts from federal AGI and substitutes its own $25,500 MFJ standard deduction plus the $4,500 child deduction. It compounded this with the superseded 4.75% rate rather than 2026's 3.99% flat rate and an invented $2,716 NC dependent credit, then submitted $5,238 in place of the $2,522 its own steps produced." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Used the $2,500-per-child deduction tier (joint AGI $40,000–$60,000) instead of the $1,500 tier that applies at $82,301 of AGI, and left the $130 deductible traditional IRA contribution in income. Its own steps yielded $2,101 at 4.25% and $1,972 at 3.99%, yet it submitted $3,041, a figure no line of its derivation produces." -us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Omitted North Carolina's child deduction entirely — three qualifying children at the $1,500 tier for joint AGI between $80,000 and $100,000 is $4,500 — and left the $130 traditional IRA contribution in AGI. It therefore taxed $56,931 rather than $52,300.52 at the correct 3.99% rate." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Set the NC joint standard deduction at $21,500 instead of $25,500 and claimed the child deduction at the top $3,000-per-child tier ($9,000) instead of the $1,500 tier applicable at $82,301 of AGI. The two errors partly cancel, leaving $51,801 of taxable income instead of $52,300.52 at the otherwise-correct 3.99% rate." -us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added the $7,746 FLSA overtime premium on top of the $85,209 gross wages, which the prompt states already include overtime pay, inflating AGI to $90,047. It then used a 4.25% rate rather than 2026's 3.99%, skipped the $4,500 child deduction, and marked its own $2,743 result up to $3,350 for an unstated retirement-contribution addback that North Carolina does not impose." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Built AGI of $90,047 by double-counting the $7,746 FLSA overtime premium already contained in the $85,209 gross wages and never subtracting the $2,778 traditional 401(k) or $130 traditional IRA contributions. It also omitted the $4,500 child deduction, so its correct 3.99% rate landed on $64,547 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Reached the correct AGI of $82,301 and the correct 3.99% rate but omitted North Carolina's child deduction, which grants $1,500 per qualifying child at joint AGI between $80,000 and $100,000, or $4,500 for three children. That single omission raised taxable income from $52,300.52 to $56,801 and the tax by $179.57." -us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Omitted the $4,500 NC child deduction (three children at the $1,500 tier for joint AGI $80,000–$100,000) and rounded the 2026 flat rate to 4.0% instead of the statutory 3.99%. It taxed $56,801 rather than $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 employer-sponsored insurance premium from AGI, but employer-plan premiums are already excluded from the reported wage figure and produce no second AGI reduction, dropping AGI to $61,093 instead of $82,300.52. It then used the $2,000-per-child deduction tier that its own understated AGI implied rather than the $1,500 tier the true $82,301 AGI requires." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation, and $3,538 equals the entire $82,300.52 of AGI taxed at roughly 4.3%, so it applied a rate above North Carolina's 3.99% flat rate to income that received neither the $25,500 joint standard deduction nor the $4,500 child deduction. The correct chain subtracts both to reach $52,300.52 of taxable income and $2,086.79 of tax." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $61,093 by subtracting the $21,208 employer-sponsored insurance premium, which is already excluded from the wage figure and generates no further AGI reduction; NC AGI is $82,300.52. It then took the $2,000-per-child deduction tier that its understated AGI implied instead of the $1,500 tier for joint AGI between $80,000 and $100,000." -us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Submitted a bare figure with no derivation; $3,538.77 is the full $82,300.52 AGI taxed at about 4.3%, meaning it used a rate above the 3.99% statutory flat rate and allowed neither the $25,500 joint standard deduction nor the $4,500 child deduction. Both subtractions are mandatory, giving $52,300.52 of taxable income." -us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied a 3.74% rate, which is not North Carolina's 2026 flat rate of 3.99%, and compounded it with a $130 spousal traditional IRA deduction for the Head that the household facts never list (unlisted numeric inputs are 0) and with the complete omission of the $4,500 child deduction. Taxable income is $52,300.52, not the $56,671 it taxed." -us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Claimed a $4,000-per-child NC child deduction, an amount that appears nowhere in the AGI-graduated schedule whose maximum is $3,000 and whose joint $80,000–$100,000 tier is $1,500; the correct total is $4,500, not $12,000. Its AGI and 3.99% rate were right, so the $7,500 deduction overstatement is the whole error." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted the household's deductions eliminate NC taxable income, but the only subtractions available are the $25,500 joint standard deduction and the $4,500 child deduction, totaling $30,000 against $82,300.52 of AGI and leaving $52,300.52 taxable. North Carolina grants no personal exemptions and no nonrefundable credit that offsets the resulting $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Returned a boilerplate explanation with no computation; $4,875 is 2.3 times the correct liability and corresponds to roughly 5.7% applied to undeducted gross wages. The correct chain subtracts the $2,778 traditional 401(k) and $130 traditional IRA contributions, then the $25,500 standard and $4,500 child deductions, and taxes $52,300.52 at 3.99%." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,957.29 is exactly $49,055 × 3.99%, the base produced by stripping the $7,746 FLSA overtime premium out of AGI to $74,555 and then allowing only the $25,500 standard deduction. That premium sits inside the $85,209 gross wages and the federal overtime deduction is below-the-line, so NC AGI remains $82,300.52, and the $4,500 child deduction is missing as well." -us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $2,027 is $50,801 × 3.99%, which is the $2,000-per-child deduction tier for joint AGI between $60,000 and $80,000, or $6,000 total. At AGI of $82,300.52 the schedule steps down to $1,500 per child, making the correct child deduction $4,500 and taxable income $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Treated roughly $85k of gross wages as the starting point, so it never subtracted the $2,778 traditional 401(k) and $130 traditional IRA contributions and never applied the $4,500 child deduction. Its $2,780 corresponds to about $69,700 of taxable income at 3.99%, a third above the correct $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Reached the correct $82,301 AGI and the correct 3.99% rate but never applied the NC child deduction of $1,500 per qualifying child for joint filers with AGI between $80,000 and $100,000, worth $4,500 for three children. It taxed $56,801 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Cut AGI to $74,555 by removing the $7,746 FLSA overtime premium, which is already contained in the $85,209 gross wages and, as a below-the-line federal overtime deduction, never reduces the federal AGI North Carolina begins from. It also omitted the $4,500 child deduction, and the two errors partly offset to $49,055 of taxable income instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime premium to wages that already include overtime, reaching AGI of $90,047, applied 2025's 4.25% rate instead of 2026's 3.99%, and omitted the $4,500 child deduction. Its $64,547 base exceeds the correct $52,300.52 by $12,246." -us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Removed the $7,746 FLSA overtime premium from AGI, though it is inside the $85,209 gross wages and the federal overtime deduction is below-the-line and leaves federal AGI at $82,300.52. It then took the $2,500-per-child deduction tier (joint AGI $40,000–$60,000) instead of the $1,500 tier, taxing $41,555 rather than $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no substantive North Carolina computation was submitted. The required chain is federal AGI of $82,300.52 less the $25,500 joint standard deduction and the $4,500 child deduction, taxed at 3.99% for $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Started from the full $85,209 of gross wages without subtracting the $2,778 traditional 401(k) and $130 traditional IRA contributions that reduce federal AGI to $82,300.52, and omitted the $4,500 child deduction. Its correct 3.99% rate therefore hit $59,709 instead of $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Zeroed the liability on 'personal exemptions for a family of five'; North Carolina repealed personal exemptions, and its only subtractions here are the $25,500 joint standard deduction and the $4,500 child deduction. Those leave $52,300.52 taxable at 3.99%, or $2,086.79." -us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Derived its $49,055 base by subtracting the $7,746 FLSA overtime premium from AGI, but that premium is inside the $85,209 gross wages and the federal overtime deduction is below-the-line, so North Carolina's starting AGI stays $82,300.52. It also skipped the $4,500 child deduction for three children at the $1,500 joint tier." -us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime premium to wages that already include it, left the $2,778 401(k) and $130 IRA contributions in income, used the superseded 4.5% rate rather than 2026's 3.99%, and omitted the $4,500 child deduction. Those four errors produced a $67,455 base against the correct $52,300.52." -us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Reported NC taxable income of $81,509, a figure that applies neither the $25,500 joint standard deduction nor the $4,500 child deduction, then taxed it at 4.5% instead of 2026's 3.99% and netted a $19.41 credit with no counterpart in North Carolina law. The correct base is $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used $500 per child ($1,500 total) for the NC child deduction. At AGI $80k–$100k, joint filers get $1,500 per child, or $4,500. It then submitted $2,882, which does not match its own calculation of $2,206.51." +us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from federal taxable income using a $29,200 federal standard deduction and applied a 4.75% rate. It then invented a dependent credit and submitted $5,238, which its own work does not support. NC starts from federal AGI, subtracts a $25,500 standard deduction and a $4,500 child deduction, and taxes the rest at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model left the $130 traditional IRA deduction out of AGI and used $2,500 per child for the child deduction instead of $1,500 per child at the $80k–$100k tier. It also switched between 4.25% and 3.99%, and its final $3,041 matches none of its own calculations." +us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model left out the NC child deduction ($1,500 × 3 = $4,500) and did not subtract the $130 traditional IRA from AGI. As a result, it taxed $56,931 instead of $52,300.52 at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an outdated $21,500 joint standard deduction instead of $25,500. It also applied the $3,000-per-child amount from the lowest AGI tier, but at $82,301 AGI the deduction is $1,500 per child. The two errors nearly cancel, which is why its $2,066.86 lands close to the right answer by accident." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the $7,746 FLSA overtime premium on top of gross wages, even though wages already include it. It also left out the $4,500 NC child deduction and used a 4.25% rate instead of 3.99%. Finally, it applied an invented retirement add-back to reach $3,350." +us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used $1,000 per child, which belongs to the $100k–$120k tier. At $82,301 AGI, joint filers fall in the $80k–$100k tier, which gives $1,500 per child ($4,500 total)." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model added the $7,746 overtime premium to wages, which already include it, giving an AGI of $90,047 instead of $82,300.52. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the NC child deduction of $1,500 per child ($4,500). It therefore taxed $56,801 instead of $52,300.52 at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model left out the $4,500 NC child deduction for three children at the $80k–$100k AGI tier. It also applied a 4.0% rate instead of the 2026 rate of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the $4,500 NC child deduction. It therefore taxed $56,801 instead of $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the head's $21,208 employer-sponsored insurance premiums from AGI, even though the spouse's gross wages are not reduced by that input, and got an AGI of $61,093 instead of $82,300.52. Because AGI was too low, it then chose the $2,000-per-child tier instead of $1,500." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct method is $82,300.52 AGI minus $30,000 in NC deductions, taxed at 3.99%, which gives $2,086.79. The model's $3,538 corresponds to about $88,700 taxed at 3.99%, which means it effectively applied neither the $25,500 standard deduction nor the $4,500 child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 employer-sponsored insurance premiums from AGI and got $61,093 instead of $82,300.52. It then applied the $2,000-per-child tier instead of the $1,500-per-child tier that applies at the correct AGI." +us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct method is $82,300.52 AGI minus the $25,500 standard deduction and $4,500 child deduction, taxed at 3.99%, which gives $2,086.79. The model's $3,538.77 corresponds to about $88,700 taxed at 3.99%, which means it effectively applied none of the NC deductions." +us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a 3.74% rate instead of the 2026 rate of 3.99% and left out the $4,500 NC child deduction. It also subtracted an extra $130 spousal IRA deduction for the head, who made no IRA contribution." +us,scenario_026,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used an invented $4,000-per-child deduction ($12,000 total). At $82,301 joint AGI, the NC child deduction is $1,500 per child ($4,500)." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed deductions wipe out NC tax. The $25,500 standard deduction plus the $4,500 child deduction total only $30,000 against $82,300.52 of AGI, which leaves $52,300.52 taxable at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $4,875 is more than double the correct $2,086.79 ($52,300.52 × 3.99%), which means it taxed close to gross wages at a rate well above 3.99% and never applied the $30,000 of NC deductions." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,957.29 equals $49,055 × 3.99%. That means it subtracted the $7,746 overtime premium from AGI, even though overtime remains in federal and NC AGI, and it left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $2,027 equals ($82,301 − $25,500 − $6,000) × 3.99%. That means it used the $2,000-per-child tier ($60k–$80k) instead of the $1,500-per-child tier that applies at $80k–$100k joint AGI." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model left out the $4,500 NC child deduction and did not subtract the $130 traditional IRA from AGI. It therefore taxed $56,931 instead of $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used $1,000 per child ($3,000), which is the $100k–$120k tier. At $82,301 joint AGI, the correct amount is $1,500 per child ($4,500)." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model started from about $85k of gross wages, without subtracting the 401(k) and IRA contributions, and left out the $4,500 NC child deduction. Its $2,780 also implies a rate above the 2026 rate of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted only the $25,500 standard deduction and left out the NC child deduction of $1,500 per child ($4,500)." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium to get an AGI of $74,555, but overtime stays in NC AGI. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model removed the $7,746 overtime premium from AGI, getting $74,555 instead of $82,300.52. Because of that lower AGI, it applied the $2,000-per-child tier instead of the correct $1,500 per child." +us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added the overtime premium on top of wages, which already include it, and got an AGI of $90,047. It also left out the $4,500 child deduction and applied 4.25% instead of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium from AGI and got $74,555. It then applied $2,500 per child, which belongs to the $40k–$60k tier. At the correct AGI of $82,300.52, the deduction is $1,500 per child." +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used gross wages of $85,209 as AGI without subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also left out the $4,500 NC child deduction." +us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied personal exemptions for a family of five, but North Carolina has no personal exemptions. The only deductions are the $25,500 standard deduction and the $4,500 child deduction, which leave $52,300.52 taxable at 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model subtracted the $7,746 overtime premium from AGI and got $74,555, but overtime stays in NC AGI. It also left out the $4,500 NC child deduction, so it taxed $49,055 instead of $52,300.52." +us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model added the overtime premium on top of wages, which already include it, and did not subtract the 401(k) and IRA contributions. It also left out the $4,500 child deduction and used a 4.5% rate instead of the 2026 rate of 3.99%." +us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied a federal standard deduction instead of NC's $25,500 standard deduction and $4,500 child deduction, and it used a 4.5% rate instead of 3.99%. It also subtracted $19.41 of nonrefundable credits that do not exist." us,scenario_026,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output, violating the required structured-output contract." us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly derived zero taxable income, zero gross tax, and no nonrefundable credits, but submitted -3195 instead of the zero produced by its calculation. Federal income tax before refundable credits does not become negative here, and its explanation explicitly ends with a value inconsistent with its own derivation." @@ -1988,88 +2256,101 @@ us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly identified Connecticut's BBCE (200% FPG gross screen, no asset test) and then re-imposed the 100%-FPG net income test as a disqualifier, which categorical eligibility through TANF non-cash assistance waives. Having declared the household ineligible, it never reached the benefit step, where the $546 maximum allotment minus the $635.10 expected contribution is negative and the one-to-two-person minimum allotment of $23.84 rising to $24.37 per month applies." -us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It computed net income of about $2,308 and denied the household for exceeding the 100%-FPG net limit of $1,763, but Connecticut's broad-based categorical eligibility makes that test non-binding for this unit. The correct continuation is the benefit formula, whose negative result triggers the two-person minimum allotment of $23.84 per month through September 2026 and $24.37 thereafter, totaling $287.68." -us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied the standard 130%-FPG gross test plus a $2,250/$3,500 resource test and asserted that Connecticut still enforces asset limits, when CT's BBCE eliminates the resource test entirely and raises the gross screen to 200% FPG ($3,525/month), which $2,536 clears. Its own arithmetic contradicts its conclusion — it wrote that $2,513 exceeds a $2,900 limit — and it never reached the minimum allotment an eligible two-person household receives when the formula returns a negative benefit." -us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"It denied the household on resources alone, testing $48,000 against a roughly $4,500 elderly/disabled limit; Connecticut's categorical eligibility through TANF non-cash assistance removes the SNAP resource test, so the bank balance is irrelevant to eligibility. Stopping at ineligibility, it skipped the benefit step where the maximum allotment minus the $635.10 expected contribution is negative and the minimum allotment of $23.84/$24.37 per month is paid." -us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"It recognized the elderly/disabled household rules but kept an asset test, disqualifying the unit for $48,000 against a claimed $4,500 resource limit that Connecticut's broad-based categorical eligibility abolishes. With eligibility established through TANF non-cash assistance, the negative formula result yields the two-person minimum allotment of $23.84 per month, $24.37 from October 2026, not zero." -us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It compared $2,536 of gross monthly income directly against the net income limit without subtracting the standard deduction or the elderly/disabled excess medical deduction, and treated that comparison as terminal; categorical eligibility in Connecticut waives the net income test for this household. The remaining step produces a negative benefit and therefore the minimum allotment of $23.84 rising to $24.37 per month, $287.68 for the year." -us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It worked the gross test, standard deduction, and excess medical deduction correctly and then voided the result with a $4,500 asset limit against $48,000, which contradicts its own premise that Connecticut uses BBCE at 200% FPG — broad-based categorical eligibility confers eligibility precisely by waiving the resource test. Eligibility intact, the negative formula result pays the one-to-two-person minimum allotment of $23.84/$24.37 per month." -us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"It ruled the household categorically ineligible for holding $48,000 against a $4,250 elderly/disabled resource limit, but Connecticut's TANF non-cash categorical eligibility strikes the resource test, so the bank balance does not disqualify. It therefore never applied the minimum allotment that an eligible two-person household receives when the maximum allotment minus 30% of net income is negative." -us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It denied eligibility on a $4,250 asset limit that Connecticut's broad-based categorical eligibility does not impose, treating $48,000 in bank assets as countable resources when the state's TANF non-cash conferral removes the resource test. The eligible household's negative computed benefit converts to the minimum allotment of $23.84 per month, $24.37 from October 2026, for $287.68 annually." -us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It accepted the 200%-FPG gross pass and then disqualified the household on a net income test ($2,300 against a $1,703 limit) that categorical eligibility waives, also using a two-person poverty guideline below the actual $1,762.50. Continuing past that test gives a negative formula result and the two-person minimum allotment of $23.84/$24.37 per month rather than $0." -us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It stopped at the net income test, declaring $2,536 minus deductions still above the two-person net limit, a test that does not screen out a categorically eligible Connecticut household. The benefit step it skipped produces the maximum allotment minus a $635.10 expected contribution — negative — which is paid at the minimum allotment of $23.84 per month rising to $24.37." -us,scenario_027,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It got eligibility exactly right — a 200%-FPG BBCE pass — and correctly found that the maximum allotment minus 30% of net income is zero or less, then reported $0 because it omitted the minimum allotment rule. Eligible one- and two-person households receive that floor: $23.84 per month under the FY2026 parameters and $24.37 from October 2026, totaling $287.68." -us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It disqualified the household because $48,000 exceeds ""the SNAP eligibility resource limit for most households,"" ignoring that Connecticut's broad-based categorical eligibility eliminates the resource test for this unit. With the asset test gone and the formula returning a negative benefit, the two-person minimum allotment of $23.84/$24.37 per month applies." -us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It affirmed categorical eligibility and correctly computed that 30% of net income ($635.10) exceeds the two-person maximum allotment ($546), then reported $0 instead of the minimum allotment that a negative formula result triggers for one- and two-person households. That floor is $23.84 per month through September 2026 and $24.37 afterward, summing to $287.68." -us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reasoned the case correctly all the way to the minimum allotment but priced that floor at the stale $23 per month instead of the values in force for the 2026 tax year — $23.84 through September 2026 and $24.37 from October, matching the $546 and $558.24 maximum allotments. Nine months at $23.84 plus three at $24.37 is $287.68, not $276." -us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no derivation, asserting only that ""income and asset levels"" produce zero; its answer is consistent with applying both the resource test and the net income test, each of which Connecticut's TANF non-cash categorical eligibility waives. The correct derivation keeps the household eligible, yields a negative formula benefit, and pays the two-person minimum allotment of $23.84/$24.37 per month for $287.68." -us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It identified BBCE eligibility, the zero formula result, and the minimum allotment rule, then used the outdated $23 monthly floor rather than the $23.84 in effect through September 2026 and the $24.37 applying from October 2026. The correctly indexed sum is $287.68, not $276." -us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,It applied the right rule — a categorically eligible two-person household draws the minimum allotment — with a stale $23 per month parameter instead of the $23.84 and $24.37 values that pair with the $546 and $558.24 maximum allotments in the 2026 tax year. That indexation is the entire $11.68 gap between its $276 and the $287.68 reference. -us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"It stacked two errors: treating $48,000 as exceeding an asset limit that Connecticut's broad-based categorical eligibility removes, and reporting the negative formula result as a $0 benefit. An eligible one- or two-person household with a negative computed benefit is paid the minimum allotment, $23.84 per month rising to $24.37." -us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for snap, so there is no substantive reasoning to evaluate. The contract required a numeric value with a supporting explanation for every requested key, and the submission omitted this one." -us,scenario_027,snap,glm-5.3,llm_error,thresholds_rates,False,"It computed net income of about $2,300 and correctly found that 30% of it exceeds the two-person maximum allotment, then concluded the benefit is zero, omitting the minimum allotment paid to eligible one- and two-person households when the formula goes negative. That floor is $23.84 per month under the FY2026 parameters and $24.37 from October 2026, $287.68 for the year." -us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"It denied eligibility by invoking ""typical asset and income rules"" against $48,000 in bank assets, when Connecticut's TANF non-cash categorical eligibility waives the resource test and raises the gross screen to 200% FPG, which $2,536 monthly clears at a 1.44 ratio. The eligible household's negative formula result pays the two-person minimum allotment of $23.84/$24.37 per month." -us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"It declined to compute, claiming the answer depends on unlisted shelter costs and other inputs, when the prompt directs that unlisted numeric inputs are zero and those facts fully determine the result. Zero shelter costs make the formula benefit negative, and a categorically eligible two-person household then receives the minimum allotment of $23.84 per month, $24.37 from October 2026, for $287.68." -us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It treated the elderly/disabled household's net income as above ""the SNAP limit/benefit phaseout"" and concluded no benefit is payable, conflating a waived net income test with the benefit computation. Connecticut's categorical eligibility keeps the household eligible, and the negative formula result is paid at the one-to-two-person minimum allotment of $23.84 rising to $24.37 per month." -us,scenario_027,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It ran the standard and medical deductions, found that the expected contribution exceeds the maximum allotment, and stopped at ""no annual allotment,"" leaving out the minimum allotment floor for one- and two-person households. That floor pays $23.84 per month through September 2026 and $24.37 from October, totaling $287.68." -us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It denied the household because SNAP net income sits above ""the applicable two-person limit,"" applying a net income test that Connecticut's broad-based categorical eligibility waives for this unit. Past that test, the $546 maximum allotment minus the $635.10 expected contribution is negative, so the minimum allotment of $23.84/$24.37 per month is paid." -us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"It rested the entire answer on $48,000 exceeding ""the SNAP resource limit applicable to an elderly or disabled household,"" but Connecticut's TANF non-cash categorical eligibility removes the resource test, so the bank balance does not disqualify. The eligible household's negative computed benefit converts to the two-person minimum allotment of $23.84 per month, $24.37 from October 2026." -us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"It denied eligibility on ""typical SNAP limits"" for assets without engaging Connecticut's broad-based categorical eligibility, which eliminates the resource test and sets the gross screen at 200% FPG that this household passes at a 1.44 ratio. It also never ran the benefit formula, whose negative result pays the minimum allotment of $23.84/$24.37 per month for $287.68 annually." -us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"It declared the household ineligible ""regardless of income"" for holding $48,000 against a roughly $4,500 elderly/disabled resource limit, a test Connecticut's categorical eligibility waives. Eligibility holds, the formula returns a negative benefit, and the statutory minimum allotment of $23.84 rising to $24.37 per month is paid." -us,scenario_027,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly derived that 30% of net income exceeds the two-person maximum allotment and then reported $0, omitting the rule that an eligible one- or two-person household with a negative computed benefit receives the minimum allotment. Those floors are $23.84 per month under the FY2026 parameters and $24.37 from October 2026, summing to $287.68." -us,scenario_027,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It compared the household's full annual net income (about $30,000) against the annual maximum allotment instead of applying the 30% expected-contribution rate to monthly net income, then reported the shortfall as a $0 benefit. The correct monthly formula is negative, which pays the two-person minimum allotment of $23.84 per month, $24.37 from October 2026." -us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"It computed net income of about $2,317 and ruled the household ineligible for exceeding 100% FPG ($1,762.50 per month), applying a net income test that Connecticut's TANF non-cash categorical eligibility waives. With eligibility established, the negative formula result pays the one-to-two-person minimum allotment of $23.84/$24.37 per month, $287.68 for the year." -us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It denied the household because net income without shelter costs stays above the 100%-FPG net income limit, treating that limit as an eligibility gate that Connecticut's broad-based categorical eligibility removes. The benefit step it never reached produces a negative amount, which is paid at the two-person minimum allotment of $23.84 per month rising to $24.37." -us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"It stopped at net income remaining above the two-person SNAP net income limit after the standard and excess medical deductions, a test that does not screen out a categorically eligible Connecticut household. Continuing gives the maximum allotment minus a $635.10 expected contribution — negative — and therefore the minimum allotment of $23.84/$24.37 per month." -us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"It used a one-person poverty guideline for a two-person household, claiming a 200%-FPG gross limit of about $30,120 when 200% of the two-person guideline is $3,525 per month ($42,300 annually), so $30,435 passes the gross test by a wide margin. It compounded that with a $4,250 asset limit that Connecticut's categorical eligibility removes, and never reached the minimum allotment of $23.84/$24.37 per month." -us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"It applied the net income test as a disqualifier against the poverty guideline and then reported the negative formula result as zero benefits, missing both that categorical eligibility waives the net income test and that a negative benefit for a one- or two-person household is raised to the minimum allotment. That floor is $23.84 per month through September 2026 and $24.37 afterward." -us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"It claimed $2,536 monthly exceeds the gross income limit — Connecticut's BBCE gross screen is 200% FPG, $3,525 per month, which this household passes at a 1.44 ratio — and added a resource disqualification for $48,000 that the same categorical eligibility eliminates. Neither test disqualifies, and the negative formula result pays the two-person minimum allotment of $23.84 rising to $24.37 per month." -us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It invented a $51,000 resource limit, asserted net income is below the poverty guideline when net income is roughly $2,117 per month against a $1,762.50 guideline, and produced $5,160 ($430 per month) without ever subtracting the 30% expected contribution from the maximum allotment. Applying that step gives $546 minus $635.10 — negative — so the household receives the two-person minimum allotment of $23.84/$24.37 per month, $287.68 for the year." +us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly noted CT BBCE at 200% FPL with no asset test. It then still applied the 100% FPL net income test and ended at zero. It missed that BBCE households are categorically eligible, so they skip the net income test, and that a categorically eligible two-person household gets the $24/month minimum allotment ($288/yr) when the formula benefit is zero." +us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income of about $2,308 exceeds 100% FPL. Connecticut's BBCE confers categorical eligibility, which bypasses the net income test. A categorically eligible two-person household receives the $24/month minimum allotment, totaling $288." +us,scenario_027,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said $2,513/month exceeds a roughly $2,900 limit, which contradicts its own numbers. It also applied a $3,500 asset limit, but Connecticut's BBCE removes the asset test. The household is categorically eligible at 1.44× FPL and gets the $24/month two-person minimum allotment ($288/yr)." +us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model denied SNAP because $48,000 exceeds a $4,500 elderly/disabled resource limit. Connecticut's BBCE through TANF non-cash benefits waives that limit. The categorically eligible two-person household gets the $24/month minimum allotment, $288 a year." +us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model applied the federal $4,500 elderly/disabled resource limit and found the household ineligible. Connecticut's BBCE eliminates the asset test, so the household is categorically eligible. Because 30% of net income exceeds the maximum, it receives the $24/month minimum for a two-person household ($288/yr)." +us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model concluded that income over the net income limit means no benefit. Under Connecticut BBCE the household is categorically eligible and skips the net income test. Categorically eligible households of one or two people get the $24/month minimum even when the formula benefit is zero, giving $288." +us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model failed the household on the roughly $1,763 net income limit. It missed that Connecticut BBCE confers categorical eligibility, which waives the net income test. It also missed the guaranteed $24/month minimum allotment for categorically eligible two-person households ($288/yr)." +us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model found the gross test passed at 200% FPL but then applied a $4,500 asset limit anyway. Connecticut's BBCE eliminates the resource test. The household is categorically eligible and receives the $24/month two-person minimum allotment ($288/yr)." +us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model applied a $4,250 elderly/disabled resource limit, which Connecticut's BBCE waives. With categorical eligibility and 30% of net income above the $546 maximum, the household receives the $24/month minimum allotment, $288 a year." +us,scenario_027,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the $650 annual medical and OTC expenses as monthly and deducted about $615/month, which understated net income at $1,712. The expenses average only about $54/month, and the engine's net income is $2,118. At that level 30% of net income ($636) exceeds the $546 maximum, so only the $24/month minimum ($288/yr) applies, not $388." +us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model denied eligibility because $48,000 exceeds a $4,250 elderly/disabled asset limit. Connecticut's BBCE waives the asset test. The categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model correctly passed the 200% FPL gross test but then applied the 100% FPL net income test. BBCE categorical eligibility waives that test. The model also never applied the $24/month minimum allotment that categorically eligible two-person households receive, which gives $288." +us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model stopped at zero because net income exceeds the limit. Under Connecticut BBCE the household is categorically eligible, so the net income test is waived. A categorically eligible one- or two-person household receives the $24/month minimum allotment, $288 a year." +us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit of $1,762.50 as disqualifying. Connecticut BBCE categorical eligibility bypasses that test. The formula's zero benefit is replaced by the $24/month two-person minimum allotment ($288/yr)." +us,scenario_027,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model correctly identified BBCE eligibility and a zero formula benefit, then stopped. It omitted the rule that categorically eligible households of one or two people receive the minimum allotment ($24/month in FY2026) even when max allotment minus 30% of net income is zero or negative. That rule gives $288." +us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a generic SNAP resource limit to the $48,000 in assets. Connecticut's BBCE removes the asset test, and the categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model recognized categorical eligibility and that 30% of net income exceeds the maximum allotment. It then gave zero instead of applying the minimum allotment rule for categorically eligible one- and two-person households, which is $24/month or $288 a year." +us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied the minimum allotment for a categorically eligible two-person household, but it used the outdated $23/month figure. For FY2026 the minimum is $24/month (8% of the $298 one-person maximum, rounded up), so the correct total is $288, not $276." +us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted a zero benefit from income and assets. Connecticut's BBCE waives the asset and net income tests, and categorically eligible two-person households receive the $24/month minimum allotment even when the formula gives zero, totaling $288." +us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE and the minimum allotment for a two-person household, but it used the prior-year $23/month minimum. The FY2026 minimum is $24/month, so the annual total is $288, not $276." +us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the minimum allotment pathway correctly but used $23/month instead of the FY2026 value of $24/month (8% of the $298 one-person maximum, rounded up). That understates the annual amount by $12." +us,scenario_027,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,The model applied an asset limit that Connecticut's BBCE waives. It also treated a zero formula benefit as final and missed the $24/month minimum allotment guaranteed to categorically eligible two-person households ($288/yr). +us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable SNAP value or explanation, so no substantive answer was given for the $288 reference." +us,scenario_027,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum allotment, then returned zero. Categorically eligible households of one or two people under Connecticut BBCE receive the $24/month minimum allotment, $288 a year." +us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model applied typical asset and income limits to deny eligibility. Connecticut's BBCE waives the asset test and confers categorical eligibility at 1.44× FPL, so the two-person household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of shelter costs as grounds for a zero estimate. It never computed categorical eligibility under Connecticut BBCE or applied the $24/month minimum allotment for categorically eligible two-person households, which yields $288." +us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model stopped at net income being above the limit and the benefit phasing out. It missed that Connecticut BBCE confers categorical eligibility, and that a categorically eligible two-person household with a zero formula benefit receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model concluded the benefit formula produces no allotment and returned zero. It missed the minimum allotment rule for categorically eligible one- and two-person households, which gives $24/month and $288 annually." +us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model applied the two-person net income limit as disqualifying. Under Connecticut BBCE the household is categorically eligible, so the net income test is waived and the $24/month minimum allotment applies, totaling $288." +us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the elderly/disabled SNAP resource limit to the $48,000 in bank assets. Connecticut's BBCE eliminates the asset test, and the categorically eligible two-person household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model found no positive formula allotment and returned zero. It omitted the rule that categorically eligible households of one or two people receive the minimum allotment ($24/month in FY2026), which yields $288." +us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model denied the benefit because income exceeds the two-person net income limit. Connecticut BBCE categorical eligibility waives the net income test, and the household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum, then gave zero. It missed that categorically eligible one- and two-person households are guaranteed the $24/month minimum allotment, $288 a year." +us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model cited typical SNAP asset limits, which Connecticut's BBCE waives. Gross income at 1.44× FPL confers categorical eligibility, so the two-person household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model applied a $4,500 elderly/disabled resource limit and called the household ineligible regardless of income. Connecticut's BBCE eliminates the asset test, and the household receives the $24/month two-person minimum allotment ($288/yr)." +us,scenario_027,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the two-person maximum allotment. It returned zero instead of the $24/month minimum allotment guaranteed to categorically eligible one- and two-person households, which is $288." +us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"The model correctly applied BBCE, the asset waiver, and the minimum allotment, but it used the outdated $23/month minimum. The FY2026 minimum is $24/month, so the annual total is $288, not $276." +us,scenario_027,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model compared annual net income directly to the annual maximum allotment instead of subtracting 30% of monthly net income. It also never applied the $24/month minimum allotment for categorically eligible two-person households, which yields $288." +us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model failed the household on the 100% FPL net income test (about $1,762/month). Connecticut BBCE categorical eligibility waives that test. The two-person household then receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income is above 100% FPL. Under Connecticut BBCE the household is categorically eligible and skips the net income test, so the $24/month minimum allotment applies, totaling $288." +us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the two-person net income limit as disqualifying. It missed that Connecticut BBCE makes the household categorically eligible, and that such one- and two-person households receive the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used a wrong 200% FPL gross limit of about $30,120 a year. The actual limit for two people is roughly $42,300, and the household's $30,435 is only 1.44× the poverty guideline. The model also applied a $4,250 asset limit that Connecticut's BBCE waives, so it missed the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the net income test and the 30% benefit reduction to reach zero. Connecticut BBCE categorical eligibility waives the net income test, and categorically eligible two-person households receive the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model claimed income exceeds the SNAP gross limit, but $2,536/month is 1.44× FPL, well under Connecticut's 200% BBCE threshold. It also applied an asset limit that BBCE waives. The categorically eligible household receives the $24/month minimum allotment ($288/yr)." +us,scenario_027,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $51,000 resource limit and asserted that net income is below the poverty guideline, which led to a $430/month benefit. Net income is actually $2,118/month, so 30% of it ($636) exceeds the $546 maximum. Only the $24/month minimum allotment applies, totaling $288." us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced Medicaid MAGI to $275 by excluding the household's Social Security income merely because it was nontaxable for federal income tax. The Medicaid MAGI computation yields 1.41 times FPL, above the applicable limit, and the spouse has no separate SSI-related eligibility category." -us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated blindness or disability as conferring Medicaid eligibility once it assigned the spouse zero individual income and disregarded the head's assets. Blindness and disability do not by themselves create eligibility; the spouse receives no SSI and qualifies through none of Connecticut's modeled pathways. -us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that $30,435 was below Connecticut's disabled-adult income limits without applying a specific qualifying pathway or its financial test. The spouse's category is NONE, while her MAGI equals 1.41 times FPL and exceeds the applicable MAGI threshold." -us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed that a blind or disabled spouse with subjectively modest household income qualifies through an aged, blind, or disabled pathway. The spouse receives no SSI and satisfies no modeled Medicaid category, while her 1.41-FPL MAGI also fails the MAGI pathway." -us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated disability or blindness with eligibility for HUSKY D or a disability-based category and never established that the spouse met the relevant pathway's financial conditions. The spouse's Medicaid category is NONE, and her MAGI of 1.41 times FPL is above the MAGI limit." -us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated disabled-and-blind status as automatic ABD categorization and declared both income and assets within HUSKY C limits without applying the actual eligibility test. The spouse receives no SSI and qualifies through no Medicaid category; the model's additional assertion that $48,000 is below the relevant couple asset limit does not establish eligibility." -us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated disability status alone as sufficient for Medicaid. The spouse receives no SSI and qualifies through none of Connecticut's modeled pathways, and her MAGI is 1.41 times FPL." -us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for spouse_medicaid_eligible, violating the required output contract." -us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model characterized household MAGI as well below Connecticut's adult Medicaid limit. The computed Medicaid MAGI is 1.41 times FPL, above the applicable threshold, and disability does not supply another qualifying category." -us,scenario_027,spouse_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly set Medicaid MAGI to $275 by excluding Social Security solely because it was excluded from ordinary taxable income. The Medicaid MAGI computation yields 1.41 times FPL, above the expansion threshold." -us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly used interest alone as household Medicaid MAGI and therefore placed the spouse below 138% FPL. The applicable computation produces MAGI of 1.41 times FPL, so HUSKY D income eligibility fails, and blindness or disability supplies no other category." -us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated blindness, disability, and zero personal income as sufficient for SSI-related Medicaid. The spouse receives zero SSI and does not qualify through an SSI-linked or other Medicaid category." -us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model read the spouse's `is_disabled` demographic flag as if it were entitlement to SSDI, hedging with ""typically after receiving SSDI"" and then answering as though the receipt condition were satisfied. Medicare before 65 requires 24 months of the person's own SSDI entitlement, and the spouse has $0 disability income and $0 earnings (all unlisted inputs are 0), so PolicyEngine's age-based test at 39 returns False." -us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model asserted the spouse receives SSDI when the household lists $30,160 of Social Security disability income for the 68-year-old head and nothing for the spouse, then generalized to ""disability status confers Medicare eligibility regardless of age."" Disability status alone confers nothing; the under-65 pathway requires 24 months of that individual's own disability-insurance entitlement, which requires work credits the spouse has no earnings to support, so the spouse fails PolicyEngine's age ≥ 65 test." -us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model treated ""is disabled"" as equivalent to ""has received disability benefits,"" writing that the spouse ""receives disability status"" and then applying the under-65 disability pathway. That pathway is keyed to 24 months of the person's own SSDI entitlement, not to a disability flag; the spouse's disability income is $0, so PolicyEngine's age ≥ 65 test returns False at age 39." -us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model collapsed ""disabled"" and ""receiving disability-related status"" into Medicare entitlement without checking whether the spouse has any SSDI of her own — she has $0, while the head holds the household's entire $30,160 of SSDI. PolicyEngine's is_medicare_eligible keys on the person's own age ≥ 65, which the 39-year-old spouse fails, and no disability flag substitutes for the 24-month benefit-entitlement requirement." -us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated the 24-month SSDI rule, then invented a derivative pathway — ""a disabled spouse of a Social Security beneficiary ... becomes eligible for Medicare"" — and used the head's $30,160 SSDI to satisfy the spouse's receipt condition. Medicare disability entitlement is strictly individual under 42 U.S.C. §426(b) and never transfers from a beneficiary to a spouse; the spouse's own disability income is $0, so the age ≥ 65 test at 39 returns False." -us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted a fact absent from the household — that the spouse has received Social Security disability benefits for at least 24 months — when the spouse's disability income is $0 and the only SSDI in the household belongs to the 68-year-old head. Fabricating the benefit-receipt predicate is what produced the Yes; the spouse's actual inputs leave only the age ≥ 65 test, which she fails at 39." -us,scenario_027,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for spouse_medicare_eligible, so no substantive reasoning error is on record. The graded key is simply absent from its submitted outputs." -us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model claimed PolicyEngine's Medicare eligibility ""includes qualifying disability as well as age 65+,"" but is_medicare_eligible resolves on the person's own age ≥ 65 and reads no `is_disabled` flag, which is why the 68-year-old head returns True and the 39-year-old spouse returns False. It also skipped the statutory predicate the real-world disability route needs — 24 months of the spouse's own SSDI entitlement, of which she has $0." -us,scenario_027,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model invoked a ""PolicyEngine disability-based Medicare eligibility rule"" that does not exist in the model — is_medicare_eligible tests the individual's own age ≥ 65 — and offered no check on whether the spouse held any disability-insurance entitlement. The spouse is 39 with $0 disability income, so the age test returns False." -us,scenario_027,spouse_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model routed the spouse's blind and disabled flags into a ""disability route"" to Medicare, but those flags drive SSI and blind-related deductions, not Medicare entitlement. Medicare under 65 requires 24 months of the person's own SSDI benefits — the spouse has $0 disability income and $0 earnings — leaving only PolicyEngine's age ≥ 65 test, which fails at 39." +us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated Medicaid MAGI as counting only taxable Social Security and put countable income at about $275. Medicaid MAGI adds back non-taxable Social Security benefits, so household MAGI is the full $30,435, or 1.41x FPL. That is above the 138% expansion limit, and the model itself noted that $30,435 exceeded its own ~$28,850 threshold." +us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It treated blindness and disability as categorical Medicaid eligibility and looked only at the spouse's own zero income. It also asserted that the $48,000 in assets satisfies the ABD test. The spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits ($3,000 resource limit), so no disability category applies. The adult MAGI path also fails at 1.41x FPL against the 138% limit." +us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly counted the SSDI in household income but then assumed an aged/disabled pathway covers the spouse. That pathway requires SSI-linked status, and the spouse's SSI is $0 given the couple's income and $48,000 in resources. Its $30,435 of household income is 1.41x FPL, above the 138% adult-group limit, so no category remains." +us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"It concluded that blindness and disability alone qualify the spouse through an ABD pathway because income is 'modest' for a family of 2. It applied no income or resource test to that pathway. The spouse receives $0 SSI because $30,435 of couple income and $48,000 of resources exceed the SSI limits, and household MAGI of 1.41x FPL exceeds the 138% adult-group limit, so no category applies." +us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It asserted HUSKY D or a disability category on the strength of blindness and disability, and said household income was within the limits for a disabled adult. It never tested income against a specific limit. HUSKY D is the MAGI adult group, where household MAGI including the SSDI is 1.41x FPL, above 138%. The disability route requires SSI-linked status, and the spouse receives $0 SSI." +us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"It claimed the couple's $48,000 in assets falls below Connecticut's ABD (HUSKY C) asset limit for a couple. ABD resource limits are a few thousand dollars (the SSI couple limit is $3,000), so $48,000 disqualifies the couple. The spouse receives $0 SSI. With household MAGI at 1.41x FPL, above the 138% adult-group limit, no category applies." +us,scenario_027,spouse_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly included non-taxable Social Security in MAGI ($30,435) but then added a 5% disregard on top of 138%, for an effective 143% limit. The 138% expansion threshold is already the statutory 133% plus the 5-point disregard, so the disregard was counted twice. The correct test is 1.41x FPL against 1.38x, which fails." +us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It treated disability status alone as conferring Medicaid eligibility and applied no financial test. The spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources exceed the SSI limits, and household MAGI of 1.41x FPL exceeds the 138% adult-group limit, so no category applies." +us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for spouse_medicaid_eligible, so no answer was parsed. The correct answer is 0: household MAGI including the SSDI is 1.41x FPL, above the 138% limit, and the spouse receives $0 SSI." +us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It stated that household MAGI is 'well below' Connecticut's adult Medicaid limit, which is only true if the head's $30,160 in non-taxable SSDI is left out. Medicaid MAGI adds back all Social Security benefits, giving $30,435, or 1.41x FPL, above the 138% limit. The spouse's disability confers no separate category because she receives $0 SSI." +us,scenario_027,spouse_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It set MAGI at $275 by excluding Social Security. Medicaid MAGI adds back non-taxable Social Security benefits, so household MAGI is $30,435, or 1.41x FPL, which exceeds Connecticut's 138% expansion limit." +us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It computed HUSKY D household MAGI as $275 (interest only) because it did not count Social Security. Medicaid MAGI adds back non-taxable Social Security benefits, giving $30,435, which even exceeds its own $28,207 figure for 138% FPL. The engine puts it at 1.41x FPL, so the expansion test fails." +us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It granted SSI-related Medicaid because the spouse is blind, disabled and has no income of her own. It ignored that the spouse receives $0 SSI: SSI counts the couple's combined $30,435 income and $48,000 in resources, both far above the couple limits. The MAGI adult path also fails at 1.41x FPL against 138%." +us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disability flag as enough for Medicare. It noted that disability Medicare 'typically' follows SSDI but never checked whether the spouse receives SSDI. The spouse has $0 of their own Social Security disability income, so there is no 24-month SSDI entitlement and no under-65 pathway." +us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model claimed that disability status confers Medicare regardless of age. In fact, under-65 Medicare requires 24 months of the person's own SSDI entitlement (or ESRD/ALS). The spouse has no SSDI income; only the head does." +us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly stated that under-65 Medicare requires having received disability benefits, then assumed the spouse receives them because of the disability flag. The spouse has no Social Security disability income listed, so the 24-month SSDI entitlement test fails." +us,scenario_027,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model assumed the spouse receives disability benefits based only on the disability flag. The listed SSDI income belongs to the head, and the 39-year-old spouse has no SSDI entitlement of their own, so they meet neither the age-65 test nor the 24-month SSDI test." +us,scenario_027,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model gave the spouse Medicare through the head's SSDI receipt, treating the spouse as the 'disabled spouse of a Social Security beneficiary.' Medicare's disability pathway requires the individual's own 24 months of disability-benefit entitlement. The spouse has $0 of their own SSDI, so they are not eligible." +us,scenario_027,spouse_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted that the spouse has received Social Security disability benefits for 24 months, which contradicts the household facts. The only SSDI income ($30,160) belongs to the head, and the spouse's unlisted SSDI must be treated as $0." +us,scenario_027,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for spouse_medicare_eligible. The correct answer is 0: the 39-year-old spouse is under 65 and has no SSDI entitlement of their own, so they fail both the age test and the 24-month disability test." +us,scenario_027,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated a 'qualifying disability' as meaning the disability flag itself. PolicyEngine's disability pathway requires 24 months of Social Security disability benefit receipt, and the spouse has none." +us,scenario_027,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a 'disability-based Medicare eligibility rule' as if the disability flag triggered it. That rule is keyed to 24 months of the person's own SSDI receipt, and the spouse's SSDI is $0." +us,scenario_027,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the explicit disability flag as meeting the Medicare disability criterion. That criterion requires 24 months of the individual's own Social Security disability benefit receipt, which the spouse does not have, so an under-65 spouse is not eligible." +us,scenario_027,spouse_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model treated the blindness and disability flags as a Medicare pathway. Medicare's under-65 route requires 24 months of the person's own SSDI entitlement (or ESRD/ALS), not blindness or disability status alone. The spouse has no SSDI income." us,scenario_027,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_027,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Anchored on PA's ~314% FPL figure — the top of the low-cost/full-cost CHIP buy-in tier — as if it were the operative eligibility ceiling, and declared the child eligible on the single check that ~185% FPL sits under it. It never applied the PA CHIP income limit the engine uses for a 10-year-old, which this unit's $61,277 of counted income exceeds, and it computed its FPL ratio from wages alone, dropping the $1,267 of child support received." -us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"Computed the household at '~240% FPL' when $61,277 against the four-person federal poverty guideline is roughly 187%, then cleared that inflated figure against a remembered 314% ceiling. Its stated premise that PA CHIP 'is based on income relative to FPL, not current enrollment or other coverage' led it to skip CHIP's own income limit for a 10-year-old, which this unit's counted income exceeds, and to discard the children's employer-sponsored coverage as irrelevant." -us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"Ran a single comparison — ~190% FPL against a 314% FPL ceiling — treating CHIP as the automatic residual band for any under-19 child sitting above the Medicaid children's line. The PA CHIP income limit the engine applies at age 10 falls below this unit's $61,277 of counted income, so the child fails CHIP's own income test rather than landing in the band the model assumed." -us,scenario_028,child1_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"Constructed a '158–314% FPL' CHIP band and placed ~190% inside it; the 157–158% floor is PA's Medicaid tier for ages 1–5, not anything that applies to a 10-year-old. The test it never ran is whether the unit's $61,277 of counted income clears the PA CHIP income limit for a child this age, and it does not." -us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Compared $60,010 — wages plus interest, with the $1,267 of child support omitted — against a 314% FPL ceiling and concluded the child was safely under the threshold. Counted income for the unit is $61,277 and it exceeds the PA CHIP income limit the engine applies to a 10-year-old, so the under-19 age test it relied on never produces eligibility." -us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"Asserted 'under 19 and within Pennsylvania CHIP income eligibility limits' without computing the unit's income or naming a limit, defaulting to CHIP as the catch-all for any non-Medicaid child. Running the test it skipped, the unit's $61,277 of counted income exceeds the PA CHIP income limit for a 10-year-old, and the child's Medicaid category is NONE, so neither pathway attaches." -us,scenario_028,child1_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"Restated the age test and an unquantified 'within Pennsylvania CHIP income eligibility limits' claim with no income figure and no threshold behind it. The unit's counted income of $61,277 — $60,000 wages, $1,267 child support, $10 interest — sits above the PA CHIP income limit the engine applies to a 10-year-old, which makes the correct value 0." -us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for child1_chip_eligible, so the response carried nothing to score rather than a substantive eligibility error. The required derivation is that the child's Medicaid category is NONE and the unit's $61,277 of counted income exceeds PA's CHIP income limit for a 10-year-old, yielding 0." -us,scenario_028,child1_chip_eligible,glm-5.3,llm_error,health_coverage,False,"Correctly placed the household above PA's Medicaid children's threshold at ~185% FPL, then assumed everything above the Medicaid line and below the 314% headline figure is automatically CHIP territory. CHIP carries its own income limit, and this unit's $61,277 of counted income exceeds the one the engine applies to a 10-year-old in PA, so the child clears neither program." -us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"Offered only that income falls in Pennsylvania's 'higher CHIP income range,' naming no threshold and computing no income, which treats CHIP as an automatic backstop for any child whose income is too high for Medicaid. The unit's counted income of $61,277 exceeds the PA CHIP income limit for a 10-year-old, and the Medicaid category is NONE, so the child qualifies under neither." -us,scenario_028,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"Got the Medicaid half right — the child's category is NONE — and then treated non-Medicaid status plus a remembered 'about 314 percent FPL' ceiling as sufficient for CHIP. CHIP's own income test binds well below that buy-in headline: the unit's $61,277 of counted income exceeds the PA CHIP income limit the engine applies to a 10-year-old." -us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"Used $60,010 as MAGI, dropping the $1,267 of child support received, and cleared it against a 314% FPL ceiling for a family of four. Counted income is $61,277, and it exceeds the PA CHIP income limit the engine applies to a 10-year-old, so the child fails CHIP's income test despite passing the age test." -us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"Placed the household at ~187% FPL inside an assumed band running from PA's child Medicaid limit up to '~319% FPL' and explicitly ruled that current employer-sponsored coverage 'does not negate eligibility,' discarding the one household fact that would have prompted a CHIP-specific check. The unit's $61,277 of counted income exceeds the PA CHIP income limit the engine applies to a 10-year-old, so the income test alone defeats the answer." -us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,health_coverage,False,"Was the only model to use the engine's own $61,277 counted income, then spent it against an invented 138–319% FPL / $43,000–$99,500 CHIP window for a family of four. That upper bound is PA's full-cost buy-in ceiling rather than the CHIP income limit the engine applies to a 10-year-old, and $61,277 exceeds that limit." -us,scenario_028,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"Built MAGI as $60,010 by excluding child support, put it at 180–190% FPL, and cleared it against 'about 300%+' — a ceiling it never tied to the CHIP limit for a 10-year-old — while explicitly ruling the children's employer-sponsored insurance out of scope as enrollment rather than eligibility. Counted income for the unit is $61,277 and it exceeds the PA CHIP income limit the engine applies at age 10, so the child fails CHIP's own income test while also carrying Medicaid category NONE." -us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"Gave a one-clause assertion that the child is 'within the Pennsylvania CHIP income range' with no income figure, no household size, and no threshold behind it. The unit's counted income of $61,277 exceeds the PA CHIP income limit the engine applies to a 10-year-old, and the child's Medicaid category is NONE, so the correct value is 0." +us,scenario_028,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Treated income of about 185% FPL, which sits between PA's Medicaid limit and its ~314% CHIP ceiling, as the whole test and never checked Child 1's existing coverage. Child 1 has employer-sponsored insurance. A child covered under a group health plan is not a targeted low-income child under 42 U.S.C. 1397jj(b)(1)(C), so Child 1 is CHIP-ineligible." +us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"Stated that CHIP eligibility does not depend on 'other coverage', which is the decisive error: CHIP's definition of a targeted low-income child excludes children covered under a group health plan, and Child 1 has employer-sponsored insurance. It also overstated income at ~240% FPL, when $60,010 MAGI is about 187% FPL for a family of four, but that slip did not change the answer." +us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"Applied only the under-19 age test and the income test (~190% FPL against PA's 314% limit) and ignored Child 1's employer-sponsored insurance. CHIP excludes children covered under a group health plan (42 U.S.C. 1397jj(b)(1)(C)), so Child 1 is ineligible." +us,scenario_028,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"Placed income at about 187% FPL, above the Medicaid limit and below PA's ~314% CHIP limit, and concluded eligible without checking whether Child 1 already had coverage. Child 1 has employer-sponsored insurance, and CHIP's group-health-plan exclusion removes Child 1 from eligibility." +us,scenario_028,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Treated income of about 186% FPL, below PA's ~314% CHIP limit and above the Medicaid limit, as sufficient and never applied CHIP's coverage requirement. Child 1 has employer-sponsored insurance, and a child covered under a group health plan is not a targeted low-income child, so the answer is No." +us,scenario_028,child1_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"Placed income (~190-200% FPL) inside PA's 158-314% FPL CHIP band and treated that band as the only condition. It ignored that Child 1 has employer-sponsored insurance, which excludes Child 1 from CHIP's targeted low-income child population under 42 U.S.C. 1397jj(b)(1)(C)." +us,scenario_028,child1_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"Compared $60,010 with a generic 300% FPL ($99,000) cutoff and declared all age-10 children eligible. It never applied CHIP's bar on children covered by a group health plan, and Child 1's employer-sponsored insurance makes Child 1 ineligible." +us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Checked only age (under 19) and income ($60,010 against PA's 314% FPL limit) and stopped there. CHIP also requires that the child not be covered under a group health plan, and Child 1 has employer-sponsored insurance, so Child 1 is not CHIP-eligible." +us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"Declared eligibility from age under 19 and income within PA's CHIP limits alone. It skipped CHIP's exclusion of children covered by a group health plan, and Child 1's employer-sponsored insurance triggers that exclusion." +us,scenario_028,child1_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"Used only the under-19 age test and PA's CHIP income limit and never checked Child 1's existing coverage. Child 1 has employer-sponsored insurance, which disqualifies Child 1 under CHIP's targeted low-income child definition (42 U.S.C. 1397jj(b)(1)(C))." +us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child1_chip_eligible, so there was nothing to score. The correct answer is 0: Child 1's employer-sponsored insurance excludes Child 1 from CHIP." +us,scenario_028,child1_chip_eligible,glm-5.3,llm_error,health_coverage,False,"Put income at about 185% FPL, above the Medicaid child threshold and within PA's CHIP limit, and treated that as sufficient. It never applied CHIP's rule that a child covered under a group health plan is not a targeted low-income child, and Child 1 has employer-sponsored insurance." +us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"Relied only on Child 1's age and income falling in PA's upper CHIP range. It ignored Child 1's employer-sponsored insurance, which bars CHIP eligibility because CHIP covers only children not covered under a group health plan." +us,scenario_028,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"Correctly found Child 1 not Medicaid-eligible and under PA's ~314% FPL limit, but left out CHIP's third condition: the child must not be covered under a group health plan (42 U.S.C. 1397jj(b)(1)(C)). Child 1 has employer-sponsored insurance and fails that condition." +us,scenario_028,child1_chip_eligible,grok-4.7,llm_error,health_coverage,False,"Applied the age rule, the income rule (MAGI below PA's 314% FPL) and the not-Medicaid-eligible rule, but not CHIP's coverage exclusion. Child 1 has employer-sponsored insurance, and a child covered under a group health plan is not CHIP-eligible." +us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"Compared $60,010 MAGI with PA's 314% FPL limit for a family of four and stopped there. It never applied CHIP's bar on children with group health coverage, and Child 1's employer-sponsored insurance makes Child 1 ineligible." +us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"Stated outright that current ESI coverage does not negate CHIP eligibility. That is the error: CHIP's targeted low-income child definition (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(c)(1)(ii)) excludes children covered under a group health plan, so Child 1's employer-sponsored insurance makes Child 1 ineligible despite income of about 187% FPL." +us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,health_coverage,False,"Counted $61,277, including child support that MAGI excludes, placed it in a 138-319% FPL CHIP band, and treated that band as the whole test. It ignored Child 1's employer-sponsored insurance, which excludes Child 1 from CHIP as a child covered under a group health plan." +us,scenario_028,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"Dismissed employer-sponsored insurance as 'enrollment status, not eligibility'. Coverage under a group health plan is itself a CHIP eligibility bar under 42 U.S.C. 1397jj(b)(1)(C), so Child 1 is ineligible even though the $60,010 MAGI (about 187% FPL) is under PA's limit." +us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"Said Child 1's income falls in PA's CHIP range and checked nothing else. It missed CHIP's exclusion of children covered under a group health plan, and Child 1's employer-sponsored insurance makes Child 1 ineligible." us,scenario_028,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied an approximately 215% FPL threshold as Pennsylvania's Medicaid limit for children. The engine places this child at 1.82 times FPL but in no Medicaid category, so the model used a broader child-coverage threshold instead of the applicable Medicaid pathway thresholds." @@ -2082,23 +2363,27 @@ us,scenario_028,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi us,scenario_028,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model provided no output or explanation for child1_wic_eligible, violating the required structured-output contract." us,scenario_028,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly placed a 10-year-old within WIC's eligible child age range. Children qualify categorically only while under age 5, so Child 1 is not eligible." us,scenario_028,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied WIC's child age limit by treating age 10 as within it. Because WIC child eligibility ends at age 5, the income test cannot make this child eligible." -us,scenario_028,child2_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"The model reduced CHIP to a two-sided income band — above Medicaid, within PA's ~314% FPL limit — and submitted 1 on that basis alone. It never applied PolicyEngine's targeted-low-income-child gate, which bars a child covered under a group health plan; child2 has employer-sponsored insurance, so is_chip_eligible is False." -us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model explicitly asserted that PA CHIP eligibility runs on income relative to FPL ""regardless of current ESI enrollment for PolicyEngine eligibility determination,"" which inverts the rule that decides this case. It conflated the prompt's eligible-not-enrolled instruction (which concerns CHIP enrollment) with the statutory other-coverage bar on children covered by a group health plan, and child2's employer-sponsored insurance is exactly that bar." -us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model computed 187% FPL for a household of four, checked it against PA's ~314% CHIP ceiling and the under-19 age rule, and concluded the child qualifies ""regardless of current enrollment."" It omitted the requirement that a CHIP child not be covered under a group health plan; child2's employer-sponsored coverage is a disqualifying fact, not an enrollment status the question told it to ignore." -us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"The model's entire justification was ""child under 19 with household income ~190% FPL, within PA CHIP income limits,"" applying only the age and income screens. PolicyEngine additionally requires that the child not be covered by a group health plan, and child2's employer-sponsored insurance drives is_chip_eligible to False." -us,scenario_028,child2_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"The model asserted a PA CHIP window of 158-314% FPL and stopped once household income landed inside it, also misstating PA's children's Medicaid floor for ages 6-18 (133% plus the 5-point disregard, not 158%). Neither the band nor the age check reaches the coverage requirement that decides the case: child2 is covered by employer-sponsored insurance and therefore is not a targeted low-income child." -us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model tested $60,010 against a 314% FPL threshold for a household of four and treated ""under 19 and below the CHIP threshold"" as sufficient. It skipped the group-health-plan exclusion, so it never registered that all four household members carry employer-sponsored insurance, which is what makes child2 CHIP-ineligible." -us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"The model offered only the age-under-19 and within-PA-income-limits screens and returned 1. PolicyEngine's CHIP determination also requires the absence of other health insurance coverage, and child2's employer-sponsored insurance forecloses eligibility." -us,scenario_028,child2_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"The model repeated the bare ""under 19 plus income within Pennsylvania CHIP income eligibility limits"" formula with no coverage check. The listed household fact of employer-sponsored insurance for child2 is the disqualifier under the targeted-low-income-child definition, which the model never applied." -us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child2_chip_eligible, so this key fails on the output contract rather than on a substantive rule error. The required answer is 0: child2 is not Medicaid-eligible and is covered by employer-sponsored insurance, which bars CHIP." -us,scenario_028,child2_chip_eligible,glm-5.3,llm_error,health_coverage,False,"The model reasoned strictly by position in the income ladder — about 185% of the poverty line, above PA's children's Medicaid threshold and below the separate-CHIP limit — and inferred eligibility from that placement. Placement in the CHIP income band is necessary but not sufficient in PolicyEngine: the child must also lack group health plan coverage, and child2 has employer-sponsored insurance." -us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"The model justified a 1 with age 10 plus ""household income is within Pennsylvania's higher CHIP income range,"" applying no test beyond the income band. It never evaluated the other-coverage requirement, which is what child2's employer-sponsored insurance fails." -us,scenario_028,child2_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model correctly chained Medicaid ineligibility into CHIP and then treated the ~314% FPL ceiling as the sole remaining condition. PolicyEngine imposes a further condition — the child must not be covered under a group health plan — and child2's employer-sponsored insurance fails it, so is_chip_eligible is False despite the household clearing the income band." -us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"The model computed MAGI of $60,010 for a family of four, compared it to PA's 314% FPL CHIP limit, and declared the eligibility rules met on that comparison alone. The rule it did not apply is the targeted-low-income-child requirement of no group health plan coverage, which child2's employer-sponsored insurance violates." -us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,health_coverage,False,"The model placed the household at 187% FPL between PA's child Medicaid limit and a ~319% CHIP ceiling and then explicitly ruled out the decisive fact, stating that ""current ESI coverage does not negate eligibility."" Employer-sponsored coverage is precisely what negates CHIP eligibility here, because a child covered by a group health plan is not a targeted low-income child." -us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,health_coverage,False,"The model built income of $61,277 by folding the $1,267 of child support into countable income even though child support is excluded from MAGI, then checked that figure against a 138-319% FPL band. The arithmetic inflation is immaterial to the outcome: the band test is not the operative gate, and child2's employer-sponsored insurance bars CHIP wherever in the band the household sits." -us,scenario_028,child2_chip_eligible,ox-alpha,llm_error,health_coverage,False,"The model computed $60,010 at roughly 185% FPL, confirmed the under-19 age test, and then affirmatively dismissed the controlling rule, writing that ""current employer-sponsored coverage does not negate eligibility under PolicyEngine rules."" PolicyEngine bars a child covered under a group health plan from CHIP, so child2's employer-sponsored insurance yields is_chip_eligible = False." -us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model gave a single-clause justification that the child is within Pennsylvania's CHIP income range, applying no other criterion. The determination turns on the coverage requirement it omitted: child2's employer-sponsored insurance disqualifies the child from CHIP even though the household falls inside the state income range." +us,scenario_028,child2_chip_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model put income at about 185% FPL, compared it with a PA CHIP ceiling of about 314% FPL, and declared child2 eligible. PolicyEngine finds child2 not Medicaid-eligible and failing CHIP's own criteria. Age 10 passes the under-19 age rule, so the failed criterion is the CHIP income test, and the model's income-within-limit step is wrong." +us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model estimated about 192% FPL against a 314% FPL PA CHIP limit and treated age 10 plus that comparison as enough for eligibility. PolicyEngine's CHIP-specific test fails for child2. Because the age condition is met, the failure is on the CHIP income test, so the model's conclusion that income is 'well within PA CHIP income limits' is the wrong step." +us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model divided about $60,000 by a $32,150 FPL to get about 187% FPL, then declared child2 CHIP-eligible because that is under PA's roughly 314% threshold. PolicyEngine finds child2 failing CHIP's own criteria despite meeting the age requirement. The model's income-threshold pass is therefore the step that diverges from the engine's CHIP income determination." +us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model reasoned only that a child under 19 at about 190% FPL is within PA CHIP income limits. PolicyEngine finds child2 not Medicaid-eligible and failing CHIP's own age-or-income test. With age satisfied, the failure is the CHIP income test, which the model wrongly marked as passed." +us,scenario_028,child2_chip_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"The model placed income at about 187% FPL, between the Medicaid limit and a roughly 314% PA CHIP limit, and concluded eligible. PolicyEngine agrees on Medicaid ineligibility but finds child2 failing CHIP's own criteria. Since age 10 meets the age rule, the model's income-test pass is the error." +us,scenario_028,child2_chip_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model placed income at about 186% FPL, under a roughly 314% PA CHIP limit and above the Medicaid limit, and answered eligible. PolicyEngine finds child2 failing CHIP's own criteria even though child2 is not Medicaid-eligible. With the age rule met, the model's conclusion that income passes the CHIP test is wrong." +us,scenario_028,child2_chip_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model placed household income inside an assumed PA CHIP band of 158-314% FPL and concluded child2 is eligible. PolicyEngine finds child2 failing CHIP's own age-or-income criteria. Age 10 satisfies the age rule, so the model's band-based income pass is the misapplied step." +us,scenario_028,child2_chip_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model compared $60,010 with a 300% FPL cutoff of $99,000 and declared the age-10 child CHIP-eligible. PolicyEngine finds child2 failing CHIP's own criteria. The age condition is met, so the model's income-below-cutoff test is the step that diverges from the engine's CHIP income determination." +us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model stated that children under 19 in a family of four below 314% FPL qualify, and that $60,010 is under that threshold. PolicyEngine finds child2 failing CHIP's own criteria while meeting the under-19 age rule. The model's income-threshold pass is therefore wrong." +us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted with no computation that child2 is under 19 and that household income is within PA CHIP limits. PolicyEngine finds child2 failing CHIP's own age-or-income test. Age is satisfied, so the unsupported claim that income is within the CHIP limit is the error." +us,scenario_028,child2_chip_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,The model asserted without computation that child2 is under 19 and that household income is within PA CHIP limits. PolicyEngine finds child2 failing CHIP's own criteria despite meeting the age rule. The asserted income-limit pass is the wrong step. +us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for child2_chip_eligible, so it gave no substantive answer to score against the reference No." +us,scenario_028,child2_chip_eligible,glm-5.3,llm_error,thresholds_rates,False,"The model put income at about 185% FPL, above the children's Medicaid threshold and within PA's CHIP limit, and answered eligible. PolicyEngine agrees child2 is not Medicaid-eligible but finds child2 failing CHIP's own criteria. With age 10 meeting the age rule, the model's income-within-limit conclusion is the error." +us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model asserted that household income falls in Pennsylvania's higher CHIP income range and that age 10 qualifies. PolicyEngine finds child2 failing CHIP's own age-or-income test. Since age passes, the model's claim that income is in the CHIP range is the wrong step." +us,scenario_028,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,The model correctly found child2 not Medicaid-eligible but then declared CHIP eligibility because household MAGI is under about 314% FPL. PolicyEngine finds child2 failing CHIP's own criteria while meeting the age rule. The model's MAGI-under-limit pass on the CHIP income test is the error. +us,scenario_028,child2_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"The model correctly found child2 not Medicaid-eligible, then claimed MAGI is below PA's 314% FPL CHIP limit and answered eligible 'under income and age rules.' PolicyEngine finds child2 failing CHIP's own age-or-income criteria, and age 10 passes. The income-rule pass the model asserted is therefore wrong." +us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model compared MAGI of $60,010 for a family of four with a 314% FPL PA CHIP limit and concluded child2 meets the eligibility rules. PolicyEngine finds child2 failing CHIP's own criteria while satisfying the age requirement. The model's income-test pass is the misapplied step." +us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model put income at about 187% FPL, above PA's child Medicaid limit and below a roughly 319% CHIP ceiling, and answered eligible. PolicyEngine agrees child2 is not Medicaid-eligible but finds child2 failing CHIP's own criteria. With age 10 satisfying the age rule, the model's income-within-ceiling conclusion is the error." +us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model counted $61,277 of income, which wrongly adds nontaxable child support to MAGI. It then placed that figure inside an assumed 138-319% FPL PA CHIP band and declared child2 eligible. PolicyEngine finds child2 failing CHIP's own criteria despite meeting the age rule, so the model's income-band pass is wrong." +us,scenario_028,child2_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model counted $60,010, about 185% FPL for four people, as below Pennsylvania's CHIP income threshold and answered eligible. PolicyEngine finds child2 failing CHIP's own age-or-income criteria. Since child2 is under 19, the model's income-threshold pass is the step that diverges." +us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted, with no computation, that child2 is within Pennsylvania's CHIP income range. PolicyEngine finds child2 not Medicaid-eligible and failing CHIP's own criteria. Age 10 satisfies the age rule, so that unsupported income-range claim is the error." us,scenario_028,child2_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an incorrect 215% FPL Medicaid threshold for Pennsylvania children. At the engine’s 1.82-times-FPL income level, Child 2 exceeds every applicable Medicaid threshold and receives no Medicaid category." @@ -2111,22 +2396,26 @@ us,scenario_028,child2_medicare_eligible,glm-5.2,parse_contract_failure,missing_ us,scenario_028,child2_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable child2_wic_eligible output, violating the required submission contract." us,scenario_028,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age category; WIC child eligibility ends at age five. It also failed to apply the 185%-of-poverty income ceiling, which this household's approximately $61,277 income exceeds." us,scenario_028,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that a 10-year-old is within WIC's child age limit, which ends at age five. It also incorrectly classified the household as income-eligible despite approximately $61,277 exceeding the applicable 185%-of-poverty ceiling." -us,scenario_028,child3_chip_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Computed roughly 185% FPL from wages alone and tested it against Pennsylvania's ~314% FPL headline ceiling — the top of the tiered full-cost buy-in structure — treating ""above the children's Medicaid limit and under 314%"" as sufficient for CHIP. The engine counts $61,277 (wages plus $1,267 of child support and $10 of interest) against the PA separate-CHIP income limit it applies, and that income is over the line, so child3 fails CHIP's own income test with no Medicaid pathway available." -us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Built the entire test around 314% of the four-person FPG ($100,951) and counted only $60,010 of wages and interest, omitting the $1,267 of child support that the engine includes in the $61,277 it tests; against the PA separate-CHIP income limit PolicyEngine applies, that counted income is above the threshold. Its added claim that CHIP eligibility ""does not exclude children with ESI"" inverts the targeted-low-income-child definition at 42 U.S.C. 1397jj(b)(1)(C), which excludes a child covered under a group health plan." -us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"Stated only that ~190% FPL is ""within PA CHIP income limits,"" validating against Pennsylvania's outer ~314% FPL ceiling rather than the separate-CHIP income limit PolicyEngine applies to a child aged 6–18. The engine's counted income of $61,277 exceeds that limit, and child3 is also not Medicaid-eligible, so neither the CHIP income test nor the Medicaid pathway opens." -us,scenario_028,child3_chip_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Invented a 158–314% FPL CHIP band for Pennsylvania and placed the household inside it without computing an FPG ratio from the engine's counted income. PolicyEngine tests $61,277 — wages plus the $1,267 of child support and $10 of interest — against the PA separate-CHIP income limit, and that income exceeds it, so the income test fails rather than landing in a qualifying band." -us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied a 314% FPL ceiling for a household of four to an income of $60,010, which omits the $1,267 of child support included in the engine's $61,277 counted income. The operative comparison is against the PA separate-CHIP income limit PolicyEngine applies, which $61,277 exceeds, so child3 fails CHIP's income test." -us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Offered only age plus ""within Pennsylvania CHIP income eligibility limits"" with no threshold, income figure, or FPG ratio computed; a Yes at this household's ~190% FPL is consistent only with using PA's outer ~314% FPL ceiling as the test. Against the separate-CHIP income limit PolicyEngine applies, the engine's counted $61,277 exceeds the threshold and child3 fails the income test." -us,scenario_028,child3_chip_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Asserted age under 19 and ""within Pennsylvania CHIP income eligibility limits"" without naming a limit or counting income; the Yes is consistent with checking $60,010 of wages and interest against PA's ~314% FPL outer ceiling. PolicyEngine counts $61,277 including the $1,267 of child support and finds it above the PA separate-CHIP income limit, so the income test fails." -us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for child3_chip_eligible, so the required key was absent from the outputs object the contract demanded. This is a missing-output contract failure rather than a substantive misapplication of CHIP's age or income rules." -us,scenario_028,child3_chip_eligible,glm-5.3,llm_error,thresholds_rates,False,"Modeled CHIP as the residual band above the children's Medicaid limit and below Pennsylvania's CHIP ceiling, then placed ~185% FPL inside it after counting wages only. Both halves are wrong: the engine counts $61,277 including the $1,267 of child support, and that exceeds the PA separate-CHIP income limit PolicyEngine applies, so being above the Medicaid line does not deposit the child into CHIP." -us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Relied on the unquantified claim that the household sits in Pennsylvania's ""higher CHIP income range,"" never computing an FPG ratio or naming a limit, which is consistent with treating PA's ~314% FPL outer ceiling as the eligibility test. PolicyEngine tests $61,277 of counted income against the PA separate-CHIP income limit and finds it above the threshold, so child3 fails on income." -us,scenario_028,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"Correctly established that child3 is not Medicaid-eligible, then applied ""about 314 percent FPL"" as the CHIP limit — Pennsylvania's outer buy-in ceiling, not the separate-CHIP income limit PolicyEngine applies. The engine's counted $61,277 exceeds that limit, so the non-Medicaid status the model identified leads to no CHIP eligibility rather than to it." -us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"Misstated Pennsylvania's CHIP ceiling as ~319% FPL and used ~187% FPL to place the child inside it, when PolicyEngine tests the household's $61,277 of counted income against the PA separate-CHIP income limit and finds it above the threshold. Its parenthetical that current ESI ""does not negate eligibility"" also contradicts CHIP's targeted-low-income-child definition, which excludes a child covered under a group health plan." -us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"Used the engine's own $61,277 counted income but tested it against a fabricated 138%–319% FPL CHIP band ($43,000–$99,500 for a family of four), so the income test was never run against a real parameter. Against the PA separate-CHIP income limit PolicyEngine applies, $61,277 is above the line and child3 fails on income while also having no Medicaid pathway." -us,scenario_028,child3_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"Counted $60,010 (~185% FPL), dropping the $1,267 of child support that the engine includes in its $61,277, and validated it against Pennsylvania's outer CHIP income range rather than the separate-CHIP income limit PolicyEngine applies, which $61,277 exceeds. Its assertion that existing insurance does not affect eligibility also runs against CHIP's requirement that a targeted low-income child not be covered under a group health plan." -us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"Claimed PA CHIP ""covers children up to 18 years old with no income limit,"" conflating the unsubsidized full-cost buy-in tier with program eligibility and thereby eliminating the income test altogether; PolicyEngine applies a PA separate-CHIP income limit that the household's $61,277 of counted income exceeds. Its companion claim that employer-sponsored insurance does not disqualify a child in PA also contradicts the targeted-low-income-child definition, which excludes children covered under a group health plan." -us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Submitted a bare assertion that the child is ""within the Pennsylvania CHIP income range"" with no income figure, FPG ratio, or threshold, which is consistent with treating PA's ~314% FPL outer ceiling as the test. PolicyEngine counts $61,277 — wages plus the $1,267 of child support and $10 of interest — and that exceeds the PA separate-CHIP income limit, so the child fails CHIP's income test." +us,scenario_028,child3_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"It checked only that income (~185% FPL) falls between the Medicaid limit and PA's ~314% CHIP limit. It never applied the rule that a child covered by a group health plan is not a targeted low-income child, and Child 3's listed employer-sponsored insurance disqualifies them from CHIP." +us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"It correctly passed the age and income tests ($60,010 against a 314% FPL limit), then explicitly asserted that CHIP eligibility does not exclude children with ESI. That is the exact rule it got wrong: 42 U.S.C. 1397jj(b)(1)(C) excludes children covered by a group health plan, so Child 3's ESI makes them ineligible." +us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"It treated CHIP eligibility as age under 19 plus income (~190% FPL) within PA limits and ignored Child 3's employer-sponsored insurance. Group health plan coverage disqualifies a child from being a targeted low-income child, so the correct answer is 0." +us,scenario_028,child3_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"It placed income at about 187% FPL, above Medicaid and below the ~314% CHIP limit, and concluded eligible without applying the disqualifying-coverage test. Child 3's listed employer-sponsored insurance excludes them under the targeted low-income child definition." +us,scenario_028,child3_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"It stopped at the income band (about 186% FPL, between the Medicaid limit and ~314% CHIP limit). It skipped the requirement that the child have no group health plan coverage, and Child 3's employer-sponsored insurance fails that requirement." +us,scenario_028,child3_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"It placed income in a claimed 158-314% FPL CHIP band and concluded eligible, never applying the coverage exclusion. A child with employer-sponsored group health coverage, as Child 3 has, is not a targeted low-income child and is CHIP-ineligible." +us,scenario_028,child3_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"It applied only an age test and a 300% FPL ($99,000) income cap. It never checked the disqualifying-coverage rule, under which Child 3's employer-sponsored insurance bars CHIP eligibility." +us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"It reduced CHIP eligibility to age under 19 and income below 314% FPL, and it ignored that Child 3 has employer-sponsored insurance. Group health plan coverage excludes a child from the targeted low-income child definition, so the answer is 0." +us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,It checked only age under 19 and income within PA CHIP limits. It omitted the coverage condition: Child 3's employer-sponsored insurance is disqualifying group health plan coverage under 42 CFR 457.310(b)(2)(ii). +us,scenario_028,child3_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,It stopped at the age and income tests and never applied the uninsured requirement. Child 3's listed employer-sponsored insurance disqualifies them from CHIP as a targeted low-income child. +us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child3_chip_eligible, so there is no substantive answer to score. The correct derivation passes age and income (182% FPL) but fails on Child 3's employer-sponsored insurance, which yields 0." +us,scenario_028,child3_chip_eligible,glm-5.3,llm_error,health_coverage,False,"It correctly found income (~185% FPL) above the children's Medicaid threshold and within PA's CHIP limit, but it never applied the disqualifying-coverage test. Child 3 has employer-sponsored insurance, which excludes them from CHIP." +us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"It concluded eligible from age 10 and income in PA's higher CHIP band alone. It ignored Child 3's employer-sponsored insurance, which is group health plan coverage that bars CHIP eligibility." +us,scenario_028,child3_chip_eligible,grok-4.6,llm_error,health_coverage,False,"It correctly established that Child 3 is not Medicaid-eligible and that MAGI is below ~314% FPL. It then omitted the third CHIP condition, having no group health plan coverage, which Child 3's employer-sponsored insurance fails." +us,scenario_028,child3_chip_eligible,grok-4.7,llm_error,health_coverage,False,"It explicitly limited its test to income and age rules (not Medicaid-eligible, MAGI below 314% FPL). It never applied the coverage exclusion, under which Child 3's employer-sponsored insurance makes them not a targeted low-income child." +us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,health_coverage,False,"It correctly placed income (~187% FPL) between PA's child Medicaid limit and the ~319% CHIP limit, then asserted that current ESI coverage does not negate eligibility. That reverses the rule: a child covered by a group health plan is excluded from CHIP under 42 U.S.C. 1397jj(b)(1)(C)." +us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,health_coverage,False,"It counted child support in income ($61,277 instead of $60,010 MAGI), which does not change the income result, and concluded eligible from the 138%-319% FPL band alone. It never applied the rule that Child 3's employer-sponsored insurance disqualifies CHIP eligibility." +us,scenario_028,child3_chip_eligible,ox-alpha,llm_error,health_coverage,False,"It computed income correctly (~185% FPL) but explicitly declared that existing insurance coverage does not affect eligibility. Group health plan coverage is disqualifying for a targeted low-income child, so Child 3's ESI makes them CHIP-ineligible." +us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"It explicitly stated that employer-sponsored insurance does not disqualify CHIP eligibility in PA, and it wrongly claimed PA CHIP has no income limit. Federal CHIP rules exclude any child covered by a group health plan, so Child 3's ESI yields 0." +us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"It based eligibility solely on income falling within the PA CHIP range. It ignored Child 3's employer-sponsored insurance, which is disqualifying group health plan coverage under 42 CFR 457.310(b)(2)(ii)." us,scenario_028,child3_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated a higher CHIP-funded coverage threshold of 215% FPL as a Medicaid threshold. Child3's 1.82-times-FPL MAGI exceeds the applicable Medicaid limits, and CHIP eligibility cannot be substituted for Medicaid eligibility." @@ -2158,112 +2447,97 @@ us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error, us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model explicitly assumed TCJA expiration for 2026 — a $12,286 head-of-household standard deduction, pre-2018 brackets with a 15% second rate, and a $1,000-per-child CTC. For 2026 the standard deduction is $24,150, the second bracket rate is 12%, and the CTC is $2,200 per child, giving $3,949.20 of tax fully offset by $6,600 of nonrefundable credit." us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for this variable, so the failure is a missing output rather than a computational error. The requested figure is $0: $6,600 of nonrefundable CTC exceeds the $3,949.20 of tax on $35,860 of taxable income." us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used the lapsed $2,000-per-child credit and reversed the ordering, subtracting an assumed $5,100 of refundable ACTC from the $6,000 total to leave only $900 of nonrefundable CTC against a $4,065 liability. The 2026 credit is $2,200 per child, and the nonrefundable portion is applied first against the entire $3,949.20 of tax, with the ACTC being only the $2,650.80 residual." -us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its own scratch work reached about $2,916, then it submitted $5,100 — the 3 × $1,700 statutory refundable ceiling — which binds only when unused CTC exceeds it; here $3,949.20 of tax before credits leaves exactly $2,650.80 of the $6,600 CTC refundable. It also anchored the EITC on an $8,046 maximum and a $23,000 phase-out start instead of the 2026 $8,231 maximum and $23,890 HoH start, producing ~$251 rather than $624.13." -us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It used the correct 2026 three-child EITC maximum of $8,231 and the 21.06% phase-out rate but started the phase-out at $23,000 instead of the 2026 HoH figure of $23,890, cutting the credit to $436.70 instead of $624.13. Its refundable CTC of $2,644.80 is $6 low because it put tax before credits at $3,955 rather than $3,949.20." -us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It treated $1,700 × 3 = $5,100 as the refundable CTC and then reduced it to $3,850 by an invented limitation; the refundable amount is the CTC left after the nonrefundable offset, $6,600 − $3,949.20 = $2,650.80, and the 15%-of-earnings-over-$2,500 formula ($8,625) never binds. It also zeroed the EITC on the claim that $61,267 exceeds the three-child phase-out, when the 2026 HoH three-child credit runs to about $62,970 and is worth $624.13 here; child support of $1,267 is additionally not income for either credit." -us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It set the EITC to zero on a stale phase-out endpoint of ~$59,899, but the 2026 three-child HoH schedule ($8,231 maximum, 21.06% from $23,890) still pays $624.13 at $60,010. It also used a pre-OBBBA $2,000-per-child CTC ($6,000) and a $23,000 standard deduction, so its unused-CTC remainder came to $2,037 instead of $6,600 − $3,949.20 = $2,650.80." -us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning derived a refundable ACTC of about $1,200 and an EITC of zero, then it submitted $4,500, a figure no step of its own derivation produces. Both inputs to that derivation were wrong as well: the 2026 CTC is $2,200 per child ($6,600 total) against $3,949.20 of tax, leaving $2,650.80 refundable, and the three-child EITC still pays $624.13 at $60,010 because the 2026 phase-out does not close until roughly $62,970." -us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It reported the per-child refundable ceiling ($1,700 × 3 = $5,100) as the credit itself, skipping the step that makes CTC refundable only to the extent it exceeds tax: $6,600 of CTC less $3,949.20 of tax before credits gives $2,650.80. It also dismissed the EITC as income-barred, when the 2026 three-child HoH schedule ($8,231 maximum, 21.06% phase-out from $23,890) still yields $624.13 at $60,010." -us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It priced the CTC at the pre-OBBBA $2,000 per child ($6,000) and used a $22,500 HoH standard deduction, yielding $4,170 of tax and a $1,830 remainder; the 2026 values are $2,200 per child ($6,600) and a $24,500 standard deduction, giving $3,949.20 of tax and $2,650.80 refundable. It then zeroed the EITC using a 2025-based phase-out endpoint of ~$58,000–58,500, whereas the 2026 three-child HoH credit runs to about $62,970 and pays $624.13 here." -us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"After stating both the $5,100 cap and the tax-offset limit it never performed the subtraction, submitting a round $2,000 'estimate' instead of $6,600 − $3,949.20 = $2,650.80. Its EITC of zero rests on a phase-out start of ~$56,838 misused as an endpoint; the 2026 three-child HoH credit of $8,231 phases out at 21.06% from $23,890 and is still $624.13 at $60,010." -us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied a $2,000-per-child CTC against $4,170 of tax for a $1,830 remainder; with the 2026 $2,200-per-child credit ($6,600) and $3,949.20 of tax before credits the refundable portion is $2,650.80. Its EITC of about $165 understates the 2026 three-child computation, $8,231 − 21.06% × ($60,010 − $23,890) = $624.13." -us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It asserted that tax liability exceeds the CTC, but taxable income of $35,510 (AGI $60,010 less the $24,500 HoH standard deduction) produces $3,949.20 of tax against $6,600 of CTC, leaving $2,650.80 refundable. Its second claim, that the EITC is phased out at $60,000, ignores that the 2026 three-child HoH schedule does not close until about $62,970 and still pays $624.13." -us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It computed an EITC in the right neighborhood ($662.87 against the correct $624.13, from a slightly low phase-out start) but then declared the CTC fully consumed nonrefundably. Tax before credits is only $3,949.20 while the 2026 CTC totals $6,600, so $2,650.80 is refundable and its omission accounts for essentially the entire shortfall." -us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $66 CTC remainder implies tax before credits of roughly $6,534, far above the $3,949.20 that $35,510 of taxable income generates in the 10% and 12% HoH brackets; the correct remainder is $6,600 − $3,949.20 = $2,650.80. Its EITC of $524 also falls short of the $624.13 produced by the 2026 $8,231 maximum phasing out at 21.06% from $23,890." -us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It submitted zero with no computation, asserting that income level and credit structure leave nothing refundable. The 2026 $6,600 CTC exceeds $3,949.20 of tax before credits by $2,650.80, and the three-child EITC adds $624.13 at $60,010 of AGI." -us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It claimed the CTC was fully utilized against tax liability, but $6,600 of 2026 CTC against $3,949.20 of tax leaves $2,650.80 refundable — the larger of the two components it dropped. Its EITC of $574 also sits below the $624.13 from the 2026 three-child HoH parameters ($8,231 maximum, 21.06% from $23,890)." -us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 total CTC — the pre-TCJA $1,000 per child — and a tax liability of $3,039, so the credit appeared exhausted; the 2026 credit is $2,200 per child ($6,600) against $3,949.20 of tax, leaving $2,650.80 refundable. It also treated $60,010 as above the three-child EITC ceiling, when the 2026 HoH phase-out closes near $62,970 and pays $624.13 here." -us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It returned zero with a one-line assertion about phasing and liability rules and no parameters. The correct derivation is $6,600 of 2026 CTC less $3,949.20 of tax before credits, giving $2,650.80 refundable, plus a three-child EITC of $624.13." -us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It set total CTC at $6,000 ($2,000 per child) against $4,089.20 of tax to get $1,910.80; the 2026 OBBBA credit is $2,200 per child ($6,600) and tax before credits is $3,949.20, so the refundable portion is $2,650.80. It also reported no EITC, omitting the $624.13 the 2026 three-child HoH schedule still pays at $60,010." -us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It zeroed the EITC as above the phase-out and declared the CTC fully absorbed nonrefundably; the 2026 three-child HoH credit runs to roughly $62,970 and is worth $624.13 at $60,010. The $6,600 CTC also exceeds $3,949.20 of tax before credits, leaving $2,650.80 refundable." -us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $1,838.80 comes from a $6,000 CTC ($2,000 per child) net of about $4,161 of tax; the 2026 credit is $6,600 and tax before credits is $3,949.20, so the refundable portion is $2,650.80. Its EITC of $327.51 understates the $624.13 given by the 2026 $8,231 maximum phasing out at 21.06% from a $23,890 HoH start." -us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the submission never engaged the computation. The required answer is $2,650.80 of refundable CTC plus $624.13 of EITC." -us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used an $1,800-per-child refundable figure (the 2026 refundable ceiling is $1,700) and, more decisively, reported that ceiling as the credit itself rather than the CTC left after the nonrefundable offset: $6,600 − $3,949.20 = $2,650.80. Its EITC of zero also ignores that the 2026 three-child HoH phase-out ends near $62,970, leaving $624.13 at $60,010." -us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It made the entire CTC refundable at roughly $2,000 per child, ignoring both the $1,700-per-child refundable ceiling and the rule that CTC is refundable only to the extent it exceeds tax. With $6,600 of 2026 CTC and $3,949.20 of tax before credits, the refundable portion is $2,650.80, to which the EITC adds $624.13." -us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It returned zero on the bare assertion that no refundable-credit triggers apply at this income, performing no CTC or EITC computation. Three qualifying children generate $6,600 of 2026 CTC against $3,949.20 of tax, leaving $2,650.80 refundable, plus a $624.13 EITC." -us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 total CTC but put tax before credits at $2,184.20, roughly $1,765 below the $3,949.20 that $35,510 of taxable income produces, so its remainder was $4,415.80 instead of $2,650.80 — and it then reported an unexplained $4,444. Its zero EITC also drops the $624.13 the 2026 three-child HoH schedule pays at $60,010." -us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It computed the refundable CTC exactly right at $2,650.80 and then declared the EITC unavailable at this income. The 2026 three-child HoH EITC of $8,231 phases out at 21.06% from $23,890, so it is fully alive at $60,010 and worth $624.13 — the entire shortfall." -us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It reported $1,700 × 3 = $5,100 as the refundable CTC, treating the statutory ceiling as the amount; the refundable portion is the CTC remaining after tax, $6,600 − $3,949.20 = $2,650.80, well under that ceiling. It also omitted the $624.13 EITC that the 2026 three-child HoH schedule still pays at $60,010." -us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,other,False,"It asserted that no refundable credits qualify at this income without computing either component. The 2026 CTC of $6,600 exceeds $3,949.20 of tax before credits by $2,650.80, and the three-child EITC contributes $624.13 at $60,010 of AGI." -us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its ACTC of about $40 implies tax before credits near the full CTC amount, roughly $6,560, whereas $35,510 of taxable income at the 10%/12% HoH rates yields $3,949.20 and leaves $2,650.80 refundable. Its EITC of ~$660 is close, using a $24k phase-out start and ~$8,250 maximum against the 2026 values of $23,890 and $8,231 that give $624.13." -us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It left only $48 of unused CTC, which requires tax before credits of about $6,552; the actual figure is $3,949.20 on $35,510 of taxable income, so $2,650.80 of the $6,600 CTC is refundable. Its EITC of $636 is near the mark, differing only by using a $23,910 phase-out start and $8,239 maximum instead of the values producing $624.13." -us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed a post-TCJA reversion to $1,000 per child ($3,000 total) fully absorbed by tax, so it reported no refundable CTC; the 2026 OBBBA credit is $2,200 per child ($6,600) against $3,949.20 of tax, leaving $2,650.80 refundable. Its EITC of $739 also overshoots the $624.13 from the 2026 $8,231 maximum and $23,890 HoH phase-out start." -us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"It got the CTC leg essentially right — $6,600 total less a $3,951 nonrefundable offset for $2,649 — but estimated the EITC at about $340 instead of $624.13. That gap comes from an understated maximum and overstated phase-out start; the 2026 three-child HoH computation is $8,231 − 21.06% × ($60,010 − $23,890)." -us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for federal_refundable_credits. The required answer combines $2,650.80 of refundable CTC with $624.13 of EITC." -us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It reproduced the EITC exactly ($8,231 − 21.06% × ($60,010 − $23,890) = $624.13) but put federal income tax before credits at $3,838.10 rather than $3,949.20, so its unused CTC came to $2,761.90 instead of $2,650.80. The $111.10 tax understatement — a slightly off HoH standard deduction or bracket uprating for 2026 — is the whole error." -us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It reported min($8,625, $5,100) = $5,100, applying only the phase-in and per-child ceiling and never the binding constraint that CTC is refundable only above tax: $6,600 − $3,949.20 = $2,650.80. It also placed the three-child phase-out completion at ~$61,500; the 2026 HoH schedule runs to about $62,970 and still pays $624.13 at $60,010." -us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It derived the refundable CTC exactly, $6,600 − $3,949.20 = $2,650.80, so the entire miss is the EITC: it used a $22,210 phase-out start against the 2026 HoH value of $23,890 (and an $8,234 maximum against $8,231), producing $273.32 instead of $624.13. A phase-out start $1,680 too low costs 21.06% of that amount, about $354." -us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It fixed the CTC at $2,000 per child ($6,000) against $4,081.20 of tax for a $1,918.80 remainder; the 2026 credit is $2,200 per child ($6,600) and tax before credits is $3,949.20, giving $2,650.80. It then cycled through five inflation guesses for the EITC and settled on $1,042, well above the $624.13 from the actual 2026 parameters of an $8,231 maximum and a $23,890 phase-out start." -us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It called the child credit 'fully refundable' and limited by 15% of earnings over $2,500, a formula that yields $8,625 rather than the $3,200 it submitted, so the number follows from no stated rule. The CTC is refundable only above tax: $6,600 less $3,949.20 gives $2,650.80, and the 2026 three-child EITC adds $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Its own working got a total of about $2,916 (ACTC $2,665 plus EITC $251), but it submitted $5,100, the full $1,700-per-child refundable cap, which ignores the limit to the CTC left after offsetting tax. Its EITC inputs were also wrong: an $8,046 maximum and a $23,000 phase-out start in place of $8,231 and $23,890, which gave $251 instead of $624.13." +us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The ACTC was about right ($2,644.80 against $2,650.80). The EITC phase-out, however, started at $23,000 instead of the 2026 threshold of $23,890 for non-joint filers with 3 children, which understated the EITC at $436.70 against $624.13." +us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It counted nontaxable child support in income and denied the EITC entirely, even though $60,010 is below the 3-child phase-out end of about $62,974 (reference EITC $624.13). It then asserted a $3,850 ACTC without subtracting the $3,949.20 tax offset from the $6,600 CTC ($2,200 per child), which leaves $2,650.80." +us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600), which gave a $2,037 ACTC instead of $2,650.80. It also zeroed the EITC by putting the phase-out end at about $59,899, when the credit actually runs to about $62,974 and is worth $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It overstated the pre-credit tax at about $4,800 (correct: $3,949.20) against a $6,000 CTC, worked out an ACTC of about $1,200, and then submitted an unsupported $4,500. It also zeroed the EITC with a phase-out end of about $59,899, which misses the $624.13 still available at $60,010." +us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It paid the full $1,700 × 3 = $5,100 ACTC, missing that the ACTC is capped at the CTC left after offsetting tax: $6,600 minus $3,949.20 is $2,650.80. It also claimed no EITC, when $60,010 is still inside the 3-child phase-out range and yields $624.13." +us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It combined a $2,000-per-child CTC ($6,000) with a $22,500 standard deduction and 2025-level brackets, which put tax at $4,170 and the ACTC at $1,830. The OBBBA figures are $2,200 per child and $24,150, giving tax of $3,949.20 and an ACTC of $2,650.80. It also placed the 3-child EITC phase-out end at about $58,000, which zeroes the $624.13 EITC." +us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC and zeroed the EITC with an invented phase-out threshold of $56,838. The actual phase-out runs from $23,890 at 21.06% and leaves $624.13. The submitted $2,000 is a round guess, not the $6,600 − $3,949.20 = $2,650.80 unused-CTC ACTC." +us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It took the ACTC as $6,000 − $4,170 = $1,830, using the pre-OBBBA $2,000-per-child CTC and an overstated tax. The correct figures are $6,600 − $3,949.20 = $2,650.80. It also estimated the EITC at $165 instead of $624.13 ($8,231 maximum, $23,890 phase-out start)." +us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It claimed tax liability exceeds the CTC. In fact the $3,949.20 tax is well below the $6,600 CTC, which leaves a $2,650.80 refundable ACTC. It also claimed the EITC phases out by $60,000, when the 3-child EITC still pays $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The EITC was close ($662.87 against $624.13). It dropped the ACTC entirely by assuming the nonrefundable CTC was fully used, missing that the $6,600 CTC ($2,200 per child) exceeds the $3,949.20 tax by a refundable $2,650.80." +us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC ($2,000 per child) against a $4,073.20 tax, which gave a $1,926.80 ACTC instead of $6,600 − $3,949.20 = $2,650.80. Its EITC parameters were also wrong: an $8,271 maximum and a $24,024 phase-out start in place of $8,231 and $23,890, which gave $692.35 instead of $624.13." +us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $66 of leftover CTC implies a pre-credit tax of about $6,500, far above the correct $3,949.20 (taxable income of $35,860 after the $24,150 head-of-household standard deduction), so it lost most of the $2,650.80 ACTC. It also understated the EITC at $524 instead of $624.13." +us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It returned $0, missing both refundable components. The $6,600 CTC exceeds the $3,949.20 tax and leaves a $2,650.80 ACTC, and $60,010 is below the 3-child EITC phase-out end (about $62,974), leaving $624.13 of EITC." +us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It set the refundable CTC to $0 on the claim that the CTC was fully used against tax. The $6,600 CTC ($2,200 per child) exceeds the $3,949.20 pre-credit tax, and the $2,650.80 difference is refundable within the $5,100 cap. Its EITC of $574 also fell short of $624.13." +us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, the pre-TCJA amount) against a $3,039 tax, when the 2026 CTC is $2,200 per child ($6,600) and exceeds the $3,949.20 tax by a refundable $2,650.80. It also denied the EITC, which pays $624.13 at $60,010 because the 3-child phase-out ends near $62,974." +us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It returned $0 with only a generic phase-out rationale. It never computed the $2,650.80 of the $6,600 CTC left after the $3,949.20 tax offset, or the $624.13 EITC still available in the 3-child phase-out range." +us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed the ACTC from a $6,000 CTC ($2,000 per child) less a $4,089.20 tax, giving $1,910.80, instead of the OBBBA $6,600 less $3,949.20 = $2,650.80. It also left out the $624.13 EITC entirely." +us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It claimed the CTC was fully absorbed as nonrefundable and the EITC fully phased out, and both claims are false. The $6,600 CTC exceeds the $3,949.20 tax by a refundable $2,650.80, and the 3-child EITC still pays $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $1,838.80 ACTC implies a $6,000 CTC ($2,000 per child) less a $4,161.20 tax, when the correct figure is $6,600 − $3,949.20 = $2,650.80. Its EITC of $327.51 also understates the $624.13 that follows from an $8,231 maximum, a $23,890 phase-out start, and a 21.06% rate." +us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to score against the $3,274.93 reference." +us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It paid an invented $1,800 per child refundable amount ($5,400), ignoring the $1,700 cap and the rule that the ACTC cannot exceed the CTC left after offsetting tax ($6,600 − $3,949.20 = $2,650.80). It also zeroed the EITC, which pays $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated the entire $2,000-per-child CTC as refundable ($6,000). That misses both the offset against the $3,949.20 tax and the $1,700-per-child refundable cap, so the ACTC is $2,650.80. It also omitted the $624.13 EITC." +us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It returned $0 and never computed the ACTC, which is the $2,650.80 of the $6,600 CTC left after the $3,949.20 tax. It also wrongly treated the 3-child EITC as fully phased out at $60,010, where $624.13 remains." +us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but applied only $2,184.20 of pre-credit tax instead of $3,949.20 (tax on $35,860 of taxable income after the $24,150 standard deduction). That inflated the ACTC to about $4,416, and it then submitted an unexplained $4,444. It also dropped the $624.13 EITC by assuming full phase-out." +us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The refundable CTC of $2,650.80 was correct, but it declared the EITC unavailable at this income. The 3-child EITC does not fully phase out until about $62,974 ($23,890 + $8,231 / 0.2106), so it still pays $624.13 at $60,010 AGI." +us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It paid the full $1,700 × 3 = $5,100 refundable cap, missing that the ACTC is limited to the CTC left after the $3,949.20 tax offset, which is $2,650.80. It also omitted the $624.13 EITC." +us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The ACTC was correct ($2,651), but the EITC was overstated at $650 instead of $624.13. The exact 2026 figure is $8,231 − 0.2106 × ($60,010 − $23,890), and its rough estimate of the 3-child phase-out missed it by about $26." +us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It got the refundable CTC exactly right ($6,600 − $3,949.20 = $2,650.80) but left out the EITC. With 3 children, the EITC phases down from $8,231 at 21.06% above $23,890 and leaves $624.13 at $60,010." +us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It declared that no refundable credits qualify. That misses the $2,650.80 ACTC (the $6,600 CTC less $3,949.20 of tax) and the $624.13 EITC, which is still inside the 3-child phase-out range at $60,010." +us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its remaining ACTC of about $40 implies a pre-credit tax of roughly $6,560 instead of $3,949.20, which wiped out most of the $2,650.80 ACTC. Its EITC used an $8,250 maximum and a phase-out start near $24,000 (instead of $8,231 and $23,890) and came out at $660 instead of $624.13." +us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its unused CTC of $48 implies a pre-credit tax of about $6,552 against the $6,600 CTC, far above the correct $3,949.20, so it lost almost all of the $2,650.80 ACTC. Its EITC of $636 used a slightly wrong $8,239 maximum and $23,910 threshold instead of $8,231 and $23,890." +us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,other,False,"It offset $3,948 of tax against the $6,600 CTC but miscalculated the remainder as $2,952 instead of $2,652, overstating the ACTC by $300. It also overstated the EITC at $661 instead of $624.13." +us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the CTC reverts to $1,000 per child ($3,000 in total) and is fully absorbed by tax. OBBBA made $2,200 per child permanent, so the $6,600 CTC exceeds the $3,949.20 tax by a refundable $2,650.80. Its EITC also used reverted parameters ($8,302 maximum, $24,099 start) and gave $739 instead of $624.13." +us,scenario_028,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The ACTC was essentially right ($2,649 against $2,650.80). The EITC, however, came out at about $340 instead of $624.13; the $8,231 maximum, $23,890 phase-out start and 21.06% rate produce the correct figure." +us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to score against the $3,274.93 reference." +us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The EITC was exactly right at $624.13, but it put the pre-credit tax at $3,838.10 instead of $3,949.20. The correct figure is $1,770 plus 12% of the $35,860 taxable income above $17,700, after the $24,150 head-of-household standard deduction. The low tax inflated the ACTC to $2,761.90 instead of $2,650.80." +us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It paid the full $5,100 ACTC cap without limiting it to the $6,600 CTC less the $3,949.20 tax ($2,650.80). It also zeroed the EITC despite stating a phase-out end of about $61,500, which is above the $60,010 AGI; the correct EITC is $624.13." +us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The ACTC was correct at $2,650.80. The EITC phase-out, however, started at $22,210 instead of the 2026 non-joint threshold of $23,890, which cut the EITC to $273.32 instead of $624.13." +us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC ($6,000) against a $4,081.20 tax, which gave a $1,918.80 ACTC instead of $6,600 − $3,949.20 = $2,650.80. It then inflated 2024 EITC parameters by 5% to a $25,914 phase-out start and got a $1,042 EITC instead of $624.13." +us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed the earned-income limit capped the ACTC at $3,200, but 15% × ($60,000 − $2,500) = $8,625 does not bind. The binding limit is the CTC left after tax: $6,600 − $3,949.20 = $2,650.80. It also left out the $624.13 EITC." us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its explanation derived the correct components and stated the correct total twice — $3,720 Social Security + $870 Medicare + $42 PA employee UC at 0.07% = $4,632 — but the submitted value was $5,730, a figure no step in its own arithmetic produces. The substantive rule work was right; the model broke the value-must-match-explanation contract by writing a total it never computed." -us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It stopped at federal FICA ($3,720 OASDI + $870 Medicare = $4,590) and never added any state component. Pennsylvania withholds an employee unemployment compensation contribution of 0.07% on all covered wages with no wage cap, adding $42 on $60,000 and bringing the total to $4,632." -us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that PA imposes no employee-side payroll tax, characterizing PA's state levies as only a corporate net income tax and capital stock tax. Pennsylvania imposes an employee unemployment compensation contribution withheld at 0.07% of all covered wages with no wage base cap, which adds $42 to the $4,590 of federal FICA for a total of $4,632." -us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"After computing federal FICA correctly it declared that ""PA has no mandatory employee payroll tax counted here."" PA's employee unemployment compensation contribution is mandatory and employee-side, withheld at 0.07% of all wages, adding $42 to reach $4,632." -us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Its reasoning covered only employee Social Security ($3,720) and Medicare ($870), with no state-level step attempted at all. The requested output includes mandatory employee state payroll taxes, and PA's 0.07% employee unemployment compensation withholding on $60,000 adds $42 for a total of $4,632." -us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It reasoned that Pennsylvania has ""a flat state income tax but no separate employee payroll tax,"" conflating PA's 3.07% income tax withholding with its distinct unemployment insurance system. PA separately withholds a 0.07% employee UC contribution on all covered wages, adding $42 to the $4,590 FICA figure." -us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It explicitly concluded there is ""no state disability/UI employee contribution"" in Pennsylvania. PA is one of the few states with an employee-side UI contribution: 0.07% of all covered wages, uncapped, which is $42 on $60,000 and raises the total from $4,590 to $4,632." -us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It stated there is ""no PA state employee payroll tax"" and submitted federal FICA alone. Pennsylvania's employee unemployment compensation contribution of 0.07% on uncapped wages adds $42 to the $4,590, giving $4,632." -us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its derivation listed only the two federal FICA components and the Additional Medicare Tax check, omitting the state layer entirely. The PA employee unemployment compensation contribution, withheld at 0.07% of all wages, adds $42 to make $4,632." -us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It concluded there was ""no ... state payroll tax"" on top of the $3,720 Social Security and $870 Medicare amounts. PA withholds an employee unemployment compensation contribution at 0.07% of all covered wages with no cap, adding $42 for a total of $4,632." -us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its explanation named only the 6.2% Social Security and 1.45% Medicare components, with no state payroll step attempted. The requested output includes mandatory employee state payroll taxes, and PA's uncapped 0.07% employee unemployment compensation contribution adds $42 to reach $4,632." -us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It defined employee payroll tax as consisting solely of the 6.2% Social Security and 1.45% Medicare rates on $60,000. Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% on all wages adds $42, making the correct total $4,632 rather than $4,590." -us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It computed federal FICA only ($3,720 + $870) and treated that as the complete employee payroll tax. PA's employee unemployment compensation withholding of 0.07% on uncapped wages adds $42, giving $4,632." -us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key was absent from the submitted outputs object. This is a contract failure rather than a substantive computation error: the correct derivation is $3,720 Social Security + $870 Medicare + $42 PA employee unemployment compensation contribution = $4,632." -us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It stated that ""PA has no mandatory employee state payroll tax"" after correctly computing $3,720 + $870 of federal FICA. Pennsylvania withholds a mandatory employee unemployment compensation contribution at 0.07% of all covered wages, which is $42 on $60,000 and brings the total to $4,632." -us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It applied a flat 7.65% FICA rate to $60,000 and asserted that ""no mandatory state payroll tax applies in PA."" Pennsylvania's employee unemployment compensation contribution is mandatory and uncapped at 0.07%, adding $42 for a correct total of $4,632." -us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It omitted the PA employee unemployment compensation contribution of 0.07% on all wages ($42) and additionally mis-added its own federal components, since $3,720 + $870 is $4,590, not the $4,600 it reported. The correct sum of $3,720 Social Security, $870 Medicare, and $42 PA employee UC is $4,632." -us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It computed 60,000 × 0.0765 = $4,590 and explicitly excluded any ""mandatory PA employee payroll tax."" PA's employee unemployment compensation contribution of 0.07% on uncapped wages is exactly such a tax and adds $42, giving $4,632." -us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly identified the Pennsylvania employee unemployment tax as an in-scope component but priced it at $7 instead of $42, implying a rate near 0.0117% rather than the statutory 0.07% employee contribution rate on all covered wages. Applying 0.0007 × $60,000 = $42 on top of $3,720 Social Security and $870 Medicare yields $4,632." -us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated employee payroll tax as exactly 7.65% of wages, covering only the two federal FICA components. Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% on all wages adds $42, so the total is $4,632." -us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its one-line derivation applied 7.65% to $60,000 with no state component considered. The PA employee unemployment compensation contribution, withheld at 0.07% of uncapped wages, adds $42 and makes the correct answer $4,632." -us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It stated there is ""no PA employee payroll tax"" after computing $3,720 Social Security and $870 Medicare. Pennsylvania withholds an employee unemployment compensation contribution at 0.07% of all covered wages, adding $42 for a total of $4,632." -us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It concluded that ""PA has no mandatory employee state payroll tax"" and stopped at $4,590. PA's employee unemployment compensation contribution is mandatory, employee-side, uncapped, and levied at 0.07%, adding $42 to reach $4,632." -us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It listed only the two federal FICA components and the Additional Medicare Tax threshold check, never reaching a state payroll step. The 0.07% PA employee unemployment compensation contribution on $60,000 adds $42, giving $4,632." -us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"Its derivation covered employee Social Security and Medicare only, with no consideration of Pennsylvania's employee-side unemployment insurance withholding. That contribution is 0.07% of all covered wages, adding $42 to the $4,590 for a total of $4,632." -us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No numeric value or explanation was returned for payroll_tax, leaving the required key unpopulated. The substantive derivation is $3,720 employee Social Security plus $870 employee Medicare plus $42 PA employee unemployment compensation contribution = $4,632." -us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It asserted that ""no PolicyEngine-modeled employee state payroll tax applies in PA."" PolicyEngine models the Pennsylvania employee unemployment compensation contribution at 0.07% of employment income with no wage cap, which contributes $42 on $60,000 and produces $4,632." -us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It correctly computed $3,720 Social Security and $870 Medicare but declared ""PA has no mandatory employee state payroll tax."" Pennsylvania's employee unemployment compensation contribution of 0.07% on all covered wages is mandatory and adds $42, making the total $4,632." -us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Its explanation addressed only the two federal FICA rates and the $200,000 Additional Medicare threshold, omitting any state layer. PA's uncapped 0.07% employee unemployment compensation contribution adds $42, so the correct total is $4,632." -us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It reasoned explicitly that PA has ""no state disability/unemployment deduction from employees beyond income tax withholding,"" which is the exact rule it needed. Pennsylvania withholds an employee unemployment compensation contribution at 0.07% of all covered wages, adding $42 to the $4,590 of federal FICA for $4,632." -us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied the combined 7.65% federal employee rate to $60,000 and treated that as the entire payroll tax. The mandatory PA employee unemployment compensation contribution of 0.07% on uncapped wages adds $42, giving the correct total of $4,632." -us,scenario_028,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It assembled countable income correctly at $61,277 but set 185% of the 2026 four-person poverty guideline at approximately $61,100, understating the 2026 guideline level; against the actual 2026 guideline the same income is 1.82 of poverty, roughly $1,000 below the reduced-price cutoff. Its 'slightly exceeds' verdict inverted a margin that runs the other way, flipping a REDUCED tier to ineligible." -us,scenario_028,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It asserted a household of 5 (the unit is 1 adult + 3 children = 4) and then declared $61,267 'significantly exceeds' the 185% limit without ever computing that limit; at its own stated size of 5 the 185% cutoff sits far above $61,277, so its conclusion contradicts the size it chose. At the correct size of 4 the 2026 ratio is 1.82, inside the reduced-price band. It also dropped the $10 of taxable interest from countable income." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It applied $59,478, which is exactly 185% of the 2025 four-person poverty guideline of $32,150, to a tax year 2026 determination. The 2026 guideline puts the same $61,277 of countable income at a school_meal_fpg_ratio of 1.82, under the 185% reduced-price ceiling, so the one-year-stale threshold is the whole error." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used '~$57,720 for a family of 4 in 2026', which is 185% of the 2024 guideline of $31,200 — two years stale — and then described the income as 'far exceeding' that figure. Applying the 2026 guideline yields a ratio of 1.82, inside the 130%–185% reduced-price band it correctly described." -us,scenario_028,reduced_price_school_meals_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It asserted the household sits at roughly 190% of poverty; the engine's school_meal_fpg_ratio on these facts is 1.82. A ratio near 1.90 requires a four-person guideline of about $32,250 — the 2025 level — so it priced the income test off a prior-year poverty guideline instead of the 2026 one." -us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It anchored on '~$56,311' as the 2025 185% four-person figure, which is below even the true 2025 value of $59,478 and corresponds to a 2023-era guideline, then adjusted it only 'slightly' for 2026. With the 2026 guideline the correctly assembled $61,277 gives a ratio of 1.82, so the household lands in the reduced-price tier rather than above it." -us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted the unit as 5 people (it is 4: one adult and three children), stated that the 185% cutoff for a household of 5 is approximately $66,000, observed income of $61,277 — below its own cutoff — and still returned 0, contradicting its own arithmetic. At the correct household size of 4 the 2026 ratio is 1.82, which is likewise inside the reduced-price band." -us,scenario_028,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It gave a bare assertion that ~$61,267 exceeds the four-person 185% limit and dropped the $10 of interest from countable income. The 2026 guideline puts the full $61,277 at a ratio of 1.82; its verdict is consistent only with a pre-2026 four-person cutoff in the $57,720–$59,478 range." -us,scenario_028,reduced_price_school_meals_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed countable income correctly at $61,277 but named no threshold, asserting only that it exceeds the four-person 185% limit. Against the 2026 poverty guideline that income is 1.82 of poverty, so the answer tracks a stale 2024–2025 cutoff ($57,720–$59,478) rather than the tax-year-2026 guideline the question specifies." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It placed 185% of the 2026 four-person guideline at approximately $61,200 and called $61,277 a narrow miss; that estimate understates the 2026 guideline, which leaves the household at a ratio of 1.82 with about a thousand dollars of headroom. The reduced-price determination hinged on a projection error of roughly two percent in the guideline level." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It supplied no threshold and no income figure, asserting only that income exceeds the reduced-price limit. The correct derivation is $60,000 wages + $1,267 child support + $10 interest = $61,277 against the 2026 four-person guideline, a ratio of 1.82 that lands in the reduced-price band; a 0 answer is consistent only with a pre-2026 185% cutoff below $61,277." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly included child support to reach $61,277 but named no 185% figure, asserting the income exceeds it. Under the 2026 guideline that income is 1.82 of poverty, so the verdict reflects a stale four-person cutoff ($57,720 for 2024 or $59,478 for 2025) carried into a 2026 determination." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave a bare assertion that income exceeds 185% of the federal poverty guideline, with no income total and no threshold. The engine's school_meal_fpg_ratio on these facts is 1.82 against the 2026 four-person guideline; the 0 answer matches a prior-year 185% cutoff in the $57,720–$59,478 range." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used '~$57,720 for a family of 4', which is 185% of the 2024 poverty guideline of $31,200, for a 2026 determination, and also omitted the $10 of interest. The 2026 guideline puts $61,277 at a ratio of 1.82, inside the reduced-price band." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It asserted the income exceeds the 185% threshold without stating either the income or the threshold. Correctly derived, countable income of $61,277 against the 2026 four-person guideline is a ratio of 1.82, so the answer is consistent only with applying a 2024- or 2025-vintage 185% cutoff instead of the 2026 one." -us,scenario_028,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It offered only the conclusion that income exceeds 185% of poverty, with no figures. The 2026 four-person guideline gives a school_meal_fpg_ratio of 1.82 for this $61,277 of countable income; a 0 verdict requires a 185% cutoff below $61,277, which only pre-2026 guidelines produce." -us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for reduced_price_school_meals_eligible, so the required key never reached the submit_outputs contract. No substantive reasoning about the 185% income test was produced to evaluate." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It stated only that household income is 'above the reduced-price meal threshold' without computing either side of the comparison. Countable income of $61,277 against the 2026 four-person poverty guideline is 1.82 of poverty, inside the 185% band; the 0 answer matches a stale threshold of $57,720–$59,478." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-nano,llm_error,other,False,"It never ran the income test, treating the output as requiring an explicit eligibility indicator in the household facts and defaulting to 0 because none was listed. The test is computable from the given facts: $61,277 of countable income against the 2026 four-person guideline is a ratio of 1.82, which sets school_meal_tier to REDUCED." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly counted child support in school-meal income and correctly used a four-person unit, then asserted the total exceeds the reduced-price threshold without naming it. Against the 2026 guideline the ratio is 1.82, so the verdict rests on a prior-year 185% cutoff rather than the 2026 level." -us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It totaled income correctly at $61,277 and called it 'slightly above the estimated 185% federal-poverty threshold'; that estimate undershoots the 2026 four-person guideline, which puts the same income at 1.82 of poverty. The determination failed on the guideline projection, not the income build-up." -us,scenario_028,reduced_price_school_meals_eligible,gpt-6-astra,llm_error,thresholds_rates,False,"It built school-meal countable income correctly at $61,277 and applied the right unit size of four, but compared it to a four-person 185%-of-poverty limit set below the 2026 level. The 2026 guideline yields a school_meal_fpg_ratio of 1.82, placing the household in the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,grok-4.3,llm_error,thresholds_rates,False,"It gave a one-clause assertion that income is above the reduced-price threshold, with no income figure and no cutoff. The 2026 four-person guideline puts $61,277 at 1.82 of poverty, under the 185% ceiling; the 0 answer is consistent only with a 2024- or 2025-vintage cutoff of $57,720–$59,478." -us,scenario_028,reduced_price_school_meals_eligible,grok-4.5,llm_error,thresholds_rates,False,"It correctly included child support to reach $61,277 and used a four-person unit, then declared the income above 185% FPL without stating the threshold. Under the 2026 guideline that income is 1.82 of poverty, so the error is a stale poverty-guideline level, not the income computation." -us,scenario_028,reduced_price_school_meals_eligible,grok-4.6,llm_error,thresholds_rates,False,"It explicitly invoked a 'projected 2026 FPG' for a four-person unit and concluded $61,277 exceeds 185% of it; its projection is too low, since the actual 2026 guideline gives a ratio of 1.82. It was right that the household lacks categorical eligibility, which is precisely why the understated guideline decided the answer." -us,scenario_028,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It computed the ratio as approximately 186% of the 2026 four-person guideline; the engine's school_meal_fpg_ratio is 1.82. Its number implies a guideline of about $32,950, roughly two percent below the 2026 level, and that two-percent shortfall is the entire distance between its ineligible verdict and the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,inkling,llm_error,thresholds_rates,False,"It used 'about $59,500 for 4' as the 185% cutoff, which is the 2025 figure ($59,478 = 185% of $32,150), for a tax year 2026 determination. The 2026 guideline puts the same $61,277 at a ratio of 1.82, inside the reduced-price band." -us,scenario_028,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"Its income build-up was exact — $60,000 wages + $1,267 child support + $10 interest = $61,277, all correctly countable for school meals — but it applied a 'projected' 2026 four-person 185% limit set below that amount. The actual 2026 guideline yields a school_meal_fpg_ratio of 1.82, so the household qualifies for reduced-price meals." -us,scenario_028,reduced_price_school_meals_eligible,kimi-k3,llm_error,thresholds_rates,False,"It cited a cutoff of '~$59,478 for four', which is 185% of the 2025 guideline of $32,150, and derived a ratio of about 191%; the 2026 guideline gives 1.82. Using a one-year-stale poverty guideline inflated the ratio by roughly nine points and pushed a reduced-price household out of the band." -us,scenario_028,reduced_price_school_meals_eligible,minimax-m3,llm_error,thresholds_rates,False,"It applied '~$57,700' as 185% of the four-person poverty guideline, which is the 2024 value ($57,720 = 185% of $31,200), two years before the tax year in question. Against the 2026 guideline the correctly totaled $61,277 is 1.82 of poverty, placing the household in the REDUCED tier." -us,scenario_028,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It declared a household of 5 (the unit is 4: one adult and three children) and then applied a $59,000–$61,000 threshold band that corresponds to prior-year four-person guidelines, so both the unit size and the cutoff were wrong. The 2026 four-person guideline puts its own correctly computed $61,277 at a ratio of 1.82, inside the 185% reduced-price limit." -us,scenario_028,reduced_price_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It asserted only that income is above the reduced-price income limit, giving neither the income nor the limit. Countable income of $61,277 against the 2026 four-person poverty guideline is a school_meal_fpg_ratio of 1.82; a 0 verdict requires a 185% cutoff below $61,277, which only pre-2026 guidelines supply." +us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got the right parts: $3,720 Social Security, $870 Medicare and $42 PA employee UC (0.07% of $60,000), and it stated the total as $4,632. It then submitted 5,730, a number its own derivation never reaches. The miss is a transcription or consistency error at the output step, not a policy error." +us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only employee OASDI plus Medicare ($4,590). It left out Pennsylvania's mandatory employee unemployment compensation withholding of 0.07% of gross wages, which adds $42." +us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It said PA has no employee payroll tax and pointed to PA's corporate taxes. Pennsylvania in fact withholds an employee UC contribution of 0.07% of all gross wages, which is $42 on $60,000, so it undercounted by $42." +us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said 'PA has no mandatory employee payroll tax counted here' and stopped at FICA of $4,590. It left out Pennsylvania's employee UC withholding of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It summed only employee Social Security and Medicare ($4,590). It never added Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% x $60,000 = $42." +us,scenario_028,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It counted only the 7.65% FICA ($4,590). It left out Pennsylvania's employee UC withholding of 0.07% of gross wages, which adds $42 for a total of $4,632." +us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It said Pennsylvania has 'no separate employee payroll tax'. PA does withhold an employee unemployment compensation contribution of 0.07% of total gross wages ($42 here), which the output definition's mandatory state payroll taxes include." +us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It said Pennsylvania has 'no state disability/UI employee contribution'. PA is one of the states that collects an employee UI contribution, 0.07% of all gross wages, so it missed $42." +us,scenario_028,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It said no PA employee state payroll tax is modeled and stopped at $4,590. It left out the PA employee UC contribution of 0.07% x $60,000 = $42." +us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It said there is no PA state employee payroll tax. Pennsylvania withholds an employee unemployment compensation contribution of 0.07% of gross wages, so it undercounted by $42." +us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare ($4,590). It never added Pennsylvania's mandatory employee UC withholding of 0.07% of $60,000 ($42)." +us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,It said there is 'no ... state payroll tax'. Pennsylvania's employee unemployment compensation contribution of 0.07% of gross wages is a mandatory employee state payroll tax worth $42 here. +us,scenario_028,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at FICA of $4,590. It left out Pennsylvania's employee UC withholding (0.07% x $60,000 = $42), which the payroll_tax definition includes." +us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It computed only 6.2% Social Security and 1.45% Medicare ($4,590). It left out the Pennsylvania employee unemployment compensation contribution of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only FICA ($4,590). It never considered mandatory employee state payroll taxes, so it missed Pennsylvania's 0.07% employee UC withholding, $42 on $60,000." +us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It computed employee FICA only ($4,590). It left out Pennsylvania's mandatory employee UC contribution of 0.07% of gross wages, which adds $42." +us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so nothing could be scored. The correct derivation is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632." +us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It said 'PA has no mandatory employee state payroll tax'. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all gross wages, so it missed $42." +us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It said 'no mandatory state payroll tax applies in PA' and stopped at 7.65% FICA. PA's employee UC withholding of 0.07% x $60,000 = $42 applies." +us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It made an arithmetic error: 6.2% and 1.45% of $60,000 sum to $4,590, but it reported $4,600. It also left out Pennsylvania's employee UC withholding of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It explicitly excluded any 'mandatory PA employee payroll tax' and computed 7.65% x $60,000 = $4,590. Pennsylvania's employee UC contribution of 0.07% of gross wages ($42) is exactly such a tax." +us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It correctly included a PA employee unemployment tax but set it at $7. That is 0.07% of the $10,000 employer taxable wage base, not of total wages. Pennsylvania's employee UC contribution has no wage cap and applies to all $60,000 of gross wages, which gives $42." +us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It computed only 7.65% FICA ($4,590). It left out Pennsylvania's mandatory employee UC withholding of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It treated the absence of an explicitly listed state payroll tax as meaning none applies. Pennsylvania's statutory employee UC contribution of 0.07% of gross wages applies to every PA wage earner, adding $42." +us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% employee FICA ($4,590). It left out Pennsylvania's employee UC withholding of 0.07% x $60,000 = $42." +us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It said there is 'no PA employee payroll tax'. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all gross wages, so it missed $42." +us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax. The PA employee UC contribution of 0.07% of gross wages is mandatory and adds $42 to the $4,590 of FICA." +us,scenario_028,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee state payroll tax. Pennsylvania withholds 0.07% of gross wages as an employee UC contribution ($42 on $60,000)." +us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It summed only Social Security and Medicare ($4,590). It left out Pennsylvania's employee unemployment compensation withholding of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"It computed employee FICA only ($4,590). It never added Pennsylvania's mandatory employee UC contribution of 0.07% x $60,000 = $42." +us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so nothing could be scored. The correct derivation is $3,720 Social Security + $870 Medicare + $42 PA employee UC = $4,632." +us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It said 'no PolicyEngine-modeled employee state payroll tax applies in PA'. PolicyEngine does model Pennsylvania's employee UC contribution at 0.07% of gross wages, which adds $42." +us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It said PA has no mandatory employee state payroll tax. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all wages, so it missed $42." +us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It counted only employee Social Security and Medicare ($4,590). It left out Pennsylvania's mandatory employee UC withholding of 0.07% of gross wages ($42)." +us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It said PA has 'no state disability/unemployment deduction from employees'. Pennsylvania does deduct an employee UC contribution of 0.07% of total gross wages, which is $42 on $60,000." +us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It computed 7.65% FICA only ($4,590). It left out Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% of gross wages ($42)." +us,scenario_028,reduced_price_school_meals_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It counted the $61,277 correctly and first found it above the 185% line. It then switched to an assumed uprated 2026 guideline with 185% at about $61,300. 185% of the $33,000 guideline for four is $61,050, so $61,277 exceeds the reduced-price limit by $227 and the household does not qualify." +us,scenario_028,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used $60,010 of income (wages plus interest) and left out the $1,267 of child support received, which counts as school-meal income. Countable income is $61,277, above the $61,050 threshold it cited itself, so the household is over 185% of the guideline and not eligible." +us,scenario_028,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It counted $61,277 correctly but set 185% of the guideline at about $61,975, which implies a guideline of roughly $33,500. The correct limit is 185% of $33,000, or $61,050, and $61,277 exceeds it (ratio 1.86), so the household is not eligible for reduced-price meals." +us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"It returned no usable value or explanation for reduced_price_school_meals_eligible. The correct derivation counts $61,277 of wages, interest and child support. That is 1.86 times the $33,000 guideline, above the 185% reduced-price limit, so the answer is 0." +us,scenario_028,reduced_price_school_meals_eligible,glm-5.3,llm_error,thresholds_rates,False,"It counted about $61,300 (wages, child support and interest) but said this was just under 185% of the 2026 poverty line. 185% of the $33,000 guideline for four is $61,050, so $61,277 is above the reduced-price cutoff and the household is not eligible." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It counted $61,277 correctly but said this was within the reduced-price limit for four people. The limit is 185% of $33,000, or $61,050. Income exceeds it by $227 (1.86 times the guideline), so no reduced-price support applies." +us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said, without any figures, that income was within the reduced-price threshold for four, so its comparison put income at or below 185% of the guideline. Countable income including the $1,267 of child support is $61,277. That is 1.86 times the $33,000 guideline and above the $61,050 cutoff, so the household is not eligible." +us,scenario_028,reduced_price_school_meals_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used $60,010 of income and got 181% of the guideline (dividing by the correct $33,000), but it left out the $1,267 of child support received, which counts as school-meal income. With child support, income is $61,277, or 186% of the guideline, above the 185% reduced-price limit." us,scenario_028,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_028,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -2277,44 +2551,49 @@ us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_erro us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted a Pennsylvania ""standard deduction and nonrefundable credits"" and returned $3,480, which implies an effective rate of 5.8% on $60,010 — nearly double Pennsylvania's statutory flat rate. Pennsylvania levies a single 3.07% rate on the eight classes of income with no standard deduction, giving $1,842.31." us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model equated PA taxable income with federal AGI and then subtracted a $17,800 head-of-household standard deduction, a purely federal construct; Pennsylvania has no standard deduction and no filing-status-based allowances. Its base is the sum of the PA income classes — $60,000 compensation plus $10 interest — taxed in full at 3.07% for $1,842.31." us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model correctly derived $1,842.31 and then subtracted $1.55 for the PA Child and Dependent Care Enhancement Tax Credit, which is a refundable credit reported under state_refundable_credits and therefore excluded from this before-refundable-credits output. The credit is also zero for this household: it is a percentage of the federal CDCC, and no childcare expenses are listed, so the federal credit base is $0." -us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model denied the existence of Pennsylvania's refundable EITC match outright (""no refundable individual income tax credits like a state EITC"") and then spent its reasoning on Tax Forgiveness, which is a nonrefundable Schedule SP credit and irrelevant to this output line. The required step is the state match on the federal credit: this HOH filer with three qualifying children and $60,010 AGI still earns $624.13 of federal EITC inside the 2026 phase-out range, and 10% of that is $62.41." -us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only one Pennsylvania refundable credit — the Child and Dependent Care Enhancement Credit — correctly zeroed it for lack of child care expenses, and stopped, never reaching the state EITC match. That match is the whole value: 10% of the household's residual federal EITC of $624.13 = $62.41." -us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model recognized a Pennsylvania earned income credit but mislabeled its refundability, asserting ""Pennsylvania's earned income tax credit (PA EIC) is non-refundable"" and therefore excluding it from the refundable-credit total. The credit is refundable and is computed as a percentage of the federal EITC, so with a federal EITC of $624.13 the state refundable total is $62.41, not $0." -us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model's only substantive check was the Tax Forgiveness phase-out (""too high for PA's Tax Forgiveness credit""), a nonrefundable provision that has no bearing on the refundable-credit line, and it dismissed all other PA credits without evaluating the federal-EITC match. Running that match yields 10% × $624.13 = $62.41, since $60,010 of AGI with three qualifying children leaves a residual 2026 federal EITC rather than zero." -us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model asserted that Tax Forgiveness is ""Pennsylvania's main refundable state income tax credit mechanism"" and, after correctly finding it unavailable at $60,010, concluded PA has no other refundable individual credit. It missed the state EITC, which is refundable and set as a percentage of the federal credit — 10% × $624.13 = $62.41 for this three-child head-of-household." -us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model gave a bare denial (""PA has no applicable refundable income tax credits for this household"") with no enumeration of PA credits and no computation. The answer is consistent with skipping the state EITC match entirely; the correct derivation carries the household's $624.13 federal EITC through the 10% Pennsylvania match to $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model made the flat factual claim that ""Pennsylvania does not have a state Earned Income Tax Credit,"" which is the exact credit that produces this value, and reasoned no further than PA's 3.07% flat rate. Pennsylvania's refundable EITC is a percentage of the federal credit, so $624.13 federal × 10% = $62.41." -us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model declared that PA offers no state-EITC-comparable refundable credit and substituted a Tax Forgiveness phase-out analysis — which it also mis-sized as a ""family of 5"" when the unit is a head of household plus three children — instead of matching the federal EITC. The refundable line is 10% of the federal EITC of $624.13, which the household retains because $60,010 of AGI falls inside the 2026 three-child phase-out range, giving $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model tied its zero to the household's income level (""no refundable income tax credits for this household at this income level""), but the Pennsylvania credit has no independent income test — it is a flat percentage of whatever federal EITC survives the federal phase-out. That federal credit is $624.13 here, so the state refundable total is $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model evaluated only the Tax Forgiveness income thresholds and concluded zero from that nonrefundable program, never touching the refundable state EITC match. Applying the match to the household's residual 2026 federal EITC of $624.13 gives 10% × $624.13 = $62.41." -us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model issued a one-line denial that Pennsylvania provides any refundable income tax credit, with no credit enumerated and no arithmetic. The state credit is a percentage of the federal EITC, and this three-child head-of-household keeps $624.13 of federal EITC at $60,010 AGI, producing $62.41." -us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model asserted that no PA refundable credit applied ""to this household's income level and profile"" for 2026, treating the state refundable line as income-gated in its own right. The credit is instead a 10% match on the federal EITC, which is $624.13 for three qualifying children at $60,010 of AGI, so the answer is $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model gave a bare assertion that no state-level refundable credits apply, without identifying a single Pennsylvania credit or computing the federal EITC that feeds the state match. Carrying $624.13 of federal EITC through Pennsylvania's percentage match yields $62.41." -us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model checked exactly one PA refundable credit, the Child and Dependent Care Enhancement Credit, correctly zeroed it for absence of child care expenses, and waved off ""other state refundable credits"" without evaluating the state EITC. That EITC match is the sole source of the value: 10% × $624.13 = $62.41." -us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stated categorically that Pennsylvania offers no refundable state income tax credit this household qualifies for, skipping the state EITC match. With three qualifying children and $60,010 AGI the household retains $624.13 of federal EITC, and the 10% Pennsylvania match makes the state refundable total $62.41." -us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model produced an unsupported blanket denial for the ""household profile"" with no credit named and no derivation. The correct computation is the state percentage match on the surviving federal EITC of $624.13, giving $62.41." -us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model concluded the household does not qualify for any PA refundable credit without applying the state EITC match, which requires only a nonzero federal EITC. The household's federal EITC is $624.13, so 10% of it — $62.41 — is the refundable state total." -us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model asserted non-qualification in a single clause with no credit identified and no arithmetic behind it. The state credit rides on the federal EITC, which is $624.13 for this three-child head-of-household at $60,010 AGI, producing a $62.41 state refundable credit." -us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model flatly denied eligibility for any Pennsylvania refundable income tax credit without evaluating the state EITC match. The match is a fixed percentage of the federal credit, so $624.13 × 10% = $62.41." -us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so the key was absent from its submitted outputs rather than substantively wrong. The graded quantity is Pennsylvania's refundable EITC match, 10% of the household's $624.13 federal EITC, or $62.41." -us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model surveyed only the Property Tax/Rent Rebate — a standalone rebate program, not an income tax credit — and used its age-65-or-disability requirement to justify zero, never considering the state EITC match. The refundable income tax credit here is 10% of the federal EITC of $624.13, i.e. $62.41." -us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model treated the state refundable credit as requiring an explicit triggering fact (""no refundable Pennsylvania income tax credits are indicated by the household facts""), when the trigger is simply a nonzero federal EITC, which $60,000 of wages with three qualifying children produces. The federal EITC is $624.13 and the state match is 10%, so the answer is $62.41." -us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model reasoned that no indicator showed ""income low enough for major refundable credits,"" but $60,010 of AGI with three qualifying children is inside the 2026 federal EITC phase-out range, which runs to roughly $63,000, leaving $624.13 of federal credit. Pennsylvania matches 10% of that, so the state refundable total is $62.41." -us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for refundable-credit inputs such as child care expenses and, finding none, concluded no PA refundable credit is ""supported by the facts."" It missed that the state EITC match needs no separate input beyond earned income and qualifying children: 10% × $624.13 federal EITC = $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model dismissed all Pennsylvania refundable credits as inapplicable to ""the reported household facts"" without naming or testing the state EITC. Those facts — $60,000 of wages and three qualifying children — generate a $624.13 federal EITC, and the 10% state match makes the answer $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model returned a one-line denial that any PA refundable individual income tax credit applies, performing no federal-EITC computation to feed the state match. Completing that step gives 10% of $624.13, or $62.41." -us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model rejected any PA refundable credit ""at this income,"" applying an income screen that the state EITC match does not have — the match keys only off the federal credit amount. The federal EITC survives at $624.13 for this three-child head-of-household, so the state refundable total is $62.41." -us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model answered with the three-word claim ""No state refundable credits,"" providing no enumeration of Pennsylvania credits and no computation. The state EITC match on the household's $624.13 federal EITC produces $62.41." -us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model asserted that no PA personal-income-tax refundable credit applies without evaluating the state EITC, which is refundable and set as a percentage of the federal credit. With $624.13 of federal EITC, the state credit is $62.41." -us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model claimed Pennsylvania has no refundable individual income tax credit applicable to this household, omitting the state EITC match entirely. That match yields 10% × $624.13 = $62.41 because the household's $60,010 AGI leaves a residual federal EITC with three qualifying children." -us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model explicitly justified zero on the ground that PA has ""no state EITC,"" which is precisely the credit that generates the reference value. The state EITC is 10% of the federal credit, so $624.13 × 10% = $62.41." -us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model reasoned entirely within Tax Forgiveness — correctly noting it is nonrefundable and unavailable at this income — and treated that as disposing of the whole refundable-credit line. The state EITC is a separate refundable credit computed as 10% of the federal EITC, giving $62.41 on a $624.13 federal credit." -us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model tested only Pennsylvania's Tax Forgiveness limits against roughly $60,010 of income and then asserted no other refundable state credit applies. The state EITC match applies without any additional income test: 10% of the household's $624.13 federal EITC is $62.41." -us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model based its zero on the categorical claim that Pennsylvania has ""no state EITC or similar refundable credit,"" which is the exact provision producing the value. Matching 10% of the $624.13 federal EITC gives $62.41." -us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model hedged that PA has ""limited refundable income tax credits"" and dismissed all of them from the listed facts without computing the federal EITC that drives the state match. Those facts yield a $624.13 federal EITC, and the 10% state match makes the refundable state total $62.41." -us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model restricted its analysis to Tax Forgiveness at $60,010 of eligibility income and generalized that result to all Pennsylvania refundable credits. The state EITC match is independent of Schedule SP: $624.13 federal EITC × 10% = $62.41." -us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model stated directly that Pennsylvania ""does not offer a refundable Earned Income Tax Credit... at the state level,"" denying the very credit that produces the reference value. Pennsylvania's refundable EITC is a percentage of the federal credit, and with $624.13 of federal EITC the state amount is $62.41." -us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model gave a single-sentence denial with no credit named and no derivation supporting it. The correct path is the state EITC match on the household's surviving federal EITC of $624.13, which gives $62.41." +us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit such as a state EITC and looked only at nonrefundable Tax Forgiveness. It missed PA's refundable EITC, which is 10% of the federal EITC. The household's $624.13 federal EITC (three children, $60,000 earnings) gives a PA credit of $62.41." +us,scenario_028,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only the PA child and dependent care credit and dropped it because there are no child care expenses. It missed PA's refundable EITC, which is 10% of the household's $624.13 federal EITC, or $62.41." +us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model knew PA has an earned income tax credit but wrongly called it nonrefundable. PA's EITC is refundable and equals 10% of the federal EITC. With a $624.13 federal EITC, this household gets $62.41." +us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated Tax Forgiveness as PA's only relevant credit and said no other refundable credit applies. It missed the refundable PA EITC at 10% of the federal EITC. The household still has a $624.13 federal EITC in the phase-out range, so the PA credit is $62.41." +us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model called Tax Forgiveness PA's main refundable mechanism and said no other refundable credit exists. It missed the refundable PA EITC, which is 10% of the federal EITC ($624.13 × 10% = $62.41)." +us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies and missed PA's refundable EITC at 10% of the federal EITC. This head of household with three children has a $624.13 federal EITC, which yields $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated outright that Pennsylvania has no state EITC. That is wrong: PA now has a refundable EITC equal to 10% of the federal EITC. Here that is 10% of $624.13, or $62.41." +us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model said PA has no refundable credit comparable to a state EITC and looked only at Tax Forgiveness. It missed PA's refundable EITC at 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It never applied PA's refundable EITC at 10% of the federal EITC, which is $62.41 given the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model said PA offers no refundable credit at this income level. It missed that PA's refundable EITC is 10% of the federal EITC, and the federal EITC is still $624.13 at $60,000 of earnings with three children. That makes the PA credit $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness income limits, and it also miscounted the household as four people. It never applied PA's refundable EITC at 10% of the $624.13 federal EITC, which is $62.41." +us,scenario_028,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said PA provides no refundable credit here. It missed the refundable PA EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said PA has no refundable individual income tax credits at all. That ignores PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this household." +us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model said no PA refundable credit applies at this income level. It missed the refundable PA EITC at 10% of the federal EITC. The household's federal EITC is $624.13 near the end of the phase-out, so the PA credit is $62.41." +us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no state refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 from a $624.13 federal EITC." +us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model looked only at the PA child and dependent care enhancement credit and found no child care expenses. It missed the refundable PA EITC at 10% of the federal EITC, which is 10% of $624.13, or $62.41." +us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It missed the PA EITC, a refundable credit equal to 10% of the federal EITC, which yields $62.41 here." +us,scenario_028,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model said no state refundable credit is available. It missed PA's refundable EITC at 10% of the federal EITC, and the household's $624.13 federal EITC gives $62.41." +us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this three-child household with a $624.13 federal EITC." +us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said the household qualifies for no PA refundable credit. It never computed PA's refundable EITC at 10% of the federal EITC, which is $62.41." +us,scenario_028,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model said the household is ineligible for any PA refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the $624.13 federal EITC." +us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_refundable_credits, so there was nothing to score. The correct value is 10% of the household's $624.13 federal EITC, or $62.41, from PA's refundable EITC." +us,scenario_028,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model looked only at the Property Tax/Rent Rebate, which requires age 65 or a disability, and concluded PA has no refundable credit. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here." +us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said the household facts support no PA refundable credit. The facts do support one: three qualifying children and $60,000 of earnings give a $624.13 federal EITC. PA's refundable EITC is 10% of that, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed the income was too high for any major refundable credit. At $60,000 of earnings with three children, the household is still in the federal EITC phase-out with a $624.13 credit. PA's refundable EITC pays 10% of that, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for child care expenses or other special inputs and found none. It missed that PA's refundable EITC depends only on the federal EITC, which this household receives ($624.13). That makes the PA credit 10% of it, or $62.41." +us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It never applied PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this household." +us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model said no PA refundable credit applies at this income. It missed that the household still gets a $624.13 federal EITC, and PA's refundable EITC pays 10% of it, or $62.41." +us,scenario_028,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model used the right rule, PA's refundable EITC at 10% of the federal EITC, but guessed the federal EITC at about $650. The correct federal EITC is $624.13, from the 2026 three-child phase-out at $60,010 of AGI. Its $65 answer is 10% of that overestimate, not the correct $62.41." +us,scenario_028,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said there are no state refundable credits. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here." +us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said no refundable PA personal income tax credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 from the $624.13 federal EITC." +us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model said Pennsylvania has no refundable individual income tax credit that applies. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41." +us,scenario_028,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model stated outright that PA has no refundable earned income credit and looked only at Tax Forgiveness. PA's refundable EITC pays 10% of the federal EITC, which is $62.41 for this household." +us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model stated outright that PA has no state EITC. PA's refundable EITC is 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness and dropped it as nonrefundable and income-ineligible. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here." +us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness income limits, and it also miscounted the household as four people. It missed PA's refundable EITC at 10% of the household's $624.13 federal EITC, which is $62.41." +us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model stated outright that PA has no state EITC or any refundable credit. PA's refundable EITC is 10% of the federal EITC, which is $62.41 for this household." +us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model said none of PA's limited refundable credits apply. It missed PA's refundable EITC at 10% of the federal EITC, which yields $62.41 from the $624.13 federal EITC." +us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model looked only at Tax Forgiveness eligibility at $60,010. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 here." +us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model stated outright that PA has no refundable state EITC. PA's refundable EITC is 10% of the federal EITC, which gives $62.41 from the household's $624.13 federal EITC." +us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said no PA refundable credit applies. It missed PA's refundable EITC at 10% of the federal EITC, which is $62.41 for this three-child household." us,scenario_028,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It stated ""AGI is $50,312 ($50,000 taxable financial assistance + $312 taxable interest),"" treating the non-taxable financial-assistance input as ordinary taxable income when that variable enters no AGI component, leaving actual AGI at $312 of taxable interest alone. It compounded this by applying a pre-TCJA 2026 schedule — an $8,600 single standard deduction with 10%/15% brackets — instead of the 2026 single standard deduction of about $16,100 under the permanent OBBBA rates. Either correction alone puts the $312 of interest below the standard deduction, yielding $0 rather than $5,676.80." us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It explicitly ""treated [the $50,000 financial assistance] as taxable income for federal tax purposes"" and added it to the $312 of interest, but financial assistance is a non-taxable receipt that contributes nothing to AGI, so the return reports only $312 of taxable interest. Its $5,236 is consistent with running a bracket schedule on roughly $45,700 of taxable income after subtracting a standard deduction from a $50,312 base. The correct derivation is $312 AGI minus the 2026 single standard deduction of about $16,100, giving $0 taxable income and $0 tax before refundable credits." @@ -2343,44 +2622,50 @@ us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model skip us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model built Oklahoma taxable income of $50,312 by adding the $50,000 of financial assistance to the $312 of taxable interest, but financial assistance is a non-taxable transfer that is excluded from federal AGI, which is Oklahoma's starting point, so Oklahoma AGI is $312. It compounded this with an $8,600 standard deduction in place of Oklahoma's single standard deduction plus the $1,000 personal exemption, then ran the graduated 0.25%–4.75% schedule over $41,712 of income that does not exist; with only $312 of interest, Oklahoma taxable income is zero and the tax is $0.00." us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no arithmetic, but $1,827 is what Oklahoma's graduated schedule yields on roughly $43,000 of taxable income — $50,312 of gross income less about $7,350 of standard deduction and personal exemption — which is reachable only by treating the $50,000 of financial assistance as taxable income. Financial assistance is non-taxable and never enters federal AGI or Oklahoma AGI; the sole taxable item is $312 of interest, which falls below Oklahoma's standard deduction and produces $0.00 of tax." us,scenario_029,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model's $36,662 base is exactly $50,312 minus a $13,650 deduction, showing it counted the non-taxable $50,000 financial assistance as Oklahoma taxable income and then subtracted a federal-style standard deduction rather than Oklahoma's single standard deduction plus $1,000 personal exemption. Excluding the non-taxable assistance leaves $312 of taxable interest, below every Oklahoma deduction and filing threshold, so the correct liability is $0.00 instead of $1,160." -us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It identified the correct credit and structure — $40 per exemption under a $20,000 ceiling for a childless under-65 filer — then pulled the $50,000 financial assistance into Oklahoma gross household income and declared the ceiling breached. That assistance is not Oklahoma gross household income; the countable figure is the $312 of taxable interest, so the single exemption pays the full $40, and its parallel OK EITC analysis is irrelevant because the sales tax relief credit has no earned-income component." -us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that Oklahoma has no refundable individual income tax credits and that none are structured for general income. Oklahoma's sales tax relief credit is refundable and pays $40 per exemption on gross household income under $20,000, a test the $312 of taxable interest clears." -us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It named the sales tax relief credit and its ~$20,000 single-filer ceiling, then counted the $50,000 financial assistance and the $54,985 of bank and stock assets against that ceiling. Neither enters Oklahoma's gross household income test — countable income is $312 — and the credit carries no asset test, so one exemption yields $40." -us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It conditioned the sales tax relief credit on having earned income and treated the filer's assets and financial assistance as disqualifying. The credit turns solely on gross household income under $20,000 — here $312 — and pays $40 per exemption whether the income is earned or unearned, with no asset test." -us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It required earned income or qualifying dependents and asserted the sales tax relief thresholds were unmet. The credit requires neither: $312 of Oklahoma gross household income sits far under the $20,000 ceiling, and the single exemption pays $40." -us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Its enumeration of Oklahoma refundable credits stopped at the state EITC and it concluded no other refundable credit applied given no childcare expenses and no child tax credit base. It omitted the sales tax relief credit, which pays $40 per exemption to a resident with $312 of gross household income against the $20,000 ceiling and requires no wages, dependents, or expenses." -us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It reasoned only about the Oklahoma EITC and concluded no other general refundable Oklahoma credit reaches a childless 21-year-old with no earned income and no elderly or disability status. The sales tax relief credit is precisely such a credit — $40 per exemption for any Oklahoma resident with gross household income under $20,000, met by the $312 of taxable interest." -us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It denied the credit on the grounds of zero Oklahoma tax liability and unspecified unmet qualifying conditions. The sales tax relief credit is refundable and paid without regard to liability; $312 of gross household income under the $20,000 ceiling delivers $40 for the one exemption." -us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It evaluated only Oklahoma's EITC at 5% of a $0 federal EITC and asserted no other state refundable credit applies. The sales tax relief credit pays $40 per exemption on $312 of gross household income against the $20,000 ceiling and has no earned-income requirement." -us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It tied Oklahoma's refundable credits to the $0 federal EITC and stopped there. The refundable sales tax relief credit is independent of the EITC: one exemption at $40, granted because gross household income of $312 is under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added the $50,000 financial assistance to the $312 of interest to reach a $50,312 income figure and ruled the household over the $20,000 sales tax relief ceiling. The financial assistance is not Oklahoma gross household income; the test income is $312, and the single exemption pays $40." -us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It asserted without analysis that no state-level refundable credit applies, a shortcut consistent with assuming refundable state credits require children or earned income. Oklahoma's refundable sales tax relief credit pays $40 per exemption to a resident whose $312 of gross household income is under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It stated that the sales tax relief credit counts financial assistance in total income and therefore exceeded the $20,000 single-filer threshold. The $50,000 financial assistance stays outside Oklahoma gross household income, leaving $312 of taxable interest under the ceiling and one exemption worth $40." -us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It flatly declared the household unqualified for any Oklahoma refundable credit with no rule cited, a $0 consistent with treating state refundable credits as EITC-style earnings-based credits. Oklahoma's sales tax relief credit is income-tested only: $312 of gross household income under $20,000 pays $40 for the one exemption." -us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It restated the $0 with no rule or computation, a result consistent with assuming a zero-earnings childless filer receives no state credit. The correct derivation applies Oklahoma's refundable sales tax relief credit — $40 per exemption at gross household income of $312, under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It treated being 65 or older, disabled, or having dependents as prerequisites for the sales tax relief credit and denied it to a non-disabled 21-year-old with no dependents. Those characteristics only raise the income ceiling from $20,000 to $50,000; at the base ceiling the filer's $312 of gross household income qualifies for the full $40." -us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It asserted no refundable Oklahoma credit applies without naming a rule or test, a $0 consistent with equating state refundable credits with an earnings-based EITC. Oklahoma's sales tax relief credit is refundable, income-tested at $20,000, and pays $40 for this filer's single exemption on $312 of gross household income." -us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It anchored the entire answer on the OK EITC being zero because the federal EITC is zero. The sales tax relief credit is a separate refundable credit with no earned-income requirement: $40 per exemption on $312 of gross household income under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It stated that Oklahoma offers no refundable individual income tax credit to this household. Oklahoma's sales tax relief credit is refundable and pays $40 per qualified exemption whenever gross household income is under $20,000, which the $312 of taxable interest satisfies." -us,scenario_029,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It gave a bare 'no refundable OK credits' with no rule or arithmetic, a $0 consistent with treating Oklahoma as having only an EITC-linked refundable credit. The sales tax relief credit pays $40 per exemption to this resident because gross household income of $312 is under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It concluded that the household facts support no Oklahoma refundable credit. The stated facts are exactly sufficient: Oklahoma residency, one exemption, age 21, and $312 of gross household income under the $20,000 ceiling trigger the refundable sales tax relief credit at $40." -us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It treated the absence of any explicit credit-eligibility fact as grounds for zero. The sales tax relief credit requires only Oklahoma residency and gross household income under $20,000 — the $312 of taxable interest clears it, and the single exemption pays $40." -us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It required 'qualifying earned income or other listed refundable-credit basis' before allowing any Oklahoma refundable credit. The sales tax relief credit has no earned-income basis: it pays $40 per exemption on unearned income alone, since $312 of gross household income is under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It conditioned every Oklahoma refundable credit on qualifying earned income or a qualifying dependent. The sales tax relief credit conditions on neither — one exemption at $40 because gross household income of $312 is under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It denied the credit for lack of earned income and dependents. Those are not eligibility conditions for Oklahoma's sales tax relief credit, which pays $40 per exemption to a resident whose $312 of gross household income falls under the $20,000 ceiling." -us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It asserted no listed fact establishes eligibility for an Oklahoma refundable credit, without testing the state's income-based credit. The listed state (OK), age (21), and $312 of taxable interest are the full eligibility test for the refundable sales tax relief credit, which pays $40 for one exemption." -us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It reached the right credit but disqualified it by counting the $50,000 financial assistance toward the sales-tax-relief income limit. Financial assistance is outside Oklahoma gross household income; the $312 of taxable interest is under the $20,000 ceiling and one exemption pays $40." -us,scenario_029,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It required earned income, dependents, or specific expenses before any Oklahoma refundable credit could apply. The sales tax relief credit requires none of these — only residency and gross household income under $20,000, met by the $312 of taxable interest, yielding $40 for one exemption." -us,scenario_029,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It denied the credit for lack of qualifying children and treated the household's income and composition as an independent bar. A single childless filer is a qualified exemption under Oklahoma's sales tax relief credit, and $312 of gross household income under the $20,000 ceiling pays the full $40." -us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It combined the $50,000 financial assistance with the investment assets to place the household over the sales-tax-relief income rules. Neither the assistance nor the $54,985 of bank and stock assets enters Oklahoma gross household income, which is $312 — under the $20,000 ceiling, so one exemption pays $40." -us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It asserted a blanket 'no refundable state credits apply' with no Oklahoma-specific rule, a $0 consistent with defaulting every no-earnings, no-children household to zero state credits. Oklahoma's refundable sales tax relief credit pays $40 per exemption at gross household income of $312 against a $20,000 ceiling." -us,scenario_029,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It checked for dependents, earned income for a state EITC, and an income/asset bar, and concluded no refundable Oklahoma low-income credit results. The sales tax relief credit is the low-income credit it failed to apply: no dependents, no earned income, and no asset test are required, and $312 of gross household income under the $20,000 ceiling pays $40 for one exemption." -us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"It denied the sales tax relief credit because the filer owes zero Oklahoma tax, is under 65, and has no disability or dependents. The credit is refundable and therefore not limited by liability, and the 65-or-older/disability/dependent conditions only lift the income ceiling from $20,000 to $50,000 — at the base ceiling, $312 of gross household income qualifies for $40." -us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It ruled the $50,000 financial assistance into gross household income and thereby put the under-65, childless filer over the sales-tax-relief limit. That assistance is excluded from Oklahoma gross household income; the $312 of taxable interest is the test amount, under $20,000, so the single exemption pays $40." -us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It stated that Oklahoma has no state income tax and derived $0 from that premise. Oklahoma levies a graduated individual income tax and grants a refundable $40-per-exemption sales tax relief credit to residents with gross household income under $20,000, which this filer's $312 of taxable interest satisfies." -us,scenario_029,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It named both the OK EITC and the Sales Tax Relief Credit but denied both for want of earned income. Only the EITC depends on earnings; the sales tax relief credit pays $40 per exemption on gross household income of $312 against the $20,000 ceiling, regardless of whether any income is earned." -us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It screened Oklahoma refundable credits solely on earned income, qualifying children, and child-related characteristics. The sales tax relief credit uses none of those screens — residency plus $312 of gross household income under the $20,000 ceiling produces $40 for the one exemption." -us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It stated that no refundable Oklahoma credit is indicated by the household facts, without applying the state's income test, a $0 consistent with recognizing only earnings- or child-linked state credits. The refundable sales tax relief credit applies here: $40 per exemption because gross household income of $312 falls under the $20,000 ceiling." +us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Correctly named the $40-per-exemption Sales Tax Relief Credit but added the $50,000 of financial assistance to gross household income and disqualified the head against the $20,000 limit. The credit's gross income is only the $312 of interest, which is far under the limit, so one exemption yields $40." +us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Oklahoma has no refundable individual income tax credits, overlooking the refundable Sales Tax Relief Credit of $40 per exemption for households under the $20,000 gross-income limit. The head's $312 gross income qualifies for $40." +us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Named the Sales Tax Relief Credit but counted the $50,000 of financial assistance (and pointed to the stock and bank assets) toward the $20,000 income limit. The credit's gross income is only the $312 of interest and the credit has no asset test, so the head receives $40 for one exemption." +us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Denied the Sales Tax Relief Credit on the grounds of no earned income and 'substantial assets/financial assistance'. The credit has no earned-income or asset test, and gross income of $312 is under the $20,000 limit, so it pays $40 for the single exemption." +us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said the sales tax relief thresholds were not met and treated take-up as zero, but the prompt assumes filing and take-up. With $312 of gross income under the $20,000 limit, the Sales Tax Relief Credit is $40 for one exemption." +us,scenario_029,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Treated earned income or dependents as a prerequisite for any Oklahoma refundable credit and never applied the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit." +us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Considered only the Oklahoma EITC and dismissed every other refundable credit, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with gross income under $20,000; here gross income is $312." +us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Found no general refundable credit and treated the elderly/disability pathway as the only non-EITC basis, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income under $20,000, which covers this head's $312." +us,scenario_029,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Said no refundable credits apply 'at this income level' when a very low income is exactly what qualifies for the Sales Tax Relief Credit. Gross income of $312 is under the $20,000 limit, giving $40 for one exemption." +us,scenario_029,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Tied refundable credits to having tax liability, but the Sales Tax Relief Credit is refundable regardless of tax. It pays $40 per exemption when gross income ($312) is under $20,000." +us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Correctly zeroed the 5% Oklahoma EITC but wrongly concluded no other refundable credit applies, omitting the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit." +us,scenario_029,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Evaluated only the EITC-based credit and missed the refundable Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under $20,000." +us,scenario_029,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,Reasoned only from the $0 federal EITC and never applied the Oklahoma Sales Tax Relief Credit. That credit gives $40 per exemption at gross income of $312. +us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Computed gross income as $50,312 by adding the $50,000 of financial assistance and compared it with the $20,000 Sales Tax Relief Credit limit. The credit's gross income is only the $312 of interest, which qualifies the single exemption for $40." +us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no state refundable credits apply, missing the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit." +us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Included the $50,000 of financial assistance in total income for the Sales Tax Relief Credit and so exceeded the $20,000 limit. The credit's gross income is $312 of interest, so the head receives $40 for one exemption." +us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Found no qualifying refundable credit, overlooking the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to an adult filer with gross income ($312) under the $20,000 limit." +us,scenario_029,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned $0 without applying the Oklahoma Sales Tax Relief Credit. That credit pays $40 for the head's single exemption because gross income of $312 is under $20,000." +us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"Treated age 65+, disability, or dependents as a prerequisite for the Sales Tax Relief Credit. Those factors only raise the income limit to $50,000; an adult single filer qualifies under the base $20,000 limit, and with $312 of gross income receives $40." +us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Concluded no Oklahoma refundable credit applies, missing the Sales Tax Relief Credit. That credit gives $40 per exemption when gross income is $312, under the $20,000 limit." +us,scenario_029,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,Checked only the Oklahoma EITC (zero because the federal EITC is zero) and never applied the Sales Tax Relief Credit. That credit pays $40 for one exemption at gross income of $312. +us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Asserted that Oklahoma offers no refundable credit for this household, overlooking the Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit." +us,scenario_029,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Reported no refundable Oklahoma credits, missing the Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under $20,000." +us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no factual basis for a refundable Oklahoma credit, but the $312 of gross income is itself the basis for the Sales Tax Relief Credit. That income is under the $20,000 limit, giving $40 for one exemption." +us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Defaulted to zero for lack of listed eligibility facts, never applying the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to an adult filer with gross income ($312) under $20,000." +us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Treated earned income as the only basis for an Oklahoma refundable credit, missing the Sales Tax Relief Credit. That credit has no earned-income requirement and pays $40 per exemption at gross income of $312." +us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Required earned income or a qualifying dependent, but the Sales Tax Relief Credit requires neither. It pays $40 per exemption when gross income ($312) is under $20,000." +us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Required earned income, dependents, or other special facts, missing the Sales Tax Relief Credit. For an adult filer that credit depends only on gross income under $20,000, and $312 yields $40." +us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Found no listed facts establishing eligibility, overlooking that the $312 of gross income alone qualifies the head for the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption." +us,scenario_029,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance toward the Sales Tax Relief Credit income limit and denied the credit. The credit's gross income is the $312 of interest only, under $20,000, so the head receives $40." +us,scenario_029,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Tied Oklahoma refundable credits to children or earned income and never applied the Sales Tax Relief Credit. That credit pays a single adult $40 per exemption when gross income ($312) is under $20,000." +us,scenario_029,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Required earnings or qualifying children, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to any adult filer with gross income under $20,000; here gross income is $312." +us,scenario_029,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Required earned income, dependents, or specific expenses, overlooking the Sales Tax Relief Credit. That credit needs none of these and pays $40 for one exemption at gross income of $312." +us,scenario_029,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Treated qualifying children as necessary for an Oklahoma refundable credit, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless adult with gross income ($312) under $20,000." +us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Put the household over the sales tax relief income limit by counting the $50,000 of financial assistance and investment assets. The credit uses gross income of only $312 and has no asset test, so it pays $40 for one exemption." +us,scenario_029,state_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Measured 'gross household resources' including the $50,000 of financial assistance against the sales tax relief limit. The credit's gross income is only the $312 of interest, under $20,000, which yields $40." +us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded no refundable state credits apply, missing the Oklahoma Sales Tax Relief Credit. That credit pays $40 per exemption to a household with $312 of gross income, under the $20,000 limit." +us,scenario_029,state_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Said income and assets rule out a refundable low-income credit, but the Sales Tax Relief Credit has no asset test and its gross income here is only the $312 of interest. That is under $20,000, so the credit is $40." +us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"Treated being under 65 with no disability or dependents as disqualifying for the Sales Tax Relief Credit. Those factors only raise the limit to $50,000; an adult single filer with $312 of gross income qualifies under the base $20,000 limit for $40." +us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Counted the $50,000 of financial assistance in gross household income for the Sales Tax Relief Credit, exceeding the $20,000 under-65 limit. The credit's gross income is $312 of interest, so the head receives $40 for one exemption." +us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Falsely stated that Oklahoma has no state income tax. Oklahoma levies an income tax, and its refundable Sales Tax Relief Credit pays $40 per exemption to this household with $312 of gross income." +us,scenario_029,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Named the Sales Tax Relief Credit but wrongly grouped it with the EITC as needing earned income. The credit is based only on gross income under $20,000, and at $312 it pays $40 for one exemption." +us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Considered only the EITC and child-related credits, missing the Sales Tax Relief Credit. That credit pays $40 per exemption to a childless adult with gross income ($312) under $20,000." +us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Reported no indicated refundable credits, overlooking the Oklahoma Sales Tax Relief Credit. That credit pays $40 for the head's one exemption because gross income of $312 is under the $20,000 limit." us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $5,000 of educational assistance and $12,000 of financial assistance in AGI, inflating it from $13,000 to $30,000. With only $13,000 of taxable wages, the single-filer standard deduction eliminates all taxable income, so applying the 10% and 12% brackets to $14,600 was erroneous." us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without first applying the minimum-age requirement. At age 23, the filer is ineligible for the 2026 childless EITC, so the phase-in and phaseout arithmetic never applies." us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated the 23-year-old as eligible for the childless EITC despite the minimum-age requirement. It also mislabeled $1,940 as the childless maximum and submitted that amount even though its own 7.65% calculation produced $994.50." @@ -2409,45 +2694,52 @@ us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50." us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It treated the 130% FPG gross test (~$1,696/month) as the only gate and stopped there. Texas confers broad-based categorical eligibility through TANF-funded non-cash services, which raises the gross limit to 165% FPG (~$2,152/month) and waives the net income test, so this household's $2,083/month countable gross income qualifies; it then never applied the one- and two-person minimum allotment of 8% of the maximum ($23.84/month)." -us,scenario_030,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"Its income mechanics were right — net income of ~$1,658/month and 30% of net exceeding the $298 maximum allotment — but it converted that to $0. For an eligible one- or two-person household, the minimum allotment of 8% of the maximum applies: $23.84/month, rising to $24.37 when the October COLA lifts the maximum to $304.68, for $287.68 annually." -us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It counted the $5,000 educational assistance to reach a ""$30,000"" gross figure, which SNAP excludes, and denied eligibility on a 130% FPG gross test it misquoted as $1,868/month. Countable gross is $2,083/month (wages plus the $12,000 financial assistance), which clears Texas's 165% FPG BBCE limit, and the household then receives the minimum allotment rather than nothing." -us,scenario_030,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly built gross income of $2,083/month and net income of $1,657/month and correctly identified Texas BBCE at 165% FPG, then used a stale poverty guideline that put the 165% limit at $2,072/month instead of ~$2,152, flipping an eligible household to ineligible by $11/month. With eligibility established, 30% of net ($497.10) exceeds the $298 maximum, so the $23.84 minimum allotment is payable." -us,scenario_030,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It excluded the $12,000 financial assistance from countable income, which SNAP counts as unearned cash income, producing a net income of ~$663/month instead of $1,657/month. It then abandoned its own formula, asserting the household ""receives roughly the maximum"" ($191/month) after computing a ~$99/month benefit; the correct net income drives the expected contribution above the maximum allotment, leaving only the $23.84 minimum." -us,scenario_030,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It computed the benefit from the $13,000 of wages alone and omitted the $12,000 financial assistance, which counts as unearned income, so its net income was roughly $658/month rather than $1,657/month. At the correct net income, 30% ($497.10) exceeds the $298 maximum allotment and the household receives the one-person minimum allotment of $23.84–$24.37 per month, not $134." -us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It asserted that ""Texas does NOT have broad-based categorical eligibility"" and therefore applied the 130% FPG gross test as binding. Texas grants BBCE through a TANF-funded non-cash service, setting the gross limit at 165% FPG and eliminating the net income test, so gross income at 160% FPG qualifies and the household draws the minimum allotment." -us,scenario_030,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It dropped the $12,000 financial assistance from countable unearned income, then contradicted its own arithmetic — $298 maximum minus 30% of net (~$200) is $98/month, not the ""$350/month"" it submitted. Counting all income yields net income of $1,657/month, an expected contribution of $497.10 above the maximum allotment, and the $23.84 minimum benefit." -us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted the $5,000 educational assistance, which SNAP excludes, and applied 130% FPG (~$1,743/month) as the operative gross limit. The operative Texas limit is the 165% FPG BBCE threshold, which $2,083/month of countable gross clears, making the household eligible for the minimum allotment." -us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It summed all three amounts to ""$30,000,"" including educational assistance that SNAP excludes, and compared it to $19,578 — a 130% FPG figure that Texas supersedes with its 165% FPG BBCE limit. Countable gross is $25,000/year (160% FPG), under the BBCE limit, so the household is eligible and receives the minimum allotment." -us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It counted the excluded educational assistance, but its decisive error is returning $0 after finding that the net-income contribution exceeds the one-person maximum allotment. When 30% of net income ($497.10) exceeds the $298 maximum, an eligible one- or two-person household receives the minimum allotment of 8% of the maximum — $23.84/month — not zero." -us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It applied the 130% FPG gross limit (""approximately $19,000-$20,000"") as the eligibility gate for the $25,000 of countable gross income. Texas's BBCE pathway through TANF non-cash services sets the gross limit at 165% FPG (~$25,800) and waives the net test, so the household qualifies and receives the $23.84–$24.37 monthly minimum allotment." -us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It based the calculation on ""$13,000 in annual earnings"" alone, omitting the $12,000 financial assistance that counts as unearned income, and produced $217/month with no stated arithmetic. With all countable income, net income is $1,657/month, the expected contribution of $497.10 exceeds the $298 maximum allotment, and the payable amount is the $23.84 minimum." -us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It correctly held household size at one but denied eligibility on the 130% FPG gross limit. Texas's 165% FPG BBCE limit governs here and admits gross income at 160% FPG, after which the expected contribution exceeding the maximum allotment triggers the one-person minimum benefit of $23.84/month." -us,scenario_030,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It named the correct gate — Texas's 165% FPG BBCE gross limit — but used $24,849 as the value; 165% of the 2026 one-person poverty guideline is about $25,800, so $25,000 of countable gross passes rather than fails. Eligibility then produces the minimum allotment of $23.84–$24.37 per month rather than $0." -us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It asserted a zero benefit with no derivation. The household passes Texas's 165% FPG BBCE gross test at 160% FPG, has its net test waived by categorical eligibility, and receives the one-person minimum allotment because 30% of its $1,657 net income exceeds the $298 maximum; its $0 is the output of applying the 130% federal gross test or dropping the minimum-benefit floor." -us,scenario_030,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It stopped at ""monthly income exceeds the net income limits to receive a positive SNAP allotment."" Categorical eligibility through Texas BBCE waives the 100% FPG net test, and an eligible one- or two-person household whose 30%-of-net contribution exceeds the maximum allotment is paid the minimum allotment of 8% of the maximum, $23.84–$24.37 per month." -us,scenario_030,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It concluded that net income bars a positive allotment for a household of one. The minimum allotment for one- and two-person households is paid whenever the household is eligible and the computed benefit falls below it, so the $497.10 expected contribution against the $298 maximum yields $23.84/month, not $0." -us,scenario_030,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly counted the recurring financial assistance as unearned income and correctly found the resulting net income too high for a formula benefit, then reported $0. The one- and two-person minimum allotment of 8% of the maximum applies at that point: $23.84/month, $24.37 after the October COLA raises the maximum to $304.68." -us,scenario_030,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"It set household size at 2 for the pregnancy; SNAP counts only born members, so the unit is one person with a $298 maximum allotment. It compounded this by taking an $8,080 excess medical deduction, which is available only to households containing an elderly (60+) or disabled member — the 23-year-old head is neither, so the $8,500 of medical expenses is not deductible and net income is $1,657/month." -us,scenario_030,snap,glm-5.3,llm_error,thresholds_rates,False,"It counted the $12,000 assistance and $13,000 wages correctly but treated the household as failing both the gross and net limits. Gross income at 160% FPG passes Texas's 165% FPG BBCE limit, categorical eligibility waives the net test, and the resulting benefit is the $23.84–$24.37 monthly minimum allotment rather than $0." -us,scenario_030,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It characterized the household as having ""very low countable cash income,"" which ignores the $12,000 of financial assistance counted as unearned income, and produced $176/month from no stated formula. Countable income of $2,083/month yields net income of $1,657, an expected contribution of $497.10 above the $298 maximum, and a payable minimum allotment of $23.84/month." -us,scenario_030,snap,gpt-5.4-nano,llm_error,other,False,"It never ran the SNAP formula, declaring $0 because ""no household size or other needs"" were specified; household size is one and every input needed is given. Even on its own premise of $13,000 in wages alone the formula yields a positive benefit, and with the full $25,000 of countable income the household is BBCE-eligible and receives the $23.84–$24.37 monthly minimum allotment." -us,scenario_030,snap,gpt-5.5,llm_error,thresholds_rates,False,"It carried the calculation correctly to the point where 30% of net income exceeds the one-person maximum allotment, then declared no benefit payable. The minimum allotment rule pays 8% of the maximum allotment to eligible one- and two-person households in exactly that situation: $23.84/month, $24.37 after the COLA, totaling $287.68." -us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It placed countable monthly income above ""the applicable one-person SNAP gross-income limit,"" applying the 130% FPG federal test. The applicable limit in Texas is the 165% FPG BBCE threshold, which $2,083/month clears; the household is categorically eligible through TANF non-cash services and receives the minimum allotment." -us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It correctly excluded the educational assistance and counted the $12,000, then applied a Texas gross-income limit below the $2,083/month figure. Texas's operative limit under BBCE is 165% FPG (~$2,152/month), so the household qualifies and its benefit is the one-person minimum allotment of $23.84–$24.37 per month." -us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It denied eligibility because $25,000 exceeds ""the one-person SNAP gross-income limit,"" i.e. 130% FPG. Texas's BBCE gross limit of 165% FPG is the binding one, and the household at 160% FPG passes it, drawing the minimum allotment because 30% of its $1,657 net income exceeds the $298 maximum." -us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"It denied eligibility on ""applicable SNAP income limits"" and cited the absence of shelter and elderly/disabled medical deductions as supporting zero. Those deductions are correctly unavailable, but the household is still categorically eligible under Texas's 165% FPG BBCE limit, and the minimum allotment of $23.84/month is paid precisely because the deduction-free net income of $1,657 pushes the expected contribution above the maximum allotment." -us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"It gave no derivation, asserting that assets and income produce zero. The $190 in assets is immaterial under BBCE, gross income at 160% FPG passes the 165% Texas limit, and the household receives the one-person minimum allotment of 8% of the maximum ($23.84/month, $24.37 post-COLA); its $0 reflects applying the 130% federal gross test instead." -us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"It counted income correctly at $25,000 but tested it against the 130% FPG limit. Texas's TANF non-cash categorical eligibility sets the gross limit at 165% FPG and removes the net test, so the unit is eligible and receives $23.84–$24.37 per month as the minimum allotment." -us,scenario_030,snap,grok-4.6,llm_error,thresholds_rates,False,"It identified the Texas BBCE 165% FPG test but asserted the household was ""well above"" it; 165% of the 2026 one-person poverty guideline is about $25,800/year and countable gross is $25,000 (160% FPG), so it passes. It also proposed counting educational assistance, which SNAP excludes, and never reached the minimum allotment that eligibility produces." -us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It correctly excluded educational assistance and computed $25,000 of countable gross, then applied a 130% FPG limit of ~$20,800 as the deciding test. Texas's BBCE gross limit of 165% FPG governs, admitting this household, which then receives the minimum allotment because 30% of net income exceeds the $298 maximum." -us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"It counted the $5,000 educational assistance, which SNAP excludes, and applied the 130% FPG gross test as dispositive. Countable gross of $2,083/month is 160% FPG and passes Texas's 165% BBCE limit, leaving the household eligible for the $23.84–$24.37 monthly minimum allotment." -us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for snap, so no substantive reasoning exists to evaluate. The contract required a numeric value with an explanation for every requested key, and this key was absent." -us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"It computed $2,083.33/month of countable income accurately and rejected the household against ""the one-person SNAP gross-income limit,"" i.e. 130% FPG. Texas's BBCE limit of 165% FPG is the applicable one and admits that income, after which the minimum allotment of 8% of the maximum ($23.84/month) is payable." -us,scenario_030,snap,minimax-m3,llm_error,period_annualization,False,"It compared an annual figure to a monthly limit — ""$10,400 net, which exceeds the income limit of ~$1,255/month"" — after applying the 20% earned income deduction to the full $13,000 of annual wages. Monthly net income is $1,657 counting the $12,000 assistance, the household is categorically eligible under Texas's 165% FPG BBCE limit, and it receives the one-person minimum allotment of $23.84–$24.37 per month." -us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"It built countable income correctly at $25,000/year, then applied the 130% FPG limit of ~$20,350 as the eligibility test. Texas confers BBCE through a TANF-funded non-cash service, raising the gross limit to 165% FPG and waiving the net test, so the household qualifies and is paid the minimum allotment." -us,scenario_030,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $12,000 financial assistance from countable income and then submitted $2,576 while its own worked arithmetic produced $31–$32 per month (about $372/year), so the value contradicts the explanation. Counting the assistance gives net income of $1,657/month, an expected contribution of $497.10 above the $298 maximum, and the $23.84 minimum allotment." -us,scenario_030,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a second household member, which SNAP does not do, and claimed net income of $0 despite $25,000 of countable income; the actual net income is $1,657/month against a one-person $298 maximum allotment. It also submitted $3,576 while its explanation states $4,272, and the correct result is the $23.84–$24.37 monthly minimum allotment." +us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but then used the 130% FPL gross test. It never applied Texas's broad-based categorical eligibility at 165% FPL (~$2,152/month), which this household passes at 160% FPL. As an eligible 1-person household, it gets the $24 monthly minimum allotment." +us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It correctly got net income of about $1,658, but it then applied the net income test, which Texas broad-based categorical eligibility waives. It also turned a 30% contribution above the $298 maximum into $0 instead of the $24 minimum allotment that an eligible 1-person household receives." +us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It counted a $30,000 total that includes the excluded $5,000 educational assistance, and it compared income to a 130% gross limit. It missed Texas's 165% FPL categorical eligibility, which countable gross income of $2,083/month passes, and the $24 minimum allotment that follows." +us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly counted $2,083/month gross and raised Texas BBCE, but it put the 165% FPL limit at ~$2,072 when the actual figure is $2,151.88 (165% of $1,304.17). That made it wrongly fail the household. Once categorically eligible, the household gets the $24 minimum allotment even though its contribution is more than the maximum." +us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It excluded the $12,000 of financial assistance, which SNAP counts as $1,000/month of unearned income. That left net income far too low. It then dropped its own ~$99/month result and submitted $2,298, which comes from neither its formula nor the maximum. Counting the assistance gives net income of $1,658, a $498 contribution, and the $24 minimum." +us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It counted only the $13,000 of wages and left out the $12,000 of financial assistance, which is countable unearned income. That understated net income, which is actually $1,658/month. Once the assistance is counted, the $498 contribution is more than the $298 maximum, so the benefit is the $24 minimum rather than $134/month." +us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It explicitly treated the $12,000 of financial assistance as non-countable, which gave net income of $657.67 and a $100.70 benefit. SNAP counts the assistance as $1,000/month of unearned income, so net income is $1,658 and the contribution is $498. That exceeds the $298 maximum, leaving the $24 minimum allotment." +us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It counted income correctly ($2,083/month) but stated that Texas has no broad-based categorical eligibility. Texas grants categorical eligibility through TANF non-cash benefits up to 165% FPL, and this household is at 160%, so it is eligible and gets the $24 minimum allotment." +us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"It left the $12,000 of financial assistance out of countable income and then made an arithmetic error: $298 minus ~$200 is ~$98, not $350. Counting the assistance gives net income of $1,658 and a $498 contribution, so the benefit is the $24 minimum." +us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the financial assistance as excluded and counted wages only, which gave net income of ~$658 and ~$101/month. SNAP counts the $1,000/month of financial assistance as unearned income, which raises net income to $1,658 and the contribution to $498. The benefit is then the $24 minimum allotment." +us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, reaching $2,500/month gross, and it applied a 130% gross test. The correct countable gross is $2,083/month, which passes Texas's 165% FPL categorical eligibility limit. The household therefore gets the $24 minimum allotment." +us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It counted a $30,000 total that includes the excluded educational assistance, and it applied the 130% FPL gross limit. It ignored Texas's 165% FPL broad-based categorical eligibility, which the correct $25,000 countable income passes, and the $24 monthly minimum allotment that follows." +us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It wrongly counted the $5,000 of educational assistance, and it treated a contribution above the one-person maximum as a $0 benefit. A categorically eligible 1-person household whose contribution exceeds the $298 maximum still receives the $24 minimum allotment." +us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It counted the pregnant head as a household of 2 and used the $546 two-person maximum. SNAP does not count an unborn child, so the unit is 1 person with a $298 maximum. Its correct $497 contribution exceeds that maximum, so the benefit is the $24 minimum allotment." +us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It correctly used a household of one and $25,000 of countable income, but it applied the 130% FPL gross test. It missed Texas's 165% FPL broad-based categorical eligibility (~$25,823/year), which the household passes, and so it missed the $24 monthly minimum allotment." +us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It based the benefit on $13,000 of earnings alone and left out the $12,000 of financial assistance, which is countable unearned income. Its $217/month result also does not follow from the SNAP formula. Correct net income is $1,658, so the contribution is more than the maximum and the benefit is the $24 minimum." +us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test even though Texas broad-based categorical eligibility replaces it with a 165% FPL limit. The household's 160%-of-FPL gross income passes that limit, which entitles it to the $24 minimum allotment." +us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It correctly raised the 165% FPL BBCE limit but calculated it from the outdated $15,060 guideline, getting $24,849. The guideline in effect is $15,650, which makes the limit $25,822.50, so $25,000 passes. The household is categorically eligible for the $24 minimum allotment." +us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without any computation. This household is categorically eligible under Texas BBCE (gross at 160% FPL, $190 in assets) and has a $498 contribution that exceeds the $298 maximum. Such a household receives the $24 monthly minimum allotment, not $0." +us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It applied net income limits that Texas broad-based categorical eligibility waives, and it treated a formula benefit of zero as no benefit. An eligible 1-person household whose contribution exceeds the maximum receives the $24 minimum allotment." +us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It applied a net income threshold that categorical eligibility waives for this household, whose gross income is 160% FPL, under Texas's 165% limit. It also missed the $24 minimum allotment for eligible 1-person households whose computed benefit is zero." +us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance as unearned income but turned a formula benefit of zero into $0. The household is categorically eligible, so the $24 minimum allotment for 1-person households applies." +us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"It counted the pregnancy as a second household member and allowed an $8,080 medical deduction. SNAP excludes unborn children from household size, and it allows the medical deduction only for elderly or disabled members. With 1 person and no medical deduction, net income is $1,658, the contribution exceeds the $298 maximum, and the benefit is the $24 minimum." +us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"It counted income correctly but applied the standard gross and net limits, which Texas broad-based categorical eligibility replaces with a 165% FPL gross test. The household passes that test and receives the $24 minimum allotment." +us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated the household as having very low countable income, which means it left out the $12,000 of financial assistance that SNAP counts as unearned income. Once it is counted, net income is $1,658 and the contribution is $498, so the benefit is the $24 minimum rather than ~$176/month." +us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 without applying any SNAP test. The household is categorically eligible under Texas BBCE at 160% FPL, and an eligible 1-person household with a contribution above the maximum receives the $24 monthly minimum allotment." +us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the $298 one-person maximum, but it concluded that no benefit is payable. The household is categorically eligible, so the $24 minimum allotment for 1- and 2-person households applies." +us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It used a one-person gross income limit without applying Texas's 165% FPL broad-based categorical eligibility threshold, which the household passes at 160% FPL. It therefore missed the $24 monthly minimum allotment." +us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It correctly counted $25,000 and excluded the educational assistance, but it wrongly concluded that this exceeds Texas's gross limit. The Texas BBCE limit is 165% FPL ($25,822.50), so the household is eligible and gets the $24 minimum allotment." +us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It tested $25,000 against the standard one-person gross limit (130% FPL) instead of Texas's 165% FPL broad-based categorical eligibility limit, which the household passes. It then missed the $24 minimum allotment." +us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,It applied SNAP income limits that Texas broad-based categorical eligibility waives for households at or below 165% FPL; this household is at 160%. It did not apply the $24 minimum allotment for an eligible 1-person household. +us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It correctly counted the financial assistance and found that the 30% contribution exceeds the one-person maximum, but it returned $0. The categorically eligible household instead receives the $24 minimum allotment each month." +us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It based net income on wages alone and left out the $1,000/month of financial assistance, which is countable unearned income. Including it raises net income to $1,658 and the contribution to $498, so the benefit drops to the $24 minimum ($288/year)." +us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,It correctly counted wages and financial assistance but treated a formula allotment of zero as no benefit. A categorically eligible 1-person household receives the $24 minimum allotment. +us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 without a computation. The household passes Texas BBCE at 160% FPL with $190 in assets, and its $498 contribution exceeds the $298 maximum, which triggers the $24 minimum allotment." +us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test to $25,000 without recognizing that Texas broad-based categorical eligibility raises the limit to 165% FPL ($25,822.50). The household passes that limit and receives the $24 minimum allotment." +us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"It claimed the income exceeds the Texas BBCE 165% FPL limit, but countable income of $25,000 (wages plus financial assistance, with educational assistance excluded) is below $25,822.50. The household is categorically eligible and receives the $24 minimum allotment." +us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"It correctly computed net income of ~$19,900/year ($1,658/month) but failed the household on the 100% FPL net income test. Texas broad-based categorical eligibility waives that test, so the household is eligible and receives the $24 minimum allotment." +us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It correctly counted $25,000 but applied the 130% FPL gross limit instead of Texas's 165% FPL broad-based categorical eligibility limit, which the household passes. It therefore missed the $24 monthly minimum allotment." +us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"It counted the $5,000 of educational assistance, which SNAP excludes, and applied the 130% FPL gross test. Correct countable gross income of $2,083/month passes Texas's 165% FPL categorical eligibility limit, so the household gets the $24 minimum." +us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable SNAP value or explanation, so the requested output is missing." +us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly computed $2,083.33/month but compared it to the standard one-person gross limit instead of Texas's 165% FPL BBCE limit ($2,151.88). It therefore missed eligibility and the $24 minimum allotment." +us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"It left out the $12,000 of countable financial assistance and compared an annual net figure ($10,400) with a monthly limit ($1,255), mixing time periods. With correct monthly countable income under Texas BBCE, the household is eligible and its $498 contribution produces the $24 minimum allotment." +us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"It correctly counted $25,000 but applied the 130% FPL gross test (~$20,350). Texas broad-based categorical eligibility uses a 165% FPL limit ($25,822.50), which the household passes, so it receives the $24 minimum allotment." +us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It left out the $12,000 of financial assistance and the standard deduction, and it then submitted $2,576, which contradicts its own $31–32/month calculation. Counting the assistance gives net income of $1,658, a $498 contribution, and the $24 minimum allotment." +us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It counted the unborn child to make a 2-person household and assumed $0 net income despite $2,083/month of countable wages and financial assistance. The unit is 1 person with a $298 maximum and a $498 contribution, so the benefit is the $24 minimum. Its submitted $3,576 also contradicts its own $4,272." us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation." us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply." @@ -2455,48 +2747,55 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income." us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation." us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic." -us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,health_coverage,False,"It found the right pathway (California's Aged & Disabled category at 138% FPL) but then counted income under MAGI rules, subtracting the $1,165 alimony expense instead of applying the non-MAGI SSI-related deductions. The senior-or-disabled test deducts health insurance premiums — the Medicare Part B premium above all — plus the $20 general exclusion, which pulls the $23,853 gross (142% FPL) down under the 138% A&D FPL limit; its $1,890/month countable figure overstates income by roughly $450 a month." -us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,health_coverage,False,"It applied only the $20/month general income exclusion ($240/year) to the $23,853 gross and stopped there. The senior-or-disabled category also deducts health insurance premiums, and the Medicare Part B premium alone (roughly $2,400/year at age 67) drops countable income from 142% FPL to below the 138% A&D FPL limit it itself cited." -us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It set the aged income standard at about $18,564/year while calling it 138% FPL — a figure below 100% FPL for a single person in 2026 — and compared gross income to it with no deductions. Under the non-MAGI senior-or-disabled test the Medicare Part B premium comes off countable income first, and California eliminated the Medi-Cal asset limit on January 1, 2024, so the $4,200 bank balance it flagged is irrelevant." -us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the aged category as MAGI-determined and then tested against a 100% FPL limit of about $15,650. California's Aged & Disabled pathway is non-MAGI: it measures countable income after deducting health insurance premiums including Medicare Part B against 138% FPL, and the head clears that standard." -us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,health_coverage,False,"It compared the $23,853 gross straight to an unadjusted 100–138% FPL band. The senior-or-disabled category deducts health insurance premiums, including the Medicare Part B premium, from countable income, which moves the head from 142% FPL to under the 138% A&D FPL standard." -us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"After cycling through pathways it settled on gross $23,853 against a 138% FPL figure it understated as $20,783, and never applied non-MAGI income counting. The aged/disabled category subtracts health insurance premiums including Medicare Part B from gross income, and that deduction alone brings countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It blended two methods: it excluded most Social Security the way MAGI does to reach $14,408, then tested that against the SSI federal benefit rate of about $13,596. The non-MAGI aged category counts Social Security in full, deducts the Medicare Part B premium, and measures the result against California's 138% Aged & Disabled FPL standard rather than the SSI payment level — a test the head passes." -us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"It compared $23,853 of gross income to 138% FPL with no deductions at all. Under the senior-or-disabled pathway countable income is gross less health insurance premiums including Medicare Part B, which puts the head at roughly 127% FPL and inside the limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Both sides of its comparison were wrong: it used undeducted gross of $1,987.75/month and a 138% FPL limit of $1,732/month, when 138% FPL for one person exceeds $1,800/month in 2026. Deducting the Medicare Part B premium as the senior-or-disabled category requires brings monthly countable income to roughly $1,780, under the true limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It treated the absence of SSI receipt as closing off the non-MAGI route and then compared gross $23,853 to the FPL limit. The senior-or-disabled category requires only aged-or-disabled status — age 67 satisfies it with SSI at $0 — and its income test deducts the Medicare Part B premium, which brings the head within California's limit." -us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,health_coverage,False,"It named the correct pathway and a 138% FPL limit of $22,080 but compared undeducted gross income of $23,853 against it. Deducting the Medicare Part B premium, which the non-MAGI senior-or-disabled income test requires, yields countable income near $21,400 — under its own stated threshold." -us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,health_coverage,False,"It gave no computation; its answer is consistent with comparing $23,853 of gross retirement income to a single-adult FPL-based limit. The correct derivation runs the non-MAGI senior-or-disabled test, deducting health insurance premiums including Medicare Part B, which moves the head from 142% FPL to inside California's 138% Aged & Disabled limit." -us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,health_coverage,False,"It tested gross income of $23,853 against the 138% FPL threshold for the Aged, Blind and Disabled program. That program's countable income is gross less health insurance premiums, and the Medicare Part B premium at age 67 drops the head from 142% FPL to roughly 127% FPL, inside the limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"It compared undeducted monthly gross of $1,988 to the 138% FPL non-MAGI limit. The aged/disabled income test subtracts health insurance premiums including the Medicare Part B premium before the comparison, and that deduction puts the head under the limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,health_coverage,False,"It supplied no rule or arithmetic; the answer tracks a gross-income-versus-FPL screen on $23,853. The senior-or-disabled pathway applies first: age 67 establishes the aged category, the Medicare Part B premium comes off countable income, and the remainder sits below California's 138% Aged & Disabled FPL standard." -us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"It compared the $23,853 gross total to the eligibility threshold for an individual aged 65 or older with no deductions. Countable income under the senior-or-disabled category is gross minus health insurance premiums including Medicare Part B, which brings the head inside California's 138% Aged & Disabled FPL limit." -us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"It asserted that income exceeds the non-MAGI Medi-Cal limits without computing countable income. Those limits are 138% FPL applied after deducting health insurance premiums, and the Medicare Part B deduction moves the head from 142% FPL to under 138%." -us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"It characterized $23,853 as 'well exceeding' 138% FPL when that gross is 142% FPL — a four-point margin the Medicare Part B premium deduction erases. The senior-or-disabled category requires that deduction before the FPL comparison, so the head qualifies." -us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It invented a rule that Medicare eligibility at 65+ precludes Medicaid. Medicare and Medicaid are concurrently held by dual eligibles, and the senior-or-disabled category exists precisely for people 65 and older; the head passes its income and asset tests once the Medicare Part B premium is deducted from countable income." -us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It routed the head through the under-65 ACA expansion group and computed $1,988/month as 199% FPL, when that gross is 142% FPL for a single person in 2026. The applicable test is the non-MAGI senior-or-disabled one, which deducts the Medicare Part B premium and applies a 138% FPL limit the head falls under." -us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,age_disability,False,"It concluded that no disability or low-income qualifying flag was present. Age 67 by itself satisfies the aged prong of the aged/blind/disabled criterion, opening the non-MAGI senior-or-disabled pathway whose income test deducts the Medicare Part B premium and clears the head at roughly 127% FPL." -us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,health_coverage,False,"It measured gross income against an aged spend-down standard and treated the $50 of over-the-counter health expenses as the only medical offset. The controlling program is California's Aged & Disabled FPL category at 138% FPL, and its health insurance premium deduction is the Medicare Part B premium implied by Medicare eligibility at 67 — not the listed out-of-pocket expenses — which brings countable income under the limit." -us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"It asserted countable income is above the applicable California threshold without identifying the category or performing the deduction. Countable income under the senior-or-disabled category is $23,853 less health insurance premiums including Medicare Part B, roughly 127% FPL, which is inside the 138% Aged & Disabled limit." -us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,health_coverage,False,"It compared undeducted retirement income to the aged pathway's income limit. That pathway subtracts health insurance premiums, and the Medicare Part B premium alone accounts for the four-percentage-point gap between the head's 142% FPL gross and the 138% FPL standard." -us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested the head against an SSI-linked income level rather than California's Aged & Disabled FPL standard. The optional senior-or-disabled category uses a 138% FPL limit and deducts health insurance premiums including Medicare Part B from countable income, and the head passes that test even though income far exceeds the SSI federal benefit rate." -us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,health_coverage,False,"It applied the $20/month general income exclusion and nothing else. The senior-or-disabled income test also deducts health insurance premiums, and the Medicare Part B premium at age 67 takes countable income from 142% FPL to under California's 138% Aged & Disabled limit." -us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It claimed full dual eligibles are excluded from Medicaid eligibility. Full duals hold Medicaid alongside Medicare by definition; the head qualifies through the senior-or-disabled category, whose income test deducts the very Medicare Part B premium that dual status implies." -us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It failed the head on both a resource limit and an income limit. California eliminated the Medi-Cal asset test on January 1, 2024, so $4,200 in bank assets passes, and the senior-or-disabled income test deducts the Medicare Part B premium, bringing $23,853 gross (142% FPL) under the 138% Aged & Disabled FPL standard." -us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,health_coverage,False,"It ran the $23,853 gross against Aged & Disabled and Medicare Savings Program limits without deducting health insurance premiums, and treated Medicare Savings Program limits as controlling. Full-scope Medi-Cal under the senior-or-disabled category deducts the Medicare Part B premium from countable income and applies a 138% FPL limit, which the head meets." -us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied a $2,000 Medi-Cal resource limit that California eliminated entirely on January 1, 2024, so $4,200 in bank assets does not disqualify anyone. The head clears both the asset test and the senior-or-disabled income test once the Medicare Part B premium is deducted from countable income." -us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650) for aged Medi-Cal. California's Aged & Disabled program sets the standard at 138% FPL and deducts health insurance premiums including Medicare Part B, so countable income near $21,400 falls inside the limit." -us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,age_disability,False,"It stated the head is not disabled, blind, pregnant, or an SSI recipient and therefore fails the non-MAGI categories. Age 65 or older satisfies the aged prong of the aged/blind/disabled criterion on its own — at 67 the head is in that category — and its income test, which deducts the Medicare Part B premium, brings countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,health_coverage,False,"It correctly excluded the under-65 expansion group but then compared listed gross retirement income to the aged Medi-Cal limit without deductions. Countable income under that category is gross less health insurance premiums including Medicare Part B, which puts the head at roughly 127% FPL against a 138% FPL standard." -us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,health_coverage,False,"It recognized that 65+ Californians have a higher income limit, then guessed that ~$24,000 exceeds it without computing countable income. Deducting the Medicare Part B premium under the senior-or-disabled category yields roughly $21,400, below California's 138% Aged & Disabled FPL limit." -us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It failed the head on a $2,000 resource limit California eliminated on January 1, 2024, and on gross income compared to an aged income standard. The senior-or-disabled income test deducts health insurance premiums including Medicare Part B, dropping $23,853 gross (142% FPL) below the 138% Aged & Disabled FPL limit." -us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"It reasoned that Medicare eligibility at 67 replaces Medicaid and that no disability, institutional, or SSI status was indicated. Age 67 alone establishes the aged category with SSI at $0, Medicare and Medicaid coexist for duals, and the category's income test deducts the Medicare Part B premium — putting countable income inside California's 138% FPL limit." -us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,age_disability,False,"It concluded the head falls into no listed eligibility category because they are not disabled, blind, or pregnant. Being 67 places the head in the aged category of the optional senior-or-disabled pathway, whose income test deducts health insurance premiums including Medicare Part B and admits the head at roughly 127% FPL." -us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Treated California's 200%-FPL broad-based categorical eligibility as waiving every income test and then paid the 1–2 person minimum allotment as a guaranteed floor. MCE waives the gross income and resource tests only; the net income test still governs, and countable unearned income of $1,987.75/month less the ~$207 one-person standard deduction (no earned-income, shelter, or excess-medical deduction applies) leaves ~$1,780/month against a ~$1,305 net income limit, so the household is ineligible and no minimum allotment attaches. Its own finding that 30% of net income exceeds the maximum allotment — $534 against a ~$298 one-person max — is the signal it should have followed, and it also priced the floor at the FY2025 $23 rather than the FY2026 figure." -us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Stopped at the 200%-FPL CalFresh categorical-eligibility screen and awarded the minimum monthly benefit, skipping the net income test that MCE does not waive. After the ~$207 standard deduction — the only deduction available, since there are no earnings, no rent or mortgage payment, and $50/year of over-the-counter costs falls below the $35/month excess-medical threshold — net income is ~$1,780/month versus the ~$1,305 one-person 100%-FPL limit, making the household ineligible so the minimum allotment never applies. It also used the FY2025 $23 minimum instead of the FY2026 amount." -us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Performed no income accounting at all, asserting ""very low countable income"" for a household whose $23,853 of Social Security, pension, and IRA income is $1,987.75/month — roughly 152% of the one-person poverty guideline and above even the 130% gross limit. Its $1,728/year is $144/month, the allotment produced by a ~$298 one-person maximum minus 30% of about $513 of net income, i.e. it assumed roughly a quarter of the household's actual ~$1,780/month net income. The correct derivation fails the ~$1,305 net income test outright, and 30% of net income ($534) exceeds the maximum allotment, giving $0." -us,scenario_031,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Computed gross monthly countable income of $1,987.75 correctly, then treated clearing the 200%-FPL categorical-eligibility limit as establishing eligibility and floored the benefit at the 1–2 person minimum. The minimum allotment is payable only to an eligible household, and for a non-cash-assistance MCE household eligibility still turns on the net income test: $1,987.75 minus the ~$207 standard deduction is ~$1,780/month against the ~$1,305 one-person 100%-FPL net income limit, so SNAP is $0 rather than $24/month." -us,scenario_031,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"Identified the decisive fact — no deductible shelter costs, leaving only the ~$207 standard deduction against $1,987.75/month of countable Social Security, pension, and IRA income — and then overrode it by paying the one-person minimum allotment on the strength of the 200%-of-poverty categorical-eligibility screen. California's MCE waives the gross income and asset tests but not the net income test, and net income of ~$1,780/month exceeds the ~$1,305 one-person limit, so the household is ineligible and the minimum benefit has nothing to floor." +us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly picked the aged pathway and its 138% FPL limit. But it built countable income by subtracting only alimony ($22,688) and never subtracted the Medicare Part B premium (about $2,435/year). With that deduction, countable income is about $21,400, which is under the limit." +us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It applied only the $240 general income exclusion ($23,613) against the 138% FPL aged/disabled limit. It left out the senior-or-disabled deduction for health insurance premiums, including Medicare Part B (about $2,435/year), which puts countable income below the roughly $21,600 limit." +us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with a 138% FPL figure understated at $18,564 and never deducted the Medicare Part B premium from countable income. With Part B subtracted, countable income is about $21,400, under 138% FPL (about $21,600). It also doubted the $4,200 in assets, although that amount easily passes California's senior-or-disabled asset test." +us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It changed its answer to a 100% FPL limit (about $15,650). California's senior-or-disabled pathway uses 138% FPL. It also counted the full $23,853 without deducting the Medicare Part B premium, which brings countable income to about $21,400, within the limit." +us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against a 100–138% FPL range and never subtracted health insurance premiums. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium (about $2,435/year), which puts income under the 138% FPL limit." +us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct 138% FPL aged limit but compared it with gross income of $23,853. Under this pathway, countable income is net of the Medicare Part B premium (about $2,435/year), leaving about $21,400, which is below the limit." +us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It switched between the ACA expansion test, which does not cover someone aged 67, and a 100% FPL aged limit. It settled on gross $23,853 against an understated 138% FPL figure of $20,783. It never used the senior-or-disabled pathway's deduction of the Medicare Part B premium, which brings countable income under the real 138% FPL limit." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It first stated the correct Aged & Disabled FPL limit (about $21,597), which its own $14,408 figure would pass. Then it switched to the SSI federal benefit rate (about $13,596) as the binding limit. It also treated Social Security as MAGI-excluded, although the non-MAGI count includes Social Security in full and instead deducts the Medicare Part B premium." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head is in no categorical group, but at 67 the head is aged, which is exactly the senior-or-disabled category. It also used unadjusted income of about $23.9k and skipped the Medicare Part B premium deduction, which brings countable income below 138% FPL." +us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 138% FPL without deducting health insurance premiums. The senior-or-disabled pathway subtracts the Medicare Part B premium (about $2,435/year), leaving countable income of about $21,400, under the limit." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used gross monthly income of $1,987.75 against an understated 138% FPL of $1,732/month. It never deducted the Medicare Part B premium (about $203/month), which brings countable income to about $1,785/month, under the correct 138% FPL of about $1,800/month." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable income and required SSI receipt for a categorical route. The senior-or-disabled pathway does not require SSI and deducts the Medicare Part B premium, leaving countable income below 138% FPL." +us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with its own 138% FPL figure of $22,218. Deducting the Medicare Part B premium (about $2,435/year), as the senior-or-disabled income count requires, leaves about $21,400, below even the model's own threshold." +us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It picked the correct 138% FPL Aged & Disabled limit (about $22,080 by its estimate) but tested gross income of $23,853. The pathway deducts Medicare Part B premiums from countable income, which brings it to about $21,400, under the limit." +us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its 'No' is consistent with comparing gross income of $23,853 with an FPL limit. The senior-or-disabled pathway subtracts the Medicare Part B premium, which leaves countable income of about $21,400, below 138% FPL, so the head is eligible." +us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against the 138% FPL Aged & Disabled limit without deducting health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) puts countable income under the limit." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It correctly identified the non-MAGI 138% FPL limit but measured gross monthly income against it. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium, which brings income below the limit." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is the aged (senior-or-disabled) category: $23,853 gross minus the Medicare Part B premium gives about $21,400, under 138% FPL, with assets passing. The 'No' reflects skipping that premium deduction." +us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It compared the full $23,853 with the aged Medi-Cal threshold without subtracting the Medicare Part B premium. The senior-or-disabled pathway deducts that premium, leaving countable income below 138% FPL." +us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It correctly identified the non-MAGI elderly pathway but did not deduct health insurance premiums, including Medicare Part B, from countable income. That deduction brings income from $23,853 to about $21,400, under the 138% FPL limit." +us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It said total income 'well' exceeds 138% FPL, but gross income is only about 150% FPL. After the senior-or-disabled pathway deducts the Medicare Part B premium, countable income falls below 138% FPL." +us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility rules out Medicaid. Medicare-eligible seniors qualify for Medicaid (dual eligibility) through the aged senior-or-disabled pathway, which this head meets on income after the Part B deduction and on assets." +us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It tested the head against the ACA adult expansion limit, which does not apply at age 67, and overstated income as about 199% FPL; $23,853 is about 150% FPL. The applicable pathway is senior-or-disabled, where countable income after the Medicare Part B premium deduction is under 138% FPL." +us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It looked for a disability flag and missed that age 67 alone meets the aged criterion for the senior-or-disabled pathway. Under that pathway, income net of the Medicare Part B premium is below 138% FPL and the $4,200 in assets passes the asset test." +us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied a spend-down (medically needy) standard and counted only the $50 over-the-counter expense as a medical deduction. The applicable pathway is the 138% FPL senior-or-disabled program, which deducts the Medicare Part B premium (about $2,435/year) and leaves countable income under the limit." +us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its 'above threshold' conclusion matches testing gross income. The senior-or-disabled pathway subtracts the Medicare Part B premium from $23,853, leaving about $21,400, below 138% FPL." +us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It correctly picked the aged pathway but compared unadjusted retirement income with the limit. That pathway deducts health insurance premiums, including Medicare Part B, which brings countable income under 138% FPL." +us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used an SSI-linked income level as the aged Medicaid test. California's optional senior-or-disabled pathway uses 138% FPL, and after the Medicare Part B premium deduction the head's countable income is under that limit." +us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied only the $20/month general income exclusion and left out the senior-or-disabled deduction for health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) brings countable income below 138% FPL." +us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It compared gross retirement income with the aged threshold without deducting the Medicare Part B premium. The senior-or-disabled income count requires that deduction, which leaves the head at about $21,400, under 138% FPL." +us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted only the general income exclusion and missed the health insurance premium deduction, including Medicare Part B, that the senior-or-disabled pathway applies. With that deduction, countable income falls below 138% FPL." +us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It claimed PolicyEngine excludes Medicare-eligible (dual-eligible) people from Medicaid. The aged senior-or-disabled pathway is built for Medicare-age individuals, and this head meets its income test (after the Part B deduction) and its asset test." +us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It treated $4,200 in bank assets as over the aged resource limit, but $4,200 easily passes California's senior-or-disabled asset test ($130,000 for an individual in 2026). It also tested gross income of $23,853 without deducting the Medicare Part B premium, which brings countable income under 138% FPL." +us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It mixed MAGI and ABD counting and compared the full $23,853 with the Aged & Disabled FPL limit. The non-MAGI count deducts the Medicare Part B premium (about $2,435/year), leaving about $21,400, under 138% FPL." +us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied only the $20 monthly unearned-income disregard before testing against the 138% FPL Aged & Disabled limit. It missed the deduction of health insurance premiums, including Medicare Part B, which brings countable income below the limit." +us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit. California's senior-or-disabled Medi-Cal asset limit is $130,000 for an individual in 2026, so $4,200 passes, and income after the Medicare Part B deduction is under 138% FPL." +us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (about $15,650) for aged Medi-Cal, but California's senior-or-disabled pathway uses 138% FPL. It also counted gross income without deducting the Medicare Part B premium, which brings countable income to about $21,400, under the limit." +us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It said the head fits no non-MAGI aged or disabled pathway because they are not disabled or on SSI. At 67 the head is aged, which qualifies them for the senior-or-disabled category without SSI receipt, and they pass its income test (after the Part B deduction) and its asset test." +us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly excluded the adult expansion group but compared unadjusted retirement income with the aged limit. The senior-or-disabled pathway deducts the Medicare Part B premium, which leaves countable income under 138% FPL." +us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It guessed that roughly $24k of income exceeds the senior limit without using any threshold or deduction. The correct calculation subtracts the Medicare Part B premium (about $2,435) from $23,853, which gives about $21,400, under 138% FPL." +us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the SSI $2,000 resource limit to the $4,200 in bank assets. California's senior-or-disabled asset test ($130,000 individual limit in 2026) is passed. It also compared gross income with the income standard without deducting the Medicare Part B premium, which brings income below 138% FPL." +us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 'typical' ABD limits and treated missing SSI or disability status as disqualifying. The aged senior-or-disabled pathway requires neither, and it deducts the Medicare Part B premium, which leaves countable income under 138% FPL." +us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It concluded the 67-year-old is in no eligibility category. Age 65+ is itself the aged criterion for the senior-or-disabled pathway, where income net of the Medicare Part B premium is below 138% FPL and assets pass." +us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated gross income under 200% FPL as automatic BBCE eligibility and never ran the net income test PolicyEngine applies to this elderly household. Net income of about $1,779/mo ($1,987.75 minus the $209 standard deduction, with no shelter or medical deduction) is above the roughly $1,305/mo net limit. The household is ineligible, so the $23/mo minimum benefit ($276/yr) does not apply." +us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model assumed BBCE at 200% FPL makes a 1-person household categorically eligible and entitled to the minimum benefit. It skipped the 100% FPL net income test that PolicyEngine applies to this elderly household. Net income of about $1,779/mo is above the roughly $1,305/mo limit, so the household is ineligible and gets $0, not $276." +us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated countable income as very low. In fact, all $23,853 of Social Security, pension and IRA income counts as unearned income, which is $1,987.75/mo gross. Its $144/mo ($1,728/yr) benefit only works if net income is about $513/mo, which means it left out most of the pension and Social Security income. The real net income of about $1,779/mo fails the net income test, and 30% of it ($534) is more than the $298 maximum allotment, so SNAP is $0." +us,scenario_031,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model computed gross income correctly at $1,987.75/mo. But it used the 200% FPL categorical-eligibility limit to grant the $24/mo minimum benefit without applying the net income test that PolicyEngine requires for this elderly household. Net income of about $1,779/mo after the $209 standard deduction is above the roughly $1,305/mo limit, so the household is ineligible and SNAP is $0, not $288." +us,scenario_031,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"The model relied on California's 200% FPL BBCE limit and awarded the $24/mo minimum benefit. It never checked the 100% FPL net income test that this elderly household must pass. With no shelter deduction, net income is about $1,779/mo, well above the roughly $1,305/mo limit, so the household is ineligible and the minimum benefit never applies." +us,scenario_031,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"The model correctly found no shelter deduction and a medical expense below the $35 threshold. But it then gave the $24/mo minimum benefit on the basis of 200% FPL categorical eligibility, without applying the net income test that PolicyEngine requires for this elderly household. Net income of about $1,779/mo is above the roughly $1,305/mo 100% FPL limit, so the household is ineligible and SNAP is $0." us,scenario_031,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model explicitly applied no nonrefundable credits, omitting the California aged-or-blind exemption credit available to the 67-year-old filer. It also substituted an estimated $6,043 standard deduction for the applicable $5,706 deduction, but the decisive error is failing to reduce the resulting $75.37 tax by $312.93 of nonrefundable credits, which yields $0." us,scenario_032,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It applied only an age test plus a ~200% FPL CHIP income screen to household income of $42,664 and never ran the Medicaid-first screen that CHIP requires: Minnesota's Medical Assistance covers children 6-18 (the engine's OLDER_CHILD category) well above this ~160% FPL household, and a Medicaid-eligible child is categorically excluded from CHIP. It also sized the household at four people when only three are listed, inflating the FPL denominator it compared against." us,scenario_032,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"It named the correct Minnesota number — 275% FPL for children — but that band is Medical Assistance (Medicaid) for children ages 2-18, not CHIP, so computing ~160% FPL established Medicaid eligibility under the OLDER_CHILD category. Treating 'MN CHIP/Medicaid expansion thresholds' as one interchangeable ceiling let it skip the mutual-exclusivity step that makes a Medicaid-eligible child ineligible for CHIP." @@ -2534,132 +2833,150 @@ us,scenario_032,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibilit us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, substituted a roughly $16,150 MFJ standard deduction and a $1,000 CTC, and thereby produced $1,477 of pre-credit tax minus $1,000. Under the applicable 2026 parameters, the $2,200 CTC supplies $500 of nonrefundable credit, which fully reduces the liability to $0; its remaining $1,700 and the EITC are refundable and do not enter this output." us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits." -us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model built the correct structure — plateau EITC for one qualifying child plus the $1,700 refundable CTC cap — but then discarded the exact 2026 one-child maximum it had already named ($4,427) and substituted a rounded ""approximately $4,420,"" producing $6,120 instead of $6,127. The entire $7 gap is that self-inflicted rounding of a statutory table value." -us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model counted ""three qualifying dependents (spouse age 18, child age 6, and using the 3-person standard deduction structure),"" but the 18-year-old is the filing spouse on a joint return and is never a qualifying child, so this household has exactly one EITC qualifying child. It then compounded that with a fabricated three-child maximum of ""approximately $3,600"" (the actual three-child 2026 maximum is over $8,000), a stale $1,600 ACTC cap instead of the 2026 cap of $1,700, and an unexplained phase-out haircut to $3,995 even though AGI of $30,916 sits below the joint one-child phase-out start." -us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly placed the household on the EITC plateau and correctly capped the refundable CTC at $1,700, but plugged in $4,328 — the 2025 one-child maximum — after itself citing the 2026 figure of $4,427, and then bolted on an arbitrary ""$215 (estimate refinement)"" with no rule behind it. Using the 2026 maximum with no refinement term gives $4,427 + $1,700 = $6,127." -us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model asserted the EITC ""phases down from max"" at $29,000 and cut it to $3,372, but the joint-filer one-child phase-out for 2026 does not begin until above this household's $30,916 AGI, so the credit stays at the full $4,427 plateau amount — the model applied single-filer phase-out geometry to a married-filing-jointly return. Its $1,700 refundable CTC was right, so the entire $1,055 shortfall is the phantom phase-out." -us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household ""in the phaseout range"" and reduced the EITC to roughly $3,412, when the joint one-child phase-out starts above the household's $30,916 AGI and the credit remains at the full $4,427. It then papered over its own shortfall with an undefined ""$1,100 remaining"" component stacked on top of the $1,700 ACTC, so its $6,212 total is an unphased EITC-sized answer assembled from a phased EITC plus a fabricated third credit." -us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied the single-filer $34,000 second-tier Social Security threshold to a joint return, whose second tier is $44,000, inflating taxable Social Security to $2,557 instead of the correct 0.5 × ($35,832 − $32,000) = $1,916 and AGI to $31,557 instead of $30,916. It then used a $30,000 standard deduction rather than the 2026 married-joint amount of $32,200, manufacturing $156 of tax liability and a $1,844 ""remaining CTC"" ACTC off a $2,000 per-child credit, when 2026 law gives a $2,200 credit with the refundable portion hard-capped at $1,700 and zero taxable income; it also shaved the EITC to $4,330 with a phase-out that does not start at this income." -us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model reached the right structure and near-right total (""$4,300 + $1,700 = $6,000"") and then abandoned it, inflating the answer to $8,200 on the strength of an invented ""additional refundable credit for spouse as a qualifying dependent under expanded family credit provisions"" — no such credit exists, and a filing spouse generates no child-related credit. Correcting the EITC to the 2026 one-child maximum of $4,427 and dropping the fictitious spouse credit yields $6,127." -us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a one-child EITC maximum of $5,177 and a joint phase-out threshold of $27,460 — neither is a 2026 parameter; the 2026 one-child maximum is $4,427 and the joint phase-out begins above this household's $30,916 AGI. Because it started too high and then subtracted a phase-out that does not apply, its $4,625 EITC overshoots the plateau amount by $198, which is the whole error given its correct $1,700 refundable CTC." -us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the pre-TCJA additional child tax credit rules — ""15% of earned income over $3,000, capped at $1,000"" — but 2026 law sets the credit at $2,200 per child with a refundable cap of $1,700 and a $2,500 earnings floor, so the refundable CTC is $1,700, not $1,000. Its EITC of $4,488 also overstates the 2026 one-child plateau maximum of $4,427 by $61." -us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model phased the EITC down to $3,797 even though the joint one-child phase-out begins above this household's $30,916 AGI, leaving the credit at the full $4,427, and paired it with the repealed pre-TCJA $1,000 refundable CTC instead of the 2026 $1,700 cap. Its submitted figure also diverges from its own explanation, which totals $4,797 while the recorded answer is $5,430 — neither reconstructs $4,427 + $1,700." -us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the pre-TCJA $1,000 additional child tax credit rather than the 2026 refundable cap of $1,700, and reported an EITC of $3,599.34 — $828 below the $4,427 one-child plateau maximum that applies because joint phase-out does not begin until above this household's $30,916 AGI." -us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no parameters, but $5,462 decomposes as a roughly $4,462 EITC plus the repealed pre-TCJA $1,000 refundable child tax credit; 2026 law caps the refundable portion of the $2,200 credit at $1,700, and the one-child plateau EITC is exactly $4,427. It also anchored on ""an AGI of $29,000,"" ignoring the $1,916 of taxable Social Security that brings AGI to $30,916." -us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly applied ""the pre-TCJA Additional Child Tax Credit of $1,000,"" assuming the 2017 expansion lapsed for 2026; the 2026 credit is $2,200 per child with $1,700 refundable, and with zero tax liability the full $1,700 is paid out. Its EITC of ""approx. $4,440"" also runs $13 above the 2026 one-child maximum of $4,427." -us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the lapsed pre-TCJA $1,000 additional child tax credit instead of the 2026 refundable cap of $1,700, and set the EITC at $4,014, which is $413 under the $4,427 one-child plateau maximum that applies at this household's $30,916 AGI." -us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model submitted a bare $3,745 with no derivation, $2,382 below the correct $4,427 EITC plus $1,700 refundable CTC; the figure is consistent with pairing the repealed $1,000 pre-TCJA additional child tax credit with an EITC phased down to roughly $2,745, when no phase-out applies at $30,916 AGI for a joint filer with one child." -us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model stated outright that ""under 2026 tax law following the TCJA expiration, the Child Tax Credit is $1,000 per child, and the ACTC refundable portion is 15% of earned income above $3,000, capped at $1,000"" — the expansion did not expire; 2026 gives $2,200 per child with a $1,700 refundable cap and a $2,500 earnings floor, so the refundable CTC is $1,700. Its EITC of $4,428 was within a dollar of the correct $4,427, making the $700 CTC understatement the entire error." -us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model named only ""EITC and the refundable Child Tax Credit"" with no parameters; $5,248 decomposes as roughly $4,248 of EITC plus the repealed pre-TCJA $1,000 refundable CTC. The 2026 values are the $4,427 one-child plateau maximum and a $1,700 refundable CTC cap, totaling $6,127." -us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model asserted eligibility without computing anything and submitted $4,056, $2,071 short; the number is consistent with the repealed $1,000 pre-TCJA additional child tax credit plus an EITC phased down to about $3,056. At $30,916 AGI a joint filer with one qualifying child sits on the EITC plateau at $4,427, and the 2026 refundable CTC cap is $1,700." -us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_refundable_credits, so the response never entered the substantive computation. The contract required every requested key in `outputs` with a numeric value; this key was absent." -us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the entire 2026 CTC of $2,200 as refundable once the 15% earnings phase-in was satisfied, but the refundable portion is separately capped at $1,700 per child and the remaining $500 is nonrefundable — with zero taxable income that $500 is unusable. Its EITC of $4,095 also fell $332 below the $4,427 one-child plateau maximum it claimed to be applying." -us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model declared that ""no refundable federal credit is identified from the provided facts"" and returned zero, despite $29,000 of wages and a 6-year-old qualifying child on a joint return — facts that produce the full $4,427 one-child EITC and, with zero tax liability, the full $1,700 refundable child tax credit. It treated an unstated credit as an absent one rather than computing eligibility from the stated household facts." -us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned zero on the reasoning that ""no qualifying refundable credit amounts are specified"" in the inputs, but refundable credits are computed from wages and household composition, not supplied: $29,000 of earnings with one qualifying child yields a $4,427 EITC on the joint plateau and a $1,700 refundable CTC. Its ""low earnings"" characterization also inverts the EITC, which is maximized rather than eliminated at this earnings level." -us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model nailed the EITC at $4,427 and the zero-liability logic, then used a refundable child tax credit limit of $1,800; the 2026 refundable cap is $1,700 per child, and the $100 overstatement is the entire $6,227 versus $6,127 gap." -us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the right two components — plateau EITC for one qualifying child and the refundable CTC — but its $6,047 is $4,347 + $1,700, understating the 2026 one-child EITC maximum by $80. That $4,347 is an interpolated inflation bump off the 2025 figure of $4,328 rather than the published 2026 maximum of $4,427." -us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,other,False,"The model submitted a round $2,500 with no parameters, rates, or arithmetic — less than the $4,427 EITC alone, before any child tax credit. The correct derivation is the one-child joint plateau EITC of $4,427 plus the $1,700 refundable CTC (earnings of $29,000 clear the $2,500 floor and 15% × $26,500 = $3,975 exceeds the cap), giving $6,127; $2,500 corresponds to no step in that computation." -us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly kept the EITC on the plateau but applied the pre-TCJA refundable child tax credit formula — ""15 percent of earned income over 3000, capped at 1000"" — when 2026 uses a $2,500 earnings floor and a $1,700 refundable cap against a $2,200 credit. Its $4,436 EITC also runs $9 above the 2026 one-child maximum of $4,427." -us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset, calling the refundable amount ""post-sunset additional CTC"" of $1,000 under the $3,000 floor rule; 2026 law keeps a $2,200 credit with $1,700 refundable and a $2,500 floor, so the refundable CTC is $1,700. It also used $4,318 for the one-child EITC maximum instead of the 2026 value of $4,427, despite correctly computing AGI as $30,916 and correctly finding the household below phase-out." -us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model got the hard parts right — AGI of $30,916 including taxable Social Security, and the household below the joint one-child EITC phase-out — but then paid out ""the full $1,000 Child Tax Credit,"" the pre-TCJA amount, instead of the 2026 $2,200 credit whose refundable portion is capped at $1,700. Its $4,439 EITC is also $12 above the 2026 one-child maximum of $4,427." -us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model reasoned correctly throughout — AGI $30,916, below the joint one-child phase-out start, refundable CTC capped at $1,700 — but used $4,449 as the 2026 one-child EITC maximum instead of $4,427, a $22 overstatement that accounts for the entire $6,149 versus $6,127 gap." -us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_refundable_credits, so no substantive computation was attempted. The required key was absent from `outputs`." -us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model computed the ""EITC for MFJ with two children,"" counting the 18-year-old spouse as a second qualifying child; a filing spouse on a joint return is never a qualifying child, so only the 6-year-old counts and the credit is the one-child plateau amount of $4,427. It then contradicted itself by naming a $2,000 refundable CTC and omitting it entirely from the total, submitting the EITC figure alone instead of adding the $1,700 refundable child tax credit." -us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model phased the EITC out from a ""$22,190 joint-filer threshold,"" cutting it to about $3,348, but the 2026 joint one-child phase-out does not begin until above this household's $30,916 AGI, leaving the full $4,427. Its $1,700 refundable CTC and zero-liability reasoning were correct, so the fictitious $1,079 phase-out is the entire error." -us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model cycled through fabricated one-child EITC maximums ($5,640, $5,386) and joint phase-out thresholds ($22,000–$23,500), applying a 15.98% phase-out that does not begin until above this household's $30,916 AGI, and settled on $4,509 rather than the $4,427 plateau amount. It then abandoned its own $1,700 ACTC computation for an unexplained ""refundable CTC ~$800 (limited),"" when the credit is limited only by the $1,700 refundable cap, which 15% × $26,500 = $3,975 of phase-in fully satisfies." -us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model reported an EITC of $1,171 and a refundable CTC of $937, both drastically phased down: the joint one-child EITC is at its $4,427 plateau maximum at $30,916 AGI, and the refundable CTC hits its $1,700 cap because 15% × ($29,000 − $2,500) = $3,975 exceeds it. Both components were reduced against income thresholds that this household is nowhere near, understating the total by $4,019." -us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model applied only the two federal NSLP pathways — the 130% FPG income test (computing ~$34,645 for a household of three) and SNAP/TANF direct certification — and stopped there. It never applied Minnesota's universal free school meals law (Minn. Stat. § 124D.111), which PolicyEngine flags as state_has_universal_free_school_meals = True and which sets school_meal_tier = FREE for the K-12 child irrespective of the household's 1.56 FPG ratio." -us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model rejected eligibility on the federal 130% FPG test and the absence of TANF/SNAP direct certification, missing Minnesota's universal free school meals program (Minn. Stat. § 124D.111), which grants FREE tier to every enrolled K-12 student regardless of income. It also overstated the income position, calling $42,664 'about 170% FPL' for a family of three when the engine's school_meal_fpg_ratio is 1.56." -us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model asserted that free meals 'in Minnesota through the National School Lunch Program require household income at or below 130% of the Federal Poverty Level,' explicitly denying the existence of the state pathway that decides this case. Minnesota's 2023 Free School Meals for Kids Act (Minn. Stat. § 124D.111) provides free breakfast and lunch to all students statewide, which PolicyEngine encodes as state_has_universal_free_school_meals = True and which yields $1,130.96 for the one K-12 child." -us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model computed the FPG ratio correctly (~157%, against the engine's 1.56) but treated the federal 130% free-meal cutoff as the only route to FREE tier. It omitted Minnesota's universal free school meals statute (Minn. Stat. § 124D.111), the state_has_universal_free_school_meals override that sets the tier to FREE without any income test." -us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model reduced the question to a single comparison, '$42,664 > 130% FPL $35,326; not free-eligible,' applying only the federal NSLP income screen. It never checked the state code, missing Minnesota's universal free school meals program (Minn. Stat. § 124D.111) that makes every K-12 student FREE-tier regardless of the 1.56 FPG ratio." -us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model concluded the child is ineligible for 'federal free school meals' and treated that federal determination as the final answer. Minnesota supplies its own universal free school meals program (Minn. Stat. § 124D.111), which PolicyEngine applies as a state-level FREE-tier override on top of the federal test, producing positive free-meal support for the 6-year-old." -us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model correctly assembled $42,664 of school-meal countable income ($29,000 wages plus $13,664 Social Security dependent benefits) but then applied only the federal 130% FPG free threshold. It missed Minnesota's universal free school meals law (Minn. Stat. § 124D.111), which PolicyEngine models as state_has_universal_free_school_meals = True and which grants FREE tier independent of income." -us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model stopped at the federal income screen, placing the household at roughly 153% of the poverty guideline (engine value 1.56) and above the 130% free cutoff. It never applied Minnesota's universal free school meals statute (Minn. Stat. § 124D.111), the state pathway that sets school_meal_tier = FREE for the enrolled K-12 child regardless of that ratio." -us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model defaulted to ineligible purely because no categorical SNAP/TANF receipt was given, using absence of direct certification as the deciding fact. Categorical eligibility is indeed False here, but Minnesota's universal free school meals program (Minn. Stat. § 124D.111) independently confers FREE tier on every K-12 student, which is what produces the $1,130.96 of free-meal support." -us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model applied the federal free-meal income limit for a three-person household plus the absence of categorical free-meal status and concluded ineligibility. It omitted the Minnesota state pathway — Minn. Stat. § 124D.111 universal free school meals, encoded as state_has_universal_free_school_meals = True — which overrides the income tier and returns positive support for the K-12 child." -us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model's entire justification was that household income exceeds 'the federal free-school-meal threshold for three people,' treating the federal 130% FPG test as exhaustive. Minnesota's universal free school meals program (Minn. Stat. § 124D.111) sets the tier to FREE for all enrolled students, so the 1.56 FPG ratio does not block eligibility." -us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model checked only two conditions — income above the modeled federal school-meal threshold and no direct-certification benefit — and never consulted the state of residence. Minnesota's universal free school meals law (Minn. Stat. § 124D.111) is the operative pathway, giving the 6-year-old K-12 student FREE tier and $1,130.96 in annual benefit value." -us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model computed the FPG ratio accurately (~156%, matching the engine's 1.56) and then applied the federal 130% free-meal limit as the sole test. It missed Minnesota's universal free school meals program (Minn. Stat. § 124D.111), the state_has_universal_free_school_meals override that makes the income ratio irrelevant to the FREE tier determination." -us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model checked the federal 130% FPG limit and SNAP/TANF categorical eligibility, and additionally mislabeled school-meal countable income as 'MAGI' — school meals use a gross-income-and-household-size test, not MAGI. Its decisive omission is Minnesota's universal free school meals statute (Minn. Stat. § 124D.111), which PolicyEngine applies as a state-level FREE-tier grant for every K-12 student." -us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model's poverty-guideline arithmetic was sound (~156.7% of the three-person guideline, matching the engine's 1.56) but it treated the federal 130% FPG threshold as dispositive. It never applied Minnesota's universal free school meals program (Minn. Stat. § 124D.111), which sets school_meal_tier = FREE for the enrolled 6-year-old regardless of income." -us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model compared $42,664 against a 130% FPG figure of $33,566 for three people and declared the household ineligible on that single federal test. It omitted the Minnesota universal free school meals pathway (Minn. Stat. § 124D.111), flagged in the engine as state_has_universal_free_school_meals = True, which confers FREE tier without any income screen." -us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for free_school_meals_eligible, so the submission carried no substantive determination to score. The correct derivation is that Minnesota's universal free school meals program (Minn. Stat. § 124D.111) sets school_meal_tier = FREE for the K-12 child, yielding $1,130.96 of annual free-meal support and an answer of 1." -us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model explicitly framed the test as 'PolicyEngine's federal school-meal income test,' applying the 130% free-meal limit against a ~156% poverty ratio. PolicyEngine also carries a state layer, and Minnesota's universal free school meals law (Minn. Stat. § 124D.111) sets the tier to FREE for all K-12 students, which is the pathway the model never evaluated." -us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model correctly counted the $13,664 Social Security dependent benefit into school-meal income but then applied only the federal 130% cutoff (and overstated the ratio as ~160% against the engine's 156%) plus the absent SNAP/TANF pathway. It missed Minnesota's universal free school meals statute (Minn. Stat. § 124D.111), the state override that grants FREE tier to the enrolled 6-year-old regardless of income or direct certification." +us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly identified the 2026 one-child maximum EITC of $4,427 and placed $29,000 below the MFJ phase-out start. It then rounded the credit down to an 'approximately' $4,420 instead of using the exact $4,427, so its total of $4,420 + $1,700 = $6,120 is $7 short." +us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It counted the 18-year-old spouse as a qualifying dependent and referred to '3 qualifying children', but a spouse on a joint return is never a qualifying child, so only the 6-year-old counts. It also used an invented $3,600 EITC and a $1,600 refundable CTC, and it applied a phase-out at $29,000 that does not reach this household, which sits below the MFJ one-child phase-out start. The correct figures are the full $4,427 EITC plus the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It stated that the 2026 one-child maximum is $4,427 and that $29,000 is on the plateau. It then plugged in the 2025 maximum of $4,328 and added an arbitrary $215 'estimate refinement', producing $6,243 instead of $4,427 + $1,700 = $6,127." +us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It phased the EITC down to $3,372 at $29,000 of earnings. The 2026 MFJ one-child phase-out starts above $31,000, above both earned income ($29,000) and AGI ($30,916), so the full $4,427 applies. Its reduction matches a single-filer threshold." +us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It put $29,000 of MFJ earnings in the EITC phase-out range and cut the credit to $3,412 instead of the full $4,427. It then added a $1,100 'remaining' CTC amount on top of the $1,700 ACTC, but refundability is capped at $1,700 per child and the non-refundable remainder of the $2,200 CTC is unused at zero tax." +us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a stale $27,638 MFJ phase-out start inflated to $28,329, which trimmed the EITC to $4,329. The actual 2026 MFJ start is above $31,000, so the full $4,427 applies. It also assumed a $2,000 CTC with a $30,000 standard deduction and computed taxable Social Security as $2,557 instead of $1,916, which created phantom tax. Its ACTC of $1,844 ignores the $1,700 per-child refundable cap; the correct 2026 standard deduction of $32,200 gives zero tax and exactly $1,700 of ACTC." +us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It reached about $6,000 from the EITC plus the $1,700 ACTC. It then inflated the total to $8,200 by inventing an 'additional refundable credit for spouse as a qualifying dependent'. A spouse on a joint return cannot be a dependent, and no such credit exists. Its EITC maximum of $4,200-4,300 is also below the 2026 figure of $4,427." +us,scenario_032,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a one-child EITC maximum of $5,177 instead of the 2026 figure of $4,427. It also used a $27,460 MFJ phase-out threshold, well below the 2026 start of above $31,000. Its phased-out $4,625 still exceeds the true maximum, so the total overshoots at $6,325." +us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the post-TCJA-sunset refundable CTC: 15% over $3,000, capped at $1,000. OBBBA made the $2,200 CTC permanent with a 2026 refundable cap of $1,700 and a $2,500 earnings floor, so the refundable CTC is $1,700. Its EITC of $4,488 also overstates the 2026 one-child maximum of $4,427." +us,scenario_032,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It phased the EITC down to $3,797 even though earned income and AGI are below the 2026 MFJ one-child phase-out start, so the full $4,427 applies. It used the expired $1,000 refundable CTC instead of the $1,700 OBBBA cap. Its reasoning totals $4,797 but it submitted $5,430, and neither matches $6,127." +us,scenario_032,federal_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It started the EITC phase-out at $23,052, a single-filer-level threshold that omits the MFJ increase. It also understated the maximum at $4,297, which drove the EITC down to $3,040. The 2026 MFJ one-child phase-out starts above $31,000, so the full $4,427 applies; its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the expired post-TCJA $1,000 refundable CTC instead of the 2026 OBBBA refundable cap of $1,700. It also reduced the EITC to $3,599.34 even though the household is below the MFJ one-child phase-out start and receives the full $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no breakdown. Its $5,462 fits an EITC near the maximum plus the expired $1,000 refundable CTC, not the 2026 $1,700 ACTC cap. The correct total is the full $4,427 EITC plus the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly applied the 'pre-TCJA' $1,000 Additional Child Tax Credit, but OBBBA made the $2,200 CTC permanent with a 2026 refundable cap of $1,700. Its $4,440 EITC also overstates the 2026 one-child maximum of $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $1,000 ACTC, the expired post-sunset cap, instead of the 2026 $1,700 refundable cap. It also set the EITC at $4,014 even though the household is on the plateau below the MFJ phase-out start and receives the full $4,427." +us,scenario_032,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no breakdown. The correct figure is the full $4,427 EITC plus the $1,700 ACTC, because earned income of $29,000 and AGI of $30,916 are below the 2026 MFJ one-child phase-out start and tax is zero. Its $3,745 is $2,382 short, which requires phasing out an EITC that is not in phase-out and/or using the expired $1,000 ACTC." +us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used a $1,000 ACTC with a $3,000 floor, but OBBBA made the $2,200 CTC permanent with a $1,700 refundable cap and a $2,500 floor. Its EITC of $4,428 is within $1 of the correct figure, so the $700 ACTC shortfall drives the error. It also misattributed the head's Social Security benefits to the child." +us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $5,248 is $879 short of the full $4,427 EITC plus the $1,700 ACTC. That gap fits the expired $1,000 refundable CTC paired with a slightly low EITC, instead of the 2026 $1,700 cap and the full one-child maximum." +us,scenario_032,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It gave no breakdown. Its $4,056 is $2,071 short of the full $4,427 EITC plus the $1,700 ACTC. It either phased out an EITC that sits on the MFJ plateau or understated the refundable CTC below the $1,700 cap." +us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so no answer was scored." +us,scenario_032,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the entire $2,200 2026 CTC as refundable. The refundable portion is capped at $1,700 per child, so the ACTC is $1,700. It also used an EITC maximum of $4,095 instead of the 2026 one-child maximum of $4,427." +us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no refundable credit. The household has $29,000 of earned income and a qualifying 6-year-old, so it qualifies for the full $4,427 EITC and the $1,700 refundable CTC." +us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed refundable credits were zero because no credit amounts were 'specified' in the facts. It never computed the EITC ($4,427 for one child, MFJ, on the plateau) or the $1,700 refundable CTC that the household's $29,000 of earnings and qualifying child generate." +us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its $4,427 EITC was correct, but it used a refundable CTC limit of $1,800. The 2026 per-child refundable cap is $1,700, so it overstated the total by $100." +us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It named the correct two credits but gave no amounts. Its $6,047 is $80 below $4,427 + $1,700, which implies an EITC of about $4,347 instead of the 2026 one-child maximum of $4,427 that applies on the MFJ plateau." +us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,other,False,"It gave only a one-line justification with no computation. Its $2,500 falls far below the full $4,427 EITC plus the $1,700 refundable CTC, so it did not carry out either credit's calculation." +us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the post-sunset $1,000 CTC (15% over $3,000, capped at $1,000) instead of OBBBA's $2,200 CTC with a 2026 refundable cap of $1,700. Its projected EITC of $4,436 also overstates the 2026 one-child maximum of $4,427." +us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a 'post-sunset' additional CTC of $1,000, but OBBBA sets the 2026 refundable cap at $1,700. It also set the one-child EITC maximum at $4,318 instead of $4,427." +us,scenario_032,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $1,700 ACTC and plateau placement were right, but it projected the 2026 one-child EITC maximum as $4,436 instead of the actual $4,427, overshooting by $9." +us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It treated the CTC as $1,000 and fully refundable, but the 2026 CTC is $2,200 with a $1,700 refundable cap, which this household reaches because tax is zero and 15% of $26,500 is $3,975. It also projected the EITC maximum as $4,439 instead of $4,427." +us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its $1,700 ACTC and plateau placement were right, but it projected the 2026 one-child EITC maximum as $4,449 instead of $4,427, overshooting by $22." +us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so no answer was scored." +us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"It computed the EITC for two children even though only the 6-year-old qualifies, because the spouse cannot be a qualifying child. It then left the refundable CTC out of the total entirely and reported only the $4,205 EITC. The correct total is the one-child EITC of $4,427 plus the $1,700 ACTC." +us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It began the EITC phase-out at about $22,190, a single-filer-level threshold, and cut the credit to $3,348. The 2026 MFJ one-child phase-out starts above $31,000, so the full $4,427 applies; its $1,700 ACTC was correct." +us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It invented one-child EITC maximums of $5,386 to $5,640 and phase-out starts near $22,000, which yielded an EITC of $4,509 instead of the full $4,427. It then arbitrarily 'limited' the refundable CTC to $800, even though tax is zero and 15% of $26,500 exceeds the $1,700 cap, so the ACTC is $1,700." +us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It reported an EITC of $1,171 and an ACTC of $937, both badly understated. The EITC reflects a deep phase-out that does not apply: earned income of $29,000 and AGI of $30,916 are below the MFJ one-child phase-out start, so the full $4,427 applies. The ACTC ignores that 15% of ($29,000 - $2,500) = $3,975 hits the $1,700 cap." +us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Compared $42,664 with 130% FPL (~$34,645) and ruled out SNAP/TANF categorical eligibility, but never applied Minnesota's universal free school meals program. That program makes the 6-year-old K-12 child eligible for free meals whatever the household earns." +us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Treated the 130% FPL income test and TANF/SNAP direct certification as the only free-meal pathways and overlooked Minnesota's universal free school meals program, which gives the K-12 child FREE tier status. Its ~170% FPL figure also overstates the engine's 1.56 ratio, but the missed state program is what decides the answer." +us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,Applied only the federal 130% FPL free-meal limit to income at ~160% FPL and ignored that Minnesota provides universal free school meals. Under that program the household's K-12 child qualifies for free meals whatever the income. +us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Explicitly framed Minnesota eligibility as the NSLP 130% FPL test and concluded the household gets reduced-price meals at most. It missed that Minnesota's universal free school meals law makes every K-12 student eligible for free meals, so the income comparison is not decisive." +us,scenario_032,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Compared countable income of ~$42,700 with 130% FPL (~$34,600) and stopped there. It never accounted for Minnesota's universal free school meals program, which puts the K-12 child in the FREE tier." +us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Correctly found income at ~157% FPL, above the 130% federal free-meal threshold, but ignored Minnesota's universal free school meals program. That program grants the K-12 child free meals without any income test." +us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied only the federal income test ($42,664 > 130% FPL of $35,326) and missed that Minnesota's universal free school meals program makes the household's K-12 child free-meal eligible regardless of income." +us,scenario_032,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Compared $42,664 with 130% FPL (~$35,750) and concluded ineligibility, never applying Minnesota's universal free school meals program. That program extends free meals to every K-12 student in the state." +us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Explicitly limited its analysis to federal free school meals under the 130% FPL limit. It skipped Minnesota's universal free school meals program, which makes the 6-year-old eligible for free meals regardless of income." +us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Correctly summed $29,000 wages and $13,664 Social Security into $42,664 and found it above 130% FPL. It then treated that federal income test as final and missed Minnesota's universal free school meals program, which grants the K-12 child the FREE tier." +us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,Used only the federal 130% poverty-guideline cutoff (income ~153% FPL) and never applied Minnesota's universal free school meals program. Under that program the K-12 child qualifies for free meals without an income test. +us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed free meals require SNAP/TANF categorical eligibility and defaulted to ineligible when those were absent. It ignored Minnesota's universal free school meals program, which makes the K-12 child eligible with no categorical or income condition." +us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Noted the school-age child but applied only the federal free-meal income limit and the categorical-status check. It missed that Minnesota's universal free school meals program places the child in the FREE tier regardless of the $42,664 income." +us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,Relied solely on the federal free-meal income threshold for a three-person household and never applied Minnesota's universal free school meals program. That program makes the K-12 child free-meal eligible whatever the household earns. +us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Checked only the federal income threshold and direct certification through benefit receipt. It omitted Minnesota's universal free school meals program, which is the decisive pathway giving the K-12 child free meals." +us,scenario_032,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied only the free-meal income threshold for a three-person household and missed Minnesota's universal free school meals program. That program grants free meals to the K-12 child without regard to income. +us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Correctly placed income at ~156% of the 2026 FPL, above the 130% federal free-meal limit, but treated that limit as dispositive. It ignored Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals." +us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"Found income at ~156% FPL and no SNAP/TANF categorical eligibility, then concluded PolicyEngine returns no free-meal support. It missed that PolicyEngine applies Minnesota's universal free school meals flag, which sets the child's tier to FREE regardless of income." +us,scenario_032,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Ruled out free meals through the 130% FPL income test and SNAP/TANF categorical eligibility, but never applied Minnesota's universal free school meals program. That program qualifies the K-12 child for free meals on its own." +us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Computed income at ~156.7% FPL and applied only the federal 130% free-meal threshold. It overlooked Minnesota's universal free school meals program, which grants the K-12 child free meals regardless of income." +us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"Compared gross income of ~$42,664 with 130% FPL ($33,566) and stopped at the federal income test. It missed Minnesota's universal free school meals program, which makes the household's K-12 child free-meal eligible." +us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for free_school_meals_eligible, so no answer was scored. The correct answer is Yes, because Minnesota's universal free school meals program puts the K-12 child in the FREE tier." +us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"Explicitly applied only PolicyEngine's federal school-meal income test (~156% of poverty, above 130%). It ignored the state_has_universal_free_school_meals pathway, which is true for Minnesota and sets the tier to FREE regardless of income." +us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Counted the $13,664 Social Security benefit, found income ~160% of poverty, and ruled out a SNAP/TANF categorical pathway. It missed the third pathway, Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals." us,scenario_032,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits from the Medicaid MAGI calculation and therefore compared only $29,000 of wages with 138% FPL. The applicable MAGI computation is 1.56 times FPL, above the adult limit, and the head has no other qualifying category." -us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged that including Social Security produces income above 138% FPL but then invented a higher Minnesota parent/caretaker pathway. The head qualifies through no parent, caretaker, or other Medicaid category at the traced 1.56-FPL MAGI, so the assigned category is NONE." -us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and treated wages alone as Medicaid MAGI. The traced MAGI is 1.56 times FPL, not below the 138% adult threshold, and no alternate category applies." -us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model invented a 275%-FPL Medicaid threshold for Minnesota parents or caretaker relatives after recognizing that total income exceeds 138% FPL. No such qualifying pathway applies to the head in this computation; at 1.56 times FPL the engine assigns Medicaid category NONE. -us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a household size of four even though the household contains three people; an income item is not an additional household member. That inflated the poverty threshold and obscured the traced MAGI ratio of 1.56 FPL, which is above the applicable adult limit." -us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted low-income eligibility without applying the Minnesota adult Medicaid threshold. The head's MAGI is 1.56 times FPL, above the applicable MAGI limits, and the head qualifies through no other category." -us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model placed household MAGI inside Minnesota's adult expansion range without performing the controlling comparison. The traced MAGI equals 1.56 FPL, exceeding the applicable adult limit, and the eligibility category is NONE." -us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,The model substituted a generic low-income characterization for the required eligibility test. At 1.56 FPL the head exceeds the applicable adult Medicaid income limits and has no categorical pathway to eligibility. -us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as $30,916 and 113% FPL, omitting income included in the engine's Medicaid MAGI computation. The correct computation places MAGI at 1.56 FPL, above Minnesota's 138% expansion-adult limit." -us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated the head's Medicaid income as approximately 113.6% FPL. The traced MAGI is 1.56 FPL, so the head is above the 138% adult limit and has no alternate eligibility category." -us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value for the requested output. It therefore failed the required structured-output contract before any substantive Medicaid determination could be evaluated. -us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated the household as being at the Medicaid expansion cutoff and concluded eligibility. The actual MAGI ratio is 1.56 FPL, above the applicable adult threshold, and being under age 65 and off Medicare does not create another Medicaid category." -us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used only about $29,000 to $30,916 as MAGI and derived a poverty ratio of 109% to 116%. The controlling MAGI calculation yields 1.56 FPL, above the 138% expansion-adult limit, with no other qualifying pathway." -us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model recognized that $42,664 exceeds the expansion threshold but then excluded the Social Security dependent benefits by treating them as benefits received by the child, despite the facts assigning them to the head. The traced Medicaid MAGI is 1.56 FPL, and the head consequently qualifies through no Medicaid category." +us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It excluded the $13,664 in Social Security dependent benefits as non-taxable and tested only about $29,000 against 138% FPL. Medicaid MAGI adds back all non-taxable Social Security, so countable income is $42,664, or 156% FPL, which is above the 138% limit." +us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly counted the Social Security benefits and found MAGI of about $42,664, which is above 138% FPL. It then claimed eligibility through a higher Medicaid limit for parents and caretakers, citing MinnesotaCare. MinnesotaCare is a Basic Health Program, not Medicaid, and Minnesota's Medical Assistance limit for parents is 133% FPL plus the 5% disregard, which the Head's 156% FPL exceeds." +us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as excluded from MAGI and counted only the $29,000 in wages. Medicaid MAGI includes the non-taxable part of Social Security, which raises income to $42,664, or 156% FPL, above the 138% expansion-adult limit." +us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly found that $42,664 exceeds 138% FPL, then invented a 275% FPL Medicaid limit for parents and caretakers in Minnesota. Minnesota's Medical Assistance limit for parents is 133% FPL plus the 5% disregard, so at 156% FPL the Head has no Medicaid pathway." +us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It counted the Social Security benefit as a fourth household member, which made the household size 4 and the limit about $43,000. The Medicaid household is 3 people (Head, Spouse, Child 1), so the $42,664 MAGI is 156% of the 3-person FPL and fails the 138% limit." +us,scenario_032,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its MAGI of $30,916 is wages plus only the taxable part of Social Security (about $1,916). Medicaid MAGI adds back the non-taxable Social Security as well, giving $42,664, or 156% FPL for 3 people, which exceeds the 138% limit." +us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted low income without computing MAGI. That is consistent with leaving out the $13,664 in Social Security benefits, which Medicaid MAGI counts in full. The correct MAGI is $42,664, or 156% FPL for a 3-person household, above Minnesota's 138% adult limit." +us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It placed household MAGI within the 138% FPL expansion range, which means it left out the non-taxable Social Security dependent benefits. With the full $13,664 added back, as Medicaid MAGI requires, income is $42,664, or 156% FPL, and the Head is ineligible." +us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,taxable_income_or_deductions,False,"It labeled the household low-income without computing MAGI, which implies it counted only wages. Medicaid MAGI includes all $13,664 in Social Security benefits, which brings income to $42,664, or 156% FPL, above Minnesota's 138% expansion-adult limit." +us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a MAGI of $30,916, meaning wages plus only the taxable part of Social Security, and got 113% FPL. Medicaid MAGI adds back the non-taxable Social Security, giving $42,664, or 156% FPL, which exceeds the 138% limit." +us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its 113.6% FPL figure comes from counting only wages plus the taxable part of Social Security. Medicaid MAGI includes the full $13,664 in Social Security benefits, which puts the Head at 156% FPL, above Minnesota's 138% limit for adults aged 19 to 64." +us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for head_medicaid_eligible, so this output is missing rather than a substantive Medicaid determination." +us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It asserted income at 138% of poverty without computing MAGI and treated that as qualifying. With wages of $29,000 plus the full $13,664 in Social Security, which Medicaid MAGI counts, the Head's MAGI is 156% FPL, above the 138% expansion limit." +us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its MAGI range of $29,000 to $30,916 counts wages plus at most the taxable part of Social Security. Medicaid MAGI adds back all non-taxable Social Security, which gives $42,664, or 156% of the 3-person FPL, above the 138% adult limit." +us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed $42,664 in MAGI, then dropped the Social Security dependent benefits on the theory that a child receives them. It tested only the $29,000 in wages. The benefits are the Head's own income, and Medicaid MAGI counts them in full, so the Head is at 156% FPL, above the 138% limit." us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either." -us,scenario_032,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It computed federal employee FICA correctly ($1,798 OASDI + $420.50 HI) but asserted ""Minnesota has no mandatory employee state payroll tax in 2026,"" missing the Minnesota Paid Leave program whose premiums begin January 1, 2026. The employee share is half of the 0.88% total premium, 0.44% × $29,000 = $127.60, which is the entire $127.60 gap to $2,346.10." -us,scenario_032,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It stopped at 7.65% of $29,000 and stated there is ""no modeled mandatory MN employee payroll tax,"" omitting the Minnesota Paid Leave employee contribution that takes effect for wages paid in 2026. Adding the employee half of the 0.88% premium (0.44% × $29,000 = $127.60) to $2,218.50 yields the $2,346.10 reference." -us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It declared ""Minnesota has no mandatory state payroll tax on employees,"" dropping the 2026 Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60, then compounded the error by submitting $2,175.90 after its own reasoning three times concluded $2,218.50, an unexplained ""rounding"" adjustment that matches no payroll-tax computation." -us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It applied only the 7.65% federal employee rate and stated ""MN has no mandatory employee state payroll tax,"" missing the Minnesota Paid Leave contribution that starts with 2026 wages. The employee half of the 0.88% premium on $29,000 of covered wages is $127.60, exactly the shortfall from $2,218.50 to $2,346.10." -us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Its OASDI and HI components are right, but the claim ""MN has no mandatory employee payroll tax"" is the error: Minnesota Paid Leave premiums begin January 1, 2026, and the employer may deduct up to half the 0.88% premium from the employee, giving 0.44% × $29,000 = $127.60." -us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It concluded there is ""no mandatory MN employee payroll tax"" and reported only federal FICA of $2,218.50, omitting Minnesota's 2026 paid leave employee contribution of 0.44% on $29,000 of taxable wages, $127.60." -us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It explicitly reasoned that ""MN does not impose a separate employee payroll tax beyond federal,"" which is false for tax year 2026 when Minnesota Paid Leave premiums begin; the employee share of the 0.88% premium on $29,000 is $127.60. Its federal components ($1,798 + $420.50) match the reference exactly, so the state omission is the whole error." -us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It stated Minnesota has ""no state disability or paid family leave employee contribution in 2026,"" the exact rule it needed: Minnesota Paid Leave collects a 0.88% premium on covered wages starting in 2026 with up to half charged to the employee, i.e. 0.44% × $29,000 = $127.60 on top of the $2,218.50 federal FICA." -us,scenario_032,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied 7.65% to $29,000 and asserted ""no employee state payroll tax,"" skipping the Minnesota Paid Leave employee premium effective for 2026 wages, 0.44% × $29,000 = $127.60." -us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly identified the Minnesota PFML employee contribution as an additional component but used a 0.35% rate ($101.50) instead of the employee share of Minnesota's 0.88% 2026 premium, 0.44% ($127.60). The $26.10 rate error is the entire difference between its $2,320 and the $2,346.10 reference." -us,scenario_032,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its explanation states ""no employee state payroll tax applies, for a total of $2,218.50"" while it submitted $2,320, a value consistent with tacking on an unstated 0.35% state premium; either way it never applied Minnesota Paid Leave's 2026 employee rate of 0.44% × $29,000 = $127.60." -us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It treated employee payroll tax as federal FICA alone (7.65% × $29,000), omitting Minnesota's paid leave employee contribution that begins with 2026 wages and adds 0.44% × $29,000 = $127.60." -us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It reported only the 6.2% and 1.45% federal components (rounded to $2,219), leaving out the Minnesota Paid Leave employee premium of 0.44% on $29,000 of taxable wages, $127.60, that applies for tax year 2026." -us,scenario_032,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It stopped at 29000 × 0.0765 = $2,218.50, treating the federal employee share as the complete answer and never adding the mandatory state component; Minnesota Paid Leave's 2026 employee premium of 0.44% × $29,000 = $127.60 brings the total to $2,346.10." -us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It computed only FICA at 7.65% of $29,000 and omitted the mandatory Minnesota Paid Leave employee contribution effective January 1, 2026, which is half the 0.88% premium on $29,000 of covered wages, $127.60." -us,scenario_032,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its answer covers Social Security and Medicare only; the missing piece is Minnesota's 2026 paid leave employee premium, 0.44% × $29,000 = $127.60, which the requested output includes as a mandatory employee state payroll tax." -us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It equated employee payroll tax with 7.65% of gross wages, ignoring state-mandated employee contributions; Minnesota Paid Leave premiums start in 2026 and charge the employee up to 0.44% of covered wages, $127.60 on $29,000." -us,scenario_032,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"It reported federal FICA of $2,218.50 as the total, omitting the Minnesota Paid Leave employee premium (0.44% of $29,000 = $127.60) that becomes payable on 2026 wages." -us,scenario_032,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $1,798 OASDI and $420.50 HI components are correct, but it left out the third component of the requested output: Minnesota's mandatory paid leave employee contribution of 0.44% on $29,000 of taxable wages, $127.60." -us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive computation was submitted. The required derivation is $1,798 OASDI + $420.50 Medicare + $127.60 Minnesota Paid Leave employee premium (0.44% of $29,000) = $2,346.10." -us,scenario_032,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It applied 7.65% to $29,000 and checked only the Additional Medicare Tax threshold, never considering state-mandated employee payroll taxes; Minnesota Paid Leave takes 0.44% of the $29,000 in covered wages from the employee in 2026, $127.60." -us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its explanation describes only the 6.2% and 1.45% employee rates on $29,000 ($2,218.50), yet it submitted $4,428.60 — essentially double, consistent with charging the combined employer-plus-employee 15.3% share that the prompt explicitly excludes. It also omitted the Minnesota Paid Leave employee premium of $127.60 that the correct $2,346.10 includes." -us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"After correctly deriving $2,218.50 it abandoned that figure and invented a 5.24% composite rate to force $1,520, citing a nonexistent ""capped Social Security wage base interaction"" — the $176,100-class OASDI wage base is far above $29,000 and reduces nothing here. The correct build is 6.2% + 1.45% federal plus Minnesota's 0.44% paid leave employee premium, $2,346.10." -us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It concluded there is ""no additional or state payroll tax,"" missing Minnesota Paid Leave, whose 2026 employee contribution of 0.44% on $29,000 of covered wages adds $127.60 to the $2,218.50 federal FICA." -us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It reported 7.65% of $29,000 rounded to $2,219 and considered only the Additional Medicare Tax as a possible addition, omitting the mandatory Minnesota Paid Leave employee premium of $127.60 (0.44% × $29,000) for tax year 2026." -us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It listed only employee OASDI and Medicare on $29,000, leaving out Minnesota's paid leave employee contribution that begins with 2026 wages; 0.44% of $29,000 is $127.60, the exact gap to $2,346.10." -us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It affirmatively stated ""no mandatory Minnesota employee payroll tax applies,"" which fails for 2026: Minnesota Paid Leave levies a 0.88% premium on covered wages with up to half deducted from the employee, giving $127.60 on $29,000." -us,scenario_032,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Its OASDI and HI figures match the reference, but it asserted there is ""no Minnesota employee-side payroll tax,"" omitting the 2026 Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60." -us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It computed federal employee FICA only and closed with ""no other wages,"" never testing for state-mandated employee contributions; Minnesota Paid Leave charges the employee 0.44% of the $29,000 in taxable wages in 2026, $127.60." -us,scenario_032,payroll_tax,inkling,llm_error,state_local_rule,False,"It stated that ""no state employee payroll taxes apply,"" missing Minnesota Paid Leave premiums, which begin January 1, 2026 and take the employee half of the 0.88% rate — $127.60 on $29,000 of covered wages." -us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so the model submitted no computation to evaluate. The required total is $1,798 + $420.50 federal employee FICA plus $127.60 of Minnesota Paid Leave employee premium = $2,346.10." -us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It derived federal FICA of $2,218.50 and rounded to $2,219, checking only the Additional Medicare Tax threshold; it never added Minnesota's 2026 paid leave employee contribution of 0.44% on $29,000 in taxable wages, $127.60." -us,scenario_032,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It concluded ""Minnesota levies no mandatory employee payroll tax,"" the single error: Minnesota Paid Leave premiums are due on 2026 wages and the employee share of the 0.88% premium on $29,000 is $127.60, lifting $2,218.50 to $2,346.10." -us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It reasoned that Minnesota has ""no mandatory employee state payroll tax (no state disability insurance tax like CA/NY),"" overlooking Minnesota Paid Leave, which starts collecting in 2026 and deducts 0.44% of $29,000 = $127.60 from the employee, and it also rounded $2,218.50 to $2,219." -us,scenario_032,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied 7.65% federal FICA only (misstating the Medicare piece as $419.50 when 1.45% of $29,000 is $420.50) and omitted the Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60 required for tax year 2026." -us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It applied only the federal income band, placing $42,664 between 130% and 185% of the 3-person guideline (~$49,302) and declaring reduced-price support positive. It never checked Minnesota's universal free school meals law (Minn. Stat. § 124D.111), which puts every MN student in the FREE tier at any income; the FREE tier supersedes reduced-price, so the reduced-price flag is 0 even though the 1.56 FPG ratio sits inside the federal reduced-price window." -us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted the household as four people (""head, spouse, and 2 children"" — there is only one child) and then tested $42,664 against 185% of a 4-person guideline, a single-threshold test that ignores tier ordering entirely. The decisive omission is Minnesota's universal free school meals program, which assigns the FREE tier to every enrolled student regardless of income; because FREE supersedes reduced-price, reduced_price_school_meals_eligible is 0 at the household's 1.56 FPG ratio." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"It explicitly ruled out free meals on the grounds that $42,664 exceeds the federal 130% FPG free-meal limit (~$33,700), treating that federal threshold as the only route into the FREE tier. Minnesota's universal free school meals program grants FREE to all students without any income test, so the household is in the FREE tier at a 1.56 FPG ratio and the reduced-price flag is 0." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"It reasoned solely from the federal band — income of $42,664 above 130% (~$33,800) and below 185% (~$48,000) of the 3-person guideline — and concluded the 6-year-old qualifies for reduced-price meals. It omitted Minnesota's universal free school meals statute, which classifies every MN student as FREE regardless of income; that superior tier displaces reduced-price, making the flag 0." -us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"It asserted an FPG ratio of roughly 170% (the engine computes 1.56) and used only the federal 130%–185% band to award reduced-price meals to the school-age child. The controlling rule it skipped is Minnesota's universal free school meals program, which assigns the FREE tier at any income level; FREE supersedes reduced-price, so the reduced-price flag is 0." -us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"After several passes it settled on the federal band, stating that ""since income exceeds 130% FPL, the household does not qualify for free meals but could qualify for reduced-price meals"" — treating the federal 130% free-meal cutoff as the exhaustive path into the FREE tier. Minnesota's universal free school meals program supplies a second path with no income test, placing this household in the FREE tier at a 1.56 FPG ratio and zeroing the reduced-price flag." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed roughly 157% of the poverty guideline and applied the federal rule that income above 130% and at or below 185% yields reduced-price eligibility. It never applied Minnesota's universal free school meals program, which grants the FREE tier to every enrolled student irrespective of income; because the FREE tier supersedes reduced-price, the correct value is 0." -us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Its entire derivation was a single comparison, $42,664 < $50,272 (185% FPG), with no tier ordering and no state check. Minnesota's universal free school meals law places the child in the FREE tier at a 1.56 FPG ratio, and reduced-price support is zero whenever the FREE tier applies." -us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It placed $42,664 between 130% and 185% of the 3-person federal poverty line and awarded reduced-price meals to Child 1 on that basis alone. Minnesota's universal free school meals program overrides the federal income tiering by giving all students the FREE tier, which supersedes reduced-price and drives the flag to 0." -us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"It tested $42,664 against the 130%/185% federal guideline band for a 3-person household and stopped at that band membership. The missed step is Minnesota's universal free school meals program, which classifies every MN student as FREE regardless of the 1.56 FPG ratio; the superior FREE tier leaves no positive reduced-price support, so the value is 0." -us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"It computed about 153% of the federal poverty guideline and concluded the household sits in the 130–185% reduced-price band. It omitted Minnesota's universal free school meals statute, which assigns the FREE tier with no income test; FREE supersedes reduced-price, making reduced_price_school_meals_eligible 0." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"It reasoned that countable income is ""below the federal reduced-price meal limit for a 3-person household while above the free-meal limit,"" treating the federal 130% cutoff as the sole gate to free meals. Minnesota's universal free school meals program admits every student to the FREE tier without an income test, so at a 1.56 FPG ratio this household is FREE and the reduced-price flag is 0." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It applied only the federal reduced-price income threshold for three people and declared positive reduced-price support. Minnesota's universal free school meals program assigns the FREE tier to all enrolled students regardless of income, and the FREE tier displaces reduced-price, so the correct value is 0." -us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Its stated basis was that reported income falls ""within the modeled reduced-price school-meal income range for a three-person household"" — a federal band test with no tier ordering and no state overlay. Minnesota's universal free school meals program places the child in the FREE tier at a 1.56 FPG ratio, and reduced-price support is zero whenever FREE applies." -us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"It computed approximately 156% of the projected 2026 3-person guideline — matching the engine's 1.56 ratio — and then applied only the federal 185% reduced-price limit. The step it skipped is Minnesota's universal free school meals program, which grants the FREE tier at any income; FREE supersedes reduced-price, so the flag is 0 despite the correct ratio." -us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"It framed the test as household MAGI (school meals use gross household income, not MAGI) at about 156% of FPG and applied the federal 130–185% band. It never applied Minnesota's universal free school meals program, which assigns the FREE tier to every student irrespective of income and thereby zeroes reduced-price support." -us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Its arithmetic was right — $42,664 at about 156.7% of a ~$27,207 3-person 2026 guideline, matching the engine's 1.56 ratio — but it concluded eligibility from the federal 185% cutoff alone. Minnesota's universal free school meals program places the child in the FREE tier regardless of that ratio, and the FREE tier supersedes reduced-price, so the value is 0." -us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,"It bracketed $42,664 inside a federal $33,566–$47,767 (130–185% FPG) window for three people and stopped there. Minnesota's universal free school meals program grants the FREE tier with no income test, and because FREE supersedes reduced-price, reduced_price_school_meals_eligible is 0." -us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for reduced_price_school_meals_eligible, so the required key was absent from its outputs object and no substantive derivation exists to evaluate. The contract required every requested key exactly once with a numeric value; the correct answer is 0 because Minnesota's universal free school meals program places this household in the FREE tier, which supersedes reduced-price." -us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"It computed about 156% of poverty and applied the federal rule that income between the 130% free-meal and 185% reduced-price limits yields reduced-price eligibility for a school-age child. It omitted Minnesota's universal free school meals statute, which assigns the FREE tier to all students regardless of income; the FREE tier supersedes reduced-price, so the flag is 0." -us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"It put the household at roughly 160% of the 3-person poverty line and concluded eligibility from being ""at or below the 185% cutoff"" — a bare federal threshold test. Minnesota's universal free school meals program admits every enrolled student to the FREE tier without an income test, and FREE supersedes reduced-price, making the correct value 0." +us,scenario_032,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Got FICA right ($1,798 + $420.50) but said Minnesota has no mandatory employee state payroll tax in 2026. That drops the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate and said Minnesota has no modeled mandatory employee payroll tax. It left out the 2026 Minnesota Paid Leave employee premium of $127.60 (0.44% of $29,000)." +us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said Minnesota has no mandatory state payroll tax on employees, so it missed the $127.60 Minnesota Paid Leave contribution. It then cut its own $2,218.50 FICA total to $2,175.90, citing unexplained 'state-specific considerations', with no rule behind the change." +us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Computed 7.65% × $29,000 = $2,218.50 and said Minnesota has no mandatory employee state payroll tax. It left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60 that took effect in 2026." +us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Got Social Security and Medicare right but said Minnesota has no mandatory employee payroll tax. It missed the $127.60 Minnesota Paid Leave employee premium (0.44% of $29,000 in taxable wages)." +us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Stopped at federal FICA of $2,218.50 and said there is no mandatory Minnesota employee payroll tax. It omitted the 2026 Minnesota Paid Leave employee contribution of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only 7.65% FICA to $29,000 and never considered Minnesota's mandatory Paid Leave employee premium. That premium adds 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Said Minnesota imposes no employee payroll tax beyond the federal ones. It missed the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60, which applies from 2026." +us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said Minnesota has no paid family leave employee contribution in 2026. In fact Minnesota Paid Leave premiums start in 2026, and the employee share is 0.44% of $29,000 = $127.60, which it left out." +us,scenario_032,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed no mandatory Minnesota employee payroll tax is modeled and stopped at $2,218.50. It left out the Minnesota Paid Leave employee contribution of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Applied 7.65% FICA and said there is no employee state payroll tax. It missed Minnesota's 2026 Paid Leave employee premium of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"Correctly counted the Minnesota Paid Leave employee share but used a 0.35% rate ($101.50). PolicyEngine applies a 0.44% employee contribution to the $29,000 in paid-leave taxable wages, which gives $127.60, so the answer came out $26.10 short." +us,scenario_032,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its explanation says no employee state payroll tax applies and totals only FICA at $2,218.50, so it missed the $127.60 Minnesota Paid Leave contribution. Its recorded value of $2,320 does not match that explanation and matches no correct derivation. It lines up with FICA plus a 0.35% paid-leave rate instead of the 0.44% employee share." +us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Computed only the 7.65% employee FICA ($2,218.50). It never applied Minnesota's mandatory 2026 Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Counted only Social Security and Medicare, rounded to $2,219, and left out the Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60 that belongs in employee state payroll tax." +us,scenario_032,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied 7.65% to $29,000 and never considered Minnesota's mandatory Paid Leave employee contribution. That contribution adds $127.60 (0.44% of $29,000)." +us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"Computed only federal FICA at 7.65% ($2,218.50). It missed the 2026 Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare on $29,000 and left out the Minnesota Paid Leave employee premium of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Treated payroll tax as 7.65% of wages only. It omitted Minnesota's mandatory Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Computed 7.65% employee FICA only and never added the 2026 Minnesota Paid Leave employee premium of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Got Social Security ($1,798) and Medicare ($420.50) right but left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"Submitted no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10." +us,scenario_032,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Applied 7.65% FICA and ruled out Additional Medicare Tax but never applied Minnesota's mandatory Paid Leave employee premium of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its explanation lists only 6.2% Social Security and 1.45% Medicare on $29,000 and says Minnesota has no employee payroll tax, which would total $2,218.50. The $4,428.60 it submitted is about double that, close to the 15.3% employer-plus-employee FICA total. It also missed the $127.60 Minnesota Paid Leave employee contribution." +us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"First computed $2,218.50 correctly, then threw it out in favor of an invented 5.24% rate tied to a Social Security wage-base cap that does not bind at $29,000. It also treated Minnesota's employee payroll tax as unspecified and never added the $127.60 Paid Leave contribution." +us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"Said there is no state payroll tax and stopped at FICA of $2,218.50. It missed the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied 7.65% FICA, rounded to $2,219, and left out the Minnesota Paid Leave employee premium of $127.60 (0.44% of $29,000) that applies in 2026." +us,scenario_032,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Concluded that no employee-side payroll tax applies beyond FICA. It missed Minnesota's mandatory 2026 Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,gpt-6-sol,llm_error,state_local_rule,False,"Computed only 7.65% FICA on $29,000 and never added the Minnesota Paid Leave employee contribution of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Applied only Social Security and Medicare to $29,000. It omitted the Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"Said no mandatory Minnesota employee payroll tax applies. It missed the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60 that took effect in 2026." +us,scenario_032,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Said there is no Minnesota employee-side payroll tax and stopped at FICA of $2,218.50. It left out the $127.60 Minnesota Paid Leave employee premium (0.44% × $29,000)." +us,scenario_032,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"Said Minnesota paid-leave withholding is optional. Minnesota Paid Leave premiums are mandatory from 2026, and the employee share is 0.44% of the $29,000 in taxable wages ($127.60), which it left out." +us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Summed only Social Security ($1,798) and Medicare ($420.50). It never applied Minnesota's mandatory Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,inkling,llm_error,state_local_rule,False,"Said no state employee payroll taxes apply. It missed the 2026 Minnesota Paid Leave employee premium of $127.60 (0.44% × $29,000)." +us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave = $2,346.10." +us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Computed FICA of $2,218.50, rounded to $2,219, and never applied the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Said Minnesota levies no mandatory employee payroll tax. It missed the Minnesota Paid Leave employee premium of 0.44% × $29,000 = $127.60 that took effect in 2026." +us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Said Minnesota has no mandatory employee state payroll tax because it lacks a CA/NY-style disability insurance tax. It overlooked Minnesota's 2026 Paid Leave program, whose employee contribution is 0.44% × $29,000 = $127.60." +us,scenario_032,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied only 7.65% FICA and miswrote Medicare as $419.50 instead of $420.50, though its total was still $2,218.50. It left out the Minnesota Paid Leave employee contribution of 0.44% × $29,000 = $127.60." +us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly placed $42,664 between 130% and 185% FPL for a 3-person household. It then applied the federal reduced-price band alone and ignored Minnesota's universal free school meals program. That program puts every student in the FREE tier, which replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted a 4-person household with two children, but the household has 3 people and one child. Its answer rests on the federal 185% FPL reduced-price limit alone. It missed that Minnesota's universal free school meals program gives the child the FREE tier, so reduced-price support is not returned." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that income above 130% FPL rules out free meals, which leaves only reduced-price. That reasoning ignores Minnesota's universal free school meals program, which gives free meals to every student regardless of income. The household is therefore in the FREE tier, not the reduced-price tier." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model placed the 3-person household's $42,664 income in the federal 130-185% reduced-price band and stopped. It never applied Minnesota's universal free school meals program, which puts the 6-year-old in the FREE tier and so replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model used only the federal 130-185% FPL band, putting the household at about 170% FPL (the engine ratio is 1.56). It ignored Minnesota's universal free school meals program, which assigns the FREE tier to every student and so makes reduced-price eligibility false." +us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model read income of about 160% FPL as reduced-price eligible under the federal 130-185% band. It missed that Minnesota provides universal free school meals, which puts the household in the FREE tier instead of reduced-price." +us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model worked repeatedly through the federal 130%/185% FPL thresholds for a family of 3. It concluded that income above 130% FPL rules out free meals. It never considered Minnesota's universal free school meals program, which gives the child free meals regardless of income and so removes reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model applied only the federal rule that income between 130% and 185% FPL means reduced-price. It ignored Minnesota's universal free school meals program, under which the student gets the FREE tier and reduced-price support is not returned." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model correctly put income at about 157% FPL but treated the federal 130-185% band as decisive. It missed Minnesota's universal free school meals program, which gives the child the FREE tier and so replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model's only test was income below 185% FPL. It ignored Minnesota's universal free school meals program, which places every student in the FREE tier and so makes reduced-price eligibility false." +us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model placed income between 130% and 185% FPL for 3 and concluded the household was reduced-price eligible. It never applied Minnesota's universal free school meals program, which puts the household in the FREE tier instead." +us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model used the federal 130-185% FPL band for a household of 3. It missed that Minnesota's universal free school meals program gives Child 1 free meals regardless of income, which replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model placed $42,664 between 130% and 185% of the 2026 guideline and concluded reduced-price. It ignored Minnesota's universal free school meals program, which assigns the FREE tier to every student." +us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model correctly placed income at about 153% of the guideline, inside the 130-185% band. It missed that Minnesota's universal free school meals program puts the household in the FREE tier, so no reduced-price support is returned." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model found income above the federal free-meal limit and below the reduced-price limit, and concluded reduced-price. That treats the federal income limits as final and ignores Minnesota's universal free school meals program, which gives every student free meals regardless of income." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model checked income against the federal reduced-price threshold and nothing else. It never applied Minnesota's universal free school meals program, which gives the household the FREE tier in place of the reduced-price tier." +us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model placed household income in the reduced-price income range for 3 people. It ignored Minnesota's universal free school meals program, which gives the FREE tier to every student and so makes reduced-price eligibility false." +us,scenario_032,reduced_price_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model based its answer only on income falling in the reduced-price range for 3 people. It missed that Minnesota's universal free school meals program moves the student into the FREE tier, which replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly put income at about 156% FPL and applied the 185% reduced-price limit. It ignored Minnesota's universal free school meals program, which gives the household free meals and so no reduced-price support." +us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model placed income at about 156% FPL, between 130% and 185%, and concluded reduced-price. It did not apply Minnesota's universal free school meals program, under which PolicyEngine assigns the FREE tier regardless of income." +us,scenario_032,reduced_price_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model found gross income between 130% and 185% FPL with a school-age child and concluded reduced-price. It missed Minnesota's universal free school meals program, which puts Child 1 in the FREE tier and so replaces reduced-price eligibility." +us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model put income at about 156.7% of the 2026 FPL, below the 185% reduced-price threshold, and stopped there. It ignored Minnesota's universal free school meals program, which gives the student free meals regardless of income." +us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model placed income between its computed 130% and 185% FPL dollar limits ($33,566 to $47,767) and concluded reduced-price. It never applied Minnesota's universal free school meals program, which puts the household in the FREE tier." +us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for reduced_price_school_meals_eligible, so its response has no answer to score. The correct answer is 0: Minnesota's universal free school meals program puts the household in the FREE tier." +us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model placed income at about 156% of poverty, between the 130% free-meal limit and the 185% reduced-price limit, and treated those federal limits as final. It missed that Minnesota's universal free school meals program gives every student free meals regardless of income." +us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model applied only the federal 185% reduced-price cutoff to income of about 160% of poverty. It ignored Minnesota's universal free school meals program, which gives the school-age child the FREE tier and so makes reduced-price eligibility false." us,scenario_032,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to bypass the failed SNAP gross-income screen, then contradicted its own calculation showing that net income produced a negative allotment. It replaced that result with an unsupported $493 monthly benefit despite no listed shelter expense or other deduction capable of generating it." us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as sufficient to pass the household through the gross-income screen. Its own max-allotment-minus-30%-of-net-income calculation was negative, but it then invented a $476 monthly benefit; SNAP's minimum benefit does not convert a negative allotment for this ineligible three-person household into that amount." @@ -2673,21 +2990,23 @@ us,scenario_032,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_elig us,scenario_032,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated being under 19 and below Minnesota's CHIP income threshold as sufficient for CHIP eligibility. It failed to apply the Medicaid-exclusion step: the spouse qualifies for Medicaid as an OLDER_CHILD and therefore cannot qualify for CHIP. us,scenario_032,spouse_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model concluded that age, income, and lack of other listed health coverage established CHIP eligibility. It never tested the controlling exclusion for Medicaid-eligible children; the spouse is Medicaid-eligible under the OLDER_CHILD category, so CHIP eligibility is false." us,scenario_032,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no spouse_chip_eligible value or explanation, violating the required structured-output contract. The required result is 0 because the spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP eligibility." -us,scenario_032,spouse_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"It applied the 138% FPL adult-expansion test for a family of three to a person the engine classifies as OLDER_CHILD, because the household label ""Spouse"" overrode the age fact. At age 18 the spouse falls in Minnesota's Medical Assistance category for children ages 6–18, whose income ceiling is far above the 1.56 × FPL MAGI here, so being married does not push the person into the adult pathway." -us,scenario_032,spouse_medicaid_eligible,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Its MAGI computation matched the engine (~157% vs 1.56 × FPL), but it declared the spouse ""an adult"" and tested that MAGI against the 138% FPL expansion limit. An 18-year-old is under 19 and is tested against Minnesota's children's Medical Assistance limit, which the 1.56 × FPL MAGI clears." -us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It carried the head's adult-expansion determination straight over to the spouse (""same income limit exceeded"") without re-running category selection for a person aged 18. The engine assigns OLDER_CHILD by age 6–18, and Minnesota's under-19 income ceiling is far above the 138% FPL limit the model reused." -us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro-0813,llm_error,other,False,"Its reasoning reached the correct determination — age 18, household MAGI below Minnesota's under-19 child Medical Assistance limit, therefore eligible, stating value = 1 — but the submitted numeric value was 0, contradicting its own explanation. The substantive Medicaid analysis was right; the answer was lost to the explanation/value mismatch the output contract forbids." -us,scenario_032,spouse_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It routed the 18-year-old spouse through Minnesota's adult Medicaid expansion pathway and rejected on the 138% FPL ceiling. PolicyEngine categorizes ages 6–18 as OLDER_CHILD, and the children's Medical Assistance limit that applies is far above the household's 1.56 × FPL MAGI." -us,scenario_032,spouse_medicaid_eligible,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It cited the 138% FPL expansion limit ""for adults in Minnesota"" for a person who is 18 and therefore under 19. The controlling test is Minnesota's Medical Assistance income limit for children ages 6–18, which the 1.56 × FPL MAGI satisfies, so the eligibility answer flips to yes." -us,scenario_032,spouse_medicaid_eligible,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It computed household income at about 155% FPL — essentially the engine's 1.56 × FPL — and then compared it to the adult expansion limit of 138% FPL. The comparison used the wrong ceiling: an 18-year-old is tested under the OLDER_CHILD category, whose Minnesota limit far exceeds 155% FPL." -us,scenario_032,spouse_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It correctly summed MAGI as $42,664 (wages $29,000 plus $13,664 of Social Security dependent benefits, which count in Medicaid MAGI) but benchmarked it against a 138% FPL adult threshold of roughly $37,812. That threshold governs the head, not the spouse: at age 18 the spouse is in the OLDER_CHILD category, tested against Minnesota's much higher children's Medical Assistance limit." -us,scenario_032,spouse_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It assigned the spouse to the parent/caretaker-relative category and applied Minnesota's ~138% FPL parent limit plus disregards to a MAGI of about 153% FPL. Both steps are wrong: no parental relationship to the 6-year-old is stated, and the engine's age-based OLDER_CHILD category (ages 6–18) governs anyone under 19, with an income limit the 1.56 × FPL MAGI clears." -us,scenario_032,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It searched only for a ""qualifying low-income adult category"" and, finding none, returned not eligible — never testing the under-19 pathway. Age 18 places the spouse in the OLDER_CHILD category, whose Minnesota income limit is satisfied by the household's 1.56 × FPL MAGI." -us,scenario_032,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It checked only disability, pregnancy, SSI and TANF triggers for an ""adult"" and defaulted to ineligible when none appeared. Eligibility here comes from age alone: at 18 the spouse is an OLDER_CHILD under PolicyEngine's category logic, and MAGI of 1.56 × FPL is within Minnesota's children's Medical Assistance limit." -us,scenario_032,spouse_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"It summed $29,000 of wages and $13,664 of Social Security dependent benefits correctly, then compared the total to ""the applicable Minnesota Medicaid income limit"" chosen as the adult expansion ceiling because the person is a spouse. The applicable limit for an 18-year-old is Minnesota's under-19 children's limit, which the resulting 1.56 × FPL MAGI is well below." -us,scenario_032,spouse_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It explicitly invoked ""the Minnesota adult Medicaid income limit"" for a household member who is 18 and therefore under 19. The engine tests that person under OLDER_CHILD, where Minnesota's income limit far exceeds the household's 1.56 × FPL MAGI, yielding eligible." -us,scenario_032,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for spouse_medicaid_eligible, so the required key was absent from the outputs object. The failure is a contract violation rather than a substantive Medicaid error." -us,scenario_032,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It gave a bare income-and-characteristics rejection with no category named; that verdict is reachable only by applying the 133/138% FPL adult limit, since Minnesota's children's Medical Assistance limit clears 1.56 × FPL by a wide margin. The correct derivation assigns OLDER_CHILD on age 18 and returns eligible." +us,scenario_032,spouse_medicaid_eligible,claude-opus-5,llm_error,age_disability,False,"Tested the spouse against the 138% FPL adult expansion limit and did not recognize that an 18-year-old is under 19, which makes the spouse Medicaid-eligible as a child. Minnesota's older-child (ages 6-18) income limit is well above the household's 1.56x FPL MAGI." +us,scenario_032,spouse_medicaid_eligible,deepseek-v4-flash-0731,llm_error,age_disability,False,"Computed household MAGI correctly at about 157% FPL but called the 18-year-old spouse an adult and applied the 138% FPL expansion limit. Being under 19 puts the spouse in Minnesota's older-child Medicaid category, whose limit covers 156% FPL." +us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro,llm_error,age_disability,False,"Reused the adult income limit it had applied to Head (""same income limit exceeded"") and never considered that the spouse, at 18, qualifies under the under-19 child category. Minnesota's older-child limit is well above the spouse's 1.56x FPL MAGI." +us,scenario_032,spouse_medicaid_eligible,deepseek-v4-pro-0813,llm_error,other,False,"Its explanation correctly placed the 18-year-old spouse under Minnesota's under-19 child Medical Assistance limit and concluded eligible, ending in value = 1. The recorded answer is No (0), which contradicts that explanation, so the miss comes from the submitted value field and not from the eligibility reasoning." +us,scenario_032,spouse_medicaid_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Compared the $42,664 household MAGI with 138% FPL for a family of 3 (about $37,950), which is the adult expansion test, and never applied the child category that covers the 18-year-old spouse. Under Minnesota's older-child limit, 1.56x FPL is eligible." +us,scenario_032,spouse_medicaid_eligible,gemini-3.6-flash,llm_error,age_disability,False,Applied the 138% FPL adult expansion limit to the spouse and missed that someone aged 18 is still in Medicaid's child category. Minnesota's older-child income limit is well above the household's 1.56x FPL MAGI. +us,scenario_032,spouse_medicaid_eligible,gemini-3.7-flash,llm_error,age_disability,False,"Treated the 18-year-old spouse as an adult under the 138% FPL expansion limit. Being under 19 puts the spouse in Minnesota's older-child (ages 6-18) Medicaid category, whose limit covers 1.56x FPL." +us,scenario_032,spouse_medicaid_eligible,gemini-3.8-flash,llm_error,age_disability,False,"Correctly put household income at about 155% FPL but measured it against the 138% adult expansion limit. The 18-year-old spouse falls in the under-19 child category, where Minnesota's limit is well above that income." +us,scenario_032,spouse_medicaid_eligible,glm-5.2,llm_error,age_disability,False,"Measured the $42,664 household income against a 138% FPL threshold of $37,812, which is the adult expansion test, and never applied the child category that covers an 18-year-old spouse. Minnesota's older-child limit covers 1.56x FPL, so the spouse is eligible." +us,scenario_032,spouse_medicaid_eligible,glm-5.3,llm_error,age_disability,False,"Placed the spouse in the parent/caretaker category and tested about 153% FPL against that roughly 138% limit. It missed that at 18 the spouse qualifies under the under-19 child category, where Minnesota's older-child limit is well above the household's MAGI." +us,scenario_032,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,age_disability,False,"Looked only for a low-income adult category for the 18-year-old spouse and found none. It overlooked the MAGI child category for people under 19, which in Minnesota covers the spouse's 1.56x FPL income." +us,scenario_032,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,age_disability,False,"Searched only non-MAGI pathways (disability, pregnancy, SSI, TANF) and ignored the MAGI child category that covers the 18-year-old spouse. Minnesota's older-child limit is well above the household's 1.56x FPL MAGI." +us,scenario_032,spouse_medicaid_eligible,gpt-5.5,llm_error,age_disability,False,"Noted that the spouse is 18 but still compared the $42,664 household income ($29,000 wages plus $13,664 Social Security dependent benefits) with an adult-level Minnesota limit. The correct limit for an 18-year-old is the older-child limit, which covers 1.56x FPL." +us,scenario_032,spouse_medicaid_eligible,gpt-5.6-luna,llm_error,age_disability,False,"Applied Minnesota's adult Medicaid income limit to the spouse. At 18 the spouse is under 19 and falls in the older-child category, whose income limit covers the household's 1.56x FPL MAGI." +us,scenario_032,spouse_medicaid_eligible,gpt-6-luna,llm_error,age_disability,False,"Compared household income, including the Social Security benefits, with Minnesota's adult Medicaid limit. It missed that the 18-year-old spouse falls in the under-19 older-child category, whose limit is well above 1.56x FPL." +us,scenario_032,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for spouse_medicaid_eligible. The correct answer is eligible (1), because the 18-year-old spouse falls in Minnesota's older-child Medicaid category at 1.56x FPL MAGI." +us,scenario_032,spouse_medicaid_eligible,qwen3.8-max,llm_error,age_disability,False,"Concluded ineligible from household income without naming a category, which implies it used an adult income limit. The 18-year-old spouse qualifies in Minnesota's under-19 older-child category, whose limit is well above the 1.56x FPL MAGI." us,scenario_032,spouse_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_032,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly stated that the spouse lacks pregnancy, postpartum, and breastfeeding status and therefore is not categorically eligible, but then submitted 1 in direct contradiction to its reasoning. WIC adjunctive eligibility does not create categorical status through another household member." us,scenario_032,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being an 18-year-old woman of childbearing age in a low-income household with a child as a WIC category. WIC requires the adult woman herself to be pregnant, breastfeeding, or postpartum, and none of those statuses is present." @@ -2699,84 +3018,89 @@ us,scenario_032,state_income_tax_before_refundable_credits,glm-5.3,llm_error,sta us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted that Minnesota AGI equals federal AGI ""with no modifications,"" discarding the Minnesota Social Security subtraction that removes the entire $1,916 of federally taxable benefits at this income, and it also omitted the Minnesota dependent exemption of roughly $5,300 for Child 1. Applying that subtraction against its own $30,699 standard deduction gives $29,000 − $30,699 < 0, so taxable income is zero and the tax is $0, not the $217 base it taxed at 5.35%." us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs object. This is a failure to deliver the answer under the output contract, not a substantive error in the Minnesota computation." us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model declared the full $13,664 of Social Security ""fully taxable"" for Minnesota purposes, contradicting the §86 provisional-income formula (provisional income $35,832 against the $32,000 joint base makes only $1,916 taxable) and the Minnesota Social Security subtraction that removes even that $1,916, and it used a $28,500 joint standard deduction below Minnesota's 2026 amount with no dependent exemption. Its own stated chain of $42,664 − $28,500 at 5.35% yields $758, not the $128 it submitted, so the figure is an unsupported approximation layered on an inflated income base; the correct $29,000 base falls below the deduction and yields $0." -us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It attached the Working Family Credit's ~$925 per-child amount to the 6-year-old; that amount applies only to qualifying older children aged 18-23, because children under 18 are covered by the separate Child Tax Credit and the WFC contribution for this filer is 4% of the first ~$8,750 of earned income. It then blended its own $1,750 + $925 build down to an unexplained $2,337 and used the unindexed 2023 CTC dollar figure rather than the 2026 inflation-adjusted amount." -us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It correctly identified the WFC's 4%-of-earnings component (~$360 cap) but then added a '~$950 one-child add-on' on top of it; that per-child WFC amount is reserved for qualifying older children (18-23), and the age-6 child already generates the Child Tax Credit. Double-counting the child in both credits inflated the total by roughly $950 above the $2,187.60 sum of the indexed CTC plus the 4% earned-income WFC." -us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that 'Minnesota does not have state-level refundable individual income tax credits' and that its credits are nonrefundable or deductions. Minnesota's Child Tax Credit and Working Family Credit are both fully refundable under the 2023 restructuring, and this household's one child under 18 plus $29,000 of earned income triggers both, yielding $2,187.60." -us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the two-component structure right — CTC for the under-18 child plus a small earned-income WFC — but used the unindexed 2023 statutory $1,750 CTC and a guessed '$410-$450' WFC cap instead of the 2026 inflation-adjusted parameters. It also placed the joint phaseout start at '~$33,000' when the indexed joint threshold is near $37,000; no phaseout applies either way, so the entire $27.60 shortfall comes from failing to index the two credit amounts to 2026." -us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It computed only the Child Tax Credit at the unindexed $1,750 and explicitly set the Working Family Credit aside as 'a small amount' without adding it. The WFC's 4%-of-the-first-~$8,750-of-earned-income component on the head's $29,000 of wages is a mandatory part of the combined Child and Working Families Credit, and omitting it plus leaving the CTC unindexed to 2026 accounts for the full $437.60 shortfall." -us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It claimed the Minnesota CTC 'phased out above ~$31,000 AGI' and applied a partial shared reduction to the Working Family Credit portion; the indexed joint-filer phaseout threshold is near $37,000, so the household's $30,916 AGI triggers no 12% reduction at all. Its $2,185 lands $2.60 low because it approximated the 2026-indexed CTC and 4% earned-income WFC amounts rather than computing them from the indexed parameters." -us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It computed the Working Family Credit as 35% of the federal EITC ($3,209 x 0.35 = $1,123) — the pre-2023 structure Minnesota repealed effective tax year 2023 — and omitted the refundable Minnesota Child Tax Credit entirely, which alone supplies over $1,750 of the $2,187.60. It also inflated AGI to $40,614 by applying the 85% maximum Social Security inclusion instead of the first-tier 50% of provisional income above the $32,000 joint base, which yields only $1,916 of taxable benefits." -us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It conditioned Minnesota's refundable credits on rent/CRP, property tax, or education expenses and concluded none were triggered. The Child Tax Credit and Working Family Credit require only a qualifying child under 18 and earned income below the phaseout threshold — both satisfied by the age-6 child and $29,000 of wages — and it acknowledged the qualifying child while still submitting zero." -us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It set the Working Family Credit at 45% of the federal EITC ($2,081), applying the percentage-of-EITC formula Minnesota repealed in 2023. Under the current flat structure the WFC here is 4% of the first ~$8,750 of earned income, roughly $370, so the EITC-linked figure overstated that component by about $1,700." -us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It computed the Working Family Credit as 33% of the federal EITC ($1,481), using the pre-2023 percentage-of-EITC formula that Minnesota replaced with a flat 4%-of-the-first-~$8,750-of-earned-income credit worth roughly $370 here. It also placed the joint phaseout start at $32,235 when the indexed joint threshold is near $37,000, though no phaseout applies at $30,916 either way." -us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $1,633 Working Family Credit is reachable only by scaling the federal EITC (roughly 37% of the ~$4,400 one-child EITC at $29,000 of earnings), the structure Minnesota repealed effective 2023. The current WFC for a household whose only child is under 18 is 4% of the first ~$8,750 of earned income, about $370, making the $1,633 figure an overstatement of roughly $1,260." -us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It reduced the Child Tax Credit to $983.60 by applying a phaseout 'starting at $35,000 income' to a household whose AGI is $30,916 — a reduction that is arithmetically impossible since income sits below its own stated threshold, and the indexed joint threshold is in fact near $37,000. It compounded this with a $550 Working Family Credit in place of the 4%-of-first-~$8,750 amount of roughly $370." -us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It asserted without derivation that no state refundable credits apply to this profile. Minnesota grants a fully refundable Child Tax Credit for the age-6 dependent plus a Working Family Credit equal to 4% of the first ~$8,750 of the head's $29,000 in earnings, and AGI of $30,916 is below the joint phaseout threshold, so the full $2,187.60 is payable." -us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It doubled the Working Family Credit into a '$700 combined base amount for the taxpaying couple,' treating the 4%-of-first-~$8,750 credit as a per-spouse amount; the WFC is a single per-return credit computed on the filing unit's earned income, and the spouse has no earnings at all. Combined with an unindexed $1,750 CTC, this overstated the total by $262.40." -us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It added an 'approximately $1,200' Working Family Credit to the unindexed $1,750 CTC, a magnitude obtainable only from the repealed percentage-of-federal-EITC formula. The post-2023 WFC for this household is 4% of the first ~$8,750 of earned income, roughly $370, since the $925 per-child WFC amount reaches only qualifying older children aged 18-23." -us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its bare $600 is consistent with a Working-Family-Credit-only estimate that omits the Minnesota Child Tax Credit entirely. The age-6 dependent generates the refundable CTC, which supplies over $1,750 of the $2,187.60, with the WFC's 4%-of-first-~$8,750 earned-income piece adding the remainder." -us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It reported the Child Tax Credit alone at the unindexed 2023 statutory $1,750 and included no Working Family Credit. The WFC's 4% of the first ~$8,750 of the head's earned income is a separate component of the combined Child and Working Families Credit, and both dollar amounts are inflation-indexed to 2026, together producing $2,187.60." -us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It added a $1,085 Working Family Credit to the unindexed $1,750 CTC, a figure on the scale of a percentage of the ~$4,400 federal EITC rather than Minnesota's post-2023 flat credit. The correct WFC contribution is 4% of the first ~$8,750 of earned income, about $370, because the per-child WFC amount applies only to qualifying children aged 18-23." -us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $2,450 decomposes as the unindexed $1,750 CTC plus a $700 Working Family Credit — twice the statutory 4%-of-the-first-~$8,750 maximum. The WFC is a single per-return credit on the filing unit's earned income, and both it and the CTC are indexed to 2026, summing to $2,187.60." -us,scenario_032,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It indexed the Child Tax Credit to $1,912 but then declared 'no other refundable state credits apply (no rent or education expenses listed),' dropping the Working Family Credit. The WFC does not require rent or education expenses: it equals 4% of the first ~$8,750 of the head's $29,000 in earnings and is part of the same combined Child and Working Families Credit." -us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It applied a 12% phaseout using the $29,500 single/head-of-household threshold to a married-joint return, cutting about $170 from the CTC; the joint threshold is indexed to roughly $37,000, so AGI of $30,916 produces no reduction whatsoever. It then set the WFC at ~$180 as though it phases out like the federal EITC, when the post-2023 credit is a flat 4% of the first ~$8,750 of earned income." -us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It concluded no Minnesota refundable credit was identifiable from the facts and submitted zero. The facts given — one dependent age 6, $29,000 of wages, AGI $30,916, married-joint — are exactly the inputs to Minnesota's refundable Child Tax Credit and Working Family Credit, which together pay $2,187.60 with no phaseout at this income." -us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It claimed no Minnesota refundable credits are implied by the provided facts and returned zero. A qualifying child under 18 and $29,000 of earned income are the only conditions the Minnesota Child Tax Credit and Working Family Credit require, and AGI of $30,916 is below the joint phaseout threshold, so the full indexed $2,187.60 is due." -us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It reproduced the correct post-2023 structure — Child Tax Credit for the under-18 child plus the earned-income Working Family Credit, with income below the joint phaseout threshold — but used the 2023 statutory dollar amounts ($1,750 and $350) verbatim. Both parameters are inflation-indexed, and the 2026 values sum to $2,187.60, leaving its answer $87.60 short purely from skipping indexation." -us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It set the Working Family Credit at 25% of the federal EITC ($4,427 x 0.25 = $1,106.75), applying the percentage-of-EITC formula Minnesota repealed effective tax year 2023. The current WFC is 4% of the first ~$8,750 of earned income, roughly $370, and the CTC it paired with it was left at the unindexed $1,750." -us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $1,675 Working Family Credit is on the scale of a percentage of the federal EITC rather than Minnesota's post-2023 flat credit of 4% of the first ~$8,750 of earned income (about $370). Adding that inflated component to an unindexed $1,750 CTC overstated the combined credit by $1,237.40." -us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $3,572 decomposes as roughly $1,750 of Child Tax Credit plus an ~$1,822 Working Family Credit, the latter equal to about 41% of the ~$4,400 federal EITC for one child at $29,000 of earnings — the EITC-percentage structure Minnesota repealed in 2023. The replacement WFC is 4% of the first ~$8,750 of earned income, about $370, since the per-child WFC amount reaches only qualifying children aged 18-23." -us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It built the credit correctly as CTC plus a $388 earned-income Working Family Credit with no phaseout at this income, but anchored the Child Tax Credit at the unindexed 2023 statutory $1,750. Indexing both components to 2026 produces $2,187.60, and the missing indexation of the CTC accounts for the $49.60 gap." -us,scenario_032,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It gestured at 'MN state refundable credits like WFC' and submitted $500, omitting the Minnesota Child Tax Credit that supplies over $1,750 of the total. The age-6 dependent generates that fully refundable CTC, and the WFC adds only 4% of the first ~$8,750 of earned income on top of it." -us,scenario_032,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It computed the Working Family Credit as 40% of the federal EITC (0.4 x $4,436 = $1,774), the pre-2023 formula Minnesota repealed. The post-2023 WFC equals 4% of the first ~$8,750 of earned income — roughly $370 — because the $925 per-child WFC amount applies only to qualifying older children aged 18-23, not to a 6-year-old." -us,scenario_032,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It added an estimated $1,560 Working Family Credit 'at this earnings level' to a $1,750 CTC, a magnitude that only follows from the repealed percentage-of-federal-EITC structure. Minnesota's current WFC for a household whose only child is under 18 is 4% of the first ~$8,750 of earned income, about $370, and both credit amounts are indexed to 2026." -us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It set the Working Family Credit at 33% of the federal EITC ($4,439 x 0.33 = $1,465), applying the formula Minnesota repealed effective tax year 2023. The replacement WFC is a flat 4% of the first ~$8,750 of earned income, roughly $370, so the EITC-linked figure overstated the total by about $1,100 even before the unindexed $1,750 CTC." -us,scenario_032,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It estimated a '$1,300 Working Family Credit for 1 child with $29,000 earned income,' treating the WFC as a child-scaled EITC-style credit. Post-2023 the per-child WFC amount is limited to qualifying older children aged 18-23; the 6-year-old generates the CTC only, leaving the WFC as 4% of the first ~$8,750 of earnings, about $370." -us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_refundable_credits, so the submission failed the output contract rather than the tax computation. The required answer is the Minnesota Child and Working Families Credit of $2,187.60 for one child under 18 with $29,000 of earned income and AGI below the joint phaseout threshold." -us,scenario_032,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It used a 'full one-child Working Family Credit of $1,250,' treating the WFC as scaling with the number of children under 18. Minnesota's post-2023 WFC pays a per-child amount only for qualifying older children aged 18-23; for this household it is 4% of the first ~$8,750 of earned income, about $370, and the $1,750 CTC it paired with was left unindexed." -us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It denied the Working Family Credit on two grounds that do not apply: that the spouse has no earned income, and that household income exceeds the eligibility threshold. The credit is computed on the filing unit's combined earned income — the head's $29,000 suffices — and AGI of $30,916 is below the indexed joint phaseout threshold near $37,000; it also omitted the refundable Child Tax Credit for the age-6 child entirely." -us,scenario_032,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It invented a Working Family Credit maximum of 'roughly $2,900' for a joint filer with one child and then phased it down to $1,650; the statutory maximum for this household is 4% of the first ~$8,750 of earned income, about $370, and no 12% phaseout applies because AGI of $30,916 is below the indexed joint threshold. Its $1,750 CTC was also the unindexed 2023 figure." -us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It derived a $1,895 Working Family Credit from 'federal EITC percentages,' the structure Minnesota repealed effective 2023, and omitted the refundable Minnesota Child Tax Credit for the age-6 child altogether. The correct build is the indexed CTC plus a WFC of 4% of the first ~$8,750 of earned income, totaling $2,187.60." -us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It reported a $1,648 EITC-scaled Working Family Credit plus a $241 Child and Dependent Care Credit while omitting the Minnesota Child Tax Credit entirely. The MN CDCC requires qualifying care expenses, which are zero here under the instruction to treat unlisted inputs as zero, and the WFC is a flat 4% of the first ~$8,750 of earned income — roughly $370 — with the indexed CTC supplying the rest of the $2,187.60." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It reproduced the reference derivation exactly — AGI $83,208, $35,500 standard-plus-aged deduction, $12,000 senior deduction, taxable income $35,708, tax $3,789 — and then submitted $3,699 as a 'PolicyEngine-style rounding' adjustment. Its own bracket arithmetic ($2,480 + 12% × $10,908.23) is $3,788.99, and no rounding convention subtracts $90 from it; the error is discarding a correct computation at the moment of reporting." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $243 state/local tax refund to gross income, which is excluded under the §111 tax benefit rule for a standard-deduction filer in South Dakota, and it claimed the $1,225 OBBBA auto-loan-interest deduction, which requires a qualifying new US-assembled vehicle placed in service after 2024 — a flag that is unlisted and therefore false. Those two errors move taxable income to $34,726 instead of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $11,400 self-employment loss as a capital loss capped at $3,000 under §1211(b) rather than a business loss that fully offsets other income, subtracted the standard deduction inside its AGI computation, and used a $30,400 standard deduction with neither the $1,650-per-spouse aged addition nor the $6,000-per-spouse senior deduction. It then abandoned its own $6,959 result and submitted $5,661 with no supporting derivation." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It reached the correct taxable income of $35,708 with the $35,500 standard-plus-aged and $12,000 senior deductions, but topped the 10% bracket at $24,150 instead of the 2026 figure of $24,800, yielding $3,801.96. It then submitted $3,416, a number produced by no step in its reasoning and labeled only as a 'slightly different bracket estimate'." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It included the $243 state tax refund in gross income and used a $31,500 MFJ base standard deduction instead of the 2026 figure of $32,200, arriving at ~$3,798 — and then submitted $4,575, which its own arithmetic never produces. The stated reasoning and the submitted value are $777 apart with no reconciling step." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It included the $243 refund and then lost $1,000 in a transcription slip (79,000 + 775 + 243 − 11,400 − 1,487 = 67,131, written as 66,131), and it applied the correct $47,500 deduction stack to get taxable income $34,951. Its own bracket math — $2,480 plus 12% of $10,151 — sums to $3,698, yet it reported $3,080." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the OBBBA $6,000-per-spouse senior deduction entirely and used 2025 parameters ($30,000 base, $1,600 aged addition) rather than 2026's $32,200 and $1,650, deducting $33,200 instead of $47,500, and it added the $243 refund to income. Taxable income came out $50,251 against the correct $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $1,600 aged addition and omitted the $12,000 senior deduction, deducting $35,400 instead of $47,500 and computing $5,241 of tax on $47,808 of taxable income. It then submitted $3,800 as an unexplained adjustment 'for the self-employment loss offset and minor credits' — the self-employment loss was already in AGI and no credit applies to this childless couple." -us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the $6,000-per-spouse senior deduction and used a $31,000 base instead of $32,200, deducting $34,300 rather than $47,500, and it included the $243 refund in AGI. Taxable income of $49,151 instead of $35,708.23 accounts for the entire $1,613 overstatement." -us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It explicitly applied 2024 parameters — a $33,100 standard deduction and a 10% bracket topping out at $23,200 — for a 2026 return, and it omitted the OBBBA $6,000-per-spouse senior deduction and included the $243 refund. The 2026 stack is $32,200 + $1,650 × 2 + $6,000 × 2 = $47,500 with the 10% bracket ending at $24,800." -us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset for 2026, using a $16,700 pre-TCJA base standard deduction, $5,350 personal exemptions per person, and a 15% second bracket. For 2026 the TCJA structure is permanent: the MFJ standard deduction is $32,200, personal exemptions are zero, the second bracket is 12%, and an additional $12,000 senior deduction applies." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed AGI correctly at $83,208 and then applied post-TCJA-expiration parameters: a $16,000 standard deduction, $10,000 of personal exemptions, and a 15% bracket above $23,200. For 2026 the deduction is $32,200 plus $3,300 of aged additions plus a $12,000 senior deduction, exemptions are zero, and the second bracket rate is 12%." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reported AGI of ~$78,818, $4,390 below the correct $83,208.23, and a $32,500 deduction containing neither the $3,300 of aged additions nor the $12,000 senior deduction. Its own stated taxable income of $46,318 produces $5,062 of tax at 2026 rates, so the submitted $3,724 does not follow from any figure it gave." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It supplied no derivation, and $6,762 is unreachable from the correct 2026 stack: AGI $83,208.23 less $47,500 of standard, aged and senior deductions leaves $35,708.23 taxed at $2,480 + 12% × $10,908.23 = $3,788.99. Its value corresponds to a TCJA-sunset computation — roughly a $20,000 standard deduction plus two ~$5,300 personal exemptions and a 15% second bracket — none of which apply in 2026." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached the correct AGI of $83,208 and then applied 'TCJA expiration' parameters: a $16,500 standard deduction, $10,500 of personal exemptions, and reverted 15% brackets. The 2026 figures are a $32,200 base plus $1,650 per aged spouse plus a $6,000-per-spouse senior deduction, zero exemptions, and a 12% bracket above $24,800." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that deductions, exemptions and credits zero out the liability without any computation. AGI is $83,208.23 and the entire allowable deduction stack is $47,500, leaving $35,708.23 of taxable income; a zero result would require roughly $36,000 more deduction than exists, and no nonrefundable credit is available to this childless couple." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI correctly at $83,208 but deducted only $34,000, omitting the OBBBA $6,000-per-spouse senior deduction that both a 69-year-old and a 66-year-old qualify for with MAGI far below the $150,000 joint phaseout. Taxable income of $49,208 instead of $35,708.23 is the whole error." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI correctly at $83,208 but subtracted 'the standard deduction and personal exemptions' — exemptions are zero through 2028 — while omitting the $12,000 senior deduction; its $5,045 implies roughly $37,000 of total deduction against the correct $47,500. Taxable income is $35,708.23, not the ~$46,175 its figure requires." -us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used 2025 parameters ($30,000 base, $1,600 aged addition, 10% bracket ending at $23,850) instead of 2026's $32,200/$1,650/$24,800, omitted the $6,000-per-spouse senior deduction, and included the $243 refund in income. That leaves taxable income $50,251 rather than $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It deducted only ~$34,030, omitting the OBBBA $6,000-per-spouse senior deduction, and it included the $243 state tax refund in AGI. Its bracket estimates were also off ($24,445 rather than $24,800 for the 10% ceiling), but the $12,000 missing deduction drives virtually all of the $1,653 overstatement." -us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as $12,085 by invoking a nonexistent '$64,000 joint threshold'; the §86 second threshold for joint filers is $44,000, and provisional income of $76,731 drives the calculation to the 85% cap of $16,320. It also dropped the $1,650-per-spouse aged addition (deducting $44,200 instead of $47,500) and added the $243 refund, landing at taxable income $35,016." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed 'limited itemized deductions from mortgage interest, SALT refund income, and HSA deduction' on top of the standard deduction; the household lists a $220,000 mortgage balance with no interest amount, and its $400 of premiums plus $50 of OTC expenses fall far below the 7.5%-of-AGI medical floor, so itemized deductions are zero. Its $3,677 implies taxable income of $34,775, about $933 below the correct $35,708.23, consistent with subtracting those phantom amounts from the $47,500 standard-plus-senior deduction." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated Social Security as non-taxable; with $79,000 of wages, provisional income is $76,731, far above the $44,000 joint threshold, so the 85% cap applies and $16,320 enters AGI. Even excluding all Social Security, AGI of $66,888 less the $47,500 deduction stack leaves $19,388 of taxable income, so zero is unreachable — the correct figure is $35,708.23 taxed at $3,788.99." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'estimated 2026 post-TCJA' parameters — $29,900 of combined standard, age and personal-exemption amounts and a 15% bracket above $23,900. For 2026 personal exemptions are zero, the second bracket is 12% above $24,800, and the deduction stack is $32,200 + $3,300 aged + $12,000 senior = $47,500; it also included the $243 refund in AGI." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $35,951 is exactly the correct $35,708.23 plus the $243 state/local tax refund, which produces no gross income under the tax benefit rule because the couple takes the standard deduction and South Dakota levies no income tax. The whole $29.13 overstatement is 12% of that refund." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the correct $32,200 base, $3,300 of aged additions and $12,000 senior deduction, but explicitly folded 'the taxable refund' of $243 into income, giving taxable income $35,951 rather than $35,708.23. That refund is excluded under the §111 tax benefit rule for a standard-deduction filer in a state with no income tax, and 12% of it is the entire $29.13 error." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It built the correct $47,500 deduction stack and correct 10%/12% bracket application, but its implied taxable income of $35,951 includes the $243 state/local tax refund, which yields no gross income for a standard-deduction filer in South Dakota. The $29.13 excess is 12% of the improperly included refund." -us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It stated AGI of $83,451, which is the correct $83,208.23 plus the $243 state/local tax refund; that refund is excluded under the tax benefit rule since the couple claims the standard deduction and South Dakota imposes no income tax. Every other step — 85% Social Security inclusion, the $11,400 loss, the $1,487 HSA, and the $35,500 + $12,000 deductions — matches, so the error is exactly 12% of $243." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $5,523 implies taxable income of about $50,158 — AGI $83,451 (including the $243 refund) less roughly $33,300 of standard and aged deduction with no senior deduction. The OBBBA $6,000-per-spouse senior deduction applies to both spouses here, bringing the deduction to $47,500 and taxable income to $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It deducted $34,030 of 'standard deduction including elderly additions' and never applied the $6,000-per-spouse senior deduction, and it included the $243 refund in AGI. Taxable income of $49,421 instead of $35,708.23 accounts for the entire $1,653 overstatement." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed 'sunset' law for 2026: a $16,600 standard deduction, $5,300 personal exemptions each, and a 15% bracket. In 2026 exemptions are zero, the MFJ standard deduction is $32,200 with $1,650 aged additions and a $6,000-per-spouse senior deduction, and the bracket above $24,800 is 12%." -us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $34,196 standard deduction 'for MFJ both over 65' with no OBBBA senior deduction, and it included the $243 refund in AGI. The correct stack is $32,200 + $1,650 × 2 + $6,000 × 2 = $47,500, giving taxable income $35,708.23 rather than its $49,255." -us,scenario_033,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It invented a '$1,500 temporary' addition to the MFJ standard deduction (the 2026 base is $32,200 with only $1,650 per aged spouse on top), took a $1,225 auto-loan-interest deduction that requires an unlisted — and therefore false — qualifying new-vehicle status, and included the $243 refund. Those distortions net to taxable income of $33,226 against the correct $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for this variable, so there is no substantive computation to evaluate. The required derivation is AGI $83,208.23 less the $47,500 standard, aged and senior deduction stack = $35,708.23 of taxable income, taxed at $2,480 + 12% × $10,908.23 = $3,788.99." -us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,225 auto-loan-interest deduction and a $333.33 'qualified overtime premium' deduction on top of the correct $47,500 stack, and included the $243 refund. The overtime deduction covers only the FLSA-required premium portion of pay, and with usual weekly hours of 40 there are no overtime hours; the auto-loan deduction requires a qualifying new US-assembled vehicle flag that is unlisted and therefore false — so both are zero and taxable income is $35,708.23, not $34,392.67." -us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It double-counted Social Security (adding the full $19,200 and then 85% of it), never subtracted the $11,400 self-employment loss, used a $33,900 deduction with no senior deduction, computed roughly $11,040, and then submitted −$250 as a 'small refund'. Federal income tax before refundable credits is floored at zero and cannot be negative; the correct value is $3,788.99 on taxable income of $35,708.23." -us,scenario_033,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Every element matches the reference except that it added the $243 state/local tax refund to AGI, giving $83,451 instead of $83,208.23 and taxable income $35,951 instead of $35,708.23. That refund generates no gross income under the tax benefit rule for a standard-deduction filer in South Dakota, and 12% of it is the entire $29.13 error." -us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It identified the $11,400 self-employment loss but then built AGI as $79,000 + $775 + $16,320 = $96,095 without ever subtracting it, and it omitted the $6,000-per-spouse senior deduction while using a $34,400 deduction. It also submitted $5,998 after computing $6,748, so its reported figure matches neither the correct $3,788.99 nor its own arithmetic." -us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction plus a 'taxable Social Security subtraction' drives taxable income to zero. Provisional income of $76,731 puts Social Security at the 85% cap, so $16,320 is includible, and AGI of $83,208.23 less the full $47,500 of standard, aged and senior deductions leaves $35,708.23 of taxable income producing $3,788.99 of tax." +us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model added a working-family amount of about $925 for a filer with one child. Under the CWFC, the working-family part does not grow with children: it is 4% of earnings capped at about $365. Its $2,337 total is about $150 above the correct $2,187.60, which is the indexed child credit plus the capped working-family part." +us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model added a one-child add-on of about $950 to the working-family part. The 2023 CWFC removed any per-child add-on, so the working-family part is only 4% of earnings capped at about $365. The model's $3,100 overstates the correct $2,187.60 by roughly $900." +us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model said Minnesota has no refundable individual income tax credits. The Child and Working Families Credit is fully refundable, and this household qualifies through one child aged 6 and $29,000 of earnings, for $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used the unindexed 2023 child credit of $1,750 and a working-family part of about $410. It also gave the working-family part its own phaseout starting near $33,000. The correct 2026 figures are the indexed child credit plus a working-family part capped at about $365, which together equal $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model counted only the $1,750 child credit and dismissed the working-family part as small. It left out the 4%-of-earnings part (capped at about $365) and did not inflation-index the child credit, so it fell short of $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model put the phaseout threshold near $31,000 and reduced the credit through a shared phaseout. The 2026 joint-filer threshold is far above the $30,916 AGI, so no reduction applies. It also used the unindexed $1,750 child credit, and its offsetting approximations produced $2,185 instead of the full indexed total of $2,187.60." +us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model had the right structure and no phaseout, but used the 2023 base amounts ($1,750 child credit and $350 working-family cap). Minnesota indexes both amounts for inflation, which brings the 2026 total to $2,187.60 rather than $2,100." +us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model treated the Working Family Credit as 35% of the federal EITC, which is the pre-2023 structure. It also left out the per-child credit entirely and included 85% of the Social Security benefits in AGI. Under the CWFC, the answer is the indexed child credit plus 4% of earnings capped at about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model concluded that no refundable credit applies without rent, education, or other expenses. The CWFC requires only earnings and a qualifying child. The 6-year-old child and $29,000 of wages produce $2,187.60." +us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model estimated a working-family part of about $1,900 for one child using the repealed EITC-linked schedule. Under the CWFC, that part is only 4% of earnings capped at about $365, so the total is $2,187.60, not $3,700." +us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 45% of a $4,625 federal EITC, a method Minnesota repealed in 2023. It then added that $2,081 to the child credit. The actual working-family part is 4% of earnings capped at about $365, so the correct total is $2,187.60." +us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model calculated the working-family part as 33% of the federal EITC ($1,481), which is the old structure. The actual part is 4% of earnings capped at about $365. It also overstated the indexed child credit at $1,912, so its $3,393 exceeds $2,187.60." +us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model gave a $1,633 working-family credit and a $1,950 child credit. The working-family part is capped at about $365 (4% of earnings), and the indexed child credit is about $1,823, so the correct total is $2,187.60, not $3,583." +us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 45% of the federal EITC ($1,368), which is the repealed structure. Under the CWFC, the working-family part is 4% of earnings capped at about $365, and it combines with the indexed child credit for $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model cut the child credit to $983.60 even though it said the phaseout starts at $35,000, above the $30,916 AGI. In fact no phaseout applies. The model also overstated the working-family part at $550, when the cap is about $365. The full credit is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said no Minnesota refundable credits apply. The household qualifies for the fully refundable CWFC through the 6-year-old child and $29,000 of earnings, which yields $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model gave a $700 working-family amount to the couple, effectively $350 per spouse. The working-family part is one amount per return: 4% of earnings capped at about $365. With the unindexed $1,750 child credit on top, the model reached $2,450 instead of $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model assigned about $1,200 to the Working Family Credit using the old EITC-style schedule. Under the CWFC, the working-family part is 4% of earnings capped at about $365, so the total is $2,187.60, not $2,950." +us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave $600 without any breakdown. That figure fits a working-family-sized amount with no child credit. The per-child credit of about $1,823 (indexed) is missing, and it makes up most of the $2,187.60 CWFC." +us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model counted only the unindexed $1,750 child credit. It left out the working-family part (4% of earnings capped at about $365) and the 2026 inflation indexing that brings the total to $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model added a working-family amount of about $1,085 to the $1,750 child credit, using the repealed EITC-linked schedule. The CWFC working-family part is capped at about $365, so the correct total is $2,187.60." +us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"The model's $2,450 matches $1,750 plus a doubled $700 working-family amount, one $350 cap per spouse. The working-family part is a single per-return amount capped at about $365, and the child credit is indexed, so the total is $2,187.60." +us,scenario_032,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model stated that no other refundable credits apply and counted only an indexed child credit of $1,912. It left out the CWFC working-family part (4% of $29,000 of earnings, capped at about $365) and overstated the indexed child amount, which is about $1,823. The combined credit is $2,187.60." +us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model phased down the child credit from a $29,500 threshold, which is the 2023 threshold for non-joint filers. This household files jointly, and the 2026 joint threshold is well above the $30,916 AGI, so the full credit of $2,187.60 applies. The model also understated the working-family part at $180." +us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable Minnesota credit. The household qualifies for the refundable CWFC through the 6-year-old child and $29,000 of wages, for $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required explicitly stated credit amounts and so returned zero. The CWFC is calculated from the facts given (a qualifying child, earnings, and AGI below the phaseout threshold) and comes to $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the 2023 base amounts ($1,750 child credit and $350 working-family cap) with no inflation indexing. The indexed 2026 amounts total $2,187.60, not $2,100." +us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model calculated the working-family part as 25% of a $4,427 federal EITC, a method that no longer exists. Under the CWFC, that part is 4% of earnings capped at about $365, so the total is $2,187.60, not $2,856.75." +us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model assigned a $1,675 Working Family Credit using the pre-2023 schedule tied to the number of children. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60." +us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model's $3,572 fits a child credit plus an old-style working-family amount of about $1,800 tied to the EITC. Under the CWFC, the working-family part is capped at about $365, so the total is $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the unindexed 2023 child credit of $1,750 and added a $388 working-family amount. The 2026 indexed child credit is about $1,823, and with the capped working-family part the total is $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model's $4,500 overstates the working-family part as a large EITC-style amount. The CWFC working-family part is capped at about $365 per return, and the only other part is the indexed child credit, for $2,187.60 in total." +us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model assigned a $1,275 working-family credit using the repealed EITC-linked schedule. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60." +us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model used the unindexed $1,750 child credit and a $387 working-family amount. The 2026 child credit is inflation-indexed to about $1,823, and the combined unphased credit is $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model counted only a $500 working-family amount and left out the refundable per-child credit, which is about $1,823 after indexing. The CWFC combines both parts for $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model calculated the working-family part as 40% of the federal EITC ($1,774), which is the repealed pre-2023 structure. The CWFC working-family part is 4% of earnings capped at about $365, so the total is $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model estimated a $1,560 Working Family Credit using the old schedule tied to the number of children. The CWFC working-family part is capped at about $365, which with the indexed child credit gives $2,187.60." +us,scenario_032,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model applied a one-child maximum Working Family Credit of $1,577, which comes from the schedule repealed in 2023. The CWFC working-family part is a flat 4% of earnings capped at about $365 regardless of the number of children, so the total is $2,187.60." +us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model calculated the Working Family Credit as 33% of the federal EITC ($1,465), which is the repealed structure. The CWFC working-family part is 4% of earnings capped at about $365, and it combines with the indexed child credit for $2,187.60." +us,scenario_032,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model estimated a one-child Working Family Credit of about $1,300 using the old schedule. The CWFC working-family part is capped at about $365, so the total is $2,187.60, not $3,050." +us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so there is no answer to score." +us,scenario_032,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used a full one-child Working Family Credit of $1,250 from the repealed schedule. The CWFC working-family part is 4% of earnings capped at about $365, which with the indexed child credit gives $2,187.60." +us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model ruled the household out because the spouse has no earnings and because it believed income exceeded the thresholds. The CWFC needs only one earner on the joint return, and the $30,916 AGI is below the joint phaseout threshold, so the full $2,187.60 is paid." +us,scenario_032,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model assumed a Working Family Credit maximum of about $2,900 phased down to $1,650. The CWFC working-family part is capped at about $365, and no phaseout applies at this AGI, so the total is $2,187.60." +us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model counted only an EITC-linked Working Family Credit of about $1,895 and left out the refundable per-child credit. The CWFC is the indexed child credit plus a working-family part capped at about $365, for $2,187.60." +us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model claimed a $241 child and dependent care credit even though the household has no care expenses. It also used an old-style $1,648 Working Family Credit and left out the per-child credit. The correct CWFC total is $2,187.60." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $243 state tax refund from gross income, which gave AGI of $83,208 and taxable income of $35,708 instead of $35,951.23, and its own brackets then produced $3,789. It submitted $3,699 anyway, citing an unexplained 'PolicyEngine-style rounding' adjustment that nothing in its computation supports." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI ($83,451), standard deduction ($35,500) and senior deduction ($12,000) were all correct, but it also subtracted $1,225 of auto-loan interest as an OBBBA vehicle-loan deduction. The household facts do not establish a qualifying new vehicle loan, so that deduction does not apply. Taking it cut taxable income to $34,726 and tax by $147." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $11,400 Schedule C loss as a capital loss capped at $3,000, when the full loss offsets ordinary income. It subtracted the standard deduction while computing AGI, used a $30,400 standard deduction with no aged additions and no $12,000 senior deduction, and then submitted $5,661, which does not follow from its own $61,451 taxable income." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 state refund, which gave AGI of $83,208, and put the top of the 10% bracket at $24,150 instead of $24,800, arriving at $3,801.96. It then replaced that with $3,416, citing 'slightly different bracket estimates' that it never computed." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It added pre-Social Security income as $67,618 instead of $68,618, a $1,000 slip. It also used the 2025 base standard deduction of $31,500 instead of $32,200 and ended the 10% bracket at $24,000. That work produced about $3,798, but it submitted an unsupported $4,575." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It computed non-Social Security AGI as $66,131 instead of $67,131, a $1,000 subtraction error, which lowered taxable income to $34,951. It then added $2,480 and $1,218 and reported $3,080 instead of $3,698." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used the 2025 pre-OBBBA standard deduction ($30,000 plus two $1,600 aged additions, $33,200) and left out the OBBBA $6,000-per-senior deduction for both spouses. That understated deductions by $14,300 and gave taxable income of $50,251 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund and left out the $12,000 OBBBA senior deduction. Its standard deduction of $35,400 gave taxable income of $47,808 and tax of $5,241. It then submitted $3,800, citing unspecified 'additional adjustments' that it never quantified." +us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $31,000 base standard deduction instead of $32,200 and never applied the $6,000-per-spouse senior deduction for filers 65 and older. Taxable income came out at $49,151 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2024 tax rules, with a $33,100 standard deduction and a 10% bracket ending at $23,200, instead of 2026 law. That meant it left out the $32,200 base, the $1,650 aged additions and the $12,000 OBBBA senior deduction, and taxed $50,351 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025. It applied a pre-TCJA $16,700 standard deduction, $10,700 of personal exemptions and a 15% bracket, instead of the permanent OBBBA rules: a $32,200 standard deduction plus aged additions, the $12,000 senior deduction, and a 12% bracket." +us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a TCJA sunset: a $16,600 standard deduction, $10,600 of personal exemptions and a 15% rate above $23,850. It skipped the 2026 OBBBA $32,200 standard deduction and the $12,000 senior deduction, and its taxable income came out at $52,951." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly assumed the TCJA had expired in 2026 and used a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket. It also excluded the $243 refund. It never applied the $32,200 OBBBA standard deduction or the $12,000 senior deduction." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at about $78,818 instead of $83,451.23 and left out the $12,000 senior deduction, which gave taxable income of $46,318. Its $3,724 answer does not follow from that figure, since the 2026 brackets put tax on $46,318 at about $5,300." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no computation. Its $6,762 matches the TCJA-sunset figures other models produced with pre-TCJA personal exemptions and a 15% bracket (around $6,700 to $6,750), not the 2026 OBBBA computation: $47,500 of deductions, taxable income of $35,951.23, and tax of $3,818.15." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired and applied a $16,500 standard deduction, $10,500 of personal exemptions and 'reverted' brackets. It also dropped the $243 refund. It skipped the $32,200 standard deduction, $3,300 of aged additions and the $12,000 senior deduction that apply under 2026 law." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed that deductions, exemptions and credits eliminate all tax. In fact the $47,500 of standard and senior deductions leave $35,951.23 of taxable income on AGI of $83,451.23, and no nonrefundable credit applies to a childless couple with this AGI." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund, which gave AGI of $83,208. It applied only a $34,000 standard deduction and left out the OBBBA $6,000-per-senior deduction for both spouses, so it taxed $49,208 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 refund and subtracted personal exemptions, which are zero under 2026 law. It also never applied the $12,000 senior deduction. Its $5,045 answer implies taxable income of about $46,000, roughly $10,000 above the correct $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used 2025 pre-OBBBA values: a $30,000 standard deduction, $1,600 aged additions and a 10% bracket ending at $23,850. It left out the $6,000-per-senior deduction for both spouses, which put taxable income at $50,251 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $34,030 standard deduction built from a $30,750 base and $1,640 aged additions, and it left out the OBBBA $6,000 senior deduction for each spouse. Taxable income came out at $49,421 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed taxable Social Security against a nonexistent $64,000 joint threshold and got $12,085. The MFJ formula with the $32,000 and $44,000 thresholds makes the full $16,320 (85%) taxable. It also left out the two $1,650 aged additional standard deductions. The two errors partly offset each other and produced $35,016 of taxable income." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It cited 'limited itemized deductions from mortgage interest' and the SALT refund, but no mortgage interest is listed, and the $243 refund is income, not a deduction. Its $3,677 implies taxable income of about $34,775, roughly $1,176 below the correct $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as non-taxable and reported zero tax. With provisional income far above $44,000, $16,320 of the benefits is taxable, AGI is $83,451.23, and taxable income after the $47,500 of deductions is $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'post-TCJA' 2026 law: $29,900 of combined standard deduction, aged additions and personal exemptions, taxed at 15% above $23,900. It skipped the OBBBA-extended $32,200 standard deduction, $3,300 of aged additions, the $12,000 senior deduction and the 12% bracket." +us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI and its $47,500 of standard and senior deductions were correct, but it also took a $333.33 qualified-overtime deduction by treating the $1,000 by which wages exceed $25 x 40 x 52 as overtime premium. The spouse's usual weekly hours are 40, so no hours exceed the FLSA 40-hour threshold and there is no qualified overtime compensation. That cut tax by $40." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no computation. Its $5,523 implies taxable income of about $50,150, meaning roughly $33,300 of deductions: a standard deduction with aged additions but without the $12,000 OBBBA senior deduction. The correct deductions total $47,500." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it applied only a $34,030 standard deduction with aged additions and left out the $6,000-per-spouse OBBBA senior deduction. It taxed $49,421 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It modeled a 'sunset' of the TCJA: a $16,600 standard deduction, $5,300 personal exemptions and a 15% rate above $24,800. It skipped the permanent OBBBA $32,200 standard deduction and the $12,000 senior deduction, so it taxed $52,951 at 15% instead of $35,951.23 at 12%." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the $243 state tax refund from income, which gave taxable income of $35,708 instead of $35,951.23. It also ended the 10% bracket at $24,400 instead of the 2026 MFJ figure of $24,800." +us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it applied only a $34,196 standard deduction and never applied the OBBBA $6,000 senior deduction for each spouse aged 65 or older. It taxed $49,255 instead of $35,951.23." +us,scenario_033,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $1,225 of auto-loan interest, which is not allowed because no qualifying new-vehicle loan is established. It also added an invented 'temporary $1,500' to the $32,200 base standard deduction. Together these overstated deductions by $2,725 and gave taxable income of $33,226." +us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score. The correct derivation gives taxable income of $35,951.23 and tax of $3,818.15." +us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, standard deduction and senior deduction were correct, but it also subtracted $1,225 of auto-loan interest and a $333.33 qualified-overtime premium. With 40 usual weekly hours there is no FLSA overtime, and no qualifying vehicle loan is established. Those extra deductions cut taxable income by $1,558.33 and tax by $187." +us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $11,400 self-employment loss and the $12,000 senior deduction and computed about $10,800 to $11,040 of tax. It then submitted -$250, a negative figure, even though tax before refundable credits has a floor of zero." +us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $11,400 self-employment loss and the $243 refund from AGI, which gave $94,608, and left out the $12,000 senior deduction. It computed $6,748 of tax and then submitted $5,998, which its own work does not support." +us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It treated taxable Social Security as a subtraction and set taxable income to zero. In fact $16,320 of Social Security is added to income, and after the $47,500 of standard and senior deductions $35,951.23 of taxable income remains, taxed at $3,818.15." us,scenario_033,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,043.50, then added a fabricated $64.15 “rounding adjustment.” The component calculations are already exact to the cent and require no adjustment." us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $6,043.50 and then improperly rounded it to $6,045. Payroll tax is requested as an annual amount, and no rule permits rounding the exact household total to a five-dollar increment." @@ -2789,147 +3113,158 @@ us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"The model correctl us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly calculated each spouse's employee FICA tax and the $6,043.50 total, then improperly rounded it to $6,042. No payroll-tax computation or rounding rule produces that submitted value." us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct $79,000 wage base and combined 7.65% Social Security and Medicare rate but computed their product incorrectly. The product is $6,043.50, not $6,123." us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss produces no self-employment tax, then contradicted that rule by reporting an invented negative liability. A negative SSTB amount cannot be multiplied by the 15.3% rate to create a self-employment tax credit; the correct liability is floored at zero." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It derived the reference exactly — $16,100 standard deduction, $43,120 of taxable income, $1,240 + 12% x $30,720 = $4,926 — and then overrode its own completed arithmetic with $4,849, attributing the $77 cut to 'projected 2026 bracket indexing' after it had already used the enacted 2026 bracket edges of $12,400 and $50,400. No deduction or bracket parameter produces $4,849; the submitted value contradicts the calculation stated one clause earlier." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied the 2025 single standard deduction of $15,000 instead of the 2026 amount of $16,100 and the stale $11,600 ceiling on the 10% band instead of $12,400, then garbled the bracket arithmetic ('$3,809 on first $44,725') into $8,074. Its own $44,220 taxable-income figure yields only $1,160 + 12% x $32,620 = $5,074, and the correct 2026 base of $43,120 yields $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It overstated the 2026 single standard deduction as about $16,600 (the enacted figure is $16,100), computed $4,866 from that base, and then cut the answer to $4,297 'for bracket inflation' — a $569 reduction with no rule behind it, since the 12% band runs unbroken from $12,400 to $50,400. Applying its own bracket parameters to the correct $43,120 of taxable income gives $1,240 + 12% x $30,720 = $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It used the correct 2026 parameters — $16,100 standard deduction, $43,120 taxable income, 10% to $12,400 then 12% — and reached $4,926, then submitted $4,969 as a 'rounding to bracket estimate.' Rounding cannot move a bracket computation by $43; the final value simply abandons its own correct result." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the reference to the cent — $59,220 less the $16,100 standard deduction gives $43,120, taxed at $1,240 plus 12% of $30,720 for $4,926.40 — and then submitted $5,194 as a 'rounded estimate accounting for bracket parameter estimates.' Its own stated thresholds are the enacted 2026 ones, so there was no parameter uncertainty left to adjust for." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It guessed a projected 2026 single standard deduction of $15,750 and a 10%-band ceiling of $11,950 rather than the enacted $16,100 and $12,400, giving $43,470 of taxable income and $4,977. That $350 of missing deduction plus the $450 low bracket ceiling accounts for the entire $50.60 overstatement; its conclusions that the mortgage balance yields no deductible interest, that medical costs fall under the 7.5% AGI floor, and that no nonrefundable credits apply were all correct." -us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It derived $4,926.40 exactly — $16,100 standard deduction, $43,120 taxable income, $12,400 x 10% plus $30,720 x 12% — stated that no credits apply, and then submitted $6,499. That value appears nowhere in its reasoning and no 2026 rate schedule produces it from $59,220 of pension income." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It reduced income by roughly $15,400 instead of the 2026 single standard deduction of $16,100 and applied the 2025 rate schedule with its $11,925 ceiling on the 10% band, producing $43,820 of taxable income and $5,020. The 2026 parameters give $43,120 of taxable income taxed at $1,240 plus 12% of $30,720, or $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It carried tax year 2025 parameters into 2026 — the $15,000 standard deduction and the $11,925 top of the 10% bracket — instead of the indexed 2026 values of $16,100 and $12,400. The $1,100 of missing deduction taxed at 12% plus the $475 of income shifted from 10% to 12% is exactly the $141.50 by which its $5,067.90 exceeds $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA individual provisions lapse for 2026 and rebuilt the return on a pre-TCJA base: an $8,300 standard deduction, a restored $5,300 personal exemption, and a 15% second bracket. OBBBA made the larger standard deduction, the zero personal exemption, and the 10/12/22 schedule permanent, so 2026 gives $59,220 − $16,100 = $43,120 taxed at 10% to $12,400 then 12%, or $4,926.40 rather than $6,233." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It explicitly computed 'post-TCJA sunset' 2026 tax with an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above an $11,600 bracket top. The TCJA structure was made permanent before this tax year, so 2026 uses a $16,100 standard deduction, no personal exemption, and 12% up to $50,400 — $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It stated no parameters, and its $5,122 is the amount the 2024 schedule produces on $59,220 — a $14,600 standard deduction leaving $44,620, taxed at 10% to $11,600 plus 12% on the remaining $33,020. The 2026 schedule uses a $16,100 deduction and a $12,400 ceiling on the 10% band, giving $43,120 of taxable income and $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied an 'expiration of the TCJA' base for 2026 — an $8,350 standard deduction plus a reinstated $5,050 personal exemption and a 15% bracket above roughly $11,450 — reaching $45,820 of taxable income. The 2026 law provides a $16,100 standard deduction, no personal exemption, and a 12% bracket to $50,400, so taxable income is $43,120 and tax is $4,926.40, not $6,301." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It combined a pre-TCJA $8,300 standard deduction with a $5,050 personal exemption and taxed the resulting $45,870 at 10% to $11,600 then 15%. Personal exemptions remain zero in 2026 and the standard deduction is $16,100 with a 12% second bracket, so the correct base is $43,120 and the tax is $4,926.40 rather than $6,300.50." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"Its $3,858 corresponds to the head-of-household schedule — a $24,150 standard deduction leaving $35,070, taxed at 10% up to $17,700 plus 12% on the remainder for about $3,854 — applied to a person who lives alone with no dependents and therefore files single. The single schedule gives $59,220 − $16,100 = $43,120 taxed at $1,240 plus 12% of $30,720, or $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a 'post-TCJA sunset' 2026 base of an $8,300 standard deduction plus a $5,300 personal exemption and taxed the resulting $45,620 at 15% above $11,925. The 2026 parameters are a $16,100 standard deduction, no personal exemption, and a 12% bracket running to $50,400, which yields $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It assumed pre-TCJA rules govern 2026, deducting $8,300 of standard deduction and a $5,300 personal exemption and applying a 15% rate above $11,925 to $45,620 of taxable income. Under the permanent post-OBBBA schedule the deduction is $16,100, the personal exemption is zero, and the second bracket is 12% to $50,400, giving $4,926.40 instead of $6,246.75." -us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its income and filing-status reasoning was right, but it used the 2025 standard deduction of $15,000 and the 2025 bracket edge of $11,925 for a 2026 return where the indexed values are $16,100 and $12,400. That understated deduction plus the lower 10%-band ceiling raises the result from $4,926.40 to $5,067.90." -us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It substituted the 2024 schedule for 2026 by its own admission — a $14,600 standard deduction and a 10% band ending at $11,600 — giving $44,620 of taxable income and $5,122.40. The 2026 values of $16,100 and $12,400 give $43,120 of taxable income and $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It used the 2023 standard deduction of $13,850, added a fabricated $15,660 item to reach $50,630.50 of 'taxable income,' and taxed it on a 15%/22% schedule with a $23,300 breakpoint that exists in no year of current law; even its own stated arithmetic ($3,495 plus $6,012.71) totals $9,508, not the $4,407 it submitted. The 2026 computation is $59,220 − $16,100 = $43,120 taxed at 10% to $12,400 then 12%, or $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It performed no computation and asserted $3,344, a figure that requires about $29,900 of taxable income and therefore roughly $29,300 of deductions — nearly double the $16,100 standard deduction that is this single filer's only allowance. The correct base is $59,220 − $16,100 = $43,120, taxed at $1,240 plus 12% of $30,720 for $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It named no deduction or bracket and submitted $3,720, which corresponds to roughly $33,070 of taxable income and therefore about $26,150 of deductions against $59,220 of pension income. The 2026 single standard deduction is $16,100, leaving $43,120 taxed at 10% to $12,400 then 12%, for $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed deductions 'including mortgage interest' inferred from the $226,000 loan balance, although no interest is listed and unlisted inputs are zero, yet its $4,966 implies only about $15,800 of total deductions — less than the $16,100 standard deduction it should have taken and inconsistent with the itemization it says it used. The correct computation is $43,120 of taxable income taxed at $1,240 plus 12% of $30,720, or $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied a 'TCJA sunset' 2026 base of an $8,300 standard deduction plus a $5,300 personal exemption and taxed the remaining $33,220 at 15%. The permanent post-OBBBA parameters give a $16,100 standard deduction, no personal exemption, and 12% above $12,400, so tax on $43,120 of taxable income is $4,926.40, not $6,223." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It invented pre-TCJA 2026 parameters — an $8,532 standard deduction, a $5,442 personal exemption, a $12,529 10%-band ceiling, and a 15% second rate — leaving $45,246 taxable. Under actual 2026 law the deduction is $16,100, the exemption is zero, and the second bracket is 12% to $50,400, giving $43,120 of taxable income and $4,926.40 of tax." -us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It computed 'under 2026 pre-TCJA rules' with an $8,315 standard deduction, a $5,303 personal exemption, and a 15% rate above $12,211, taxing $45,602. Personal exemptions are zero and the standard deduction is $16,100 in 2026 with a 12% second bracket, so taxable income is $43,120 and tax is $4,926.40 rather than $6,230." -us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It correctly identified $59,220 of pension income, single filing, and the standard deduction, but deliberately substituted the 2025 figures — a $15,000 deduction and an $11,925 10%-band ceiling — 'as a practical estimate for 2026.' The 2026 values are $16,100 and $12,400, so taxable income is $43,120 and tax is $4,926.40, not the $5,067.90 the 2025 schedule produces." -us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It used a projected $15,750 standard deduction and a $12,150 top of the 10% band instead of the enacted $16,100 and $12,400, computed $4,973, and then submitted $6,437 'after rounding and bracket threshold adjustments.' No 2026 threshold change turns $43,470 of taxable income into a $1,464 larger tax; the correct result on $43,120 of taxable income is $4,926.40." -us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It started from the correct base — $16,100 standard deduction and $43,120 of taxable income — then applied a nonexistent 22% bracket beginning at $43,100 when 2026's 12% band runs to $50,400, subtracted a $3,497.60 'elderly credit' that a 56-year-old cannot claim and that is capped at $750 for a qualifying single filer, and finally added $6,995.20 of a 'medical expense deduction item' directly to the tax. Stripping those three fabrications leaves $1,240 plus 12% of $30,720, or $4,926.40, against the $8,440.50 submitted." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the correct $16,100 standard deduction, $43,120 of taxable income and roughly $4,926 of tax from the $12,400/$50,400 brackets. It then cut the figure to $4,849 by applying 'projected bracket indexing' a second time to thresholds that were already indexed for 2026." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction instead of $16,100. It then mangled the bracket arithmetic by mixing 2024 thresholds ($11,600, $44,725) and reported $8,074, a figure that does not follow from its own 10%/12% brackets. Applying 10%/12% correctly to $43,120 gives $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model overstated the 2026 standard deduction as $16,600 instead of $16,100 and got about $4,866. It then 'reduced slightly for bracket inflation' to an unsupported $4,297, even though its $12,400/$50,400 brackets were already the indexed 2026 values." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed the exact correct chain: a $16,100 standard deduction, $43,120 of taxable income, and $1,240 + $3,686 = $4,926. It then submitted $4,969 by 'rounding to bracket estimate', an adjustment with no basis once the 2026 brackets are applied." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reached $4,926 from the $16,100 standard deduction and the 2026 10%/12% brackets. It then submitted an unexplained $5,194 as an allowance for 'bracket parameter estimates', which discarded its own correct result." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $15,750 as the 2026 single standard deduction, which is the 2025 OBBBA figure; the 2026 amount is $16,100. It also put the top of the 10% bracket at $11,950 instead of $12,400, so taxable income came to $43,470 and tax to $4,977. It also considered itemizing imputed mortgage interest, but mortgage interest is unlisted and therefore $0." +us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's reasoning computed exactly $1,240 + $3,686.40 = $4,926.40 from the $16,100 standard deduction and the $12,400/$50,400 brackets. It then submitted $6,499, a number that contradicts its own derivation." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income of $43,820 implies a $15,400 standard deduction instead of $16,100. Its $5,020 of tax matches the 2025 10% threshold of $11,925 rather than the 2026 threshold of $12,400. Both parameters are stale, and together they overstate the tax." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the 2025 pre-OBBBA $15,000 standard deduction and the 2025 10% threshold of $11,925, giving $44,220 of taxable income and $5,067.90 of tax. The 2026 values are $16,100 and $12,400, which give $43,120 of taxable income and $4,926.40 of tax." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption and a 12% bracket through $50,400." +us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $15,600 standard deduction instead of $16,100 and a $12,200 top for the 10% bracket instead of $12,400. That gives $43,620 of taxable income and $4,990.40 of tax instead of $43,120 and $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above $11,600. OBBBA made the $16,100 standard deduction, the zero personal exemption and the 12% bracket through $50,400 permanent for 2026." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no working, but its $5,122 exactly matches 2024 parameters: a $14,600 standard deduction and a 10% bracket to $11,600, giving 1,160 + 12% × $33,020 = $5,122.40. It never applied the 2026 $16,100 deduction and $12,400 threshold." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired and applied an $8,350 standard deduction, a $5,050 personal exemption and a 15% bracket above about $11,450. OBBBA permanently kept the TCJA rates, set the 2026 single standard deduction at $16,100, and eliminated the personal exemption." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied pre-TCJA 2026 rules: an $8,300 standard deduction plus a $5,050 exemption, and 15% above $11,600. Under OBBBA's permanent extension, 2026 has a $16,100 standard deduction, no personal exemption and 12% up to $50,400, which gives $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model said it subtracted 'deductions and exemptions', but no personal exemption exists in 2026. Its $3,858 implies taxable income of about $34,200 under the 2026 brackets, which means it subtracted roughly $9,000 more than the $16,100 standard deduction that is the only deduction available." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15% above $11,925. OBBBA made the TCJA structure permanent, so the correct figures are a $16,100 standard deduction, no exemption, and 12% on income between $12,400 and $50,400." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied 'post-TCJA' rules: an $8,300 standard deduction, a $5,300 exemption, and 15% above $11,925. OBBBA repealed the sunset, so 2026 taxable income is $59,220 − $16,100 = $43,120, taxed at 10%/12%." +us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 pre-OBBBA $15,000 standard deduction and the 2025 10% threshold of $11,925, giving $5,067.90. The 2026 values are a $16,100 deduction and a $12,400 threshold." +us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model stated that it used the 2024 $14,600 standard deduction and the 2024 brackets (10% to $11,600) as proxies for 2026. That overstates taxable income by $1,500 and understates the 10% band compared with the 2026 $16,100 deduction and $12,400 threshold." +us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model used the 2023 $13,850 standard deduction and invented a medical-expense add-back that pushed taxable income to $50,630.50, above AGI minus the deduction. It then mixed a pre-TCJA 15% rate with a 22% bracket, and its stated terms do not even sum to $4,407. The correct figure is $43,120 taxed at 10%/12%." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model gave no numeric working. Its $3,344 implies taxable income of about $29,900 under the 2026 brackets, so it subtracted roughly $13,000 more than the $16,100 standard deduction. The only deduction available is the standard deduction, which leaves $43,120." +us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model gave no computation. Its $3,720 implies taxable income of about $33,100, so it subtracted roughly $10,000 beyond the $16,100 standard deduction. The correct taxable income is $59,220 − $16,100 = $43,120, and the tax is $4,926.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model deducted 'mortgage interest' imputed from the $226,000 mortgage balance, but mortgage interest is unlisted and therefore $0. Its $4,966 implies total deductions of about $15,770, below the $16,100 standard deduction it should have taken instead." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a TCJA sunset: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above $12,400. OBBBA made the 12% bracket (through $50,400), the $16,100 standard deduction and the zero exemption permanent for 2026." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a 'current-law TCJA sunset' with an $8,532 standard deduction, a $5,442 personal exemption and a 15% bracket. Current law after OBBBA is a $16,100 standard deduction, no personal exemption and 12% up to $50,400." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model got the $16,100 standard deduction and the $43,120 taxable income right. It then put the top of the 10% bracket at $12,200 instead of the 2026 value of $12,400, which taxed an extra $200 at 12% and overstated the tax by $4, at $4,930.40." +us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied 'pre-TCJA' 2026 rules: an $8,315 standard deduction, a $5,303 exemption, and 15% above $12,211. OBBBA made the TCJA parameters permanent, so the 2026 figures are a $16,100 deduction, no exemption and a 12% bracket through $50,400." +us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model stated that it used the 2025 $15,000 standard deduction and the 2025 rate schedule (10% to $11,925) as an estimate for 2026. The 2026 values are $16,100 and $12,400, which give $4,926.40 rather than $5,067.90." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used a $15,750 standard deduction and a $12,150 10% threshold instead of $16,100 and $12,400, and got about $4,973. It then submitted an unexplained $6,437 for 'rounding and bracket threshold adjustments', which departs from its own computation." +us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model reached the correct $43,120 of taxable income but applied a nonexistent 22% bracket starting at $43,100; in 2026 the 12% bracket runs to $50,400. It also subtracted a Credit for the Elderly, which requires age 65 or older or disability, and then added the medical deduction back as tax. The result was an incoherent $8,440.50 instead of $4,926.40." us,scenario_036,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly declared the $59,220 pension income below a New Jersey adult Medicaid threshold without converting the household's MAGI to its income-to-FPL level or identifying a qualifying category. The correct computation places MAGI at 3.71 times FPL and assigns medicaid_category NONE, so the head is not eligible." us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that payroll tax therefore equals zero, but submitted $3,715 instead. That amount applies a payroll-tax-style rate to the $59,220 taxable private pension, which is not employee compensation subject to payroll tax." us,scenario_036,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted a $915.60 NJ medical-expense deduction ($2,100 of non-ESI medical costs less 2% of $59,220) on top of the $1,000 exemption, while nj_taxable_income here is nj_agi less the exemption alone, so its taxable base of $57,304 undershoots $58,220. Its own arithmetic then produced $1,673.55, and it submitted $187 — a figure its derivation never yields." -us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It stretched the 3.5% NJ bracket across $35,000-$60,000 and omitted the 5.525% bracket that begins at $40,000, and it never subtracted the $1,000 personal exemption, giving $1,417.50. It then abandoned that number for $3,185 attributed to a 'standard allowance and adjustments' — NJ's gross income tax has no standard deduction and no such allowance." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Its first pass reproduced the reference exactly: $59,220 less the $1,000 exemption = $58,220 taxed at 1.4/1.75/3.5/5.525% = $1,724.15. It then discarded that by subtracting an $8,405 medical-expense deduction built from $9,589 that includes the $7,389 employer-sponsored premium; ESI premiums are withheld pre-tax and outside NJ income, and PolicyEngine takes no medical deduction against nj_agi here, so cutting taxable income to $49,815 produced $1,260." -us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It correctly denied the pension exclusion at age 56 but then reduced taxable income by an $8,405 medical-expense deduction that included the $7,389 pre-tax employer-sponsored premium, reaching $1,260 instead of the $1,724.16 that nj_agi minus the $1,000 exemption yields. It then cut that again to $613 on an invented 'PolicyEngine treatment of the ESI premium deduction' that appears nowhere in the NJ tax computation." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied every step of the NJ schedule correctly but inserted an $8,404.60 medical-expense deduction (the full $9,589 of listed premiums and expenses, including the $7,389 employer-sponsored premium, less 2% of gross income), dropping taxable income from $58,220 to $49,815.40. NJ taxable income here is nj_agi less only the $1,000 exemption, and the 5.525% bracket on the missing $8,405 is exactly the $464.36 gap between its $1,259.80 and $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It granted the NJ retirement-income exclusion to a 56-year-old; that exclusion requires the taxpayer to be 62 or older or disabled, so the entire $59,220 pension stays in nj_agi. It then reported $1,591 without any bracket computation, a number consistent with neither its own 'fully excluded, near $0' conclusion nor the $1,724.16 that the schedule yields on $58,220." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the NJ single rate schedule correctly to the full $59,220 but never subtracted the $1,000 single personal exemption that produces nj_taxable_income of $58,220. The $55.25 excess over the reference is precisely $1,000 taxed at the 5.525% top-applicable rate." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It applied NJ's up-to-$75,000 pension and retirement income exclusion on the strength of the income test alone, ignoring the statutory age requirement that the taxpayer be 62 or older (or disabled); the head is 56, so no part of the pension is excluded. With the full $59,220 in nj_agi and the $1,000 exemption, the schedule yields $1,724.16 rather than $0." -us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly kept the pension fully taxable and took the $1,000 exemption, then subtracted an additional $915.60 medical-expense deduction ($2,100 of non-ESI medical costs less the 2%-of-gross-income floor). PolicyEngine's nj_taxable_income is nj_agi less nj_total_exemptions only, so the base is $58,220 and the tax is $1,724.16, not $1,673.57." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It got the age test, the exemption, and the rate schedule right but subtracted an $8,404.60 medical-expense deduction assembled from all $9,589 of listed health costs, including the $7,389 employer-sponsored premium withheld pre-tax and therefore already outside NJ income. Taxable income is $58,220, not $49,815.40, and the tax is $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, only an assertion of 'NJ-specific exemptions and deductions.' Its $1,113 corresponds to taxable income near $47,160, meaning roughly $12,060 of deductions beyond the $1,000 exemption; the only subtraction from nj_agi of $59,220 is that exemption, giving $58,220 and $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It applied NJ's $75,000 pension exclusion using only the under-$100,000 gross income condition and skipped the age-62-or-disabled requirement, which a 56-year-old fails. The pension therefore stays fully in nj_agi, and $58,220 of taxable income after the $1,000 exemption produces $1,724.16 rather than $0." -us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It asserted that the pension is taxed only above 'the NJ pension exclusion' and gave no computation; at age 56 the head qualifies for no exclusion at all. Its $1,057 implies taxable income near $46,145, i.e. about $13,075 excluded, whereas the full $59,220 less the $1,000 exemption is taxed for $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It reached the correct taxable income of $58,220 but used 2% on the first $20,000 and 2.5% on the next $15,000 in place of NJ's actual single-filer rates of 1.4% and 1.75%. Those two rate errors add exactly $120 and $112.50, the full $232.50 by which $1,956.66 exceeds $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It invented a $20,000 NJ standard deduction and a flat 5.525% rate; NJ's gross income tax has no standard deduction, and its graduated schedule applies 1.4%/1.75%/3.5% below $40,000. It then subtracted a $972.46 'NJ-1040-H property tax credit' computed as 18% of a $226,000 mortgage balance, but NJ-1040-H requires property tax paid and income at or below $20,000, and no property tax is listed." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It performed no bracket computation, stating only that the tax is positive. Its $2,860 exceeds even the tax on the undeducted, unexempted $59,220 ($1,779.41) and corresponds to a flat rate near 4.8% on gross, whereas the graduated NJ schedule on $58,220 gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It reduced the tax with unspecified 'standard credits,' but NJ grants no standard nonrefundable credit against the gross income tax; the only offset is the $1,000 personal exemption already in nj_taxable_income. Its $600 corresponds to taxable income near $38,300, roughly $20,900 below the actual $58,220 base that yields $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took the $1,000 exemption correctly, then also subtracted a $7,304.60 medical deduction ($8,489 of premiums and expenses less 2% of income), which PolicyEngine does not apply against nj_agi. Its taxable base of $50,915.40 is $7,304.60 short of the correct $58,220, and 5.525% of that shortfall is the $403.58 gap between $1,320.58 and $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It excluded the entire $59,220 pension under NJ's retirement income exclusion based on a total-income threshold alone, omitting the requirement that the taxpayer be 62 or older or disabled; the head is 56 and qualifies for no exclusion. The full pension is in nj_agi, so $58,220 of taxable income is taxed for $1,724.16, not $0." -us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It supplied no derivation beyond 'after deductions and credits.' Its $1,125 corresponds to taxable income near $47,375, about $11,845 below the actual base; nj_agi of $59,220 is reduced only by the $1,000 exemption, and the NJ single schedule on $58,220 yields $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It replaced NJ's four-rate single schedule with a fabricated '$350 plus 2.75% of excess over $20,000' formula, and it first cut income by an $8,405 medical-expense deduction including the pre-tax employer-sponsored premium. The correct base is $58,220 taxed at 1.4/1.75/3.5/5.525% for $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical-expense deduction (all $9,589 of listed health costs, including the $7,389 pre-tax employer-sponsored premium, less the 2% floor), which PolicyEngine does not take against nj_agi. It then compounded the error by reporting $1,008.54 on its own $49,815 base, when the NJ schedule on that amount is $1,259.80; the correct base of $58,220 gives $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so no substantive tax reasoning was submitted. The required computation is nj_agi $59,220 less the $1,000 single exemption, taxed on the NJ single schedule for $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It recognized that the age-62 rule bars the head at 56, then invented a separate income-based exclusion of up to $100,000 for filers under $150,000 of income and excluded the whole pension. NJ has no such age-free exclusion; the $59,220 pension is fully in nj_agi, and $58,220 after the exemption is taxed for $1,724.16." -us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It correctly denied the pension exclusion and took the $1,000 exemption, then subtracted an extra $1,015.60 medical-expense deduction ($2,200 of non-ESI medical costs less 2% of $59,220). nj_taxable_income is nj_agi less nj_total_exemptions only, so the base is $58,220 and the tax is $1,724.16, not $1,668." -us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning arrived at the reference exactly — $59,220 less the $1,000 exemption is $58,220, taxed at 1.4/1.75/3.5/5.525% for $1,724.16 — after first mis-stating a 6.37% bracket at $50,000 and correcting it. It then discarded that result for $2,283.06 from an unexplained 'withholding-style flat estimation,' submitting a number its own corrected schedule contradicts." -us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It applied a $16,100 federal-style standard deduction to NJ income, but NJ's gross income tax has no standard deduction — only the $1,000 personal exemption — and it used a fabricated '$983.60 on the first $40,000' plus 5.5% schedule instead of the 1.4/1.75/3.5/5.525% brackets. It then added $507.48 of 'property tax deduction' to the tax itself, though no property tax is listed and a deduction reduces income rather than increasing liability." +us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning subtracted a $915.60 NJ medical deduction ($2,100 of premiums and expenses minus the 2% floor) that the reference does not allow, which gave $1,673.55. It then submitted value = 187, a number that matches none of its own arithmetic. The correct figure is $58,220 of taxable income (AGI less the $1,000 exemption), taxed to $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It imported a federal-style $6,500 standard deduction, which NJ does not have, and left out both the $1,000 personal exemption and the 5.525% bracket above $40,000, taxing income up to $60,000 at 3.5% for $1,417.50. It then jumped without explanation to $3,185. The correct base is $58,220 (AGI less the exemption), with 5.525% applied to $18,220 above $40,000, for $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It correctly reached $1,724.15 on $58,220 of taxable income, then threw that out to subtract an $8,405 medical deduction. That deduction counted the $7,389 employer-sponsored premium and the OTC spending in $9,589 of 'medical expenses'. The reference allows no medical deduction, so taxable income stays at $58,220 rather than $49,815." +us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction that included the $7,389 employer-sponsored premium, cutting taxable income to $49,815 and tax to $1,260. It then applied an invented further 'ESI premium deduction' to get $613. The reference taxes $58,220 (AGI less the $1,000 exemption) with no medical deduction, for $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated the $7,389 employer-sponsored premium, both $1,000 premium lines, and the OTC and other medical spending as $9,589 of deductible NJ medical expenses, and deducted $8,404.60 above the 2% floor. That lowered taxable income to $49,815.40 instead of the reference's $58,220, which allows no medical deduction." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It applied NJ's retirement income exclusion (up to $75,000 for single filers) to a 56-year-old who is not disabled. That exclusion requires age 62+ or disability. It then reported an unexplained 'phase-out remainder' of $1,591 instead of taxing the full $59,220 less the $1,000 exemption for $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It correctly declined the pension exclusion, then subtracted a $1,016 medical deduction ($2,200 of premiums, OTC and other medical costs minus the 2% floor) that the reference does not allow. That put taxable income at $57,204 instead of $58,220 and the tax at $1,668 instead of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the correct NJ single brackets to the full $59,220 but left out the $1,000 single personal exemption. That taxed $19,220 instead of $18,220 at 5.525% and gave $1,779.41 instead of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It excluded the entire $59,220 pension under NJ's pension exclusion using only the $100,000 income test and ignored the requirement that the filer be 62+ or disabled. The head is 56, so the whole pension is taxable, which gives $58,220 of taxable income after the exemption and $1,724.16 of tax." +us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted a $915.60 medical deduction ($2,100 of premiums and medical costs above the 2% floor of $1,184.40) that the reference does not allow. That left $57,304.40 of taxable income instead of $58,220 (AGI less the $1,000 exemption only), and $1,673.57 of tax instead of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It correctly denied the pension exclusion, then deducted $8,404.60 of medical expenses built from $9,589 that included the $7,389 employer-sponsored premium. That dropped taxable income to $49,815.40, whereas the reference allows no medical deduction and taxes $58,220." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation beyond unnamed 'NJ-specific exemptions and deductions'. On the NJ single schedule, $1,113 corresponds to about $47,150 of taxable income, so it subtracted roughly $11,000 of deductions beyond the $1,000 exemption. Only the exemption applies, which leaves $58,220 of taxable income and $1,724.16 of tax." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"It excluded the full $59,220 pension under NJ's $75,000 pension exclusion using only the income limit and never applied the age 62+ or disability requirement. The head is 56, so the pension is fully taxable and the tax on $58,220 is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,age_disability,False,"It taxed only the pension 'above the NJ pension exclusion', applying a partial exclusion to a 56-year-old who is not disabled and does not qualify for it. $1,057 corresponds to about $46,150 of taxable income instead of the correct $58,220 (full pension less the $1,000 exemption), which is taxed to $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly got taxable income of $58,220 but used 2% and 2.5% for the first two NJ single brackets instead of 1.4% and 1.75%. That added $232.50 of tax and gave $1,956.66 instead of $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It invented a $20,000 NJ standard deduction (NJ has none, only the $1,000 exemption) and taxed the remainder at a flat 5.525% instead of the graduated schedule. It then subtracted a property tax credit made up as 18% of the mortgage balance, although no property tax is listed. The correct tax on $58,220 is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation. $2,860 exceeds even the NJ schedule's tax on the full $59,220 with no exemption ($1,779.41), so it used a rate schedule far above NJ's 1.4%/1.75%/3.5%/5.525% single brackets. The correct tax on $58,220 is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It subtracted unnamed 'standard credits' from NJ tax to reach about $600, but no NJ nonrefundable credit applies to this single pensioner. The tax after nonrefundable credits equals the $1,724.16 computed on $58,220 of taxable income." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $7,304.60 of medical expenses, built from $8,489 of 'qualifying' costs that included the $7,389 employer-sponsored premium, above the 2% floor. That brought taxable income down to $50,915.40, whereas the reference allows no medical deduction and taxes $58,220 for $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It excluded the entire private pension under NJ's pension exclusion on the income threshold alone, skipping the age 62+ or disability requirement that a 56-year-old head fails. The full $59,220 is taxable, which gives $1,724.16 after the $1,000 exemption." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted a medical expense deduction of about $900 (out-of-pocket premiums and expenses above the 2% floor) along with the $1,000 exemption. That lowered taxable income to about $57,320, whereas the reference allows no medical deduction, taxes $58,220, and arrives at $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It declared the head an 'eligible single filer' for NJ's pension exclusion and excluded the whole $59,220, but the exclusion requires age 62+ or disability and the head is 56. The pension is fully taxable, and the tax on $58,220 is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It cited unnamed 'deductions and credits'. $1,125 corresponds to about $47,400 of NJ taxable income, meaning roughly $10,800 of deductions beyond the $1,000 exemption that NJ does not allow here. The correct taxable income is $58,220, taxed to $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction built from $9,589 that included the $7,389 employer-sponsored premium, cutting taxable income to $49,815 instead of $58,220. It then applied an invented '$350 plus 2.75% over $20,000' schedule instead of NJ's 1.4%/1.75%/3.5%/5.525% single brackets." +us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,405 medical deduction from $9,589 of listed costs, including the $7,389 employer-sponsored premium, which lowered taxable income to $49,815. It then misapplied the brackets, giving $1,008.54 where the NJ schedule gives $1,259.78 on that base. The reference allows no medical deduction and taxes $58,220 for $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no substantive answer to evaluate against the $1,724.16 reference." +us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It acknowledged that the pension exclusion is for residents 62+ and that the head is 56, but still applied an invented $100,000 income-based exclusion that wiped out the pension. Below age 62 without disability, no exclusion applies, and the tax on $58,220 is $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It correctly taxed the full pension but subtracted a $1,015.60 medical deduction ($2,200 of premiums, OTC and other medical costs minus the 2% floor) that the reference does not allow. That produced $57,204.40 of taxable income and $1,668 of tax instead of $58,220 and $1,724.16." +us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning explicitly reached the correct $1,724.16 ($58,220 of taxable income after the $1,000 exemption, taxed on the NJ single brackets). It then dropped that result for an unexplained 'withholding-style flat estimation' of $2,283.06, after first also inventing a 6.37% rate above $50,000." +us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted a $16,100 'standard deduction' that NJ does not have, using it in place of the $1,000 personal exemption, and applied 5.5% instead of 5.525% on the top portion. It then added a $507.48 'property tax deduction' onto the tax even though no property tax is listed. The correct tax on $58,220 of taxable income is $1,724.16." us,scenario_036,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It explicitly rejected the OBBBA qualified-overtime deduction, calling the $9,629 FLSA premium non-deductible, when it is a $9,629.03 below-the-line deduction that non-itemizers stack on top of the standard deduction. It compounded this by subtracting the $5,789 employer-sponsored insurance premium from wages, which does not reduce the stated employment income, and by using the stale $14,600 standard deduction instead of the 2026 single amount of $16,100; its submitted $1,646 also contradicts the $2,308 its own arithmetic produced." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of $41,539, the correct $16,100 standard deduction and the correct 2026 brackets, but omitted the $9,629.03 OBBBA qualified-overtime deduction entirely, leaving taxable income at $25,439 rather than $15,810.43. It then submitted $2,884 against the $2,805 its own bracket arithmetic yielded." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629.03 OBBBA qualified-overtime deduction, which stacks with the $16,100 standard deduction to bring taxable income to $15,810.43 instead of the $25,439 it used. It then inflated its own correctly computed $2,805 to $3,160 by invoking 'slightly higher inflation-adjusted brackets,' a move that raises tax where higher brackets would lower it." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It identified the OBBBA overtime deduction but never subtracted it: removing the full $9,629.03 from taxable income of $25,439 gives $15,810.43 and tax of $1,649.25. Instead of running that arithmetic it estimated a $235 tax reduction, roughly a fifth of the $1,155 the deduction actually delivers at the 12% marginal rate." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It recognized the overtime relief and removed the $9,629 premium from income, the correct magnitude of adjustment, but priced the result with 2025 parameters — a $15,000 to $15,750 standard deduction and an $11,925 top of the 10% bracket — rather than the 2026 $16,100 deduction and $12,400 bracket top. It also submitted $1,569 after its final stated computation produced $1,700, so the answer matches none of its own derivations." -us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629.03 OBBBA qualified-overtime deduction and set the top of the 10% bracket at $12,150 instead of $12,400, computing $2,809.68 on taxable income of $25,439. It then submitted $1,808, a number no step in its reasoning produces." -us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It omitted the $9,629.03 qualified-overtime deduction and used a $15,600 standard deduction with the 2025 $11,925 bracket top instead of the 2026 $16,100 and $12,400, leaving taxable income of $25,939 against the correct $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset parameters — an $8,250 standard deduction, a $5,250 personal exemption and a 15% second bracket — all superseded for 2026 by the permanent $16,100 standard deduction, zero personal exemption and 12% bracket. It also removed the $5,789 employer health premium from wages, which does not reduce the stated employment income, and never applied the $9,629.03 overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It built taxable income from pre-TCJA 2026 parameters — an $8,250 standard deduction, a $5,300 personal exemption and a 15% bracket — when 2026 uses a $16,100 standard deduction, no personal exemption and a 12% second bracket. It also omitted the $9,629.03 OBBBA qualified-overtime deduction, so its $27,989 of taxable income overstates the correct $15,810.43 by $12,179." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,789 employer-sponsored insurance premium from wages, which leaves the stated employment income unchanged in the reference, and then applied sunset parameters of an $8,300 standard deduction, a $5,300 personal exemption and a 15% rate rather than 2026's $16,100 deduction, no exemption and 12% rate. It also omitted the $9,629.03 qualified-overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It deducted only the $386 401(k) deferral, skipping both the $18.03 traditional IRA deduction and the $9,629.03 qualified-overtime deduction, and used a $15,800 standard deduction instead of $16,100. Its own stated inputs — AGI $41,557 less $15,800 — give taxable income of $25,757 and tax of $2,843, not the $1,109 it submitted." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It removed the $5,789 employer health premium from wages, assumed a TCJA sunset with a personal exemption and 15% bracket, and omitted the $9,629.03 qualified-overtime deduction that carries taxable income down to $15,810.43. It then subtracted a $50 saver's credit, which is worth zero here because AGI of $41,539 exceeds the 2026 single AGI ceiling for the credit." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expiration and used an $8,500 standard deduction, a $5,150 personal exemption and a 15% second bracket, none of which apply in 2026 under the permanent $16,100 deduction, no exemption and 12% bracket. It also subtracted the $5,789 employer health premium from wages and omitted the $9,629.03 qualified-overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $2,301 is consistent with taxing wages against a standard deduction alone with no overtime relief. The correct 2026 path subtracts the $385.90 401(k) deferral, the $18.03 IRA deduction, the $16,100 standard deduction and the $9,629.03 OBBBA overtime deduction to reach taxable income of $15,810.43 and tax of $1,649.25." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $5,789 employer-sponsored insurance premium, which does not reduce the stated employment income, and used a $15,450 standard deduction with the 2025 $11,925 bracket top rather than $16,100 and $12,400. It never applied the $9,629.03 qualified-overtime deduction, so its taxable income of $20,300 exceeds the correct $15,810.43 even after the erroneous premium subtraction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,789 employer health premium from wages and omitted the $9,629.03 OBBBA qualified-overtime deduction, then reduced the result by a saver's credit that is worth zero at AGI $41,539, above the 2026 single AGI ceiling for that credit. The correct sequence keeps the full $41,943.38 of wages, removes only the $385.90 deferral and $18.03 IRA deduction, and takes the $16,100 standard deduction plus the $9,629.03 overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of $41,539 but applied 2025 parameters — a $15,000 standard deduction and an $11,925 top of the 10% bracket — instead of the 2026 $16,100 and $12,400. It also omitted the $9,629.03 qualified-overtime deduction, leaving taxable income of $26,539 against the correct $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used 2024-vintage parameters, a $14,600 standard deduction and an $11,600 top of the 10% bracket, instead of the 2026 $16,100 and $12,400. Layered on that, it omitted the $9,629.03 OBBBA qualified-overtime deduction, so its $26,939 of taxable income overstates the correct $15,810.43 by $11,129." -us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It correctly applied the $9,629 OBBBA overtime deduction along with the 401(k) and IRA adjustments, but used a $15,300 standard deduction instead of $16,100 and put the top of the 10% bracket at $12,200 instead of $12,400, inflating taxable income to $16,610 against the correct $15,810.43. It then subtracted a $50 saver's credit that is unavailable because AGI of $41,539 exceeds the 2026 single AGI ceiling for the credit." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction and unnamed credits fully offset $41,943 of wages, when the full deduction stack — $16,100 standard plus $9,629.03 overtime, against AGI of $41,539.45 — still leaves $15,810.43 of taxable income. No nonrefundable credit is available to a childless single filer at this AGI, so the $1,649.25 of bracket tax stands in full." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed taxable income was low enough that the standard deduction and unspecified credits zeroed out liability; the actual deduction stack of $25,729.03 against AGI of $41,539.45 leaves $15,810.43 taxable, which produces $1,649.25 at the 2026 10% and 12% brackets. It named no credit this childless 44-year-old single filer qualifies for, and none applies." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $12,400 top of the 10% bracket but a $15,750 standard deduction instead of $16,100, dropped the $385.90 traditional 401(k) deferral from its AGI computation, and omitted the $9,629.03 qualified-overtime deduction. Those three omissions put taxable income at $26,175 rather than $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied both the $16,100 standard deduction and the $9,629 overtime deduction correctly, but additionally subtracted the $5,789 employer-sponsored insurance premium from wages, which does not reduce the stated employment income, pushing taxable income down to $10,021 instead of $15,810.43. It also took a $50 saver's credit that is worth zero at AGI $41,539, above the 2026 single AGI ceiling for that credit." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave a one-line assertion with no parameters, and $2,894 is consistent with taxing roughly $25,400 to $26,500 of income, that is, applying the standard deduction and retirement adjustments while omitting the $9,629.03 OBBBA qualified-overtime deduction. With that deduction the taxable base is $15,810.43 and the tax is $1,649.25." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It built 2026 taxable income from sunset parameters — an $8,300 standard deduction, a $5,300 personal exemption and a 15% second bracket — when 2026 carries a $16,100 standard deduction, no personal exemption and a 12% second bracket. It also omitted the $9,629.03 qualified-overtime deduction, giving $27,939 of taxable income against the correct $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly applied the $9,629 qualified FLSA overtime deduction along with the 401(k) and IRA adjustments, but used a $15,400 standard deduction instead of $16,100 and a $12,250 top of the 10% bracket instead of $12,400. Those two parameter errors left taxable income at $16,510 rather than $15,810.43 and overstated tax by $87." -us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It invented reverted-bracket parameters of an $8,579 standard deduction and a $5,472 personal exemption with a 15% second bracket, when 2026 uses a $16,100 standard deduction, no exemption and a 12% bracket. It further subtracted the $5,789 employer health premium from wages and omitted the $9,629.03 qualified-overtime deduction." -us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It removed the $5,789 employer health premium from wages, which does not reduce the stated employment income, and omitted the $9,629.03 OBBBA qualified-overtime deduction, arriving at $19,650 of taxable income instead of $15,810.43. It then subtracted a $40.40 saver's credit that is worth zero because AGI of $41,539 exceeds the 2026 single AGI ceiling for that credit." -us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this variable, so nothing was submitted to score. The required computation reduces $41,943.38 of wages by the $385.90 401(k) deferral and $18.03 IRA deduction to AGI of $41,539.45, subtracts the $16,100 standard deduction and $9,629.03 overtime deduction, and taxes $15,810.43 at 10% to $12,400 plus 12% above for $1,649.25." -us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It stated that $41,943 of wages net of a $386 deferral falls below the standard deduction, when the 2026 single standard deduction is $16,100 and the total deduction stack including the $9,629.03 overtime deduction is $25,729.03, leaving $15,810.43 taxable. It also invoked a personal exemption, which 2026 law sets at zero." -us,scenario_037,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Every input was right — AGI of $41,539, the $16,100 standard deduction, the $12,400 top of the 10% bracket, and no nonrefundable credits — except that it omitted the OBBBA deduction for the $9,629.03 qualified FLSA overtime premium, which non-itemizers claim on top of the standard deduction. Subtracting it drops taxable income from its $25,439 to $15,810.43 and tax from $2,804.68 to $1,649.25." -us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted only the $386 401(k) contribution, dropping the $18.03 traditional IRA deduction, and used a $15,600 standard deduction with the 2025 $11,925 bracket top rather than $16,100 and $12,400. Most consequentially it noted the wages include FLSA overtime yet never claimed the $9,629.03 OBBBA overtime deduction, leaving taxable income at $25,957 instead of $15,810.43." -us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It zeroed out liability with a Child Tax Credit for a 44-year-old single filer with no dependents in the household, a credit that requires a qualifying child and is $0 here. Its pre-credit figure of $2,613 was also built without the $9,629.03 qualified-overtime deduction, which alone brings taxable income to $15,810.43 and tax to $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model said outright that the $9,629 FLSA overtime premium is not deductible, which ignores the 2026 OBBBA qualified-overtime deduction. It also subtracted the $5,789 ESI premiums from the stated wages, skipped the $18 IRA deduction, and used the 2024 $14,600 standard deduction and 2024 brackets. It then reported $1,646, a number unrelated to its own $2,308 calculation." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model got AGI ($41,539), the $16,100 standard deduction and the $12,400 10% bracket right, but it never subtracted the $9,629 qualified-overtime deduction. Taxable income should have been $15,810, not $25,439. It also reported $2,884 instead of the $2,805 its own work produced." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 OBBBA qualified-overtime deduction, so taxable income came out $9,629 too high ($25,439 instead of $15,810). It then pushed its own $2,805 result up to $3,160 on an unsupported 'recheck' of the brackets." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model identified the OBBBA overtime deduction but cut tax by only about $235. Deducting the full $9,629 at the 12% marginal rate lowers tax by $1,155, from $2,805 to $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the overtime exclusion correctly, but it used a 2025-based standard deduction ($15,000, then $15,750) and a 10% bracket of $11,925–$12,150 instead of the 2026 values of $16,100 and $12,400. It then reported $1,569, which does not match its own $1,700 result." +us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model never subtracted the $9,629 qualified-overtime deduction and used a 10% bracket of $12,150 instead of $12,400, which gave $2,809.68. It then reported $1,808 with no derivation. The correct taxable income is $15,810, and the tax is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used a $15,600 standard deduction and the 2025 10% bracket of $11,925 instead of the 2026 values of $16,100 and $12,400. That inflated taxable income to $25,939." +us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using an $8,250 standard deduction, a personal exemption and a 15% bracket. The OBBBA made the TCJA structure permanent, with a $16,100 standard deduction and a 12% bracket. The model also subtracted the $5,789 ESI premiums from wages that were already stated and left out the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used pre-TCJA 2026 parameters ($8,250 standard deduction, a $5,300 personal exemption and a 15% second bracket), but the OBBBA made the $16,100 standard deduction and 12% bracket permanent. It also left out the $9,629 qualified-overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model applied a TCJA-sunset regime ($8,000 standard deduction plus a $5,000 exemption, taxed at 15%) instead of the OBBBA-extended 2026 parameters ($16,100 standard deduction, 12% bracket). It also never took the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA had expired, using an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, instead of the $16,100 standard deduction and 12% bracket. It also subtracted the $5,789 ESI premiums from the stated wages and left out the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model left out both the $9,629 qualified-overtime deduction and the $18 IRA deduction. Its own stated inputs (AGI of $41,557 less a $15,800 standard deduction) produce about $2,850 of tax, not the $1,109 it reported." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset with personal exemptions, subtracted the $5,789 ESI premiums from the stated wages, and left out the $9,629 overtime deduction. It also claimed a $50 Saver's Credit, even though AGI of $41,539 is above the 2026 single eligibility limit." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used TCJA-sunset parameters ($8,500 standard deduction, a $5,150 exemption and a 15% bracket) instead of the permanent $16,100 standard deduction and 12% bracket. It also subtracted the ESI premiums from the stated wages and never took the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no working beyond 'standard deduction', and its $2,301 ignores the $9,629 qualified-overtime deduction. The figure matches subtracting ESI premiums and using older standard-deduction and bracket values. The correct taxable income is $41,539 − $16,100 − $9,629 = $15,810, and the tax is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from wages that were already stated, left out the $9,629 qualified-overtime deduction, and used a $15,450 standard deduction and the 2025 $11,925 bracket instead of the 2026 values of $16,100 and $12,400." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the ESI premiums and never took the $9,629 qualified-overtime deduction. It also subtracted a Saver's Credit, even though AGI of $41,539 is above the 2026 single limit, so no credit applies." +us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used the 2025 $15,000 standard deduction and $11,925 bracket instead of the 2026 values of $16,100 and $12,400. That produced taxable income of $26,539 instead of $15,810." +us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and used the 2024 $14,600 standard deduction and $11,600 10% bracket instead of the 2026 values of $16,100 and $12,400." +us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction and a $12,200 10% bracket instead of the 2026 values of $16,100 and $12,400. It also treated the overtime deduction as above-the-line, which lowered AGI to $31,910 and wrongly qualified the filer for a $50 Saver's Credit. The overtime deduction does not reduce AGI, so AGI stays at $41,539, above the Saver's Credit limit." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model said the standard deduction and credits wipe out the tax. In fact, $41,539 AGI less the $16,100 standard deduction and $9,629 overtime deduction leaves $15,810 of taxable income, and a childless single filer at this AGI has no nonrefundable credits, so tax is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as a credit and assumed unspecified credits wipe out the liability. Taxable income is still $15,810 after the $16,100 standard deduction and the $9,629 overtime deduction, and no nonrefundable credit applies, so tax is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction, did not exclude the $386 traditional 401(k) deferral, and used a $15,750 standard deduction instead of $16,100. That inflated taxable income to $26,175 instead of $15,810." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from gross wages that were already stated, which cut AGI to $35,750 instead of $41,539. It also claimed a $50 Saver's Credit, but that AGI is above the 2026 single Saver's Credit limit. Correct taxable income is $15,810, giving $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the $5,789 ESI premiums even though the stated gross wages are taxable wages, which understated AGI by $5,789. It then subtracted a $50 Saver's Credit that is unavailable at the correct $41,539 AGI." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied only the standard deduction and the 401(k)/IRA reductions and left out the $9,629 OBBBA qualified-overtime deduction. That left taxable income near $25,400 instead of $15,810." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a TCJA-sunset regime ($8,300 standard deduction, a $5,300 personal exemption and a 15% bracket) instead of the permanent 2026 values ($16,100 standard deduction and 12% bracket). It also left out the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied the overtime deduction correctly but used a $15,400 standard deduction and a $12,250 10% bracket instead of the 2026 values of $16,100 and $12,400. That overstated taxable income by $700." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model reached the correct $15,810 of taxable income, but it treated the overtime deduction as reducing AGI to $31,910 and claimed a $50 Saver's Credit. AGI is actually $41,539, which is above the 2026 single limit. It also used a $12,200 10% bracket instead of $12,400." +us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed 'reverted' pre-TCJA brackets with a personal exemption and a 15% rate, but the OBBBA made the $16,100 standard deduction and 12% bracket permanent. It also subtracted the ESI premiums from the stated wages and left out the $9,629 overtime deduction." +us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 ESI premiums from the stated wages, left out the $9,629 qualified-overtime deduction, and claimed a Saver's Credit that is unavailable at the correct AGI of $41,539." +us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to grade." +us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed $41,543 of taxable wages falls 'well below' the standard deduction and applied a personal exemption that no longer exists. The $16,100 standard deduction plus the $9,629 overtime deduction still leaves $15,810 of taxable income, so tax is $1,649.25." +us,scenario_037,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model got AGI, the $16,100 standard deduction, the brackets and the Saver's Credit exclusion right, but it left out the $9,629 OBBBA qualified-overtime deduction. That left taxable income at $25,439 instead of $15,810." +us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model left out the $9,629 qualified-overtime deduction and the $18 IRA deduction, and used a $15,600 standard deduction and the 2025 $11,925 bracket instead of the 2026 values of $16,100 and $12,400." +us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied a Child Tax Credit to a single filer with no children, which is impossible, and used it to wipe out the tax. It also left out the $9,629 overtime deduction. The correct liability, with no nonrefundable credits, is $1,649.25." us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract. us,scenario_037,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented dependent-child credit eligibility despite the household containing only one 44-year-old adult: refundable CTC is $0 because there is no qualifying child. It also applied a child-based EITC amount even though the filer has no qualifying children, and $41,943 exceeds the childless EITC income limit, so EITC is $0." us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated North Carolina’s adoption of Medicaid expansion as sufficient for eligibility and failed to compare the head’s MAGI income of 2.60 times FPL with the expansion-adult income limit. The head exceeds that limit and has no alternative Medicaid pathway, yielding medicaid_category NONE and ineligibility." -us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It derived the correct base and rates in its own reasoning — $41,943 × 6.2% = $2,600.47 plus $41,943 × 1.45% = $608.27, total $3,208.74 — and then discarded that result for an invented ""minor adjustment for exact wage calculations,"" submitting $3,211.64, $2.90 above its own sum. The submitted number corresponds to no rule: the correct derivation is 7.65% of unreduced gross wages, which is exactly what its explanation already computed. It additionally cited the stale $168,600 Social Security wage base instead of the 2026 base, though wages fall far below either cap." -us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $5,789 employer-sponsored insurance premium from gross wages to build a $36,154 FICA base, treating an employer-paid premium as a section 125 salary-reduction exclusion. Employee OASDI and HI apply to the full $41,943 of wages, so removing $5,789 from the base understated the tax by $442.89 (7.65% of the wrongly excluded premium)." -us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It stated that FICA is 7.65% of wages ""after subtracting pre-tax health insurance premiums"" and applied the rate to $41,943 − $5,789 = $36,154. The $5,789 is an employer-sponsored insurance premium input, not an employee cafeteria-plan salary reduction, so the payroll-tax base remains the full $41,943; the exclusion cost it $442.89 of tax." -us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced the FICA wage base to $36,154 by excluding the $5,789 employer-sponsored insurance premium, then applied 6.2% and 1.45% to that reduced figure ($2,241.55 + $524.23). Both rates attach to the full $41,943 of gross wages, which yields $2,600.49 + $608.18 = $3,208.67; the premium exclusion is the entire $442.89 shortfall." -us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It asserted that Social Security and Medicare taxes apply to wages net of pre-tax health insurance premiums and computed 6.2% and 1.45% on $36,154. Employer-sponsored insurance premiums listed as a household input do not reduce covered wages, so the correct base is the unreduced $41,943 and the correct total is $3,208.67." -us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied the combined 7.65% employee rate to $41,943 − $5,789 = $36,154, carving the employer-sponsored insurance premium out of the payroll-tax base as if it were an employee pre-tax deferral. The premium leaves FICA wages intact, so 7.65% runs on the full $41,943 and the answer is $3,208.67, $442.89 above what it submitted." -us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its reasoning reached the right components — 6.2% and 1.45% on $41,943, ""$2,601 + $608 = $3,209"" — and it correctly ruled out both the Additional Medicare Tax and any North Carolina employee payroll tax, then submitted $3,151, a $58 reduction its own text attributes to nothing. Ruling out the state tax leaves the total unchanged at $3,208.67; the submitted value is inconsistent with every step it wrote." -us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It correctly held that traditional 401(k) contributions and the FLSA overtime deduction leave payroll-tax wages untouched, then negated that by netting the $5,789 employer-sponsored insurance premium out of wages to reach a $36,154 base. The premium is not an employee section 125 salary reduction, so 7.65% applies to the full $41,943 for $3,208.67." -us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"It gave a one-clause justification with no components and submitted a flat $3,210, a figure consistent with taking 7.65% of $41,943 and rounding up to the nearest ten dollars. The exact computation is $2,600.49 of employee Social Security plus $608.18 of employee Medicare for $3,208.67; the model never carried the cent-level component amounts the answer requires." -us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It defined ""wages subject to payroll tax"" as $41,943 − $5,789 = $36,154, treating the employer-sponsored insurance premium as a pre-tax exclusion from FICA wages, then rounded each component to whole dollars ($2,242 + $524 = $2,766). Employee OASDI and HI attach to the entire $41,943, giving $2,600.49 + $608.18 = $3,208.67." -us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It set FICA wages at $36,154 by excluding $5,789 of employer-sponsored health premiums from $41,943 gross wages before applying 6.2% and 1.45%. That exclusion has no basis here — the premium input is employer-paid coverage cost, not an employee salary-reduction election — so the tax runs on the full $41,943 and totals $3,208.67." -us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key was absent from the submitted outputs object. The failure is a contract violation in the response rather than a substantive misapplication of the FICA rules." -us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"It used the correct unreduced base and rates and wrote the right components — 6.2% × $41,943 ≈ $2,601 and 1.45% × $41,943 ≈ $608 — which sum to $3,209, then declared and submitted $3,213. The $4 gap is a summation error inside its own explanation; carrying the components to cents gives $2,600.49 + $608.18 = $3,208.67." -us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used North Carolina's superseded $10,750 single standard deduction and a 4.5% rate (NC's 2024 rate) instead of the 2026 $12,750 deduction and 3.99% flat rate, computing ~$1,386, then discarded that figure and submitted $0 on the assertion that deductions and nonrefundable credits zero out the liability. North Carolina has no low-income exemption or nonrefundable credit available to a childless single filer, so $28,789.45 of taxable income produces $1,148.70 of tax." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Reached the correct $28,807 base and $1,149 result, then reversed itself by assuming the stated $41,943 gross wages were already net of the traditional 401(k) deferral, adding the $386 back and dropping the $18 deductible IRA. That inflated the base to $29,193, and 3.99% of it yields the submitted $1,165; the AGI in the reference is $41,539.45 precisely because both contributions are subtracted from the stated gross wage." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Built the correct $28,789 taxable base but applied a 4.0% rate rather than North Carolina's 2026 statutory 3.99% flat rate, producing $1,151.56, and then inflated that to $1,180 for 'traditional contribution effects on the state base' that were already captured in the AGI it computed. The correct product is 0.0399 × $28,789.45 = $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Derived the exact correct chain — AGI $41,539, less the $12,750 NC standard deduction, times 3.99% = $1,149 — then overwrote it with $883 for an unnamed 'decoupling/deduction treatment.' North Carolina applies no decoupling addback or extra deduction to a wage-only single filer with these facts; the base stays $28,789.45 and the tax stays $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Cycled through candidate parameters, landed on the exactly correct $41,539 − $12,750 = $28,789 at 3.99% = $1,148.68 twice, then submitted $803 after inventing a 'doubled' NC single standard deduction of $21,500. North Carolina's 2026 single standard deduction is $12,750, and no statute doubles it." -us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Constructed the correct $41,539 AGI and $28,789 taxable base but applied 4.25% — North Carolina's 2025 rate — instead of the 2026 rate of 3.99%, yielding $1,223.50, and then submitted $1,685 for unspecified 'rounding and estimated updates.' The submitted figure is 5.85% of the correct base and corresponds to no North Carolina rate." -us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used the correct $41,539 AGI and 3.99% rate but substituted a $13,750 single standard deduction for North Carolina's actual $12,750, understating taxable income by $1,000. That $1,000 excess deduction is the entire gap: 0.0399 × $1,000 = $39.90, exactly the difference between its $1,109 and the correct $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Started from a federal AGI of $35,750, which is $41,943 less the $5,789 employer-sponsored insurance premium plus the $386 401(k) and $18 IRA — treating the ESI premium as an above-the-line subtraction. The Section 125 exclusion is already reflected in the stated wage figure and is not deducted again, so AGI is $41,539.45, taxable income $28,789.45, and tax $1,148.70 rather than 3.99% × $23,000." -us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Applied the correct $41,539 AGI and 3.99% rate but used a fabricated $15,400 single standard deduction in place of North Carolina's $12,750, cutting taxable income to $26,139. The $2,650 of phantom deduction costs 0.0399 × $2,650 = $105.74, the entire shortfall between its $1,042.95 and the correct $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Compounded two base errors: it subtracted the $5,789 employer-sponsored insurance premium to get a $35,750 AGI, and used a $12,850 standard deduction instead of North Carolina's $12,750. Neither the ESI premium nor a $12,850 figure enters the NC computation; the correct chain is $41,539.45 − $12,750 = $28,789.45 at 3.99% = $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Applied an invented ~4.3% rate rather than North Carolina's 2026 flat 3.99%, and its $1,787.52 equals 4.3% of the full $41,539 AGI with no standard deduction subtracted at all. The $12,750 single standard deduction is mandatory before the flat rate, giving $28,789.45 taxable and $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Took federal AGI as $35,750, which subtracts the $5,789 employer-sponsored insurance premium from wages on top of the 401(k) and IRA contributions. That premium is already excluded from the reported wage amount and is not an AGI adjustment, so NC taxable income is $41,539.45 − $12,750 = $28,789.45, and 3.99% of it is $1,148.70, not 3.99% × $23,000 = $917.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Used the correct $12,750 deduction and 3.99% rate but a state AGI of $35,750, which removes the $5,789 employer-sponsored insurance premium from wages a second time. AGI is $41,539.45 after only the $386 traditional 401(k) and $18 traditional IRA, so taxable income is $28,789.45 and tax is $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Applied North Carolina's 2025 rate of 4.25% instead of the 2026 rate of 3.99%, and its $1,281 implies a deduction near $11,400 rather than the $12,750 single standard deduction. The correct computation is 0.0399 × ($41,539.45 − $12,750) = $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Set the NC starting AGI at $35,750, subtracting the $5,789 employer-sponsored insurance premium in addition to the $386 401(k) and $18 IRA contributions. Employee ESI premiums are already excluded from the stated wages and are not a further AGI reduction, so the base is $28,789.45 and the 3.99% tax is $1,148.70 rather than $917.70." -us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Used the right $12,750 deduction and 3.99% rate but an NC AGI of $35,750, implied by its $23,000 taxable income — the $5,789 employer-sponsored insurance premium subtracted from wages that already exclude it. Correct AGI is $41,539.45, taxable income $28,789.45, tax $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Computed the correct $41,539 AGI and used the correct 3.99% rate, then applied an invented $15,500 single standard deduction instead of North Carolina's $12,750. The $2,750 of extra deduction costs 0.0399 × $2,750 = $109.73, accounting for the whole gap between its $1,039 and the correct $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Cut AGI to $31,910 by subtracting the $9,629 FLSA overtime premium, but the federal overtime deduction is a below-the-line deduction that never reduces AGI and North Carolina does not conform to it, so NC AGI stays $41,539.45 and taxable income $28,789.45. Its arithmetic also misfires: 0.0399 × $19,160 is $764.48, not the $765.28 it submitted." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Supplied no derivation and submitted $1,268, which is 4.40% of the correct $28,789.45 base — a pre-2026 North Carolina rate rather than the 2026 flat 3.99%. Applying 3.99% to $41,539.45 AGI less the $12,750 single standard deduction gives $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that wages are low relative to North Carolina's standard deduction and credits, leaving nothing to tax. The $12,750 single standard deduction leaves $28,789.45 of taxable income, North Carolina's 3.99% flat rate applies from the first taxable dollar, and no nonrefundable credit is available to this childless single filer, so the tax is $1,148.70, not $0." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"Derived the computation exactly right — $41,943 less $386 traditional 401(k) and $18 traditional IRA, less the $12,750 single standard deduction, times 3.99% = $1,148.68 — then submitted $1,147. It rounded its own $1,148.68 down by $1.68 instead of reporting $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Arrived at $19,160 of NC taxable income, which requires subtracting the $9,629 FLSA overtime premium from AGI on top of the $12,750 standard deduction. The federal overtime deduction sits below the line and North Carolina does not conform to it, so taxable income is $28,789.45 and 3.99% of it is $1,148.70, not $764.48." -us,scenario_037,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Correctly ruled out the federal below-the-line overtime deduction, then still used a $35,750 starting AGI, which subtracts the $5,789 employer-sponsored insurance premium from wages that already exclude it. The only AGI adjustments are the $386 traditional 401(k) and $18 traditional IRA, giving $41,539.45 AGI, $28,789.45 taxable, and $1,148.70 of tax." -us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Gave no derivation and submitted $1,370, which is 4.76% of the correct $28,789.45 taxable base — North Carolina's 2023 rate of 4.75%, not the 2026 flat 3.99%. The 2026 computation is 0.0399 × ($41,539.45 − $12,750) = $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Started from a federal AGI of $31,910, which is $41,539 less the $9,629 FLSA overtime premium. That premium supports a below-the-line federal deduction that does not reduce AGI and to which North Carolina does not conform, so NC taxable income is $28,789.45 and the 3.99% tax is $1,148.70 rather than $764." -us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Made two errors that partly offset: it started from a $35,750 AGI, subtracting the $5,789 employer-sponsored insurance premium already excluded from the stated wages, and applied a 4% rate rather than North Carolina's 2026 flat 3.99%. The correct base is $41,539.45 − $12,750 = $28,789.45, taxed at 3.99% for $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used the correct $12,750 deduction and 3.99% rate on a wrong $35,750 AGI, which removes the $5,789 employer-sponsored insurance premium from wages that already exclude it. AGI is $41,539.45 after only the $386 401(k) and $18 IRA subtractions, giving $28,789.45 taxable and $1,148.70 of tax." -us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Explicitly subtracted the $5,789 pre-tax employer-sponsored insurance premium from wages when computing federal AGI, arriving at $35,750. The Section 125 exclusion is embedded in the reported wage figure and is not an additional AGI adjustment, so AGI is $41,539.45, NC taxable income $28,789.45, and the 3.99% tax $1,148.70 rather than $917.70." -us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed a 4.5% North Carolina rate (the 2024 rate, superseded by 3.99% for 2026) and that pre-tax adjustments drive taxable income near zero. The only pre-tax adjustments are $386 of traditional 401(k) and $18 of traditional IRA, totaling $404, so after the $12,750 standard deduction $28,789.45 of taxable income remains and the tax is $1,148.70, not $0." -us,scenario_037,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Derived the base and rate correctly in its explanation — $41,539 AGI less the $12,750 standard deduction at 3.99% = $1,148.68 — but submitted $1,223.53, which is exactly 4.25% of $28,789, North Carolina's 2025 rate. The submitted number applies the prior-year rate that its own explanation had rejected." -us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Applied 4.5%, North Carolina's 2024 rate, rather than the 2026 flat 3.99%, computing $1,296.32 on a base that also dropped the $18 deductible IRA, then submitted $1,773.56, which equals 4.27% of the undeducted $41,539 AGI with no standard deduction applied. The correct figure is 0.0399 × ($41,539.45 − $12,750) = $1,148.70." -us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Gave no derivation and submitted $1,234, which is 4.29% of the correct $28,789.45 taxable base — a pre-2026 North Carolina rate rather than the 2026 flat 3.99%. Applying 3.99% to $41,539.45 AGI less the $12,750 single standard deduction produces $1,148.70." +us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It used the correct $41,943 base and correct rates, but it got Medicare wrong: 1.45% × $41,943 is $608.17, not $608.27. It then added an unexplained 'minor adjustment' that raised its own $3,208.74 to $3,211.64, which no rule supports. The correct total is $2,600.49 + $608.18 = $3,208.67." +us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the $5,789 employer-sponsored insurance premium as a pre-tax deduction and cut FICA wages to $36,154. The facts list no cafeteria-plan salary reduction, so Social Security and Medicare apply to the full $41,943 of gross wages. Applying 7.65% to that base gives $3,208.67, not $2,765.78." +us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted the $5,789 employer-sponsored insurance premium from wages as a pre-tax health premium, leaving $36,154. The payroll-tax wage base is the full $41,943 of gross wages. Leaving out $5,789 of taxable wages understated the tax by about $442.89." +us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It excluded the $5,789 employer-sponsored insurance premium from FICA wages and applied 6.2% and 1.45% to $36,154. No employee salary-reduction election is listed, so both taxes apply to the full $41,943: Social Security $2,600.49 and Medicare $608.18." +us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages by the $5,789 employer-sponsored insurance premium, as if it were a Section 125 pre-tax deduction, and taxed only $36,154. Payroll tax applies to the full $41,943 of gross wages, which gives $3,208.67." +us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It computed FICA on gross wages minus 'pre-tax ESI premiums' ($36,154). The $5,789 premium is not a salary reduction in the stated facts, so the 7.65% combined rate applies to the full $41,943, giving $3,208.67." +us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It correctly found about $2,601 in Social Security and $608 in Medicare on the full $41,943, totaling about $3,209. It then lowered the total to $3,151 without any rule to justify it. No deduction or state payroll adjustment applies, so the answer is $3,208.67." +us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It correctly kept 401(k) contributions and the overtime deduction in FICA wages, but it wrongly subtracted the $5,789 employer-sponsored insurance premium to get $36,154. Nothing in the facts makes that premium a pre-tax payroll deduction, so the base is the full $41,943 and the tax is $3,208.67." +us,scenario_037,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It applied 7.65% to $36,154 after taking out the $5,789 employer-sponsored insurance premium, treating it as a cafeteria-plan exclusion that the facts do not include. Employee Social Security and Medicare apply to the full $41,943 of gross wages, which yields $3,208.67." +us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"It used the full-wage base and the standard 6.2% + 1.45% employee rates, but it rounded the result to $3,210 instead of reporting the exact $3,208.67. Its own base calls for $2,600.49 in Social Security plus $608.18 in Medicare." +us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $5,789 employer-sponsored insurance premium as pre-tax ESI, leaving $36,154 of FICA wages, and then rounded each component to whole dollars to get $2,766. The FICA base is the full $41,943 of gross wages, which gives $3,208.67." +us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It excluded the $5,789 employer-sponsored health premium from FICA wages as a pre-tax deduction, which the facts do not support, and taxed $36,154. Social Security at 6.2% and Medicare at 1.45% apply to the full $41,943, for a total of $3,208.67." +us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is 7.65% of the full $41,943 of wages, which is $3,208.67." +us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"It used the correct $41,943 base and correct rates, getting $2,601 in Social Security and $608 in Medicare. It then added these to $3,213 instead of $3,209. The exact total is $2,600.49 + $608.18 = $3,208.67." +us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model used the outdated $10,750 deduction and a 4.5% rate to reach $1,386. It then zeroed the liability by pointing to nonrefundable credits that do not exist, because no NC nonrefundable credit applies to this childless single filer. The correct result is $28,789.45 of taxable income × 3.99% = $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model first had nearly the right figure (about $1,149). It then reversed itself by assuming the stated gross wages already excluded the 401(k) deferral, so it taxed $29,193. The stated wages are gross, so the $386 401(k) and $18 IRA contributions must be subtracted to reach AGI of $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model had the correct $28,789 taxable income but applied a 4.0% rate instead of the 2026 NC rate of 3.99%. It then added an unexplained 'traditional contribution effects' adjustment to reach $1,180, when no such NC adjustment exists." +us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model computed the correct $28,789 × 3.99% ≈ $1,149. It then cut this to $883 by citing an NC 'decoupling/deduction treatment' that does not apply to this wage earner, since no further NC subtraction reduces the $28,789.45 of taxable income." +us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model reached the correct $28,789 × 3.99% = $1,148.68 at one point but kept switching standard deduction amounts. It finally settled on an invented $21,500 'doubled' deduction and a 401(k)-only AGI. The 2026 NC single standard deduction is $12,750." +us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 4.25% rate instead of the 2026 rate of 3.99% on the correct $28,789 base, which gave $1,223.5. It then raised this to $1,685 for unexplained 'rounding and updates'. The correct figure is $28,789.45 × 3.99% = $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $13,750 NC standard deduction instead of the 2026 single amount of $12,750. That understated taxable income by $1,000 ($27,789 instead of $28,789.45)." +us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI subtracts the $5,789 employer-sponsored insurance premiums from the stated gross wages. Those premiums do not reduce AGI here, so AGI is $41,539.45 (wages less only the 401(k) and IRA contributions)." +us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied an invented $15,400 NC standard deduction instead of the 2026 single amount of $12,750. That understated taxable income, which is $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,789 employer-sponsored insurance premiums from wages to get an AGI of $35,750, when the correct AGI is $41,539.45. It also used a $12,850 standard deduction instead of NC's $12,750." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $1,787.52 answer is about 4.3% of nearly the whole $41.5k AGI. The model left out the $12,750 NC standard deduction and used a rate higher than the 2026 flat rate of 3.99%." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from a federal AGI of $35,750, which subtracts the $5,789 employer-sponsored insurance premiums from gross wages. Those premiums do not reduce AGI, so NC taxable income is $41,539.45 − $12,750 = $28,789.45, not $23,000." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's $35,750 state AGI removes the $5,789 employer-sponsored insurance premiums from wages. Only the $386 401(k) and $18 IRA contributions reduce AGI, which makes taxable income $28,789.45 rather than $23,000." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model applied an outdated rate above the 2026 NC flat rate of 3.99%; it stated 4.25%. Its $1,281 answer equals the correct $28,789 base taxed at about 4.45%, when the correct result is $28,789.45 × 3.99% = $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $35,750 starting AGI subtracts the $5,789 employer-sponsored insurance premiums from gross wages. Those premiums do not reduce AGI here, so the NC base is $41,539.45 − $12,750 = $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $917.70 answer implies $23,000 of taxable income, which comes from a $35,750 AGI that subtracts the $5,789 employer-sponsored insurance premiums. The correct AGI is $41,539.45, which gives $28,789.45 of taxable income." +us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model had the correct AGI of $41,539 and the correct 3.99% rate, but used an invented $15,500 standard deduction instead of NC's 2026 single amount of $12,750." +us,scenario_037,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model subtracted the $9,629 overtime premium to get a $31,910 AGI. The federal overtime deduction is a below-the-line deduction that does not reduce the federal AGI NC starts from, so AGI stays at $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The correct derivation is $28,789.45 × 3.99% = $1,148.70. The $1,268 answer is about $119 higher, which matches applying an outdated rate of about 4.4% to the correct base instead of the 2026 rate of 3.99%." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model claimed NC deductions and credits wipe out the liability. But $41,539.45 of AGI far exceeds the $12,750 standard deduction, and NC has no nonrefundable credit for this childless filer, so tax is $28,789.45 × 3.99% = $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly calculated $1,148.68 and then misrounded it to $1,147. Rounding $1,148.68 to the nearest dollar gives $1,149, and the unrounded figure is $1,148.70." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $19,160 taxable income comes from a $31,910 AGI that subtracts the $9,629 overtime premium. The federal overtime deduction is below the line and does not reduce the AGI NC starts from, so taxable income is $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model correctly left out the overtime deduction but started from a $35,750 AGI that subtracts the $5,789 employer-sponsored insurance premiums. The correct AGI is $41,539.45, which gives $28,789.45 of taxable income." +us,scenario_037,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI removes the $5,789 employer-sponsored insurance premiums from gross wages. Only the $386 401(k) and $18 IRA contributions reduce AGI, so the correct base is $41,539.45 − $12,750 = $28,789.45." +us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The $1,370 answer equals $30,789 × 4.45%, which means the model used the outdated $10,750 standard deduction and a 4.45% rate. The 2026 values are $12,750 and 3.99%." +us,scenario_037,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model started from a $31,910 AGI that subtracts the $9,629 overtime premium. The federal overtime deduction is below the line and does not reduce the federal AGI NC starts from, which is $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model treated federal AGI as already reflecting the overtime deduction ($31,910). The overtime deduction is taken below the line, so AGI and NC's starting point stay at $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $35,750 AGI subtracts the $5,789 employer-sponsored insurance premiums, which do not reduce AGI; the correct AGI is $41,539.45. It also used 4% instead of the 2026 NC rate of 3.99%." +us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's $35,750 federal AGI removes the $5,789 employer-sponsored insurance premiums from gross wages. AGI is actually $41,539.45, which gives $28,789.45 of taxable income instead of $23,000." +us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the $5,789 employer-sponsored insurance premiums as a pre-tax reduction to wages. Those premiums do not reduce AGI here, so only the 401(k) and IRA contributions come out, and AGI is $41,539.45." +us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model claimed taxable income was near zero after the standard deduction, but $41,539.45 of AGI less the $12,750 deduction leaves $28,789.45 taxable. It also used a 4.5% rate instead of the 2026 rate of 3.99%." +us,scenario_037,state_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model's explanation correctly derives $28,789 × 3.99% = $1,148.68. The value it submitted, $1,223.53, equals $28,789 × 4.25%, so the number does not match its own derivation and uses an outdated rate." +us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of the 2026 rate of 3.99%, and it left out the $18 IRA deduction. It then replaced its own $1,296 result with an unexplained $1,773.56." +us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The $1,234 answer is $85 above the correct $1,148.70, which is the size of error produced by using the outdated $10,750 standard deduction ($30,789 × 3.99% ≈ $1,228) or a 4.25% rate. The correct values are the $12,750 deduction and 3.99% applied to $41,539.45 of AGI." us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model applied only a 200% FPL CHIP upper income limit and concluded that $22,993 for a household of four being under it, plus age 7 being under 19, established CHIP eligibility. It never ran the Medicaid screen first: at roughly 71% FPL the child falls under Louisiana's Medicaid threshold for ages 6–18 and qualifies under the OLDER_CHILD category, and CHIP covers only children who do not qualify for Medicaid." us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model tested child1 against a single ~250% FPL LaCHIP ceiling and treated ""well below threshold"" as qualifying, which inverts the CHIP test. Income far below that ceiling places the child under Louisiana's children's Medicaid limit for ages 6–18, making them Medicaid-eligible under the OLDER_CHILD category, and Medicaid eligibility bars CHIP." us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model refined the income figure (wages plus self-employment net of the SE tax adjustment) but used it only against the ~250% FPL upper CHIP limit, concluding that a dependent child inside that limit is CHIP-eligible. The missing step is the Medicaid floor: at roughly 71% FPL the 7-year-old qualifies for Louisiana Medicaid under the OLDER_CHILD category, which makes CHIP eligibility 0 because CHIP covers only children Medicaid does not." @@ -2961,42 +3296,46 @@ us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili us,scenario_038,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old within the modeled WIC child age category. Because eligibility ends at the fifth birthday, Child 2 fails the categorical age test." us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable and therefore failed the required submission contract. us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly derived zero taxable income, zero tax before credits, and zero nonrefundable credits used, but its submitted value of -$1,644 contradicts that derivation. It inserted an unsupported negative amount even though nonrefundable credits cannot reduce liability below zero and refundable credits are excluded from this output." -us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It computed the refundable CTC exactly right — 15% × ($21,896 − $2,500) = $2,909, below the two-child refundable cap — but set the 2026 two-child EITC maximum at $7,428 and then arbitrarily shaved the total to $10,148, implying an EITC of $7,239 that appears nowhere in its derivation. The 2026 indexed maximum for two qualifying children is $7,316, and earned income of $21,896 sits on the plateau between the end of the 40% phase-in and the joint phase-out start, so the full $7,316 applies." -us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It ran the 40% phase-in against a fabricated earned-income ceiling of $15,628 (yielding $6,251 instead of the 2026 two-child maximum of $7,316), then subtracted a $1,656 EITC phase-out even though earned income of $21,896 is roughly $8,600 below the 2026 joint phase-out start for two children. It also used a flat $3,400 refundable CTC instead of the binding 15%-of-earnings-over-$2,500 phase-in of $2,909.35, and its stated components ($6,251 + $3,400 − $1,656) sum to $7,995, not the $3,995 it submitted." -us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It set the 2026 two-child EITC maximum at $7,150 rather than the indexed $7,316, and then reported an ACTC of $2,383 after correctly writing min($2,909, $3,400) — its own computation yields $2,909.35. Correcting both components gives $7,316 + $2,909.35 = $10,225.35." -us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It described earned income of $21,896 as only 'near the plateau' and cut the EITC to roughly $5,500, when $21,896 is fully past the end of the 40% phase-in and far below the joint phase-out start, giving the full $7,316. It then computed the ACTC phase-in as $3,074 and halved it to $1,527 by misreading the $1,700-per-child refundable cap as a further limit; the correct phase-in on earned income of $21,896 is $2,909.35." -us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It put the 2026 plateau EITC at roughly $7,000 instead of the indexed two-child maximum of $7,316 and ran the ACTC phase-in on gross earnings of $22,993 rather than earned income net of the deductible half of self-employment tax ($21,896), producing $3,074 instead of $2,909.35. Its submitted $9,199 matches neither of its own stated components, which would total $10,074." -us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It derived the refundable CTC exactly right at 15% × ($21,896 − $2,500) = $2,909 and correctly found the household on the EITC plateau, but applied a stale two-child maximum of $6,604 (a pre-2024 figure) as its 2026 proxy instead of the indexed $7,316, understating the total by $712." -us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It estimated the 2026 two-child plateau EITC at about $6,900 rather than $7,316 and computed the ACTC phase-in as $2,896 off an earned income of $21,806 (92.35% of self-employment income rather than gross less the deductible half of SE tax), then discarded its own $9,796 total in favor of an unexplained $8,500 'adjustment' for a phase-in curve that does not apply on the plateau." -us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum at $7,430 instead of $7,316, and defined earned income as wages plus 92.35% of self-employment income ($21,805) rather than gross self-employment income less the deductible half of SE tax ($21,896), so its ACTC phase-in came to $2,895.80 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA additional child tax credit rules — a $1,000-per-child refundable cap with a $3,000 earned-income floor — capping the refundable CTC at $2,000, when the 2026 refundable cap is $1,700 per child and the binding limit is the 15% phase-in on earnings over $2,500, which yields $2,909.35. Its EITC of $7,354 also overshoots the indexed 2026 two-child maximum of $7,316." -us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed the ACTC phase-in with a $3,000 floor and then capped it at $1,000 per child ($2,000 total), applying repealed pre-TCJA parameters; for 2026 the floor is $2,500 and the refundable cap is $1,700 per child, so the phase-in of 15% × ($21,896 − $2,500) = $2,909.35 governs. Its EITC of $7,344 is $28 above the indexed 2026 two-child maximum of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly placed the household on the EITC plateau but used $7,300 rather than the indexed 2026 two-child maximum of $7,316, and limited the refundable CTC to $1,000 per child using a $3,000 earned-income floor — pre-TCJA parameters. For 2026 the refundable cap is $1,700 per child and the 15% phase-in over $2,500 gives $2,909.35." -us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It assigned an EITC of $4,100 to a household sitting squarely on the two-child plateau, where the 2026 maximum of $7,316 applies in full, and treated the entire $4,000 nonrefundable-eligible CTC as refundable, ignoring the 15%-of-earnings-over-$2,500 limit that caps the refundable portion at $2,909.35." -us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a rounded $7,300 EITC instead of the indexed 2026 two-child maximum of $7,316 and capped the refundable child credit at $2,000 total — the pre-TCJA $1,000-per-child limit — when the governing 2026 limit is the 15% phase-in on earnings above $2,500, which yields $2,909.35." -us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It capped the additional child tax credit at $1,000 per child ($2,000 total), a parameter repealed for 2026, where the refundable cap is $1,700 per child and the binding constraint is the 15% earned-income phase-in of $2,909.35. Its EITC of $7,250 is $66 below the indexed 2026 two-child maximum of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It named both the EITC and the ACTC but submitted $7,466 — a figure consistent with the two-child EITC alone (overstated by $150 against the 2026 maximum of $7,316) with the refundable CTC omitted entirely. The 15% phase-in on earned income of $21,896 over $2,500 adds $2,909.35, which its total does not contain." -us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It identified the right two components but gave no parameters; $9,753.44 falls $471.91 short of $7,316 + $2,909.35 and is consistent with pairing the correct $2,909.35 phase-in with a roughly $6,844 two-child EITC maximum, i.e. a pre-2024 indexed value rather than the 2026 figure of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It named the EITC and ACTC with no parameters; $10,185 is $40.35 short of $7,316 + $2,909.35 and is consistent with an approximated two-child EITC maximum near $7,276 rather than the indexed 2026 value of $7,316." -us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It projected a two-child EITC maximum of $7,326 ($10 above the indexed 2026 value of $7,316) and fixed the refundable CTC at $2,000 — the pre-TCJA $1,000-per-child cap. For 2026 the refundable cap is $1,700 per child, so the 15% phase-in on earned income of $21,896 above $2,500 governs at $2,909.35." -us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_refundable_credits, so the submission carries no substantive computation to evaluate. The required derivation is the 2026 two-child EITC plateau maximum of $7,316 plus the refundable CTC phase-in of 15% × ($21,896 − $2,500) = $2,909.35." -us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It rounded the 2026 two-child EITC maximum to 'about $7,300' instead of the indexed $7,316, and defined earned income as wages plus 92.35% of self-employment income ($21,805) rather than gross self-employment income less the deductible half of SE tax ($21,896), so its phase-in gave $2,896 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It named the EITC and refundable CTC but supplied no parameters or arithmetic; $8,724 is $1,501.35 below $7,316 + $2,909.35 and is consistent with a two-child EITC near $5,815, far under the 2026 plateau maximum of $7,316 that applies at earned income of $21,896." -us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted that low earnings and the household composition generate no positive refundable amount, inverting the structure of both credits: the EITC phases in at 40% of earned income and the refundable CTC at 15% of earnings above $2,500, so low earnings maximize rather than eliminate them. With $21,896 of earned income, two qualifying children under 17, and a married-joint return, the EITC is at its 2026 plateau maximum of $7,316 and the refundable CTC is $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It got the 2026 two-child EITC maximum exactly right at $7,316 but treated the refundable child credit as a flat $1,800 per child ($3,600), a per-child amount that does not exist for 2026 and that skips the binding limit entirely. The refundable CTC is capped by 15% × (earned income − $2,500) = 15% × ($21,896 − $2,500) = $2,909.35, well below the $1,700-per-child cap." -us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $8,231, the 2026 EITC maximum for three or more qualifying children, for a household with two children, whose 2026 maximum is $7,316. It also ran the refundable CTC phase-in on gross earnings of $22,993 rather than earned income net of the deductible half of self-employment tax ($21,896), giving $3,073.95 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It matched the 2026 two-child EITC maximum of $7,316 exactly but double-reduced self-employment earnings when computing earned income for the CTC phase-in, applying both the 92.35% net-earnings factor and the deduction for one-half of SE tax. Earned income here is $21,896 (gross $22,993 less the $1,097 SE-tax deduction), so the phase-in is $2,909.35 rather than the $2,731.32 its double reduction produced." -us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It read the instruction against inferring unlisted facts as a bar on computing credits from the listed wages and self-employment income, and concluded zero because tax before credits is zero. Both the EITC and the refundable CTC are payable with zero income tax liability: $21,896 of listed earned income and two qualifying children yield the $7,316 plateau EITC plus a $2,909.35 refundable CTC." -us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed a post-2025 reversion to a $1,000-per-child refundable cap, writing ACTC = min($2,000, 15% × ($21,805 − $2,500)) = $2,000. The 2026 refundable cap is $1,700 per child, so the 15% phase-in binds at $2,909.35 on earned income of $21,896; its uprated EITC of $7,312 is also $4 below the indexed maximum of $7,316." -us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied a repealed pre-TCJA ACTC structure — $1,000 per child with a $3,000 earned-income floor — capping the refundable CTC at $2,000. For 2026 the floor is $2,500 and the refundable cap is $1,700 per child, so the phase-in of 15% × ($21,896 − $2,500) = $2,909.35 controls; its $7,330 EITC also exceeds the indexed 2026 two-child maximum of $7,316." -us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It set the refundable CTC at $1,000 per child ($2,000) under an assumed post-TCJA reversion with a $3,000 floor, when 2026 uses a $2,500 floor, a $1,700-per-child refundable cap, and therefore a binding phase-in of $2,909.35. It also used gross earnings of $22,993 as earned income rather than $21,896 net of the SE-tax deduction, and put the EITC at $7,338 against the indexed maximum of $7,316." -us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 married-joint two-child EITC at about $7,422 rather than the indexed maximum of $7,316, and ran the 15% phase-in on an earned income near $21,693 rather than $21,896 (wages plus self-employment income less the $1,097 deductible half of SE tax), yielding $2,879 instead of $2,909.35." -us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_refundable_credits, so there is no substantive computation to evaluate. The required derivation is the 2026 two-child EITC plateau maximum of $7,316 plus a refundable CTC of 15% × ($21,896 − $2,500) = $2,909.35." -us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It identified the 2026 two-child EITC maximum of $7,316 correctly but ran the refundable CTC phase-in on gross earnings of $22,993 without subtracting the deduction for one-half of self-employment tax ($1,097). Earned income is $21,896, so the phase-in is 15% × $19,396 = $2,909.35, not the $3,073.95 it used." -us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It put the two-child EITC maximum at about $6,500 rather than the indexed 2026 value of $7,316, and set the refundable CTC at the flat $1,700-per-child cap of $3,400 without applying the 15%-of-earnings-over-$2,500 limit, which binds first at $2,909.35 on earned income of $21,896." -us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It computed the refundable CTC exactly right — 15% × ($21,897 − $2,500) = $2,910, below the $1,700-per-child cap — and correctly placed the household on the EITC plateau, but used a maximum of about $7,370 instead of the indexed 2026 two-child maximum of $7,316." -us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied a phase-out threshold of $21,370 — a single-filer figure from an earlier year — to a married-joint return whose 2026 two-child phase-out does not begin until roughly $30,470, so it manufactured a reduction on income that is squarely on the plateau, and it anchored on a stale $6,960 maximum instead of $7,316. Its submitted $6,868.75 is an EITC-only figure that drops the $2,909.35 refundable CTC it had itself computed." -us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed earned income of $22,993 exceeds the EITC phase-out threshold and cut the credit to $46, when the 2026 married-joint two-child phase-out starts near $30,470 and runs to roughly $64,000, leaving the household at the full $7,316 plateau maximum. It also fixed the refundable CTC at a flat $3,000 rather than the 15% phase-in on earned income of $21,896 over $2,500, which is $2,909.35." +us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909 but then cut its own $7,428 EITC maximum to an arbitrary $7,239, submitting $7,239 + $2,909 = $10,148. The 2026 two-child EITC plateau maximum is $7,316." +us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It gave only the 40% phase-in amount ($6,251), even though earnings of about $21,896 are on the plateau and earn the $7,316 maximum. It then subtracted an invented $1,656 EITC phase-out that does not apply below the MFJ phase-out start. It also granted a flat $3,400 ACTC without the 15% × (earned income − $2,500) limit, which yields $2,909." +us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly derived the ACTC phase-in amount of $2,909 (below the $3,400 cap) but then submitted an unexplained $2,383. It also used an EITC of $7,150 instead of the 2026 two-child plateau maximum of $7,316." +us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It placed the household 'near the plateau' and gave only a $5,500 EITC, although $21,896 of earnings is squarely on the plateau at $7,316. It computed an ACTC phase-in of about $3,074 below the $3,400 cap and then submitted $1,527 anyway. The correct ACTC is 15% × ($21,896 − $2,500) = $2,909." +us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It estimated the EITC at about $7,000 instead of the $7,316 2026 two-child plateau maximum. It based the ACTC on gross earnings of $22,993 rather than earnings net of half the SE tax, and its $9,199 total implies an ACTC of about $2,199 instead of $2,909." +us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It computed the ACTC correctly at $2,909 but used a stale two-child EITC maximum of $6,604 as a proxy for 2026. The 2026 plateau maximum is $7,316, so the EITC was $712 too low." +us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It projected the two-child EITC maximum at about $6,900 instead of $7,316, and it measured earned income as wages plus 92.35% of SE income ($21,806). It then cut its own $9,796 sum to an arbitrary $8,500 by citing a 'phase-in curve', although the household is on the EITC plateau." +us,scenario_038,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as $7,430 instead of $7,316. It also measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax (about $21,896), which put the ACTC at $2,895.80 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child CTC with a $3,000 threshold, capping the ACTC at $2,000. For 2026 the CTC is $2,200 with a $1,700 refundable cap, so the ACTC is 15% × (earned income − $2,500) = $2,909.35. It also set the EITC maximum at $7,354 instead of $7,316." +us,scenario_038,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It capped the ACTC at 2 × $1,000 = $2,000 using the reverted pre-TCJA credit and a $3,000 threshold. The 2026 law has a $1,700-per-child refundable cap and a $2,500 threshold, which gives $2,909.35. It also overstated the EITC maximum at $7,344 instead of $7,316." +us,scenario_038,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a two-child EITC maximum of $7,350 instead of $7,316. It also measured ACTC earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus half the SE tax, which gave $2,895.80 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated the refundable CTC as the pre-TCJA $1,000 per child with a $3,000 threshold, capping it at $2,000. The 2026 refundable cap is $1,700 per child, so the 15% × ($21,896 − $2,500) = $2,909.35 phase-in binds. It also used a $7,300 EITC instead of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It counted the full $4,000 CTC as refundable, ignoring the $1,700-per-child refundable cap and the 15% earned-income phase-in that limits the ACTC to $2,909.35. It also gave an EITC of only $4,100, although earnings of about $21,896 are on the two-child plateau at $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $2,000 CTC figure, which corresponds to the reverted $1,000-per-child credit. The 2026 refundable CTC is 15% × (earned income − $2,500) = $2,909.35, under a $3,400 cap. It also used a $7,300 EITC instead of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed an ACTC of $1,000 per child ($2,000) under the reverted pre-TCJA credit. The 2026 $1,700-per-child refundable cap makes the 15% earned-income phase-in ($2,909.35) binding. It also understated the EITC maximum at $7,250 instead of $7,316." +us,scenario_038,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Its $7,466 total is barely above the $7,316 EITC alone, which means it dropped nearly all of the $2,909.35 ACTC. That ACTC equals 15% of earned income above $2,500 for a family with two children and no income tax liability." +us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $9,753.44 is $472 below the correct $7,316 EITC plus $2,909.35 ACTC; with the ACTC at about $2,909, that implies an EITC of about $6,844. So it understated the EITC below the $7,316 plateau maximum that applies at $21,896 of earnings." +us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no breakdown. Its $10,185 is $40 below the correct $7,316 EITC plus $2,909.35 ACTC, which fits a slightly low 2026 two-child EITC maximum (about $7,276) paired with an otherwise correct ACTC phase-in." +us,scenario_038,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It capped the ACTC at $2,000 ($1,000 per child), as if the TCJA had lapsed. For 2026 the refundable cap is $1,700 per child, so the ACTC is the 15% × (earned income − $2,500) phase-in of $2,909.35. It also projected the EITC maximum at $7,326 instead of $7,316." +us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to grade." +us,scenario_038,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $7,300 EITC instead of the $7,316 2026 two-child maximum. It also measured earned income as wages plus 92.35% of SE income ($21,805) instead of SE income minus the deductible half of SE tax (about $21,896), which understated the ACTC as $2,896 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It gave no breakdown. Its $8,724 is $1,501 short of the correct $7,316 EITC plus $2,909.35 ACTC, so it understated one or both credits. The household is on the two-child EITC plateau, and its ACTC is the full 15% × (earned income − $2,500) phase-in amount." +us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It said a family with two children and $22,993 of earnings generates no refundable credits. That family qualifies for the maximum two-child EITC ($7,316) and a $2,909.35 refundable ACTC." +us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It had the EITC right at $7,316 but granted a flat $1,800-per-child refundable CTC ($3,600). It skipped the 15% × (earned income − $2,500) limit, which caps the ACTC at $2,909.35, below the $3,400 two-child refundable cap." +us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $8,231, the three-or-more-child EITC maximum, for a two-child family whose maximum is $7,316. It also computed the ACTC on gross earnings of $22,993 without subtracting the deductible half of SE tax, which gave $3,073.95 instead of $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $7,316, but its ACTC of $2,731.32 implies earned income of about $20,709. That figure comes from taking 92.35% of SE income and then also subtracting the half-SE-tax deduction, reducing SE earnings twice. The correct earned income is SE income minus half the SE tax plus wages (about $21,896), which gives $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It had the EITC right at $7,316 but granted the full $1,700-per-child refundable CTC ($3,400). It never applied the 15% × (earned income − $2,500) phase-in, which binds at $2,909.35." +us,scenario_038,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It computed the ACTC phase-in correctly (about $2,909), but it overstated the 2026 two-child EITC maximum as $7,427 instead of $7,316." +us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It misread the no-inference instruction as barring refundable credits and returned $0. The listed wages, SE income and two qualifying children directly produce a $7,316 EITC and a $2,909.35 refundable ACTC." +us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had lapsed and capped the ACTC at $2,000 ($1,000 per child). In 2026 the CTC is $2,200 with a $1,700-per-child refundable cap, so the 15% × ($21,896 − $2,500) = $2,909.35 phase-in governs. It also uprated a stale 2024 EITC maximum to $7,312 instead of $7,316." +us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a reverted $1,000-per-child ACTC with a $3,000 threshold, which gives $2,000. The 2026 refundable cap is $1,700 per child with a $2,500 threshold, so the ACTC is $2,909.35. It also projected the EITC maximum at $7,330 instead of $7,316." +us,scenario_038,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied a 'post-2025' CTC of $1,000 per child, as if the TCJA had expired, capping the refundable CTC at $2,000. The 2026 $1,700-per-child refundable cap lets the 15% earned-income phase-in ($2,909.35) bind. It also set the EITC maximum at $7,332 instead of $7,316." +us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'post-TCJA reversion rules' ($1,000 per child, $3,000 threshold) to cap the ACTC at $2,000. For 2026 the ACTC is 15% × (earned income − $2,500) = $2,909.35 under a $3,400 cap. It also used gross earnings of $22,993 and an EITC maximum of $7,338 instead of $7,316." +us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It overstated the 2026 two-child EITC maximum as about $7,422 instead of $7,316. It also understated the ACTC at $2,879 instead of 15% × ($21,896 − $2,500) = $2,909.35." +us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_refundable_credits, so there is no substantive answer to grade." +us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $7,316 but computed the ACTC phase-in on gross earnings of $22,993. ACTC earned income counts SE income net of the deductible half of SE tax (about $21,896), which gives $2,909.35 instead of $3,073.95." +us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It set the plateau EITC at about $6,500 instead of the $7,316 2026 two-child maximum. It then granted the full $3,400 refundable CTC without the 15% × (earned income − $2,500) limit, which caps the ACTC at $2,909.35." +us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It computed the ACTC correctly at about $2,910 but overstated the 2026 two-child EITC plateau maximum as $7,370 instead of $7,316." +us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It cycled through several parameter sets, including an invented $21,370 MFJ phase-out start that is below the true 2026 threshold. It then dropped its computed ACTC to $0 'since I'm uncertain' and submitted an arbitrary $6,868.75. The correct result is the $7,316 plateau EITC plus a $2,909.35 ACTC." +us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed $22,993 of earnings exceeds the two-child EITC phase-out and gave only $46. MFJ earnings of about $21,896 are below the 2026 phase-out start, so the full $7,316 plateau credit applies. It also set the ACTC at an unsupported $3,000 instead of 15% × ($21,896 − $2,500) = $2,909.35." us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted nonqualification without applying either qualifying pathway. PolicyEngine derives categorical eligibility from the household's $7,286.94 annual SNAP benefit and independently derives income eligibility because the school-meal FPG ratio is 0.70, below 1.30." us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly computed gross income below 130% FPG but then incorrectly imposed SNAP receipt or direct certification as an additional requirement and treated an unlisted SNAP input as proof of no SNAP participation. Income at or below 130% FPG independently qualifies the children for free meals, and PolicyEngine also computes $7,286.94 of SNAP under the assumed-take-up instruction, which supplies categorical eligibility and produces positive meal support." us,scenario_038,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -3021,45 +3360,44 @@ us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,Fals us,scenario_038,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model explicitly settled for an approximation of the 15.3% SECA rate on net earnings and returned a whole-dollar $2,192, which corresponds to a base of $14,326.80 rather than the exact 92.35% base of $14,337.34 on $15,525. Carrying the same formula to the cent gives $2,193.61; the $1.61 shortfall is rounding the base down instead of computing it." us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for self_employment_tax, so the submission carries no substantive position on the computation. The required output — 15.3% × 0.9235 × $15,525 = $2,193.61 — was simply absent from the response contract." us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model computed the correct answer — $15,525 × 0.9235 = $14,337.34, × 0.153 = $2,193.61 — then subtracted a second adjustment for the $1,096.81 one-half-of-SE-tax deduction, claiming it feeds back into the SECA base through an iterative computation. The §164(f) deduction reduces adjusted gross income for income tax purposes only and never re-enters the self-employment tax base; the 92.35% factor is the sole adjustment for the employer-share equivalent, so applying it twice drove the answer $139.53 below the correct $2,193.61." -us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its own worked chain was essentially the reference chain — $1,916 gross, $383 earned-income deduction, ~$212 standard deduction, $994-ish allotment, ~$603/month, ~$7,236/year — but it submitted $4,200, a figure no step in its text produces. $4,200 corresponds to $350/month, which would require net monthly income of $2,147, more than the household's entire gross income of $1,916.04." -us,scenario_038,snap,claude-fable-5.1,llm_error,period_annualization,False,"It reproduced the reference month exactly — $1,916.08 gross, $383.22 earned-income deduction, $223 standard deduction, $1,309.86 net, $994 allotment less 30% = $601.30 — then flat-multiplied by 12 to get $7,212. PolicyEngine computes SNAP month by month, so the October 1, 2026 FY2027 uprating of the four-person maximum allotment and standard deduction raises the last quarter of calendar 2026 and adds the missing $75." -us,scenario_038,snap,claude-haiku-4.5,llm_error,other,False,"It manufactured a ~$400/month shelter deduction out of the $75,000 mortgage balance even though no mortgage payment, property tax, insurance, or utility cost is listed (shelter costs are $0 here, so no excess shelter deduction exists), and paired it with a $1,044 allotment to reach $7,772. It then discarded that entire calculation for an unexplained '$84/month,' submitting $1,008 against the correct $601.30/month." -us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It mixed annual and monthly frames: after taking the 20% earned-income deduction on annual earnings ($18,394.40) it subtracted the monthly $224 standard deduction once from that annual figure instead of $2,676 for the year, producing net monthly income of $1,514 rather than $1,309.83. It then abandoned even that result's $540/month benefit and submitted $9,114, which implies $759.50/month and a net income of $781 — impossible given $1,916.04 of gross earnings." -us,scenario_038,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It named every correct ingredient — $1,916/month gross, the 20% earned-income deduction, a ~$217 standard deduction, and the $994 four-person maximum allotment — but never carried out the subtraction, jumping straight to '~$7,848.' That is $654/month, which implies net income of $1,133 rather than the $1,309.83 its own stated deduction stack yields." -us,scenario_038,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It skipped the net-income computation entirely and placed the benefit 'near the middle of the range' at $564/month. That implies net monthly income of $1,433, $123 above the actual $1,309.83, meaning it never applied the full $223 FY2026 four-person standard deduction on top of the $383.21 earned-income deduction." -us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its methodology was correct — full counting of the $15,525 self-employment income, 20% earned-income deduction, no medical deduction (no elderly or disabled member), no shelter deduction (a mortgage balance is not a housing payment) — but it deliberately reverted to FY2025 parameters, using a $975 maximum allotment and a $204 standard deduction instead of the FY2026 $994 and $223. It even computed $7,298 with FY2026-style figures and then discarded that result for $6,912." -us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It applied a 20% reduction to the $15,525 self-employment income before computing gross income, cutting gross to $19,888/year ($1,657/month) instead of the full $22,993 ($1,916.04/month), and then applied the 20% earned-income deduction a second time. It compounded this with a nonexistent dependent-care and medical deduction (child-care costs are $0 and the medical deduction is limited to elderly or disabled members), driving net income to $900–1,000 instead of $1,309.83." -us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a gross income of $21,896 instead of the actual $22,993 ($7,468 wages plus $15,525 self-employment) and an annual maximum allotment of $13,164 — $1,097/month — instead of the FY2026 four-person $994. The inflated allotment alone adds roughly $1,236 to the annual figure." -us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"Its income and deduction work was nearly right ($1,916 gross, $208 standard deduction, $1,325 net), but it used a $1,050 four-person maximum allotment instead of the FY2026 $994. That $56/month overstatement is the entire $672 annual gap." -us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied FY2025 parameters to tax year 2026: a $204 standard deduction instead of $223 and a $975 maximum allotment instead of $994. With the correct FY2026 values its own arithmetic ($1,916.08 gross, $383.22 earned-income deduction, 30% of net) yields $601.30/month rather than $576.34." -us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It invented a '$310/month dependent deduction' that has no counterpart in SNAP — the dependent care deduction requires actual paid child-care expenses, which are $0 for this household — pushing net income down to $1,025 instead of $1,309.83. It compounded that with the FY2024 $973 maximum allotment rather than the FY2026 $994." -us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It submitted a round $9,000 with no computation at all. That is $750/month, implying net monthly income of $813 against the actual $1,309.83 — a phantom $497/month of deductions beyond the $383.21 earned-income and $223 standard deductions that are the only ones this household qualifies for." -us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It asserted the correct structure (gross, net, and asset tests passed; allotment minus 30% of net) but produced $573.17/month against the correct $601.30. That shortfall matches use of the stale FY2025 $975 maximum allotment together with a standard deduction below the FY2026 four-person $223." -us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a ~$990 maximum allotment and a standard deduction near $193 (net income $1,333 instead of $1,309.83), where FY2026 sets the four-person allotment at $994 and the standard deduction at $223. Those two parameter slips cut the monthly benefit from $601.30 to $590." -us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It declared zero benefit despite the household clearing every test: gross income is 72% of the poverty guideline, net income 49%, countable assets are $500, and TANF non-cash assistance confers categorical eligibility. The $994 maximum allotment less 30% of $1,309.83 net income is $601.30 — positive, not zero." -us,scenario_038,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It affirmed eligibility but produced $523/month, which implies net monthly income of $1,570 rather than $1,309.83. That corresponds to omitting the $223 four-person standard deduction and part of the 20% earned-income deduction from the net-income calculation." -us,scenario_038,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation and submitted $9,708, or $809/month, which implies net monthly income of $617 against the actual $1,309.83. Reaching $617 would require roughly $1,300/month of deductions this household does not have — there is no shelter deduction (no rent, mortgage payment, or utility cost listed), no dependent care deduction, and no medical deduction absent an elderly or disabled member." -us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the right deduction stack (20% earned-income plus the four-person standard deduction) but used the FY2025 $975 maximum allotment instead of the FY2026 $994. With correct net income of $1,309.83 that yields $582/month rather than $601.30." -us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It overstated the FY2026 four-person maximum allotment as $1,020 instead of $994 and used a $220 standard deduction instead of $223. The $26/month allotment error accounts for essentially the whole $227 annual overshoot." -us,scenario_038,snap,glm-5.3,llm_error,period_annualization,False,"It landed within a dollar of the reference month — $995 allotment, $220 standard deduction, net ~$1,310, benefit ~$600 — then rounded to $600 and flat-multiplied by 12 for $7,200. PolicyEngine computes SNAP monthly, so the FY2027 uprating effective October 1, 2026 lifts the last three months of calendar year 2026 above the flat $601.30 baseline." -us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It concluded SNAP is not positive for this household, but gross income of $1,916.04/month is only 72% of the poverty guideline, well under the 130% gross test, net income of $1,309.83 is 49%, countable assets are $500, and TANF non-cash receipt confers categorical eligibility. The benefit formula $994 − 30% × $1,309.83 gives $601.30/month." -us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted a zero benefit without applying any test. The household passes the gross (72% of the poverty guideline), net (49%), and $500 asset tests and is categorically eligible through TANF non-cash assistance, and $994 less 30% of $1,309.83 net income is a positive $601.30/month." -us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"Its income and deduction work was close (net ~$1,316 against $1,309.83), but it used an $879 four-person maximum allotment — below every recent fiscal year's value and $115 under the FY2026 $994. That single parameter error explains the entire $1,400 annual shortfall." -us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It named the correct formula (maximum allotment less 30% of net income after the earned-income and standard deductions) but produced $609.74/month against $601.30. That implies net income of $1,280.87, $28.96 below the actual $1,309.83 — a standard deduction near $252 rather than the FY2026 four-person $223." -us,scenario_038,snap,gpt-5.6-sol,llm_error,period_annualization,False,"It computed the reference monthly benefit exactly — $601 after the 20% earned-income and four-person standard deductions, with no shelter costs — then multiplied by a flat 12 for $7,212. PolicyEngine evaluates SNAP month by month, so the October 1, 2026 FY2027 uprating of the allotment and standard deduction raises the final quarter of calendar 2026 above that flat rate." -us,scenario_038,snap,gpt-5.6-terra,llm_error,period_annualization,False,"It reached the correct $601/month using the $994 allotment less 30% of net income after the earned-income and standard deductions, then annualized at a flat 12 months. The last three months of calendar 2026 fall in FY2027, whose uprated four-person allotment and standard deduction lift the monthly benefit and add roughly $75 to the year." -us,scenario_038,snap,gpt-6-astra,llm_error,period_annualization,False,"It used every FY2026 parameter correctly — $994 maximum allotment, $223 monthly standard deduction, 20% earned-income deduction on $22,993, annual net countable income of $15,718 — and computed $11,928 less 30% of that, i.e. a flat twelve months at $601.04. Calendar year 2026 straddles the October 1, 2026 FY2027 uprating, so the last quarter carries a higher allotment and standard deduction than the flat annualization assumes." -us,scenario_038,snap,grok-4.3,llm_error,other,False,"It returned 0 by invoking the prompt's no-inference rule, but that rule governs unlisted household facts (income, rent, assets), not statutory program parameters, which the benchmark expects the model to supply. Every input needed was listed: $22,993 of earnings, $500 in assets, and a four-person Louisiana household yield $994 − 30% × $1,309.83 = $601.30/month." -us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It rounded the FY2026 four-person maximum allotment to $1,000 and the standard deduction to $225 instead of the actual $994 and $223, giving $608/month. It then submitted $7,280 rather than the $7,296 its own $608 monthly figure annualizes to." -us,scenario_038,snap,grok-4.6,llm_error,period_annualization,False,"It got the deduction stack exactly right ($223 standard deduction, 20% earned-income deduction, no countable shelter costs, net ~$1,310) and used a $993 allotment, then rounded the monthly benefit to $600 and multiplied by a flat 12. PolicyEngine computes SNAP monthly, and the FY2027 uprating effective October 1, 2026 raises the allotment for the last three months of the tax year." -us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"Working in annual terms, it used a standard deduction of $3,985/year — $332/month against the FY2026 four-person $223 — and a maximum allotment of $12,337/year, $1,028/month against $994. Those two inflated parameters together add about $728 to the annual benefit." -us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"It rounded the FY2026 four-person maximum allotment up to $1,000 from $994 and computed net income as $1,316 rather than $1,309.83, yielding $605/month instead of $601.30. It then annualized flat at 12 months, missing the October 2026 FY2027 uprating as well." -us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $1,020 four-person maximum allotment instead of the FY2026 $994 and a $208 standard deduction instead of $223, giving net income of $1,325 and a benefit of $622/month. The $26/month allotment overstatement drives nearly the entire $177 annual overshoot." -us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It matched the reference deduction stack exactly — $1,916.08 gross, 20% earned-income deduction, $223 four-person standard deduction, net $1,309.87 — but used a $997 maximum allotment rather than the FY2026 $994, and then annualized flat at 12 months. The $3 allotment error plus the omitted October 2026 FY2027 uprating account for the gap." -us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"Its net income was close ($1,329 against $1,309.83, reflecting a standard deduction slightly under $223), but it used a $1,050 four-person maximum allotment instead of the FY2026 $994. That $56/month overstatement produces the bulk of the $513 annual overshoot." -us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It used the correct $994 FY2026 four-person maximum allotment but a $207 standard deduction instead of $223, leaving net income at $1,325.83 rather than $1,309.83 and the benefit at $596.25/month instead of $601.30. It also annualized flat at 12 months, omitting the October 2026 FY2027 uprating." -us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After computing $572.54/month from a $973 FY2024 allotment and a $198 standard deduction, it discarded that result and submitted $879/month on the strength of an unquantified 'shelter deduction and other adjustments.' No shelter deduction exists here — no rent, mortgage payment, property tax, or utility cost is listed, and the $75,000 mortgage balance is not a housing expense — so net income stays at $1,309.83 and the benefit at $601.30/month." -us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It drove net income to $0 with a phantom shelter deduction and paid the household the full maximum allotment, which it also set at $964/month rather than the FY2026 four-person $994. Shelter costs are $0 for this household, so the only deductions are the $383.21 earned-income and $223 standard deductions, leaving net income of $1,309.83 and requiring a $392.95 monthly contribution against the allotment." +us,scenario_038,snap,claude-fable-5,llm_error,other,False,"Its reasoning worked out about $7,236 a year: a $1,000 guessed maximum minus 30% of $1,321 net, using a $212 standard deduction. It then submitted $4,200, which its own work does not support. Even the computed figure used the wrong FY2026 parameters; the correct ones are a $223 standard deduction and a $994 maximum, giving $601 a month." +us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never applied the 20% earned income deduction and used a $194 standard deduction. It also invented a $400/month shelter deduction from the $75,000 mortgage balance, although no mortgage payment, property tax or utility cost is listed. It then dropped to an unexplained $84/month. The correct result is $1,916 − $383 − $223 = $1,310 net, and $994 − $393 = $601/month." +us,scenario_038,snap,claude-opus-4.7,llm_error,period_annualization,False,"It took 20% off annual earnings ($18,394) and then subtracted a monthly standard deduction (~$224) from that annual figure. That shrank the standard deduction to about $19 a month and gave net income of $1,514 instead of $1,310. It then abandoned its own $540/month result for an unsupported $759/month ($9,114), well above the correct $601/month." +us,scenario_038,snap,claude-opus-4.8,llm_error,other,False,"It named the right inputs ($1,916 gross, 20% earned income deduction, ~$217 standard deduction, $994 maximum) but never did the arithmetic. Those inputs give about $599/month. Its $7,848 ($654/month) implies a 30% contribution of only ~$340, which none of its stated deductions support. The correct result is $994 − $393 = $601/month." +us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"It described the right method (20% earned income deduction, standard deduction, 30% contribution) but produced $564/month without showing parameters, $37 below the correct $601. The shortfall comes from not using the FY2026 $994 maximum allotment and $223 standard deduction, which give net income of $1,310 and a $393 contribution." +us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the FY2025 figures of a $975 maximum allotment and a $204 standard deduction instead of FY2026's $994 and $223. That raised net income to $1,328.86 and gave $576/month. The FY2026 parameters give $1,310 net, a $393 contribution and $601/month." +us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It cut self-employment income by 20% before counting it, which understated gross income at $1,657/month. It then took the 20% earned income deduction a second time and added dependent care and medical deductions, even though there are no care costs and no elderly or disabled members. The resulting net of $900-$1,000 was also discarded in favour of an unsupported $6,800. The correct net income is $1,310, giving $601/month." +us,scenario_038,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It understated gross income at $21,896/year instead of $22,993. It also used an annual maximum allotment of $13,164 ($1,097/month) instead of $994 × 12 = $11,928, which inflated the benefit to $8,697. The correct figures are $1,310 net and $601/month." +us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $1,050 maximum allotment and a $208 standard deduction instead of FY2026's $994 and $223. It also left the contribution and benefit unrounded, giving $652.54/month instead of $994 − $393 = $601." +us,scenario_038,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 $975 maximum allotment and $204 standard deduction instead of FY2026's $994 and $223. That gave net income of $1,328.87 and a benefit of $576.34/month, below the correct $601." +us,scenario_038,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $213 standard deduction and a $1,000 maximum allotment instead of $223 and $994. It also skipped rounding net income to $1,310 and the contribution to $393, which produced $604.04/month instead of $601." +us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It made up a $310/month 'dependent deduction'. SNAP has no per-dependent deduction, and the dependent care deduction needs actual care costs, which are $0 here. It also used the 2024 $973 maximum and $198 standard deduction, which pushed net income down to $1,025 and the benefit up to $665.50/month instead of $601." +us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave a round $9,000 ($750/month) without doing any calculation. The correct steps are $1,916.04 − $383.21 earned income deduction − $223 standard deduction = $1,310 net, then $994 − $393 = $601/month. The $750 figure overstates that by $149 a month." +us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It stated the right formula (maximum allotment minus 30% of net income) but produced $573/month. That matches pre-FY2026 parameters (about $975 maximum and $204 standard deduction), not FY2026's $994 and $223, which give $601/month." +us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $1,333 net income means it used a standard deduction of about $200 instead of FY2026's $223. It also used a ~$990 maximum instead of $994, which gave $590/month instead of $994 − $393 = $601." +us,scenario_038,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It concluded the household's net income rules out any benefit. In fact, net income of $1,310 is only 48-49% of the poverty guideline, far under the 100% net income limit, and gross income is 70-72% of the guideline. The benefit is $994 − $393 = $601/month." +us,scenario_038,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It said the household passes all tests but gave $523/month. That matches a 30% contribution on about $1,507-$1,570 of net income, which is what you get by dropping the $223 standard deduction. The correct net of $1,310 gives a $393 contribution and $601/month." +us,scenario_038,snap,gemini-3.7-flash,llm_error,other,False,"It gave $809/month ($9,708) without any calculation. Against the $994 maximum, that implies net income of only about $617, roughly half the correct $1,310 ($1,916.04 − $383.21 − $223). No deduction beyond the earned income and standard deductions applies, so the correct benefit is $601/month." +us,scenario_038,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the right structure (20% earned income deduction plus standard deduction, then 30% of net off the maximum) but reached $580.50/month. That matches pre-FY2026 parameters instead of the $994 maximum and $223 standard deduction, which give $601/month." +us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"It estimated the standard deduction at $220 and the maximum allotment at $1,020 instead of FY2026's $223 and $994, and left the result unrounded. That gave $626.14/month instead of $994 − $393 = $601." +us,scenario_038,snap,glm-5.3,llm_error,other,False,"It reached the correct $1,310 net income but took 30% of it as $394 instead of $393 and used a $995 maximum. It then rounded the benefit to 'about $600/month' instead of computing $994 − $393 = $601, which left the annual figure $12 short at $7,200." +us,scenario_038,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It declared a zero benefit because of earnings and assets. But gross income is 70-72% of the poverty guideline, net income is 48-49%, and the $500 of assets is well under the limit; TANF non-cash categorical eligibility also applies. The formula gives $994 − 30% × $1,310 = $601/month." +us,scenario_038,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It asserted a zero benefit without applying any test. The household passes the gross, net and asset tests, with net income of $1,310 after the 20% earned income deduction and the $223 standard deduction. That gives $994 − $393 = $601/month." +us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"Its net income of about $1,316 was close to right, but it used a maximum allotment of $879 for four people instead of FY2026's $994. That cut the benefit to $484/month instead of $601." +us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It described the right formula but produced an unrounded $609.74/month. That implies an overstated maximum allotment or understated contribution compared with the FY2026 $994 maximum and $393 contribution (30% of $1,310). It also skipped SNAP's whole-dollar rounding." +us,scenario_038,snap,gpt-6-luna,llm_error,thresholds_rates,False,"It got $595/month, which implies a $399 contribution on net income of about $1,330. That is the result of using the FY2025 $204 standard deduction instead of FY2026's $223, which gives $1,310 net, a $393 contribution and $601/month." +us,scenario_038,snap,grok-4.3,llm_error,other,False,"It entered 0 because program parameters 'cannot be inferred', misreading the no-inference rule. That rule covers unlisted household facts, not statutory SNAP parameters. With the listed earnings, the FY2026 formula gives $601/month." +us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"It used a ~$225 standard deduction and a ~$1,000 maximum allotment instead of $223 and $994, which gave $608/month. It also mis-annualized: $608 × 12 is $7,296, not the $7,280 it submitted. The correct figure is $601/month." +us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction and reached $1,310 net. But it used a $993 maximum allotment instead of FY2026's $994, which gave $600/month instead of $994 − $393 = $601." +us,scenario_038,snap,grok-4.7,llm_error,thresholds_rates,False,"It used an $11,700 annual maximum allotment ($975/month, the FY2025 figure) and a $2,604 annual standard deduction ($217/month). FY2026's $994 maximum and $223 standard deduction are needed to get $1,310 net and $601/month." +us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used a ~$3,985 annual standard deduction ($332/month) and a ~$12,337 annual maximum allotment ($1,028/month). Both overstate FY2026's $223 standard deduction and $994 maximum and push the benefit to $8,014. The correct result is $601/month." +us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"Its net income of about $1,316 reflects a standard deduction of roughly $217 instead of $223. It also used a ~$1,000 maximum allotment instead of $994, which gave $605/month instead of $601." +us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $208 standard deduction and a $1,020 maximum allotment instead of FY2026's $223 and $994. That gave net income of $1,325 and $622/month instead of $1,310 net and $601/month." +us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly used the $223 standard deduction but used a $997 maximum allotment instead of $994. It also skipped rounding net income to $1,310 and the contribution to $393, which gave $604.04/month instead of $601." +us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"Its $1,329 net income reflects the FY2025 $204 standard deduction instead of $223, and it used a ~$1,050 maximum allotment instead of FY2026's $994. That inflated the benefit to $651/month instead of $601." +us,scenario_038,snap,ox-alpha,llm_error,thresholds_rates,False,"It used the correct $994 maximum but a ~$207 standard deduction instead of $223. That raised net income to $1,326 and the contribution to $398, giving about $596/month instead of $994 − $393 = $601." +us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It first computed $572.54/month using an outdated $198 standard deduction and $973 maximum. It then made up a shelter deduction to raise the benefit to $879/month, although no rent, mortgage payment or utility cost is listed, so no excess shelter deduction applies. The correct figures are $1,310 net and $601/month." +us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It set net income to $0 with a shelter deduction the household cannot claim, because no rent, mortgage payment or utility cost is listed. It also used a $964 maximum instead of $994. The standard and earned income deductions leave $1,310 of net income, so the $393 contribution applies and the benefit is $601/month, not the maximum." us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only the CHIP age and income conditions and skipped the prerequisite that the person not qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP." us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the question asking about eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP even though the spouse is under 19 and below the CHIP income limit." us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model concluded that satisfying the CHIP age and income ceiling was sufficient and never evaluated Medicaid eligibility first. The spouse is Medicaid-eligible under the OLDER_CHILD category, so the mutually exclusive CHIP pathway yields No." @@ -3072,147 +3410,170 @@ us,scenario_038,spouse_wic_eligible,claude-opus-4.7,llm_error,categorical_eligib us,scenario_038,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly recognized that the spouse lacked a qualifying categorical status, then contradicted that conclusion by treating low income and membership in a childbearing household as sufficient. Income or adjunctive income eligibility satisfies only the financial prong and does not create pregnancy, breastfeeding, postpartum, infant, or under-five status." us,scenario_038,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred that the spouse was pregnant, postpartum, or breastfeeding from her age and the presence of children, contrary to the instruction that unlisted statuses are false. The listed children are ages six and seven, and neither their presence nor low household income gives the spouse WIC categorical eligibility." us,scenario_038,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The answer treats being 18 and living with children as a WIC eligibility pathway. The spouse has no listed pregnancy, breastfeeding, or postpartum status, and the children are six and seven rather than under five, so the required categorical basis is absent." -us,scenario_038,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived zero Louisiana taxable income and zero tax after its $25,000 joint standard deduction, then submitted $218 anyway. Its numeric output contradicts its own completed computation." -us,scenario_038,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 joint standard deduction and thereby manufactured $13,696 of taxable income. The applicable Louisiana calculation for this joint filer with two dependents leaves no taxable income, so applying 1.85% to that artificial base produced the erroneous $253.38." -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model combined a $9,000 joint deduction with only $2,000 of dependent exemptions, leaving an artificial $10,696 tax base. Louisiana’s applicable joint-filer and dependent treatment reduces taxable income to zero before any rate is applied." -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $152 estimate implies that the model retained a positive Louisiana taxable-income base after exemptions. The joint-filer and two-dependent calculation instead leaves zero taxable income, so no lower-bracket tax arises." -us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction plus $4,000 of personal exemptions and incorrectly retained $8,896 of taxable income. The applicable Louisiana deductions and dependent treatment eliminate the taxable base for this household, making its 1.85% calculation inapplicable." -us,scenario_038,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied Louisiana’s 3% rate to an implied positive base of about $9,400 after an unspecified married deduction. The applicable joint-filer and two-dependent calculation leaves zero taxable income, so the 3% rate has no positive base to tax." -us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted a $4,500-per-person exemption scheme and a 1.75% lowest bracket, producing $3,896 of taxable income. Those parameters do not govern this 2026 Louisiana calculation; the applicable joint-filer treatment leaves the household below the taxable-income threshold." -us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than completing the Louisiana tax calculation. -us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an obsolete graduated 2% and 4% rate schedule together with an exemption-only taxable-income calculation. The applicable 2026 Louisiana joint-filer calculation leaves zero taxable income, so neither the $10,896 base nor the graduated-bracket tax applies." -us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Got the entire structure right — Louisiana's refundable EITC at 5% of the federal credit, earned income of ~$21,806 (wages $7,468 plus 92.35% of $15,525 in self-employment income) sitting on the two-child plateau — but plugged in $7,478 as the 2026 maximum two-child EITC when the correct figure is $7,316. The $162 overstatement of the federal ceiling propagates through the 5% match to produce $374.00 instead of $365.80." -us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Louisiana offers no refundable EITC and that the state's credits for this household are all nonrefundable; La. R.S. 47:297.8 grants a fully refundable state EITC equal to 5% of the federal credit. Because the household's federal EITC is $7,316 (two children, earned income on the plateau), the refundable state credit is $365.80, and the model's added observation that the household owes little Louisiana tax is irrelevant — a refundable credit is paid out regardless of liability." -us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Identified the correct 5% Louisiana refundable EITC match but computed the federal EITC as $3,680 for a joint filer with two children at ~$21,400 of earned income, roughly half the true amount. Earned income of ~$21,805 lies above the two-child phase-in completion point and below the joint phase-out threshold, so the federal credit is at its 2026 maximum of $7,316; the model treated the household as far down a phase-out that has not begun, halving the base and yielding $184 instead of $365.80." -us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Applied the correct 5% Louisiana refundable EITC rate and even computed combined earned income of ~$21,896 and noted the household is 'near the plateau,' then contradicted itself by using $2,640 as the federal EITC — about 36% of the plateau maximum it had just cited. The 2026 two-child maximum of $7,316 applies in full at this earned income, so the credit is $365.80, not $132." -us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Correctly ruled out the School Readiness Credit and applied Louisiana's 5% refundable EITC match, but used a rounded $7,000 federal EITC instead of the 2026 two-child maximum of $7,316 that applies on the plateau at ~$21,805 of earned income. The $316 shortfall in the base produces $350.00 rather than $365.80." -us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Claimed Louisiana 'eliminated its refundable EITC' and has no broadly applicable refundable individual income tax credit for 2026; the credit under La. R.S. 47:297.8 remains in force and was raised from 3.5% to 5% of the federal EITC for tax years beginning on or after January 1, 2019, surviving the flat-rate income tax reform. Applying the live 5% match to this household's $7,316 federal EITC gives $365.80." -us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Stated that Louisiana's EITC is nonrefundable; the statute makes it refundable, paid out in full even when Louisiana income tax liability is zero, which is exactly this household's situation with AGI of ~$21,894 against the joint standard deduction. The refundable 5% match on the $7,316 federal EITC is $365.80, not $0." -us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used Louisiana's repealed 3.5% EITC match rate, which was superseded by the 5% rate for tax years beginning on or after January 1, 2019, and compounded it with a $7,430 federal EITC base — a three-or-more-child ceiling figure rather than the 2026 two-child maximum of $7,316. Both errors cut the same direction: the correct computation is 5% × $7,316 = $365.80, not 3.5% × $7,430 = $260.05." -us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied the correct 5% Louisiana refundable EITC match to a federal EITC of $7,354, overstating the 2026 maximum two-child credit of $7,316 by $38. The structure and rate are right; only the federal maximum-credit parameter is off, producing $367.70 instead of $365.80." -us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Declared no applicable Louisiana refundable credit without engaging the state EITC, which La. R.S. 47:297.8 sets at 5% of the federal credit and makes refundable. With a federal EITC of $7,316 for this joint filer with two children, the state refundable credit is $365.80." -us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Asserted flatly that no state-level refundable credits apply in Louisiana, overlooking the refundable state EITC set at 5% of the federal credit. The household's $7,468 of wages plus $15,525 of self-employment income puts earned income on the two-child EITC plateau for a federal credit of $7,316, so the state match is $365.80." -us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly identified Louisiana's refundable EITC as 5% of the federal credit but used $7,250 as the federal EITC instead of the 2026 two-child maximum of $7,316 that applies on the plateau. The $66 shortfall in the base yields $362.50 rather than $365.80." -us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned zero with a bare assertion, missing Louisiana's refundable EITC equal to 5% of the federal credit. Earned income of ~$21,805 places this two-child joint filer at the federal maximum of $7,316, so the correct state refundable credit is $365.80." -us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Named the right mechanism — Louisiana's refundable earned income credit as a percentage of the federal EITC — but its $353.43 answer implies a federal base of $7,068.6 at the 5% rate, short of the 2026 two-child maximum of $7,316. The household is on the EITC plateau, so the full maximum applies and the credit is $365.80." -us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly identified the Louisiana refundable earned income credit but its $350 answer corresponds to a rounded $7,000 federal EITC at the 5% match rate rather than the actual 2026 two-child maximum of $7,316. Using the true federal credit gives $365.80." -us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Made two compounding errors: it called Louisiana's EITC nonrefundable when La. R.S. 47:297.8 makes it refundable, and it then conditioned the credit on positive state income tax liability, which is precisely what refundability overrides. The 5% match on the $7,316 federal EITC is paid out as a refund of $365.80 despite zero Louisiana tax." -us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Correctly ruled out the School Readiness Credit for lack of child care expenses but wrongly grouped the Louisiana EITC with it as nonrefundable and treated zero state tax liability as dispositive. The state EITC is refundable at 5% of the federal credit, so it pays $365.80 on this household's $7,316 federal EITC with no liability required." -us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded that no Louisiana refundable credit applies 'from the facts provided,' but the listed wages of $7,468 and self-employment income of $15,525 are exactly the facts that generate the federal EITC on which the state credit is based. That federal EITC is $7,316 for two qualifying children, and Louisiana's refundable 5% match makes the answer $365.80." -us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Claimed no positive Louisiana refundable credit is expected 'at this income level,' inverting the relationship: ~$21,805 of earned income sits on the two-child EITC plateau, the income range that produces the maximum federal credit of $7,316. Louisiana's refundable 5% match on that maximum is $365.80." -us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Treated the state refundable credit as requiring an explicitly listed credit input, concluding that 'qualifying refundable-credit inputs are absent.' Louisiana's refundable EITC is derived from earned income already in the facts — $7,468 in wages plus $15,525 in self-employment income yields a $7,316 federal EITC — and the statutory 5% match produces $365.80." -us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Applied the correct 5% Louisiana refundable match but used $8,231 as the federal EITC, which is the 2026 maximum for three or more qualifying children; this household has exactly two children, whose 2026 maximum is $7,316. The wrong child-count tier inflated the answer to $411.55 instead of $365.80." -us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Computed the federal EITC exactly right at $7,316 but applied Louisiana's superseded 3.5% match rate, which was raised to 5% for tax years beginning on or after January 1, 2019 and remains at 5% for 2026. The single rate error is the whole gap: 5% × $7,316 = $365.80, not $256.06." -us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Asserted that no Louisiana refundable credit is triggered by the listed facts, missing that the wage and self-employment income listed produce a federal EITC of $7,316 for this two-child joint filer. Louisiana matches that credit at a refundable 5%, giving $365.80." -us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Stated Louisiana has no state EITC; La. R.S. 47:297.8 provides a refundable state EITC equal to 5% of the federal credit, and it is the only Louisiana refundable credit that fires here. On a $7,316 federal EITC that produces $365.80." -us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Denied the existence of a Louisiana refundable EITC for this filing unit; the state credit exists, is refundable, and equals 5% of the federal EITC with no separate eligibility test beyond qualifying for the federal credit. This joint filer with two children and ~$21,805 of earned income has a $7,316 federal EITC, so the state credit is $365.80." -us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded there is 'no qualifying expenses or EITC match' in Louisiana, but the state does operate an EITC match — refundable at 5% of the federal credit. The household's federal EITC of $7,316 therefore generates $365.80 in state refundable credits, independent of any expense-based credit." -us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"Correctly applied Louisiana's refundable 5% EITC match but used $7,422 as the federal credit rather than the 2026 two-child maximum of $7,316, overstating the base by $106. That single parameter error produces $371 instead of $365.80." -us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_refundable_credits, so the failure is a missing submission rather than a substantive modeling error. The required derivation is Louisiana's refundable EITC at 5% of the household's $7,316 federal EITC, which equals $365.80." -us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Used the correct 5% Louisiana match on a federal EITC of $6,500, understating the 2026 two-child maximum of $7,316 by $816. Earned income of ~$21,805 sits on the plateau where the maximum applies in full, so the credit is $365.80, not $325." -us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Conditioned the answer on zero Louisiana taxable income, treating a nil liability as barring any credit; the Louisiana EITC is refundable, so it is paid as a cash refund precisely when liability is zero. Its correct amount here is 5% of the $7,316 federal EITC, or $365.80, and the School Readiness Credit it also ruled out is a separate credit." -us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Reasoned correctly that Louisiana's refundable EITC is 5% of the federal credit and that this household is on the plateau, then used $7,430 as the 2026 two-child maximum — a stale three-or-more-child ceiling — rather than the correct $7,316. The $114 overstatement of the federal maximum yields $371.50 instead of $365.80." -us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Claimed Louisiana provides no applicable refundable individual income tax credit and that no amount is 'indicated by the household facts.' The state EITC is refundable at 5% of the federal credit and is computed from the listed earnings, which give a federal EITC of $7,316 and a Louisiana refundable credit of $365.80." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It ran three contradictory computations and submitted the last: estate income of $25,950 was added to AGI, pushing provisional income to $66,572.50 and forcing Social Security to the 85% ceiling, then it switched to joint surviving-spouse provisional thresholds ($32,000/$44,000) and the $32,200 joint standard deduction. With estate income out of AGI, non-Social-Security income is $22,569.97, provisional income is $40,622.47, taxable benefits are $10,129.10, and the single $16,100 standard deduction plus the $3,319.81 QBI deduction leave $13,279.25 of taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in AGI, which raised provisional income to $66,572.50 and drove Social Security to the full 85% ($30,689.25) instead of the $10,129.10 the tiered worksheet yields at $40,622.47 of provisional income. It then applied surviving-spouse joint brackets and the $32,200 joint standard deduction rather than single status with the $16,100 standard deduction, and took no QBI deduction for the $3,319.81 limited by taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It built an $84,395 AGI that both included the $25,950 of estate income and took Social Security at the 85% cap, invented a $2,050 additional standard deduction for disability when the aged/blind add-on requires age 65 or blindness and this filer is 61 and sighted, then wiped the entire liability to zero with an 'elderly/disabled' nonrefundable credit that does not exist in the Code. The correct derivation gives $13,279.25 of taxable income and $1,345.51 of tax with no nonrefundable credit available." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It correctly rejected qualifying-surviving-spouse status but carried the $25,950 of estate income into a $75,209 AGI, which forced Social Security to the 85% cap of $30,689 instead of the $10,129.10 produced by $40,622.47 of provisional income. It also used a $15,750 standard deduction rather than the 2026 single amount of $16,100 and never applied the QBI deduction, which is $3,319.81 here under the 20%-of-taxable-income limitation." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It put the $25,950 of estate income into the $48,520 of pre-Social-Security income, lifting provisional income to $66,572.50 and making $25,186.63 of benefits taxable instead of $10,129.10, and then applied qualifying-widow(er) joint brackets with a $30,000 standard deduction rather than single status with $16,100. It also omitted the $3,319.81 QBI deduction, so its $43,707 taxable income overstates the correct $13,279.25 base." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added the $25,950 of estate income to reach a $79,209 AGI with Social Security at the 85% cap, when excluding estate income leaves AGI at $32,699.07 with only $10,129.10 of benefits taxable. It then guessed a roughly $16,700 standard deduction instead of the exact $16,100 and applied no QBI deduction, skipping the $3,319.81 write-off equal to 20% of the $16,599.07 taxable-income limit." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in the $48,520 pre-benefit income base, which triggered the joint 85% tier and made $25,186.63 of Social Security taxable rather than $10,129.10, and it applied qualifying-surviving-spouse joint rates with a $30,000 standard deduction instead of single status with $16,100. Its careful medical-itemization and Social-Security-worksheet work was applied to the wrong income base, and it never claimed the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income to reach a $79,209 total income with Social Security at the 85% cap, used joint surviving-spouse brackets with a $31,500 standard deduction rather than single with $16,100, and then adjusted its computed $5,248 down to $5,187 by fiat rather than by any rule. The correct base is $32,699.07 of AGI, $10,129.10 of taxable benefits, and $13,279.25 of taxable income after the $16,100 standard deduction and the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $25,950 of estate income to AGI, which put provisional income at $66,573 and forced the 85% Social Security ceiling of $30,689 in place of the $10,129.10 the tiers actually produce. It then used a $15,000 single standard deduction rather than the 2026 amount of $16,100 and took no QBI deduction, leaving $64,209 of taxable income against the correct $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income and the 85% Social Security cap in a $79,209 AGI, then applied a pre-TCJA regime — an $8,800 standard deduction and a 15% second bracket — when 2026 uses the $16,100 single standard deduction and the 10%/12% schedule with the 10% bracket ending at $12,400. It also omitted the $3,319.81 QBI deduction, so its $70,409 taxable income exceeds the correct $13,279.25 by more than fivefold." -us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income, which raised provisional income above the $34,000 upper tier and produced $30,689.25 of taxable Social Security instead of $10,129.10, and it used a $15,400 standard deduction rather than $16,100. Its $63,809.25 taxable income also omits the $3,319.81 QBI deduction that 20% of the $16,599.07 limit produces." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in a $79,209.25 AGI with Social Security at 85%, and then assumed TCJA expiration, using a combined $13,750 standard deduction plus personal exemption and 15%/25% brackets when 2026 law provides a $16,100 single standard deduction, no personal exemption, and 10%/12%/22% rates. Excluding estate income yields $10,129.10 of taxable benefits, $32,699.07 of AGI, and $13,279.25 of taxable income after the QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $5,354 exceeds the $3,675.89 that taxing the entire $32,699.07 AGI with no deduction at all would produce, so the figure necessarily includes the $25,950 of estate income in AGI and Social Security at the 85% ceiling. The correct path excludes estate income, taxes $10,129.10 of benefits, and applies the $16,100 standard deduction plus a $3,319.81 QBI deduction to reach $13,279.25 of taxable income and $1,345.51 of tax." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It states an AGI of $79,209, which is the $32,699.07 correct AGI plus the $25,950 of estate income and the extra $20,560.15 of Social Security that estate income pushes into taxability above the $10,129.10 the provisional tiers give. It also applied no QBI deduction, so its taxable income base is roughly five times the correct $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It built a $79,209 AGI containing the $25,950 of estate income and $30,689 of taxable Social Security, then subtracted a $13,400 combined standard deduction and personal exemption on a TCJA-sunset assumption; 2026 grants a $16,100 single standard deduction and no personal exemption. Correcting the AGI to $32,699.07 and adding the $3,319.81 QBI deduction leaves $13,279.25 of taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, and $6,233 exceeds the $3,675.89 that taxing the whole $32,699.07 AGI with zero deductions would produce, so the answer is built on an income base that includes the $25,950 of estate income and Social Security at the 85% ceiling. The correct base is $13,279.25 of taxable income after the $16,100 standard deduction and the $3,319.81 QBI deduction, taxed at 10% to $12,400 and 12% above." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in a $79,209.25 AGI and therefore took $30,689.25 of Social Security as taxable rather than the $10,129.10 that $40,622.47 of provisional income yields. It also used a $15,700 standard deduction instead of $16,100 and claimed no QBI deduction, leaving $63,509.25 of taxable income against the correct $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It states an AGI of about $79,209 that includes the $25,950 of estate income and $30,689 of taxable Social Security, both of which fall away once estate income is out of AGI: benefits are taxable only to $10,129.10 and AGI is $32,699.07. It also took no QBI deduction, omitting $3,319.81 of the $19,419.81 in total deductions." -us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its stated income build includes estate income alongside the IRA, pension, and taxable Social Security, which is the entire source of the gap: estate income sits outside AGI, so benefits are taxable only to $10,129.10 and AGI is $32,699.07. It also applied only a standard deduction, omitting the $3,319.81 QBI deduction that reduces taxable income to $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It ran the Social Security worksheet correctly but on an income base that wrongly included the $25,950 of estate income, giving $66,572.50 of provisional income and the 85% cap of $30,689.25 rather than the $10,129.10 that $40,622.47 of provisional income produces. It also used a $15,432 standard deduction instead of $16,100 and concluded no deductions beyond it applied, missing the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It correctly kept estate income out of AGI but then applied a flat 85% to Social Security ($30,691) instead of running the provisional-income tiers, which cap taxable benefits at $10,129.10, and it dropped the $6,260.03 self-employment loss from AGI entirely. It further subtracted a fabricated $1,920 of 'tax on previously-taxed Social Security,' a credit that does not exist, and omitted the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted without computation that taxable income is not positive after the standard deduction, but AGI is $32,699.07 and the $16,100 standard deduction plus the $3,319.81 QBI deduction leave $13,279.25 of taxable income. There is no 'elderly/surviving-spouse treatment' that exempts this filer: the age-65 additional standard deduction and senior deduction both require age 65, and the head is 61." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted zero on the unsupported claim that nonrefundable credits exceed the computed liability, when no nonrefundable credit applies to this childless, non-elderly filer with no care or education expenses. The actual computation is $32,699.07 of AGI less $19,419.81 of deductions, giving $13,279.25 of taxable income and $1,345.51 of tax." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It counted the $25,950 of estate income in the $48,520 of non-Social-Security income, which made 85% of benefits taxable and gave an AGI of $79,209 instead of $32,699.07 with $10,129.10 of taxable benefits. It then used surviving-spouse brackets and the $32,200 joint standard deduction rather than single status with $16,100, and claimed no QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in the $48,520 income base, so it computed $25,186.63 of taxable Social Security off joint thresholds instead of the $10,129.10 the single tiers give on $40,622.47 of provisional income. It also applied joint surviving-spouse rates and standard deduction rather than single with $16,100, and omitted the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied qualifying-surviving-spouse joint thresholds and the $32,200 joint standard deduction, and included the $25,950 of estate income, producing $25,187 of taxable Social Security and $73,707 of AGI rather than $10,129.10 and $32,699.07. It also took no QBI deduction, so its $41,507 taxable income overstates the correct $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It carried the $25,950 of estate income into a $79,209 AGI with Social Security at the 85% cap, then subtracted a joint standard deduction plus an 'age-61 additional deduction' and the senior deduction, both of which require age 65 and are unavailable to this 61-year-old. Excluding estate income gives $32,699.07 of AGI, and the $16,100 single standard deduction plus the $3,319.81 QBI deduction leave $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income, taking Social Security to the 85% ceiling of $30,689.25 and AGI to $79,209.25 instead of $10,129.10 and $32,699.07, and applied surviving-spouse status with the $32,200 joint standard deduction rather than single with $16,100. It also asserted no further deductions when the QBI deduction of $3,319.81 applies." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave a round $8,000 with no computation, and that figure sits above the $3,675.89 that taxing the full $32,699.07 AGI with no deduction would produce, so it is built on an income base including the $25,950 of estate income and the 85% Social Security ceiling. The correct derivation taxes $13,279.25 after the $16,100 standard deduction and $3,319.81 QBI deduction, giving $1,345.51." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income to reach a $79,209 AGI with Social Security at 85%, then applied a post-TCJA-sunset parameter set — an $8,255 standard deduction plus a $5,265 personal exemption and 15%/25% brackets — when 2026 provides a $16,100 single standard deduction, no personal exemption, and 10%/12%/22% rates. It also skipped the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It built a $79,209 AGI containing the $25,950 of estate income and $30,689 of taxable Social Security, then used an $8,300 standard deduction plus a $5,300 personal exemption under a TCJA-sunset assumption that 2026 law does not follow — the single standard deduction is $16,100 with no personal exemption. Removing estate income drops taxable benefits to $10,129.10 and, after the QBI deduction, taxable income to $13,279.25." -us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income in AGI, which forced Social Security to the 85% cap of $30,689 rather than the $10,129.10 the tiers give at $40,622.47 of provisional income, and used a $15,420 standard deduction instead of $16,100. It also declared no further deductions available, missing the $3,319.81 QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It got single status, the $16,100 standard deduction, and the 10%/12%/22% schedule right, so its entire error is including the $25,950 of estate income in AGI — which lifted provisional income to $66,572.50 and Social Security to the 85% cap of $30,689 instead of $10,129.10 — plus omitting the $3,319.81 QBI deduction. Those two corrections take its $63,109 taxable income down to $13,279.25 and its $8,656 tax to $1,345.51." -us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so no substantive computation was submitted for scoring. The required output is $1,345.51, from $13,279.25 of taxable income taxed at 10% to $12,400 plus 12% on the remaining $879.25." -us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly rejected qualifying-surviving-spouse status, used the $16,100 single standard deduction, and applied the right 2026 brackets, so its whole $7,250 gap comes from adding the $25,950 of estate income to AGI — which drove Social Security to the 85% ceiling of $30,689.25 rather than the $10,129.10 the provisional tiers produce — and from taking no QBI deduction. With estate income excluded, taxable income is $16,599.07 before the $3,319.81 QBI deduction and $13,279.25 after." -us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It double-counted Social Security, first adding the full $36,105 of gross benefits to gross income and then adding $30,689 of taxable benefits again to reach roughly $99,214, and it invented additional standard deductions of about $1,950 each for being 61 and for disability, neither of which exists under age 65 absent blindness. The correct build is $32,699.07 of AGI with $10,129.10 of taxable benefits, less $19,419.81 of deductions, for $13,279.25 of taxable income." -us,scenario_039,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It chose single status, the $16,100 standard deduction, and the correct 2026 brackets, so its error is entirely the $25,950 of estate income in AGI — which forced Social Security to the 85% cap of $30,689 instead of $10,129.10 — and the omitted $3,319.81 QBI deduction. Those corrections take taxable income from its $63,109 to $13,279.25 and the tax from $8,596 to $1,345.51." -us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After oscillating through several Social Security worksheets, it settled on a base that includes the $25,950 of estate income, giving $66,572.50 of provisional income, $25,196.63 of taxable benefits, and $73,716.63 of AGI instead of $40,622.47, $10,129.10, and $32,699.07. It compounded that with a $32,200 joint surviving-spouse standard deduction in place of the $16,100 single amount and took no QBI deduction." -us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It included the $25,950 of estate income and the 85% Social Security ceiling, halved the self-employment loss to -$3,130 when the full $6,260.03 is an above-the-line reduction, and claimed a $1,716 'credit for other dependents' derived from pension income, a credit that requires a qualifying dependent and has no relation to pension amounts. Its own figures also fail to reconcile — $9,260.15 less $1,716 is not the $4,694.15 submitted — against a correct base of $13,279.25 of taxable income." +us,scenario_038,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning used the right rules: a flat 3% rate, the $25,000 MFJ standard deduction and AGI of $21,896. It concluded that taxable income is $0 and wrote 'tax before refundable credits is $0', but then submitted 218. The submitted value contradicts its own correct derivation." +us,scenario_038,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It applied pre-reform Louisiana law: the old $9,000 MFJ combined exemption–standard deduction and the 1.85% lowest bracket. It never used the post-2025 $25,000 MFJ standard deduction and flat 3% rate. It also overstated AGI at $22,696 by not fully subtracting half of SE tax. Under the correct deduction, taxable income on AGI of $21,896 is zero." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used the repealed pre-2025 structure: the $9,000 MFJ exemption–standard deduction, $1,000 dependent exemptions and the 1.85% bottom rate. That left $10,696 of taxable income. The 2026 law gives a $25,000 MFJ standard deduction, which exceeds the roughly $21,896 AGI, so taxable income and tax are zero." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Its reasoning cites 'personal exemptions and lower bracket rates', which describes Louisiana's repealed graduated system. It ignored the 2025 reform's flat 3% rate and $25,000 MFJ standard deduction. That deduction wipes out the household's roughly $21,896 AGI and leaves $0 tax." +us,scenario_038,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It subtracted a $9,000 standard deduction plus $4,000 of personal exemptions from AGI of $21,896 and taxed the remaining $8,896 at the old 1.85% bracket. That is pre-reform law, with the exemptions double-counted on top. In 2026 Louisiana allows a $25,000 MFJ standard deduction at a flat 3%, so taxable income is zero." +us,scenario_038,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It correctly used the flat 3% rate, but $282 implies taxable income of about $9,400. That equals AGI of about $21,896 minus only the $12,500 single-filer deduction. It missed that the married-filing-jointly standard deduction is $25,000, which exceeds AGI and yields $0 tax." +us,scenario_038,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It called $12,500 the 'joint deduction' and subtracted $2,000 of dependent deductions to get $7,396.39 taxed at 3%. $12,500 is the single-filer amount; the MFJ standard deduction is $25,000. Applying $25,000 to AGI of $21,896 leaves no taxable income and $0 tax." +us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It invented a $4,500 per-person exemption for all four members, totaling $18,000, and a 1.75% 'post-2024' bottom rate. The post-2024 reform actually set a flat 3% rate with a $25,000 MFJ standard deduction. That deduction exceeds the $21,896 AGI, so tax is $0 rather than $68." +us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits. The correct derivation is AGI of about $21,896 minus Louisiana's $25,000 MFJ standard deduction, which gives zero taxable income and $0 tax." +us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used the pre-reform $4,500 per-spouse exemptions and $1,000 dependent exemptions, then taxed $10,896 at the pre-2022 2%/4% bracket rates with a wrong $10,000 bracket break. It never applied the 2025 reform's $25,000 MFJ standard deduction and flat 3% rate, which bring taxable income to zero on AGI of $21,896." +us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly placed the household at the federal EITC maximum and applied Louisiana's 5% rate. However, it used $7,478 as the 2026 two-child maximum instead of $7,316, which gave $374 rather than $365.80." +us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Louisiana has no refundable EITC and only nonrefundable credits. It missed Louisiana's refundable EITC of 5% of the federal EITC, which is 0.05 x $7,316 = $365.80 for this household." +us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model applied the correct 5% Louisiana rate but put the federal EITC at about $3,680. Joint filers with two children and about $21,900 of earned income are on the plateau below the MFJ phase-out, so the federal EITC is the full $7,316 and the LA credit is $365.80." +us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model said earned income of about $21,900 was near the plateau, then used a federal EITC of only $2,640, which contradicts its own placement. On the plateau the federal EITC is the full two-child maximum of $7,316, so 5% gives $365.80 rather than $132." +us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied Louisiana's 5% refundable EITC rate, but its $350 answer implies a federal EITC of about $7,000. The 2026 two-child maximum is $7,316, which gives $365.80." +us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model said Louisiana eliminated its refundable EITC. Louisiana still pays a refundable EITC of 5% of the federal credit, which is $365.80 on this household's $7,316 federal EITC." +us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Louisiana's EITC as nonrefundable. The credit is refundable at 5% of the federal EITC, so this household receives $365.80 regardless of its state tax liability." +us,scenario_038,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model identified the federal EITC correctly as $7,316 but applied a 50% state match instead of Louisiana's 5% rate. That multiplied the correct $365.80 by ten." +us,scenario_038,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied a 3.5% state rate instead of Louisiana's current 5%. It also used a federal EITC of $7,430 instead of the 2026 two-child maximum of $7,316. The correct result is 0.05 x $7,316 = $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but used $7,354 as the federal EITC instead of the 2026 two-child maximum of $7,316. That gave $367.70 instead of $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model said no Louisiana refundable credit applies and missed the refundable LA EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model said Louisiana eliminated its state EITC. Louisiana still pays a refundable EITC of 5% of the federal credit, which is $365.80 here." +us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said no Louisiana refundable credits apply, overlooking the refundable LA EITC. At 5% of the household's $7,316 federal EITC, that credit is $365.80." +us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the correct 5% Louisiana rate but used $7,250 as the federal EITC instead of the 2026 two-child maximum of $7,316. That gave $362.50 instead of $365.80." +us,scenario_038,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave Louisiana refundable credits as zero, omitting the refundable LA EITC. That credit is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model identified Louisiana's refundable EITC, but its $353.43 answer is 5% of about $7,069. That understates the 2026 two-child federal maximum of $7,316, which is where this household sits on the plateau." +us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model identified the Louisiana refundable EITC, but its $350 answer is 5% of a rounded federal EITC of $7,000. The actual 2026 two-child maximum is $7,316, which gives $365.80." +us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model treated Louisiana's EITC as nonrefundable and capped at zero tax liability. The LA EITC is refundable at 5% of the federal credit and pays $365.80 even with no state tax owed. +us,scenario_038,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model said the LA EITC is nonrefundable, so zero state liability left no refundable credit. The LA EITC is refundable at 5% of the $7,316 federal EITC, which is $365.80." +us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no Louisiana refundable credits and missed the refundable LA EITC. That credit applies automatically to any federal EITC recipient: 5% of $7,316 = $365.80." +us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed no Louisiana refundable credits at this income level. This low-income family with two children receives the maximum federal EITC of $7,316, which triggers Louisiana's refundable 5% EITC of $365.80." +us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model looked for listed credit inputs and found none. It missed that the refundable LA EITC comes directly from the federal EITC the household earns, at 5% of $7,316 = $365.80." +us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied the 5% Louisiana rate to a federal EITC of $8,231, which is the 2026 maximum for three or more children. The household has two children, so the federal EITC is $7,316 and the LA credit is $365.80." +us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the correct $7,316 federal EITC but applied an outdated 3.5% Louisiana rate instead of the current 5%. That gave $256.06 instead of $365.80." +us,scenario_038,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but put the federal EITC at $7,427 instead of the 2026 two-child maximum of $7,316. That overstated the LA credit as $371.35 instead of $365.80." +us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model said the listed facts trigger no Louisiana refundable credits. The refundable LA EITC follows automatically from the household's $7,316 federal EITC, at 5% = $365.80." +us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model said Louisiana has no state EITC. Louisiana pays a refundable EITC of 5% of the federal credit, which is $365.80 for this household." +us,scenario_038,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model said Louisiana has no refundable EITC. Louisiana's EITC is refundable at 5% of the federal EITC, which is $365.80 on the $7,316 federal credit." +us,scenario_038,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of $7,332 instead of the 2026 two-child maximum of $7,316. It also rounded to whole dollars, answering $367 instead of $365.80." +us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model found no applicable Louisiana refundable credit and missed the refundable LA EITC, which requires no qualifying expenses. That credit is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of about $7,422 instead of the 2026 two-child maximum of $7,316. That produced $371 instead of $365.80." +us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so no numeric answer was scored. The correct derivation is 5% of the $7,316 federal EITC, or $365.80." +us,scenario_038,state_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model applied the correct 5% rate but used a federal EITC of $6,500. That understates the 2026 two-child maximum of $7,316, which this household receives on the plateau." +us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tied Louisiana refundable credits to positive state taxable income or child-care activity. The LA EITC is refundable at 5% of the federal EITC regardless of state liability, which is $365.80 here." +us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly placed the household on the federal EITC plateau and applied the 5% rate. However, it used $7,430 as the 2026 two-child maximum instead of $7,316, which gave $371.50 instead of $365.80." +us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model said Louisiana offers no applicable refundable credit, omitting the refundable LA EITC. That credit is 5% of the household's $7,316 federal EITC, or $365.80." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Used qualifying-surviving-spouse status (joint $32k/$44k SS thresholds, $32,200 standard deduction, joint brackets), but QSS under IRC §2(a) requires a dependent child and none is listed, so the head files single. It also put the $25,950 estate income into AGI and provisional income, whereas the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and adds a $3,319.81 QBI deduction, taxing $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Applied the joint $32,200 standard deduction and joint brackets under surviving-spouse status even though, with no dependent child, the head files single ($16,100 deduction, 10% bracket to $12,400). It also included the $25,950 estate income, which made 85% of SS taxable; the reference keeps estate income out of AGI (taxable SS $10,129.10) and takes a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that an elderly/disabled credit and a disability add-on to the standard deduction wipe out the tax. Disability does not increase the standard deduction, and the §22 credit is fully phased out at this income. It also mis-added AGI ($84,395); the reference taxes $13,279.25 at single rates for $1,345.51." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Filed single but used a stale $15,750 standard deduction and made-up brackets (10% to $12,150, 12% to $49,350) instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, so 85% of SS became taxable. The reference excludes estate income (taxable SS $10,129.10, AGI $32,699.07) and subtracts $16,100 plus a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Treated the head as a qualifying surviving spouse, using joint $32k/$44k SS thresholds, a ~$30,000 standard deduction and joint brackets, though no dependent child is listed and the head files single. It also included estate income in provisional income and AGI; the reference excludes it, giving taxable SS $10,129.10, AGI $32,699.07, a $3,319.81 QBI deduction and taxable income $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Used single status but a ~$16,700 standard deduction instead of $16,100, and its $6,210 does not follow from its own ~$62,500 of taxable income (2026 single rates give about $8,460). It also included the $25,950 estate income that the reference leaves out of AGI (taxable SS $10,129.10, AGI $32,699.07, QBI deduction $3,319.81)." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used the joint $32,200 standard deduction and joint brackets for surviving-spouse status even though no dependent child exists, so single status and the $16,100 deduction apply. It also counted the $25,950 estate income, which pushed SS to $30,689.25 taxable; the reference excludes it (taxable SS $10,129.10) and adds a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed qualifying-surviving-spouse status (joint $44k SS threshold, ~$30,000 deduction, 2025 joint $23,850 bracket), but that status requires a dependent child, so the head files single with the $16,100 deduction and 2026 single brackets. It also included estate income in provisional income and AGI, which the reference excludes (AGI $32,699.07, QBI deduction $3,319.81)." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse status with a joint ~$31,500 deduction and 2025 joint brackets, though no dependent child is listed and single status applies. It then fudged the result to $5,187, and it included the $25,950 estate income that the reference keeps out of AGI (taxable SS $10,129.10, taxable income $13,279.25)." +us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied the joint $32,200 standard deduction and joint brackets under surviving-spouse status even though the head has no dependent child and files single. It also counted estate income, which made 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and deducts $16,100 plus $3,319.81 of QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used the pre-OBBBA 2025 single standard deduction of $15,000 and 2025 brackets ($11,925/$48,475) instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, which the reference excludes from AGI (taxable SS $10,129.10) while applying a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA had sunset, using an $8,800 standard deduction and a 10%/15% schedule (and never reaching the 25% bracket). OBBBA made the TCJA rates and the $16,100 single deduction permanent for 2026. It also included the estate income that the reference leaves out of AGI (AGI $32,699.07)." +us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Used a $15,400 single standard deduction and $12,250/$49,800 brackets instead of the 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income, making 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and subtracts a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Had the correct $16,100 deduction but wrong 2026 single bracket edges ($12,200/$49,600 instead of $12,400/$50,400). It also put the $25,950 estate income into AGI and provisional income; the reference excludes it, yielding taxable SS $10,129.10, a $3,319.81 QBI deduction and taxable income $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration law (a $13,750 standard deduction plus exemption and 10%/15%/25% brackets), but OBBBA made TCJA rates permanent, so 2026 uses the $16,100 deduction and 10%/12% brackets. It also included estate income that the reference excludes from AGI ($32,699.07)." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. Its $5,354 corresponds to taxable income of roughly $47,000–$49,000 at joint rates, which matches full inclusion of the estate income (85% of SS taxable) with a joint surviving-spouse deduction. The reference uses single status, excludes the estate income (AGI $32,699.07) and taxes $13,279.25 after the $16,100 deduction and a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"On its $79,209 AGI, $11,415 exceeds what 2026 single rates produce with the $16,100 deduction ($8,596). That places it on TCJA-sunset parameters (smaller deduction/exemption, 15%/25% brackets) rather than OBBBA's permanent schedule. It also included the estate income that the reference excludes from AGI ($32,699.07)." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly used post-TCJA-sunset parameters (a ~$13,400 standard deduction plus personal exemption), but OBBBA keeps the $16,100 single deduction and 10%/12%/22% brackets in 2026. It also included the $25,950 estate income that the reference leaves out of AGI and provisional income." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. $6,233 corresponds to about $52,400 of taxable income at 2026 single rates, meaning it counted the $25,950 estate income and 85% of SS. The reference excludes the estate income (taxable SS $10,129.10, AGI $32,699.07) and taxes only $13,279.25 after $16,100 standard and $3,319.81 QBI deductions." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used a $15,700 deduction instead of $16,100 and miscomputed tax on its own $63,509.25 of taxable income (2026 single rates give $8,684, not $7,117). It also included the estate income that the reference excludes from AGI (AGI $32,699.07, taxable income $13,279.25)." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"From AGI of $79,209, a $9,414 result implies a deduction of only about $12,400 instead of the $16,100 single standard deduction. It also counted the $25,950 estate income, which the reference excludes (taxable SS $10,129.10) while adding a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted estate income toward AGI, so SS hit the 85% cap. Its $7,578 corresponds to about $58,500 of taxable income, which does not match any correct 2026 single deduction. The reference excludes the estate income (AGI $32,699.07) and taxes $13,279.25 after the $16,100 and $3,319.81 QBI deductions." +us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Estimated a $15,432 standard deduction and $12,259/$49,838 brackets instead of the published 2026 values of $16,100 and $12,400/$50,400. It also included the $25,950 estate income that the reference excludes, which pushed taxable SS to $30,689.25 instead of $10,129.10." +us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Dropped the estate income but also ignored the $6,260 SE loss and taxed a flat 85% of SS without the provisional-income test, which on non-estate income gives $10,129.10. It then invented a $1,920 'tax on previously-taxed Social Security' credit and skipped the $3,319.81 QBI deduction, instead of taxing $13,279.25 at 10%/12%." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Asserted that taxable income is not positive after the standard deduction and elderly/surviving-spouse treatment, but no age-65 or surviving-spouse relief applies at age 61 without a dependent. Even the reference's AGI of $32,699.07 leaves $13,279.25 taxable after the $16,100 and $3,319.81 QBI deductions, which is taxed at $1,345.51." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Claimed that deductions and nonrefundable credits eliminate the liability, but no credits apply and AGI of $32,699.07 exceeds the $16,100 deduction plus the $3,319.81 QBI deduction. That leaves $13,279.25 taxable at 10%/12%, or $1,345.51." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse joint brackets and the $32,200 deduction, but with no dependent child the head files single ($16,100 deduction). It also included the estate income, making 85% of SS taxable; the reference excludes it (taxable SS $10,129.10, AGI $32,699.07) and adds a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Applied surviving-spouse/MFJ treatment (joint $32k/$44k SS thresholds, $32,200 deduction, joint brackets) without a dependent child, so single status applies. It also included the $25,950 estate income in provisional income and AGI; the reference excludes it and taxes $13,279.25 after a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used qualifying-surviving-spouse joint SS thresholds, the $32,200 deduction and joint brackets, though that status requires a dependent child and the head files single. It also counted estate income that the reference excludes from AGI (taxable SS $10,129.10, AGI $32,699.07, taxable income $13,279.25)." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse joint status and then added an age-based additional standard deduction and the OBBBA senior deduction, both of which require age 65, for a 61-year-old. It also included the estate income; the reference uses single status, excludes estate income (AGI $32,699.07) and deducts $16,100 plus $3,319.81 QBI." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Applied surviving-spouse joint brackets and the $32,200 deduction even though no dependent child is listed, so single status and the $16,100 deduction apply. It also counted the $25,950 estate income, which drove taxable SS to $30,689.25 instead of the reference's $10,129.10." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Correctly filed single with the 2026 $16,100 deduction and brackets, but included the $25,950 estate income in AGI and provisional income (85% of SS taxable, AGI $79,209), producing $8,596. The reference leaves estate income out of AGI (taxable SS $10,129.10, AGI $32,699.07) and grants a $3,319.81 QBI deduction, so it taxes $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used single status, the $16,100 deduction and 2026 single brackets, and included the $25,950 estate income so SS was taxable at the $30,689.25 cap, giving $8,596.04. The reference excludes estate income from AGI and provisional income (taxable SS $10,129.10, AGI $32,699.07) and subtracts a $3,319.81 QBI deduction, leaving $13,279.25 taxable." +us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used surviving-spouse status with the $32,200 joint deduction and joint brackets, though with no dependent child the head files single. It also included the $25,950 estate income that the reference excludes from AGI, where taxable SS is $10,129.10 and the QBI deduction is $3,319.81." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Gave a rounded $8,000 guess based on estate income, 85%-taxable SS and a medical-expense deduction. Medical expenses above the 7.5% floor fall far short of the $16,100 standard deduction, and the reference excludes estate income from AGI ($32,699.07) and applies a $3,319.81 QBI deduction, taxing $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Used TCJA-sunset parameters ($8,255 deduction plus $5,265 exemption, 10%/15%/25% brackets), but OBBBA made the $16,100 single deduction and 10%/12%/22% brackets permanent for 2026. It also included the $25,950 estate income, which the reference excludes from AGI ($32,699.07)." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA had sunset ($8,300 deduction plus $5,300 exemption, 15%/25% brackets) instead of applying OBBBA's permanent $16,100 single deduction and 10%/12% brackets. It also counted the estate income that the reference excludes (taxable SS $10,129.10, taxable income $13,279.25)." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset 2026 law ($8,500 deduction, $5,400 exemption, 15%/25% brackets), but OBBBA keeps the $16,100 single deduction and TCJA rate schedule. It also included the $25,950 estate income that the reference leaves out of AGI and provisional income." +us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Projected a $15,420 deduction and $12,259/$49,832 brackets instead of the published 2026 values of $16,100 and $12,400/$50,400. It also included estate income, so SS was 85% taxable; the reference excludes it (AGI $32,699.07) and adds a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Had the correct $16,100 deduction but wrong bracket amounts (about $1,225 in the 10% band and $4,509 in the 12% band instead of $1,240 and $4,560). It also included the $25,950 estate income, which the reference excludes from AGI (taxable SS $10,129.10) while adding a $3,319.81 QBI deduction." +us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $1,345.51 reference." +us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Correctly rejected QSS status and applied single status, the $16,100 deduction and 2026 brackets, but included the $25,950 estate income in AGI and provisional income, reaching $8,596.04. The reference omits estate income from AGI (taxable SS $10,129.10, AGI $32,699.07) and grants a $3,319.81 QBI deduction, taxing $13,279.25." +us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Double-counted Social Security by adding the full $36,105 benefit and then 85% of it again. It also invented extra standard deductions for being 61 and for disability; neither exists, and the $16,100 single deduction is the only one. The reference's AGI is $32,699.07, which excludes estate income and includes only $10,129.10 of SS." +us,scenario_039,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Correctly filed single with the $16,100 deduction and 2026 brackets but included the $25,950 estate income, which made SS taxable at $30,689 and AGI $79,209, for $8,596. The reference excludes estate income from AGI and provisional income (taxable SS $10,129.10) and subtracts a $3,319.81 QBI deduction, leaving $13,279.25 taxable." +us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Settled on qualifying-surviving-spouse treatment (joint $32k/$44k SS thresholds, $32,200 deduction, 2025 joint $23,850 bracket) without a dependent child, where single status applies. It also mis-multiplied the SS worksheet and included the estate income that the reference excludes from AGI ($32,699.07)." +us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Produced taxable income of $90,564, more than its own AGI, after subtracting a $31,900 deduction. It then invented a credit for other dependents funded by the pension and an AMT floor, none of which applies to a single filer with no dependents. The reference taxes $13,279.25 for $1,345.51 with no nonrefundable credits." us,scenario_039,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"Virginia's expansion adult group covers 19-64-year-olds only up to 138% FPL, yet the model declared that countable MAGI ""falls within eligible range for a household size of 1"" without ever summing the listed income: $36,105 of Social Security counts in full under MAGI rules, on top of $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, against which the $6,260 self-employment loss is trivial, putting the head at 368% FPL. Its correct observation that the expansion group applies no asset test is irrelevant once income blows through the 138% FPL ceiling, and the SSI-related aged/blind/disabled group it gestured at requires income near the SSI standard with SSI receipt at $0 here." us,scenario_039,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model named the correct rule — Virginia expanded to 138% FPL — then asserted the head's MAGI was below that threshold without computing it. Medicaid MAGI here is 368% FPL: full Social Security of $36,105, $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, less the $6,260 self-employment loss. With income at 2.7x the expansion limit and no SSI receipt to open the aged/blind/disabled route, the head qualifies through no Virginia category." us,scenario_039,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the ""is disabled"" flag as a standalone categorical qualifier and explicitly declined to weigh the income facts, stating no disqualifying income was provided when the prompt lists $36,105 Social Security, $26,800 IRA distributions, $2,030 pension, and $25,950 estate income. Every disability-linked Virginia pathway is income-tested — SSI-linked eligibility requires SSI receipt, which is $0, and the ABD and expansion groups cap income at or below 138% FPL against this head's 368% FPL — so disability determines which income test applies, not whether eligibility is automatic." -us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model asserted that ""disabled individuals receiving Social Security qualify for Medicare,"" conflating the head's listed Social Security retirement income with entitlement to disability insurance benefits. Medicare's under-65 pathway requires 24 months of DIB entitlement (or ESRD/ALS); SSDI is unlisted and therefore zero here, leaving only the age-65 test, which age 61 fails." -us,scenario_039,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model invented a ""PolicyEngine disability pathway"" keyed on the is_disabled flag plus Social Security receipt. PolicyEngine's Medicare eligibility turns on age ≥ 65, and the statutory under-65 route requires 24 months of Social Security disability insurance benefits — the head's Social Security is retirement income and SSDI is zero, so age 61 controls and the answer is not eligible." -us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated $36,105 of Social Security retirement income as evidence of disability-benefit entitlement and then declared that disability alone confers Medicare ""regardless of age."" Retirement benefits carry no Medicare entitlement before 65, and the disability route requires 24 months of DIB receipt, which is unlisted and therefore zero, so the head at 61 is not eligible." -us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model correctly named the 24-month SSDI condition parenthetically and then waived it, concluding eligibility from the bare is_disabled flag. Because SSDI entitlement is unlisted and set to zero by the prompt's default rule, the 24-month clock never starts and only the age-65 test applies, which age 61 fails." -us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model extended Medicare to ""disabled individuals who qualify"" without checking the qualifying condition, which is 24 months of entitlement to Social Security disability insurance benefits, ESRD, or ALS. None of those is listed, all default to zero, and the head is 61, below the age-65 threshold that governs the determination." -us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model claimed PolicyEngine treats the is_disabled input as Medicare-eligible under 65; PolicyEngine's Medicare eligibility keys on age ≥ 65, and is_disabled feeds SSI, the SNAP elderly/disabled deduction, and state disability provisions instead. With SSDI entitlement and ESRD unlisted and therefore zero, age 61 yields not eligible." -us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model stated the correct legal test — 24 months of Social Security disability benefits — then satisfied it with nothing more than the is_disabled flag, and additionally leaned on surviving-spouse status, which confers no Medicare entitlement. The head's Social Security is retirement income, disability-benefit receipt is unlisted and zero, so the age-65 test governs and age 61 fails it." -us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model asserted that age 61 plus the disabled flag ""meets requirements for Medicare eligibility"" with no rule cited. The under-65 requirement is 24 months of Social Security disability insurance entitlement, ESRD, or ALS — all unlisted and therefore zero — so the head is not eligible." -us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model treated ""is disabled"" as itself the qualifying disability condition under PolicyEngine's rules. The qualifying condition is 24 months of entitlement to Social Security disability insurance benefits (or ESRD/ALS), which the household facts do not include and the prompt zeroes out, leaving age 61 against a 65 threshold." -us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model explicitly ""assumed yes under the policy rule for disabled individuals,"" inferring an unlisted disability-benefit entitlement in direct violation of the instruction to treat unlisted inputs as zero. With SSDI and ESRD at zero, Medicare eligibility rests solely on age ≥ 65, and the head is 61." -us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model applied a ""disability eligibility rule"" that grants Medicare to any disabled person under 65; no such rule exists, since 42 U.S.C. §426(b) requires 24 months of disability insurance benefit entitlement. Those benefits are unlisted and zero — the listed Social Security is retirement income — so age 61 leaves the head ineligible." -us,scenario_039,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model routed eligibility through ""the stated disability"" alone, skipping the entitlement condition that actually opens the under-65 pathway: 24 months of Social Security disability insurance benefits, ESRD, or ALS. All are unlisted and therefore zero, so only the age-65 test applies and age 61 fails it." -us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model equated the is_disabled flag with Medicare entitlement in a single unqualified assertion. Medicare before 65 requires 24 months of Social Security disability insurance benefits, ESRD, or ALS; none is listed, each defaults to zero, and the head at 61 falls short of the age threshold." -us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model recited the rule as ""age 65 or older or disabled,"" which misstates the statute: the under-65 route requires 24 months of entitlement to disability insurance benefits, ESRD, or ALS, not a disability status flag. With those inputs unlisted and zeroed and the head's Social Security coded as retirement income, age 61 produces not eligible." -us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model wrote the correct counterfactual — ""after 24 months of SSDI the Head would qualify"" — and then treated the disabled flag as if it established that SSDI history. SSDI receipt is not among the household facts and is therefore zero, the listed Social Security is retirement income, and age 61 is below the 65 threshold that governs." +us,scenario_039,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"Treated the $36,105 of Social Security retirement income as if it were disability benefits. Medicare's under-65 pathway requires 24 months of SSDI entitlement. The head has no SSDI and no SSDI months (both count as 0), so the disability pathway fails. The head is under 65, so the age pathway fails too." +us,scenario_039,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"Claimed the head qualifies through the disability pathway because they are disabled and receive Social Security. That income is retirement income, not SSDI. The pathway requires at least 24 months of SSDI receipt, and the head has 0 months, so they are not eligible at 61." +us,scenario_039,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"Said disabled people are eligible for Medicare regardless of age, and read Social Security retirement income as proof of benefit eligibility. The under-65 pathway requires 24 months of Social Security disability receipt, or ESRD/ALS. The head has none of these, so a 61-year-old is not eligible." +us,scenario_039,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"Named the correct rule (Medicare after 24 months of SSDI) and then did not apply it. The head has no SSDI income and no SSDI months, so the 24-month requirement is not met, and the head is not eligible." +us,scenario_039,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"Treated disabled status alone as enough for Medicare before 65. The disability pathway requires 24 months of SSDI entitlement, and the head has none. At 61 with no ESRD or ALS, the head is not eligible." +us,scenario_039,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Assumed PolicyEngine gives Medicare eligibility to anyone flagged disabled. The under-65 rule instead requires at least 24 months of Social Security disability receipt, or ESRD/ALS. The head has 0 SSDI months and neither condition, so they are ineligible." +us,scenario_039,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Stated the 24-month SSDI requirement and then skipped it, going straight from 'is disabled' to eligible. The head receives retirement benefits, not SSDI, and has 0 SSDI months. Surviving-spouse status is not a Medicare pathway for someone under 65, so the head is not eligible." +us,scenario_039,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Assumed PolicyEngine treats disabled people as Medicare eligible. Eligibility under 65 actually requires 24 or more months of SSDI receipt, or ESRD/ALS. The head has 0 SSDI months, so a 61-year-old is not eligible." +us,scenario_039,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"Treated 'age 61 and disabled' as meeting the Medicare requirements. Nothing in the facts satisfies the under-65 pathway, which requires 24 months of SSDI entitlement (or ESRD/ALS). The head has no SSDI, so they are not eligible." +us,scenario_039,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"Took the disabled flag as a qualifying condition. Medicare does not look at disability status by itself; under 65 it requires 24 months of Social Security disability benefits or ESRD/ALS. The head has none of these, so they are ineligible." +us,scenario_039,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"Assumed coverage starts once someone is disabled. Medicare disability coverage starts only after 24 months of SSDI entitlement. The head has 0 months of SSDI receipt, so the head is not eligible at 61." +us,scenario_039,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"Applied a disability rule that treats the flag alone as enough. The real rule requires at least 24 months of SSDI receipt, and the head has 0 because only retirement Social Security is listed. The head is therefore not eligible under 65." +us,scenario_039,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Said the stated disability makes the head eligible, but the modeled rule depends on months of SSDI receipt (24 or more), not on disability status. With 0 SSDI months and age 61, the head is not eligible." +us,scenario_039,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"Concluded that disability gives Medicare before 65. That pathway requires 24 months of Social Security disability entitlement, and the head has no SSDI income or months. At 61 the head is not eligible." +us,scenario_039,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"Relied on a disability-based Medicare rule, but that rule is triggered by 24 or more months of SSDI receipt, not by the disabled flag. The head's SSDI months are 0, so they are not eligible at 61." +us,scenario_039,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"Went straight from 'disabled' to Medicare eligibility without checking the 24-month SSDI requirement. The head has no SSDI receipt and no ESRD/ALS, so at 61 they are not eligible." +us,scenario_039,head_medicare_eligible,grok-4.7,llm_error,age_disability,False,"Assumed PolicyEngine counts disability itself as a Medicare qualifier. PolicyEngine's under-65 test uses months of Social Security disability receipt (24 or more) or ESRD/ALS, all of which are 0 or false here. The head is not eligible." +us,scenario_039,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"Stated Medicare eligibility as 'age 65+ or disabled'. The disability prong actually requires 24 months of SSDI entitlement, and the head has 0 months. At 61 the head is ineligible." +us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"Named the 24-month SSDI rule and then treated the disabled flag as meeting it. No SSDI income or SSDI months are listed, so both are 0, and the head's Social Security is retirement income. The head is not eligible at 61." us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage earnings and applied the 7.65% employee Social Security and Medicare rate to an unstated payroll-tax base. The household has no listed wage or salary income, while its listed estate, retirement, IRA, pension, and negative self-employment income are not subject to employee payroll tax, yielding $0." us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract." us,scenario_039,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Carried the $25,950 of estate income into the state base, deriving $48,520 of non-Social-Security income where the reference AGI holds only the $26,800 IRA distribution and $2,030 pension net of the $6,260 self-employment loss, i.e. $22,570. It also omitted the $930 personal exemption and then raised its own bracket total from the $2,029 its arithmetic produced to the $2,192 it submitted, with no stated basis." -us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Set Virginia AGI at $48,520 by keeping the $25,950 of estate income in the base that the reference AGI of $32,699.07 excludes, and applied a $17,500 joint-style standard deduction where Virginia gives this filer the $8,750 single amount. Its bracket arithmetic on $30,090 was internally consistent, so the whole $958.18 error is those two inputs against the correct $12,889.97 of taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"Zeroed the liability with a nonexistent Virginia ""retirement income credit and disability income exclusion."" Virginia's retirement-related relief is the age deduction, which requires age 65 or older and is unavailable to this 61-year-old, and the §58.1-322.02(6) subtraction for income received for permanent and total disability, which does not reach IRA or pension distributions. Virginia AGI of $22,570 less the $8,750 standard deduction and $930 exemption leaves $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Built its $48,520 starting figure around the $25,950 of estate income that sits outside the federal AGI Virginia starts from, and took a $17,000 married-filing-jointly standard deduction for the surviving-spouse status instead of Virginia's $8,750 single amount. The resulting $30,590 base is more than double the correct $12,889.97 and reaches the 5.75 percent bracket, which the correct base never touches." -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Kept the $25,950 of estate income in the state base ($48,520 instead of $22,570), added an $800 exemption for disability when Virginia's additional $800 exemption is for taxpayers 65 or older or blind, and used an $8,500 standard deduction instead of $8,750. It then cut its own $1,944 bracket result to the $1,545 submitted for unspecified ""medical/other adjustments.""" -us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Started from the $48,520 non-Social-Security total that embeds the $25,950 of estate income excluded from the reference AGI, used an $8,500 standard deduction rather than $8,750, and converted the $930 personal exemption into a $30 credit effect instead of deducting it from income. Its roughly $40,000 base is more than triple the correct $12,889.97 of Virginia taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"Invented a $20,000 Virginia subtraction for IRA and pension income of disabled filers under 65; §58.1-322.02(6) covers income received for permanent and total disability, not retirement distributions. It paired that with a $17,000 joint standard deduction and $2,400 of exemptions on a base that still included the $25,950 of estate income, and the offsetting errors landed at $326 against the correct $12,889.97 of taxable income." -us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Used $48,520 as Virginia AGI, which includes the $25,950 of estate income the reference AGI excludes, took an $8,500 standard deduction and dropped the $930 exemption, reaching a $40,020 base against the reference's $12,889.97. It then submitted $3,400 after its own brackets produced $2,044, adjusting upward with no stated basis." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Subtracted the full $36,105 of Social Security from a federal AGI in which it had included only $30,689 of it, layered on a fabricated $12,000 retirement subtraction for disabled filers, and used Virginia's repealed $4,000 standard deduction with a $970 exemption instead of $8,750 and $930. It also kept the $25,950 of estate income in the base, which alone accounts for the gap between its $26,134 and the reference's $12,889.97." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Never applied Virginia's subtraction for federally taxable Social Security, leaving $30,689 of benefits in the state base, and additionally kept the $25,950 of estate income in it, taxing $69,529 instead of $12,889.97. Its $8,750 standard deduction and $930 exemption match the reference exactly, so the entire $3,225.50 error is those two unremoved income items." -us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Carried the $25,950 of estate income into Virginia AGI, using $48,520 where the reference has $22,570, and took an $8,500 standard deduction instead of Virginia's $8,750 for 2026. Removing the estate income alone drops its base from $39,090 to $13,140 and the tax to $527; the remaining $12.50 is the 5 percent bracket applied to its $250 standard-deduction shortfall." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $48,520 Virginia AGI includes the $25,950 of estate income that the reference federal AGI of $32,699.07 excludes, and it used an $8,500 standard deduction rather than $8,750. Virginia AGI is the $26,800 IRA distribution plus $2,030 pension less the $6,260 self-employment loss, and after $8,750 and $930 the taxable amount is $12,889.97, taxed entirely at the 2, 3 and 5 percent rates." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $2,420 is Virginia tax on about $46,565 of taxable income, which corresponds to the $48,520 non-Social-Security total containing the $25,950 of estate income with roughly $1,955 of allowances rather than the $8,750 standard deduction and $930 exemption. The correct chain is $22,570 of Virginia AGI less $9,680, leaving $12,889.97 taxed at $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $1,990 is the tax on $39,090, i.e. the $48,520 that includes the $25,950 of estate income less an $8,500 standard deduction and the $930 exemption. Estate income is outside the federal AGI Virginia starts from, so Virginia AGI is $22,570 and the 2026 standard deduction is $8,750, leaving $12,889.97 taxable." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income in the $48,520 base, used a $9,000 standard deduction instead of $8,750 and an $800 exemption instead of $930. Its $38,720 result is three times the reference's $12,889.97 and puts $21,720 into the 5.75 percent bracket, which the correct base never reaches." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Gave no derivation; $3,371 is the tax on about $63,100 of taxable income, which is federal AGI with both the $25,950 of estate income and the federally taxable Social Security left in the state base. Virginia subtracts all federally taxable Social Security and starts from an AGI that excludes estate income, leaving $22,570 before the $8,750 standard deduction and $930 exemption and $12,889.97 taxable." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Kept the $25,950 of estate income in Virginia AGI ($48,520 versus $22,570) and used an $8,000 standard deduction instead of $8,750, taxing $39,590 rather than $12,889.97. Its own brackets on $39,590 total $2,018.93, ten dollars below the $2,028.93 it reported." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,990 matches Virginia tax on $39,090, the $48,520 non-Social-Security total that includes the $25,950 of estate income less an $8,500 standard deduction and the $930 exemption. The reference base excludes estate income, so Virginia AGI is $22,570 and, after $8,750 and $930, only $12,889.97 is taxable, all of it below the $17,000 start of the 5.75 percent bracket." -us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income and took an $8,000 standard deduction instead of $8,750, taxing $39,590 rather than $12,889.97. Its stated brackets on $39,590 give $2,018.93, not the $2,024 submitted." -us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Ran the provisional-income test with the $25,950 of estate income in it and, decisively, carried that estate income into Virginia AGI as $48,520 rather than $22,570. Because Virginia subtracts the federally taxable Social Security in full, the benefit calculation cancels out entirely and only the non-Social-Security items drive the answer; it also used an $8,500 standard deduction instead of $8,750." -us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Applied a flat 2 percent to $20,330 instead of Virginia's graduated 2/3/5/5.75 percent schedule, omitted both the $6,260 self-employment loss and the $930 personal exemption, and used an $8,500 standard deduction instead of $8,750. It then added roughly $2,000 of ""prior-year estimated payments"" to reach $2,516, and payments are not part of tax before credits; the correct base is $22,570 less $9,680, taxed at $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"Asserted a non-positive Virginia base on ""age-based treatment,"" but at 61 the head qualifies for no Virginia age deduction, which requires age 65 or older, and no disability subtraction reaches IRA or pension income. Virginia AGI of $22,570 less the $8,750 standard deduction and $930 exemption leaves $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Submitted zero on an assumed low-income scenario with no derivation. Virginia AGI of $22,570 exceeds the $9,680 of standard deduction plus exemption and sits far above the federal poverty guideline ceiling for Virginia's Credit for Low-Income Individuals, leaving $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used $48,520 as the non-Social-Security base, embedding the $25,950 of estate income that the reference AGI excludes, and an $8,500 standard deduction instead of $8,750. Its $720-plus-5.75-percent shortcut presumes a base above $17,000, while the correct $12,889.97 of taxable income falls entirely in the 2, 3 and 5 percent brackets." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income in the state base and applied roughly a $12,500 surviving-spouse standard deduction with the $930 exemption, taxing about $35,090. Virginia allows this filer the $8,750 single standard deduction against $22,570 of Virginia AGI, leaving $12,889.97 taxable." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income to reach a $48,520 base and applied a $17,500 joint standard deduction where Virginia allows $8,750. Virginia AGI is $22,570 and taxable income is $12,889.97, not the $30,090 it taxed." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Kept the $25,950 of estate income in the Virginia base and used roughly a $12,000 surviving-spouse standard deduction plus the $930 exemption, taxing about $35,590. The reference applies the $8,750 single standard deduction and $930 exemption to $22,570 of Virginia AGI, leaving $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Set its pre-deduction figure at $48,520 by including the $25,950 of estate income excluded from the reference AGI, then took a $17,500 surviving-spouse standard deduction against Virginia's $8,750. Correcting both leaves $12,889.97 of taxable income and $514.50 of tax rather than $30,090 and $1,472.68." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and exemptions offset the entire base. The $8,750 standard deduction plus the $930 exemption total $9,680 against $22,570 of Virginia AGI, which is the $26,800 IRA distribution plus $2,030 pension less the $6,260 self-employment loss, leaving $12,889.97 taxable and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income in Virginia AGI ($48,520 rather than $22,570) and used an $8,000 standard deduction instead of $8,750, taxing $39,590 instead of $12,889.97. That base reaches the 5.75 percent bracket, which the correct taxable income never enters." -us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Carried the $25,950 of estate income into Virginia AGI and used an $8,500 standard deduction instead of $8,750, taxing $39,090. It also justified dropping the age deduction on an AGI phase-out when the deduction is simply unavailable below age 65; the correct base is $22,570 less $8,750 and $930, or $12,889.97." -us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Included the $25,950 of estate income to reach $48,520 of Virginia AGI and applied an $8,000 standard deduction instead of $8,750, taxing $39,590. Virginia AGI is $22,570, taxable income is $12,889.97, and the tax is $60 + $60 + 5 percent of $7,889.97 = $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Used $48,520 of Virginia AGI, which keeps the $25,950 of estate income the reference AGI excludes, and an $8,000 standard deduction instead of $8,750, giving $39,590 of taxable income. The correct base of $12,889.97 stays entirely within the 2, 3 and 5 percent brackets, so no part of it is taxed at 5.75 percent." -us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value or explanation for state_income_tax_before_refundable_credits, so no substantive computation was returned. The required answer is Virginia AGI of $22,570 less the $8,750 standard deduction and $930 personal exemption, giving $12,889.97 of taxable income and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Applied exactly the reference's $8,750 standard deduction and $930 exemption but started from $48,520 because it kept the $25,950 of estate income in the state base. That single item is the whole $1,461.30 error: $22,570 less $9,680 is $12,889.97 of taxable income, taxed at $514.50." -us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"Stacked three nonexistent Virginia items on a 61-year-old — a $6,000 age-65 deduction, another $6,000 for disability, and a $1,000 surviving-spouse age-65 adjustment — and never subtracted the federally taxable Social Security that Virginia removes in full. It then computed about $4,100 and submitted $1,500 as a rough estimate with no derivation, against the correct $12,889.97 base and $514.50 of tax." -us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Denied Virginia's Social Security subtraction on an invented ""$50,000 single AGI"" ceiling; Virginia subtracts all federally taxable Social Security with no income test, and the AGI-based reduction belongs to the age deduction. It also kept the $25,950 of estate income in the base, taxing $69,779 instead of the reference's $12,889.97." -us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated the $930 personal exemption as the standard deduction, dropping Virginia's $8,750 standard deduction entirely, and kept the $25,950 of estate income in the base, reaching $47,590 rather than $12,889.97. It then submitted $3,260.45 while its own bracket arithmetic produced $1,878.93." -us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,age_disability,False,"Invented an ""age-61 exemption"" of $11,898 and a ""nonrefundable pension deduction"" of $2,030 that wipes out the liability; Virginia grants no age exemption before 65 and no credit for pension income. Virginia AGI of $22,570 less the $8,750 standard deduction and $930 exemption leaves $12,889.97 taxable, producing $514.50 of tax." +us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Put the $25,950 of estate income into AGI, which gave a Virginia AGI of $48,520 instead of $22,569.97. It then left out the $930 personal exemption and, with no stated basis, raised its own bracket result of about $2,029 to $2,192. The reference taxes $12,889.97 after the $8,750 deduction and $930 exemption, for $514.50." +us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Gave the surviving spouse the $17,500 joint standard deduction, but Virginia has no surviving-spouse status, so the filer gets the single $8,750 deduction. It also counted the $25,950 of estate income in AGI (Virginia AGI $48,520 instead of $22,569.97), so its taxable income of $30,090 is far from the reference's $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Relied on a Virginia retirement-income credit and disability exclusion that do not exist in the form it described. A 61-year-old gets no age deduction (that starts at 65), and the disability subtraction covers only disability pay received in place of wages, not IRA or pension income. After the Social Security subtraction, $8,750 deduction and $930 exemption, $12,889.97 stays taxable, so tax is $514.50, not $0." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used a joint-level $17,000 standard deduction for the surviving spouse instead of Virginia's single $8,750, and counted the $25,950 of estate income in AGI. It also got the brackets wrong, putting the tax at $17,000 at $670 instead of $720." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI (Virginia AGI $48,520), used the outdated $8,500 deduction, and added an $800 exemption for disability, although Virginia gives the extra $800 only for age 65+ or blindness. It then cut its own $1,944 result to $1,545 for unnamed 'medical/other adjustments'." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI, which made taxable Social Security $30,689 and Virginia AGI $48,520. PolicyEngine's AGI of $32,699.07 has no estate income, so its taxable Social Security is $10,129.10 and Virginia taxable income is $12,889.97. The model also used the 2024 deduction of $8,500 instead of $8,750 and treated the exemption as a $30 credit." +us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in federal AGI, giving taxable Social Security of $30,689.25 and Virginia AGI of $48,520. PolicyEngine's AGI of $32,699.07 has no estate income, which leaves taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied a $20,000 retirement subtraction for a disabled filer to the pension and IRA income, but Virginia's disability subtraction covers only disability income paid in place of wages. It also started from gross Social Security instead of the taxable portion, used the joint $17,000 deduction, and claimed three $800 exemptions instead of one $930 exemption." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI and left out the $930 personal exemption. It then raised its own bracket result of $2,044 to $3,400 with no rule to support it. The reference taxes $12,889.97 for $514.50." +us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used the $17,500 joint standard deduction for a surviving spouse, but Virginia applies the single $8,750 deduction. It also counted the $25,950 of estate income in Virginia AGI ($48,520 instead of $22,569.97)." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the full $36,105 of Social Security from an AGI that included only $30,689 of taxable benefits. It added a $12,000 retirement subtraction for disability that Virginia does not provide, and used an outdated $4,000 deduction and a $970 exemption instead of $8,750 and $930." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Never applied Virginia's full subtraction of federally taxable Social Security and taxed a federal AGI of $79,209 that also included the $25,950 of estate income. The result was $69,529 of taxable income instead of $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI (taxable Social Security $30,689.25, Virginia AGI $48,520). PolicyEngine's AGI of $32,699.07 leaves it out, giving a Virginia AGI of $22,569.97. The model also used the 2024 deduction of $8,500 instead of the 2026 amount of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, while PolicyEngine's AGI is $32,699.07 without it. That difference raised taxable Social Security and Virginia AGI to $48,520. The model also used the outdated $8,500 deduction instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out, so its Virginia AGI was $48,520 instead of $22,569.97. It also used the 2024 deduction of $8,500 instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. $2,420 implies Virginia taxable income of about $46,565, which fits counting the estate income and taking almost none of the $8,750 deduction and $930 exemption. The reference taxes $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"$1,990 matches a Virginia AGI of $48,520 (estate income included) less the 2024 deduction of $8,500 and a $930 exemption. PolicyEngine's AGI leaves out the $25,950 of estate income, giving a Virginia AGI of $22,569.97 and taxable income of $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in federal AGI, which PolicyEngine's AGI leaves out, so its Virginia AGI was $48,520 instead of $22,569.97. It also used a $9,000 deduction and an $800 exemption instead of Virginia's $8,750 and $930." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. $3,371 implies Virginia taxable income of about $63,100, which fits taxing federal AGI including estate income without Virginia's full subtraction of taxable Social Security. The reference taxes $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the old $8,000 deduction instead of the 2026 amount of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"$1,990 matches a Virginia AGI of $48,520 that includes the $25,950 of estate income, less the outdated $8,500 deduction and a $930 exemption. PolicyEngine's AGI leaves out estate income, so Virginia taxable income is $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97. It also used the pre-2024 $8,000 deduction instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in non-Social Security income ($48,520), which made provisional income $66,572.50 and taxable Social Security $30,689.25. PolicyEngine leaves estate income out of AGI, which gives provisional income of $40,622.47, taxable Social Security of $10,129.10 and a Virginia AGI of $22,569.97. The model also used $8,500 instead of the $8,750 deduction." +us,scenario_039,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Treated Virginia as having a flat 2% rate instead of its 2%/3%/5%/5.75% brackets, and set the $6,260 self-employment loss to $0. It then added $2,000 of made-up 'prior-year estimated payments' to reach $2,516." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Assumed age-based relief would wipe out Virginia taxable income, but the age deduction starts at 65 and the filer is 61. After the Social Security subtraction, the $8,750 deduction and the $930 exemption, $12,889.97 stays taxable." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Assumed a low-income case with no tax and did no computation. Virginia AGI of $22,569.97 less $9,680 of deduction and exemption leaves $12,889.97 taxable, for $514.50." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Used $48,520 of non-Social Security income that includes the $25,950 of estate income, which PolicyEngine's AGI leaves out, giving a Virginia AGI of $22,569.97. It also used the 2024 deduction of $8,500 instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Applied a surviving-spouse/joint standard deduction, but Virginia taxes a surviving spouse as single with the $8,750 deduction. It also counted the $25,950 of estate income in Virginia AGI." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used the $17,500 joint standard deduction for a surviving spouse instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI ($48,520 instead of $22,569.97)." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Applied a larger surviving-spouse standard deduction (about $12,900 implied) instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI. The reference taxes $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Used a $17,500 surviving-spouse standard deduction, but Virginia applies the single $8,750. It also counted the $25,950 of estate income, so its Virginia AGI was $48,520 instead of $22,569.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Taxed $69,529, which is federal AGI including estate income with no Virginia Social Security subtraction. Virginia fully subtracts federally taxable Social Security, and the reference's taxable income is $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in AGI, giving taxable Social Security of $30,689.25 and $48,520 of remaining income. PolicyEngine's AGI of $32,699.07 leaves estate income out, which gives taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference." +us,scenario_039,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used a $17,500 surviving-spouse standard deduction instead of Virginia's single $8,750, and counted the $25,950 of estate income in Virginia AGI. It also made an arithmetic error: tax on $30,090 under its own brackets is $1,472.68, not $1,442.68." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed the standard deduction and exemptions cancel out all income. The $8,750 deduction plus the $930 exemption is only $9,680, which leaves $12,889.97 of the $22,569.97 Virginia AGI taxable, for $514.50." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97. It also used the old $8,000 deduction instead of the 2026 amount of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the $8,500 deduction instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out, giving $22,569.97 and taxable income of $12,889.97. It also used the 2024 deduction of $8,500 instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $25,950 of estate income in its $48,520 Virginia AGI, which PolicyEngine's AGI leaves out. It also used the pre-2024 $8,000 deduction instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of about $79,209 that included the $25,950 of estate income, which PolicyEngine's AGI of $32,699.07 leaves out. It also used the outdated $8,000 deduction instead of $8,750." +us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $514.50 reference." +us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Started from a federal AGI of $79,209.25 that included the $25,950 of estate income, giving taxable Social Security of $30,689.25. PolicyEngine's AGI of $32,699.07 leaves estate income out, which gives taxable Social Security of $10,129.10 and Virginia taxable income of $12,889.97. The model's $8,750 deduction, $930 exemption and brackets match the reference." +us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Gave a 61-year-old age 65+ deductions and an extra 'surviving spouse age 65' adjustment, and never subtracted taxable Social Security. It then cut its own $4,100 estimate to $1,500 with no computation." +us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Wrongly required AGI of $50,000 or less for Virginia's Social Security subtraction. Virginia subtracts all federally taxable Social Security at any income level, so the model taxed $69,779 instead of $12,889.97." +us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Put all $36,105 of Social Security into federal AGI instead of only the taxable portion. It used the $930 exemption as the only deduction, leaving out the $8,750 standard deduction, and then reported $3,260.45, which does not match its own computed $1,878.93." +us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Made up an age-61 exemption of $11,898 and a nonrefundable 'pension deduction' credit of $2,030. Virginia has neither, and the reference's $514.50 comes only from the Social Security subtraction, the $8,750 deduction, the $930 exemption, and the brackets." us,scenario_039,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed AGI correctly at $32,300 but assumed the TCJA sunset took effect in 2026, subtracting a $18,850 standard deduction plus two revived personal exemptions of $10,600 for $29,450 of offsets. Personal exemptions remain repealed in 2026 and the MFJ standard deduction is $32,200, plus $3,300 of aged additional standard deduction and the $12,000 OBBBA senior deduction for two 65+ spouses, totaling $47,500 — more than AGI, so taxable income is $0 rather than the $2,850 it taxed at 10%." us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model reached the correct $32,300 AGI but then paired a $20,000 MFJ standard deduction with $10,600 of personal exemptions, a combination that exists under no 2026 rule set — personal exemptions are permanently repealed and the 2026 MFJ standard deduction is $32,200. Adding the $3,300 aged additional standard deduction and the $12,000 senior deduction for two spouses aged 65+ wipes out the entire $32,300 of AGI, leaving $0 taxable income instead of the $1,700 it taxed at 10%." @@ -3244,88 +3605,102 @@ us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error, us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed $21,420 of Social Security as part of the Arizona base, missing both that provisional income of $40,600 puts only $4,300 of the $25,200 in federal AGI and that Arizona subtracts 100% of the Social Security benefits included in federal AGI. It also contradicted its own arithmetic, computing $484.25 and then submitting $486.25; with Social Security removed, Arizona AGI is $28,000, below the roughly $32,200 MFJ standard deduction, so taxable income and tax are both zero." us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0." us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It assembled the $24,150 deduction stack correctly but built provisional income from $32,807 of non-SS income that folds in the $3,753 non-Schedule-D capital gains, which PolicyEngine keeps out of IRS gross income; its $47,597 provisional income produced $16,057 of taxable Social Security instead of the $13,655.95 that the tier-2 formula gives on the correct $44,771.71 base. That single overstatement lifted taxable income from $19,487.65 to $24,714 and the tax from $1,979.16 to $2,133." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It filed the head at joint rates as a surviving spouse and claimed a $33,850 married standard deduction; with no dependent this filer is Single, taking $16,100 + $2,050 aged = $18,150 plus the $6,000 senior deduction. Compounded by $16,846.25 of taxable Social Security computed on a provisional income that includes the excluded $3,753 of capital gains, it left only $9,789 of ordinary income at 10% instead of $18,559.65 spanning the 10% and 12% brackets." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included all $29,580 of Social Security in gross income rather than applying the §86 formula that makes $13,655.95 taxable, used a $28,900 married standard deduction in place of the single $18,150 plus the $6,000 senior deduction, and then subtracted an invented $783 'dependent exemption credit' that does not exist in the federal code." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security dependent benefits as a child's income and excluded them from the head entirely, while PolicyEngine attributes them to the head and taxes $13,655.95 of them. It also omitted the $6,000 senior deduction, so its $14,657 of taxable income falls $4,830 short of the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It applied the $44,000 married provisional-income threshold to force Social Security to the 85% maximum of $25,143 and used a $32,350 surviving-spouse standard deduction; the single $25,000/$34,000 thresholds on $44,771.71 of provisional income give $13,655.95, and the deduction stack is $18,150 + $6,000. It then discarded its own $2,603 computation for an unexplained $1,245." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It asserted that high other income made 85% of benefits taxable ($25,143), but provisional income of $44,771.71 lands in the tier-2 formula, $4,500 + 85% × $10,771.71 = $13,655.95, so its $58,878 AGI overstates the true $43,637.65 by $15,240. Its submitted $2,699 also does not follow from its own stated $34,700 of taxable income, which would produce roughly $3,900." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the $6,000 OBBBA senior deduction, stopping at a $17,750 standard deduction rather than $24,150, and put the $3,753 non-Schedule-D capital gains and $71 BDC income into gross and provisional income, raising taxable Social Security to $16,907 against $13,655.95. Together those left $32,949 of taxable income versus the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It excluded the entire $29,580 of Social Security as dependent benefits not includible to the head, though $13,655.95 is taxable to them, and then submitted $0 despite its own arithmetic showing about $16,792 of taxable income, which would itself produce roughly $1,767 of tax. The correct base is $43,637.65 of AGI less $24,150 of deductions." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It claimed only $17,650 of deductions, omitting the $6,000 senior deduction that brings the stack to $24,150, and included the $3,753 capital gains and $71 BDC income in AGI and provisional income, taxing $16,907 of Social Security instead of $13,655.95. Taxable income came out $33,063 against the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $50,712.60 AGI includes $3,824 of non-Schedule-D capital gains and BDC income that PolicyEngine excludes from gross income, which pushed taxable Social Security to $16,906.60 rather than $13,655.95, and its $17,425 deduction leaves out the $6,000 senior deduction. The result was $33,287.60 of taxable income instead of $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed 2026 under a pre-TCJA sunset, using an $8,600 standard deduction, a $2,100 aged addition and a $5,500 personal exemption; 2026 applies a $16,100 single standard deduction, a $2,050 aged addition, the $6,000 senior deduction and no personal exemption. It also taxed $16,907 of Social Security rather than $13,655.95, giving $34,513 of taxable income against $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied expired pre-TCJA parameters to 2026 — a ~$10,300 standard deduction, a $5,200 personal exemption and a 15% bracket — where the law gives $18,150 of standard deduction plus the $6,000 senior deduction and 10%/12% rates. It also taxed $16,907 of Social Security instead of $13,655.95, producing $35,213 of taxable income versus $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used a head-of-household standard deduction for a filer who is Single and omitted the $6,000 senior deduction. Its $3,156 corresponds to about $28,367 of ordinary taxable income — an AGI near $50,700 with Social Security taxed around $16,907, less roughly $17,650 of deductions — where the correct chain is $43,637.65 of AGI less $24,150, leaving $18,559.65 of ordinary income." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It built AGI of $50,581 by including the $3,753 capital gains and $71 BDC income in gross and provisional income, taxing $16,846 of Social Security rather than $13,655.95, and then subtracted only an elderly single standard deduction with no $6,000 senior deduction. Its taxable income runs roughly $13,000 above the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It combined a $15,150 pre-TCJA standard-deduction-plus-personal-exemption figure with pre-TCJA brackets reaching 15%, where 2026 gives $18,150 of standard deduction, the $6,000 senior deduction, and 10%/12% rates on this income. Its $16,846 of taxable Social Security also exceeds the $13,655.95 produced on the $44,771.71 provisional-income base, leaving $30,750 of ordinary income versus $18,559.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation; $2,878 corresponds to about $26,050 of ordinary taxable income, consistent with an AGI near $48,900 (Social Security taxed well above $13,655.95) reduced only by the $18,150 standard deduction with the $6,000 senior deduction omitted. The correct chain is $43,637.65 of AGI less $24,150, leaving $18,559.65 of ordinary income taxed at 10% to $12,400 and 12% above." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It took a $31,550 surviving-spouse standard deduction in place of the single $18,150 plus the $6,000 senior deduction, and understated taxable Social Security at $9,846 against $13,655.95. Its $12,017 of taxable income leaves only $7,336 in the 10% bracket, where the correct $18,559.65 of ordinary income puts $6,159.65 into the 12% bracket." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation states $3,584 while the submitted value is $3,090, and both rest on a $50,581 AGI that folds in the $3,753 capital gains and taxes about $16,846 of Social Security, reduced only by a single age-65 standard deduction with no $6,000 senior deduction. The correct base is $43,637.65 of AGI less $24,150 of deductions." -us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation; $3,734 implies roughly $33,183 of ordinary taxable income, which requires taxing Social Security far above $13,655.95 and skipping the $6,000 senior deduction. The correct chain is $43,637.65 of AGI less $24,150 of deductions, giving $19,487.65 taxable with $18,559.65 ordinary after the $928 of qualified dividends." -us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It put the $3,753 non-Schedule-D gains and $71 BDC income into its $34,806 non-SS income base, lifting provisional income to $49,596 and taxable Social Security to $17,757 instead of $13,655.95, and stopped at a $17,850 standard deduction without the $6,000 senior deduction. Taxable income came out $34,713 against the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It declared the Social Security 'largely nontaxable' when the single-filer tier-2 formula on $44,771.71 of provisional income makes $13,655.95 taxable, and submitted a digit-sequence placeholder of $1,234.56 backed by no arithmetic. The correct computation taxes $18,559.65 of ordinary income at 10% to $12,400 and 12% above, for $1,979.16." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age-related deductions fully offset income; the deduction stack is $24,150 ($16,100 + $2,050 + $6,000) against $43,637.65 of AGI, which leaves $19,487.65 of taxable income. It also assumed age-related nonrefundable credits absorb the rest, but the elderly credit is fully phased out at this AGI and nothing reduces the $1,979.16." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction plus nonrefundable credits drive the liability to zero; $24,150 of deductions against $43,637.65 of AGI leaves $19,487.65 of taxable income and no nonrefundable credit is available here. The $18,559.65 ordinary portion is taxed at 10% up to $12,400 and 12% above, giving $1,979.16." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It applied a $33,200 surviving-spouse age-65 standard deduction; with no dependent the filer is Single with $18,150 plus the $6,000 senior deduction, a $9,050 difference. Its $16,846 of taxable Social Security also overstates $13,655.95, and the resulting ~$12,700 of ordinary income taxed at 10% replaces $18,559.65 taxed at 10% and 12%." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It drove Social Security to the 85% maximum for a $58,949 AGI when the tier-2 formula on $44,771.71 of provisional income yields $13,655.95 and an AGI of $43,637.65, and it then taxed the qualified dividends and capital gains at 15% although $19,487.65 of taxable income keeps preferential income entirely in the 0% band." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It claimed the qualifying-surviving-spouse standard deduction of $33,850 on top of the $6,000 senior deduction; the correct stack is the single $18,150 plus $6,000 = $24,150. That extra $15,700 of deduction, together with $16,846.25 of taxable Social Security instead of $13,655.95, cut ordinary taxable income to $6,036 from $18,559.65." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse standard deduction with the age add-on and senior deduction, leaving taxable income entirely within the 10% bracket. The single $18,150 + $6,000 stack against $43,637.65 of AGI leaves $19,487.65 of taxable income, $18,559.65 of it ordinary, with $6,159.65 taxed at 12%." -us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It filed as qualifying surviving spouse with a $33,850 age-adjusted standard deduction rather than the single $18,150, and included the $3,753 capital gains and $71 BDC income in provisional income, taxing $16,846.25 of Social Security instead of $13,655.95 while claiming a $14.20 QBI deduction on income PolicyEngine excludes. Ordinary taxable income of $6,036.05 replaced the correct $18,559.65." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation and invoked a mortgage interest deduction that does not exist here, since only a $1,000 loan balance is listed and no interest is an input. Its $3,125 corresponds to about $28,100 of ordinary taxable income, which requires taxing Social Security above $13,655.95 and skipping the $6,000 senior deduction that brings total deductions to $24,150." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA sunset for 2026, using a $10,507 standard deduction plus a $5,387 personal exemption and 10%/15% brackets, where the law gives $16,100 + $2,050 + $6,000 of deductions, no personal exemption, and a 12% top rate on this income. It also taxed $16,907 of Social Security instead of $13,655.95, leaving $34,819 of taxable income against $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used an $8,300 standard deduction with a $5,300 personal exemption and the 15% bracket for 2026; the correct parameters are a $16,100 base, $2,050 aged addition, $6,000 senior deduction, no exemption, and 10%/12% rates. Its $16,907 of taxable Social Security also exceeds $13,655.95, giving $35,063 of taxable income versus $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It combined a pre-TCJA $10,233 standard deduction and 15% bracket with $24,205 of taxable Social Security derived by pushing provisional income toward the 85% cap; 2026 gives $24,150 of deductions and $13,655.95 of taxable benefits. Its $47,778 of taxable income is roughly 2.5 times the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It double-counted the farm loss as −$2,254 by adding farm income and farm operations income, which describe the same single $1,127 loss, stopped at the $18,150 standard deduction with no $6,000 senior deduction, and taxed $15,949 of Social Security instead of $13,655.95. Its $30,478 of taxable income replaced the correct $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for this variable, so no substantive computation exists to evaluate. The required chain is $43,637.65 of AGI less $24,150 of deductions, leaving $18,559.65 of ordinary taxable income after the $928 qualified-dividend exclusion, taxed at 10% to $12,400 and 12% above for $1,979.16." -us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It got Single status and the $16,100 + $2,050 + $6,000 = $24,150 deduction stack exactly right, then botched §86: on its own $48,525 provisional income the tier-2 formula gives $4,500 + 85% × $14,525 = $16,846, not the $24,108.75 it used, and PolicyEngine's $44,771.71 base — which excludes the $3,753 capital gains and $71 BDC income — gives $13,655.95. That $10,453 overstatement raised taxable income to $33,693.75 from $19,487.65." -us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"It applied the married $44,000 provisional-income threshold and a $32,150 surviving-spouse standard deduction, and its $8,407 of taxable Social Security follows from no §86 tier; the single $25,000/$34,000 thresholds give $13,655.95 and the deduction stack is $18,150 + $6,000. Its $10,063 of taxable income left $5,311 at 10% instead of $18,559.65 spanning the 10% and 12% brackets." -us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed at married rates with a $39,850 deduction stack ($32,200 base + $1,650 age + $6,000 senior) and used the $44,000 married Social Security threshold to reach $9,907 of taxable benefits. Single status gives $24,150 of deductions and $13,655.95 of taxable benefits, so taxable income is $19,487.65 rather than its $3,863." -us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $4,700 of tax-exempt IRA distributions into provisional income, where only tax-exempt interest belongs, producing $25,698 of taxable Social Security against $13,655.95, and used a $31,675 qualifying-surviving-spouse standard deduction rather than the single $18,150 plus the $6,000 senior deduction. It then subtracted a $562 'savings' on preferential income from bracket tax it had already computed on the ordinary portion." -us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It submitted $21,824 with no derivation — about half of AGI, and roughly ten times the largest liability the 10%/12% brackets can generate on $19,487.65 of taxable income. The correct computation is $43,637.65 of AGI less $24,150 of deductions, then 10% on the first $12,400 and 12% on the remaining $6,159.65 of the $18,559.65 ordinary portion." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left the $928 of qualified dividends out of gross income (non-SS income $32,807 instead of $33,735). That understated both provisional income and taxable Social Security ($16,057 instead of $16,846.25), and it cut taxable income to $24,714 instead of $26,430.71. It also counted the $71 BDC income as preferential income and did loose bracket arithmetic, which is how it reached $2,133." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It filed as Qualifying Surviving Spouse and used the $32,200 joint standard deduction with joint brackets. QSS status requires a dependent child, so this filer is Single, with a $16,100 + $2,050 deduction and a 10% bracket ending at $12,400. It also taxed the $3,753 of capital gains as ordinary income instead of at the 0% preferential rate." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It put the entire $29,580 of Social Security into income without applying the provisional-income formula, which makes only $16,846.25 taxable. It used an invented $28,900 surviving-spouse deduction and left out the $6,000 senior deduction. It then taxed the $4,681 of gains at 15% instead of 0% and subtracted a nonexistent 'dependent exemption credit'." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security dependent benefits as belonging to someone else and excluded them. In fact $16,846.25 is taxable to the head once provisional income of $48,525 is run through the 25k/34k formula. It also left out the $928 qualified dividends and the $6,000 senior deduction, which produced a $14,657 taxable income unrelated to the correct $26,430.71." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used Qualifying Surviving Spouse parameters (a $30,700 deduction and joint brackets) even though there is no dependent child, so the filer is Single. It also made 85% of all benefits ($25,143) taxable instead of the $16,846.25 the two-tier formula gives, and then applied an unexplained downward 'adjustment' to land on $1,245." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It made 85% of all benefits ($25,143) taxable instead of applying the formula min(85% of benefits, 0.85 × (48,525 − 34,000) + 4,500) = $16,846.25. That inflated AGI to about $58,878 instead of $50,580.71. It then produced $2,699 through loose arithmetic instead of computing tax on $21,749.71 of ordinary taxable income." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with joint brackets and the $32,200 + $1,650 joint deduction. With no dependent child QSS is unavailable, so the Single $16,100 + $2,050 deduction and the Single brackets apply. That leaves $21,749.71 of ordinary income taxed at 10%/12% rather than $6,036 at 10%." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $17,750 deduction and omitted the $6,000 OBBBA senior deduction for filers 65 and older, so taxable income came out about $6,000 too high. It also added the $71 BDC income on top of dividends and used outdated bracket figures." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It excluded the $29,580 of Social Security on the theory that dependent benefits are not the taxpayer's, but $16,846.25 of them is taxable to the head. It then reported $0 even though its own $16,792 taxable income produces positive tax at 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with the $32,200 joint deduction and joint brackets. With no dependent child the filer is Single, entitled to $16,100 + $2,050 + $6,000. That leaves $21,749.71 of ordinary income taxed at 10%/12% instead of $6,050 at 10%." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $17,650 standard deduction and omitted the $6,000 senior deduction for filers 65 and older, which overstated taxable income by about $6,000. It also counted the $71 BDC income as extra income, which raised taxable SS to $16,907." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied only a $17,425 standard deduction and never took the $6,000 senior deduction, so taxable income was $33,287.60 instead of $26,430.71. It also counted the $71 BDC income on top of dividends." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law ($8,600 deduction, a $5,500 personal exemption, and the 15% bracket), but OBBBA made the TCJA parameters permanent. The correct 2026 figures are a $16,100 + $2,050 deduction plus the $6,000 senior deduction, with 10%/12% brackets." +us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used a $17,350 deduction with no $6,000 senior deduction, which produced $33,362.60 of taxable income instead of $26,430.71. Its SS amount of $16,907 also includes the $71 BDC income double-counted on top of dividends." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and used a $10,300 deduction, a $5,200 personal exemption, and the 15% bracket. OBBBA kept TCJA's $16,100 + $2,050 deduction and 10%/12% brackets, and it added the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used head-of-household status even though there is no qualifying person, and its $3,156 is roughly $794 above the correct tax. That gap matches omitting the $6,000 senior deduction at the 12% marginal rate, which adds about $720." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It had the correct $50,581 AGI but subtracted only the elderly Single standard deduction and skipped the $6,000 senior deduction. That leaves $32,431 of taxable income instead of $26,430.71, and it rounded up further to $3,500." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA law with a combined $15,150 deduction and exemption and the 15% bracket. OBBBA made the $16,100 + $2,050 deduction and the 10%/12% brackets permanent and added the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $1,240 + 12% × $9,349.71 = $2,361.96 on $21,749.71 of ordinary taxable income. Its $2,878 is about $516 higher, roughly the effect of taxing the $4,681 of qualified dividends and capital gain at 12% instead of the 0% rate." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $31,550 surviving-spouse deduction, but with no dependent child the filer is Single. It also understated taxable Social Security at $9,846 instead of $16,846.25 and omitted the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the Single age-65 standard deduction from the $50,581 AGI and omitted the $6,000 senior deduction. The submitted $3,090 matches tax on $32,431 of taxable income, and its explanation contradicts that value by citing $3,584." +us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and never applied the $6,000 senior deduction. Its $3,734 also exceeds the roughly $3,082 that the no-senior-deduction path gives, which is consistent with taxing the $4,681 of preferential income at ordinary rates." +us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $17,850 deduction with no $6,000 senior deduction. It also made a $1,000 addition error (non-SS income $34,806 instead of $33,805 including BDC), which inflated taxable SS to $17,757 and AGI to $52,563." +us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It returned the placeholder figure $1,234.56 with no computation. Its premise that Social Security is 'largely nontaxable' is wrong, because provisional income of $48,525 makes $16,846.25 taxable and leaves $26,430.71 of taxable income." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age deductions fully offset income. In fact $33,735 of other income plus $16,846.25 of taxable Social Security exceeds the $24,150 of deductions by $26,430.71, of which $21,749.71 is ordinary income taxed at 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits eliminate all tax, but AGI of $50,580.71 minus $24,150 of deductions leaves $26,430.71 taxable. No nonrefundable credit applies, because the elderly credit is phased out at this AGI." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $33,200 surviving-spouse deduction and taxed the remainder at 10% under joint brackets. QSS requires a dependent child, so Single parameters apply ($16,100 + $2,050 + $6,000 senior deduction, with the 10% bracket ending at $12,400), and it also never took the senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $58,949 AGI treats 85% of all benefits as taxable instead of the $16,846.25 the provisional-income formula gives. It also taxed qualified dividends and capital gains at 15%, although they sit entirely within the 0% band at $26,430.71 of taxable income." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the qualifying-surviving-spouse deduction and joint brackets. With no dependent child the filer is Single, which leaves $21,749.71 of ordinary taxable income rather than $6,036." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the surviving-spouse (joint) standard deduction and kept all ordinary income in the 10% bracket. The filer has no dependent child and must use Single parameters, under which $9,349.71 of ordinary income falls in the 12% bracket." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status with the $33,850 joint age-adjusted deduction. QSS requires a dependent child, so the Single $18,150 deduction applies, and ordinary taxable income is $21,749.71 rather than $6,036.05." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $34,785 taxable income after $24,150 of deductions implies roughly $25,143 of taxable SS (85% of all benefits) instead of $16,846.25. It also gave 0% treatment only to the $928 qualified dividends and taxed the $3,753 of capital gains as ordinary income." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It added the $71 qualified BDC income as extra ordinary income on top of dividends, which also raised taxable Social Security by 85% of that amount. Together that put about $131 more in ordinary taxable income, and 12% of it is the roughly $16 overstatement." +us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used surviving-spouse (joint) deductions, although a filer with no dependent child is Single. It also wrongly concluded that only preferential income remains; even under its own QSS assumption about $6,036 of ordinary income is left taxable." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and omitted the $6,000 senior deduction, and it cited an inapplicable mortgage-interest deduction. Its $3,125 matches tax on roughly $32,400 of taxable income instead of $26,430.71." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law ($10,507 deduction, $5,387 exemption, and the 15% bracket). OBBBA made the TCJA $16,100 + $2,050 deduction and 10%/12% brackets permanent and added the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used 'post-TCJA' expired-law parameters ($8,300 deduction, $5,300 personal exemption, and the 15% bracket). The 2026 law keeps the $16,100 + $2,050 deduction and 10%/12% brackets and adds the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It assumed TCJA sunset and used an $8,300 deduction, a $5,300 exemption, and 10%/15% brackets. OBBBA made TCJA permanent, so the deductions total $24,150 including the $6,000 senior deduction and the ordinary rates are 10%/12%." +us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied reverted pre-TCJA parameters ($10,233 deduction and the 15% bracket) and omitted the $6,000 senior deduction. It miscomputed taxable SS as $24,205 instead of $16,846.25 and taxed the $3,753 of capital gains as ordinary income." +us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It deducted only the $18,150 standard deduction and never took the $6,000 senior deduction, so taxable income came out at $30,478. It also double-counted the farm loss (−$2,254 instead of −$1,127), which understated taxable SS at $15,949." +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It had the correct $48,525 provisional income and $24,150 of deductions but computed taxable Social Security as $24,108.75. The formula gives 0.85 × ($48,525 − $34,000) + $4,500 = $16,846.25, so its taxable income was overstated by $7,262.50." +us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used joint (MFJ/QSS) $32k/$44k Social Security thresholds and a $32,150 surviving-spouse deduction. A widow with no dependent child is Single, with $25k/$34k thresholds, $16,846.25 of taxable SS, and $24,150 of deductions." +us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as surviving spouse with MFJ rates and a $32,200 base deduction, and used joint SS thresholds that produced $9,907 of taxable SS. Single status gives $16,846.25 of taxable SS and $18,150 plus $6,000 of deductions, and it also taxed the $3,753 of capital gains as ordinary income." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used QSS brackets and deduction even though there is no dependent child. It also added the $4,700 tax-exempt IRA distributions to provisional income, computed $25,698 of taxable SS (above the 85% cap), and omitted the $6,000 senior deduction." +us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no computation, and $21,824 is about nine times the correct tax. The correct tax comes from $21,749.71 of ordinary taxable income at 10%/12% with preferential income at 0%, whereas its figure implies taxing nearly all gross income at high marginal rates." us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it." us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base. us,scenario_042,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Invented a ""2026 expanded WI retirement income exclusion up to ~$24,000"" and applied it against the $19,200 IRA plus $4,886 pension; Wisconsin's only retirement subtraction is $5,000 and is unavailable above $15,000 of federal AGI, so all of that income remains in the $29,981.71 Wisconsin AGI. It also used a surviving-spouse standard deduction of ~$13,960 that it said ""fully covers income,"" then submitted $953 — a number its own narrative (income fully absorbed) cannot produce — while never applying the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Treated the ""surviving spouse"" flag as joint filing status and applied a sliding standard deduction of ~$25,728 instead of the single schedule's $12,671.36, cutting taxable income to $7,057 against the correct $16,360.34. It correctly rejected the retirement exclusion and used the $700 + $250 exemptions, but dismissed property taxes as ineligible for the itemized deduction credit without noticing the separate school property tax credit, so the deduction error and the missing $300 credit partly offset at $247." -us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Built its base from 100% of the $29,580 Social Security benefit rather than the federally taxable portion Wisconsin then subtracts, reaching $64,513 and a ~$55,000 taxable income, then abandoned that arithmetic and asserted the homestead property tax credit zeroed the liability. The homestead credit is refundable and phases out well below this household's income; the applicable nonrefundable relief is the school property tax credit, capped at $300, which reduces $584.74 to $284.74 rather than to zero." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own computation — ~$595 of tax less the $300 school property tax credit — lands within $11 of the reference, but it discarded that result and submitted $0 by adding a fictitious $5,000 age-65 retirement subtraction (barred above $15,000 federal AGI in Wisconsin) and imaginary ""$20 + $20"" personal and age exemption credits. Wisconsin's $700 and $250 exemptions are income deductions already embedded in the $16,360.34 taxable income, not credits against the $584.74 of tax." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Used a ~$9,200 standard deduction instead of the sliding-scale $12,671.36 and stacked on a $5,000 retirement subtraction that Wisconsin denies above $15,000 of federal AGI, then computed $731 of tax and overrode it with ""retirement subtractions and credits reduce liability to near zero."" The only nonrefundable credit available is the $300-capped school property tax credit, which leaves $284.74 of the $584.74 tax standing." -us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Phased Wisconsin's single standard deduction down to about $5,000 when the schedule yields $12,671.36 at $29,981.71 of Wisconsin AGI, leaving taxable income of ~$27,500 instead of $16,360.34. That inflated base carries roughly $1,090 of bracket tax, and the $1,050 submitted is that figure barely touched by the $300 school property tax credit it claimed to have applied." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Treated the surviving spouse as married filing jointly and applied a $21,151 MFJ standard deduction against $23,806, after first removing a nonexistent $10,000 age-65 retirement subtraction; the correct single-schedule deduction is $12,671.36 against Wisconsin AGI of $29,981.71. It then subtracted the $700 exemption from the tax as a credit — in Wisconsin the $700 and $250 exemptions reduce income, and the residual $584.74 is offset only by the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Reached a tax of ~$790 from a broadly correct structure, then wiped it to zero with an ""itemized deduction credit"" computed on the $5,011 of real estate taxes. Wisconsin's itemized deduction credit is 5% of qualifying federal itemized deductions (medical, mortgage interest, charitable) in excess of the standard deduction and specifically excludes state and local taxes; property taxes flow instead into the school property tax credit, capped at $300, leaving $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used a fabricated ""~$14,800 including elderly addition"" standard deduction — Wisconsin's schedule gives $12,671.36 at this income and provides no additional elderly standard deduction, its age-65 relief being the $250 exemption — and kept Wisconsin AGI at $33,806 rather than $29,981.71, producing taxable income of $19,006 against the correct $16,360.34. It also omitted the $300 school property tax credit on the $5,011 of real estate taxes." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Declared ""no nonrefundable state credits apply,"" dropping the $300 school property tax credit that Wisconsin grants as 12% of the first $2,500 of the $5,011 in real estate taxes. It compounded that with a flat $13,410 standard deduction, zero exemptions in place of the $700 + $250, and 3.54%/4.65% rates instead of Wisconsin's 3.50%/4.40%, yielding $787.37 where the pre-credit tax is $584.74." -us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Subtracted $5,000 of retirement income that Wisconsin allows only to filers with federal AGI under $15,000, counted a single $700 exemption while dropping the $250 age-65 exemption, and then omitted the $300 school property tax credit entirely. Its $534 is the tax on an understated $15,225 base with no credit applied, where the correct path is $16,360.34 taxable → $584.74 → less $300 = $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Taxed $29,446 — essentially Wisconsin AGI of $29,981.71 relabeled as taxable income — because it never subtracted the $12,671.36 sliding-scale standard deduction or the $950 of exemptions, producing $1,170 of bracket tax against the correct $584.74. Its $300 school property tax credit was applied correctly but to a base nearly double the right one, landing at $820 instead of $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation beyond ""AGI after standard deduction""; $1,137 at Wisconsin's 3.50%/4.40% brackets corresponds to a taxable base near $28,800, i.e. Wisconsin AGI of $29,981.71 taxed essentially intact. It therefore skipped the $12,671.36 standard deduction, the $700 + $250 exemptions that bring taxable income to $16,360.34, and the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $860 requires a taxable base of about $22,550 at Wisconsin's 3.50%/4.40% brackets, roughly $6,200 above the correct $16,360.34, so the standard deduction it claims to have taken was far below Wisconsin's $12,671.36 and the $700 + $250 exemptions were never subtracted. It also made no allowance for the $300 school property tax credit on the $5,011 of real estate taxes." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Landed on $25,235 of Wisconsin taxable income — $8,875 above the correct $16,360.34 — by understating the sliding-scale standard deduction relative to $12,671.36 and omitting the $700 personal and $250 age-65 exemptions. It correctly identified and applied the $300 school property tax credit, but against an inflated $981 of bracket tax rather than the actual $584.74." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Asserted $1,973 with no derivation; at Wisconsin's 3.50%/4.40% brackets that implies a taxable base near $47,800, larger than total federal AGI of $43,637.65. The number is consistent with taxing gross income including the full Social Security benefit with none of the $13,655.95 of Wisconsin subtractions, no $12,671.36 standard deduction, no $950 of exemptions, and no $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Set Wisconsin AGI at $33,735 — $3,754 above the reference $29,981.71 and missing the $71 of BDC income from its own sum — and its $786 implies a taxable base near $20,850 rather than $16,360.34, so the deduction and exemptions it invoked were far short of $12,671.36 plus $950. It also never applied the $300 school property tax credit generated by the $5,011 of real estate taxes." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Submitted $1,348 while its explanation states a net liability of $1,617, breaking the requirement that the explanation support the submitted value. Both figures are two to three times the $584.74 of pre-credit tax, which places the taxable base near $30,000 — Wisconsin AGI untouched by the $12,671.36 standard deduction and the $700 + $250 exemptions that reduce it to $16,360.34." -us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Asserted $1,428 with only a label; at Wisconsin's 3.50%/4.40% brackets that requires a taxable base near $35,450, which exceeds Wisconsin AGI of $29,981.71 itself. No standard deduction of $12,671.36 and no $950 of exemptions were applied, and adding back the $300 school property tax credit it claims to have used pushes the implied base higher still." -us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Claimed that Wisconsin subtracts all federally taxable retirement income — IRA distributions and private pension — for filers 65 or older, collapsing Wisconsin AGI to ~$9,720 and zeroing the tax. Wisconsin has no such provision: the $19,200 of taxable IRA distributions and $4,886 of pension remain fully in Wisconsin AGI of $29,981.71, which after the $12,671.36 standard deduction and $950 of exemptions leaves $16,360.34 taxable and $584.74 of tax." -us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Invoked a ""retirement income subtraction for those 65+"" without computing it; Wisconsin caps that subtraction at $5,000 and denies it entirely above $15,000 of federal AGI, so none applies to this $43,637.65 AGI filer. Its $123.45 corresponds to no step in the Wisconsin schedule, which runs $29,981.71 Wisconsin AGI less $12,671.36 standard deduction and $950 exemptions to $16,360.34 taxable, $584.74 of tax, less the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Asserted that ""pension exclusions"" plus the standard deduction fully offset Wisconsin taxable income. Wisconsin grants no pension exclusion to a filer with $43,637.65 of federal AGI, and the $12,671.36 standard deduction plus $700 + $250 exemptions still leave $16,360.34 of taxable income, yielding $584.74 of tax before the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Claimed deductions and nonrefundable credits reduce the Wisconsin tax to zero without computing either. The only nonrefundable credit here is the school property tax credit, 12% of the first $2,500 of the $5,011 of real estate taxes for a flat $300, which offsets barely half of the $584.74 of tax on $16,360.34 of taxable income." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Identified every correct step — Social Security exclusion, standard deduction, age and personal exemptions, and the school property tax credit on the $5,011 of real estate taxes — but treated that credit as covering the entire tentative tax. Wisconsin's school property tax credit for property taxes is 12% of the first $2,500 only, a hard $300, so it reduces the $584.74 of tax to $284.74 rather than to zero." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Stated that ""no state nonrefundable credit is identified,"" discarding the $300 school property tax credit that the $5,011 of real estate taxes generates. Its $800 also implies a taxable base near $21,200 instead of $16,360.34, meaning it applied a standard deduction well short of $12,671.36 and skipped the $700 + $250 exemptions." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Applied a ""retirement-income subtraction covering up to 24000"" of the taxable IRA and pension income; Wisconsin has no such provision, its retirement subtraction being $5,000 and unavailable above $15,000 of federal AGI. With that $24,086 of retirement income left in the base, Wisconsin AGI is $29,981.71 and taxable income $16,360.34 after the $12,671.36 deduction and $950 exemptions — far from being absorbed." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Omitted the $300 school property tax credit, treating the pre-credit figure as final, while its $430 implies a taxable base near $12,300 — roughly $4,000 below the correct $16,360.34, so its Wisconsin deductions overshot the $12,671.36 standard deduction plus $950 of exemptions. Over-deducting and skipping the credit pushed it $145 above the $284.74 answer." -us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Asserted an ""age-67 retirement-income subtraction of $24,000"" that does not exist in Wisconsin law, whose retirement subtraction is $5,000 and cuts off above $15,000 of federal AGI. Without it, Wisconsin AGI of $29,981.71 sits far above the $12,671.36 standard deduction and $950 of exemptions, leaving $16,360.34 taxable and $584.74 of tax before the $300 school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Claimed ""WI standard deduction and retirement exclusions eliminate tax"" with no computation; Wisconsin offers this $43,637.65-AGI filer no retirement exclusion, and its $12,671.36 standard deduction plus $700 + $250 exemptions leave $16,360.34 of taxable income. That base produces $584.74 of tax, of which only $300 is erased by the school property tax credit." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Computed a $366 school property tax credit as a percentage of the full $5,011 of real estate taxes; Wisconsin caps the credit at 12% of the first $2,500, a flat $300. Its standard deduction of $10,683 also falls $1,988 short of the $12,671.36 schedule amount and its Wisconsin AGI of $32,680 exceeds $29,981.71, so taxable income came in at $21,047 rather than $16,360.34." -us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Correctly capped the school property tax credit at $300 and used the $700 + $250 exemptions, but overstated Wisconsin AGI by $2,698 at $32,680 and understated the sliding-scale standard deduction, leaving taxable income of $19,510 against the correct $16,360.34. That $3,150 excess base produced $723 of bracket tax instead of $584.74, carrying straight through to $423 versus $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Omitted the $300 school property tax credit and the $700 + $250 exemptions entirely, and used a projected flat $13,900 standard deduction on Wisconsin AGI of $33,806 instead of $12,671.36 on $29,981.71, leaving $19,906 taxable against $16,360.34. Its 3.54%/4.65% rates are also wrong — Wisconsin's bottom brackets are 3.50% and 4.40%." -us,scenario_042,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Assembled the right structure — sliding-scale standard deduction, $700 personal and $250 elderly exemptions, $300 school property tax credit — but carried Wisconsin income of $32,679 rather than $29,981.71 and a $11,351 deduction rather than $12,671.36, so taxable income came to $20,378 instead of $16,360.34. It also applied a 4.65% upper rate in place of Wisconsin's 4.40%, pushing pre-credit tax to $779 against $584.74." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so nothing substantive was submitted to score. The required derivation runs Wisconsin AGI of $29,981.71 less the $12,671.36 standard deduction and $950 of exemptions to $16,360.34 of taxable income, $584.74 of bracket tax, less the $300 school property tax credit, giving $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Got the framework right, including the $700 + $250 exemptions and the school property tax credit correctly capped at $300 (12% of the first $2,500), but overstated Wisconsin income at $32,609 versus $29,981.71 and phased the standard deduction down to $12,023 instead of $12,671.36. The resulting $19,636 taxable income exceeds $16,360.34 by $3,276, adding about $145 of tax and producing $429.26 instead of $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Stacked two fabricated Wisconsin provisions: a $24,000 retirement income exclusion applied to the $24,086 of IRA and pension income, and a $24,000 ""surviving spouse"" standard deduction. Wisconsin's retirement subtraction is $5,000 and is barred above $15,000 of federal AGI, and the single sliding-scale standard deduction here is $12,671.36, leaving $16,360.34 taxable and $584.74 of tax before the $300 credit." -us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Conflated the municipal school levy tax credit with the income-tax school property tax credit, computing 12% of the full $5,011 and then phasing it down by income to ~$276; the income-tax credit is 12% of the first $2,500 only, an unphased flat $300. It also used a $13,300 MFJ standard deduction against Wisconsin AGI of $33,806 rather than the single $12,671.36 against $29,981.71, overstating taxable income by $4,146." -us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Computed $297 of tax and then added a $307 ""itemized deduction credit"" based on real estate taxes to reach $417.88 — credits subtract from tax, and Wisconsin's itemized deduction credit excludes state and local taxes such as the $5,011 of property tax. It also used a $10,430 head-of-household standard deduction rather than the single $12,671.36, skipped the $700 + $250 exemptions, and never applied the $300 school property tax credit that takes $584.74 down to $284.74." -us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted $1,830 with no derivation; at Wisconsin's 3.50%/4.40% brackets that implies a taxable base near $44,600, which matches total federal AGI of $43,637.65 rather than Wisconsin taxable income of $16,360.34. The number is consistent with skipping the $13,655.95 of Wisconsin subtractions, the $12,671.36 standard deduction, the $950 of exemptions, and the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"The model's own reasoning cut WI income to about $9,700 and said the standard deduction fully covers it, yet it submitted $953 without any computation connecting the two. The engine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI. The $12,671.36 standard deduction and $950 of exemptions bring that to $16,360.34 taxable and $584.74 of tax, and the model never applied the $300 school property tax credit that follows." +us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"It gave the surviving spouse the married-joint sliding-scale standard deduction (about $25,728). Wisconsin has no surviving-spouse status, so the engine uses the single deduction of $12,671.36 on $29,981.71 of WI AGI, which leaves $16,360.34 taxable and $584.74 of tax. It also said property taxes earn no credit, which leaves out the $300 school property tax credit (12% of the first $2,500 of property tax)." +us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"It left the full $29,580 of Social Security in Wisconsin taxable income (about $55,000), although Wisconsin subtracts all $13,655.95 of taxable Social Security. It then wiped out the resulting ~$2,500 tax with a homestead credit. The homestead credit is refundable and income-limited, so the only nonrefundable offset is the $300 school property tax credit against $584.74 of tax." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement income subtraction, which Wisconsin allows only when WI AGI is under $15,000 for a single filer, and treated the $700/$250 exemptions as $20 credits instead of deductions. Its own figures ($595 − $300 − $40 = $255) also do not reach its reported $0. The engine taxes $16,360.34 for $584.74 and subtracts the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement subtraction that is barred once WI AGI exceeds $15,000 (single) and used a ~$9,200 standard deduction instead of $12,671.36. It then dropped its own $731 tax estimate to zero by citing unspecified real-estate and mortgage credit effects. The only nonrefundable credit here is the $300 school property tax credit, which takes $584.74 of tax to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"It used a sliding-scale standard deduction of only about $5,000 and kept the $3,753 of capital gains in WI income, which produced about $27,500 of taxable income. At WI AGI of $29,981.71, the single standard deduction is $12,671.36, and with $950 of exemptions that leaves $16,360.34 taxable and $584.74 of tax before the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67+ retirement income exclusion to the IRA and pension income, which dropped Wisconsin income to about $8,600 and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security. That leaves $29,981.71 of WI AGI, $16,360.34 of taxable income, $584.74 of tax, and $284.74 after the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a married-joint Wisconsin filer, which gave a $10,000 retirement subtraction and a ~$21,151 joint standard deduction. Wisconsin files this person as single: the $5,000 retirement subtraction is barred above $15,000 of WI AGI, and the single deduction is $12,671.36. It also treated the $700 exemption as a tax credit, which zeroed a tax the engine computes as $584.74 less the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"It computed about $790 of tax and then erased it with an itemized deduction credit on real estate taxes. Wisconsin's itemized deduction credit excludes property taxes, and the $350 of medical expenses produces no credit, so the only nonrefundable credit is the flat $300 school property tax credit. It also kept the capital gains in WI AGI, which overstates the base above the engine's $29,981.71." +us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gain distributions in WI AGI ($33,735, versus the engine's $29,981.71). It also never subtracted the $300 school property tax credit (12% of the first $2,500 of the $5,011 in real estate taxes). Its $790 is therefore a pre-credit tax on an overstated base." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"It started from $33,806 of WI income, which includes $3,824 of capital gain and BDC income that is absent from the engine's $29,981.71 of WI AGI. It used a lumped ~$14,800 deduction instead of the $12,671.36 standard deduction plus $950 of exemptions, and it never claimed the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"It applied a $13,410 standard deduction with no phase-down to $33,806 of WI income and skipped the $700 + $250 exemptions. It used 3.54%/4.65% rates instead of 3.5%/4.4%. It also said no nonrefundable credits apply, which leaves out the $300 school property tax credit that the engine subtracts from $584.74." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"It took a $5,000 retirement income subtraction that Wisconsin allows only when WI AGI is below $15,000 (single) and left out the $250 age-65 exemption. It also never applied the $300 school property tax credit, so its $534 replaces the correct $584.74 − $300." +us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"It used an $8,837 standard deduction on $33,806 of WI income and took no $700/$250 exemptions. That puts taxable income at $24,969, versus the engine's $16,360.34, and it also left out the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"Its $29,446 of taxable income leaves the $12,671.36 sliding-scale standard deduction and $950 of exemptions essentially unapplied, which yields about $1,170 of tax against the engine's $584.74 on $16,360.34. It also subtracted a nonexistent $50 personal exemption credit alongside the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,137 matches taxing about $29,000 of Wisconsin income at 3.5%/4.4% with no standard deduction, no exemptions, and no school property tax credit. The engine reduces $29,981.71 of WI AGI by $12,671.36 and $950 to $16,360.34, taxes that at $584.74, and subtracts the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"It applied only the Social Security subtraction and a standard deduction. Its $860 matches about $22,600 of taxable income, which means no $950 of exemptions and a base that still includes the capital gains. It also left out the $300 school property tax credit that brings the engine's $584.74 down to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"It reached $25,235 of taxable income by taking only about $8,571 of deductions from $33,806 of Wisconsin income. The engine's WI AGI is $29,981.71, and it subtracts a $12,671.36 standard deduction plus $950 of exemptions to get $16,360.34 taxable. The model applied the $300 school property tax credit correctly, but against an overstated $981 of tax." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,973 matches taxing about $48,000 at 3.5%/4.4%, which is roughly the full federal AGI with taxable Social Security left in and no deduction. Wisconsin subtracts the $13,655.95 of taxable Social Security, the $12,671.36 standard deduction, and $950 of exemptions, leaving $16,360.34 taxed at $584.74 before the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gain distributions in WI AGI ($33,735, versus the engine's $29,981.71) and never applied the $300 school property tax credit. Its $786 is therefore a tax before that credit, on an overstated base." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"Its explanation states $1,617 while it submitted $1,348, and both figures reflect a tax base far above the engine's $16,360.34 of taxable income. It also relied on a nonexistent 'senior credit'. The engine's $584.74 of tax less the $300 school property tax credit is $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"It gave no computation. $1,428 after a $300 school property tax credit implies about $1,728 of tax on roughly $42,000 of taxable income, which matches leaving taxable Social Security in Wisconsin income or skipping the standard deduction. Wisconsin subtracts the $13,655.95 of taxable Social Security, then the $12,671.36 deduction and $950 of exemptions, leaving $16,360.34 taxable." +us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"It subtracted all $24,086 of IRA and pension income as senior retirement income, which cut WI AGI to about $9,720 and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security and keeps the IRA and pension income in $29,981.71 of WI AGI. That produces $584.74 of tax, less the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"It returned a placeholder-style $123.45 with no computation behind it. The engine takes $29,981.71 of WI AGI, subtracts the $12,671.36 standard deduction and $950 of exemptions to get $16,360.34 taxable, computes $584.74 of tax, and subtracts the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"It zeroed the tax by applying pension and retirement exclusions to the IRA and pension income. PolicyEngine applies no retirement exclusion here: it taxes $29,981.71 of WI AGI, less the $12,671.36 deduction and $950 of exemptions, at $584.74 and then subtracts the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"It claimed the retirement income is fully offset without naming any exclusion or deduction that does so. The standard deduction ($12,671.36) and exemptions ($950) absorb only $13,621.36 of the $29,981.71 of WI AGI, which leaves $16,360.34 taxable, $584.74 of tax, and $284.74 after the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"It claimed the school property tax credit on $5,011 of real estate taxes fully offsets the tentative tax. That credit is capped at $300 (12% of the first $2,500 of property tax), while the tentative tax is $584.74 on $16,360.34 of taxable income, which leaves $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"It said no nonrefundable state credit applies, which leaves out the $300 school property tax credit on the $5,011 of real estate taxes. It also taxed Wisconsin income that still includes the capital gain distributions, so its ~$800 is a tax before that credit on a base above the engine's $16,360.34." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67+ retirement-income subtraction to the taxable IRA and pension income, which cut Wisconsin income below the deductions and the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI, $584.74 of tax, and $284.74 after the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"Its explanation uses only the Social Security exclusion, deductions, and exemptions, with no school property tax credit. Its $430 is therefore a tentative tax below the engine's $584.74 on $16,360.34 of taxable income, and it skips the required $300 school property tax credit that lowers the tax to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"It applied a $24,000 age-67 retirement-income subtraction to the IRA and pension income and reported $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, taxes $16,360.34 (after the $12,671.36 deduction and $950 of exemptions) at $584.74, and subtracts the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"It stopped at a tax before credits and never applied the $300 school property tax credit on the $5,011 of real estate taxes. Its base still includes the capital gain distributions, so its $770 is above the engine's $584.74 even before that credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"It applied an age-qualified retirement-income subtraction to the $24,086 of IRA and pension income, which let the standard deduction absorb the remainder and gave $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, leaving $29,981.71 of WI AGI and $284.74 of tax after the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"It used a retirement deduction on the IRA and pension income, together with the standard deduction and credits, to eliminate the tax. PolicyEngine applies no retirement subtraction here: it taxes $16,360.34 at $584.74 and subtracts only the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"It eliminated the tax with retirement exclusions on the IRA and pension income. PolicyEngine subtracts only the $13,655.95 of taxable Social Security, so $29,981.71 of WI AGI remains, and after the $12,671.36 deduction, $950 of exemptions, and the $300 school property tax credit the tax is $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"It kept the $3,753 of capital gains (after a 30% exclusion) and the $71 of BDC income in WI AGI ($32,680, versus $29,981.71) and used a $10,683 standard deduction instead of $12,671.36. It also set the school property tax credit at $366, although the credit is capped at $300 (12% of the first $2,500 of property tax)." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"Its $32,680 of WI AGI keeps the non-Schedule-D capital gains (after a 30% exclusion) and the BDC income, which are not in the engine's $43,637.65 federal AGI. The engine's WI AGI is $29,981.71, which with the $12,671.36 deduction and $950 of exemptions gives $16,360.34 taxable and $584.74 of tax before the $300 credit that the model applied correctly." +us,scenario_042,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"It built federal AGI of $50,713 with the capital gain distributions included and carried $32,680 of WI income forward, against the engine's $43,637.65 of AGI and $29,981.71 of WI AGI. That raised taxable income to $19,408, versus $16,360.34, and tax to $719, versus $584.74, before the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"It applied a $13,900 standard deduction with no phase-down, took no $950 of exemptions, and used 3.54%/4.65% rates instead of 3.5%/4.4%, all on $33,806 of WI AGI that includes the capital gains. It also left out the $300 school property tax credit that reduces the engine's $584.74 to $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"It overstated WI income at $32,679 by including the capital gain distributions, used an $11,351 deduction instead of $12,671.36, and taxed the second bracket at 4.65% instead of 4.4%. That produced about $779 of tax before the $300 credit, versus the engine's $584.74 on $16,360.34." +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the engine's $284.74." +us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"It computed $24,108.75 of taxable Social Security from an inflated $57,843.75 federal AGI and kept the $3,753 of capital gains (less 30%) in WI income ($32,609.10). The engine's taxable Social Security is $13,655.95 and its WI AGI is $29,981.71, so taxable income is $16,360.34 and tax is $584.74 before the $300 credit." +us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"It excluded $24,000 of IRA and pension income as senior retirement income and also used a $24,000 surviving-spouse standard deduction, which drove the tax to $0. PolicyEngine subtracts only the $13,655.95 of taxable Social Security and applies the single deduction of $12,671.36 plus $950 of exemptions, which yields $584.74 of tax and $284.74 after the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"It treated the surviving spouse as a married-joint filer for the standard deduction, skipped the $950 of exemptions, and kept the capital gains in WI AGI ($33,806). It then taxed $20,506 entirely at 3.5%, although the first bracket ends near $15,000. It also invented an income phase-out that cut the school property tax credit to $276 instead of the flat $300 (12% of the first $2,500)." +us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"Its arithmetic is incoherent: it computed $297 of tax before credits, subtracted roughly $339 of credits, and reported a higher $417.88. It also used a head-of-household deduction and a nonexistent 'personal tax credit'. The engine taxes $16,360.34 for $584.74 and subtracts only the $300 school property tax credit." +us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"It gave no computation. Its $1,830 matches about $44,000 of taxable income at 3.5%/4.4%, which is taxing roughly the full federal AGI without subtracting taxable Social Security or applying deductions. The engine subtracts the $13,655.95 of taxable Social Security, the $12,671.36 deduction, and $950 of exemptions, then the $300 school property tax credit, for $284.74." us,scenario_042,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model understated Wisconsin homestead-credit household income by excluding substantial Social Security, taxable IRA distributions, pension income, dividends, interest, and capital gains, then applied an assumed 2024 maximum-credit formula instead of the 2026 eligibility computation. The complete computation produces no Wisconsin refundable credit." us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric state_refundable_credits output or explanation, violating the required structured-output contract." us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model applied the childless EITC phase-in (7.65% × $4,073) without checking IRC §32(c)(1)(A)(ii)(II), which limits the no-qualifying-children EITC to filers at least 25 and under 65 at the close of the tax year; the head is 66, so the credit is $0 for 2026. Its reasoning tested only the earned-income and AGI phase-out thresholds and never the upper age bar, treating the ARPA-2021 suspension of that bar as if it still applied in 2026. With EITC out, all five refundable components (EITC, refundable AOTC, refundable CTC, Recovery Rebate, refundable payroll tax credit) are $0." @@ -3346,162 +3721,166 @@ us,scenario_043,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibi us,scenario_043,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_043,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_043,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_043,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It correctly identified the FAMLI employee premium but applied the 2023–2025 rate of 0.45% instead of the 2026 employee share of 0.44% (half of the 0.88% total), giving $18.40 rather than $17.99, and then submitted $315.14 — a figure matching neither its FICA-only total of $312.73 nor its stated FICA-plus-FAMLI total of $331.14. Its final value corresponds to no derivation it performed, so both the FAMLI rate and the submission itself are wrong." -us,scenario_043,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It asserted there was 'no modeled mandatory Colorado employee payroll tax' and stopped at the 7.65% federal FICA sum. Colorado's FAMLI program charges a mandatory employee premium of 0.44% of wages in 2026, which adds $17.99 on the $4,088.15 wage base; omitting that component left it $17.99 short of $330.73." -us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It declared that 'no mandatory state payroll taxes apply in Colorado,' dropping the 0.44% employee FAMLI premium of $17.99, and additionally rounded its $312.74 FICA figure to a whole $313 rather than reporting cents. It also cited a stale $168,600 Social Security wage base, which is the 2024 figure rather than the 2026 one (harmless here only because wages are far below any base)." -us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It computed the FAMLI employee premium at 0.45% ($18.40) and then deliberately discarded it as 'state UI-like,' but FAMLI is an employee-paid mandatory state payroll premium, not an employer unemployment-insurance tax, and it belongs in this output. It also used the superseded 0.45% rate rather than the 2026 employee share of 0.44%, which yields $17.99 on the $4,088.15 base." -us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It stated 'Colorado has no mandatory employee payroll tax applicable here,' which is wrong: the FAMLI employee premium of 0.44% of wages applies in 2026 and adds $17.99 to the $312.73 of employee Social Security and Medicare tax." -us,scenario_043,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It computed only federal employee FICA at 7.65% of $4,088 and never considered any state component, missing Colorado's mandatory 0.44% FAMLI employee premium of $17.99. It also rounded to a whole $313 instead of carrying cents." -us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded 'Colorado has no mandatory employee-side state payroll tax,' omitting the FAMLI premium, whose 0.44% employee share on the $4,088.15 wage base is $17.99. Its wage base and both FICA rates were right; the entire $17.99 shortfall is the missing state paid-leave premium." -us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It ruled out a Colorado employee payroll tax by checking only for a state disability or UI employee contribution, missing that Colorado funds paid family and medical leave through FAMLI, whose 2026 employee premium of 0.44% of wages is $17.99 here. That omission is exactly the gap between its $312.73 and the $330.73 total." -us,scenario_043,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It asserted 'Colorado has no employee state payroll tax' and reported only the 7.65% FICA total. The Colorado FAMLI employee premium of 0.44% on the $4,088.15 wage base adds $17.99, producing $330.73." -us,scenario_043,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It correctly included the Colorado FAMLI employee premium but priced it at $24.53, which is 0.6% of wages, rather than the 2026 employee share of 0.44% (half the 0.88% total premium), which is $17.99 on the $4,088.15 wage base. Overstating the FAMLI rate by 0.16 points is the entire $6.53 overshoot above $330.73." -us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $8,389 of employer-sponsored insurance premiums from the $4,088 of wages and concluded the FICA and FAMLI wage base was zero. Section 125 exclusions cannot reduce the FICA wage base below zero or carry over from other income, and the household's premium figure is not an employee pre-tax payroll deduction against these wages; the base stays $4,088.15, yielding $253.47 Social Security, $59.28 Medicare, and $17.99 FAMLI." -us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only the combined 7.65% federal employee FICA rate to $4,088 and included no state component, omitting Colorado's mandatory FAMLI employee premium of 0.44% ($17.99)." -us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"Its answer covers only employee Social Security ($253.46) and Medicare ($59.28) and omits the Colorado FAMLI employee premium, which at the 2026 employee rate of 0.44% on the $4,088.15 wage base is $17.99." -us,scenario_043,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It limited the output to 'Social Security and Medicare' on $4,088 of wages, excluding the mandatory employee state payroll tax the question explicitly asks for: Colorado's FAMLI premium at 0.44% of wages, or $17.99." -us,scenario_043,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It summed only the 6.2% Social Security and 1.45% Medicare employee taxes and never added a state component, missing Colorado's 0.44% FAMLI employee premium of $17.99 on the $4,088.15 wage base." -us,scenario_043,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It applied the 7.65% federal employee rate and considered only whether Additional Medicare Tax applied, never testing for a mandatory state payroll tax; Colorado's FAMLI employee premium of 0.44% adds $17.99. It also rounded $312.73 to a whole $313." -us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It stated that 'no state employee payroll tax applies in Colorado,' dropping the FAMLI employee premium of 0.44% of wages ($17.99), and then truncated its own 7.65% computation of $312.73 to a whole $312." -us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It gave no substantive derivation, declaring the wages 'not subject/insufficient' for employee-side payroll tax and returning zero. The $4,088.15 of wages is fully subject with no floor or exemption: 6.2% Social Security is $253.47, 1.45% Medicare is $59.28, and the 0.44% Colorado FAMLI employee premium is $17.99, totaling $330.73; a zero answer is consistent only with treating wages as entirely outside the payroll tax base." -us,scenario_043,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It explicitly excluded any 'mandatory Colorado employee payroll tax' from the total. Colorado's FAMLI program imposes exactly that: a 0.44% employee premium in 2026, worth $17.99 on the $4,088.15 wage base, which the requested output includes." -us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It computed only the 7.65% federal employee FICA on $4,088 and reported no state component, missing Colorado's mandatory FAMLI employee premium of 0.44%, or $17.99 on the $4,088.15 wage base." -us,scenario_043,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It stopped at 6.2% plus 1.45% of wages and omitted Colorado's 0.44% FAMLI employee premium of $17.99, then rounded its $312.73 to a whole $313." -us,scenario_043,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It correctly kept elective 401(k) deferrals in the FICA base but concluded there is 'no Colorado employee payroll tax,' omitting the FAMLI employee premium of 0.44% of wages, which is $17.99 on the $4,088.15 base and the whole of its shortfall from $330.73." -us,scenario_043,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It asserted 'no Colorado employee payroll tax' after computing 7.65% federal FICA. Colorado's paid family and medical leave program collects a mandatory 0.44% employee premium in 2026, adding $17.99 to reach $330.73." -us,scenario_043,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It concluded that 'no Additional Medicare Tax or mandatory state payroll taxes apply,' but Colorado's FAMLI employee premium is a mandatory state payroll tax of 0.44% of wages, or $17.99 here. It compounded the omission by rounding $312.732 to a whole $313." -us,scenario_043,payroll_tax,inkling,llm_error,state_local_rule,False,"Its check covered only the Social Security wage base and the Additional Medicare Tax threshold, never a state component; Colorado's 0.44% FAMLI employee premium of $17.99 on the $4,088.15 wage base is the missing piece of $330.73." -us,scenario_043,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It stated 'Colorado has no mandatory employee state payroll tax,' omitting the FAMLI employee premium of 0.44% of wages ($17.99), and rounded its federal FICA subtotal of $312.73 to a whole $313." -us,scenario_043,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It restricted the output to employee Social Security and Medicare on $4,088 and evaluated no state payroll tax, missing Colorado's mandatory FAMLI employee premium of 0.44%, worth $17.99 on the $4,088.15 wage base." -us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It concluded 'Colorado does not have a mandatory employee state payroll tax,' dropping the 0.44% FAMLI employee premium of $17.99 that Colorado collects from wages in 2026." -us,scenario_043,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It ruled out a Colorado component by testing for a 'mandatory employee payroll income tax,' the wrong test: Colorado's employee-side obligation is the FAMLI paid-leave premium at 0.44% of wages, which adds $17.99 to its $312.73 FICA total." +us,scenario_043,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly counted FICA ($312.73) plus the Colorado FAMLI employee premium and reached $331.14 in its reasoning. That total used 0.45% instead of the 2026 employee rate of 0.44%. It then submitted $315.14, a number its own calculation does not support, so the answer is a transcription error on top of the small rate error." +us,scenario_043,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model assumed no mandatory Colorado employee payroll tax and submitted FICA only ($312.73). It left out the Colorado FAMLI employee premium of 0.44% of wages ($17.99), which the payroll_tax output includes." +us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model stated that Colorado has no mandatory state payroll tax. It missed the Colorado FAMLI employee premium (0.44% of wages, $17.99) and submitted FICA only, rounded to $313." +us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified the Colorado FAMLI employee premium and computed it, but then excluded it on the grounds that it is like unemployment insurance. FAMLI is a mandatory employee-side state payroll premium (0.44% of wages, $17.99) and belongs in payroll_tax. The model submitted FICA only ($312.73)." +us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model stated that Colorado has no applicable mandatory employee payroll tax. It left out the FAMLI employee premium of 0.44% of wages ($17.99) and submitted FICA only ($312.73). +us,scenario_043,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model computed only employee FICA on $4,088 and rounded to $313. It never added the mandatory Colorado FAMLI employee premium (0.44% of wages, $17.99)." +us,scenario_043,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,The model treated employee payroll tax as 7.65% FICA only ($312.73). It left out the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated that Colorado has no mandatory employee-side state payroll tax. Colorado's FAMLI program charges employees a premium of 0.44% of wages ($17.99), which the model omitted from its $312.73 FICA-only total." +us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model said Colorado has no employee state disability or leave contribution. It missed the mandatory Colorado FAMLI paid-leave premium (0.44% of wages, $17.99) and submitted FICA only ($312.73)." +us,scenario_043,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,The model asserted that Colorado has no employee state payroll tax and submitted 7.65% FICA only ($312.73). It omitted the Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly included the Colorado FAMLI employee premium but priced it at $24.53, which is 0.6% of wages. The 2026 employee share is half of the 0.88% total premium, or 0.44% ($17.99), so the model overstated FAMLI by about $6.54." +us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the full $8,389 employer-sponsored insurance premium from wages and reduced the FICA and FAMLI wage base to zero. The listed premiums are not a pre-tax employee salary reduction that shrinks the $4,088 of taxable wages. Applying FICA plus the 0.44% FAMLI premium to the full wages gives $330.73." +us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied only 7.65% FICA to $4,088 of wages. It left out the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99)." +us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,The model summed only employee Social Security and Medicare tax ($312.74). It never added the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model counted only Social Security and Medicare ($312.73). It omitted the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99), which the payroll_tax output includes." +us,scenario_043,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,The model computed employee FICA only ($312.73). It never included the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,glm-5.3,llm_error,state_local_rule,False,The model applied 7.65% FICA to wages and rounded to $313. It omitted the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model stated that no state employee payroll tax applies in Colorado, missing the FAMLI employee premium of 0.44% of wages ($17.99). It also truncated its FICA-only figure to $312." +us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $0 and claimed the wages were not subject to employee payroll taxes, without computing anything. The $4,088 of wages is fully subject to 6.2% Social Security, 1.45% Medicare and the 0.44% Colorado FAMLI premium, which together total $330.73." +us,scenario_043,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,The model applied 7.65% FICA and explicitly left out any mandatory Colorado employee payroll tax. It missed the FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model computed only 7.65% FICA on $4,088 ($312.73). It omitted the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99)." +us,scenario_043,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model computed only Social Security and Medicare and rounded to $313. It never added the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,grok-4.5,llm_error,state_local_rule,False,The model correctly kept 401(k) deferrals in the FICA base but asserted that Colorado has no employee payroll tax. It omitted the FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,grok-4.6,llm_error,state_local_rule,False,The model stated that no Colorado employee payroll tax applies and submitted 7.65% FICA only ($312.73). It missed the mandatory FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model stated that no mandatory state payroll taxes apply and rounded FICA to $313. It omitted the Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,inkling,llm_error,state_local_rule,False,The model computed employee FICA only ($312.73). It never included the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model stated that Colorado has no mandatory employee state payroll tax and rounded FICA to $313. It missed the FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model included only Social Security and Medicare ($312.73). It omitted the mandatory Colorado FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model asserted that Colorado has no mandatory employee state payroll tax and submitted FICA only ($312.73). It missed the FAMLI employee premium of 0.44% of wages ($17.99). +us,scenario_043,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,The model stated that Colorado imposes no mandatory employee payroll tax and submitted FICA only ($312.73). It omitted the Colorado FAMLI employee premium of 0.44% of wages ($17.99). us,scenario_043,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"Its net-income derivation was correct (deductions exceed income, so the benefit equals the maximum allotment), but it used $291/month — the FY2024 one-person maximum — for a 2026 calendar year. The applicable rates are $298 through September 2026 and $304.68 from October 2026, so the flat $291 × 12 understates the annual benefit by $104.04." -us,scenario_043,snap,claude-fable-5.1,llm_error,period_annualization,False,"It identified the correct FY2026 one-person maximum of $298 but multiplied it flat across all 12 calendar months; the allotment table resets each October 1, so October–December 2026 pay the FY2027 rate of $304.68, which is the entire $20.04 shortfall. It also asserted roughly $830/month of SSI income that the household facts do not list." -us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied the non-elderly $2,250 resource limit and declared $2,800 in bank assets disqualifying, after its own text had already found the household income-eligible. The head is 66, so the higher elderly/disabled resource limit applies, and Colorado's broad-based categorical eligibility through TANF non-cash assistance removes the asset test altogether — the household receives the full maximum allotment." -us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Its own derivation reached net income of $0 and a maximum allotment of about $300/month for $3,600/year, and it then discarded that result to submit an unexplained $200/month. The rules reasoning was sound; the failure is substituting an arbitrary allotment for the maximum it had just computed ($298 for January–September 2026, $304.68 thereafter)." -us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"After computing near-zero net income and a $292 maximum allotment, it cut the benefit to $192/month by invoking the SNAP minimum benefit. The minimum benefit is a floor of roughly $24 for one- and two-person households, never a ceiling on the maximum allotment, and it has no role when net income is zero — the household draws $298/month through September 2026 and $304.68 after." -us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"It reasoned correctly to zero net income and the maximum allotment, then priced that maximum at about $203/month. No contiguous-US one-person SNAP maximum is near $203; the FY2026 value is $298, rising to $304.68 in October 2026, so the entire $1,160 error is the allotment parameter." -us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its whole chain was right — 0.29 gross-income-to-poverty ratio, elderly resource limit, ESI premiums plus other and OTC medical costs above the $35 threshold driving net income below zero — but it priced the one-person maximum at $292, the FY2025 figure. FY2026 pays $298 through September 2026 and $304.68 for October–December." -us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It set the one-person maximum allotment at $328/month, well above the $298 contiguous-US FY2026 value, overstating the benefit by $340. It also garbled the elderly medical deduction as medical costs less a '$420 standard exclusion' when the threshold is $35/month, though that error was immaterial once net income hit zero." -us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly cleared the elderly resource limit and drove net income to zero, then used $301/month as the 2026 one-person maximum. That figure matches no SNAP allotment table: calendar 2026 pays $298 for January–September and $304.68 for October–December." -us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It explicitly labeled and applied the FY2025 maximum allotment of $292 to a 2026 calendar year. The correct rates are $298 (January–September 2026) and $304.68 (October–December 2026), a $92.04 annual difference." -us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,It reached the right structural answer — maximum allotment for a one-person elderly household with net income below the benefit-reduction threshold — but priced it at the FY2025 rate of $292/month. Calendar 2026 pays $298 through September and the uprated $304.68 from October 1. -us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed gross monthly income of $387.33, applied the elderly medical deduction above $35/month, and correctly reached zero net income and the maximum allotment, but used the FY2025 value of $292/month. FY2026 is $298, and October–December 2026 pay $304.68." -us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It asserted that income and assets disqualify the household without any computation. Gross income of $387/month is about 29% of the poverty guideline against a 130% test, $2,800 sits under the elderly/disabled resource limit, and Colorado's broad-based categorical eligibility through TANF non-cash assistance waives the asset test — the household receives the full maximum allotment." -us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its deduction stack was correct (medical costs of $13,389 far exceed the $35/month threshold, driving net income to $0), but it used $291/month — the FY2024 one-person maximum. The 2026 calendar year pays $298 for nine months and $304.68 for three." -us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly identified categorical eligibility and zero net income, then submitted a round $3,500, which corresponds to $291.67/month and no published allotment. It never retrieved the maximum allotment value: $298/month through September 2026 and $304.68 from October." -us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 with a bare restatement of the answer and no rule or arithmetic. A one-person household with $387/month gross income, $2,800 in resources, an elderly member, and Colorado broad-based categorical eligibility passes every test and receives the maximum allotment of $298–$304.68/month." -us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly netted $340.67 in wages plus $46.08 in pension against the standard, earned-income, and $381.67 excess-medical deductions to reach zero net income, then applied the FY2025 maximum of $292/month. FY2026 pays $298, and the October 2026 uprating raises it to $304.68." -us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It reached zero net countable income by the correct route but priced the one-person maximum at $292/month, the FY2025 figure. Calendar 2026 pays $298 for January–September and $304.68 for October–December." -us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly found the elderly household's medical expenses exceed income and net income is zero, then multiplied the stale FY2025 maximum of $292/month by 12. The applicable rates are $298 and, from October 2026, $304.68." -us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"It correctly waived the gross income test for the elderly household and drove net income to zero, then guessed $302/month as the one-person maximum. No allotment table lists $302: 2026 pays $298 for nine months and $304.68 for three." -us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,"It summed $13,389 in medical costs against the $35/month threshold and correctly reached zero net income, then applied roughly $305/month across all 12 months. That rate holds only from October 1, 2026 ($304.68); January–September 2026 pay the FY2026 maximum of $298." -us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It offered no derivation and submitted $1,276, about $106/month, which matches no maximum allotment and no benefit-formula result. With net income at zero after the elderly medical deduction, the benefit is the full maximum: $298/month through September 2026 and $304.68 after." -us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It declared the household not SNAP-eligible for lack of 'qualifying low-income indicators' despite $4,648 of annual income for a single person. Gross income at 29% of the poverty guideline clears the 130% test, the elderly resource limit and Colorado categorical eligibility clear the asset test, and the benefit is the maximum allotment." -us,scenario_043,snap,gpt-5.5,llm_error,period_annualization,False,"It named the correct FY2026 one-person maximum of $298 and then held it constant for all 12 months of calendar 2026. The maximum allotment resets on October 1, so the last three months of 2026 pay $304.68 — the exact $20.04 gap between its answer and the reference." -us,scenario_043,snap,gpt-5.6-luna,llm_error,period_annualization,False,"Its eligibility and deduction reasoning was correct and it used the right FY2026 maximum of $298, but it applied that single rate to all 12 calendar months. October–December 2026 fall in FY2027 and pay $304.68/month." -us,scenario_043,snap,gpt-5.6-sol,llm_error,period_annualization,False,"It correctly cleared the resource test and drove net income to zero for the $298 FY2026 maximum, then annualized that rate flat across calendar 2026. The October 1, 2026 uprating to $304.68 applies to the final three months." -us,scenario_043,snap,gpt-5.6-terra,llm_error,period_annualization,False,"It applied the earned-income, standard, and excess-medical deductions correctly to reach zero net SNAP income and the $298 monthly maximum, then multiplied by 12 without the fiscal-year change. October–December 2026 pay $304.68/month under FY2027 parameters." -us,scenario_043,snap,gpt-6-astra,llm_error,period_annualization,False,"It correctly found the resource test met and deductible medical costs eliminating net income, giving the one-person maximum of $298/month, but treated that rate as constant for the whole tax year. The allotment steps up to $304.68 on October 1, 2026." -us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"It answered $0 on the claim that assets exceed typical limits. The head is 66, so the elderly/disabled resource limit governs and $2,800 sits well under it, and Colorado's broad-based categorical eligibility through TANF non-cash assistance eliminates the asset test outright; with zero net income the household draws the full maximum allotment." -us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly passed the gross, net, and elderly resource tests and used the ESI premiums plus other and OTC medical costs to zero out net income, then priced the maximum at the FY2025 rate of $292/month. FY2026 is $298, with $304.68 from October 2026." -us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"Its deduction stack and eligibility findings were correct, but it applied the FY2025 one-person maximum of $292 for all of calendar 2026. The correct blend is $298 for January–September and $304.68 for October–December." -us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly cleared the 130% gross test and the $4,250-class elderly resource limit and reached zero net income, then rounded the one-person maximum to '~$300' for all 12 months. The actual parameters are $298 through September 2026 and $304.68 thereafter." -us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"It applied the elderly medical deduction on $13,389 of costs above the $35/month threshold to reach zero net income, then used an approximate $300/month maximum allotment. The 2026 calendar year pays $298 for nine months and $304.68 for three, totaling $3,596.04." -us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for snap, so no substantive SNAP reasoning was submitted to evaluate. The contract required a numeric value for every requested key." -us,scenario_043,snap,kimi-k3,llm_error,period_annualization,False,"It cleared the higher elderly resource limit and used the medical costs above $35/month to reach zero net income and the correct $298 FY2026 maximum, then held that rate for all 12 months. October–December 2026 pay the FY2027 rate of $304.68." -us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"It laid out the income and asset facts, noted the higher limits for a household with a member over 60, and then asserted the benefit 'results in $0' without performing the calculation. Net income is zero after the earned-income, standard, and excess-medical deductions, so the household receives the full maximum allotment of $298–$304.68/month." -us,scenario_043,snap,ox-alpha,llm_error,period_annualization,False,"Every step was right — gross income far under the elderly limit, $2,800 under the resource limit, medical and standard deductions driving net income to zero, and the correct FY2026 maximum of $298 — except that it multiplied $298 by 12. The maximum allotment rises to $304.68 for October–December 2026." -us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It priced the one-person maximum at about $243/month, a figure from no allotment table, and additionally routed the elderly medical costs through the excess shelter deduction rather than the medical deduction. With net income at zero the benefit is the full maximum: $298/month through September 2026 and $304.68 after." -us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It conceded the household's income is low and then asserted a $0 benefit by appeal to PolicyEngine's output, citing no rule and doing no arithmetic. The elderly one-person household passes the gross, net, and resource tests and receives the maximum allotment across all 12 months." +us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the earned-income, standard and elderly excess medical deductions. It then used $291 a month, the FY2024 one-person maximum, instead of the 2026 maximum of $298. That gives $3,492 instead of $3,576." +us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied the $2,250 general-household resource limit and concluded that $2,800 in bank assets makes the household ineligible. Because the 66-year-old head is elderly, the higher elderly resource limit applies, and $2,800 passes it. The household is also categorically eligible through Colorado's TANF non-cash test. It is eligible for the $298 monthly maximum." +us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly found net income of $0 and a maximum of about $300 a month, which gives about $3,600. It then dropped that result and put in an unsupported $200 a month. With net income at $0, the benefit is the full one-person maximum of $298 a month." +us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It found net income near zero and cited a maximum of about $292. It then invented a reduced benefit of $192 a month on the false idea that a minimum-benefit rule applies. When net income is $0, the household gets the full one-person maximum allotment, which is $298 in 2026." +us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly found net income near zero after the elderly medical deduction. It then used a one-person maximum allotment of about $203 a month, far below the 2026 contiguous-states maximum of $298. That gives $2,436 instead of $3,576." +us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly found that the household passes the asset test and has net income of $0. It then used $292 a month, the FY2025 one-person maximum, instead of the 2026 value of $298. That gives $3,504 instead of $3,576." +us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly found net income of $0 and eligibility for the maximum allotment. It then overstated the one-person maximum as about $328 a month instead of $298, giving $3,936." +us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly applied the elderly asset limit and the deductions and found net income of $0. It then used $301 a month as the 2026 one-person maximum instead of $298, giving $3,612." +us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It found net income below zero but said outright that it was using the FY2025 maximum of $292. The 2026 one-person maximum of $298 applies, so it answered $3,504 instead of $3,576." +us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly treated this one-person elderly household as receiving the maximum benefit. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298." +us,scenario_043,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly found net income below zero and awarded the maximum allotment. It then put the 2026 one-person maximum at $299 a month instead of $298, giving $3,588 instead of $3,576." +us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly brought net income to $0 through the elderly medical, standard and earned-income deductions. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504." +us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It claimed the household's assets and income disqualify it. The household's gross income of $387 a month is about 30% of the poverty guideline, and its $2,800 in assets is under the elderly resource limit. It is also categorically eligible through Colorado's TANF non-cash test, so it gets the $298 monthly maximum." +us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the elderly medical, standard and earned-income deductions. It then used $291 a month, the FY2024 one-person maximum, instead of the 2026 value of $298, giving $3,492." +us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly found categorical eligibility under Colorado's rules and net income of $0. It then gave a rounded $3,500, which is about $292 a month, instead of the actual 2026 maximum of $298 × 12 = $3,576." +us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It answered $0 on vague income and resource grounds. The household passes the gross income, net income and asset tests and is categorically eligible through Colorado's TANF non-cash test. With net income at $0, it gets the $298 monthly maximum." +us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly computed a $381.67-a-month excess medical deduction that brings net income to $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504." +us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the standard, earned-income and excess medical deductions. Its $3,504 answer is $292 a month, the FY2025 maximum, instead of the 2026 one-person maximum of $298." +us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly found that medical expenses bring net income to $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298, giving $3,504." +us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"It correctly found net income of $0 and awarded the maximum allotment. Its $3,624 implies $302 a month, which overstates the 2026 one-person maximum of $298." +us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,"It correctly brought net income to $0 through the elderly medical deduction. It then overstated the one-person maximum as about $305 a month instead of $298, giving $3,660." +us,scenario_043,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave an unsupported $1,276, about $106 a month, without computing net income. It never applied the elderly excess medical deduction on $5,000 of out-of-pocket medical costs or the standard and earned-income deductions. Those bring net income to $0, which yields the full $298 monthly maximum." +us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It said the household is ineligible because it lacks low-income indicators. Its gross income of $387 a month is about 30% of the poverty guideline, which is far below every income limit. The household is also categorically eligible through Colorado's TANF non-cash test and receives the $298 monthly maximum." +us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"It claimed the $2,800 in assets exceeds typical limits. Because the head is 66, the household is elderly and $2,800 is under the elderly resource limit. The household is also categorically eligible through Colorado's TANF non-cash test. It also ignored the $5,000 in stated medical expenses, which feed the excess medical deduction that brings net income to $0." +us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly found that the household passes all tests and has net income of $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 one-person maximum of $298, giving $3,504." +us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly applied the elderly resource limit and the three deductions and found net income of $0. It then used $292 a month, the FY2025 maximum, instead of the 2026 value of $298." +us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly found net income of $0 and eligibility for the maximum. It then approximated the 2026 one-person maximum as $300 a month instead of $298, giving $3,600 instead of $3,576." +us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"It correctly found net income of $0 after the earned-income, standard and elderly medical deductions. It then rounded the one-person maximum to about $300 a month instead of the actual $298, giving $3,600." +us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there is nothing to score. The correct derivation is net income of $0 for this elderly one-person household, giving the $298 monthly maximum, or $3,576 a year." +us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"It listed the income and assets but asserted $0 without applying any test. Gross income is about 30% of the poverty guideline, and $2,800 is under the elderly resource limit. After the deductions, net income is $0, so the household receives the $298 monthly maximum." +us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It filed the medical expenses under the shelter deduction instead of the separate elderly excess medical deduction. It then used an understated 2026 one-person maximum of about $243 a month. The correct maximum is $298, and it applies in full because net income is $0." +us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It said without reasoning that PolicyEngine produces no benefit. The household passes the income and asset tests, is categorically eligible through Colorado's TANF non-cash test, and has net income of $0 after the elderly medical deduction. It therefore receives $298 a month." us,scenario_043,ssi,claude-fable-5.1,llm_error,period_annualization,False,"The model annualized the monthly SSI federal benefit rate and monthly income exclusions, then mechanically subtracted annual countable income to produce $9,954. It also identified that $2,800 exceeds the stated $2,000 resource limit but discarded that result; the controlling engine computation finds the individual’s approximately $4,642 of income above the SSI income limit, yielding zero." us,scenario_043,ssi,claude-opus-4.7,llm_error,period_annualization,False,"The model converted annual wages and pension income into a monthly countable-income figure and subtracted it from an estimated monthly federal benefit rate, bypassing the SSI income-eligibility gate. The individual’s approximately $4,642 of income exceeds the applicable limit, so aged status and assumed take-up do not produce a payable benefit." us,scenario_043,ssi,claude-opus-5,llm_error,period_annualization,False,"The model first annualized the SSI benefit rate and exclusions to derive about $9,360, then replaced that calculation with an unsupported $6,720 figure. Both paths miss that the individual’s approximately $4,642 of income fails the SSI income-limit test, which eliminates the benefit entirely." us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, so it failed the required structured-output contract." -us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Enumerated Colorado's refundable credits as only the federally-derived set (CO EITC, CO CTC, child care) and correctly killed the CO EITC on the federal childless age-25-64 cap, then declared the total zero. It never reached the TABOR state sales tax refund, which is claimed on Form DR 0104 by any full-year resident 18+ who files and is keyed to AGI tier rather than earned income or children — $19 at this $4,625 MAGI." -us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,categorical_eligibility,False,"Asserted that Colorado's EITC extends to childless workers 65 and older who are barred federally by age; Colorado's credit is a flat percentage of the federal EITC actually allowed, which is $0 for a 66-year-old childless filer, so both the $312.73 'federal-equivalent' credit and the $78.18 match are constructed from a rule that does not exist. The real credit is the TABOR state sales tax refund at $19, which the model never considered." -us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Stated that Colorado's individual credits are generally nonrefundable and that low liability therefore yields nothing, which inverts the governing rule: the TABOR state sales tax refund is fully refundable and paid to every filing full-year resident 18+ regardless of tax liability. That is precisely why this near-zero-liability household still receives the bottom-tier $19." -us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Identified the correct pathway — the refundable TABOR sales tax refund at the lowest AGI tier for a single full-year resident — but substituted a large-surplus-year amount (~$1,130, the scale of the 2023 six-tier schedule) for the 2026 schedule's bottom-tier figure of $19, overstating the credit roughly sixtyfold. The tier selection was right; the parameter value was recalled from the wrong year." -us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Explicitly grouped 'TABOR-related refundable credits' with the state EITC as requiring qualifying earned income and filing conditions, then zeroed them for lack of qualifying earnings or children. The sales tax refund imposes neither test — full-year residency, age 18+, and filing a return are the whole eligibility standard, and the $4,625.40 MAGI places this filer in the bottom tier at $19." -us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Invoked Colorado 'senior housing/income-qualified' refundable provisions that are not in force for tax year 2026 and then produced $225 as an unanchored estimate rather than reading any statutory schedule. The operative credit is the TABOR state sales tax refund, a fixed bottom-AGI-tier amount of $19 for a single filer." -us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly applied the federal childless EITC age ceiling of 64 to zero the CO EITC and correctly zeroed the CO CTC for want of children, but treated those two as exhausting Colorado's refundable credits. Colorado's TABOR state sales tax refund is a third, independent refundable credit that turns on residency, filing, and AGI tier alone, paying $19 at this MAGI." -us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Named the TABOR sales tax refund but recast it as a senior-targeted flat credit for low-income 65+ filers and estimated $1,000-$1,400 from surplus-year Colorado Cash Back recall. The refund is age-neutral above 18 and graduated by AGI tier, and the 2026 bottom tier for this $4,625.40 MAGI single filer pays $19." -us,scenario_043,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Reasoned entirely through the federal-EITC dependency chain — no federal EITC at age 66, therefore no CO EITC, therefore nothing — and concluded no other Colorado refundable credit exists. Colorado's TABOR state sales tax refund is refundable, independent of the federal EITC, and pays $19 to this filer on the strength of residency and AGI tier alone." -us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Conditioned the entire state refundable total on a qualifying child and a nonzero federal EITC, both of which fail here, and stopped. The TABOR state sales tax refund requires neither and pays a bottom-tier $19 to a single full-year resident filing with $4,625.40 MAGI." -us,scenario_043,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Dismissed Colorado's refundable credits wholesale on household composition — 'single senior, no qualifying children' — without checking any credit whose eligibility is composition-independent. The TABOR state sales tax refund keys on full-year residency, age 18+, filing, and AGI tier, delivering $19 here." -us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Tied the whole state refundable total to federal EITC qualification and child-related credits, both correctly zero at age 66 with no dependents. That enumeration omits Colorado's TABOR state sales tax refund, a refundable credit granted on residency and AGI tier that pays $19 at this household's $4,625.40 MAGI." -us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Rejected all Colorado refundable credits on 'income and composition' grounds, which is exactly backwards for the credit that applies: the TABOR state sales tax refund pays more at low AGI and imposes no composition test. The bottom AGI tier for a single filer yields $19." -us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Checked only the Colorado PTC rebate, which requires property tax, rent, or heat expenses and is not an income tax credit, and zeroed the total for want of those expenses. Colorado's refundable income tax credit here is the TABOR state sales tax refund, which requires no expenses at all and pays $19 at the bottom AGI tier." -us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Made federal EITC qualification a necessary condition for any state refundable credit, so the age-66 childless bar zeroed everything. Colorado's TABOR state sales tax refund is independent of the federal EITC and pays this single full-year resident $19 based solely on the $4,625.40 MAGI tier." -us,scenario_043,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Gave a bare zero with no pathway analysis; the $0 is consistent with treating Colorado's refundable credits as exhausted by the federal-EITC-matched CO EITC and the child-conditioned CO CTC. The correct derivation runs the TABOR state sales tax refund schedule, which pays a single full-year resident with $4,625.40 MAGI $19." -us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Asserted zero Colorado refundable credits without naming any program tested, a result consistent with checking only the credits that key on children or the federal EITC. Colorado's TABOR state sales tax refund keys on residency, age 18+, filing, and AGI tier instead, and pays $19 here." -us,scenario_043,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,Derived the zero from two conditions — no qualifying children and no federal EITC — and treated those as covering Colorado's refundable set. The TABOR state sales tax refund depends on neither and pays this single filer $19 at the bottom AGI tier of the 2026 schedule. -us,scenario_043,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Limited the Colorado refundable set to the federal-EITC-matched credit and the state CTC, both zero for a childless 66-year-old, and stopped there. Colorado's TABOR state sales tax refund is the third refundable credit and pays $19 on the basis of full-year residency and a $4,625.40 MAGI." -us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly applied the federal childless EITC upper age limit to zero the CO EITC, then asserted no other refundable state credit applies. Colorado's TABOR state sales tax refund applies to every filing full-year resident 18+ regardless of EITC status, paying the bottom-tier $19 at this MAGI." -us,scenario_043,state_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"Fabricated a $625 federal EITC for a 66-year-old childless filer, whose federal childless credit is $0 because the credit is limited to ages 25 through 64, then multiplied by a 25% Colorado match to reach $156. Both inputs are wrong, and the actual $19 comes from the TABOR state sales tax refund, which the model never examined." -us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded that the reported facts trigger no Colorado refundable credit, but the facts that trigger the TABOR state sales tax refund — Colorado residence, age 66, and a filed return with $4,625.40 MAGI — are all present in the prompt. That refund pays $19 at the bottom AGI tier." -us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,Framed state refundable credits as requiring earned income thresholds or qualifying children and zeroed on that basis. Colorado's TABOR state sales tax refund imposes neither requirement — it is a per-filer surplus rebate graduated by AGI — and pays this household $19. -us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Enumerated four categories — child-based, earned-income, property/rent/heat, and 'other' — and found no supporting facts, but the TABOR state sales tax refund fits none of those categories: it is a surplus rebate paid through the income tax return to every full-year resident 18+ who files. Its bottom AGI tier gives this filer $19." -us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Declared no qualifying Colorado refundable credit without identifying the sales tax refund, the one Colorado refundable credit whose eligibility test is residency and filing rather than household characteristics or expenses. At $4,625.40 MAGI, the single-filer bottom tier pays $19." -us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Tested only two conditions — a federal EITC amount to match and listed expenses or dependents for another credit — and zeroed when both failed. Colorado's TABOR state sales tax refund requires no federal credit, no dependents, and no expenses, and pays $19 at this AGI tier." -us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Concluded that the listed facts support no Colorado refundable credit, overlooking that state of residence, age 66, and filing status alone qualify the household for the TABOR state sales tax refund. The 2026 bottom AGI tier for a single filer is $19." -us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Applied Colorado's senior housing income tax credit at its maximum $800, a credit enacted for specific earlier tax years and not in force for 2026, and skipped the credit that does apply. The 2026 refundable amount is the TABOR state sales tax refund's bottom AGI tier, $19." -us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Returned a bare zero with no program named, a result consistent with screening only the child- and federal-EITC-linked Colorado credits. The TABOR state sales tax refund screens on full-year residency, age 18+, and AGI tier instead, paying $19 to this household." -us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Made the Colorado total contingent on a federal EITC to match plus children or 'other qualifying circumstances,' none of which exist here. The unexamined circumstance that does qualify is Colorado residency itself: the TABOR state sales tax refund pays $19 at this filer's $4,625.40 MAGI tier." -us,scenario_043,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Screened exactly two programs — the federal-EITC-matched CO EITC and the senior PTC rebate, correctly zeroing both — and treated that pair as the full Colorado refundable set. The TABOR state sales tax refund is the third program, requires neither a federal EITC nor rent or property tax, and pays $19." -us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Built the answer on a $313 federal EITC that does not exist for a 66-year-old childless filer, since the federal childless credit is confined to ages 25 through 64, and compounded the error with a 50% Colorado match rate. The $19 reference is the TABOR state sales tax refund's bottom AGI tier, a credit the model dismissed as needing rent or property tax data." -us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Correctly zeroed the CO CTC for lack of dependents and the CO EITC on the federal age bar, then treated the state total as settled. Colorado's TABOR state sales tax refund is refundable and independent of both, paying this single full-year resident $19 based on the AGI tier." -us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Concluded that a 66-year-old with no dependents or qualifying expenses generates no Colorado refundable credit, applying only composition and expense tests. The TABOR state sales tax refund uses neither test and pays $19 on residency, filing, and a $4,625.40 MAGI." -us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Anchored the state total to the existence of a federal EITC or CTC to piggyback on, plus child-care expenses, and zeroed when none applied. Colorado's TABOR state sales tax refund is a standalone refundable credit with no federal base, worth $19 at this filer's bottom AGI tier." -us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Rejected Colorado refundable credits on income level and absence of qualifying children, but the applicable credit rewards low AGI rather than penalizing it and ignores household composition entirely. The TABOR state sales tax refund's bottom tier pays this single filer $19." -us,scenario_043,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Reached the TABOR sales tax refund but discarded it as inapplicable 'for this tax year under the stated assumptions'; the refund is a standing statutory credit on the 2026 Colorado return requiring no facts beyond full-year residency, age 18+, and filing, all of which the prompt supplies. Its bottom AGI tier pays $19." -us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Reasoned that earned income of $4,088 is too small to trigger a meaningful state credit and that no qualifying children exist, applying an earned-income-phase-in mental model. The TABOR state sales tax refund has no phase-in — a low AGI simply places the filer in the bottom tier, which pays $19." -us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Inferred zero from the absence of Colorado tax liability, which contradicts refundability: the TABOR state sales tax refund is paid out in full even when liability is zero. At $4,625.40 MAGI, the single-filer bottom tier delivers $19." +us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Correctly zeroed the Colorado EITC because a 66-year-old is past the federal childless EITC age cap. It then left out the Colorado sales tax refund, a refundable credit that pays $19 at modified AGI of $4,625 in 2026, and wrongly said the income was too low to trigger any other credit." +us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Invented a Colorado EITC for childless filers aged 65+. Colorado's EITC is a percentage of the federal EITC, which is $0 for this 66-year-old, so its $78.18 has no basis. It also missed the only credit that applies: the $19 sales tax refund." +us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly claimed Colorado has no refundable individual credits. The state EITC, child tax credit, family affordability credit and sales tax refund are all refundable, and the sales tax refund pays this filer $19." +us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Picked the right program, the TABOR sales tax refund, but invented a lowest-tier amount of about $1,130. For 2026, the tier for modified AGI of $4,625.40 pays $19." +us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Mentioned TABOR refunds but wrongly assumed they depend on earned income or other triggers. Every full-year resident filer gets the sales tax refund, and at modified AGI of $4,625.40 it pays $19." +us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Made up a $225 refundable senior housing or income-based credit that pays nothing in 2026. It missed the only credit that applies: the $19 Colorado sales tax refund at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Named the 2026 TABOR sales tax refund but chose to leave it out. This filer qualifies for it, and at modified AGI of $4,625.40 it pays $19." +us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly zeroed the Colorado EITC and child tax credit. It then said no other refundable credit applies and missed the sales tax refund, which pays this 66-year-old resident filer $19." +us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Invented a flat refundable senior credit of about $1,200. The senior housing credit pays nothing in 2026, and the TABOR sales tax refund at modified AGI of $4,625.40 is only $19." +us,scenario_043,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Said no Colorado refundable credit applies and missed the sales tax refund, which pays this resident filer $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Correctly ruled out the Colorado EITC because the head is 66, but wrongly said no other refundable credit exists. The Colorado sales tax refund pays this filer $19." +us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Checked only the EITC and child-based credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Assumed Colorado's refundable credits need qualifying children. The sales tax refund goes to childless resident filers and pays this household $19. +us,scenario_043,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Stopped once the Colorado EITC came out at $0 and missed the refundable sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,Looked only at the Colorado EITC and child-related credits and missed the Colorado sales tax refund. That refund does not depend on EITC eligibility and pays $19 here. +us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Said income and household makeup rule out every Colorado refundable credit. In fact the sales tax refund goes to low-income single resident filers and pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Checked only the property tax, rent and heat rebate and ignored the Colorado sales tax refund. That refund needs no qualifying expenses and pays $19 here." +us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,Wrongly tied all Colorado refundable credits to federal EITC eligibility. The sales tax refund does not depend on the EITC and pays this filer $19. +us,scenario_043,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Answered $0 without identifying any credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,Answered $0 without naming a program and missed the Colorado sales tax refund. That refund goes to every full-year resident filer and pays $19 at this income. +us,scenario_043,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Based the answer only on the missing children and the $0 federal EITC, and missed the Colorado sales tax refund. That refund pays $19 regardless of those factors." +us,scenario_043,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,Ruled out the EITC and state child tax credit but never included the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19. +us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Correctly applied the age limit to zero out the EITC, then wrongly said no other refundable credit applies. The Colorado sales tax refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"Assumed a $625 federal EITC even though a 66-year-old is past the childless EITC age limit, so federal and Colorado EITC are both $0. It also missed the only credit that applies: the $19 sales tax refund." +us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Said none of the reported facts trigger a refundable Colorado credit and missed the sales tax refund. Filing as a full-year adult resident is enough, and it pays $19 here." +us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed refundable credits need earned-income eligibility or children. It missed the Colorado sales tax refund, which pays this filer $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Checked the child, earned-income and property/rent/heat credits but left out the Colorado sales tax refund. That refund needs no expense facts and pays $19 here." +us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Said no Colorado refundable credit applies and missed the sales tax refund, which pays this resident filer $19." +us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Looked only for EITC, dependent-based and expense-based credits and missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Said none of the listed facts support a refundable credit. Colorado residency plus filing is enough for the sales tax refund, which pays $19 here." +us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Gave the maximum $800 senior housing income tax credit, but that credit pays nothing for tax year 2026. The only Colorado refundable credit here is the $19 sales tax refund." +us,scenario_043,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Correctly ruled out the EITC because of the head's age and the credits because of no children, but missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,Looked only for dependent-based and housing-cost-based credits and missed the Colorado sales tax refund. That refund needs neither and pays $19 here. +us,scenario_043,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Covered the child, EITC and property/rent/heat credits but left out the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19." +us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Answered $0 and missed the Colorado sales tax refund, a refundable state credit that pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,Ruled out the EITC match and child credits but never included the Colorado sales tax refund. That refund goes to every full-year resident filer and pays $19 here. +us,scenario_043,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Correctly zeroed the EITC and the senior property tax, rent and heat rebate but missed the Colorado sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Checked the EITC, the child tax credit and the property tax, rent and heat rebate but left out the Colorado sales tax refund. That refund pays this filer $19." +us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"Assumed a $313 federal EITC for a 66-year-old childless filer who is past the EITC age limit. Its 50% Colorado match therefore has no basis, since the Colorado EITC is $0. It also missed the $19 sales tax refund." +us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,Correctly zeroed the Colorado EITC and child tax credit but missed the Colorado sales tax refund. That refund does not depend on age limits or dependents and pays $19. +us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,Assumed a refundable credit needs dependents or qualifying expenses. The Colorado sales tax refund needs neither and pays this resident filer $19. +us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Tied Colorado's refundable credits only to the federal EITC, CTC, children and childcare costs, and missed the sales tax refund. That refund pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Said the low income and missing children rule out every Colorado credit. In fact the sales tax refund is aimed at low-income filers and pays $19 here. +us,scenario_043,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Considered the TABOR sales tax refund and wrongly declared it inapplicable for 2026. This full-year resident filer qualifies, and it pays $19 at modified AGI of $4,625.40." +us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Checked only the child tax credit and the EITC and missed the Colorado sales tax refund. That refund pays this 66-year-old resident filer $19. +us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Wrongly tied refundable credits to Colorado tax liability. Refundable credits pay out even when liability is zero, and the sales tax refund pays this filer $19." us,scenario_043,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the standard deduction directly from the full $56,806 Social Security benefit, effectively treating gross Social Security as taxable income. It failed to calculate taxable Social Security from provisional income before applying the joint-filer deduction, which leaves no taxable income." -us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,age_disability,False,"The model incorrectly described both spouses as older adults even though the spouse is 61, then produced tax without deriving the taxable portion of Social Security or taxable income. Only the 66-year-old head receives the age-based treatment, and the household's taxable Social Security remains below the applicable joint-filer deductions, producing zero tax." -us,scenario_044,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire Social Security benefit as taxable, ignoring that provisional income with no other income equals only half the benefits and is only modestly above the $25,000 joint base amount. It also computed $1,808.51 in its explanation but submitted $5,816.40, a final value unsupported by its own arithmetic." -us,scenario_044,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly stated that its claimed $24,730 taxable Social Security was below its $34,250 deduction and therefore produced zero taxable income, but then separately taxed that Social Security despite the deduction. It further misclassified the senior deduction as a nonrefundable credit and concluded with $0 while submitting $4,234, contradicting its own derivation." +us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model took the full $56,806 of Social Security, subtracted the standard deduction, and taxed the remainder. It never applied the §86 provisional-income test: half of benefits, $28,403, is below the $32,000 joint base amount, so no benefits are includable and tax is $0." +us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied only the older-couple standard deduction and nonrefundable credits, and its $4,162 answer only works if a large share of the Social Security counts as taxable income. It skipped the benefit-taxation test. Provisional income of $28,403 is below the $32,000 MFJ base amount, so taxable income and tax are both $0." +us,scenario_044,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model said 85% of the benefits ($48,285) is taxable when there is no other income. But 85% is only the statutory maximum, and it applies only after provisional income passes the thresholds. Here provisional income is 0.5 × $56,806 = $28,403, below the $32,000 MFJ base amount, so taxable benefits are $0 and there is nothing left to tax at 10%." +us,scenario_044,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model assumed 85% of the benefits ($48,285.10) was taxable without running the provisional-income test. Provisional income of $28,403 is below the $32,000 joint base amount, so no benefits are taxable. It also submitted $5,816.40, which does not match the $1,808.51 tax it computed in its own reasoning." +us,scenario_044,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model said provisional income exceeded the joint threshold and made $24,730 of benefits taxable. In fact provisional income is $28,403, below the $32,000 MFJ base amount, so taxable benefits are $0. It also treated the senior deduction as a nonrefundable credit and invented a federal retirement-income credit. Finally, it submitted $4,234 even though its own reasoning concluded $0." us,scenario_044,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the household has no wage or self-employment income and therefore no employee payroll-tax base, but then contradicted that reasoning by imposing $2,926 of unspecified “typical” wage-related payroll taxes. The $56,806 is Social Security retirement income, which is not subject to employee Social Security or Medicare tax, so the correct payroll-tax computation yields $0." us,scenario_044,spouse_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the 61-year-old spouse is not Medicare eligible and explicitly derived 0, then submitted value = 1. It violated the required consistency between the eligibility explanation and numeric output." us,scenario_044,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated Social Security retirement benefits as Kansas-taxable income and applied state brackets after exemptions. Kansas subtracts the federally taxable Social Security amount from Kansas adjusted gross income, leaving no taxable income and $0 tax before refundable credits." us,scenario_044,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model subjected the reported Social Security income to Kansas tax after a standard-deduction-style adjustment. It omitted Kansas’s subtraction for federally taxable Social Security benefits, which removes the household’s only income from the Kansas tax base and yields $0." us,scenario_044,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly asserted that Kansas taxes Social Security as ordinary income, then reduced the full $56,806 only by an $8,000 standard deduction and $4,500 of personal exemptions before applying tax brackets. Kansas instead subtracts the federally taxable Social Security benefits from Kansas adjusted gross income, so none of that benefit reaches the bracket calculation and the resulting tax is $0." -us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model classified the Kansas food sales tax credit as refundable and added $125 for each of two exemptions to state_refundable_credits. That credit is nonrefundable, so its asserted eligibility does not create any amount in this output." -us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly placed the $250 Kansas food sales tax credit in the refundable-credit total. Kansas treats this credit as nonrefundable, leaving state_refundable_credits at $0." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Reproduced the reference derivation exactly — $16,100 standard deduction, $20,176 taxable, $1,240 + 12% x $7,776 = $2,173 — and then submitted $2,540, a figure its own arithmetic never produces. The tax computation was right; the value field discarded it." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 employer-sponsored premium from the $36,276 wage figure to reach $15,068 of taxable wages, but the stated gross wage is the taxable wage amount and the ESI premium input drives health-coverage modeling only. It compounded that with the 2024 $14,600 standard deduction instead of the 2026 $16,100, then zeroed the residual with the EITC — a refundable credit this output explicitly excludes, and one that is $0 for a childless filer at $36,276 of earnings." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Self-corrected mid-reasoning to the right 2026 parameters ($16,100 standard deduction, $12,400 top of the 10% bracket) and computed $2,173, then submitted $1,631 as a 'best estimate.' It abandoned a correct derivation for a number nothing in its reasoning supports." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Derived $1,240 + 0.12 x $7,776 = $2,173 from the correct $16,100 standard deduction and $12,400 bracket ceiling, then submitted $1,387 'after rounding to bracket estimates.' No rounding of the 2026 schedule produces $1,387; the derived value was the answer." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Computed the reference figure of about $2,173 from the correct $16,100 standard deduction and $12,400 bracket top, then submitted $2,029 and attributed the $144 gap to 'bracket rounding.' The 2026 single rate schedule involves no rounding at this income level." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Projected its own 2026 standard deduction of $15,350 by inflating the 2025 $15,000, missing the statutory 2026 amount of $16,100, and used the 2025 top of the 10% bracket ($11,925) instead of 2026's $12,400. Its AGI treatment was correct — full wages, ESI already outside Box 1, child support not deductible — so the entire $99.55 gap comes from those two parameters." -us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Used an invented $15,700 standard deduction and $12,150 bracket ceiling to compute $2,226, then submitted $1,130, which its stated arithmetic never yields. The 2026 parameters are $16,100 and $12,400, giving $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Netted the $21,208 employer-sponsored premium out of the $36,276 of wages to reach $15,068 of AGI and concluded the standard deduction wiped out taxable income. The stated gross wage is already the taxable wage figure, so AGI is $36,275.59 and the $16,100 standard deduction leaves $20,175.59 taxable, taxed at $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Made both available errors at once: it subtracted the $21,208 ESI premium from wages to reach $15,068 of AGI, then applied a TCJA-sunset $8,117 standard deduction plus a $5,177 personal exemption. OBBBA governs 2026 — a $16,100 standard deduction and no personal exemption — and the premium does not reduce the stated wages, so taxable income is $20,175.59, not $1,774." -us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Kept AGI correct at $36,276 but applied pre-TCJA law to 2026: an $8,400 standard deduction, a $5,400 personal exemption, and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 uses $16,100, no exemption, and 12% above $12,400 — $2,173.07 rather than $2,775.15." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Treated the $21,208 employer-sponsored premium as already excluded from a larger gross figure and subtracted it again, cutting AGI to $15,068 and putting income below its assumed $15,700 standard deduction. The listed $36,276 is the taxable wage amount; against the actual 2026 standard deduction of $16,100 it leaves $20,175.59 taxable and $2,173.07 of tax." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Applied a head-of-household framing and a $15,000 standard deduction to a single filer with no dependents, then reported $0 — its own deduction still leaves $21,276 of taxable income, so zero follows from nothing it stated. The 2026 single standard deduction of $16,100 leaves $20,175.59 taxable and $2,173.07 of tax, and no nonrefundable credit is available to this filer." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied post-sunset pre-TCJA parameters — a standard deduction and personal exemption totaling $14,050, plus a 15% second bracket — to reach $22,226 of taxable income and $2,726. 2026 runs under OBBBA: a $16,100 standard deduction, no personal exemption, and 12% above $12,400." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 ESI premium from wages to get $15,068 of AGI and then applied a pre-TCJA-style $13,350 combined standard deduction and personal exemption, leaving $1,718 taxable. The premium does not reduce the stated wages and 2026 has a $16,100 standard deduction with no personal exemption, so taxable income is $20,175.59 and tax is $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction exceeds taxable income for a single filer with $36,276 of wages. The 2026 single standard deduction is $16,100, which leaves $20,175.59 taxable; the answer of zero corresponds to no computation being performed." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Combined both errors: it reduced wages by the $21,208 ESI premium to $15,068 of AGI and then assumed TCJA expiration, using an $8,300 standard deduction and $5,150 personal exemption. Wages stand in full at $36,275.59 and 2026 provides a $16,100 standard deduction with no exemption, giving $20,175.59 taxable and $2,173.07 of tax rather than $1,618 and $161.80." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Kept AGI at the correct $36,276 but applied pre-TCJA sunset parameters — an $8,300 standard deduction, a $5,300 personal exemption, and a 15% second bracket — producing $22,676 of taxable income. OBBBA leaves 2026 with a $16,100 standard deduction, no personal exemption, and 12% above $12,400." -us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the 2025 standard deduction of $15,000 and the 2025 top of the 10% bracket ($11,925) rather than the 2026 values of $16,100 and $12,400. Its AGI and no-credits conclusions were correct, so the $141.55 overstatement is entirely the stale inflation-adjusted parameters." -us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Estimated a $15,400 standard deduction and a $12,250 bracket ceiling; the 2026 values are $16,100 and $12,400. Its AGI treatment matched the reference exactly — full wages as AGI, child support paid not deductible, no nonrefundable credits — so the $87.05 gap is pure parameter drift." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Asserted without computation that income is too low for federal tax after the standard deduction and credits. Wages of $36,275.59 less the $16,100 standard deduction leave $20,175.59 taxable and $2,173.07 of tax, and a childless 44-year-old with no childcare, education, or retirement-contribution expenses has zero nonrefundable credits to offset it." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Assumed the standard deduction plus unnamed nonrefundable credits fully offset liability. No nonrefundable credit applies to a single 44-year-old with no dependents and no qualifying expenses, and the standard deduction of $16,100 leaves $20,175.59 of taxable income, so the full $2,173.07 stands." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used the 2025 OBBBA single standard deduction of $15,750 instead of the inflation-adjusted 2026 amount of $16,100. Its bracket parameters were right — 10% to $12,400, then 12% — so the $350 of extra taxable income taxed at 12% accounts for the entire $41.93 overstatement." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Reduced taxable wages by the $21,208 employer-sponsored premium to $15,068 and declared the standard deduction larger than that. The stated $36,276 is the taxable wage figure, so the $16,100 standard deduction leaves $20,175.59 taxable and $2,173.07 of tax." -us,scenario_045,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Itemized $18,487.30 of the employer-sponsored premium as medical expense above the 7.5%-of-AGI floor; premiums paid through an employer cafeteria plan are excluded from wages and are not deductible under section 213, and the reference's itemized total is $1,541.71, far below the $16,100 standard deduction. Deducting a phantom $19,778 instead of $16,100 understated taxable income by $3,677.87 and tax by exactly $441.34 at the 12% rate." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Used the 2025 standard deduction of roughly $15,000 rather than the 2026 amount of $16,100, and never stated bracket thresholds. Its $2,352 does not reconcile with either schedule applied to its own $21,276 of taxable income ($2,314.62 on the 2025 brackets, $2,305 on the 2026 brackets), so the bracket arithmetic is wrong on top of the stale deduction." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Stacked both errors: it cut AGI to $15,068 by subtracting the $21,208 ESI premium, then applied a TCJA-sunset $8,535 standard deduction and $5,444 personal exemption. OBBBA keeps a $16,100 standard deduction and no personal exemption in 2026, and the premium does not reduce the stated wages, so taxable income is $20,175.59 rather than $1,089." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Got the income side exactly right — full wages as AGI, child support paid not deductible, the ESI premium not an additional deduction — then applied the 2025 standard deduction of $15,000 and the 2025 bracket ceiling of $11,925. The 2026 values are $16,100 and $12,400, which move the answer from $2,314.62 to $2,173.07." -us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Assumed TCJA expiration for 2026 and used an ~$8,000 standard deduction, a ~$5,000 personal exemption, an $11,600 top of the 10% bracket, and a 15% second bracket. All four are superseded: 2026 has a $16,100 standard deduction, no personal exemption, a $12,400 bracket ceiling, and a 12% second bracket." -us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for this variable, so nothing was scored against the $2,173.07 reference. The required derivation — $36,275.59 of wages less the $16,100 single standard deduction, taxed at 10% to $12,400 and 12% above — was never attempted." -us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Derived the correct $20,176 of taxable income from the $16,100 standard deduction and then reported $0 on the ground that the EITC and other credits eliminate liability. The EITC is refundable and this output is defined before refundable credits, and the childless-filer EITC phases out entirely well below $36,276 of earned income, so it is $0 for this filer in any event." -us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Computed $2,220.12 from a $15,750 standard deduction and a $12,150 bracket ceiling, then submitted $424.76, which is unconnected to that arithmetic or to any 2026 rate schedule. Even its own stated parameters were only $47 from the reference; the submitted value discarded them." -us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction exceeds AGI; the 2026 single standard deduction of $16,100 against $36,275.59 of wages leaves $20,175.59 of taxable income and $2,173.07 of tax. It correctly noted that child support paid is not deductible and then wrote the answer as if the $5,200 had zeroed the income anyway." +us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied the Kansas food sales tax credit ($125 per exemption, age 55+ pathway, AGI under $30,615) and got $250, but that credit ended after tax year 2024, when the state food sales tax went to 0%. It is not available in 2026. The household has no earned income for the Kansas EITC, so Kansas refundable credits are $0." +us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It gave the household the Kansas food sales tax credit of $125 for each of two exemptions based on the low-income, age 55+ test, but it used pre-2025 law. Kansas ended that credit after tax year 2024, when food became exempt from state sales tax, so it pays nothing in 2026. With no earnings there is no Kansas EITC either, so the correct total is $0." +us,scenario_044,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It correctly found Kansas AGI of $0 and both spouses over 55, but it then paid out the food sales tax credit (2 x $125 = $250), which Kansas ended after tax year 2024, when the state food sales tax rate went to zero. No refundable Kansas credit is left for this household in 2026. The Kansas EITC needs earned income, and the household has only Social Security." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It used the correct $16,100 standard deduction and the $12,400 10% bracket and derived $1,240 + $933 = $2,173. It then submitted $2,540 with no supporting computation, so the submitted value contradicts its own correct derivation." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the stated gross wages, which cut income to $15,068, and it applied the 2024 standard deduction of $14,600. It then wiped out the remaining tax with the EITC, a refundable credit that this pre-refundable-credit measure excludes. The correct base is the full $36,275.59 of wages less the $16,100 standard deduction, which gives $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"On its recheck it applied the $16,100 standard deduction and $12,400 bracket and got $2,173. It then submitted $1,631, a number unsupported by any step of its own derivation." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed taxable income of $20,176 and tax of $1,240 + $933 = $2,173 with no credits. It then submitted $1,387, citing 'rounding to bracket estimates', which no bracket rounding produces." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived $2,173 from the $16,100 standard deduction and the 10%/12% brackets. It then submitted $2,029 on an unexplained 'slight bracket rounding' that contradicts its own arithmetic." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the 2025 pre-OBBBA $15,000 standard deduction to $15,350 and used the 2025 10% bracket top of $11,925. The 2026 values are a $16,100 standard deduction and a $12,400 bracket top, so it overstated taxable income ($20,926 vs $20,175.59) and put too much in the 12% bracket." +us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used an understated $15,700 standard deduction and a $12,150 bracket top to reach $2,226, instead of $16,100 and $12,400. It then submitted $1,130, roughly half its own figure, with no computation behind it." +us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the listed gross wages to get AGI of $15,068, then zeroed taxable income against the standard deduction. The listed gross wages are the AGI base ($36,275.59), and after the $16,100 standard deduction $20,175.59 is taxable." +us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages to reach AGI of $15,068. It also applied a sunset-era $8,117 standard deduction and a $5,177 personal exemption. Under the permanent TCJA/OBBBA rules the 2026 base is $36,275.59, less a $16,100 standard deduction, with no personal exemption." +us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,400 standard deduction, a $5,400 personal exemption and a 15% second bracket. The 2026 law is the permanent TCJA/OBBBA schedule: a $16,100 standard deduction, no exemption, and 10% up to $12,400 with 12% above." +us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied a sunset-era $8,000 standard deduction and a $5,000 personal exemption with a 15% bracket above $11,650. The permanent TCJA/OBBBA 2026 rules use a $16,100 standard deduction, no exemption, and 12% above $12,400." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 employer-sponsored insurance premiums as a pre-tax reduction of the stated gross wages, which put AGI at $15,068, below its $15,700 standard deduction. AGI is the full $36,275.59 of wages, which leaves $20,175.59 taxable after the $16,100 deduction." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It assigned head-of-household status to a single filer with no dependents and claimed that a $15,000 deduction eliminates tax on $36,276 of wages. Even that deduction leaves $21,276 taxable. The correct single-filer computation ($16,100 deduction, $20,175.59 taxable) yields $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: a standard deduction plus a personal exemption totaling about $14,050, and a 15% second bracket. The permanent TCJA/OBBBA 2026 schedule uses a $16,100 standard deduction, no exemption, and 10%/12% brackets split at $12,400." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from gross wages to reach AGI of $15,068, then used a sunset-era $13,350 standard deduction plus exemption. The correct base is $36,275.59 of wages less the $16,100 TCJA/OBBBA standard deduction." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction exceeds taxable income for a single filer with $36,276 of wages. That is false: $36,275.59 less the $16,100 standard deduction leaves $20,175.59 taxable. A $0 result is reachable only by removing the $21,208 ESI premiums from wages." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced gross wages by the $21,208 ESI premiums to get AGI of $15,068, then applied a TCJA-expiration $8,300 standard deduction and a $5,150 personal exemption. The correct base is $36,275.59 of AGI less the permanent $16,100 standard deduction." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used sunset-era parameters: an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, which put taxable income at $22,676. The permanent TCJA/OBBBA 2026 rules give $20,175.59 taxable, and tax of $2,173.07 at 10%/12%." +us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the 2025 pre-OBBBA $15,000 standard deduction and the 2025 $11,925 10% bracket top to tax year 2026. The 2026 values are $16,100 and $12,400, so it overstated taxable income by $1,100 and moved income into the 12% bracket." +us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $15,400 standard deduction and a $12,250 10% bracket top for 2026 instead of the OBBBA-adjusted $16,100 and $12,400. That inflated taxable income to $20,876 and tax to $2,260.12." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed that the standard deduction and nonrefundable credits eliminate the tax. $36,275.59 of wages less the $16,100 standard deduction leaves $20,175.59 taxable, and a single childless filer has no nonrefundable credits to offset the resulting $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed without computation that the standard deduction and nonrefundable credits fully offset the tax. The $16,100 standard deduction leaves $20,175.59 taxable, and no nonrefundable credit applies to this childless single filer, so tax is $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 rather than the inflation-adjusted 2026 value of $16,100. Its $12,400 bracket top was correct, but the $350 shortfall in the deduction raised taxable income to $20,526 and tax to $2,215." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from the stated gross wages to get $15,068, below the standard deduction. The listed gross wages are the AGI base of $36,275.59, leaving $20,175.59 taxable after the $16,100 deduction." +us,scenario_045,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized and treated the $21,208 employer-sponsored insurance premium as a deductible out-of-pocket medical expense above the 7.5%-of-AGI floor, claiming $18,487.30 of medical deductions. Excluding that premium, itemized deductions are only $1,541.71, so the $16,100 standard deduction applies, giving $20,175.59 taxable and $2,173.07 of tax." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA $15,000 standard deduction instead of the 2026 $16,100, which overstated taxable income at $21,276. It then produced $2,352, which exceeds even a correct 10%/12% bracket computation on that overstated base." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 ESI premiums from wages to get AGI of $15,068 and assumed a TCJA sunset with an $8,535 standard deduction and a $5,444 exemption. The correct base is $36,275.59 of AGI less the permanent $16,100 standard deduction." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the 2025 pre-OBBBA $15,000 standard deduction and the $11,925 10% bracket top to 2026 instead of $16,100 and $12,400. That produced $21,276 of taxable income and $2,315 of tax instead of $20,175.59 and $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,000 standard deduction, a $5,000 exemption and a 15% bracket above $11,600. The 2026 law keeps the TCJA structure permanently: a $16,100 standard deduction, no exemption, and 12% above $12,400." +us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so no answer was parsed. The correct derivation is $36,275.59 of AGI less the $16,100 standard deduction, which leaves $20,175.59 taxed at 10%/12% for $2,173.07." +us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It correctly reached $20,176 of taxable income with the $16,100 standard deduction. It then zeroed the tax with the EITC, even though it acknowledged the EITC is refundable and excluded from this output. A childless 44-year-old earning $36,276 also receives no EITC." +us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It used a stale $15,750 standard deduction and a $12,150 bracket top to compute $2,220.12. It then submitted $424.76, a value with no connection to its own computation." +us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction exceeds AGI. $36,275.59 of wage AGI exceeds the $16,100 standard deduction by $20,175.59, so tax is $2,173.07. Its $0 is reachable only by removing the $21,208 ESI premiums from wages." us,scenario_045,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified the childless EITC pathway and that its phaseout ends near $20,000, but then awarded $632 at $36,276. The childless EITC is fully phased out at that income, so this component is $0." us,scenario_045,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from wages when computing EITC income, despite no stated pre-tax payroll treatment. It also produced $2,410 for a childless EITC whose maximum is far below that amount; using the full $36,276 leaves the credit fully phased out." us,scenario_045,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated earned income as $15,068 by subtracting the $21,208 insurance premium from $36,276 of gross wages. The EITC test uses the full $36,276 here, at which the childless credit is fully phased out." @@ -3534,169 +3913,146 @@ us,scenario_045,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model us,scenario_045,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so it never submitted the required key through the submit_outputs contract. The correct derivation is 6.2% and 1.45% of the $36,276 wage base, $2,249.09 + $526.00 = $2,775.08." us,scenario_045,payroll_tax,minimax-m3,llm_error,other,False,"The model's reasoning was fully correct — $36,276 × 0.062 = $2,249 Social Security, $36,276 × 0.0145 = $526 Medicare, no Additional Medicare Tax, no Michigan employee payroll tax, ""Total ≈ $2,775"" — but it submitted 2837 in the value field, contradicting its own stated total by $62. The failure is the transcription of a derived answer, not the tax rule." us,scenario_045,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It applied Michigan's 200% FPL BBCE gross screen to full wages and never invoked categorical eligibility through TANF non-cash assistance, which waives the gross test. Its net income test then failed only because it counted $880 rent as the entire shelter cost and omitted Michigan's standard utility allowance, stopping at the $1,776 pre-shelter subtotal instead of the excess shelter deduction that brings net income to $1,102.06, below the $1,304.17 net limit. It also never reached the one- and two-person minimum allotment ($23.84/month, $24.37 from October) that applies once 30% of net income exceeds the $298 max allotment." -us,scenario_045,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It disqualified the household on Michigan's ~$2,610 200%-FPL BBCE gross screen alone; categorical eligibility through TANF non-cash assistance carries the household past the gross test, leaving the net income test controlling, and net income of $1,102.06 clears the $1,304.17 limit. With 30% of net income ($330.60) above the $298 max allotment, the benefit is the one- and two-person minimum allotment — $23.84/month through September and $24.37 from October — for $287.68." -us,scenario_045,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premium from countable income, a deduction SNAP grants only as an excess medical deduction to elderly or disabled members, and then set the benefit at roughly 30% of that fabricated net income rather than maximum allotment minus 30% of net income. The correct deduction set (20% earned income, standard, $433.33 child support, excess shelter with Michigan's utility allowance) yields net income of $1,102.06, an expected contribution of $330.60 that exceeds the $298 max allotment, and therefore the minimum allotment floor totaling $287.68 — not $3,100." -us,scenario_045,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct structural conclusion — a negative computed allotment paying the roughly $23/month minimum, which it itself scored at about $276/year — and then discarded that figure to submit an unexplained $2,718. The floor it identified is the one- and two-person minimum allotment of $23.84/month under FY2026 parameters and $24.37/month from October under FY2027 parameters, which sums to exactly $287.68; it also zeroed the excess shelter deduction by omitting Michigan's standard utility allowance from shelter costs." -us,scenario_045,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It asserted net income still exceeds the $1,305 net limit, which holds only because it built the shelter deduction from rent alone and left out both the full $433.33/month legally obligated child support deduction and Michigan's standard utility allowance; with those, net income is $1,102.06 and the net test passes. It then never applied the minimum allotment that governs when the 30% expected contribution of $330.60 exceeds the $298 max allotment, producing $287.68 for the year." -us,scenario_045,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It disqualified the household on a single step — gross wages of $36,276 against Michigan's ~$31,300 200%-FPL gross limit — treating the gross screen as dispositive when categorical eligibility from TANF non-cash assistance removes it. The controlling test is net income of $1,102.06 against the $1,304.17 limit, and because 30% of net income exceeds the $298 max allotment, the household receives the minimum allotment of $23.84/month rising to $24.37 in October, or $287.68 annually." -us,scenario_045,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the plain 130%-of-poverty gross income test, ignoring both Michigan's broad-based categorical eligibility and the household's categorical eligibility through TANF non-cash assistance, which waives the gross test entirely. Under the governing net income test, monthly net income of $1,102.06 sits below the $1,304.17 limit, and the resulting negative allotment ($298 max minus a $330.60 expected contribution) pays the one- and two-person minimum allotment for $287.68." -us,scenario_045,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Its own arithmetic reached net income near $1,069 and it stated this was below the poverty threshold and yields a small benefit, then it contradicted that and submitted $0 by claiming the calculation rounds to zero. Eligibility with a negative computed allotment pays the statutory floor, not zero: the one- and two-person minimum allotment of $23.84/month under FY2026 parameters and $24.37/month from October gives $287.68 for the year." -us,scenario_045,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It failed the household on the 130%-of-FPG gross income screen and reasoned that only an elderly or disabled member could exempt it, missing categorical eligibility through TANF non-cash assistance, which waives the gross test for this non-elderly household. Net income of $1,102.06 passes the $1,304.17 test, and since the $330.60 expected contribution exceeds the $298 max allotment, the minimum allotment applies for $287.68." -us,scenario_045,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It compared full gross wages of $36,276 to a 130%-of-poverty annual limit of about $21,060 and stopped there, applying a gross screen that categorical eligibility through TANF non-cash assistance removes. The net income test controls: $1,102.06 against $1,304.17, with the benefit set by the one- and two-person minimum allotment ($23.84/month, $24.37 from October) because 30% of net income exceeds the $298 max allotment." -us,scenario_045,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It declared $3,023 monthly gross income above Michigan's gross limit and explicitly refused to consider net income, but categorical eligibility through TANF non-cash assistance waives the gross test and makes the net computation the operative one. After the 20% earned income, standard, $433.33 child support, and excess shelter deductions, net income is $1,102.06, and the negative allotment result is floored at the minimum benefit, totaling $287.68." -us,scenario_045,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It disqualified the household solely on Michigan's 200% FPL gross income limit, missing that categorical eligibility through TANF non-cash assistance waives the gross test. Under the net income test the household qualifies at $1,102.06 versus $1,304.17, and because the $330.60 expected contribution exceeds the $298 max allotment, the benefit is the minimum allotment of $23.84/month rising to $24.37 in October, or $287.68." -us,scenario_045,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It gave only ""income exceeds the threshold,"" a $0 consistent with applying a gross income screen to $3,023/month and never computing net income. The correct chain is categorical eligibility from TANF non-cash assistance, net income of $1,102.06 under the $1,304.17 limit, and a minimum allotment floor of $23.84/month ($24.37 from October) because 30% of net income exceeds the $298 max allotment, giving $287.68." -us,scenario_045,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It cleared the gross test but concluded net income is too high for a positive benefit, which requires omitting Michigan's standard utility allowance from shelter costs; with rent plus the SUA the excess shelter deduction brings net income to $1,102.06, below the $1,304.17 net limit. Eligibility with a negative computed allotment ($298 max minus $330.60 expected contribution) pays the one- and two-person minimum allotment, $287.68 for the year." -us,scenario_045,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It applied a single-person gross income eligibility limit to $36,276 of wages and stopped, missing categorical eligibility through TANF non-cash assistance, which removes the gross screen. Net income of $1,102.06 passes the $1,304.17 test, and the minimum allotment ($23.84/month, $24.37 from October) applies because 30% of net income exceeds the $298 max allotment, yielding $287.68." -us,scenario_045,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no derivation; the $0 is consistent with screening $3,023/month gross income against a poverty-percentage limit and never running the deduction chain. Categorical eligibility from TANF non-cash assistance waives that gross screen, net income of $1,102.06 clears the $1,304.17 limit, and the negative allotment result is floored at the one- and two-person minimum allotment for $287.68." -us,scenario_045,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It rested entirely on gross income exceeding Michigan's 200% FPL threshold, missing that categorical eligibility through TANF non-cash assistance removes the gross test. The controlling net income test passes at $1,102.06 against $1,304.17, and since the $330.60 expected contribution exceeds the $298 max allotment, the household receives the minimum allotment totaling $287.68." -us,scenario_045,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It reached the correct final step — the deduction-adjusted contribution exceeds the one-person maximum allotment — and then reported $0 instead of the statutory floor. When maximum allotment minus the 30% expected contribution ($298 minus $330.60) is negative for an eligible one- or two-person household, the minimum allotment of 8% of the one-person max applies: $23.84/month through September and $24.37/month from October, for $287.68." -us,scenario_045,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It stated the exact condition that triggers the minimum benefit — a 30% net income contribution exceeding the one-person maximum allotment — and drew the wrong conclusion, reporting $0 rather than the floor. Eligible one- and two-person households receive the minimum allotment of 8% of the one-person max: $23.84/month under FY2026 parameters and $24.37/month from October, summing to $287.68; its companion claim that gross income disqualifies is also displaced by categorical eligibility through TANF non-cash assistance." -us,scenario_045,snap,glm-5.2,llm_error,categorical_eligibility,False,"It failed the household on the 130%-of-FPG gross income test and declared eligibility closed, ignoring both Michigan's BBCE and categorical eligibility through TANF non-cash assistance, which waives the gross test. Net income is $1,102.06 against a $1,304.17 limit, and the resulting negative allotment is floored at the one- and two-person minimum allotment, $287.68 for the year." -us,scenario_045,snap,glm-5.3,llm_error,categorical_eligibility,False,"It applied a 130%-of-poverty gross screen and used an incorrect figure of about $1,400/month for it (130% of the $1,304.17 guideline is $1,695), then never ran the net income test that actually governs once categorical eligibility from TANF non-cash assistance waives the gross test. Net income of $1,102.06 qualifies the household, and the minimum allotment floor yields $287.68." -us,scenario_045,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It asserted that wage income is too high for eligibility without computing any deduction, a $0 consistent with a bare gross income screen on $36,276. Categorical eligibility through TANF non-cash assistance removes that screen; net income after the 20% earned income, standard, $433.33 child support, and excess shelter deductions is $1,102.06, under the $1,304.17 limit, and the minimum allotment floor gives $287.68." -us,scenario_045,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It declined to compute, claiming household size and qualifying status details were missing, when the prompt fixes a one-person household (head, age 44) and directs that every unlisted fact be treated as zero or false. The listed facts fully determine the result: categorical eligibility through TANF non-cash assistance, net income of $1,102.06 against a $1,304.17 limit, and a minimum allotment floor of $23.84/month rising to $24.37 in October for $287.68." -us,scenario_045,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It applied the non-elderly, non-disabled gross income limit to $3,023 monthly gross income and declared no benefit payable before any deductions, missing categorical eligibility through TANF non-cash assistance, which waives the gross test. The net income test governs and passes at $1,102.06 versus $1,304.17, with the one- and two-person minimum allotment producing $287.68." -us,scenario_045,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It disqualified the household on ""the applicable Michigan SNAP income threshold"" applied to gross annual wages, never running deductions, and thereby skipped the categorical eligibility pathway from TANF non-cash assistance that waives the gross test. Net income of $1,102.06 clears the $1,304.17 net limit, and because 30% of net exceeds the $298 max allotment the minimum allotment applies for $287.68." -us,scenario_045,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It named the right deduction set (earned income, child support, standard, shelter) but concluded no positive allotment, which follows only if the shelter deduction counts $880 rent alone; including Michigan's standard utility allowance produces the excess shelter deduction that lands net income at $1,102.06, below the $1,304.17 limit. It then omitted the step that resolves a negative allotment for an eligible one-person household — the minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_045,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It compared annual wages of $36,276 to a one-person gross income limit and stopped, applying a screen that categorical eligibility through TANF non-cash assistance removes. Under the governing net income test the household qualifies at $1,102.06 against $1,304.17, and the negative allotment result ($298 max minus a $330.60 expected contribution) is floored at the minimum allotment for $287.68." -us,scenario_045,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"It tested gross wages against Michigan's one-person gross income limit and specifically reasoned that the child support expense does not bring gross income under it, treating the gross screen as decisive when categorical eligibility through TANF non-cash assistance waives it. The child support payment operates in the net computation, where $433.33/month plus the earned income, standard, and excess shelter deductions gives net income of $1,102.06 and a minimum-allotment benefit of $287.68." -us,scenario_045,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed income exceeds the net income limit after deductions, which requires dropping the excess shelter deduction: rent of $880 plus Michigan's standard utility allowance exceeds half of the post-deduction income, producing the shelter deduction that brings net income to $1,102.06, under the $1,304.17 limit. It also skipped the minimum allotment floor that governs when the $330.60 expected contribution exceeds the $298 max allotment, giving $287.68." -us,scenario_045,snap,grok-4.5,llm_error,categorical_eligibility,False,"It failed the household on the 130%-of-FPG gross income test alone, missing categorical eligibility through TANF non-cash assistance, which waives that test. Net income of $1,102.06 passes the $1,304.17 net test, and because 30% of net income ($330.60) exceeds the $298 one-person max allotment, the minimum allotment of $23.84/month rising to $24.37 in October yields $287.68." -us,scenario_045,snap,grok-4.6,llm_error,categorical_eligibility,False,"It computed countable income at 232% of the poverty guideline and rejected eligibility under both the 130% and 200% gross screens, reasoning that child support only enters as a deduction — but categorical eligibility through TANF non-cash assistance removes the gross test entirely, so neither screen binds. In the net computation the $433.33/month child support deduction plus the earned income, standard, and excess shelter deductions gives $1,102.06 net income, and the minimum allotment floor produces $287.68." -us,scenario_045,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross test to $36,276 and noted the head is neither elderly nor disabled, missing that categorical eligibility through TANF non-cash assistance — not an age or disability exemption — waives the gross screen here. The household passes the net income test at $1,102.06 against $1,304.17 and receives the one- and two-person minimum allotment, $287.68 for the year." -us,scenario_045,snap,inkling,llm_error,categorical_eligibility,False,"It disqualified the household on the 130% FPL gross limit and asserted no deduction structure can overcome that limit, missing categorical eligibility through TANF non-cash assistance, which waives the gross test outright. Deductions then carry the net computation to $1,102.06 against a $1,304.17 limit, and the negative allotment result is floored at the minimum allotment of $23.84/month ($24.37 from October) for $287.68." -us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for snap, so no substantive derivation exists to evaluate. The required answer is $287.68: categorical eligibility through TANF non-cash assistance, net income of $1,102.06 under the $1,304.17 limit, and a minimum allotment of $23.84/month through September plus $24.37/month from October." -us,scenario_045,snap,kimi-k3,llm_error,categorical_eligibility,False,"It screened $3,023 monthly gross income against the 200% FPL BBCE limit of about $2,608 and treated that as final, arguing child support is only a later deduction; the decisive omission is categorical eligibility through TANF non-cash assistance, which waives the gross income test regardless of where child support enters. Under the net income test the household qualifies at $1,102.06 versus $1,304.17, and the minimum allotment floor yields $287.68 for the year." -us,scenario_045,snap,minimax-m3,llm_error,categorical_eligibility,False,"It used a one-person gross income limit of roughly $1,800/month against $3,023 and concluded ineligibility, applying a gross screen that categorical eligibility through TANF non-cash assistance removes. Net income of $1,102.06 clears the $1,304.17 net limit, and since the $330.60 expected contribution exceeds the $298 max allotment, the minimum allotment applies for $287.68." -us,scenario_045,snap,ox-alpha,llm_error,categorical_eligibility,False,"It disqualified the household on a ~$1,700/month 130%-of-poverty gross limit and computed nothing further, missing categorical eligibility through TANF non-cash assistance, which waives the gross test. The net income test governs and passes at $1,102.06 against $1,304.17, with the negative allotment result floored at the one- and two-person minimum allotment for $287.68." -us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"Its explanation concluded ineligibility under a $1,766/month 130% FPL gross screen while its submitted value was $7,032 — $586/month, nearly double the $298 one-person maximum allotment, so no SNAP rule produces it. Both halves are wrong: categorical eligibility through TANF non-cash assistance waives the gross screen, and with net income of $1,102.06 the 30% contribution of $330.60 exceeds the max allotment, leaving the minimum allotment of $23.84/month rising to $24.37 in October, or $287.68." -us,scenario_045,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted net income after deductions is above the eligibility threshold without showing the shelter step; including rent of $880 plus Michigan's standard utility allowance yields the excess shelter deduction that puts net income at $1,102.06, below the $1,304.17 net limit. It then also skipped the minimum allotment floor that governs when the $330.60 expected contribution exceeds the $298 max allotment, giving $287.68 for the year." +us,scenario_045,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 in employer health insurance premiums from income, but SNAP allows no such deduction; medical expenses count only for elderly or disabled members. It also took the $5,200 child support out before any eligibility test. It even said the household exceeded the income limit, then paid a benefit anyway. It never ran the 130% gross test or Michigan's 200% FPL BBCE limit on the full $3,022.97 a month in wages (232% FPL), which the household fails, so the benefit is $0." +us,scenario_045,snap,claude-opus-4.7,llm_error,state_local_rule,False,"It claimed Michigan excludes child support paid from gross income, and used that to bring gross income ($31,076) just under the 200% FPL BBCE limit. Michigan actually deducts child support at the net income step (273.9(d)(5)), so gross income stays at $36,276 (232% FPL), above both the 130% and 200% limits, and the household is ineligible. Its own benefit arithmetic ($293 maximum minus $616, 30% of net income) also came out below zero, yet it submitted $2,718 with no calculation behind it." +us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so nothing substantive can be graded. The correct answer is $0: gross income of $3,022.97 a month is 232% FPL, above both the 130% gross limit and Michigan's 200% BBCE limit." +us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly found that $3,023 a month exceeds the 130% FPL gross limit and that the household is not eligible, which means $0. It submitted $7,032 anyway, a value its own explanation does not support. It also never checked Michigan's 200% FPL BBCE limit (which the household also fails), and it overstated the 130% limit as $1,766 when the guideline gives about $1,695 a month." us,scenario_045,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It set Michigan's 2026 single personal exemption at $5,900 instead of $5,950, then abandoned its own arithmetic: after computing $1,291 (and floating $1,272), it submitted $1,229, a figure none of its stated derivations produce. Applying the correct exemption to the full $36,276 wage base at the 4.25% flat rate gives $30,325.59 taxable and $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The structure was right — full wages as Michigan AGI, one personal exemption, flat 4.25% — but it used a $5,800 exemption instead of Michigan's 2026 value of $5,950. That $150 shortfall in the exemption inflated taxable income to $30,476 and the tax to $1,295.23 against the correct $30,325.59 and $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It imported the federal single standard deduction ($14,600) into Michigan's base, which Michigan does not allow — the state base is AGI less the $5,950 personal exemption — and then subtracted the $21,208 employer-sponsored insurance premium from AGI a second time even though the $36,276 wage figure is already the AGI. It then zeroed the result with the Michigan homestead property tax credit, which is refundable and therefore excluded from a before-refundable-credits figure, discarding the entire $1,288.84 liability." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It applied the correct Michigan method — $36,276 AGI, one personal exemption, 4.25% flat rate — with a $5,800 exemption instead of the 2026 figure of $5,950. Taxable income of $30,476 rather than $30,325.59 produced $1,295 in place of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its derivation used a $5,800 exemption and reached $1,295.23, already $6.39 high because Michigan's 2026 exemption is $5,950, and it then submitted $1,024 — a value inconsistent with its own stated arithmetic and with any Michigan exemption. Backing $1,024 out of the 4.25% rate implies a $12,182 exemption that no Michigan provision supplies." -us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It used a $5,800 exemption rather than $5,950 and then shaved roughly $60 off its $1,295 figure for unnamed nonrefundable credits; Michigan offers this single childless filer no nonrefundable credit, and its only relevant credit, the homestead property tax credit, is refundable and excluded here. The unadjusted correct computation is $30,325.59 × 4.25% = $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied a 4.05% rate, which was Michigan's one-year 2023 reduction that reverted to the statutory 4.25% for later years, and paired it with a $6,000 exemption instead of $5,950. Both errors are in the reference computation: $30,325.59 × 4.25% = $1,288.84, not $30,276 × 4.05% = $1,226.18." -us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $1,295 from a $5,800 exemption, then raised the submitted figure to $1,490 for an invented ""exemption phase-in"" adjustment; Michigan's personal exemption does not phase out with income, and the 2026 amount is a flat $5,950 for this single filer. The correct chain is $36,275.59 − $5,950 = $30,325.59 at 4.25% = $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premium from wages to get a $15,068 AGI, double-counting an exclusion already reflected in the $36,276 wage figure, which is Michigan's AGI starting point. Compounded by a $6,200 exemption instead of $5,950, this cut taxable income from $30,325.59 to $8,868 and the tax from $1,288.84 to $376.89." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 employer-sponsored insurance premium as an AGI reduction, using $15,068 rather than the $36,275.59 AGI that Michigan's tax starts from, and used a $5,670 exemption instead of $5,950. That produced $9,398 of taxable income against the correct $30,325.59, so its $399 is under a third of the $1,288.84 liability." -us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Every structural step matched the reference — full wage AGI, one exemption, 4.25% flat rate — but the exemption used was $5,600 rather than Michigan's 2026 amount of $5,950. The $350 gap raised taxable income to $30,676 and the tax to $1,303.73 instead of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It netted the $21,208 employer-sponsored insurance premium out of wages to reach a $15,068 Michigan AGI; the premium is not deductible from AGI, and Michigan's base here is the full $36,275.59. With a $5,600 exemption instead of $5,950 on top of that, it taxed $9,468 rather than $30,325.59 and reported $402.39 instead of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It named the 4.25% rate and ""exemptions"" but reported no AGI, no exemption amount, and no product, submitting $1,150. That figure implies a $9,217 exemption; Michigan's 2026 single personal exemption is $5,950, and $30,325.59 × 4.25% = $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $30,476 taxable income comes from subtracting a $5,800 exemption from the $36,276 AGI, but Michigan's 2026 personal exemption is $5,950, giving $30,325.59. At the 4.25% rate that $150 difference is the entire $6.16 gap between its $1,295 and the correct $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used $15,068 as federal AGI, having removed the $21,208 employer-sponsored insurance premium from the $36,276 of wages that already constitute AGI, and applied a $5,600 exemption instead of $5,950. Michigan taxes $30,325.59 here at 4.25% for $1,288.84, not the $9,468 base behind its $402.39." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It asserted that exemptions and standard deductions wipe out Michigan taxable income, but Michigan grants no standard deduction to a 44-year-old wage earner and its single personal exemption is $5,950 against $36,275.59 of AGI. That leaves $30,325.59 taxable at 4.25%, a $1,288.84 liability rather than zero." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted the $21,208 employer-sponsored insurance premium from wages to produce a $15,068 AGI, a subtraction Michigan's AGI does not take, and used a $5,600 exemption rather than $5,950. The correct base of $30,325.59 at 4.25% is $1,288.84, versus its $9,468 base and $402.39." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The method matched the reference exactly except for the exemption: $5,800 instead of Michigan's 2026 personal exemption of $5,950. Taxable income of $30,476 rather than $30,325.59 turned $1,288.84 into $1,295.23." -us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It subtracted a $5,800 personal exemption from the $36,276 AGI when Michigan's 2026 exemption is $5,950, overstating taxable income by $150. The 4.25% flat rate on the correct $30,325.59 yields $1,288.84, not the $1,295.23 it submitted." -us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It invented a $20,000 Michigan standard deduction, which does not exist — the state base is AGI less the $5,950 personal exemption — and then applied the homestead property tax credit as a nonrefundable offset, when that credit is refundable and excluded from this before-refundable-credits output. It compounded this by reasoning to $121.03 and submitting 0, discarding the full $1,288.84 liability." -us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a personal exemption near $5,200 rather than Michigan's 2026 amount of $5,950, taxing about $31,076 instead of $30,325.59. The 4.25% rate on the correct base is $1,288.84, not the $1,321 it reported." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It claimed Michigan's ""deduction structure"" leaves no taxable income at $36,276 of wages, but the only subtraction available to this filer is the $5,950 personal exemption, leaving $30,325.59 taxable. At the 4.25% flat rate that is $1,288.84, not zero." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It assumed unnamed standard deductions and nonrefundable credits fully offset the Michigan liability; Michigan provides no standard deduction here and no nonrefundable credit for a single childless filer, and its refundable homestead credit is excluded from this output. Wages of $36,275.59 less the $5,950 exemption at 4.25% leave $1,288.84 owing." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated Michigan's 2026 personal exemption at $5,900 instead of the actual $5,950, the sole deviation from the reference computation. Taxable income of $30,376 rather than $30,325.59 at 4.25% gives $1,290.98 instead of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It applied a 4.05% rate — Michigan's temporary 2023-only reduction, not the 4.25% statutory rate in force for 2026 — and a $5,800 exemption instead of $5,950. The correct product is $30,325.59 × 4.25% = $1,288.84, not $30,476 × 4.05% = $1,234.28." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It took federal AGI as $15,068 by removing the $21,208 employer-sponsored insurance premium from wages, but Michigan's tax starts from the full $36,275.59 AGI. With a $5,900 exemption instead of $5,950 layered on, it taxed roughly $9,168 rather than $30,325.59, reporting $390 against $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The single error is the exemption: $5,900 assumed where Michigan's 2026 personal exemption is $5,950. That $50 difference at the 4.25% flat rate accounts for the entire $2.14 gap between its $1,290.98 and the correct $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It estimated the 2026 Michigan personal exemption at $5,900 rather than $5,950, leaving $30,376 taxable instead of $30,325.59. Multiplying by the correct 4.25% rate gives $1,288.84, not the $1,290.98 submitted." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It asserted that a Michigan standard deduction plus exemptions offset all $36,276 of income; Michigan allows this filer only the $5,950 personal exemption and no standard deduction. The resulting $30,325.59 of taxable income carries $1,288.84 of tax at the 4.25% flat rate." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It started from a $15,068 AGI produced by subtracting the $21,208 employer-sponsored insurance premium from wages, a subtraction Michigan's AGI does not permit, and used a $5,880 exemption instead of $5,950. Taxing $9,188 rather than $30,325.59 cut the answer to $390 from $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly kept Michigan AGI at the full $36,276 with no additions or subtractions and applied the 4.25% flat rate, but projected the personal exemption at $5,800 rather than the 2026 figure of $5,950. Taxable income of $30,476 instead of $30,325.59 yields $1,295 rather than $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $5,000 personal exemption, roughly Michigan's level a decade earlier, against the 2026 amount of $5,950. That overstated taxable income by $950, so 4.25% of $31,276 gave $1,329 instead of 4.25% of $30,325.59, or $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Its pre-credit figure of about $1,287 came from a $6,000 exemption rather than $5,950, and it then subtracted roughly $1,159 of Michigan homestead property tax credit as if it reduced tax before refundable credits. Michigan's homestead property tax credit is refundable and is excluded from this output, leaving the full $1,288.84 in place of its $128." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so the submission failed the output contract rather than the tax computation. The required derivation is $36,275.59 AGI less Michigan's $5,950 personal exemption, taxed at the 4.25% flat rate, for $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reproduced the reference method precisely but set the Michigan personal exemption at $6,000 instead of $5,950. The $50 overstatement of the exemption at 4.25% is exactly the $2.11 by which its $1,286.73 falls short of $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It used a $5,000 exemption rather than $5,950 to compute about $1,329, then submitted 0 while describing an unspecified ""interaction with the exemption and credits"" that reduces nothing; the exemption is a flat $5,950 subtraction and no nonrefundable Michigan credit applies to this filer. The correct figure is $30,325.59 × 4.25% = $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It subtracted a $5,000 personal exemption from the $36,276 wage base, but Michigan's indexed 2026 exemption is $5,950. Taxing $31,276 instead of $30,325.59 at 4.25% produced $1,329.23 rather than $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $1,303.73 twice from a $5,600 exemption — already high because Michigan's 2026 exemption is $5,950 — and then submitted $1,116.01 under the label ""rounding,"" a number no step of its work produces. The correct chain is $36,275.59 − $5,950 = $30,325.59 at 4.25% = $1,288.84." -us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It zeroed Michigan taxable income using a standard deduction Michigan does not grant and a deduction for the $5,200 of child support paid, which is not deductible for federal AGI or for Michigan. The actual base is $36,275.59 less the $5,950 personal exemption, so $30,325.59 taxed at 4.25% is $1,288.84." -us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Ruled the household out of the Homestead Property Tax Credit because it rents rather than owns, but MI-1040CR expressly covers renters by deeming 23% of gross rent to be property tax ($10,560 × 0.23 = $2,428.80). Applying the 3.2% resource offset and the 60% non-senior rate to that amount produces the $760.79 it reported as $0." -us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Derived the reference exactly twice in its own reasoning — 23% of rent = $2,428.80, less 3.2% of household resources = $1,160.83, times 60% = $760.78 — then discarded that result and submitted $884 as a 'rounding adjustment'. No Michigan homestead parameter set produces $884 from these inputs; the correct answer was the number it had already computed." -us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Screened only the Michigan EITC and the Home Heating Credit, correctly zeroing both, and never applied the Homestead Property Tax Credit, which is the entire $760.79 reference and is claimed by renters on 23% of gross rent. It then submitted $1,209 with no derivation, contradicting its own finding that every credit it examined was zero." -us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the Michigan EITC as the only state refundable credit and dismissed the rest without evaluation. The Homestead Property Tax Credit is a refundable Michigan income tax credit claimed on MI-1040CR by renters at 23% of gross rent, and it supplies the full $760.79." -us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Rejected the Homestead Property Tax Credit on the ground that no property tax was listed, missing that a renter converts gross rent to deemed property tax at 23% ($2,428.80 on $10,560 of rent). Its EITC analysis was right, but discarding the homestead credit dropped all $760.79." -us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Correctly zeroed the Michigan EITC and then produced $1,089 from 'typical benefit outputs' instead of computing the credit that actually applies. The Homestead Property Tax Credit for this renter is 60% × (0.23 × $10,560 − 0.032 × $36,275.59) = $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Stopped at the Michigan EITC's 30%-of-federal structure and asserted no other refundable credit applies. The refundable Homestead Property Tax Credit pays a renter 60% of the excess of 23% of $10,560 rent over 3.2% of household resources, which is $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Set total household resources to $15,068 by netting the $21,208 employer-sponsored insurance premium out of wages, converted rent at 20% instead of the current 23%, and added a $102.60 state match on a federal EITC that is $0 for a childless filer at $36,276. THR is the full $36,275.59, so the offset is $1,160.82 and the credit is $760.79." -us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied three superseded parameters at once: a 20% rent conversion instead of 23%, a 3.5% resource offset instead of 3.2%, and an 80% credit rate instead of the 60% that applies to a non-senior, non-disabled claimant. The current parameters give 0.60 × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Used the correct 23% rent conversion, 3.2% offset and 60% rate but set total household resources to $15,068 — wages less the $21,208 employer-sponsored insurance premium. Health insurance premiums do not reduce THR for this claimant, so the offset is 3.2% × $36,275.59 = $1,160.82 and the credit is $760.79, not $1,167.97." -us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Asserted no qualifying Michigan refundable credit without evaluating the homestead credit, whose only required inputs — $10,560 of rent and $36,275.59 of household resources — were both given. The renter computation yields 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Reasoned solely from the Michigan EITC's link to the federal EITC, correctly $0, and never reached the refundable Homestead Property Tax Credit. That credit supplies the entire $760.79 for a renter paying $10,560 with $36,275.59 in household resources." -us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $977.89 homestead figure comes from converting rent at 20% instead of 23% and offsetting by 3.2% of $15,068 — wages net of the $21,208 insurance premium — rather than 3.2% of the full $36,275.59 THR; it also added a $62.10 state match on a federal EITC that is $0 at $36,276 for a childless filer. The correct chain is 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned zero with no derivation, omitting the Michigan Homestead Property Tax Credit. A renter paying $10,560 with $36,275.59 of household resources receives 60% of ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Claimed a $338.80 federal EITC for a 44-year-old childless filer earning $36,276, far above the childless phase-out ceiling, so the 30% Michigan match is $0 rather than $101.64. It also never computed the Homestead Property Tax Credit, which is the whole $760.79." -us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $695.48 equals 60% × ($2,428.80 − 3.5% × $36,276), so it used the pre-2018 3.5% resource offset. The current offset is 3.2%, giving $1,160.82, an excess of $1,267.98, and a credit of $760.79." -us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources to reach $31,076. MI-1040CR includes child support received in THR and allows no deduction for child support paid, so THR stays at $36,275.59, the offset is $1,160.82, and the credit is 60% × $1,267.98 = $760.79 rather than $860.62." -us,scenario_045,state_refundable_credits,glm-5.2,llm_error,other,False,"Its explanation walks through the homestead credit and lands on $570.70, then submits $0, so the value contradicts its own stated result. The $570.70 is itself short because it converted rent at 20% instead of 23%; the correct deemed tax is $2,428.80 and the credit is 60% × ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Considered only the Michigan EITC's federal linkage and declared all other Michigan refundable credits inapplicable. That dropped the Homestead Property Tax Credit, which pays this renter 60% of the excess of $2,428.80 deemed property tax over the $1,160.82 resource offset, or $760.79." -us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded no Michigan refundable credit is identifiable from the given facts, even though pre-subsidy rent of $10,560 plus household resources of $36,275.59 is exactly the input set the MI-1040CR renter computation needs. That computation is 60% × (0.23 × $10,560 − 0.032 × $36,275.59) = $760.79." -us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Treated the absence of a stated dependent or special condition as disqualifying for every Michigan refundable credit. The Homestead Property Tax Credit has no dependent requirement and pays a renter 60% of 23% of rent above 3.2% of household resources, which is $760.79 here." -us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Identified the right credit and used the correct 3.2% offset and 60% rate but converted rent at 20% rather than the 23% in force for 2026, starting from $2,112 instead of $2,428.80. With the correct conversion the excess is $1,267.98 and the credit is $760.79, not $570.70." -us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Computed household resources as $15,068 by netting out the $21,208 employer-sponsored insurance premium, which is not a deduction from THR, inflating the homestead credit to $1,168, and added a $104 Michigan EITC on a federal EITC that is $0 for a childless filer at $36,276. With THR of $36,275.59 the homestead credit alone is the answer: $760.79." -us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Dismissed all Michigan refundable credits as inapplicable to a childless household. The Homestead Property Tax Credit turns on rent and household resources, not dependents, and pays this renter 60% of ($2,428.80 − $1,160.82) = $760.79." -us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Applied the correct 23%/3.2%/60% structure but deducted the $21,208 employer-sponsored insurance premium from total household resources to reach $15,068. Medical insurance premiums do not reduce THR for this non-senior claimant, so the offset is 3.2% × $36,275.59 = $1,160.82 and the credit is $760.79, not $1,167.97." -us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Asserted with no analysis that no state refundable credit qualifies, skipping the Michigan Homestead Property Tax Credit entirely. That credit converts the $10,560 rent to $2,428.80 of deemed property tax and pays 60% of the amount above 3.2% of household resources — $760.79." -us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Compounded three errors: household resources of $15,068 (wages net of the insurance premium), a 3.5% resource offset instead of 3.2%, and a $20 state EITC computed at 6% on a federal EITC that is $0 at this income for a childless filer. The correct computation is 60% × ($2,428.80 − 0.032 × $36,275.59) = $760.79." -us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used the correct 23% conversion, 3.2% offset and 60% rate but reduced total household resources by the $5,200 of child support paid to $31,076. MI-1040CR provides no deduction for child support paid, so THR is $36,275.59, the excess is $1,267.98, and the credit is $760.79 rather than $861." -us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Declared no Michigan refundable credit applicable without evaluating the homestead credit. A renter claims it by deeming 23% of $10,560 rent ($2,428.80) to be property tax and taking 60% of the excess over 3.2% of household resources, giving $760.79." -us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Stated the Michigan EITC is 6% of federal — it is 30% for 2026, though $0 either way here — and then limited the homestead credit to the household's income tax liability. The Homestead Property Tax Credit is fully refundable and is paid in full irrespective of Michigan liability, so the $760.79 stands." -us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_refundable_credits, so no substantive computation reached the grader. The required answer is the $760.79 Michigan Homestead Property Tax Credit: 60% × (0.23 × $10,560 − 0.032 × $36,275.59)." -us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used the pre-2018 parameter pair — 20% of rent and a 3.5% resource offset — producing 60% × ($2,112 − $1,269.66) = $505.40. The 2026 parameters are a 23% rent conversion ($2,428.80) and a 3.2% offset ($1,160.82), giving 60% × $1,267.98 = $760.79." -us,scenario_045,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Claimed the renter's rent does not exceed the credit's threshold, so nothing is triggered. Deemed property tax of 23% × $10,560 = $2,428.80 exceeds the 3.2% resource offset of $1,160.82 by $1,267.98, and 60% of that excess is the $760.79 credit." -us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Converted rent at 17%, a rate that applies to none of this household's circumstances, and offset with 3.5% of resources instead of 3.2%, giving 60% × ($1,795.20 − $1,269.66) = $315.32. The correct conversion is 23% ($2,428.80) against a $1,160.82 offset, yielding $760.79." -us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Asserted that no homestead property tax credit qualifies on the listed facts, even though pre-subsidy rent of $10,560 is the only input the renter computation requires. Deemed property tax of $2,428.80 less 3.2% of $36,275.59 in resources, taken at the 60% non-senior rate, is $760.79." -us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Returned zero with no analysis, omitting the Michigan Homestead Property Tax Credit. It pays this renter 60% of ($2,428.80 deemed property tax − $1,160.82 resource offset) = $760.79." +us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model first computed the correct $1,291 using the $5,900 exemption. It then submitted $1,229, which equals 4.25% of $36,276 less the wrong $7,350 exemption it also floated. So it replaced the correct 2026 exemption with an inflated one." +us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a $5,800 personal exemption instead of Michigan's 2026 exemption of $5,900. Taxable income came out $100 too high ($30,476), and the tax was overstated by $4.25 at $1,295.23." +us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the $21,208 employer insurance premium input from wages, which gave an AGI of $15,068. It then applied the federal $14,600 standard deduction, which Michigan does not allow, in place of the $5,900 personal exemption. Finally it treated the refundable Homestead Property Tax Credit as a nonrefundable credit to reach $0, when Michigan tax on $30,375.59 at 4.25% is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a $5,800 personal exemption instead of the 2026 amount of $5,900. That gave taxable income of $30,476 and tax of $1,295 rather than $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own work gave $1,295.23 using a $5,800 exemption, which is already $100 below the correct $5,900. It then submitted $1,024, which nothing in its derivation supports. The final value contradicts its own calculation." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model used a $5,800 exemption instead of $5,900, which gave $1,295. It then subtracted about $60 of nonrefundable credits that do not exist for this filer to reach $1,235. A single childless wage earner has no Michigan nonrefundable credit, so tax before refundable credits is the full $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model subtracted a $5,800 personal exemption instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $100 and the tax by $4.25, giving $1,295.23." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% flat rate. It also used a $6,000 exemption instead of $5,900, giving $30,276 × 4.05% = $1,226.18 rather than $30,375.59 × 4.25% = $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed $1,295 with a $5,800 exemption, which is $100 below the correct $5,900. It then inflated the answer to $1,490 by citing a 'rounding' and 'exemption phase-in' adjustment that does not exist, so the submitted value contradicts its own arithmetic." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $15,068, which means it subtracted the $21,208 employer insurance premium input from wages even though that input does not reduce wage income. It also used a $6,200 exemption instead of $5,900. Taxable income should be $30,375.59, not $8,868." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $15,068 by subtracting the employer insurance premium input from wages, and used a $5,670 exemption instead of $5,900. That gave $399 instead of 4.25% of $30,375.59." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $5,600 personal exemption instead of Michigan's 2026 exemption of $5,900. Taxable income came out $300 too high and the tax was $1,303.73 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $6,000 personal exemption instead of $5,900. That understated taxable income by $100 and gave $1,286.73 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model took Michigan AGI as $15,068 by subtracting the $21,208 employer insurance premium input from wages, when AGI is the full $36,275.59. It also used a $5,600 exemption instead of $5,900, which cut taxable income to $9,468." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no derivation. At 4.25%, its $1,150 implies taxable income of about $27,060, which means it subtracted roughly $9,200 from $36,276 instead of the $5,900 personal exemption." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income of $30,476 shows it used a $5,800 exemption instead of Michigan's 2026 exemption of $5,900. The result was $1,295 instead of $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a federal AGI of $15,068 by subtracting the employer insurance premium input from wages, and a $5,600 exemption instead of $5,900. That shrank taxable income to $9,468 instead of $30,375.59." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model claimed that 'exemptions and standard deductions' eliminate the tax. Michigan has no standard deduction for a 44-year-old, and its only offset is the $5,900 personal exemption, which leaves $30,375.59 taxable and $1,290.96 of tax." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model started from an AGI of $15,068 by subtracting the $21,208 employer insurance premium input from wages, and used a $5,600 exemption instead of $5,900. That gave $402.39 instead of 4.25% of $30,375.59." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model subtracted a $5,800 personal exemption instead of the 2026 amount of $5,900. That gave taxable income of $30,476 and tax of $1,295.23." +us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used a $5,800 Michigan personal exemption instead of $5,900. That overstated taxable income by $100 and gave $1,295.23." +us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $20,000 Michigan standard deduction and subtracted the Homestead Property Tax Credit as if it were nonrefundable. That credit is refundable and does not reduce tax before refundable credits. The model then submitted $0 even though its own reasoning reached $121." +us,scenario_045,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used a personal exemption of about $5,200 instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $700 and gave $1,321." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that Michigan's 'deduction structure' leaves no taxable income. The only offset is the $5,900 personal exemption, which leaves $30,375.59 taxed at 4.25% for $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed that standard deductions and nonrefundable credits fully offset the tax. Michigan allows only the $5,900 personal exemption and no nonrefundable credit here, so tax before refundable credits is $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied a 4.05% rate instead of Michigan's 4.25% flat rate, and a $5,800 exemption instead of $5,900. That produced $30,476 × 4.05% = $1,234.28." +us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model had the right $5,900 exemption but started from an AGI of $15,068, subtracting the $21,208 employer insurance premium input from wages. AGI is the full $36,275.59 of wages, so taxable income is $30,375.59, not $9,168." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model claimed a Michigan standard deduction plus exemptions offset all $36,276 of income. Michigan has no standard deduction for this filer and only a $5,900 personal exemption, which leaves $30,375.59 taxable." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a federal AGI of $15,068 by subtracting the employer insurance premium input from wages, and a projected $5,880 exemption instead of $5,900. That gave taxable income of $9,188 instead of $30,375.59." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model projected a 2026 personal exemption of $5,800 instead of the published $5,900. That gave taxable income of $30,476 and tax of $1,295." +us,scenario_045,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model subtracted a $6,000 personal exemption instead of $5,900. That understated taxable income by $100 and gave $1,286.73." +us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption instead of Michigan's 2026 exemption of $5,900. That overstated taxable income by $900 and gave $1,329." +us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model subtracted about $1,159 of Homestead Property Tax Credit from its $1,287 tax, treating it as nonrefundable. That credit is refundable and excluded from tax before refundable credits. It also used a $6,000 exemption instead of $5,900." +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $1,290.96." +us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model allowed a $6,000 personal exemption instead of Michigan's 2026 exemption of $5,900. That gave taxable income of $30,276 and tax of $1,286.73." +us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model computed about $1,329 using a $5,000 exemption, $900 below the correct $5,900. It also correctly noted that the Homestead credit is refundable and excluded. It then submitted $0 anyway, which contradicts its own derivation." +us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption instead of $5,900. That gave taxable income of $31,276 and tax of $1,329.23." +us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's derivation gave $1,303.73 using a $5,600 exemption, $300 below the correct $5,900. It then submitted $1,116.01, which nothing in its reasoning supports, so the final value contradicts its own calculation." +us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted a standard deduction and the $5,200 of child support paid to reach $0. Michigan allows neither: child support paid is not deductible and there is no standard deduction for this filer. Only the $5,900 exemption applies, which leaves $30,375.59 taxable." +us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Said the homestead property tax credit needs home ownership and so excluded the renter. In fact Michigan treats 23% of rent as property tax, which gives a credit of 60% × ($2,428.80 − 3.2% × $36,276) = $760.79." +us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"Worked out the homestead credit correctly as $760.78: 23% of rent, minus 3.2% of resources, times 60%. It then submitted $884 citing an unexplained 'rounding' adjustment that nothing in its own working supports." +us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Considered only the EITC and the Home Heating Credit, concluded both were $0, and then submitted an unsupported $1,209. It never computed the renter's homestead property tax credit of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Counted only the Michigan EITC, which is $0, and left out the refundable Homestead Property Tax Credit. That credit is worth $760.79 to this renter, based on 23% of $10,560 rent." +us,scenario_045,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Treated 20% of rent ($2,112) as property tax instead of Michigan's 23% for renters whose heat isn't included ($2,428.80). This gave 60% × ($2,112 − $1,160.83) = $570.70 instead of $760.79." +us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Concluded the homestead credit didn't apply because no property tax was listed. It overlooked that Michigan counts 23% of the $10,560 rent as property tax, which gives a $760.79 refundable credit." +us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Correctly found federal and Michigan EITC to be $0, then made up $1,089 from a 'reduced federal EITC estimate'. It never computed the homestead property tax credit, which is the only refundable credit here, at $760.79." +us,scenario_045,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Got all three homestead parameters wrong: 20% of rent instead of 23%, a 3.5% income threshold instead of 3.2%, and a guessed ~55% rate instead of 60%. This gave $460 instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Considered only the Michigan EITC, which is $0, and said no other refundable credits apply. It left out the renter's refundable Homestead Property Tax Credit of $760.79." +us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Added a Michigan EITC based on a $342 federal EITC, but a childless filer earning $36,276 gets no federal EITC. It also cut household resources to $15,068 by subtracting the ESI premiums and used 20% of rent instead of 23%, which overstated the homestead credit at about $978." +us,scenario_045,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, a 3.5% resource threshold instead of 3.2%, and an 80% credit rate instead of 60%. This gave $673.87 instead of $760.79." +us,scenario_045,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23%, and never applied the 60% credit percentage to the excess over 3.2% of resources. It submitted the raw excess of $951.17 rather than 60% × $1,267.98 = $760.79." +us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Cut total household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums, so the 3.2% threshold fell to $482.18. The reference uses resources of $36,275.59, which gives 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Concluded that no Michigan refundable credits apply and left out the Homestead Property Tax Credit. That credit gives this renter $760.79: 60% of 23% of rent minus 3.2% of resources. +us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Considered only the Michigan EITC, which is $0, and never computed the refundable Homestead Property Tax Credit of $760.79 for the renter." +us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Added a $62.10 Michigan EITC from a nonexistent $207 federal EITC and used 20% of rent instead of 23%. Its $977.89 homestead figure also implies household resources of $15,068, with the ESI premiums wrongly subtracted, rather than $36,276." +us,scenario_045,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported $0 and left out the refundable Homestead Property Tax Credit. For this renter that credit is 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Gave a childless filer with $36,276 of wages a $338.80 federal EITC, although the childless EITC phases out below that income, and took 30% of it. It also left out the $760.79 homestead property tax credit entirely." +us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $695.48 equals 60% × ($2,428.80 − 3.5% × $36,276), so it used a 3.5% household-resource threshold instead of 3.2%. With 3.2% the credit is $760.79." +us,scenario_045,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, which lowered the 3.2% threshold to $994.43. The reference keeps resources at $36,275.59, which gives $760.79." +us,scenario_045,state_refundable_credits,glm-5.2,llm_error,other,False,"Computed a $570.70 homestead credit, itself understated because it used 20% of rent instead of 23%, and then submitted $0, contradicting its own working. The correct figure is 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Said no refundable Michigan credits apply beyond the $0 EITC, leaving out the renter's $760.79 Homestead Property Tax Credit." +us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable Michigan credits, overlooking the Homestead Property Tax Credit. That credit treats 23% of the $10,560 rent as property tax and yields $760.79." +us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Assumed that a Michigan refundable credit requires a dependent or some other unlisted condition. The Homestead Property Tax Credit needs only rent paid and resources under the limit, giving $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Treated 20% of rent as property tax instead of 23%. That gave 60% × ($2,112 − $1,160.83) = $570.70 instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Cut household resources to $15,068 by subtracting the ESI premiums, which inflated the homestead credit to $1,168. It also added a $104 state EITC, although federal EITC is $0 for a childless filer at $36,276." +us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,Treated Michigan's refundable credits as unavailable to a childless household. The Homestead Property Tax Credit has no children requirement and gives this renter $760.79. +us,scenario_045,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Deducted the $21,208 insurance premiums from household resources, giving $15,068 and a 3.2% threshold of only $482.18. The reference uses $36,275.59 of resources, which gives $760.79." +us,scenario_045,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and left out the 60% credit percentage. It submitted the raw $951 excess instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from household resources, giving $31,076, which lowered the 3.2% threshold. The reference keeps resources at $36,275.59, which gives $760.79." +us,scenario_045,state_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Subtracted the $21,208 insurance premiums from household resources, giving $15,068, and used 20% of rent instead of 23%, producing $977.89. The correct figure is 60% × ($2,428.80 − 3.2% × $36,275.59) = $760.79." +us,scenario_045,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no state refundable credits qualify, leaving out the renter's refundable Homestead Property Tax Credit of $760.79." +us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Calculated the homestead credit on $15,068 of resources (with ESI premiums subtracted) and a 3.5% threshold, giving $1,141. It then added a 6% Michigan EITC on a nonexistent $341 federal EITC, when the credit is 60% × ($2,428.80 − $1,160.82) = $760.79 and EITC is $0." +us,scenario_045,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,200 of child support paid from total household resources, giving $31,076, which inflated the credit to $861. The reference uses resources of $36,275.59, which gives $760.79." +us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Reported no Michigan refundable credits and left out the Homestead Property Tax Credit. That credit is 60% of 23% of rent minus 3.2% of resources, which is $760.79." +us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the Homestead Property Tax Credit as limited by tax liability, when it is fully refundable and paid regardless of liability. It also used an outdated 6% EITC rate, though the EITC is $0 either way. The credit is $760.79." +us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so nothing could be scored against the $760.79 homestead credit." +us,scenario_045,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used 20% of rent instead of 23% and a 3.5% resource threshold instead of 3.2%. This gave 60% × ($2,112 − $1,269.66) = $505.40 instead of $760.79." +us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Wrongly treated the listed $10,560 pre-subsidy rent as not actually paid and so denied the homestead credit. At 23% of that rent, $2,428.80 exceeds 3.2% of resources by $1,267.98, and 60% of that is $760.79." +us,scenario_045,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used 17% of rent as property tax instead of 23% and a 3.5% threshold instead of 3.2%. This gave $315.32 instead of 60% × ($2,428.80 − $1,160.82) = $760.79." +us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Concluded the homestead property tax credit did not qualify on the listed facts. The $10,560 of rent paid is enough, since 23% of it counts as property tax, and the credit is $760.79." +us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Said no refundable state credits apply, leaving out the renter's refundable Michigan Homestead Property Tax Credit of $760.79." +us,scenario_046,child1_chip_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied a generic 255% FPL CHIP limit that is not Oklahoma's rule. In Oklahoma a 10-year-old falls outside the separate CHIP's age coverage and gets children's coverage only through SoonerCare Medicaid, whose income test the household also fails. It also understated MAGI as $77,164 by subtracting the $14,839 FLSA overtime premium ($92,003 - $14,839), but the overtime deduction does not reduce AGI-based MAGI. Correct MAGI is about $92,003, well above Oklahoma's children's limit, so the answer is not eligible." us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model asserted that the child was “likely eligible” without testing Oklahoma's income limits or any categorical pathway. At 2.79 times FPL, child1 fails the regular children's Medicaid limit, CHIP limit, and all other applicable categories, yielding medicaid_category = NONE and value = 0." +us,scenario_046,child2_chip_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's MAGI of $77,164 is exactly $92,003 minus the $14,839 FLSA overtime premium. It treated the no-tax-on-overtime deduction as lowering MAGI, but that deduction is taken below the line and never reduces AGI or Medicaid/CHIP MAGI. Using the correct MAGI of $92,003, the four-person household sits at about 280% FPL. That is above even the 255% CHIP limit the model cited, so child2 is not CHIP-eligible." us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that being a child made Child 2 eligible without testing Oklahoma's categorical and income requirements. At 2.79 times FPL, Child 2 exceeds all applicable child Medicaid limits and qualifies through no other pathway, so the correct value is 0." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached taxable income of $44,864, tax of $4,888, and $4,400 of CTC, landing at roughly $488 — within rounding of the reference — and then discarded that result to submit $3,465, a figure no step of its derivation produces. The only substantive slips in the derivation were a $32,300 standard deduction instead of $32,200 and an approximated first bracket; the error that produced the wrong answer is the abandonment of its computed $488." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used a $28,700 MFJ standard deduction instead of the 2026 figure of $32,200 and never applied the OBBBA qualified-overtime deduction for the $14,838.71 FLSA premium, leaving taxable income of $63,303 instead of $44,964.29. It then fabricated a $2,500 American Opportunity Credit despite zero qualified tuition expenses and applied only $1,292 of CTC instead of the full $4,400 ($2,200 x 2) that offsets liability." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Omitted the OBBBA qualified-overtime deduction of $14,838.71 and used a $31,500 standard deduction and a $2,000-per-child CTC instead of $32,200 and $2,200, so it derived $2,777 from $60,503 of taxable income rather than $499.71 from $44,964.29. It then submitted $4,669, a value none of its own steps produces." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Never applied the qualified-overtime deduction, so it started from $59,703 of taxable income instead of $44,964.29, and then submitted its pre-credit tax of about $6,684 as the final answer after stating that nonrefundable credits drive liability near zero. The requested output is net of nonrefundable credits, and the $4,400 CTC ($2,200 x 2) reduces the engine's $4,899.71 to $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Reproduced the engine's derivation — AGI $92,003, standard deduction $32,200, overtime deduction $14,839, taxable income $44,964 — then explicitly reported the pre-child-credit tax and declined to subtract the $4,400 nonrefundable CTC the requested output requires. Its bracket arithmetic also used a $24,000/$2,400 first bracket rather than $24,800/$2,480, so the pre-credit figure it reported overstates even the engine's $4,899.71." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Anchored on the 2025 $30,000 MFJ standard deduction rather than the 2026 $32,200 and omitted the OBBBA qualified-overtime deduction entirely, producing taxable income of $62,003 instead of $44,964.29. It also used $2,000 per child for the CTC instead of $2,200, so the $4,000 it subtracted understates the $4,400 of nonrefundable credit." -us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Dismissed the $14,839 FLSA overtime premium as informational rather than the basis of the OBBBA qualified-overtime deduction, leaving taxable income at $59,803 instead of $44,964.29. It then abandoned its own $6,696 pre-credit computation and submitted $12,200, justified only as the output of a 'broader calculation model' — a figure exceeding any tax its derivation supports." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Built a $17,430 medical itemized deduction out of $24,330 of health-insurance premiums and pushed itemized deductions to $39,845, while this household's itemized total falls short of the $32,200 standard deduction the engine applied. It also omitted the $14,838.71 qualified-overtime deduction and limited the nonrefundable CTC to $400 by subtracting a $1,800 refundable ceiling per child; the full $4,400 offsets liability first, with refundability reached only when tax runs out." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied a TCJA sunset that does not occur, claiming $20,680 of personal exemptions and taxing at pre-TCJA 10%/15% rates; for 2026 OBBBA keeps personal exemptions at zero and the second bracket at 12%. It also omitted the $14,838.71 qualified-overtime deduction and used a $2,000 total CTC where two children yield $4,400 at $2,200 each." -us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the pre-TCJA regime for 2026, subtracting $21,200 of personal exemptions that no longer exist and itemizing $26,958.78 including a medical deduction, where the $32,200 standard deduction governs. It never applied the $14,838.71 qualified-overtime deduction and credited $2,000 of CTC instead of $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Grossed wages up to $106,839 by adding the $14,839 FLSA overtime premium, though the prompt states gross wages already include overtime, and separately stripped the $9,208 employer-sponsored premium from wages already stated net of the pre-tax exclusion. It compounded this with four $5,000 personal exemptions from an assumed TCJA sunset and a $2,000 CTC, and never took the qualified-overtime deduction that brings taxable income to $44,964.29." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its stated method — $92,003 AGI less a $31,200 standard deduction, taxed at 2026 MFJ rates, less $4,000 of CTC — yields about $2,900, not the $8,518 submitted, so the answer does not follow from its own arithmetic. The method itself also omits the $14,838.71 qualified-overtime deduction and uses $2,000 per child rather than $2,200." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $9,208 employer-sponsored insurance premium from AGI even though the stated $92,000 of gross wages is already the taxable wage figure, then applied $20,200 of personal exemptions from a TCJA sunset that OBBBA foreclosed for 2026. It never took the $14,838.71 qualified-overtime deduction and credited only $2,000 of CTC for two children instead of $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Reached $419 through offsetting errors: it stripped $9,208 of employer-sponsored premiums from AGI, claimed $20,200 of personal exemptions from a TCJA sunset that does not occur in 2026, and invented a $1,500 nonrefundable American Opportunity Credit with zero qualified tuition expenses listed. The correct path keeps AGI at $92,003, takes the $32,200 standard deduction plus the $14,838.71 overtime deduction, and offsets $4,899.71 of tax with $4,400 of CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $3,212 is consistent with taxing about $64,000 of income — no qualified-overtime deduction and an undersized standard deduction — at 2026 MFJ rates and then subtracting a $4,000 CTC at $2,000 per child. The correct derivation subtracts the $32,200 standard deduction and the $14,838.71 overtime deduction from $92,003, taxes $44,964.29 at $4,899.71, and applies $4,400 of CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $1,858.40 is reproduced exactly by removing the $9,208 employer-sponsored premium from AGI ($82,795), applying the 2025 $30,000 MFJ standard deduction ($52,795 taxable), taxing at $2,385 plus 12% of $28,945 = $5,858.40, and subtracting a $2,000-per-child CTC. Every one of those steps is wrong for 2026: the stated wages are already the taxable amount, the standard deduction is $32,200, the $14,838.71 overtime deduction goes untaken, and the CTC is $2,200 per child." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no arithmetic; $2,722.40 is consistent with taxing about $60,000 — AGI less a standard deduction alone — and subtracting a $4,000 two-child CTC. It omitted the $14,838.71 OBBBA qualified-overtime deduction that brings taxable income to $44,964.29 and used $2,000 rather than $2,200 per child." -us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Took the 2025 $30,000 MFJ standard deduction rather than the 2026 $32,200 and never applied the $14,838.71 qualified-overtime deduction, so taxable income came out at $62,003 instead of $44,964.29 and pre-credit tax at $6,963.36 instead of $4,899.71. Its $4,000 CTC also understates the $4,400 available at $2,200 per child." -us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so no substantive computation was submitted for scoring." -us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Reproduced the engine's taxable income of $44,964 and pre-credit tax of $4,899.68 exactly, then limited the nonrefundable CTC to $1,000 by computing 2 x ($2,200 - $1,700), treating the $1,700 per-child refundable ceiling as carved out of the nonrefundable credit. The full $4,400 applies against liability first and the additional CTC becomes refundable only to the extent tax runs out, leaving $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Abandoned the computation and submitted a 'conservative' zero rather than deriving liability from the stated facts, which fully determine it: $92,003 AGI less the $32,200 standard deduction and the $14,838.71 qualified-overtime deduction gives $4,899.71 of tax less $4,400 of CTC. Its zero also rests on an American Opportunity Credit that is zero here, since no qualified tuition expense is listed." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Submitted essentially the pre-credit tax on about $60,000 of taxable income and subtracted nothing for the $4,400 CTC, while invoking an American Opportunity Credit that is zero absent any listed qualified tuition expense. It also skipped the $14,838.71 qualified-overtime deduction that reduces taxable income to $44,964.29." -us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the qualified-overtime deduction at the $12,500 single-filer cap instead of the $25,000 joint cap, deducting $12,500 rather than the full $14,838.71 FLSA premium. That single substitution raises taxable income by $2,338.71 and tax by $280.65, which is exactly the gap between its $780.36 and the reference $499.71; its $32,200 standard deduction and $4,400 CTC are otherwise correct." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Offered no derivation beyond an AOTC phaseout that is irrelevant — the credit is zero because no qualified tuition expense is listed, not because of the $160,000 MFJ phaseout — and the $8,500 submitted exceeds the pre-credit tax on this household under any 2026 bracket set. The correct figure is $4,899.71 of tax on $44,964.29 of taxable income less $4,400 of nonrefundable CTC." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed the TCJA sunset, claiming four ~$5,460 personal exemptions ($21,840), a 15% second bracket, and $1,000 per child of CTC; for 2026 there are no personal exemptions, the second bracket is 12%, and the CTC is $2,200 per child. It also itemized $28,861 including a $5,429 medical deduction and $2,553 of estimated Oklahoma income tax, where the $32,200 standard deduction plus the $14,838.71 overtime deduction governs." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied a post-sunset 2026 with $21,168 of personal exemptions and $1,000-per-child CTCs, both eliminated by OBBBA's permanent extension, and itemized $28,896 including $5,430 of medical expenses and $2,587 of Oklahoma tax rather than taking the $32,200 standard deduction. It never applied the $14,838.71 qualified-overtime deduction, so its taxable income of $41,939 and $2,000 of credits replace $44,964.29 and $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Reported $23,508 of taxable income where the correct figure is $44,964.29, taxed the whole amount in a pre-TCJA 10% bracket, and subtracted a $2,000 CTC at $1,000 per child. For 2026 the deductions are the $32,200 standard deduction plus the $14,838.71 qualified-overtime deduction, the schedule is 10%/12% with the joint 10% bracket ending at $24,800, and the CTC is $4,400." -us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Stacked an inflated ~$19,193 medical itemized deduction built from health-insurance premiums onto $10,850 of SALT and $12,415 of mortgage interest, driving taxable income to $34,706; this household's itemized deductions fall below the $32,200 standard deduction the engine used, and the actual property tax is $8,464. With the correct $44,964.29 of taxable income, the $4,899.71 of tax exceeds the $4,400 CTC and leaves $499.71 rather than zero." -us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so no substantive computation was submitted for scoring." -us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Submitted $9,999 with no derivation — a placeholder that exceeds the household's entire pre-credit tax of $4,899.71 on $44,964.29 of taxable income and ignores the $4,400 of nonrefundable CTC that brings liability to $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Correctly applied the $14,838.71 qualified-overtime deduction and the $4,400 CTC but itemized $40,772, including an $18,080 medical deduction assembled from $24,980 of health-insurance premiums and over-the-counter expenses; this household's itemized deductions do not reach the $32,200 standard deduction the engine applied. That inflated deduction cut taxable income to $36,392 and tax to $3,871, letting the CTC absorb the entire liability instead of leaving $499.71." -us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Itemized $38,958.77 on the strength of an $18,079.77 medical deduction built from $24,980 of insurance premiums and over-the-counter expenses, where the $32,200 standard deduction governs, and it omitted the $14,838.71 qualified-overtime deduction. It also claimed a $1,000 nonrefundable AOTC with zero qualified tuition expenses listed and used $2,000 per child instead of $2,200." -us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Its own reasoning ended at $342, yet it submitted $8,838 — a number larger than the $6,842 pre-credit tax it computed. The underlying derivation was also wrong: a $31,500 standard deduction instead of $32,200, no qualified-overtime deduction for the $14,838.71 FLSA premium, and a fabricated $2,500 American Opportunity Credit with no qualified tuition expenses listed." -us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated the $1,700-per-child ACTC cap as the refundable amount itself, skipping the §24(d) reduction by the nonrefundable CTC actually allowed; its own reasoning acknowledges the household has enough tax liability to absorb the credit, which is precisely why the entire $4,400 CTC is used nonrefundably and the refundable portion is $0. It also submitted $3,200 while its explanation derived $3,400, so the number does not even match its stated cap arithmetic." -us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly zeroed EITC and the refundable CTC, then manufactured the American Opportunity Credit by explicitly assuming qualified expenses sufficient to max the credit despite no tuition or qualified-expense amount appearing in the household facts. With qualified education expenses of $0, the AOTC is $0 and its 40% refundable portion is $0, not $1,000." -us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,other,False,"It conflated nonrefundable and refundable credits: its own reasoning says the $4,400 CTC and an assumed AOTC 'offset liability,' which is the nonrefundable path, yet it reported that offsetting amount as refundable credits. It compounded this by crediting a $2,500 AOTC with zero listed qualified tuition expenses and then returning an unexplained $4,966 that matches neither the CTC nor any refundable formula." -us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Its CTC analysis was right — tax liability exceeds the full CTC, so the ACTC is $0 — but it then assigned the maximum $2,500 AOTC purely from the head's enrollment flags, inventing $2,500 of qualified tuition expenses that the household facts do not list. The AOTC credit base is qualified tuition and related expenses, which is $0 here, so the 40% refundable portion is $0." -us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It claimed the $1,000 refundable AOTC on the strength of the head qualifying as an eligible student while ignoring that the credit is 100% of the first $2,000 plus 25% of the next $2,000 of qualified tuition expenses — and no tuition or qualified expense amount is listed, so that base is $0. Its CTC reasoning correctly found liability absorbs the nonrefundable credit, leaving $0 total rather than $1,000." -us,scenario_046,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"It computed the ACTC as the raw earned-income formula capped at the per-child refundable maximum ($1,800 × 2) and never subtracted the nonrefundable CTC allowed against tax liability. Federal tax before credits (~$4,900 on ~$45,000 of taxable income after the standard and overtime deductions) exceeds the full $4,400 CTC, so no credit spills into the refundable portion and the answer is $0." -us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It correctly determined the CTC is fully used against liability and EITC is $0, then added a $1,000 refundable AOTC computed as 40% of the statutory $2,500 maximum rather than 40% of the credit actually earned on qualified tuition expenses. Those expenses are $0 in this household, so the AOTC and its refundable portion are both $0." -us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were submitted for federal_refundable_credits, so the required key never reached the outputs object. The correct derivation — EITC $0 at $92,000 of earnings with two children, the full $4,400 CTC absorbed nonrefundably by roughly $4,900 of tax, and $0 AOTC with no qualified tuition expenses — yields $0." -us,scenario_046,federal_refundable_credits,glm-5.3,llm_error,other,False,"It correctly zeroed the AOTC for lack of listed tuition expenses and the EITC for income, but then equated the refundable CTC with the $1,700-per-child cap, omitting the step that reduces the refundable amount by the nonrefundable CTC already allowed. Because tax before credits exceeds the $4,400 total CTC, all of it is nonrefundable and the ACTC is $0." -us,scenario_046,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its sequencing of the CTC was right, but it understated federal tax before credits at $3,675 — below the $4,400 CTC — which is what manufactured the $725 spillover. Taxable income of about $45,000 ($92,003 less the ~$32,200 MFJ standard deduction and the $14,839 FLSA overtime deduction) produces roughly $4,900 of tax, so the full CTC is nonrefundable and nothing remains refundable." -us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so nothing was submitted to evaluate. The correct result is $0: EITC phased out at $92,000 of earnings, the $4,400 CTC entirely absorbed as a nonrefundable credit, and no AOTC because qualified tuition expenses are $0." -us,scenario_046,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the §24(d) ordering correctly and rightly zeroed the AOTC and EITC, but pegged tax before credits at $3,871.01, under the $4,400 CTC, which is the sole source of its $528.99 refundable residue. Federal tax on this return is about $4,900 — $92,003 AGI less the ~$32,200 standard deduction and the $14,839 overtime deduction leaves roughly $45,000 taxable — so the CTC is fully nonrefundable and the answer is $0." -us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It reasoned its way to $0 refundable CTC and $0 EITC correctly, then added a $1,000 refundable AOTC computed from the statutory maximum despite $0 of listed qualified tuition expenses, which makes the AOTC itself $0. It then submitted $1,500, a figure its own explanation never derives — its stated total was $1,000." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning got close to the right answer. It applied the overtime deduction and the $4,400 CTC and reached about $488, using a $32,300 standard deduction instead of $32,200. It then submitted $3,465 after an unexplained 'adjustment' that none of its own steps support." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a $28,700 standard deduction instead of $32,200 and left out the $14,839 overtime deduction, which inflated taxable income to $63,303. It also applied a $2,500 AOTC even though no qualified tuition expenses were listed, and used a $2,000-per-child CTC instead of $2,200." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the OBBBA overtime deduction, used a $31,500 standard deduction and a $2,000-per-child CTC, and so reached $2,777 in its reasoning. It then submitted $4,669, which none of its own steps produce. The correct figure uses taxable income of $44,964 and a $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It left out the $14,839 overtime deduction, which gave it taxable income of $59,703. It then submitted about the pre-credit tax ($6,779) without subtracting any nonrefundable CTC, even though its own reasoning said the credits reduce the tax to near zero." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It correctly got taxable income of $44,964 from the $32,200 standard deduction plus the overtime deduction. It then reported the tax before child credits ($4,966, using a $24,000 10% bracket instead of $24,800) and never subtracted the $4,400 nonrefundable CTC, which the output definition requires." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the OBBBA qualified overtime deduction for the $14,839 FLSA premium. It also used the 2025 $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 and $24,800 figures, and a $2,000-per-child CTC instead of $2,200, arriving at $2,963." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It dismissed the FLSA overtime premium as 'informational' and so never took the qualified overtime deduction. It then submitted $12,200, which is higher than its own pre-credit tax of $6,696, with no nonrefundable CTC subtracted." +us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction correctly but itemized at about $40.9k by counting health insurance premiums as medical expenses above the 7.5% floor. The household's deductible itemized total is below the $32,200 standard deduction, which the reference uses. Its understated taxable income ($36.2k) made the tax smaller than the $4,400 CTC, so it answered $0 instead of $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized by counting $24,330 of health insurance premiums and medical costs as deductible medical expenses, when the standard deduction should apply. It also left out the $14,839 overtime deduction. Finally, it treated only $200 per child as the nonrefundable CTC, when the full $4,400 CTC is nonrefundable up to the tax liability." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset for 2026, so it applied personal exemptions, pre-TCJA 10%/15% brackets and a $1,000-per-child CTC. OBBBA made the TCJA structure permanent: $32,200 standard deduction, 12% bracket and $2,200 CTC. It also left out the overtime deduction." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law ($21,200 of personal exemptions, itemized deductions, pre-TCJA rates and a $2,000 total CTC) instead of the permanent OBBBA rules. Under those rules the $32,200 standard deduction plus the $14,839 overtime deduction gives $44,964 of taxable income and a $4,400 CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It got taxable income of $44,964 right. It then inflated the CTC to $2,260 per child ($4,520) instead of the statutory $2,200 ($4,400), and used a $24,400 10% bracket instead of $24,800. Together these understated the result at $387.68." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It added the overtime premium on top of wages (which already include it) and subtracted ESI premiums that were not listed as payroll deductions. It also assumed the TCJA had sunset ($5,000 personal exemptions, a $1,000-per-child CTC) and never took the OBBBA overtime deduction or the $32,200 standard deduction." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used a $31,200 standard deduction with no overtime deduction and a $4,000 CTC. Its $8,518 is also higher than the pre-credit tax those inputs produce under the 10%/12% brackets (about $6,800), so its arithmetic is internally wrong as well." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules (personal exemptions, itemizing, a $1,000-per-child CTC) and subtracted ESI premiums from wages. It never applied the OBBBA overtime deduction, the $32,200 standard deduction or the $2,200-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset (personal exemptions, a $1,000-per-child CTC), subtracted ESI premiums from AGI and left out the overtime deduction. It also claimed a $1,500 nonrefundable AOTC even though no qualified tuition expenses were listed, which makes the AOTC zero." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is taxable income of $44,964 (standard deduction plus the $14,839 overtime deduction), tax of $4,899.71, and the $4,400 CTC. Its $3,212 is several thousand dollars too high, which fits leaving out the overtime deduction and not applying the full $2,200-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its explanation mentions only standard/itemized deductions and the nonrefundable CTC, and never the qualified overtime deduction. Its $1,858.40 implies taxable income well above the correct $44,964, meaning it left out the $14,839 overtime deduction before applying the credits." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions and itemized deductions, which is TCJA-sunset logic. The permanent 2026 rules eliminate personal exemptions and apply the $32,200 standard deduction, the $14,839 overtime deduction and a $2,200-per-child CTC, leaving $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets, left out the $14,839 OBBBA overtime deduction, and applied a $2,000-per-child CTC instead of $2,200, giving $2,963.36." +us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no value or explanation for this output, so there was no answer to score." +us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It got taxable income ($44,964) and tax ($4,899.68) right. It then treated only $500 per child ($2,200 minus the $1,700 refundable cap) as nonrefundable. In fact the full $4,400 CTC is nonrefundable up to the tax liability, and the ACTC cap applies only to the unused remainder." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It skipped the calculation and assumed the American Opportunity Credit wiped out the tax. No qualified tuition expenses were listed, so the AOTC is zero. The tax of $4,899.71 on $44,964 of taxable income, less the $4,400 CTC, leaves $499.71, not $0." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It applied an AOTC even though no tuition expenses were listed, left out the overtime deduction, and never subtracted the $4,400 nonrefundable CTC. As a result its $6,794 is roughly the tax before child credits on taxable income that excludes the overtime deduction." +us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It capped the qualified overtime deduction at the $12,500 single-filer limit instead of the $25,000 joint limit. That cut the deduction from $14,839 to $12,500 and raised taxable income to $47,303, giving $780.36 instead of $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no computation. It cited an irrelevant 'AOTC phaseout' and submitted a round $8,500, which is higher than even the pre-credit tax. It never applied the $32,200 standard deduction, the $14,839 overtime deduction or the $4,400 nonrefundable CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset: four personal exemptions of about $5,460 each, 10%/15% brackets, and a $1,000-per-child CTC. It also left out the overtime deduction. The permanent 2026 rules use the $32,200 standard deduction and a $2,200-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly applied post-TCJA-sunset law ($21,168 of personal exemptions, itemized deductions, two $1,000 CTCs) and left out the OBBBA overtime deduction. OBBBA made the $32,200 standard deduction and the $2,200-per-child CTC permanent for 2026." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a 'post-TCJA' 2026 joint return with $21,200 of personal exemptions and a $1,000-per-child CTC. It never applied the permanent $32,200 standard deduction, the $14,839 qualified overtime deduction or the $2,200-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules, with $23,508 of taxable income and a $2,000 total CTC. The correct rules give $44,964 of taxable income (standard deduction plus overtime deduction) and a $4,400 nonrefundable CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction but itemized with about $19,193 of medical expenses by counting health insurance premiums above the 7.5% floor. The household's deductible itemized total is below the $32,200 standard deduction. That understated taxable income enough that the $4,400 CTC appeared to wipe out the tax, giving $0 instead of $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for this output, so there was no answer to score." +us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted a placeholder $9,999 with no derivation. That figure is higher than even the pre-credit tax. It ignored the $32,200 standard deduction, the $14,839 overtime deduction and the $4,400 nonrefundable CTC, which together give $499.71." +us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction and the $2,200-per-child CTC correctly. It then itemized with $18,080 of medical expenses by counting $24,000 of health insurance premiums, when the deductible itemized total falls below the $32,200 standard deduction. That understated taxable income ($36,392), so the tax appeared to be fully offset and it answered $0." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It rejected the overtime deduction on the grounds that wages already include the premium, but the OBBBA deduction applies to the premium included in wages. It also itemized using $24,000 of premiums as medical expenses, claimed a $1,000 AOTC with no qualified expenses, and used a $2,000-per-child CTC." +us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left out the $14,839 overtime deduction, used a $31,500 standard deduction, and claimed a $2,500 AOTC even though no tuition expenses were listed. Its reasoning ended at $342, but it submitted $8,838, a figure none of its steps produce." +us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It treated the refundable CTC as the $1,700-per-child cap ($3,400) and never applied the rule that only CTC left after tax is refundable. Tax before credits (about $4,900) uses up the whole CTC, so refundable CTC is $0. Its submitted $3,200 does not even match its own $3,400 figure." +us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly zeroed the EITC and found the CTC fully used against tax. It then assumed the head had enough tuition to max out the AOTC and claimed its $1,000 refundable part. No qualified tuition and related expenses are listed, so the AOTC and its refundable 40% are both $0." +us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It counted the $4,400 CTC as refundable along with an assumed $1,000 AOTC and reached $4,966 of credits offsetting tax. That mixes nonrefundable credits with refundable ones. The CTC is fully used against tax of about $4,900, leaving $0 refundable CTC. The AOTC is $0 because no qualified tuition expenses are listed." +us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It correctly found that tax absorbs the full CTC, so refundable CTC is $0. It then gave the head the full $2,500 AOTC and a $1,000 refundable part based only on the student flags. The credit is 100% of the first $2,000 and 25% of the next $2,000 of qualified tuition expenses, and none are listed, so it is $0." +us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It correctly said tax absorbs the CTC and the EITC is phased out. It then claimed a $1,000 refundable AOTC just because the head is an eligible student. The AOTC is a percentage of qualified tuition and related expenses, none are listed, so the refundable part is $0." +us,scenario_046,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It put tax before credits at about $3,852, below the $4,400 CTC, and paid the $548 difference as refundable CTC. Wages of $92,003, less the $32,200 standard deduction and the $14,839 overtime deduction, leave about $44,964 taxable. That is about $4,900 of tax, which uses up the whole CTC, so no refundable CTC remains." +us,scenario_046,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It set refundable CTC to a per-child cap times two ($3,600) without first applying the $4,400 CTC to tax, and it used an $1,800 cap where the 2026 cap is $1,700. Tax before credits exceeds $4,400, so no CTC is left to refund and refundable credits are $0." +us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It handled the CTC and EITC correctly, then gave the maximum $1,000 refundable AOTC without any tuition in the facts. The AOTC is built from qualified tuition and related expenses, which are $0 here, so the credit and its refundable part are $0." +us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no usable value or explanation for federal_refundable_credits. The correct derivation gives $0: the EITC is phased out, the AOTC has no qualified expenses, and tax uses up the full $4,400 CTC." +us,scenario_046,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly zeroed the EITC and AOTC, but set refundable CTC to the $1,700-per-child cap ($3,400) because the earned-income phase-in was complete. Refundable CTC is also limited to CTC left over after tax, and tax before credits (about $4,900) uses up the full $4,400, so refundable CTC is $0." +us,scenario_046,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It said only $3,675 of the $4,400 CTC was needed to eliminate tax, and refunded the remaining $725. That understates tax: even after the $32,200 standard deduction and the $14,839 overtime deduction, tax on $92,003 is about $4,900. The CTC is fully used and refundable CTC is $0." +us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable value or explanation for federal_refundable_credits. The correct derivation gives $0: the EITC is phased out, the AOTC has no qualified expenses, and tax uses up the full $4,400 CTC." +us,scenario_046,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It put tax before credits at $3,871.01, below the $4,400 CTC, and refunded the $528.99 remainder. Tax is higher: even after the $32,200 standard deduction and the $14,839 overtime deduction, tax on $92,003 is about $4,900. That uses up the whole CTC, so refundable CTC is $0." +us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It correctly found refundable CTC and EITC are $0. It then assumed a $1,000 refundable AOTC with no tuition listed, and submitted $1,500, which contradicts its own stated $1,000 total. With no qualified tuition and related expenses the AOTC is $0, so the correct total is $0." us,scenario_046,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed $5,704 of Social Security tax and $1,334 of Medicare tax but then added them incorrectly as $10,038 and invented a $518.40 Oklahoma payroll-tax adjustment. Oklahoma contributes no mandatory employee payroll tax here, so the two federal components total $7,038." us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from FICA wages. The facts do not identify that premium as a pre-tax employee payroll deduction, so Social Security and Medicare apply to the full $92,000 of wages and total $7,038." @@ -3716,45 +4072,51 @@ us,scenario_046,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The mod us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning calculated the exact $7,038 total but submitted $8,438 as the output value. It failed to carry its computed result into the required numeric field, creating a $1,400 answer inconsistency." us,scenario_046,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It built the correct $75,303 Oklahoma base but priced it on the pre-2026 six-bracket schedule topping at 4.75% for $3,221.89 instead of the 2026 schedule's $2,959.14, and valued the state credit at 5% of a $4,000 federal CTC ($200) rather than 5% of the $4,400 CTC for two children at $2,200 each ($220). It then submitted $3,364, a figure matching neither the $3,222 nor the $3,022 its own reasoning produced." -us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its base ($92,003 less $12,700 and four $1,000 exemptions = $75,303) and its $220 child care/child tax credit are both exact. The entire $262.75 error is the rate schedule: it taxed the base with the pre-2026 4.75% top rate for $3,221.89 instead of the 2026 Oklahoma schedule's $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied a $14,600 federal standard deduction and a fabricated 5.85% flat Oklahoma rate, never subtracting Oklahoma's own $12,700 MFJ standard deduction or the four $1,000 personal exemptions that produce the $75,303 state base. It then discarded its own $4,528 computation and submitted $1,614 with no derivation supporting it." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached the correct $75,303 base but taxed it on the pre-2026 4.75% six-bracket schedule for $3,221 rather than the 2026 schedule's $2,959.14. It then subtracted an unquantified 'small' credit to reach $3,010 instead of applying the $220 credit equal to 5% of the $4,400 federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied Oklahoma's $17,000 itemized cap but dropped the $5,429.78 of medical expense above the 7.5% floor that completes the $22,429.78 state itemized total, and it invented a $13,050 Oklahoma standard deduction where the MFJ amount is $12,700. Its submitted $3,563 exceeds even the pre-2026 4.75% tax on its own $71,003 base ($3,018), and it never applied the $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It assembled the correct $75,303 base from $92,003 of AGI, the $12,700 standard deduction, and $4,000 of exemptions, then estimated 'about $2,500 after the child care/child tax credit' without running the graduated schedule. The 2026 brackets on $75,303 give $2,959.14, and the $220 credit leaves $2,739.14; the $239 shortfall is bracket arithmetic it skipped." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded Oklahoma conforms to the federal standard deduction and subtracted ~$30,000; Oklahoma's MFJ standard deduction is $12,700, which with four $1,000 exemptions gives the $75,303 base. It explicitly computed that correct path, then rejected it in favor of the conformity claim, and never applied the $220 Oklahoma child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It opened by asserting Oklahoma has no income tax, then produced $3,200 by hand — the pre-2026 4.75% result on the $75,303 base — while its stated derivation subtracted $20,879 of itemized deductions that yield no such figure. It applied neither the four $1,000 exemptions in a consistent computation nor the $220 nonrefundable child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It set Oklahoma's standard deduction equal to the federal MFJ amount (~$30,900) and dropped the four $1,000 personal exemptions, producing $61,103 of taxable income instead of $75,303. It also valued the state credit at 5% of a $4,000 federal CTC ($200) rather than 5% of the $4,400 CTC for two children ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $51,795.62 base implies roughly $36,200 of deductions — a second exclusion of the $9,208 of employer-sponsored premiums already excluded from the $92,000 of wages, stacked on an uncapped federal itemized total — far beyond either Oklahoma figure, the $12,700 standard deduction or the $22,429.78 capped itemized total, that bracket the reference's $75,303 base. It also applied no $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It deducted $26,958.78 of federal itemized deductions, ignoring Oklahoma's $17,000 cap on the interest-and-taxes portion that holds the state itemized total to $22,429.78, landing at $61,044 of taxable income against the reference's $75,303. It then stated no nonrefundable state credits applied, missing the $220 child care/child tax credit equal to 5% of the $4,400 federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It correctly built Oklahoma's capped itemized figure ($17,000 plus $5,430 of medical above the 7.5% floor) but subtracted it to reach $65,573 of taxable income rather than the reference's $75,303, understating the base by $9,730. It also set the state credit at 5% of a $2,000 federal CTC ($100) instead of 5% of the $4,400 CTC for two children at $2,200 each ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It submitted $3,154 with no derivation at all. The 2026 Oklahoma brackets on the $75,303 base give $2,959.14, and the $220 child care/child tax credit leaves $2,739.14; its figure is consistent with pricing the correct base on the repealed 4.75% top rate and applying no state credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $2,105 is exactly the pre-2026 4.75% bracket tax on about $51,800 of taxable income, implying roughly $40,200 of deductions and exemptions — far beyond either Oklahoma figure, the $12,700 standard deduction or the $22,429.78 capped itemized total, that bracket the reference's $75,303 base. It also applied no $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $82,795 by subtracting the $9,208 of employer-sponsored insurance premiums that are already excluded from the $92,000 of wages, then deducted an uncapped $26,999 of itemized deductions where Oklahoma caps the non-medical portion at $17,000 (state itemized total $22,429.78), landing at $51,796 instead of the $75,303 base. It also used a $100 credit rather than the $220 equal to 5% of the $4,400 federal CTC." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted $3,043 with no derivation. The 2026 Oklahoma schedule on the $75,303 base produces $2,959.14 before credits and $2,739.14 after the $220 child care/child tax credit; its figure is what a top rate above the 2026 schedule yields on that base with no state credit subtracted." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $2,276.01 is the pre-2026 bracket tax on roughly $55,400 of taxable income, about $20,000 below the reference's $75,303 base — the gap produced by stacking an uncapped federal itemized total on an AGI further reduced for health premiums already excluded from wages. It also applied no $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,684.51 equals the correct $220 credit taken against a pre-credit tax of $2,904.51, which is what the $75,303 base yields at a 4.25% top rate. The 2026 Oklahoma schedule on that base produces $2,959.14, so its entire $54.63 error is the rate applied to the top bracket." -us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $12,200 standard deduction instead of Oklahoma's $12,700 MFJ amount and invented a schedule of '$300 on the first $15,200 plus 4.75%' — Oklahoma's lower brackets total $224.50 through $12,200 and the 2026 top rate is below 4.75%. It also valued the state credit at 5% of a $4,000 federal CTC ($200) rather than 5% of the $4,400 CTC for two children ($220)." -us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits, so nothing was submitted against the $2,739.14 reference. The required chain — $92,003 AGI, $12,700 Oklahoma standard deduction, $4,000 of exemptions, $75,303 taxable, $2,959.14 of bracket tax, $220 child care/child tax credit — never appeared." -us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It cut AGI to $77,164 by subtracting the $14,839 FLSA overtime premium, a below-the-line federal deduction that leaves AGI at $92,003 and never reaches the Oklahoma starting point. It then applied a federal-sized $32,200 standard deduction in place of Oklahoma's $12,700, dropped the four $1,000 exemptions, and taxed $44,964 at a flat 4.25% instead of applying the graduated 2026 schedule to $75,303." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It declined to compute and submitted a zero placeholder. The listed facts fully determine the result: $92,003 of AGI less the $12,700 Oklahoma MFJ standard deduction and four $1,000 exemptions is $75,303 of taxable income, the 2026 brackets give $2,959.14, and the $220 child care/child tax credit leaves $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It reduced the Oklahoma tax by the American Opportunity Credit, a federal credit with no Oklahoma counterpart; the only nonrefundable state credit here is the $220 child care/child tax credit equal to 5% of the $4,400 federal CTC. Its $2,414 falls $325 below the $2,739.14 that the $75,303 base, the 2026 brackets, and that credit produce." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its base ($75,303) and credit ($220) are both exact. It priced the base at $3,064 — the six-bracket result carrying a 4.5% top rate — instead of the 2026 Oklahoma schedule's $2,959.14, overstating the pre-credit tax by $105 and the answer by the same amount." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $4,223.29 implies an effective rate of 5.6% on the $75,303 Oklahoma base, above every Oklahoma bracket rate — the schedule tops at 4.75% before 2026 and lower for 2026, giving $2,959.14 and $2,739.14 after the $220 credit. A flat 4.75% on the entire $92,003 of AGI is $4,370, barely above the figure it submitted after claiming deductions and four exemptions." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the $17,000 Oklahoma itemized cap but omitted the $5,429.78 of medical expense above the 7.5% floor that completes the $22,429.78 state itemized total, reaching a $71,003 base instead of $75,303. It then taxed that base at the pre-2026 4.75% top rate for $3,018 rather than the 2026 schedule's $2,959.14, so its correct $220 credit still left the answer $59 high." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its taxable income of roughly $50,500 falls about $24,800 short of the $75,303 base, the size of a federal-scale standard deduction taken in place of Oklahoma's $12,700. It also subtracted 'two $100 nonrefundable Oklahoma child credits'; Oklahoma allows one child care/child tax credit equal to 5% of the federal CTC, $220 here, not $100 per child." -us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Its base ($92,003 less the $12,700 joint standard deduction and $4,000 of exemptions = $75,303) and its $220 credit are both exact. It mis-set the 2026 bracket amounts, computing $2,981.14 of tax where the schedule gives $2,959.14, and carried that $22 overstatement straight into the answer." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It submitted $3,800 with no derivation; that is the pre-2026 4.75% bracket tax on about $88,000, i.e. AGI reduced by roughly the $4,000 of exemptions alone. Oklahoma's $12,700 MFJ standard deduction was never subtracted and the $220 child care/child tax credit was never applied, so the $75,303 base and $2,959.14 tax never entered its computation." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the full $26,309 federal itemized total, ignoring Oklahoma's $17,000 cap on the interest-and-taxes portion that holds the state itemized figure to $22,429.78, producing $61,694 of taxable income against the reference's $75,303. It also applied no $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the same uncapped $26,309 of federal itemized deductions, bypassing Oklahoma's $17,000 cap on the non-medical portion that limits the state itemized total to $22,429.78, giving $61,694 of taxable income instead of $75,303. Its bracket base of '$236 plus 4.75% over $12,200' also overstates Oklahoma's $224.50 of lower-bracket tax, and it applied no $220 state credit." -us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It taxed $45,216 — a base $30,087 below the reference's $75,303, which comes from $92,003 less the $12,700 standard deduction and $4,000 of exemptions — at a flat 4.5%, where Oklahoma applies graduated brackets. It also applied no $220 child care/child tax credit, so it missed both the base and the credit." -us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started from $77,164 of AGI, having subtracted the $14,839 overtime premium that as a below-the-line federal deduction leaves AGI at $92,003, then used a ~$15,000 standard deduction rather than Oklahoma's $12,700. That produced $58,164 of taxable income instead of $75,303, and it applied no $220 child care/child tax credit." -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits, so no state tax figure was submitted against the $2,739.14 reference. The computation — $75,303 of Oklahoma taxable income, $2,959.14 of bracket tax, less the $220 child care/child tax credit — was never performed." -us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly held AGI at $92,003 and used the $12,700 Oklahoma standard deduction but omitted the four $1,000 personal exemptions, taxing $79,303 instead of $75,303. It then used the pre-2026 4.75% top rate and declared no state credit applies because federal EITC and CDCC are zero, missing that Oklahoma's credit is the greater of 20% of the CDCC or 5% of the federal CTC, which is $220 on a $4,400 CTC." -us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Its bare $3,773 is the pre-2026 4.75% bracket tax on roughly $86,900 — AGI with neither the $12,700 Oklahoma standard deduction nor the $4,000 of personal exemptions removed. The reference base is $75,303, the bracket tax $2,959.14, and the $220 child care/child tax credit brings it to $2,739.14." -us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It built the exact $75,303 base and then asserted the Oklahoma child care/child tax credit is $0, missing the $220 credit equal to 5% of this household's $4,400 federal CTC (two children at $2,200), which applies because federal AGI is under the $100,000 limit. It also priced the base at $3,064.14 by putting a 4.5% top rate on the old six-bracket structure rather than the 2026 schedule's $2,959.14." -us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It inflated AGI to $94,004 where the facts give $92,003 in wages plus interest, and omitted the four $1,000 Oklahoma personal exemptions, stating an $81,304 base. Its reported $3,224.25 is the pre-2026 4.75% tax on $75,303, matching neither its own stated base nor the 2026 schedule's $2,959.14, and it applied no $220 state credit." -us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It invented a 0.5%-to-5% Oklahoma bracket schedule topping at 5% over $22,500, where Oklahoma's top rate is 4.75% before 2026 and lower for 2026 with the top bracket beginning at $12,200 for joint filers. Its $60,503 base came from a federal-sized standard deduction rather than Oklahoma's $12,700 plus $4,000 of exemptions ($75,303), and it declared no nonrefundable credits, missing the $220 child care/child tax credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but used the pre-2026 brackets (0.25% to 4.75%) instead of the 2026 schedule (0%/2.5%/3.5%/4.5%, with 4.5% above $14,400). It based the credit on a $4,000 CTC instead of $4,400. It then submitted $3,364, which does not match its own $3,022 result." +us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and the $220 credit right, but ran the income through the pre-2026 brackets (0.25% to 4.75%) and got $3,221.89. The 2026 joint schedule (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It started from federal taxable income using an outdated $14,600 federal standard deduction and applied a nonexistent 5.85% Oklahoma rate. It never used Oklahoma's $12,700 standard deduction, $4,000 of exemptions, or the graduated 2026 schedule, and its final $1,614 does not follow from its own arithmetic." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but applied the pre-2026 brackets with a 4.75% top rate, which gives about $3,221 instead of $2,959.14. It also guessed a credit of about $211 instead of computing 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return with a $17,000 cap, but Oklahoma allows itemizing only when the taxpayer itemizes federally. This household takes the $32,200 federal standard deduction, so the $12,700 Oklahoma standard deduction applies. It also used 4.75% rates and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It got the $75,300 taxable income right but never worked through the 2026 schedule. The schedule (0%/2.5%/3.5%/4.5%, with 4.5% above $14,400) gives $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its round $2,500 is an estimate based on the old 4.75% rates." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly assumed Oklahoma matches the federal standard deduction and subtracted $30,000 instead of Oklahoma's fixed $12,700. That understated taxable income by $17,300. It also used an invented 3.75% bracket up to $14,400 with 4.75% above, and it ignored the 5%-of-federal-CTC credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It first claimed Oklahoma has no income tax, then itemized mortgage interest and property tax on the Oklahoma return. Oklahoma requires federal itemizing for that, and this household takes the federal standard deduction. It then applied pre-2026 4.75% rates instead of taxing $75,303 under the 2026 schedule and subtracting the $220 credit." +us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used Oklahoma itemized deductions to reach about $49k of taxable income. Because the household takes the $32,200 federal standard deduction, Oklahoma requires the $12,700 standard deduction, which gives $75,303. It also applied the old brackets topping out at 4.75%." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It set the Oklahoma standard deduction equal to the federal one (about $30,900) instead of Oklahoma's $12,700, and it left out the $4,000 of exemptions. It used pre-2026 4.75% brackets and computed the credit as 5% of $4,000 instead of 5% of the $4,400 federal CTC." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut Oklahoma income to $51,795.62 by subtracting the ESI premiums and itemizing, but Oklahoma income starts from the $92,003 AGI. Itemizing is not allowed because the household does not itemize federally, so taxable income is $75,303. It applied the old 4.75% brackets and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted $26,958.78 of Oklahoma itemized deductions, but Oklahoma allows itemizing only when the taxpayer itemizes federally, and this household takes the $32,200 federal standard deduction. The $12,700 Oklahoma standard deduction should have given $75,303 of taxable income. It also used pre-2026 4.75% brackets and applied no child credit, even though the $220 credit (5% of federal CTC) applies." +us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It took the $14,839 federal overtime deduction out of AGI and got $77,164. That deduction is taken after AGI is calculated and does not reduce the $92,003 AGI that Oklahoma starts from. It also used old 3.75%/4.75% brackets and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return ($17,000 capped plus $5,430 medical), but Oklahoma allows itemizing only when the taxpayer itemizes federally. This household takes the federal standard deduction, so the $12,700 standard deduction and $75,303 taxable income apply. It also used 4.75% brackets and computed the credit on one $2,000 CTC ($100) instead of $4,400 ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $3,154 equals the pre-2026 formula ($300 plus 4.75% above $15,200) applied to $75,303, with no child credit. The 2026 schedule gives $2,959.14 of tax, and subtracting the $220 5%-of-CTC credit gives $2,739.14." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used itemized deductions on the Oklahoma return. Oklahoma requires federal itemizing, and this household takes the $32,200 federal standard deduction, so the $12,700 Oklahoma standard deduction applies. Its $2,105 comes from old 4.75% brackets on about $51.8k of taxable income with no child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut Oklahoma AGI to $82,795 by subtracting the ESI premiums from the reported $92,000 of wages, and it itemized $26,999 even though the household does not itemize federally. It also computed the credit as $100 instead of 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. The correct steps are $75,303 of taxable income taxed at 0%/2.5%/3.5%/4.5% for $2,959.14, minus the $220 credit, for $2,739.14. Its $3,043 overstates the tax by $304, a result that matches using pre-2026 brackets with a 4.75% top rate." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used itemized deductions on the Oklahoma return. Oklahoma requires the $12,700 standard deduction whenever the taxpayer takes the federal standard deduction, as this household does. It also left out the $220 5%-of-federal-CTC credit." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly used the standard deduction, exemptions, and child credit, but applied the bracket schedule wrongly. The 2026 schedule on $75,303 gives $2,959.14, and subtracting the $220 credit leaves $2,739.14. Its $2,684.51 is $54.63 too low, which comes from misstated lower-bracket amounts." +us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $12,200 standard deduction instead of Oklahoma's $12,700 joint amount and applied the pre-2026 formula ($300 plus 4.75% above $15,200) instead of the 2026 schedule with 4.5% above $14,400. It based the credit on a $4,000 CTC ($200) instead of $4,400 ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no parseable value for state_income_tax_before_refundable_credits. +us,scenario_046,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the federal overtime deduction out of AGI and used the $32,200 federal standard deduction instead of Oklahoma's $12,700. It dropped the $4,000 of exemptions and applied a flat 4.25% rate instead of the graduated 2026 schedule. It also omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It submitted a $0 placeholder instead of computing the tax. The provided facts fully determine $75,303 of Oklahoma taxable income, $2,959.14 of tax under the 2026 schedule, and $2,739.14 after the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return, which Oklahoma blocks because the household takes the federal standard deduction. It also applied the federal American Opportunity Credit, which has no Oklahoma counterpart. The only Oklahoma nonrefundable credit here is 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right. Its $3,064 regular tax comes from keeping the old lower brackets (0.25% to 3.75% up to $12,200, or $224.50) with 4.5% above $12,200. The 2026 schedule is 0% up to $7,500, 2.5% up to $9,800, 3.5% up to $14,400, and 4.5% above, which gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $4,223.29 is more than the tax even the old 4.75% schedule produces on the entire $92,003 AGI (about $4,015). So it did not subtract the $12,700 standard deduction and $4,000 of exemptions to reach $75,303, and it cited an overtime deduction that has no effect on the Oklahoma base." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used a $17,000 capped itemized deduction and got $71,003 of taxable income. Because the household does not itemize federally, the $12,700 Oklahoma standard deduction applies, giving $75,303. It also used the pre-2026 4.75% brackets ($3,018) instead of the 2026 schedule." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a flat 4.5% to about $43,300 of taxable income, which reflects a deduction far larger than Oklahoma's $12,700, and it ignored the 0%/2.5%/3.5% lower brackets. It set the child credit at $100 per child instead of 5% of $2,200 ($110 per child)." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right but misstated the lower brackets. The 0% bracket up to $7,500, 2.5% up to $9,800, and 3.5% up to $14,400 produce $218.50 before the 4.5% bracket, but it counted $240.50, which led to its $2,981.14 figure instead of $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Oklahoma's $12,700 standard deduction and $4,000 of exemptions give $75,303, taxed at $2,959.14, minus the $220 credit, for $2,739.14. Its $2,513 is about $226 too low, which equals roughly $5,000 of extra deductions at the 4.5% rate on top of the $12,700 standard deduction." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right, but its $3,064.14 pre-credit tax keeps the old lower brackets up to $12,200 ($224.50) and applies 4.5% above that. The 2026 schedule (0% to $7,500, 2.5% to $9,800, 3.5% to $14,400, 4.5% above) gives $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got the $75,303 taxable income and $220 credit right but understated the lower brackets. It counted $66.50 of tax below the 4.5% bracket instead of $218.50 ($57.50 at 2.5% for $7,500 to $9,800 plus $161 at 3.5% for $9,800 to $14,400), which produced $2,807.14 instead of $2,959.14." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,800 matches taxing about $87,500 under the pre-2026 4.75% schedule, which means it left out the $12,700 Oklahoma standard deduction and did not reach $75,303 of taxable income. It also left out the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted $26,309 of itemized deductions on the Oklahoma return, but Oklahoma allows itemizing only when the taxpayer itemizes federally, and $26,309 is less than the $32,200 federal standard deduction. So the $12,700 Oklahoma standard deduction applies. It also used 4.75% brackets and omitted the $220 5%-of-CTC credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized $26,309 on the Oklahoma return even though the household takes the larger $32,200 federal standard deduction. That forces the $12,700 Oklahoma standard deduction and $75,303 of taxable income. It also taxed income above $12,200 at 4.75% instead of using the 2026 schedule, and it left out the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It deducted $26,309 of itemized deductions on the Oklahoma return. Oklahoma requires federal itemizing, which this household does not do, so the $12,700 standard deduction applies. It also wrongly said no Oklahoma nonrefundable credits apply, missing the 5%-of-federal-CTC child credit ($220), and it used the pre-2026 4.75% rate." +us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It itemized on the Oklahoma return and got $45,216 of taxable income, instead of using the $12,700 standard deduction that applies to a federal non-itemizer, which gives $75,303. It then applied a flat 4.5% with no 0%, 2.5%, or 3.5% lower brackets and no $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It took the federal overtime deduction out of AGI and got $77,164, but that deduction does not reduce the $92,003 AGI that Oklahoma starts from. It also used a $15,000 standard deduction instead of $12,700 and omitted the $220 5%-of-CTC credit." +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value for state_income_tax_before_refundable_credits. +us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly kept overtime in AGI and used the $12,700 standard deduction, but it left out the four $1,000 exemptions, so its $79,303 should have been $75,303. It used the pre-2026 4.75% brackets. It also wrongly said no Oklahoma nonrefundable credits apply, missing the child credit of 5% of the $4,400 federal CTC ($220)." +us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,773 matches taxing about $87,000 under the pre-2026 4.75% schedule, which means it skipped the $12,700 Oklahoma standard deduction and did not reach $75,303 of taxable income. It also left out the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It got the $75,303 taxable income right but kept the old lower brackets up to $12,200 ($224.50) and applied 4.5% above $12,200, instead of the 2026 schedule (0%/2.5%/3.5% up to $14,400). It set the child credit to $0 because the CDCC is $0. The Oklahoma credit is the greater of 20% of the CDCC or 5% of the federal CTC, which is $220 here." +us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It miscalculated AGI as $94,004 instead of $92,003 and left out the four $1,000 exemptions, which inflated taxable income to $81,304 instead of $75,303. It also used pre-2026 4.75% brackets and omitted the $220 child credit." +us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It started from a federal-taxable-income figure of $60,503 instead of the $92,003 AGI less Oklahoma's $12,700 standard deduction and $4,000 of exemptions. It applied an invented 0.5% to 5% bracket schedule instead of the 2026 0%/2.5%/3.5%/4.5% schedule, and it left out the $220 5%-of-CTC credit." us,scenario_046,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_046,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age and assets below $2,000 as sufficient for Ohio aged Medicaid while never establishing that the head passed the applicable income test. The head's wages and Social Security total $34,626, SSI is zero, and no Medicaid category applies." @@ -3772,45 +4134,52 @@ us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0." us,scenario_048,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate to the first dollar of taxable income, ignoring Ohio's zero-tax bracket: Ohio's rate schedule imposes no tax on Ohio taxable nonbusiness income at or below the roughly $26,050 threshold, and this filer's $7,656 of wages (Social Security survivor benefits are excluded from federal AGI here because provisional income of $21,141 is under the $25,000 base amount) falls far below it, yielding $0. It compounded the error by inventing a nonexistent $1,400 Ohio 'single standard deduction' — Ohio starts from federal AGI and allows personal exemptions, not a standard deduction — and then taxed the residual $3,856 at 2.75% to reach $106." us,scenario_048,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model taxed the entire $7,656 of Ohio taxable income at 2.75% ($210.54), skipping Ohio's zero-rate first bracket that exempts all Ohio taxable nonbusiness income up to about $26,050, under which this household's liability is $0 before any credit is considered. It also mis-specified the personal exemption as a credit equal to 2.75% of $2,400 ($66) — Ohio subtracts the $2,400 exemption from Ohio adjusted gross income and separately allows only a $20-per-exemption credit — and then netted a $50 senior citizen credit against a tax that never existed, landing on $94.54." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own worksheet closed at $30,830 (AGI $247,385, $32,200 standard deduction, full $25,000 overtime deduction, $931 on qualified dividends), and it then submitted $33,223 — a figure no step of its derivation produces. The $30,830 itself misses only the $721 deductible traditional IRA contribution and the $579.77 non-itemizer cash charitable deduction, so the decisive error is submitting a number $2,393 above its own arithmetic." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It reproduced the entire reference structure — $32,200 standard deduction, full $25,000 FLSA overtime deduction, $580 non-itemizer cash charitable deduction, $6,208 of qualified dividends at 15% — but denied the spouse's $721 traditional IRA deduction as fully phased out for an active participant. The reference allows that $721.28 adjustment; at the 22% marginal rate it is worth $158.63, exactly the gap between $30,702.54 and $30,543.91." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It claimed no deduction at all against the $44,647 FLSA overtime premium, skipping the $25,000 qualified-overtime deduction, and used a stale $29,200 standard deduction instead of 2026's $32,200, leaving taxable income of $217,364 against the reference's $188,883.95. Its $48,837 does not follow even from its own $217,364, which under the 2026 MFJ schedule with $6,208 of qualified dividends at 15% yields about $36,800." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It omitted the $25,000 qualified-overtime deduction and the $579.77 non-itemizer charitable deduction, applied a $10,000 SALT cap and a $31,500 standard deduction, and folded $27 of Additional Medicare Tax into an income-tax output that excludes payroll taxes. It ended its own arithmetic at about $36,571 and then submitted $49,861, roughly $13,300 above anything its derivation produces." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It took no deduction against the $44,647 FLSA overtime premium, used a $30,300 standard deduction rather than $32,200, and skipped the $580 non-itemizer cash charitable deduction, reaching taxable income of $217,085 versus the reference's $188,883.95. It then stated its total as $37,055 and submitted $53,503, a value $16,448 above its own computation." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added $2,743 of mortgage interest, $20,655 of real estate taxes and $3,338 of charity to ""about $46,081"" when that sum is $26,736, and on that arithmetic error it itemized instead of taking the larger $32,200 standard deduction. It also claimed nothing against the $44,647 FLSA overtime premium, and its rounded $47,500 exceeds by roughly $14,200 the $33,300 that the 2026 MFJ schedule produces on its own stated $201,300 of taxable income." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Its worksheet is internally consistent but never applies the OBBBA qualified-overtime deduction, leaving the $44,647 FLSA overtime premium fully taxed, and it used a $30,000 standard deduction rather than 2026's $32,200 while skipping the $579.77 non-itemizer cash charitable deduction. Taxable income of $217,385 against the reference's $188,883.95 accounts for the entire $6,765 overstatement." -us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It omitted the $25,000 qualified-overtime deduction and used a $32,600 standard deduction with a $10,000 SALT cap, reaching about $38,831 by its own brackets, then added roughly $23,900 ""after accounting for additional Medicare wages and phase-outs"" to submit $62,700. Additional Medicare Tax belongs to the payroll-tax output rather than to federal income tax before refundable credits, and no phase-out bites at $246,664 of MAGI." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It deducted nothing against the $44,647 FLSA overtime premium and used a $31,000 standard deduction with a $10,000 SALT cap, producing taxable income of $216,385 versus the reference's $188,883.95. The missing $25,000 overtime deduction, $579.77 cash charitable deduction and $721 IRA deduction at the 22% rate account for the full $6,250 overstatement." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It computed 2026 under a TCJA sunset — a $15,600 standard deduction, two $5,000 personal exemptions, and pre-TCJA 10/15/25/28% brackets — when the permanent structure gives a $32,200 MFJ standard deduction, no personal exemptions, 10/12/22% rates and a $25,000 qualified-overtime deduction it never claimed. It also stripped $16,408 of employer health premiums out of wages, which the reference's $237,564 of employment income does not exclude." -us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It supplied no bracket arithmetic; its $36,639.45 corresponds to taxable income near $216,600, i.e. AGI $247,385 reduced by roughly a $30,800 standard deduction and nothing else. It took no deduction against the $44,647 FLSA overtime premium, no $579.77 non-itemizer cash charitable deduction and no $721 IRA deduction — the three items that carry the reference from $246,663.72 of AGI to $188,883.95 of taxable income." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It explicitly assumed ""TCJA provisions sunset"" for 2026, claiming $10,100 of personal exemptions, itemized deductions of $26,736 and 10/15/25% brackets, when 2026 applies a $32,200 MFJ standard deduction, no personal exemptions, 10/12/22% rates and a $25,000 qualified-overtime deduction it never took. It also excluded $16,408 of employer-sponsored insurance premiums from wages to reach AGI $230,977 rather than $246,663.72." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its sole stated basis is ""taxable income of approximately $229,000,"" about $40,100 above the reference's $188,883.95 because it subtracted neither the $25,000 overtime deduction nor the $32,200 standard deduction plus $579.77 cash charitable deduction in full. Even its own $229,000 produces roughly $39,600 under the 2026 MFJ schedule with $6,208 of qualified dividends at 15%, so the submitted $52,367.59 overshoots its own premise by about $12,800." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law — $26,736 of itemized deductions with uncapped real estate taxes plus $10,800 of personal exemptions — when the couple takes the $32,200 standard deduction with no personal exemptions, and it claimed no deduction against the $44,647 FLSA overtime premium. It also removed $16,408 of employer-sponsored insurance premiums from W-2 wages, giving AGI $230,977 instead of the reference's $246,663.72." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used ""post-TCJA expiration"" 2026 rules — $26,736 itemized plus $10,100 of personal exemptions, with ordinary income reaching a 25% bracket — where 2026 gives a $32,200 standard deduction, no exemptions and a 22% top bracket at this income. Its taxable income of $194,141 also reflects the $16,408 ESI exclusion from wages and no $25,000 qualified-overtime deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation at all; $42,203 corresponds to taxable income near $239,900, which is AGI $247,385 reduced by roughly $7,500 — effectively no deductions. The reference subtracts $57,779.77: the $32,200 standard deduction, the $25,000 capped FLSA overtime deduction and the $579.77 non-itemizer cash charitable deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded $16,408 of employer-sponsored insurance premiums from wages to reach $221,156 of taxable wages and AGI $230,977, which the reference's $237,564 of employment income does not do, and then deducted nothing against the $44,647 FLSA overtime premium while using a $30,000 standard deduction rather than $32,200. The two errors run in opposite directions, leaving taxable income of $200,977 against the reference's $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It showed no deduction schedule; $37,920.81 implies taxable income of about $222,400, i.e. AGI $247,385 reduced by roughly $25,000 in total. The reference stacks $32,200 of standard deduction, $25,000 of capped overtime deduction and $579.77 of non-itemizer cash charity for $57,779.77, so its number captures the equivalent of one of those items instead of all three." -us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI correctly at $247,385 and then deducted only a $30,000 standard deduction, taking nothing against the $44,647 FLSA overtime premium and nothing for the $579.77 non-itemizer cash charitable contribution. Taxable income of $217,385 versus the reference's $188,883.95 is the whole $6,764 overstatement." -us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the submission never entered the substantive computation. The failure is the absent output rather than a mistaken derivation." -us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of $247,385 and correctly chose the $32,200 standard deduction over $26,736 of itemized deductions, then claimed no deduction against the $44,647 FLSA overtime premium and no $579.77 non-itemizer cash charitable deduction, leaving taxable income of $215,185 rather than $188,883.95. Its ordinary-tax figure of about $36,982 also overstates the 10/12/22% schedule result on its own $208,977 of ordinary income, which is $35,399." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It performed no bracket arithmetic; $20,460 implies taxable income near $143,000, about $46,000 below the reference's $188,883.95. That figure is what you get by stacking the $16,408 ESI exclusion, the $26,736 itemized total and the $25,000 overtime deduction on top of the standard deduction, where the reference takes the $32,200 standard deduction instead of itemizing and does not exclude ESI premiums from the $237,564 of employment income." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation; $38,566 corresponds to taxable income of about $225,300, i.e. only around $22,000 of deductions against AGI $247,385. It captured neither the $25,000 qualified-overtime deduction nor the full $32,200 standard deduction, which with the $579.77 cash charitable deduction make up the reference's $57,779.77." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied the 2026 structure correctly — $32,200 standard deduction, full $25,000 capped overtime deduction, $580 non-itemizer cash charitable deduction, $6,208 of qualified dividends at 15% — but reduced wages by $16,408 of employer-sponsored insurance premiums, giving AGI $230,977 instead of $246,663.72. That exclusion at the 22% rate is $3,610, offset by $159 from its omission of the $721 IRA deduction, producing the $3,451 shortfall." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It claimed the full $928 qualified-vehicle-loan interest deduction, which is reduced $200 for every $1,000 of MAGI above $200,000 for joint filers and therefore reaches zero above $204,640 — at $246,664 of MAGI it is $0 — and it omitted the $721 traditional IRA deduction. The $928 deduction at 22% is $204.16 and the missing IRA deduction is $158.63, netting the $45.53 shortfall from $30,543.91." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It got the OBBBA stack right — $32,200 standard deduction, $25,000 capped overtime deduction, $580 cash-charity deduction — but subtracted $16,408 of pre-tax employer health premiums from wages, reaching taxable income of $173,197 instead of $188,883.95. The reference's $237,564 of employment income already reflects only the $15,436 traditional 401(k) deferral, so that second exclusion is a $16,408 overstatement worth about $3,610 of tax." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $33,473 corresponds to taxable income of about $202,200 against AGI $247,385 — roughly $45,200 of deductions versus the reference's $57,779.77. It named both the overtime and vehicle-loan deductions but claimed under half of the $25,000 overtime cap that applies in full below $300,000 of joint MAGI, while adding the auto-loan interest deduction that phases out entirely above $204,640, and it omitted the $721 IRA deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied every 2026 provision correctly — $32,200 standard deduction, $25,000 qualified-overtime deduction, $580 non-itemizer cash charity, $6,208 of qualified dividends at 15% — but subtracted $16,408 of pre-tax employer health premiums from wages to get AGI $230,977 rather than $246,663.72. That single duplicate exclusion drives ordinary taxable income to $166,989 instead of $182,675.95, worth $3,610 of tax." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It carried AGI at about $263,000, never removing the $15,436 traditional 401(k) deferral, itemized about $20,000 rather than taking the larger $32,200 standard deduction, and claimed nothing against the $44,647 FLSA overtime premium. Its taxable income of roughly $230,000 exceeds the reference's $188,883.95 by about $41,000, and it placed the couple in a 24% bracket that the correct $182,676 of ordinary income never reaches." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It priced 2026 as a TCJA-sunset year, claiming $10,500 of personal exemptions and itemizing $26,736 with uncapped property taxes under pre-TCJA rates, and took no qualified-overtime deduction. Under 2026 law the couple takes the $32,200 standard deduction with no personal exemptions plus the $25,000 overtime deduction and $579.77 of cash charity, giving taxable income of $188,883.95 rather than its $210,149." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied ""post-TCJA-sunset"" 2026 rules — $26,736 of itemized deductions plus about $10,910 of personal exemptions, with ordinary income taxed in 10/15/25% brackets — where 2026 uses a $32,200 standard deduction, no personal exemptions and a 22% top rate at this income. It also claimed no deduction against the $44,647 FLSA overtime premium, leaving taxable income of $209,739 against $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It gave no arithmetic; $36,370 corresponds to taxable income near $215,400, which is AGI $247,385 less about $32,000 — the standard deduction alone. It claimed nothing against the $44,647 FLSA overtime premium and nothing for the $579.77 non-itemizer cash charitable contribution, and its stated reliance on itemized deductions is moot because $26,736 falls below the $32,200 standard deduction." -us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It excluded $16,408 of employer-sponsored insurance premiums from wages to reach AGI $230,256, which the reference's $237,564 of employment income does not do, and then took no deduction against the $44,647 FLSA overtime premium and none for the $579.77 of cash charity. The two errors partly cancel, leaving taxable income of $198,056 against the reference's $188,883.95." -us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It computed 2026 ""with expired TCJA provisions,"" itemizing $26,736 and claiming two roughly $5,346 personal exemptions under pre-TCJA brackets, when the couple takes the $32,200 standard deduction with no exemptions under 10/12/22% rates. It also removed $16,408 of pre-tax health premiums from wages and claimed no $25,000 qualified-overtime deduction, giving taxable income of about $193,549 taxed at inflated rates." -us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It matched the reference on every OBBBA item — $32,200 standard deduction, uncapped $25,000 FLSA overtime deduction with no phaseout below $300,000 MAGI, $580 non-itemizer cash charity, $6,208 of qualified dividends at 15%, no NIIT — but never subtracted the spouse's $721 deductible traditional IRA contribution, so AGI stayed at $247,385 instead of $246,663.72. At the 22% marginal rate that $721.28 is $158.63, the whole gap from $30,543.91." -us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It ran its bracket schedule across the full $190,089 of taxable income — $24,500 + $75,100 + $90,489 sums to the entire amount, including the $6,208 of qualified dividends — and then added $931 of 15% dividend tax on top, double-taxing the dividends by about $1,366. It also capped the overtime deduction at $25,675 rather than $25,000 and used a $30,900 standard deduction instead of $32,200." -us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It matched the reference's AGI of $246,664, including the $721 deductible IRA contribution, and correctly chose the $32,200 standard deduction over $26,736 of itemized deductions, then claimed nothing against the $44,647 FLSA overtime premium and nothing for the $579.77 of non-itemizer cash charity. Those $25,580 of missed deductions at the 22% rate come to $5,628 — exactly its overstatement of $36,171.52 over $30,543.91." -us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It subtracted only about $4,000 of the $15,436 401(k) deferral, writing AGI as $258,821 where $262,821 less $15,436 is $247,385, then declared $15,395 of itemized deductions to ""exceed"" the $32,000 standard deduction and used the smaller figure, and claimed no $25,000 overtime deduction. It closed with ""use regular tax: $42,933.38"" and submitted $51,214.75, a number its own derivation never produces." -us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $18,160 of subtracted retirement contributions is the $15,436 traditional 401(k) plus $2,724 of Roth 401(k) contributions, which are after-tax and never reduce taxable income, while the $721 deductible IRA contribution was skipped; it also took no deduction against the $44,647 FLSA overtime premium. It then computed $44,586.80 of ordinary tax on $205,441, where the 2026 MFJ schedule yields about $34,600, so rate misapplication adds roughly $10,000 on top of the $22,765 of overstated taxable income." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own computation reached $30,830, but it submitted $33,223, a figure that does not follow from that work. The $30,830 also leaves out the $580 OBBBA non-itemizer cash charitable deduction and the $721 above-the-line traditional IRA deduction, so its taxable income of $190,185 is $1,301 above the frozen reference's; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), leaving only the $580 missing." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its work matches the reference except for the $721 traditional IRA contribution, which it disallowed under the §219(g) active-participant phaseout on the grounds that the spouse's 401(k) deferrals make them a plan participant. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. That $721 explains the entire $158.63 gap to the frozen reference (22% × $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used an outdated $29,200 standard deduction instead of the 2026 figure of $32,200 and never took the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA overtime premium. Its tax of $48,837 is also far above what 2026 joint brackets produce even on its own $217,364 of taxable income (about $37k)." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $25,000 overtime deduction, used a $31,500 standard deduction instead of $32,200, and added the Additional Medicare Tax, which is a payroll tax, to income tax. It then submitted $49,861, which bears no relation to the roughly $36.6k it had just computed." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It used a stale $30,300 standard deduction and 2025-style brackets and never applied the $25,000 qualified overtime deduction. It then submitted $53,503 even though its own calculation came to $37,055." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It used the $40,000 SALT cap as the deduction amount instead of the actual $20,655 of real estate taxes, which pushed itemized deductions to $46,081 and wrongly made itemizing beat the $32,200 standard deduction. It also left out the $25,000 overtime deduction, and $47,500 is far more than the brackets produce on the roughly $201k of taxable income it stated." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly disallowed the $721 traditional IRA contribution because the spouse is covered by a workplace plan. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. Everything else matches, so the $158.63 gap to the frozen reference comes from the $721 (22% of $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used 2025 values, a $30,000 standard deduction and 2025 bracket thresholds, instead of the 2026 figures ($32,200 standard deduction, 22% bracket up to $211,400). It also never applied the OBBBA qualified overtime deduction, which is $25,000 on the spouse's $44,647 FLSA premium, so taxable income came out at $217,385 instead of $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and used a $32,600 standard deduction. It then raised its own $38,831 estimate to $62,700 by pointing to Additional Medicare Tax and phase-outs, neither of which belongs in income tax before credits." +us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the full $928 of auto loan interest from AGI, a deduction the reference does not allow for this household. It also left out the $580 non-itemizer cash charitable deduction and the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Against the frozen reference net deductions were $373 too low and tax $82 too high; against the corrected value, $30,702.59, net deductions were $348 too high and tax $76.59 too low." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It never applied the $25,000 qualified overtime deduction and used a $31,000 standard deduction instead of $32,200. That left taxable income at $216,385 instead of $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law (pre-TCJA brackets of 10/15/25/28% and personal exemptions), but OBBBA made the TCJA rate schedule and standard deduction permanent. It also subtracted the $16,408 of employer-paid premiums from wages that do not include them and never took the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its stated method removes the $16,408 of employer-sponsored premiums from wages, but those premiums were never part of the $253,000. Its $36,639 corresponds to taxable income of about $216k under 2026 joint brackets, roughly $27.7k above the reference, which fits with leaving out the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It correctly used the $32,200 standard deduction and the 2026 brackets but never took the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium. It also left out the $580 non-itemizer charitable deduction and the $721 IRA deduction, so taxable income was $215,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA provisions sunset in 2026 and so used personal exemptions, uncapped itemizing and 10/15/25% brackets, but OBBBA made the TCJA structure permanent and added the $25,000 overtime deduction, which it never applied. It also subtracted $16,408 of employer-paid premiums from wages that exclude them." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It stated taxable income of about $229,000, roughly $40k above the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), because it did not take the $25,000 overtime deduction on top of the $32,200 standard deduction. Even so, $52,367 is higher than 2026 joint brackets produce on $229k." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 rules (personal exemptions, uncapped itemized deductions and pre-TCJA brackets) instead of OBBBA's permanent TCJA structure, and it never took the $25,000 overtime deduction. It also cut wages by $16,408 of employer-paid premiums that are not in the $253,000 of gross wages." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used post-TCJA-expiration schedules, with $10,100 of personal exemptions, itemized deductions and a 25% bracket, instead of OBBBA's permanent $32,200 standard deduction and 10/12/22% brackets. It also left out the $25,000 qualified overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $42,203 corresponds to taxable income of about $240k under 2026 joint brackets, meaning it applied neither the $25,000 overtime deduction nor most of the $32,200 standard deduction against AGI of roughly $247k." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,408 of employer-paid premiums from wages that never included them and used a $30,000 standard deduction instead of $32,200. It also never took the $25,000 qualified overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It worked from gross wages, interest and dividends with only generic allowable deductions. Its $37,921 corresponds to taxable income of about $220k, which is what AGI yields without the $25,000 qualified overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2025 figures, a $30,000 standard deduction and 2025 brackets, instead of the 2026 figures of $32,200 and a 22% bracket up to $211,400. It also never applied the $25,000 qualified overtime deduction, so taxable income was $217,385 instead of $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the correct $32,200 standard deduction but never took the $25,000 qualified overtime deduction, leaving taxable income at $215,185. It then overstated ordinary tax on $208,977 as $36,982; the 2026 brackets give $35,399." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It built its deductions around itemizing with the old $10,000 SALT cap, which never beats the $32,200 standard deduction here. Its $20,460 corresponds to taxable income of only about $143k, some $46k below the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), so its deduction stack was badly overstated." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $38,566 corresponds to taxable income of about $219k under 2026 joint brackets, which fits with leaving out the $25,000 qualified overtime deduction that brings taxable income down to $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,408 of employer-sponsored insurance premiums from wages. Those premiums are employer-paid and were never part of the $253,000 of gross wages, so AGI came out at $230,977. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income is $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the $928 of auto loan interest as a qualified vehicle-loan-interest deduction, which the reference does not allow for this household, and it left out the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Taxable income ended up $207 below the frozen reference's, producing the $45.53 shortfall; against the corrected value, $30,702.59, the $928 deduction alone leaves it $204.21 short." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,408 of employer-paid health premiums as a pretax reduction to wages that never included them, which cut taxable income to $173,197. That is $15,687 below the frozen reference's $188,884. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so the whole gap to the corrected value comes from the $16,408." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $33,473 corresponds to taxable income of about $202k, some $13k above the frozen reference's $188,884, so it did not apply the full stack of $32,200 standard, $25,000 overtime and $580 charitable deductions (the frozen reference also deducts the $721 IRA contribution, which the exclusion's corrected value disallows under the active-participant phase-out, 26 U.S.C. 219(g)). It also claimed a vehicle-loan-interest deduction that the reference does not allow." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $16,408 of employer-sponsored premiums even though the $253,000 of gross wages never included them, giving AGI of $230,977. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income was $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It stopped at the standard deduction plus the overtime deduction. It never took the OBBBA non-itemizer cash charitable deduction of $580 (available from 2026) or the $721 above-the-line IRA deduction, so taxable income was $1,301 above the frozen reference's. The frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so against the corrected value, $30,702.59, only the $580 is missing and its $30,830.14 is $127.55 high." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its joint AGI of $247,385 leaves out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That difference alone produces the $158.63 gap to the frozen reference, and its $30,702.54 is $0.05 below the corrected value." +us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It excluded $16,408 of employer-plan premiums from wages that never contained them, which put AGI at $230,977 instead of $246,664, the frozen reference's AGI net of a $721 IRA deduction. The model left out that deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income came out $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It said AGI was about $263k, ignoring the 401(k) exclusion. It itemized about $20k under caps, which is below the $32,200 standard deduction, and never took the $25,000 overtime deduction, so taxable income came out around $230k instead of $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (uncapped itemizing, $10,500 of personal exemptions and pre-TCJA brackets) even though OBBBA made the TCJA schedule permanent. It also left out the $25,000 qualified overtime deduction, leaving taxable income at $210,149." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed 2026 income under post-TCJA-sunset 10/15/25% brackets with personal exemptions of about $10,910 instead of OBBBA's permanent $32,200 standard deduction and 10/12/22% brackets. It also omitted the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $580 non-itemizer cash charitable deduction and the $721 IRA deduction, so taxable income was $190,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so of the two omissions only the $580 is an error. It also overstated ordinary tax on $183,977 as $30,051; the 2026 brackets give $29,899." +us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It built taxable income from SALT-capped itemized deductions instead of the larger $32,200 standard deduction and never applied the $25,000 qualified overtime deduction. Its $36,370 corresponds to taxable income of about $215k." +us,scenario_049,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $230,256 comes from subtracting the $16,408 of employer-paid premiums, which are not in wages, and it then took only the $32,200 standard deduction. It never applied the $25,000 overtime deduction or the $580 non-itemizer charitable deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA provisions had expired and used itemized deductions, two personal exemptions and pre-TCJA brackets, but OBBBA made the TCJA schedule and the $32,200 standard deduction permanent for 2026. It also subtracted $16,408 of employer premiums from wages and never took the $25,000 overtime deduction." +us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It started from $247,385, leaving out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That is the only difference, and it produces the $158.63 gap to the frozen reference; its $30,702.54 is $0.05 below the corrected value." +us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It treated the overtime deduction as an above-the-line adjustment of $25,675, above the $25,000 joint cap, and used a $30,900 standard deduction instead of $32,200. It also ran all $190,089 of taxable income, qualified dividends included, through the ordinary brackets and then added 15% on the $6,208 of dividends again, taxing them twice." +us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $721 IRA contribution, as the frozen reference does although the exclusion's corrected value disallows it under the active-participant phase-out (26 U.S.C. 219(g)), and used the $32,200 standard deduction, but it never applied the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium or the $580 non-itemizer charitable deduction. Taxable income came out at $214,464 instead of $188,884." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It miscomputed AGI as $258,821; $262,821 minus $15,436 is $247,385. It then used $15,395 of itemized deductions even though that is less than the standard deduction, and it never took the $25,000 overtime deduction. It finally submitted $51,214.75, which is unrelated to its own $42,933." +us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $18,160 of retirement contributions, which includes the after-tax $2,724 Roth 401(k), and never took the $25,000 qualified overtime deduction. It then reported $44,587 of ordinary tax on $205,441, far above the roughly $34.8k that the 2026 joint brackets give." us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -3819,36 +4188,37 @@ us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's reasoning correctly determined that the 53-year-old head lacks both the age threshold and any qualifying early-Medicare condition, but it submitted value 1 instead of 0. Its numeric output directly contradicts its own eligibility analysis and required final marker." us,scenario_049,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning derives the reference exactly — $11,439 of Social Security tax on the $184,500 2026 wage base, $3,668.50 of Medicare tax on the full $253,000, and $27 of Additional Medicare Tax, stated twice as $15,134.50 — yet it submitted $14,503.98, which matches none of its own components (it implies a $174,330 wage base). The defect is the transcription of its own correct total into the value field, not the payroll-tax computation." -us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It applied the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500 ($10,453.20 rather than $11,439), used the $200,000 single-filer Additional Medicare Tax threshold instead of the $250,000 joint threshold ($477 rather than $27), and fabricated a $904.13 New Hampshire employee payroll tax where the state imposes no mandatory employee payroll tax. Even its own listed components sum to $15,502.83, so the submitted $20,502.83 also carries a $5,000 addition error." -us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"It used the 2025 Social Security wage base of $176,100 in place of the 2026 base of $184,500, computing $10,918.20 of employee Social Security tax instead of $11,439. Medicare ($3,668.50 on all wages) and the Additional Medicare Tax ($27 on the $3,000 above the $250,000 joint threshold) were correct, so the stale wage base accounts for the entire $520.80 shortfall." -us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning identifies the $184,500 2026 wage base and computes $11,439 + $3,668.50 + $27 = $15,134.50 twice, then submits $13,164.85, a number that corresponds to none of its components and to no wage base. The failure is emitting a value that contradicts its own completed and correct derivation." -us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"It computed the correct structure ($3,668.50 Medicare on the full $253,000, $27 Additional Medicare Tax on the excess over $250,000) but settled on the 2025 wage base of $176,100 rather than the 2026 base of $184,500, yielding $10,918.20 of Social Security tax instead of $11,439. It had already generated the closer $183,600 variant ($15,078.70 total) and discarded it in favor of the prior-year base, costing $520.80." -us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"It correctly refused to let the $15,436 traditional 401(k) deferral reduce FICA wages and correctly applied the $250,000 joint Additional Medicare Tax threshold, but capped Social Security wages at the 2025 base of $176,100 instead of the 2026 base of $184,500. That single substitution understates Social Security tax by $520.80 and is the whole gap between $14,613.70 and $15,134.50." -us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Every component is structurally right — 1.45% on the full $253,000 and 0.9% on the $3,000 above the $250,000 joint threshold — but it used a $180,900 Social Security wage base rather than the 2026 base of $184,500. The $3,600 shortfall in the cap times 6.2% is exactly the $223.20 by which its $14,911.30 falls short." -us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It made two compounding errors: it subtracted the $16,408 employer-sponsored insurance premium from FICA wages to get $236,592, when Medicare tax applies to the full $253,000 of gross wages, and it capped Social Security wages at the 2025 base of $176,100 instead of the 2026 base of $184,500. The wage reduction cost $237.92 of Medicare tax and additionally dropped wages below the $250,000 joint threshold, erasing the $27 Additional Medicare Tax; with the $520.80 wage-base error, that is the full $785.72 shortfall." -us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Its $14,850.984 decomposes exactly as 6.2% × $184,200 = $11,420.40 plus 1.45% × $236,592 = $3,430.58, so it used a $184,200 wage base instead of the 2026 base of $184,500 and netted the $16,408 employer-sponsored insurance premium out of Medicare wages instead of taxing the full $253,000. That reduction also pushed wages under the $250,000 joint threshold, so it omitted the $27 Additional Medicare Tax entirely." -us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $16,408 pre-tax insurance premium from wages to reach $236,592 of Medicare wages, when Medicare tax applies to the full $253,000, and it used an estimated $183,900 Social Security cap instead of the 2026 base of $184,500. The wage reduction cost $237.92 of Medicare tax and drove its stated conclusion that wages fall below the $250,000 joint threshold, dropping the $27 Additional Medicare Tax; with the $37.20 cap error that is the entire $302.12 gap." -us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"Its stated method — 6.2% to a $176,100 limit, 1.45% on all wages, 0.9% above $250,000 — uses the 2025 wage base rather than the 2026 base of $184,500 and yields $14,613.70, but the submitted $13,959.81 does not even match that, implying a $165,553 cap. Both the prior-year wage base and the disagreement between its explanation and its value are wrong." -us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It defined FICA-subject wages as gross wages minus the $16,408 employer-sponsored insurance premium ($236,592), when Medicare tax applies to the full $253,000, and capped Social Security at roughly $183,600 instead of the 2026 base of $184,500 — $11,383.20 + $3,430.58 = $14,813.78, its rounded $14,814. Cutting Medicare wages below $250,000 also caused it to omit the $27 Additional Medicare Tax, which its component list never mentions." -us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It used a projected $174,900 Social Security cap ($10,844) rather than the 2026 base of $184,500 ($11,439) and applied the 1.45% Medicare rate to $236,592 after netting out the $16,408 insurance premium instead of to the full $253,000. Those two reductions, plus the resulting omission of the $27 Additional Medicare Tax once wages appeared to fall below $250,000, produce the $860.50 shortfall." -us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"It gave no derivation and submitted $4,058.45 for a household whose employee Social Security tax alone is 6.2% × $184,500 = $11,439, with $3,668.50 of Medicare tax on the full $253,000 and $27 of Additional Medicare Tax on top. Its figure is roughly 7.65% of $53,000 — the combined employee FICA rate applied to the excess of wages over a $200,000 threshold — rather than to the wage base and to total wages." -us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It capped Social Security wages at $181,200 instead of the 2026 base of $184,500 and reduced Medicare wages to $236,592 by subtracting the $16,408 employer-sponsored insurance premium, when Medicare tax applies to the full $253,000. Its two-component total also omits the $27 Additional Medicare Tax, which applies because gross wages exceed the $250,000 joint threshold." -us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It listed the right three components, including the 0.9% surtax on the $3,000 above the $250,000 joint threshold, but its $14,613.34 total backs out to a Social Security cap of about $176,100 — the 2025 wage base — instead of the 2026 base of $184,500. The stale cap year is worth $520.80 and accounts for essentially the whole error." -us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It computed Medicare ($3,668.50 on the full $253,000) and the Additional Medicare Tax ($27 on the excess over $250,000) exactly, then capped Social Security wages at the 2025 base of $176,100 rather than the 2026 base of $184,500. That yields $10,918.20 instead of $11,439, the entire $520.80 shortfall." -us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so nothing was submitted against the required key. This is an output-contract failure rather than a substantive misapplication of the Social Security cap, Medicare rate, or Additional Medicare Tax threshold." -us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It named the correct components but never enforced the Social Security wage base: 6.2% applied to all $253,000 is $15,686 versus $11,439 on the $184,500 cap, and its $19,648 total sits $4,513.50 above the capped result, consistent with taxing wages above the cap at the full 6.2% rate. The correct build is $11,439 + $3,668.50 + $27 = $15,134.50." -us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It correctly held that employer-sponsored insurance does not reduce FICA wages and included the Additional Medicare Tax, but its $15,302 total backs out to a Social Security cap near $187,200 rather than the 2026 base of $184,500. Guessing the wage base $2,700 high overstates Social Security tax by $167.50." -us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It excluded the $16,408 employer health premium from FICA wages, taxing $236,592 of Medicare wages instead of the full $253,000, and capped Social Security at $183,600 rather than the 2026 base of $184,500. The wage exclusion cost $237.92 of Medicare tax and led it to set the Additional Medicare Tax to $0 even though gross wages exceed the $250,000 joint threshold by $3,000, forfeiting another $27." -us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"It got the Social Security piece exactly right ($11,439 on the $184,500 2026 wage base) and erred only by computing Medicare tax on wages net of the $16,408 employer-plan premium ($3,430.58 on $236,592) rather than on the full $253,000. That single exclusion costs $237.92 of Medicare tax and, by pushing wages below $250,000, also drops the $27 Additional Medicare Tax — exactly the $264.92 gap." -us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied the correct $184,500 2026 wage base for the $11,439 Social Security component but treated the $16,408 employer health premium as pre-tax for FICA, computing Medicare on $236,592 instead of the full $253,000. The $237.92 Medicare shortfall plus the $27 Additional Medicare Tax it declared inapplicable — wages exceed the $250,000 joint threshold by $3,000 — is the whole $264.92 error." -us,scenario_049,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"It used an approximate $180,000 Social Security base rather than the 2026 base of $184,500, and then rounded its own components ($11,160 + $3,668.50 + $27 = $14,855.50) down to a flat $14,000, discarding a further $855. The precise build is $11,439 + $3,668.50 + $27 = $15,134.50." -us,scenario_049,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"It kept Medicare on the full $253,000 and applied the $250,000 joint Additional Medicare Tax threshold correctly, but capped Social Security wages at $183,600 instead of the 2026 base of $184,500. The $900 cap shortfall times 6.2% is the $55.50 gap between $15,079 and $15,134.50." -us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It stated that no Additional Medicare Tax is triggered, which only follows after netting the $16,408 employer health premium out of $253,000 of wages; the surtax applies because gross wages exceed the $250,000 joint threshold by $3,000, and Medicare tax applies to the full $253,000. Its $14,808 corresponds to a roughly $183,500 Social Security cap rather than the 2026 base of $184,500 plus $3,430.58 of Medicare tax on the reduced $236,592." -us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It identified the $184,500 2026 wage base and produced the correct $11,439 Social Security component, then reduced FICA wages by the $16,408 pre-tax employer health premium, computing Medicare on $236,592 rather than on the full $253,000. That exclusion costs $237.92 of Medicare tax and eliminates the $27 Additional Medicare Tax owed on the $3,000 of gross wages above the $250,000 joint threshold." -us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It set both Social Security and Medicare wages to $236,592 by subtracting the $16,408 employer health premium, when both taxes apply to the full $253,000 of gross wages here, and it estimated the Social Security cap at $183,000 instead of the 2026 base of $184,500. The $93 cap error, $237.92 of foregone Medicare tax, and the $27 Additional Medicare Tax it waived on the ground that wages fell below $250,000 total the $357.50 shortfall." -us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"Its reasoning cycled through the 2024 ($168,600), 2025 ($176,100), and a $184,500 cap without settling, ending in text at $14,613.70, then submitted $19,354.50 — exactly 7.65% of $253,000, the combined employee rate with no Social Security wage base applied at all. The correct build caps Social Security at $184,500 for $11,439 and adds $3,668.50 of Medicare plus $27 of Additional Medicare Tax." -us,scenario_049,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It stated $11,475 of capped Social Security tax (implying a $185,081 wage base rather than the 2026 base of $184,500), $3,668.50 of Medicare, and $27 of Additional Medicare Tax, components that sum to $15,170.50, yet it submitted $23,780.75. The dominant error is an arithmetic failure summing its own three components, compounded by an overstated wage base." +us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right: $11,439 Social Security on the $184,500 base, $3,668.50 Medicare and $27 Additional Medicare Tax, for $15,134.50. It then submitted an unrelated $14,503.98 that none of its own arithmetic supports." +us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 wage base of $168,600 instead of the 2026 base of $184,500. It applied the $200,000 single threshold instead of the $250,000 married-filing-jointly threshold, which inflated Additional Medicare Tax to $477. It also added a made-up $904.13 New Hampshire payroll tax. Its listed components add to $15,502.83, not the $20,502.83 it submitted." +us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 wage base of $176,100, giving $10,918.20, instead of the 2026 base of $184,500, which gives $11,439. The result is $520.80 too low. Its Medicare and Additional Medicare figures were correct." +us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning twice arrived at the correct $15,134.50: $11,439 + $3,668.50 + $27. It then submitted $13,164.85, a number that contradicts its own computation." +us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It briefly used an estimate of $183,600, then dropped it and kept the 2025 wage base of $176,100 for Social Security, giving $10,918.20. The 2026 base of $184,500 gives $11,439. That understated the total by $520.80." +us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It labeled $176,100, the 2025 figure, as the 2026 Social Security wage base and got $10,918.20 of Social Security tax. The 2026 base of $184,500 gives $11,439. Its Medicare and Additional Medicare figures were correct." +us,scenario_049,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a made-up 2026 wage base of $180,900, giving $11,215.80, instead of the actual $184,500, which gives $11,439. That understated Social Security tax by $223.20." +us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,408 employer-sponsored insurance premiums from the $253,000 gross wages, which cut Medicare tax to $3,430.58. The lower wages also fell below the $250,000 threshold, so it dropped the $27 Additional Medicare Tax. It also used the 2025 wage base of $176,100 instead of $184,500." +us,scenario_049,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"$14,850.98 equals 6.2% of a $184,200 wage base ($11,420.40) plus 1.45% of $236,592 ($3,430.58). That means it removed the $16,408 insurance premiums from Medicare wages and so omitted the $27 Additional Medicare Tax. It also used a Social Security base slightly below the correct $184,500." +us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated $236,592, which is gross wages minus the $16,408 premiums, as Medicare wages. That undercounted Medicare tax by $237.92 and wrongly concluded there was no Additional Medicare Tax. It also used a guessed wage base of $183,900 instead of $184,500." +us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It capped Social Security at the 2025 base of $176,100 instead of the 2026 base of $184,500. Even its own method gives $14,613.70, not the $13,959.81 it submitted, so the final number also contains an arithmetic error." +us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It reduced FICA wages to $236,592 by excluding the $16,408 insurance premiums. That lowered Medicare tax to $3,430.58 and removed the $27 Additional Medicare Tax. Its $14,814 total also implies a wage base of about $183,600 rather than $184,500." +us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It computed Medicare tax on $236,592 after subtracting the insurance premiums and left out the $27 Additional Medicare Tax. It also used a projected wage base of $174,900, which gives $10,844, instead of $184,500, which gives $11,439." +us,scenario_049,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"$4,058.45 is about the size of the Medicare tax alone. The model left out nearly all of the $11,439 Social Security tax owed on the first $184,500 of the spouse's wages. The correct components are $11,439, $3,668.50 and $27." +us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get $236,592 of FICA wages, so it understated Medicare tax and dropped the $27 Additional Medicare Tax. It also capped Social Security at a guessed $181,200 instead of the 2026 base of $184,500." +us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"$14,613.34 matches a calculation using the 2025 wage base of $176,100, which gives $10,918.20 of Social Security tax. The 2026 base of $184,500 gives $11,439, so the answer is short by about $521." +us,scenario_049,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"It explicitly used a $176,100 wage base, which is the 2025 figure, to get $10,918.20 of Social Security tax. The 2026 base is $184,500, which gives $11,439." +us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"$19,648 is about $4,500 above the correct total. That gap is consistent with applying the 6.2% Social Security rate to wages above the $184,500 cap instead of stopping at $11,439, since uncapped Social Security on all $253,000 is $15,686." +us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave $15,302 with no component breakdown, $167.50 above the correct total. The correct figures are $11,439 of Social Security on the $184,500 base, $3,668.50 of Medicare on $253,000, and $27 of Additional Medicare Tax on the $3,000 above the $250,000 married-filing-jointly threshold. The model overstated at least one of these." +us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It excluded the $16,408 insurance premiums from FICA wages, which gave Medicare tax of $3,430.58 and no Additional Medicare Tax. It also used a wage base of $183,600 instead of the 2026 base of $184,500." +us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"Its Social Security figure of $11,439 was correct. However, it computed Medicare on wages reduced by the $16,408 insurance premiums, $3,430.58 instead of $3,668.50. That also put wages under $250,000, so it wrongly dropped the $27 Additional Medicare Tax." +us,scenario_049,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used the correct $184,500 base for Social Security. It then took the $16,408 insurance premiums off Medicare wages, leaving $236,592. That understated Medicare tax by $237.92 and wrongly removed the $27 Additional Medicare Tax on the $3,000 of gross wages above $250,000." +us,scenario_049,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Its $11,439 of Social Security tax was correct. It treated the $16,408 insurance premiums as pre-tax and cut Medicare wages to $236,592. That lost $237.92 of Medicare tax and the $27 Additional Medicare Tax that applies to $253,000 of gross wages." +us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a rough wage base of about $180,000 and then rounded the whole total to $14,000. The exact figures are $184,500 × 6.2% = $11,439, plus $3,668.50 and $27." +us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It capped Social Security at a guessed 2026 wage base of $183,600, giving $11,383. The actual base is $184,500, which gives $11,439. Its Medicare and Additional Medicare figures were correct." +us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It said no Additional Medicare Tax applied, but the spouse's $253,000 of wages is $3,000 above the $250,000 married-filing-jointly threshold, which adds $27. Its $14,808 total is consistent with Medicare computed on premium-reduced wages and a Social Security base below $184,500." +us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It subtracted the $16,408 insurance premiums to get FICA wages of $236,592. That gave Medicare tax of $3,430.58 instead of $3,668.50 and removed the $27 Additional Medicare Tax. Its $11,439 of Social Security tax was correct." +us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"It reduced both Social Security and Medicare wages by the $16,408 insurance premiums, so it understated Medicare tax and dropped the $27 Additional Medicare Tax. It also used a $183,000 wage base, giving $11,346, instead of $184,500, giving $11,439." +us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The $19,354.50 it submitted equals 6.2% of the full $253,000 ($15,686) plus $3,668.50 of Medicare. That means it applied Social Security tax with no wage-base cap. None of the capped totals in its own reasoning used the correct 2026 base of $184,500 either." +us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"Its components are $11,475 of Social Security, which is slightly above the correct $11,439, plus $3,668.50 and $27. They add to $15,170.50, but it reported $23,780.75, so the addition itself is wrong." us,scenario_049,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -3858,50 +4228,55 @@ us,scenario_049,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_ us,scenario_049,spouse_medicare_eligible,gpt-5.4-nano,llm_error,other,False,"The model's reasoning correctly concluded that the 52-year-old spouse lacked every Medicare eligibility pathway and explicitly stated the flag was 0, but it submitted value = 1. It inverted the binary value at the final output step." us,scenario_049,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_049,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model built the New Hampshire Interest & Dividends base correctly ($3,501 interest + $6,320 dividends = $9,821, less the $4,800 joint exemption = $5,021) but then multiplied it by a 1% rate that does not exist in 2026. The 1% figure belongs to the 2021 phase-down schedule that HB 2 (2023) superseded by repealing the I&D tax for taxable periods beginning after December 31, 2024, so New Hampshire assesses no income tax on this household and the correct amount is $0." -us,scenario_049,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the key was scored as missing rather than substantively wrong. Nothing in its output engages New Hampshire's Interest & Dividends treatment, which yields $0 for this household in 2026." -us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model asserted that New Hampshire taxes only dividends and interest and then submitted $2,350, a figure no New Hampshire computation can produce: the household's entire I&D base is $9,821, leaving $5,021 after the $4,800 joint exemption, so even the highest rate New Hampshire ever imposed (5%) caps out near $250. The $2,350 is consistent with applying a roughly 1% broad-based state income tax to the $253,000 of wages that New Hampshire does not tax at all, on top of missing that the I&D tax itself is repealed for 2026, where the correct amount is $0." -us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model computed the correct $5,021 post-exemption Interest & Dividends base ($9,821 less the $4,800 joint exemption) and then applied a 3% rate, which was New Hampshire's rate for tax year 2024 only. The I&D tax was repealed for taxable periods beginning after December 31, 2024, so no rate attaches in 2026 and the household owes $0." -us,scenario_049,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model correctly reduced $9,821 of interest and dividends by the $4,800 joint exemption to $5,021 but then applied what it called ""the 2026 rate of 1%,"" a rate drawn from the superseded 2021 phase-down schedule. HB 2 (2023) accelerated the Interest & Dividends repeal to taxable periods beginning after December 31, 2024, leaving New Hampshire with no individual income tax in 2026 and a liability of $0." -us,scenario_049,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model applied an unnamed ""applicable rate"" to the exemption-reduced Interest & Dividends base, producing $50 — 1% of the $5,021 that remains after the $4,800 joint exemption. New Hampshire's I&D tax was repealed for taxable periods beginning after December 31, 2024, so the applicable 2026 rate is zero and the correct liability is $0; the model also stated it was taxing only interest and non-qualified dividends while its number reflects the full $9,821 including qualified dividends." -us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model assembled the Interest & Dividends base exactly as PolicyEngine does ($3,501 + $6,320 = $9,821, minus the $4,800 joint exemption = $5,021) and then multiplied by 1%, a rate from the pre-2023 phase-down calendar. The repeal enacted in HB 2 (2023) eliminated the tax for taxable periods beginning after December 31, 2024, so New Hampshire imposes no income tax for 2026 and the answer is $0." -us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model itemized the I&D base perfectly ($1,500 + $2,001 interest, $112 non-qualified and $6,208 qualified dividends = $9,821) and applied the $4,800 joint exemption to reach $5,021, then multiplied by a 1% rate it attributed to 2026. That rate came from the superseded phase-down schedule; the Interest & Dividends tax is repealed for taxable periods beginning after December 31, 2024, making the 2026 New Hampshire liability $0." +us,scenario_049,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It said New Hampshire taxes interest and dividends at 1% in 2026 and got $50.21, which is 1% of the $5,021 base left after exemptions. That 1% comes from the superseded 2021 phase-down schedule. The tax was repealed for periods beginning after 2024, so the 2026 liability on that base is $0." +us,scenario_049,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its base was right: $3,501 of interest plus $6,320 of dividends minus the $4,800 exemption is $5,021. It then multiplied by a 1% rate for 2026 that no longer applies. The Interest and Dividends Tax does not apply after 2024, so the correct liability is $0, not $50.21." +us,scenario_049,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score. This is a missing-output failure, not a mistake about New Hampshire tax." +us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed New Hampshire still charges a limited tax on dividends and interest and gave $2,350 with no calculation. That number has no basis: even the old 5% rate on all $9,821 of interest and dividends comes to only $491. New Hampshire charges no Interest and Dividends Tax in 2026, so the correct answer is $0." +us,scenario_049,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly took the $4,800 joint exemption off $9,821 of interest and dividends to get $5,021. It then applied 3%, which was the 2024 rate, and got $150.63. The tax was repealed for taxable periods beginning after December 31, 2024, so the 2026 rate is effectively zero and the liability is $0." +us,scenario_049,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied what it called the 2026 rate of 1% to $9,821 less the $4,800 joint exemption and rounded the result to $50. The 1% comes from the superseded 2021 phase-down schedule. The 2023 repeal leaves no tax on interest and dividends in 2026, so the answer is $0." +us,scenario_049,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It said New Hampshire taxes interest and dividend income in 2026 and gave $50. That equals the stale 1% rate on the $5,021 post-exemption base, rounded. It also misstated the base as interest plus non-qualified dividends only, which comes to $3,613 and falls below the $4,800 exemption. Either way the correct liability is $0, because the Interest and Dividends Tax was repealed for periods after 2024." +us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It computed the $5,021 base correctly ($3,501 interest plus $6,320 dividends minus the $4,800 joint exemption). It then applied a 1% rate for 2026 from the superseded 2021 phase-down schedule and got $50.21. New Hampshire levies no Interest and Dividends Tax in 2026, so the liability is $0." +us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It itemized the interest and dividends correctly, subtracted the $4,800 joint exemption to reach $5,021, and then multiplied by a 1% rate it believed applied in 2026. That rate comes from the 2021 phase-down schedule, which the 2023 law replaced. The tax was repealed effective 2025, so the correct 2026 amount is $0, not $50.21." us,scenario_049,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_049,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work produces $2,641.60 using the correct $16,100 standard deduction and $12,400/12% schedule, overstated only by pushing the $180 of non-Schedule-D capital gain distributions into ordinary income. It then submitted $3,086, a figure no step in its derivation yields, discarding a computation that was within $21.60 of the reference." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Applied the correct 2026 single standard deduction of $16,100 and the $12,400 ceiling on the 10% bracket, but added the $180 of non-Schedule-D capital gain distributions to ordinary income and taxed it at 12%. Ordinary taxable income is $23,900, not $24,080, so the schedule gives $1,240 + 12% × $11,500 = $2,620; the entire $21.60 excess is that $180." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"Subtracted the $5,389 employer-sponsored insurance premium from the stated $40,000 of gross wages, which is already the taxable wage figure, and used the 2025 $15,000 standard deduction instead of 2026's $16,100. It then abandoned its own $2,103 result and submitted $0 by netting the EITC against tax, a refundable credit the output definition explicitly excludes from this quantity." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Reached $2,642 with the correct $16,100 standard deduction and 10%/12% schedule, then replaced it with $2,960 by invoking a $16,000 standard deduction and a $12,150 bracket edge that together yield about $2,657. The submitted number follows from neither parameter set it named." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Computed $1,240 + 12% × $11,680 = $2,642 from the correct $16,100 standard deduction and $12,400 bracket edge, then wrote ""using approx: tax = $2,867"" and submitted that. No parameter set in its reasoning produces $2,867; the only substantive error in the arithmetic it discarded was taxing the $180 of capital gain distributions at 12%." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Derived $1,240 + $1,402 = $2,642 from the correct $16,100 standard deduction and $12,400 bracket ceiling, then asserted that ""applying 2026 brackets yields about 2,955"" and submitted $2,955. The 2026 schedule on its own $24,080 of taxable income gives $2,641.60, and on the correct $23,900 gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Settled on 2025 parameters — a $15,000 standard deduction and an $11,925 ceiling on the 10% bracket — instead of 2026's $16,100 and $12,400, and added the $180 of capital gain distributions to ordinary income. The $1,280 of excess taxable income taxed at 12% plus the $475 of bracket shift taxed 2 points higher accounts for the full $163 overstatement." -us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Used the correct $16,100 standard deduction but a $12,150 ceiling on the 10% bracket instead of $12,400, reaching $2,646.60, then submitted $3,059 attributed to ""standard deduction adjustments and rounding."" No standard deduction or bracket combination in its reasoning produces $3,059." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used a $15,400 standard deduction and the 2025 $11,925 ceiling on the 10% bracket rather than 2026's $16,100 and $12,400, and taxed the $180 of capital gain distributions as ordinary income. The resulting $880 of excess taxable income at 12% plus $9.50 of bracket shift is the entire $115 overstatement." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Treated 2026 as a post-TCJA-expiration year with a $7,493 standard deduction, a $4,779 personal exemption, and a 15% second bracket; the 2026 law in force sets the single standard deduction at $16,100, allows no personal exemption, and taxes the second bracket at 12%. It compounded that by deducting the $5,389 employer-sponsored insurance premium from wages that are already stated net of pre-tax exclusions." -us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Applied a pre-TCJA schedule for 2026 — an $8,600 standard deduction plus a $4,950 personal exemption and a 15% second bracket — producing $26,630 of taxable income against the correct $23,900. The 2026 single standard deduction is $16,100 with no personal exemption and a 12% second bracket, giving $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,389 employer-sponsored insurance premium from the stated gross wages and then applied a TCJA-expiration schedule with an $8,300 standard deduction, a $5,150 personal exemption, and a 15% second bracket. Two large offsetting errors landed it within $1.15 of the reference from an entirely wrong law baseline; the correct path is $40,000 − $16,100 = $23,900 taxed at 10%/12% for $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Used the stale 2025 $15,000 standard deduction and then subtracted $1,300 of nonrefundable Child Tax Credit from a childless 25-year-old single filer, who has no qualifying child and therefore no CTC of any kind. Dropping the invented credit and using the 2026 $16,100 standard deduction gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,389 employer-sponsored insurance premium from the $40,000 of stated gross wages and then applied roughly $13,400 of combined standard deduction and personal exemption from the pre-TCJA schedule. The 2026 computation allows no personal exemption and no premium subtraction: $40,000 − $16,100 = $23,900 taxed at 10%/12% for $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Deducted the $5,389 employer-sponsored insurance premium from wages already stated as taxable, then used a post-TCJA-expiration $8,300 standard deduction, a $5,150 personal exemption, and a 15% second bracket. The 2026 schedule has a $16,100 standard deduction, no personal exemption, and a 12% second bracket, so $23,900 of taxable income yields $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Gave no derivation, only the assertion that the standard deduction and 2026 brackets produce $2,388. That figure corresponds to about $21,970 of ordinary taxable income, roughly $1,930 below the $23,900 that $40,000 of wages less the $16,100 standard deduction produces." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Removed the $5,389 employer-sponsored insurance premium from the stated gross wages, used a $15,700 standard deduction instead of $16,100, and applied the 2025 $11,925 bracket edge, giving $19,091 of taxable income against the correct $23,900. Wages of $40,000 are already the taxable amount, so no premium subtraction applies and the tax is $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Applied a personal exemption alongside the standard deduction and taxed the second bracket at 15%, a pre-TCJA structure; its $3,392 is consistent with about $13,450 of total deductions and a 10%/15% schedule. In 2026 the single standard deduction is $16,100, there is no personal exemption, and the second bracket rate is 12%, producing $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Assumed TCJA expiration for 2026 with an $8,300 standard deduction, a $5,300 personal exemption, and a 15% rate above $12,200, yielding $26,580 of taxable income. The governing 2026 parameters are a $16,100 standard deduction, no personal exemption, and 10%/12% brackets breaking at $12,400, giving $23,900 taxable and $2,620 of tax." -us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Computed $2,831.10 from a $14,600 standard deduction and the 2025 $11,925 bracket edge, then submitted $2,530 after a digression about ACA premium tax credit affordability, which has no effect on income tax before refundable credits. Neither its $14,600 deduction nor the submitted $2,530 follows from the 2026 schedule, which gives $23,900 taxable and $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Used the correct $16,100 standard deduction but taxed the $180 of non-Schedule-D capital gain distributions at the 12% ordinary rate and set the 10% bracket ceiling at $12,200 instead of $12,400. Those two errors contribute $21.60 and $4.00 respectively, the whole $25.60 gap above $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction and the small capital gain leave no federal income tax, but $40,000 of wages less the $16,100 standard deduction leaves $23,900 of taxable income and $1,240 + 12% × $11,500 = $2,620 of tax. Payroll taxes being reported in a separate output does not reduce income tax liability." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Produced $2,146 as a bare estimate with no bracket arithmetic. That amount corresponds to about $19,950 of ordinary taxable income, implying roughly $20,000 of deductions against the $16,100 standard deduction that actually applies and yields $23,900 taxable and $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used a $15,350 standard deduction and a $12,000 ceiling on the 10% bracket instead of 2026's $16,100 and $12,400, and its own parameters yield $1,200 + 12% × $12,830 = $2,739.60 rather than the $2,830.60 submitted. The correct figures give $23,900 of taxable income and $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct $16,100 standard deduction and 2026 brackets but carried the $180 of non-Schedule-D capital gain distributions into ordinary taxable income at 12%. Ordinary taxable income is $23,900, so the schedule gives $2,620 and the $21.60 excess is exactly that $180." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Stated taxable income of about $24,680, implying a roughly $15,500 standard deduction, and submitted $2,781, which corresponds instead to a $15,000 deduction with the 2025 $11,925 bracket edge. The 2026 standard deduction is $16,100 and the 10% bracket runs to $12,400, giving $23,900 taxable and $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Carried the 2025 parameters forward — a $15,000 standard deduction and an $11,925 ceiling on the 10% bracket — and added the $180 of capital gain distributions to ordinary income. The 2026 values are $16,100 and $12,400, which drop taxable income to $23,900 and tax to $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Applied the correct $12,400 ceiling on the 10% bracket but understated the standard deduction as $15,750 instead of $16,100 and taxed the $180 of capital gain distributions as ordinary income. The combined $530 of excess taxable income at 12% is the entire $63.60 gap above $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $5,389 employer-sponsored insurance premium from stated gross wages and then applied a pre-TCJA schedule with an $8,509 standard deduction, a $5,427 personal exemption, and a 15% second bracket. In 2026 the wages of $40,000 stand as taxable, the standard deduction is $16,100, there is no personal exemption, and the second bracket rate is 12%, giving $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so nothing substantive reached the grader. The failure is in producing the required output key, not in the tax computation." -us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Used the correct $16,100 standard deduction but a $12,750 ceiling on the 10% bracket to reach $2,635, then cut that to $2,090 by subtracting an invented ""nonrefundable credit for net capital gains"" of about $545. The 0% long-term rate is a preferential rate applied to the $180 of gains, not a credit against ordinary tax; the correct schedule on $23,900 of ordinary income gives $2,620." -us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Applied the correct $16,100 standard deduction and the $12,400/$50,400 bracket edges but included the $180 of non-Schedule-D capital gain distributions in ordinary income at 12%. Ordinary taxable income is $23,900, giving $1,240 + $1,380 = $2,620; the $21.60 excess is entirely the misplaced $180." -us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Computed $2,618.60 from a $16,300 standard deduction and a $12,350 bracket edge — within $1.40 of the reference — then submitted $1,410, a number no step in its derivation produces. It abandoned a correct-in-structure calculation at the reporting step." -us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction and nonrefundable credits reduce the liability to zero with no computation shown. $40,000 of wages less the $16,100 standard deduction leaves $23,900 of taxable income, and a childless 25-year-old single filer has no nonrefundable credits, so the tax is $1,240 + 12% × $11,500 = $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached $2,641.60, which was already $21.60 too high because it taxed the $180 capital gain distribution at 12% instead of 0%. It then submitted $3,086, a number with no derivation in its reasoning." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $12,400 bracket, but it labeled the $180 of non-Schedule D capital gains as short-term and taxed it at 12%. Capital gain distributions are long-term gains taxed at 0% here, so the tax on $23,900 of ordinary income is $2,620, not $2,641.60." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It subtracted the $5,389 employer-sponsored premium from gross wages and used 2025-era figures ($15,000 deduction, $11,600 bracket), reaching about $2,103. It then zeroed out the tax by pointing to the EITC, which is refundable and by definition excluded from tax before refundable credits." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It first computed $2,642 with the correct 2026 parameters, while wrongly taxing the $180 capital gain distribution at 12%. It then switched to invented figures ($16,000 deduction, $12,150 bracket) and submitted $2,960, which those figures do not produce either." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed $2,642 from the correct $16,100 deduction and $12,400 bracket, with the $180 capital gain distribution wrongly taxed at 12% instead of 0%. It then submitted $2,867 with no arithmetic behind the change." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It set up the right 2026 computation ($1,240 plus $1,402 = $2,642), with the $180 capital gain distribution wrongly taxed at 12%. It then submitted $2,955, which is not the sum of its own bracket amounts." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on 2025 parameters (a $15,000 standard deduction and a $11,925 10% bracket) instead of the 2026 values of $16,100 and $12,400, which overstated taxable income at $25,180. It also explicitly taxed the $180 of non-Schedule D gains as ordinary income rather than at the 0% capital gains rate." +us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used the correct $16,100 deduction with a $12,150 bracket and reached $2,646.60. It then added about $400 by double-counting the $180 of capital gains 'at ordinary rates', even though they were already in taxable income and are taxed at 0% as capital gain distributions." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $11,925 10% bracket instead of the 2026 values of $16,100 and $12,400. Although it said the capital gains were taxed at 0%, its arithmetic put the full $24,780, including the $180, at ordinary rates." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $7,493 pre-TCJA standard deduction, a $4,779 personal exemption and a 15% bracket. The TCJA structure was made permanent, so the correct figures are a $16,100 deduction, no exemption and 10%/12% rates. It also wrongly subtracted the $5,389 employer premium from gross wages." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a post-TCJA-expiration schedule ($8,600 standard deduction, $4,950 personal exemption, 15% second bracket) instead of the permanent 2026 rules. Those rules give a $16,100 deduction, no exemption and a 12% second bracket, for $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 of gains at 0%, but it used a $15,400 standard deduction and a $12,200 bracket instead of the 2026 values of $16,100 and $12,400. That left $24,600 of ordinary taxable income instead of $23,900." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,300 standard deduction, a $5,150 personal exemption and a 15% bracket, and it subtracted the $5,389 employer premium from wages. Those offsetting errors land near $2,620 by coincidence. The correct computation is $40,000 minus the $16,100 deduction, taxed at 10%/12%." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used a stale $15,000 standard deduction and then subtracted a $1,300 nonrefundable Child Tax Credit, even though the filer has no children and no nonrefundable credit applies." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted the $5,389 employer premium from gross wages and applied a pre-TCJA-style deduction plus personal exemption of about $13,400. The correct figures are the 2026 $16,100 standard deduction against the full $40,000 of wages, with no exemption." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It treated 2026 as a TCJA-expiration year, using an $8,300 deduction, a $5,150 exemption and a 15% rate on income above $11,600, and it subtracted the $5,389 employer premium from wages. Under the permanent 2026 rules, the $16,100 deduction and 10%/12% brackets give $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $40,000 minus $16,100 = $23,900 taxable, which gives $2,620. Its $2,388 matches about $21,970 of taxable income, so it over-deducted roughly $1,930." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $5,389 employer premium from gross wages, even though listed wages are the gross amount. It also used a $15,700 deduction and the 2025 $11,925 bracket, which understated taxable income at $19,091 instead of $23,900." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied an expired-TCJA schedule with a standard deduction, a personal exemption and a 15% second bracket. Under the permanent 2026 law, the $16,100 deduction and 12% second bracket give $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed the $180 of gains at 0%, but it assumed the TCJA expired. It used an $8,300 deduction plus a $5,300 exemption and a 15% rate instead of the 2026 $16,100 deduction and 12% second bracket." +us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It used the 2024 $14,600 deduction and the 2025 $11,925 bracket and computed $2,831.10. It then submitted $2,530 with a premium-tax-credit discussion that supplies no arithmetic. The PTC has no role in this measure." +us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It ended the 10% bracket at $12,200 instead of $12,400, and it taxed the $180 of non-Schedule D gains as ordinary income at 12% instead of at the 0% capital gains rate." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction wipes out all tax. The 2026 $16,100 deduction leaves $23,900 of taxable wage income, which is taxed at 10%/12% for $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave only a vague reference to 'nonrefundable taxes reduced'. The correct computation is $23,900 taxable, which gives $2,620. Its $2,146 matches taxable income about $3,950 too low, as if an extra deduction had been taken." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It estimated a $15,350 standard deduction and a $12,000 bracket instead of the 2026 values of $16,100 and $12,400. It also taxed the $180 capital gain distribution at 12% instead of 0%." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the correct $16,100 deduction and 2026 brackets, but it taxed the $180 of non-Schedule D capital gains at the 12% ordinary rate. Those capital gain distributions are long-term gains taxed at 0%, which drops the tax by $21.60 to $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied the correct 2026 deduction and brackets, but it put the $180 capital gain distribution in the 12% bracket. That $180 is taxed at the 0% long-term capital gains rate, so the tax is $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It included the $180 of non-Schedule D capital gains in ordinary taxable income at 12%. As capital gain distributions they are taxed at 0% in this bracket, so only the $23,900 of wage taxable income is taxed, giving $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,500, which gave $24,680 of taxable income, instead of the 2026 $16,100. It also taxed the $180 capital gain distribution at ordinary rates instead of 0%." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025 parameters (a $15,000 deduction and a $11,925 bracket) rather than the 2026 values of $16,100 and $12,400. It also taxed the $180 capital gain distribution at 12% instead of 0%." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the inflation-adjusted 2026 $16,100. It also taxed the $180 capital gain distribution at 12% instead of the 0% rate." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $15,400 deduction and a $12,225 bracket instead of the 2026 values of $16,100 and $12,400. It also treated the $180 of non-Schedule D gains as ordinary income instead of taxing it at 0%." +us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted pre-TCJA rules' ($8,509 deduction, $5,427 exemption, 15% bracket) and subtracted the $5,389 employer premium from gross wages. Under the permanent 2026 law, $40,000 minus the $16,100 deduction, taxed at 10%/12%, gives $2,620." +us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there was no answer to score." +us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It used the correct $16,100 deduction but a $12,750 bracket. It then invented a roughly $545 'nonrefundable credit for net capital gains', when the 0% capital gains rate only removes tax on the $180 of gains, which is worth $21.60." +us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly applied the $16,100 deduction and the $12,400 10% bracket, but it taxed the $180 of non-Schedule D capital gains at 12%. Those capital gain distributions are taxed at 0%, which gives $2,620 instead of $2,641.60." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $2,618.60 using a $16,300 deduction and a $12,350 bracket in place of the 2026 values of $16,100 and $12,400. It then submitted $1,410, a number that has no support in its own arithmetic." +us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It claimed the standard deduction and nonrefundable credits cut the tax to zero. The childless filer has no nonrefundable credits, and the $16,100 deduction leaves $23,900 taxable, which gives $2,620." us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model awarded $560 of childless EITC without applying the phaseout endpoint for a single filer. At $40,000 of earnings, the childless EITC has fully phased out." us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract." us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own EITC calculation correctly reached $0 and stated that no other refundable credits apply, but it submitted $2,557.60. The submitted value contradicts its completed derivation and does not represent any applicable refundable-credit component." @@ -3919,236 +4294,263 @@ us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract. us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation." us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It froze Louisiana's flat-tax standard deduction at the un-indexed statutory $12,500 instead of the 2026 inflation-adjusted $12,835 and added the $180 of non-Schedule-D capital gains to the $40,000 wage base, producing $830.40 instead of 3% x $27,165 = $814.95. It then discarded its own arithmetic and submitted $1,020, a figure no step in its reasoning supports." -us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It correctly applied Louisiana's post-Act-11 flat 3% rate but used the un-indexed $12,500 single standard deduction rather than the 2026 indexed amount of $12,835, and included the $180 of non-Schedule-D capital gains in the state base. Its base of $27,680 overstates the correct $27,165 by $515, and 3% of that gap is the entire $15.45 error." -us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that Louisiana imposes no individual income tax, confusing it with a no-income-tax state such as Texas or Florida. Louisiana levies a 3% flat individual income tax for 2026, which on $40,000 of wages less the $12,835 standard deduction yields $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It identified the correct 2024 reform (flat 3%, personal exemption folded into the standard deduction) but held the standard deduction at the base $12,500 rather than the 2026 indexed $12,835, and it pulled the $180 of non-Schedule-D capital gains into the Louisiana base. Those two additions to taxable income ($515 combined) account for the entire $15.45 overstatement." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"After computing $830.40 from the un-indexed $12,500 deduction and a $40,180 base, it subtracted roughly $198 for a Louisiana 'personal credit' that does not exist: the personal exemption was absorbed into the flat-tax standard deduction and there is no residual per-filer credit. The correct chain is $40,000 less the 2026 indexed $12,835 deduction at 3%, with no nonrefundable credits, giving $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the flat 3% tax on $40,180 less the un-indexed $12,500 deduction, got about $830, then shaved it to $812 citing unspecified 'rounding of federal AGI adjustments.' The real $15.45 reduction comes from the 2026 indexed standard deduction of $12,835 and a base of $40,000 that excludes the $180 of non-Schedule-D capital gains, neither of which it applied." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It landed on the right flat 3% rate but subtracted $2,783 for the Louisiana deduction for federal income taxes paid, which was repealed effective 2022 by the constitutional amendment and conforming legislation and does not exist under the 2025+ flat tax. It also used the un-indexed $12,500 standard deduction rather than the 2026 indexed $12,835 and included the $180 of capital gains, so its base of $24,897 understates the correct $27,165." -us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its stated derivation produces $830.40 from the un-indexed $12,500 deduction and a $40,180 base, but it submitted $1,080 after invoking a vague 'combined personal exemption/standard deduction' and unnamed nonrefundable credits, which under the flat tax reduce nothing. The correct computation is $40,000 less the 2026 indexed $12,835 deduction at 3% = $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the correct 3% flat rate to the pre-reform deduction stack of a $4,500 standard deduction plus a $4,500 personal exemption, both of which were replaced by the single $12,500 flat-tax standard deduction (indexed to $12,835 for 2026). Its $31,180 base overstates the correct $27,165 by $4,015." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the repealed 4.25% top marginal rate from the 2022-2024 graduated schedule instead of the 3% flat rate effective 2025, allowed only the eliminated $4,500 personal exemption instead of the $12,835 standard deduction, and cut AGI to $34,791 by subtracting the $5,389 employer-sponsored insurance premium from wages already reported net of Section 125 amounts. The correct base is $40,000 less $12,835 at 3%." -us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the correct flat 3% rate but used the un-indexed $12,500 standard deduction rather than Louisiana's 2026 inflation-adjusted $12,835 and added the $180 of non-Schedule-D capital gains to the base. That $515 of excess taxable income at 3% is exactly the $15.45 by which it overshot $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It reconstructed the entire pre-2025 Louisiana regime: the graduated 1.85%/3.5% brackets, the $4,500 combined standard deduction/personal exemption, and the deduction for federal income taxes paid, all of which were eliminated by the 2024 reform that installed a 3% flat rate and a $12,500 standard deduction indexed to $12,835 for 2026. It further reduced AGI to $34,791 by subtracting the $5,389 employer-sponsored insurance premium from wages already net of it." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied 'progressive individual income tax rates,' but Louisiana repealed its graduated schedule effective 2025 in favor of a single 3% rate. Its $724 is consistent with the old 1.85%/3.5% brackets on a base near $26,600; the correct computation is $40,000 less the 2026 indexed $12,835 standard deduction at a flat 3% = $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It taxed $30,291 under the repealed 1.85%/3.5% graduated brackets after allowing only the eliminated $4,500 personal exemption, and it shrank AGI to $34,791 by subtracting the $5,389 employer insurance premium from wages already reported net of Section 125 amounts. Louisiana's 2026 law is a 3% flat rate on $40,000 less a $12,835 standard deduction." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It deducted $2,621.15 of federal income tax and a $4,500 combined standard deduction/exemption and then applied the 1.85%/3.5% brackets, reproducing Louisiana law as it stood before the 2024 reform repealed the federal tax deduction, the $4,500 exemption, and the graduated schedule. The 2026 computation is 3% of $40,000 less the indexed $12,835 standard deduction." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $1,115 is roughly 3% of $37,167, meaning it taxed essentially undeducted income and never subtracted Louisiana's $12,835 standard deduction. Applying that deduction to the $40,000 base leaves $27,165 taxable at the 3% flat rate, or $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It used the pre-reform $4,500 combined standard deduction/exemption and the repealed 1.85%/3.5% bracket schedule, and it cut AGI to $34,791 by subtracting the $5,389 employer-sponsored insurance premium from wages already net of it. Louisiana taxes 2026 income at a flat 3% after a $12,835 standard deduction, giving $814.95 on the $40,000 base." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It allowed only the repealed $4,500 standard deduction/personal exemption and applied the pre-2025 graduated 1.85%/3.5% brackets to a $35,680 base. The 2026 law grants a $12,835 indexed standard deduction and imposes a single 3% rate, so the base is $27,165 and the tax $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It carried forward the pre-reform $4,500 single standard deduction and the repealed 1.85%/3.50% bracket schedule, taxing $35,680. Louisiana's 2026 flat tax applies 3% to $40,000 less the inflation-indexed $12,835 standard deduction, leaving $27,165 and a liability of $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the correct 3% flat rate but used the statutory base standard deduction of $12,500 instead of the 2026 inflation-indexed $12,835, and it added the $180 of non-Schedule-D capital gains to the $40,000 wage base. The $515 of excess taxable income at 3% produces its $15.45 overstatement." -us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It recalled the flat 3% rate correctly but not the annual indexation of the single standard deduction, using $12,500 instead of the 2026 value of $12,835, and it included the $180 of non-Schedule-D capital gains in the Louisiana base. The correct base is $27,165, not $27,680." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed Louisiana's standard deduction and exemptions fully offset the tax base, but the $12,835 single standard deduction leaves $27,165 of $40,000 in wages exposed to the 3% flat rate. Zeroing the liability required a deduction more than three times the actual amount." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It named no rate, deduction, or bracket and submitted $781, which corresponds to a 3% base of about $26,033. The statutory computation is $40,000 of wages less the 2026 indexed $12,835 standard deduction, or $27,165, taxed at the flat 3% rate for $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the right 3% flat rate to the wrong base, using the un-indexed $12,500 standard deduction instead of the 2026 indexed $12,835 and folding in the $180 of non-Schedule-D capital gains. Its $27,680 base exceeds the correct $27,165 by $515." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used the un-indexed $12,500 single standard deduction rather than Louisiana's 2026 inflation-adjusted $12,835 and included the $180 of non-Schedule-D capital gains in the state base. Both errors inflate taxable income, and 3% of the $515 excess is the whole $15.45 miss." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its 3% rate is right but its deduction is the statutory $12,500 rather than the 2026 indexed $12,835, and it taxed $40,180 rather than the $40,000 wage base. Correcting both yields $27,165 taxable and $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied Louisiana's 3% flat rate over a base built from the un-indexed $12,500 standard deduction and $40,180 of income including the $180 of non-Schedule-D capital gains. The 2026 deduction is $12,835 and the state base is $40,000, giving $27,165 taxable." -us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used the flat-tax standard deduction at its un-indexed statutory level of $12,500 rather than the 2026 inflation-adjusted $12,835 and included the $180 of non-Schedule-D capital gains in the base. Those $515 of extra taxable income at 3% account for the entire difference from $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $680 equals 3% of $22,665, which is $40,000 less the correct $12,835 standard deduction less an additional $4,500: it stacked the repealed personal exemption on top of the flat-tax standard deduction that already absorbed it. Removing that phantom exemption leaves $27,165 taxable and $814.95 of tax." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It substituted the federal standard deduction (about $15,000) for Louisiana's own $12,835 standard deduction and then applied the repealed 1.85%/3.5% graduated brackets rather than the 3% flat rate. Louisiana starts from federal AGI but allows only its own standard deduction, so the base is $27,165 taxed entirely at 3%." -us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the correct 3% flat rate but held the standard deduction at the un-indexed $12,500 instead of the 2026 value of $12,835 and taxed $40,180 rather than the $40,000 state base. It then rounded $830.40 to a whole dollar, compounding a base that was already $515 too high." -us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It rebuilt Louisiana's pre-2025 law wholesale: a $2,503 deduction for federal income tax paid, a $4,500 standard deduction plus a $4,500 personal exemption, and the 1.85%/3.5% brackets, every one of which the 2024 flat-tax reform eliminated. It also reduced AGI to $34,791 by subtracting the $5,389 employer insurance premium from wages already net of it; the correct base is $40,000 less $12,835 at 3%." -us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the correct flat 3% rate over a base using the un-indexed $12,500 standard deduction and $40,180 of income. Louisiana's 2026 single standard deduction is the indexed $12,835 and the state base is the $40,000 of wages, so taxable income is $27,165 and the tax $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no substantive computation was submitted. The required answer is 3% of $40,000 less the 2026 indexed Louisiana standard deduction of $12,835, or $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used the flat 3% rate with the un-indexed $12,500 single standard deduction rather than the 2026 inflation-adjusted $12,835, and included the $180 of non-Schedule-D capital gains in the base. The resulting $27,680 of taxable income exceeds the correct $27,165 by $515." -us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It asserted that Louisiana eliminated the individual income tax for 2026, then computed 3% on $35,500 using the repealed $4,500 standard deduction for $1,065, and finally submitted $1,208, a number its own arithmetic never produced. Louisiana's 2026 tax is 3% of $40,000 less the $12,835 indexed standard deduction, or $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied the 3% flat rate to $40,180 less the un-indexed $12,500 standard deduction. The 2026 deduction is the inflation-adjusted $12,835 and the Louisiana base is the $40,000 of wages, so taxable income is $27,165 and the liability $814.95." -us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It subtracted the federal $15,750 standard deduction and then a repealed $4,500 Louisiana personal exemption, and invented 1.75%/3.55% brackets in place of the 3% flat rate, computing $482.52 before submitting an unexplained $573. Louisiana allows only its own $12,835 standard deduction against the $40,000 base at a single 3% rate." -us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $1,324.80 is about 3.3% of the full $40,180 of income, meaning it applied a rate to a base from which no Louisiana standard deduction was subtracted. The 2026 computation subtracts the indexed $12,835 deduction from the $40,000 wage base and applies 3% to the remaining $27,165." +us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which produced $830.40. It then submitted $1,020, which contradicts its own arithmetic. The correct result is 3% × ($40,180 − $12,835) = $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 single standard deduction. It missed that Act 11 indexes the deduction to inflation starting in 2026, which gives $12,835. That overstated taxable income by $335 and tax by $10.05." +us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model claimed Louisiana has no individual income tax. Louisiana taxes individual income at a flat 3% of federal AGI minus the $12,835 standard deduction, which yields $820.35 here." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model got the flat 3% structure right but used the base-year $12,500 standard deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed $830.40 using the unindexed $12,500 deduction. It then subtracted a nonexistent 'personal credit' to reach $632, but Louisiana's post-reform law has no personal exemption credit for this filer. The correct deduction is $12,835 and the tax is $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 deduction and then made an unexplained 'rounding' cut from about $830 to $812. The correct figure is 3% of ($40,180 − $12,835) = $820.35, using the 2026 inflation-indexed deduction." +us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 standard deduction instead of the 2026 indexed $12,835. Taxable income came out $335 too high, giving $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model deducted about $2,783 of federal income tax, a deduction Louisiana repealed after 2021. It also used the unindexed $12,500 standard deduction instead of $12,835. Together these understated taxable income and produced $747 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model computed $830.40 with the unindexed $12,500 deduction. It then submitted $1,080, citing an exemption and credits that would lower tax, not raise it. The correct figure is 3% × $27,345 = $820.35, using the $12,835 indexed deduction." +us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model assumed the $12,500 standard deduction carries into 2026 unchanged. The 2026 inflation-indexed deduction is $12,835, which gives $820.35 instead of $830.40." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model paired the new 3% flat rate with the repealed pre-2025 $4,500 exemption and counted that amount twice, as a standard deduction and a personal exemption. It should have subtracted the single $12,835 standard deduction. That overstated taxable income at $31,180 instead of $27,345." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a nonexistent 4.25% flat rate instead of 3%. It reduced AGI to $34,791 by subtracting ESI premiums that are not in the AGI computation, and used a $4,500 exemption instead of the $12,835 standard deduction." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the 2025 $12,500 single standard deduction. It missed the 2026 inflation adjustment to $12,835, so it overstated tax by $10.05 at $830.40." +us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model subtracted the unindexed $12,500 standard deduction instead of the 2026 indexed $12,835. That produced $830.40 instead of 3% × $27,345 = $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the repealed pre-2025 law: 1.85%/3.5% brackets, a $4,500 exemption, and a federal income tax deduction. It also cut AGI to $34,791 by subtracting ESI premiums. It should have used the flat 3% rate on $40,180 minus the $12,835 standard deduction." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said it applied Louisiana's 'progressive' rates, but Louisiana has had a single 3% rate since 2025. The correct figure is 3% × ($40,180 − $12,835) = $820.35. The $724 answer reflects the obsolete graduated schedule, not the flat-tax computation." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used the repealed 1.85%/3.5% brackets and $4,500 exemption instead of the flat 3% rate with the $12,835 standard deduction. It also started from $34,791 instead of the $40,180 federal AGI, having wrongly subtracted ESI premiums." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied pre-2025 law: the federal income tax deduction, a $4,500 exemption, and 1.85%/3.5% brackets. It also started from a $34,791 AGI net of ESI premiums. For 2026 it should have used the flat 3% rate on $40,180 minus the $12,835 standard deduction." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $1,115 exceeds the correct $820.35 (3% × ($40,180 − $12,835)), which fits using a far smaller deduction than $12,835 or the obsolete graduated rates." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the repealed 1.85%/3.5% brackets with a $4,500 exemption instead of the flat 3% rate and $12,835 standard deduction. It also started from $34,791 instead of the $40,180 AGI, having wrongly subtracted ESI premiums." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the pre-2025 1.85%/3.5% brackets and $4,500 exemption to $40,180. It should have used the 2025-onward flat 3% rate and the 2026 $12,835 standard deduction, so it overstated taxable income and tax." +us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the obsolete $4,500 deduction and 1.85%/3.5% brackets instead of the flat 3% rate on AGI minus the $12,835 standard deduction. That produced $1,042.55 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction for 2026 without Act 11's inflation indexing, which raises it to $12,835. That overstated taxable income by $335 and tax by $10.05." +us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835. That yielded $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model claimed deductions fully offset Louisiana tax. The only deduction is $12,835, which leaves $27,345 taxable at 3% and produces $820.35 of tax." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave no derivation. Its $781 does not match the flat 3% on $40,180 minus the 2026 $12,835 standard deduction, which yields $820.35, so it did not apply the actual 2026 deduction and rate." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. That gave $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 deduction instead of the 2026 indexed $12,835. Taxable income came out $335 too high and tax $10.05 too high." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied the base-year $12,500 single deduction and missed the 2026 inflation indexing to $12,835. That produced $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated Louisiana taxable income at $27,680 instead of $27,345." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 single standard deduction for 2026 instead of $12,835. That yielded $830.40 instead of 3% × $27,345 = $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model assumed a $12,500 standard deduction for 2026 and missed that it is inflation-indexed to $12,835. That overstated tax by $10.05." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model applied the 2025 $12,500 deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model subtracted the unindexed $12,500 standard deduction instead of the 2026 $12,835 amount. That left $27,680 taxable instead of $27,345 and $830.40 of tax instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no computation. The correct figure is the flat 3% on $40,180 minus the $12,835 2026 standard deduction, which yields $820.35. The $680 answer understates that, which fits using an oversized deduction or the obsolete graduated rates." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model subtracted the federal $15,000 standard deduction instead of Louisiana's own $12,835 deduction. It then applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used the $12,500 standard deduction instead of the 2026 inflation-indexed $12,835, then rounded $830.40 to $830. The correct result is $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model applied the base-year $12,500 single deduction and missed the 2026 inflation adjustment to $12,835. That overstated tax at $830.40." +us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model stacked repealed pre-2025 provisions: a federal income tax deduction, separate $4,500 standard deduction and personal exemption, and 1.85%/3.5% brackets. It also started from a $34,791 AGI net of ESI premiums. It should have used the flat 3% on $40,180 minus $12,835." +us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used the unindexed $12,500 standard deduction instead of the 2026 $12,835 and rounded $830.40 to $830. The correct result is $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for this output, so there is nothing to score. The correct figure is 3% × ($40,180 − $12,835) = $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model subtracted the 2025 $12,500 standard deduction instead of the 2026 inflation-indexed $12,835. That produced $830.40 instead of $820.35." +us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used the obsolete $4,500 deduction on wages alone, omitting the $180 capital gain, and computed $1,065. It then submitted an unrelated $1,208. The correct computation is 3% of $40,180 minus the $12,835 standard deduction." +us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied the unindexed $12,500 single deduction instead of the 2026 $12,835. Taxable income came out $335 too high and tax $10.05 too high." +us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model subtracted the federal $15,750 standard deduction plus a repealed $4,500 exemption and applied invented 1.75%/3.55% brackets. It then submitted $573, which contradicts its own $482.52. It should have used the flat 3% on $40,180 minus Louisiana's $12,835 deduction." +us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave no deduction or rate detail. Its $1,324.80 far exceeds 3% × ($40,180 − $12,835) = $820.35, so it applied a wrong rate or omitted most of the $12,835 standard deduction." us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own bracket walk produced $107,277 (ordinary tax $106,932 + $127 qualified-dividend tax + $218 NIIT), but it then submitted $117,285 with the note ""with bracket estimates rounding, value = 117285 adjusted"" — a $10,008 substitution that follows from nothing in its derivation. Underneath that, it used a $32,600 standard deduction instead of the $32,200 MFJ figure for 2026 and denied the $43.28 traditional IRA deduction that PolicyEngine allowed inside its $3,043.28 of above-the-line deductions, and its ordinary tax of $106,932 overstates the $103,865.78 the 2026 MFJ schedule yields on $471,730.56." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Every structural step matched the reference — $499,074 of wages after the $926 pre-tax 401(k), $7,900 interest, $848 dividends, the $3,000 net-capital-loss limit, the $32,200 MFJ standard deduction, 15% on qualified dividends, and $218.42 of NIIT on $5,748 — except that it ruled the $43 traditional IRA contribution nondeductible for a 401(k)-covered filer above the phaseout. PolicyEngine deducts it, making above-the-line deductions $3,043.28 rather than $3,000 and taxable income $472,578.56 rather than $472,622; the $43.28 taxed at the 32% marginal rate is exactly the $13.89 by which its $104,225.30 exceeds $104,211.41." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It applied a $28,700 married-filing-jointly standard deduction rather than the $32,200 in effect for 2026, never separated the $848 of qualified dividends for the 15% preferential rate, and omitted the 3.8% NIIT on $5,748 of net investment income. It then discarded both of its own computed figures ($108,564 and $109,223) and submitted $103,241 as the result of ""careful calculation of all adjustments and phaseouts"" with no derivation connecting to it." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It added $2,250 of Additional Medicare Tax (0.9% on wages above the $250,000 MFJ threshold) into the income-tax total, a Form 8959 payroll levy that this output excludes, reaching $107,040 — and then submitted $110,997 on the strength of ""adjusting standard deduction and brackets to 2026 inflation estimates,"" a further $3,957 that its own arithmetic never produces. Its $32,300 standard deduction also undershoots the $32,200-plus-$43.28-IRA structure the reference uses, and the correct total is ordinary tax $103,865.78 + $127.20 qualified-dividend tax + $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It wrote that the traditional 401(k) reduces wages but then built AGI as $505,748 — $500,000 of gross wages plus $7,900 interest plus $848 dividends less the $3,000 loss limit — never subtracting the $926.16 deferral, and it used a $32,300 standard deduction instead of $32,200 and disallowed the $43.28 IRA deduction, leaving taxable income of $473,448 against the correct $472,578.56. It also stopped at ordinary tax plus qualified-dividend tax and omitted the $218.42 net investment income tax on $5,748." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It set up the computation correctly — roughly $472,200 of taxable income with a top marginal rate of 32% — and then asserted a bracket total of ""about $141,500,"" when walking the 10/12/22/24/32% MFJ schedule it itself listed over $471,350 of ordinary income yields $103,865.78. It also inflated the standard deduction to $32,600 and omitted the $218.42 NIIT, but the $37,000 gap is the unexecuted bracket arithmetic." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"It called the $2,250 Additional Medicare Tax ""an income tax line (Form 8959) included before refundable credits"" and added it to its $104,516 ordinary tax + $127 qualified-dividend tax + $218 NIIT; that 0.9% levy on wages above the $250,000 MFJ threshold belongs to the payroll-tax output, not to federal_income_tax_before_refundable_credits. It compounded this with a $30,000 standard deduction instead of the $32,200 MFJ amount for 2026, which alone added about $704 to its ordinary tax." -us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,period_annualization,False,"It read the $926 traditional 401(k) and $43 traditional IRA figures as weekly contributions, multiplied each by 52, and applied the $31,000 catch-up cap and a $2,236 IRA deduction — stripping $33,236 from AGI when the prompt states all listed amounts are annual totals and the correct above-the-line reduction is $926.16 plus $43.28. It then computed $94,754 on that understated base, excluded the NIIT as ""a separate tax,"" and submitted $121,500 ""accounting for underestimation of brackets,"" abandoning its own figure by $26,746." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"With taxable income of $474,822 it reported ordinary bracket tax of ""≈ $119,769""; the 2026 MFJ schedule (10/12/22/24/32%) on that base produces about $104,900, so its rate walk overstates the tax by roughly $15,000 and drives essentially the whole $16,018 error. It also computed NIIT on $8,748 instead of $5,748, failing to net the $3,000 allowed capital-loss deduction against net investment income, and used a $30,000 standard deduction rather than $32,200." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premium from wages to get taxable wages of $482,866, but the reference's employment income of $499,073.84 reflects only the $926.16 pre-tax 401(k) deferral — the ESI premium input drives health-program modeling and does not reduce taxable wages here. That single exclusion, plus a $30,800 standard deduction instead of $32,200 and NIIT computed on $8,748 rather than $5,748, produced AGI of $488,614 against the correct $504,778.56." -us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It priced 2026 as a TCJA-sunset year, applying a $16,900 standard deduction and $3,240 of PEP-reduced personal exemptions to reach taxable income of $484,682. For 2026 the MFJ standard deduction is $32,200 and personal exemptions remain zero, so taxable income is $472,578.56; the $12,104 of excess base plus the pre-TCJA-style rate walk yielded $126,648.72 of ordinary tax against $103,865.78, and it also computed NIIT on $8,748 rather than the $5,748 net of the $3,000 capital-loss deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It itemized $45,620 of deductions ""including mortgage interest and state/local taxes after the Pease limitation"" when no mortgage interest payment, property tax, or sales tax amount is listed — only a $685,735 loan balance — and the prompt directs unlisted numeric inputs to be treated as 0, leaving the $32,200 standard deduction. It then reported $120,856 of tax on the resulting $459,202 of taxable income, more than the $104,211.41 due on the larger correct base of $472,578.56, because it applied TCJA-sunset rates and a 20% qualified-dividend rate that begins above $600,000 of MFJ taxable income." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It reported AGI of ""approximately $476,323"" after subtracting a $28,926 capital loss it simultaneously described as capped at $3,000, and then produced $139,198 of tax on a base below the correct $504,778.56 AGI. The 2026 MFJ schedule on $472,578.56 of taxable income yields $103,865.78 of ordinary tax plus $127.20 and $218.42; $139,198 corresponds to an average rate near 29%, which the 10/12/22/24/32% schedule cannot reach at this income." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It offered no derivation beyond ""ordinary income tax brackets and net investment income tax,"" and its $123,753 sits $19,542 above the correct $103,865.78 + $127.20 + $218.42. That gap corresponds to running $472,578.56 of taxable income through a pre-TCJA 28/33/35% rate schedule with personal exemptions rather than the 2026 law, which keeps the 10/12/22/24/32% brackets and the $32,200 MFJ standard deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used ""2026 post-TCJA expiration tax rules"": a $15,300 standard deduction and two $5,050 personal exemptions, and it also stripped the $16,208 employer-sponsored insurance premium out of wages to get AGI of $488,614. For 2026 the MFJ standard deduction is $32,200 with no personal exemptions and the ESI premium does not reduce taxable wages, so taxable income is $472,578.56, not $463,214, and ordinary tax is $103,865.78 rather than the $122,573.78 its sunset rate schedule produced." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted that this household's income is ""above the top marginal tax bracket"" — the 2026 MFJ 35% bracket does not begin until $768,700, and $472,578.56 of taxable income tops out at 32% — and counted only the Head's $7,200 of interest, omitting the Spouse's $700. Its $105,744 corresponds to roughly $477,400 of taxable income under the 2026 MFJ schedule, about $4,800 more than the correct base, consistent with not applying the full $32,200 standard deduction and the $3,043.28 of above-the-line deductions." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It listed the right components — $500,000 wages, $7,900 interest, $848 qualified dividends, the $3,000 net-capital-loss deduction, pre-tax contributions, and NIIT — but its $113,421 corresponds to roughly $500,500 of ordinary income run through the 2026 MFJ brackets, which is AGI with no standard deduction removed. Subtracting the $32,200 MFJ standard deduction gives taxable income of $472,578.56 and ordinary tax of $103,865.78, and the $9,210 overage is precisely the tax on the deduction it never took." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It built AGI of $488,614 by excluding the $16,208 employer-sponsored insurance premium from wages, then reported $117,546.80 of regular tax on that base when the 2026 MFJ schedule on the resulting $456,414 of taxable income yields about $98,700 — an $18,800 overstatement from applying TCJA-sunset rates and exemptions. The correct build keeps wages at $499,073.84 after only the $926.16 401(k) deferral, giving $103,865.78 of ordinary tax plus $127.20 and the $218.42 NIIT it did get right." -us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It stated it applied ""standard deduction and personal exemptions under TCJA sunset"" and reduced wages by both the traditional 401(k) and the ESI premiums. For 2026 the MFJ standard deduction is $32,200 with no personal exemptions, and the $16,208 ESI premium does not reduce taxable wages — the reference's employment income of $499,073.84 nets only the $926.16 deferral — so its $117,769 reflects a sunset rate-and-deduction regime that is not in force." -us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It computed the whole return under expired TCJA law: a $15,900 MFJ standard deduction, $10,200 of personal exemptions phased down 92%, and the pre-TCJA 10/15/25/28/33/35% rate schedule, producing $131,672.30 of ordinary tax on $487,258. For 2026 the rates are 10/12/22/24/32/35/37%, the MFJ standard deduction is $32,200, and personal exemptions are zero, which puts taxable income at $472,578.56 and ordinary tax at $103,865.78." -us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It said it deducted the $926 traditional 401(k) but then carried AGI of ""roughly $505,800"" — the figure you get without that deferral and without the $43.28 IRA deduction, against the correct $504,778.56 — and paired it with a ~$32,400 standard deduction rather than $32,200. It also omitted the $218.42 net investment income tax entirely and treated the qualified-dividend preference as a $144 reduction rather than computing $127.20 of tax on the $848." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It submitted $50,504 with no bracket walk, an effective rate of 10.7% on $472,578.56 of taxable income — a household whose last dollars sit in the 32% MFJ bracket. Walking the 2026 MFJ schedule over $471,730.56 of ordinary income gives $103,865.78, to which $127.20 of qualified-dividend tax and $218.42 of NIIT are added; its figure is under half of that and consistent with applying a single low average rate instead of the graduated schedule." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reported $0, reasoning that ""large retirement contributions and health premiums"" left no net liability, when the listed contributions total $926 of traditional 401(k) and $43 of traditional IRA and the $16,208 ESI premium does not reduce taxable wages at all. AGI is $504,778.56 and taxable income $472,578.56 after the $32,200 standard deduction, producing $104,211.41 of tax with no nonrefundable credits available at this income." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It stopped at regular tax — $103,879.68 of ordinary bracket tax on $471,774 plus $127.20 on the qualified dividend — and never added the 3.8% net investment income tax, which applies here because MAGI of $504,778.56 exceeds the $250,000 MFJ threshold and net investment income is $5,748 (interest $7,900 + dividends $848 less the $3,000 capital-loss deduction), giving $218.42. It also denied the $43.28 traditional IRA deduction that PolicyEngine allows, putting taxable income at $472,622 instead of $472,578.56." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It taxed the $848 qualified dividend at 20% ($169.60) rather than 15% ($127.20) — the 20% MFJ rate does not begin until taxable income exceeds roughly $613,700 for 2026, well above this household's $472,578.56 — and it omitted the $218.42 NIIT on $5,748 of net investment income. Its $104,049.28 is exactly $103,879.68 of ordinary tax on the $472,622 base it used (denying the $43.28 IRA deduction) plus that 20% dividend tax." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It computed ""about $332 of net investment income tax,"" which is 3.8% of $8,748 — the raw interest plus dividends — without applying the $3,000 allowed net-capital-loss deduction against net investment income, so the NIIT base is $5,748 and the tax $218.42, not $332.42. Combined with denying the $43.28 traditional IRA deduction, its $104,339 overshoots by $127.59." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It described the correct build — $500,000 wages, $7,900 interest, $848 qualified dividends, the $3,000 capital-loss deduction, the 2026 joint standard deduction — and then reported $57,891, an effective rate of 12.25% on $472,578.56 of taxable income. The 2026 MFJ bracket walk over $471,730.56 of ordinary income yields $103,865.78; its number corresponds to about $303,000 of taxable income, roughly $170,000 short of the base its own description implies." -us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its only error was treating the $16,208 employer-sponsored insurance premium as pre-tax and removing it from wages, giving AGI of $488,614 instead of $504,778.56; the reference's employment income of $499,073.84 nets only the $926.16 traditional 401(k) deferral, since the ESI premium input feeds health-program modeling rather than the wage base. Everything else — the $32,200 standard deduction, the $218.42 NIIT, no nonrefundable credits — was right, and $16,208 taxed at the 32% marginal rate is the $5,172.67 shortfall." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no bracket arithmetic and submitted a round $140,000, an average rate of 29.6% on $472,578.56 of taxable income when the household's top marginal rate under the 2026 MFJ schedule is 32% and its average rate is 22.0%. Walking 10/12/22/24/32% over $471,730.56 of ordinary income gives $103,865.78, plus $127.20 of qualified-dividend tax and $218.42 of NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It combined two deduction-regime errors: subtracting the $16,208 ESI premium from wages to reach AGI of $488,614, and applying ""projected 2026 post-TCJA parameters"" — a $16,600 standard deduction with phased-out personal exemptions and pre-TCJA rates. For 2026 the MFJ standard deduction is $32,200 with no personal exemptions and taxable wages are $499,073.84, giving taxable income of $472,578.56 and ordinary tax of $103,865.78; it also computed NIIT on $8,748 rather than the $5,748 net of the capital-loss deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $30,750 married-filing-jointly standard deduction for 2026 instead of the $32,200 actually in effect, and left AGI at $504,822 by treating the $43 traditional IRA contribution as nondeductible rather than including it in the $3,043.28 of above-the-line deductions. Those two together put taxable income at $474,072 instead of $472,578.56, and $1,493 of extra base at the 32% marginal rate accounts for its $544.59 overage." -us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It removed the $16,208 employer-sponsored insurance premium from wages when computing ""income tax wages,"" giving AGI of $488,571 rather than $504,778.56 — the ESI premium is not an exclusion from the taxable wage base here, which nets only the $926.16 401(k) deferral. It also used a $30,750 standard deduction instead of $32,200 and computed $332 of NIIT on $8,748 rather than $218.42 on the $5,748 net of the $3,000 capital-loss deduction." -us,scenario_052,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It declared the $43 traditional IRA contribution nondeductible at this income, so its above-the-line total was $3,000 rather than the $3,043.28 PolicyEngine allows, leaving taxable income of $472,622 instead of $472,578.56. Its ordinary tax of ""about $104,035"" also runs about $155 above the $103,879.68 the 2026 MFJ schedule produces on that base; the NIIT of $218 on $5,748 and the 15% dividend treatment were correct." -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so the key was absent from its submitted outputs object. There is no substantive tax reasoning to fault — the answer simply never arrived." -us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It invented an overtime-premium deduction, reasoning that the 60-hour usual weekly schedule ""implies 88,400 of overtime premium already included in wages"" and claiming a $4,517.80 phased-down qualified-overtime deduction, when no overtime premium amount is listed and the prompt directs unlisted numeric inputs to be treated as 0. That phantom deduction plus denying the $43.28 IRA deduction cut taxable income to $468,104.20 against the correct $472,578.56, and it also omitted the $218.42 NIIT on $5,748 of net investment income." -us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Its explanation names three mutually inconsistent standard deductions ($16,100, then $30,800) and reports ""AGI approximately $504,748 (after standard deduction $16,100)"" with taxable income of $488,648 — a base that yields roughly $109,000 under the 2026 MFJ schedule, not the $104,050 submitted. The correct build is a $32,200 standard deduction on AGI of $504,778.56 giving taxable income of $472,578.56, and it omitted the $218.42 NIIT on $5,748 of net investment income." -us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It reproduced the reference's ordinary tax exactly ($103,866 on $471,731 after the $32,200 standard deduction and $3,043 of above-the-line deductions) but stated the ""$848 qualified dividend taxed at 0%"" — the 0% MFJ capital-gain bracket ends under $100,000 of taxable income, so at $472,578.56 the rate is 15% and the tax $127.20. It also omitted the 3.8% NIIT of $218.42 on $5,748 of net investment income; those two items are the entire $345.41 shortfall." -us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It deducted the full $29,922 of combined short- and long-term capital losses against income rather than the $3,000 annual limit, dropping AGI to $478,826, then concluded ""Total tax before AMT = $85,184.10"" with ""AMT = $0"" — and submitted $165,473.58, a figure neither its regular tax nor its $86,491.08 tentative minimum tax supports. It further treated the $848 qualified dividend as taxed at 0% when $472,578.56 of taxable income puts it at 15%, and omitted the $218.42 NIIT." -us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It computed ""$76,256 + 35% of $218,948,"" placing the start of the 35% MFJ bracket near $256,800; for 2026 the 35% bracket begins at $768,700 and $472,578.56 of taxable income tops out in the 32% bracket, so the correct ordinary tax is $103,865.78 rather than $152,887.80. It also left AGI at $505,748 by never subtracting the $926.16 pre-tax 401(k) deferral and used a $30,000 standard deduction instead of $32,200." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own work produced about $107,277, using a $32,600 standard deduction instead of $32,200 and ordinary tax of about $106,932 instead of about $103,866 on the 2026 brackets. It then submitted $117,285, a figure about $10,000 above its own calculation with no derivation behind it." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: 2026 brackets, the $32,200 standard deduction, $127.20 on qualified dividends and $218.42 NIIT. The one difference is that it disallowed the $43 traditional IRA contribution under the active-participant phase-out, as the exclusion's corrected value does under 26 U.S.C. 219(g), while the frozen reference subtracts $43.28 above the line (AGI $504,778.56) because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That extra $43 of income taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It reached the frozen reference's AGI ($504,779), which takes the $43 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), but used a $28,700 standard deduction instead of the 2026 OBBBA $32,200 and never ran the 2026 brackets. Its final $103,241 is an unsupported figure. The frozen reference's path is ordinary tax of $103,865.78 on $471,730.56, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, without the IRA deduction, is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax, which is a payroll tax and not part of income tax before refundable credits. It also used outdated bracket thresholds (24% ending at $400,000 instead of $403,550) and a $32,300 standard deduction. It then inflated its own $107,040 to $110,997 with no calculation to support the increase." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its guessed bracket thresholds (24% ending at $401,950 instead of $403,550, with lower thresholds throughout) overstated ordinary tax by about $378. It also left the $926 401(k) deferral in AGI, used a $32,300 standard deduction, and omitted the $218.42 NIIT on $5,748 of net investment income." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its bracket schedule applied to about $471,350 of ordinary income works out to roughly $104,000, but it reported $141,500, an arithmetic breakdown of about $37,000. It also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $2,250 Additional Medicare Tax (Form 8959) to income tax, although it is a payroll tax outside this output. It also used a $30,000 standard deduction instead of the 2026 OBBBA $32,200 and its own inflated bracket guesses instead of the 2026 schedule." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the annual $926 401(k) and $43 IRA contributions as weekly amounts, multiplied them by 52 and capped the 401(k) at $31,000, which understated AGI by about $32,000. It then replaced its own $94,754 with an arbitrary $121,500. It also used 2025 brackets and left out NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: the $32,200 standard deduction, 2026 brackets, $127 on qualified dividends and $218 NIIT. The one difference is that it treated the $43 traditional IRA contribution as phased out, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That $43 taxed at 32% is the $14 gap to the frozen reference, and its $104,225 is $0.26 below the corrected value, $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It reported ordinary tax of $119,769 on about $474k of taxable income, about $15,000 more than the 2026 brackets produce ($103,866 on $471,731 of ordinary income). It also used a $30,000 standard deduction instead of $32,200 and computed NIIT on $8,748 without subtracting the $3,000 allowable capital loss (the correct base is $5,748)." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premiums from wages. The reference reduces wages only by the 401(k) deferral, so employment income is $499,073.84. It also used a $30,800 standard deduction instead of $32,200 and computed NIIT on $8,748 instead of the $5,748 base net of the capital loss." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the expired pre-TCJA system: a $16,900 standard deduction, personal exemptions and higher pre-TCJA rates. The OBBBA made the TCJA brackets and the larger standard deduction permanent, so 2026 uses $32,200 with no exemptions. It also used $8,748 instead of $5,748 as the NIIT base." +us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA had expired, using a $16,700 standard deduction plus $10,600 of personal exemptions and pre-TCJA rates. Under the OBBBA, 2026 uses the $32,200 standard deduction, no exemptions and the permanent 10–37% brackets. It also used $8,748 instead of $5,748 as the NIIT base." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rates and made up $45,620 of itemized deductions for mortgage interest and SALT that the facts do not include (no interest paid is listed, and Texas has no income tax), then applied a Pease limitation. The OBBBA 2026 law uses the $32,200 standard deduction and the permanent TCJA brackets." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its stated AGI of $476,323 conflicts with the $3,000 capital-loss limit, which gives AGI of about $504,779. Its $139,198 is about $35,000 above what the 2026 MFJ brackets produce on $472,578.56 of taxable income (the reference is $104,211.41 including qualified-dividend tax and NIIT)." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. The $123,753 is about $19,500 above the frozen reference's $104,211.41 and the exclusion's corrected value, $104,225.26, a gap that fits the pre-TCJA sunset rate schedule (33%/35% brackets and a small standard deduction) rather than the permanent 2026 OBBBA brackets and the $32,200 standard deduction." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a TCJA sunset that did not happen, using a $15,300 standard deduction, $10,100 of personal exemptions and pre-TCJA rates. It also subtracted the $16,208 ESI premiums from wages. The correct computation uses $499,073.84 of wages, the $32,200 OBBBA standard deduction and the 2026 10–37% brackets." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It counted only the head's $7,200 of interest and dropped the spouse's $700. It gave no bracket calculation, and its $105,744 is $1,533 above the frozen reference's $104,211.41 ($1,519 above the exclusion's corrected value, $104,225.26), a gap that reflects bracket thresholds below the 2026 schedule (the 32% rate starts at $403,550)." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working, and its $113,421 is about $9,200 too high. The frozen reference's path is $472,578.56 of taxable income taxed on the 2026 MFJ brackets for $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26. The overstatement comes from using a rate schedule heavier than the 2026 OBBBA brackets." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums, giving AGI of $488,614, then taxed that income at pre-TCJA sunset rates to reach about $117.5k of regular tax. Under the 2026 OBBBA brackets and the $32,200 standard deduction, regular tax on the frozen reference's $472,578.56 of taxable income is $103,993; that taxable income takes the $43 IRA deduction, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g))." +us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly applied the TCJA sunset (personal exemptions, a smaller standard deduction and pre-TCJA brackets) and subtracted ESI premiums from wages. The OBBBA made the TCJA brackets and the higher standard deduction ($32,200 MFJ in 2026) permanent." +us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used the expired pre-TCJA structure: a $15,900 standard deduction, phased-out personal exemptions and the 10/15/25/28/33/35% brackets. The OBBBA kept the TCJA brackets and set the 2026 MFJ standard deduction at $32,200. It also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It put AGI at about $505,800, about $1,000 too high, used a $32,400 standard deduction and estimated ordinary tax at about $105,200 instead of $103,866 on the 2026 brackets. It then left out the $218.42 NIIT and rounded to $105,000." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no bracket calculation, and its $50,504 is about half the correct liability. On $472,578.56 of taxable income, the 2026 MFJ brackets run into the 32% rate and give $103,865.78 of ordinary tax, plus $127.20 on qualified dividends and $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It said retirement contributions and health premiums wiped out the liability, but the traditional 401(k) deferral and IRA contribution total only about $969 against $500,000 of wages. Taxable income is still $472,578.56 in the frozen reference, whose tax is $104,211.41; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It reported only regular tax ($103,879.68 + $127.20) and left out the $218.42 net investment income tax (3.8% of $5,748 because MAGI is above $250,000), which is part of income tax before refundable credits. It also skipped the $43 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $218.42 NIIT and taxed the $848 of qualified dividends at 20% ($169.60) instead of 15%, even though taxable income is below the 2026 MFJ 20% threshold of $613,700. It also skipped the $43 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It computed NIIT as 3.8% of $8,748 in interest and dividends ($332) without subtracting the $3,000 deductible net capital loss. The correct base is $5,748, for $218.42. It also skipped the $43 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It showed no bracket arithmetic, and its $57,891 is about $46,000 too low. The 2026 MFJ brackets on $471,730.56 of ordinary income give $103,865.78, and it also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 employer-sponsored insurance premiums from wages, giving AGI of $488,614. The reference reduces wages only by the 401(k) deferral ($499,073.84), so AGI is $504,778.56 and taxable income is $472,578.56. That $16,208 at 32% explains almost the whole $5,173 shortfall." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Every component matches the reference: the standard deduction, 2026 brackets, 15% on qualified dividends and $218.42 NIIT. The one difference is that it did not deduct the $43 traditional IRA contribution, which the exclusion's corrected value also disallows under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That amount taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It stopped at regular tax ($103,879.68 + $127.20 on qualified dividends) and left out the $218.42 NIIT owed on $5,748 of net investment income because MAGI is above $250,000. It also skipped the $43 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $16,208 employer-sponsored insurance premiums as a pretax reduction of wages, giving AGI of $488,614. The reference reduces wages only by the 401(k) deferral, so AGI is $504,778.56. The lost $16,208 at 32% explains the $5,173 shortfall." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $140,000 with no bracket calculation, about $35,800 too high. The 2026 MFJ brackets on $472,578.56 of taxable income give $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It projected post-TCJA sunset parameters ($16,600 standard deduction, personal exemptions, pre-TCJA rates) where the OBBBA 2026 law applies. It also subtracted the $16,208 ESI premiums from wages and computed NIIT on $8,748 instead of $5,748." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 standard deduction instead of the 2026 OBBBA MFJ amount of $32,200, which overstated taxable income by about $1,450 (about $464 of tax at 32%). It also skipped the $43 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA individual provisions had expired and used a $16,950 standard deduction, personal exemptions and pre-TCJA rates. The OBBBA made the TCJA brackets permanent and set the 2026 MFJ standard deduction at $32,200, so the tax is overstated by about $22,000." +us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $16,208 ESI premiums from wages (the reference reduces wages only by the 401(k) deferral). It also used a $30,750 standard deduction instead of $32,200, used projected bracket thresholds instead of the 2026 schedule, and computed NIIT on $8,748 instead of $5,748." +us,scenario_052,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It put ordinary tax on $471,774 at $104,035, about $155 more than the 2026 schedule gives (10% to $24,800, 12% to $100,800, 22% to $211,400, 24% to $403,550, then 32%, for $103,880). It also skipped the $43.28 IRA deduction that the reference subtracts." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or reasoning for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"From the 60-hour schedule it claimed a $4,517.80 OBBBA qualified-overtime deduction, but the reference allows no overtime deduction for this $500,000 highly compensated earner: taxable income is AGI minus only the $32,200 standard deduction. It also left out the $218.42 NIIT. It skipped the $43 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference takes it." +us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Its reasoning cites contradictory standard deductions ($16,100 and $30,800) and never shows a bracket calculation. It also leaves out the $218.42 NIIT, so its $104,050 is a rough estimate rather than the $103,865.78 + $127.20 + $218.42 derivation." +us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI and ordinary tax match the reference exactly ($103,865.78), but it taxed the $848 of qualified dividends at 0%. The 0% rate ends at $98,900 of 2026 MFJ taxable income, so 15% ($127.20) applies. It also left out the $218.42 NIIT." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $29,922 capital loss instead of the $3,000 limit, applied the 32% rate to a made-up $182,326 slice, and taxed made-up LTCG and qualified dividends at 0%. It then submitted $165,473.58, which does not follow from its own $85,184.10 result." +us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 35% rate to $218,948 using a made-up bracket structure, although 2026 MFJ income up to $512,450 is taxed at no more than 32%. It also left the $926 401(k) deferral in income, used a $30,000 standard deduction instead of $32,200, and treated the qualified-dividend rate differential as a credit." us,scenario_052,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. Medicare's standard age threshold is 65, and the prompt supplies no disability, end-stage renal disease, or other alternative eligibility basis, so the head is not eligible." -us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the reference figure exactly in its own reasoning — $11,439 Social Security on the $184,500 2026 wage base, $7,250 Medicare on the full $500,000, and $2,250 Additional Medicare Tax on the $250,000 excess over the MFJ threshold, summing to $20,939 — and then submitted $12,820.40, a number that matches neither that sum nor its ESI-reduced alternative of $20,558.11. The submitted value abandons every computation the model performed; it also wrongly credited PolicyEngine with excluding the $16,208 employer-sponsored premiums from FICA wages, when PolicyEngine taxes the full $500,000 of gross wages." -us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500 (understating Social Security tax by $985.80) and applied the $200,000 single withholding threshold rather than the $250,000 MFJ Additional Medicare Tax threshold, producing $2,700 instead of $2,250. It then discarded its own $20,403.20 subtotal and submitted $38,335.50, a figure consistent with applying 6.2% to all $500,000 with no wage-base cap at all." -us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"Every component except the wage base is right: Medicare $7,250 on the full $500,000 and Additional Medicare Tax $2,250 at the correct $250,000 MFJ threshold. The model applied the 2025 Social Security wage base of $176,100 rather than the 2026 base of $184,500, yielding $10,918.20 instead of $11,439 — the entire $520.80 shortfall." -us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model got the 2026 wage base right ($184,500 × 6.2% = $11,439) and Medicare right ($7,250 on the full $500,000), but applied the $200,000 employer-withholding threshold for a single filer to the Additional Medicare Tax instead of the $250,000 MFJ filing threshold, taxing $300,000 of excess wages at 0.9% for $2,700 rather than $2,250. That single threshold substitution is the entire $450 overstatement." -us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model computed all three components correctly — $11,439 Social Security on the $184,500 2026 base, $7,250 Medicare, $2,250 Additional Medicare Tax at the $250,000 MFJ threshold — and stated the total as ""about $20,939,"" then submitted $21,118. It corrupted a correct total in transcription, calling the change a rounding when $21,118 is $179 above the sum it had just computed." -us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model handled Medicare ($7,250 on full wages) and the Additional Medicare Tax ($2,250 at the $250,000 MFJ threshold) correctly but cycled through wage-base guesses ($176,100, $180,000, $183,000) and settled on the 2025 base of $176,100 for 2026, giving $10,918.20 of Social Security tax instead of $11,439 on the 2026 base of $184,500. The $520.80 wage-base shortfall is the whole error." -us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model first used the correct $184,500 base, then overrode it with a $176,100 estimate and applied the $200,000 single threshold instead of the $250,000 MFJ threshold for the Additional Medicare Tax, reaching $20,868.20; it then submitted $16,856.10, which follows from none of its stated components. The submitted number discards a subtotal that was itself already wrong by two parameters." -us,scenario_052,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Medicare ($7,250 on the full $500,000) and the Additional Medicare Tax ($2,250 on the excess over the $250,000 MFJ threshold) are both correct. The model guessed a round $180,000 Social Security wage base for 2026 instead of the actual $184,500, producing $11,160 rather than $11,439 — a $279 shortfall that is the entire difference." -us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $16,208 employer-sponsored insurance premiums from gross wages to get a $483,792 FICA base, which cut Medicare tax to $7,014.98 and the Additional Medicare Tax to $2,104.13; PolicyEngine applies both to the full $500,000 of gross wages, giving $7,250 and $2,250. It compounded that with a $181,000 wage-base guess instead of the 2026 base of $184,500, understating Social Security tax by $217." -us,scenario_052,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Medicare on the full $500,000 ($7,250) and the Additional Medicare Tax at the $250,000 MFJ threshold ($2,250) are both correct. The model used a $181,800 Social Security wage base instead of the 2026 figure of $184,500, giving $11,271.60 rather than $11,439 — the entire $167.40 gap." -us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied the correct 1.45% Medicare rate to the full $500,000 and the correct 0.9% Additional Medicare Tax to wages above the $250,000 MFJ threshold, but capped Social Security wages at a projected $171,300 instead of the 2026 wage base of $184,500. That understated the Social Security component by $818.40 ($10,620.60 versus $11,439)." -us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $9,932.50 Social Security figure implies a $160,200 wage base — the 2023 cap — instead of the 2026 base of $184,500, and its $18,958.50 Additional Medicare Tax implies roughly a 7.6% rate on the $250,000 of excess wages rather than the statutory 0.9%, which yields $2,250. The 0.9% rate error alone overstates the total by $16,708.50." -us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The submitted $20,279 decomposes as $11,160 Social Security (6.2% on a guessed $180,000 wage base rather than the 2026 base of $184,500), $7,014.98 Medicare, and $2,104.13 Additional Medicare Tax — the latter two computed on $483,792 after subtracting the $16,208 employer-sponsored insurance premiums. PolicyEngine applies Medicare and the Additional Medicare Tax to the full $500,000 of gross wages, giving $7,250 and $2,250." -us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model removed the $16,208 employer-sponsored insurance premiums from the FICA base, computing Medicare and Additional Medicare Tax on $483,792 ($7,014.98 and $2,104.13) instead of on the full $500,000 of gross wages ($7,250 and $2,250). It also capped Social Security at an estimated $172,200 rather than the 2026 wage base of $184,500, losing a further $762.60." -us,scenario_052,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"The model applied the $200,000 single-filer threshold to the Additional Medicare Tax instead of the $250,000 MFJ threshold, and its $34,980 total leaves an implied Social Security component of about $25,030 — 6.2% on roughly $403,700 of wages — so the wage-base cap it claimed to apply was never actually imposed. Capping at the 2026 base of $184,500 gives $11,439, and the correct components sum to $20,939." -us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model named the right structure — capped Social Security, 1.45% Medicare, and 0.9% Additional Medicare Tax at the $250,000 MFJ threshold — but its $20,353 total leaves an implied Social Security component near $10,853, corresponding to a wage base around $175,000 rather than the 2026 base of $184,500. Capping at $184,500 yields $11,439 and a total of $20,939." -us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model excluded the $16,208 employer-sponsored insurance premiums to build a $483,792 FICA base, so Medicare came to $7,014.98 and the Additional Medicare Tax to $2,104.13 instead of $7,250 and $2,250 on the full $500,000 of gross wages. It also used a $181,800 wage base, giving $11,271.60 of Social Security tax rather than $11,439 on the 2026 base of $184,500." -us,scenario_052,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The $20,387 total is consistent with Medicare and Additional Medicare Tax computed on $483,792 — gross wages net of the $16,208 employer-sponsored insurance premiums — plus Social Security capped near $181,700 rather than the 2026 wage base of $184,500. PolicyEngine taxes the full $500,000 for Medicare ($7,250) and the Additional Medicare Tax ($2,250) and caps Social Security at $184,500 ($11,439)." -us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model subtracted the $926 traditional 401(k) contribution from the wage base, but elective deferrals to a 401(k) remain fully subject to Social Security and Medicare tax, so Medicare wages are $500,000, not $499,074 — that costs $13.43 of Medicare and $8.33 of Additional Medicare Tax. It also used a $183,000 estimated wage base instead of the 2026 figure of $184,500, understating Social Security tax by $93." -us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model asserted that no Additional Medicare Tax applies ""because the total wages are below the joint threshold,"" when the head's $500,000 of wages exceeds the $250,000 MFJ threshold by $250,000 and triggers $2,250 of tax. Its $38,897 total also exceeds the uncapped 7.65% of $500,000, so the wage-base cap it claimed to apply to Social Security was never imposed; capping at $184,500 gives $11,439." -us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model declared the household's wage base untaxed for lack of ""payroll-tax applicability in the provided facts,"" but $500,000 of gross wages and salaries is by definition FICA-covered employment income. Applying 6.2% to the $184,500 2026 wage base, 1.45% to all $500,000, and 0.9% to the $250,000 above the MFJ threshold gives $20,939, not $0." -us,scenario_052,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model used the correct 2026 wage base ($184,500 × 6.2% = $11,439) but treated the $16,208 employer-plan premiums as pretax and computed Medicare and the Additional Medicare Tax on $483,792, giving $7,014.98 and $2,104.13. PolicyEngine applies both to the full $500,000 of gross wages, producing $7,250 and $2,250; that single base reduction accounts for the entire $380.89 shortfall." -us,scenario_052,payroll_tax,grok-4.3,llm_error,other,False,"The model used a stale ""~$168k"" wage base (the 2024 cap) rather than the 2026 base of $184,500, and then submitted $13,500 even though the three components it named — capped Social Security, 1.45% Medicare on $500,000, and 0.9% Additional Medicare Tax — sum to roughly $20,400 on its own stale base. The answer is a hand-waved approximation contradicted by its own listed components." -us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model correctly capped Social Security at the 2026 wage base of $184,500 for $11,439, but subtracted the $16,208 employer-sponsored insurance premiums to get FICA wages of $483,792, computing Medicare as $7,015 and the Additional Medicare Tax as $2,104. Medicare and the Additional Medicare Tax apply to the full $500,000 of gross wages, giving $7,250 and $2,250." -us,scenario_052,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"The model reduced FICA wages to $499,074 by deducting the $926 traditional 401(k) deferral, but elective 401(k) deferrals are excluded from income tax only, never from Social Security or Medicare wages. That understated Medicare by $13 ($7,237 versus $7,250) and the Additional Medicare Tax by $9 ($2,241 versus $2,250) on an otherwise correct calculation." -us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $16,208 employer-sponsored insurance premiums from gross wages, computing Medicare on $483,792 ($7,015) and the Additional Medicare Tax on $233,792 of excess ($2,104) instead of on the full $500,000 ($7,250 and $2,250). It also projected a $183,144 wage base rather than the 2026 figure of $184,500, giving $11,355 of Social Security tax instead of $11,439." -us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive computation was submitted for scoring. The required derivation is $11,439 of Social Security tax on the $184,500 2026 wage base, $7,250 of Medicare tax on $500,000, and $2,250 of Additional Medicare Tax on the $250,000 above the MFJ threshold, totaling $20,939." -us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model listed all three components correctly — $11,439, $7,250, and $2,250 — but summed them to ""≈$30,939"" (a $10,000 arithmetic slip) and then submitted $11,573, which equals none of its components or their sum. Its own correct components add to $20,939." -us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model recomputed $20,418.20 three times using the 2025 wage base of $176,100 instead of the 2026 base of $184,500, then submitted $32,846 — a figure unconnected to any of its arithmetic. Correcting the wage base to $184,500 raises Social Security tax from $10,918.20 to $11,439 for a total of $20,939." -us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model double-counted the Additional Medicare Tax: its $20,418.20 subtotal already contained the $2,250 of 0.9% tax on wages above the $250,000 MFJ threshold, and it then added the same $2,250 again to reach $22,668.20. It also used the 2025 wage base of $176,100 rather than the 2026 base of $184,500, understating Social Security tax by $520.80." +us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"It correctly computed $11,439 + $7,250 + $2,250 = $20,939 on full wages and also worked out an ESI-excluded alternative of $20,558.11. It then submitted $12,820.40, which matches neither calculation or any combination of its components. The submitted value is an output inconsistency, not a tax rule." +us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also applied the $200,000 single Additional Medicare threshold instead of the $250,000 MFJ threshold, giving $20,403.20. It then submitted an unsupported $38,335.50 that contradicts even that figure." +us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 Social Security wage base of $176,100 instead of the 2026 base of $184,500. That understates employee Social Security tax by $520.80 ($10,918.20 vs $11,439). Its Medicare and MFJ Additional Medicare components were correct." +us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied the Additional Medicare Tax at the $200,000 single-filer threshold instead of the $250,000 threshold for married filing jointly. That charges 0.9% on $300,000 ($2,700) instead of $250,000 ($2,250), overstating the total by $450." +us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"Its components ($11,439 SS, $7,250 Medicare, $2,250 Additional Medicare) sum exactly to $20,939. It then 'rounded' to an unsupported $21,118, so the submitted value contradicts its own correct arithmetic." +us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After cycling through several guesses, it settled on the 2025 wage base of $176,100 for 2026 instead of the actual 2026 base of $184,500. That understates Social Security tax by $520.80, producing $20,418.20." +us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It switched from the correct $184,500 wage base to $176,100 and applied the $200,000 single threshold instead of the $250,000 MFJ threshold, computing $20,868.20. It then submitted $16,856.10, which matches none of its own component sums." +us,scenario_052,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a rounded $180,000 Social Security wage base instead of the 2026 base of $184,500. That understates employee Social Security tax by $279 ($11,160 vs $11,439)." +us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $16,208 ESI premiums from gross wages as if they were paid through a pre-tax cafeteria plan, although no such plan is stated. The Medicare and Additional Medicare base should be the full $500,000. It also used a $181,000 wage base instead of $184,500." +us,scenario_052,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $181,800 Social Security wage base instead of the 2026 base of $184,500. That understates Social Security tax by $167.40 ($11,271.60 vs $11,439), while its Medicare and MFJ Additional Medicare amounts were correct." +us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a projected $171,300 Social Security wage base instead of the 2026 base of $184,500. That gives $10,620.60 instead of $11,439 for employee Social Security tax." +us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It computed $18,958.50 of Additional Medicare Tax, when 0.9% × ($500,000 − $250,000) is only $2,250, a rate error of roughly eightfold. It also capped Social Security at an outdated base ($9,932.50 instead of $11,439)." +us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Its $20,279 matches a $180,000 wage base ($11,160) plus Medicare and Additional Medicare computed on $483,792, meaning it treated the ESI premiums as pre-tax. The correct figures are the $184,500 base and the full $500,000 of Medicare wages." +us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages to $483,792 by treating the $16,208 ESI premiums as pre-tax, understating Medicare plus Additional Medicare by $380.89. It also used a $172,200 wage base instead of the 2026 base of $184,500." +us,scenario_052,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It applied the $200,000 single Additional Medicare threshold instead of the $250,000 MFJ threshold. Its $34,980 total also far exceeds the correct $11,439 + $7,250 + $2,250 = $20,939, so it overstated Social Security well beyond the $184,500 cap." +us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Its $20,353 is $586 below the correct $20,939. That gap matches treating the ESI premiums as pre-tax (−$380.89 in Medicare and Additional Medicare) combined with a Social Security wage base of about $181,200 instead of $184,500." +us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It computed Medicare and Additional Medicare on $483,792 by treating the $16,208 ESI premiums as pre-tax instead of using the full $500,000. It also used a $181,800 wage base ($11,271.60) instead of $184,500 ($11,439)." +us,scenario_052,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"Its $20,387 closely matches $20,390.71, the total from Medicare taxes on ESI-reduced wages of $483,792 plus Social Security on a base of about $181,800. It should have used the full $500,000 of Medicare wages and the $184,500 base." +us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the $926 traditional 401(k) deferral from Medicare wages, but elective 401(k) deferrals remain FICA wages under IRC §3121(v)(1)(A). It also used a $183,000 wage base instead of $184,500." +us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said no Additional Medicare Tax applied because wages were below the joint threshold, but $500,000 far exceeds the $250,000 MFJ threshold. Its $38,897 total also implies Social Security was taxed well beyond the $184,500 cap." +us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It returned $0, ignoring that the Head's $500,000 of W-2 wages is subject to employee Social Security ($11,439), Medicare ($7,250), and Additional Medicare Tax ($2,250)." +us,scenario_052,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $16,208 ESI premiums as pre-tax cafeteria-plan deductions and cut Medicare wages to $483,792, although the facts state no such plan. The correct base is the full $500,000 ($7,250 Medicare and $2,250 Additional Medicare), not $7,014.98 and $2,104.13." +us,scenario_052,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It excluded the $16,208 ESI premiums from Medicare wages ($483,792) when the full $500,000 of gross wages is the payroll tax base. That understates Medicare plus Additional Medicare by $380.89." +us,scenario_052,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It used a stale wage base of about $168,000 and then reported a total of about $13,500, which drops most of the $7,250 Medicare and $2,250 Additional Medicare it listed. The correct sum is $11,439 + $7,250 + $2,250 = $20,939." +us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"It correctly used the $184,500 wage base but treated the ESI premiums as pre-tax, computing Medicare ($7,015) and Additional Medicare ($2,104) on $483,792. Those taxes apply to the full $500,000 of wages." +us,scenario_052,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"It reduced FICA wages to $499,074 by subtracting the traditional 401(k) deferral, but elective deferrals are still subject to Medicare tax and Additional Medicare Tax. That understates those two components by $22." +us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $16,208 ESI premiums from wages to get $483,792 of FICA wages instead of using the full $500,000. It also used a projected $183,144 wage base instead of the 2026 base of $184,500." +us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no answer to score." +us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"It listed the correct components ($11,439 + $7,250 + $2,250) but added them wrongly to $30,939. It then submitted an unsupported $11,573 instead of the correct sum, $20,939." +us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of the 2026 base of $184,500, reaching $20,418.20 repeatedly. It then submitted $32,846, which contradicts its own stated total." +us,scenario_052,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It used the 2025 wage base of $176,100 instead of $184,500. It also added the $2,250 Additional Medicare Tax twice, putting it on top of a $20,418.20 subtotal that already included it." us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It stripped the $3,389 ESI premium from wages and then subtracted the $950 auto loan interest, cutting taxable income to $46,530 when the 2026 base is gross wages of $66,968.68 less the $16,100 standard deduction. Neither adjustment applies: the listed wages are already the taxable wage figure, and the OBBBA auto-loan-interest deduction requires a qualifying new-vehicle purchase loan, which a bare loan balance and interest figure do not establish. It then submitted $6,224, a value its own $5,335.60 arithmetic never produces." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It carried the correct 2026 standard deduction ($16,100) and bracket edges ($12,400/$50,400) but subtracted the $950 auto loan interest as an above-the-line deduction, cutting taxable income from $50,868.68 to $49,919. The OBBBA deduction attaches only to interest on a qualifying new personal-use vehicle purchase loan, which these facts do not create, and removing that $950 accounts for exactly the $160.83 shortfall against $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It stacked three errors: removing the $3,389 ESI premium from wages, using the 2024 $14,600 standard deduction instead of the 2026 $16,100, and applying a bracket schedule whose 22% rate begins at $11,000 — producing $12,761.60. It then submitted $5,744, a figure unrelated both to that computation and to the correct $1,240 + $4,560 + 22% × $468.68 = $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It set the 2026 single standard deduction at $16,600 rather than $16,100, reaching $5,796.24 with otherwise-correct $12,400/$50,400 bracket edges, and then discarded that result for an unexplained $6,196 attributed to 'slightly updated' parameters. The actual $16,100 deduction leaves $50,868.68 taxable and $5,903.11 of tax." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived the reference answer exactly — $16,100 standard deduction, $50,869 taxable income, $1,240 + $4,560 + $103 = $5,903 — and then submitted $6,027 instead. The abandonment is the entire error: its own bracket arithmetic was correct to the dollar." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the correct chain — $16,100 standard deduction, $50,869 taxable, 10% on $12,400 plus 12% to $50,400 plus 22% on the $469 remainder, totaling $5,903 — then overrode it with $6,379 as a 'brackets estimate.' The submitted value is $476 above the schedule its own reasoning applied correctly." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It treated 2026 as a ~2.7% inflation index of 2025 law, using a $15,700 standard deduction with the 2025 bracket edges of $11,925 and $48,475. The 2026 single standard deduction is $16,100 and the 12%/22% break sits at $50,400, so it pushed $2,794 into the 22% bracket where only $468.68 belongs, overstating tax by $289.89." -us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $50,869 taxable income but scaled the 2025 bracket edges by 2% ($11,925→$12,164, $48,475→$49,445) instead of using the actual 2026 edges of $12,400 and $50,400, dropping $1,424 into the 22% bracket rather than $468.68. Its submitted $6,154 exceeds even that mis-bracketed total of about $6,003." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction with bracket edges of $12,200 and $49,950; the 2026 values are $16,100, $12,400, and $50,400. The $450 deduction shortfall plus the low 12%/22% break taxed $1,369 at 22% instead of $468.68, adding $148.07 to the correct $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages to reach a $63,580 AGI and then applied a $15,400 standard deduction with a 10% bracket ending at $12,225. Gross wages of $66,968.68 are the AGI and the 2026 standard deduction is $16,100, so taxable income is $50,868.68 — $2,689 above its $48,180 — and $468.68 of it reaches the 22% bracket its computation never enters." -us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It priced 2026 under a TCJA sunset: an $8,300 standard deduction plus a $5,300 personal exemption and a 10/15/25% rate schedule. For 2026 the personal exemption remains repealed, the single standard deduction is $16,100, and the rates above 10% are 12% and 22%, so both its $53,369 taxable income and its 25% top rate are wrong." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It combined two errors: removing the $3,389 ESI premium from wages and then applying a TCJA-sunset schedule of an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% rates. In 2026 there is no personal exemption, the single standard deduction is $16,100, the highest rate reached at this income is 22%, and the base is the full $66,968.68 of wages." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It named a $15,300 standard deduction but ran no bracket arithmetic; that deduction under the 2026 single schedule yields $6,079.11, and across the whole range of deductions other models assumed ($8,300 to $16,100) the schedule produces $5,903.11 to $7,619.11. Its $3,939 falls below that entire range and corresponds to no application of the 10/12/22% brackets." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It excluded the $3,389 ESI premium from wages and then assumed TCJA expiration, restoring a personal exemption and using pre-TCJA 15%/25% brackets. The 2026 law has no personal exemption and a $16,100 single standard deduction against the full $66,968.68 of wages, giving $50,868.68 taxable and a 22% top bracket covering only $468.68." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a TCJA-sunset 2026 — an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets — on top of an ESI-reduced $63,580 AGI. The 2026 schedule has no personal exemption, a $16,100 standard deduction, and a 22% top rate here, so its $49,980 taxable income and $707.50 of 25% tax both come from a repealed structure." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $3,737 with no derivation. Applying the 2026 single schedule to $66,968.68 of wages less the $16,100 standard deduction gives $5,903.11, and even the smallest deduction assumption in play ($8,300) yields $7,619.11, so $3,737 is below the entire feasible range and reflects no bracket computation." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the correct $16,100 standard deduction and 2026 brackets but first reduced wages by the $3,389 ESI premium, so taxable income came in at $47,480 instead of $50,868.68. That exclusion removed the entire $468.68 that belongs in the 22% bracket plus $3,389 of 12% income; its $5,459.10 also overshoots its own $47,480 computation of $5,449.60 by $9.50." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $3,389 ESI premium to an AGI of $63,580 before the standard deduction, producing $47,480 of taxable income and exactly $1,240 + 12% × $35,080 = $5,449.60. The listed gross wages of $66,968.68 are the tax base with no ESI offset, so taxable income is $50,868.68 and $468.68 falls in the 22% bracket." -us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied 2025 parameters wholesale — a $15,000 standard deduction with bracket edges at $11,925 and $48,475 — to a 2026 return. The 2026 single standard deduction is $16,100 and the 12%/22% break is $50,400, so its $3,494 of 22% income should be $468.68, overstating the tax by $444.07." -us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and bracket edges of $11,600 and $47,150, against 2026 values of $16,100, $12,400, and $50,400. That made taxable income $1,500.32 too high and taxed $5,219 at 22% where only $468.68 belongs, adding $671.07." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no derivation and invoked 'the basic CTC' for a childless 25-year-old, who generates no child tax credit at all. The 2026 schedule applied to $66,968.68 of wages less the $16,100 standard deduction yields $5,903.11 with zero nonrefundable credits, so its $3,657 is $2,246.11 low and below the tax at any 2026-era deduction level." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted $5,118 from self-described 'bracket and standard-deduction level approximations' without running any computation. The $16,100 standard deduction leaves $50,868.68 taxable and the 2026 single schedule gives $1,240 + $4,560 + 22% × $468.68 = $5,903.11, so its figure is $785.11 low and below what any 2026 deduction level produces." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction rather than the actual 2026 single figure of $16,100, and then submitted $5,793 when its own $50,969 taxable income run through the 2026 brackets gives $5,925.11. The correct $16,100 leaves $50,868.68 taxable and $5,903.11 of tax." -us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It stated every input correctly — the $16,100 standard deduction, $50,869 of taxable income, and the 10/12/22% brackets — and then submitted $5,985 rather than the $5,903.11 that schedule produces. Only $468.68 sits above the $50,400 edge, contributing $103.11 of 22% tax, so its total overshoots its own inputs by $81.89." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It rounded the standard deduction to '~15k' — the 2025 figure, not the 2026 $16,100 — and then eyeballed $5,850 without running the brackets, when even a $15,000 deduction yields $6,145.11 under the 2026 schedule. The correct deduction leaves $50,868.68 taxable and $5,903.11 of tax." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law TCJA sunset' parameters for 2026: an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% rates. In 2026 the personal exemption is zero, the single standard deduction is $16,100, and the top rate at this income is 22%, so its $53,369 taxable income overstates the correct $50,868.68 by $2,500 and its 25% bracket does not exist." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed a TCJA sunset with an indexed $8,522 standard deduction, a restored $5,435 personal exemption, and 10/15/25% rates. For 2026 the personal exemption is repealed, the single standard deduction is $16,100, and this income tops out at 22%, so the base is $50,868.68 rather than its $53,012 and the tax is $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 law — an $8,372 standard deduction and 10/15/25% brackets — leaving $58,597 of taxable income against the correct $50,868.68. The 2026 single standard deduction is $16,100 and the schedule is 10/12/22%, which is why its $9,073 overstates the tax by $3,169.89." -us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It removed the $3,389 ESI premium from wages and then applied a TCJA-sunset schedule of an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% rates. For 2026 the wages of $66,968.68 are the AGI, the standard deduction is $16,100, the personal exemption is zero, and the top rate reached is 22%, giving $50,868.68 taxable and $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It used the 2023 single standard deduction of $13,850 instead of the 2026 $16,100, computed $6,397.98 with otherwise-correct $12,400/$50,400 brackets, and then submitted $4,272. That figure is $2,125.98 below its own arithmetic and $1,631.11 below the correct $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It computed $6,127.18 using a $16,000 standard deduction with 2025 bracket edges of $11,925 and $48,475, then submitted $8,252 — $2,124.82 above its own total. With the actual 2026 $16,100 deduction and $12,400/$50,400 edges, taxable income is $50,868.68 and the tax is $5,903.11." -us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It asserted taxable income of about $49,779, implying a standard deduction near $17,190 rather than the 2026 figure of $16,100, and then reduced the result by a nonrefundable 'child tax credit/dependent credit' for a childless 25-year-old with no dependents. Even its own $49,779 base yields $5,725.48 under the 2026 schedule; the correct $50,868.68 base yields $5,903.11 with no nonrefundable credits available." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, and it also deducted $950 of auto loan interest. That interest does not qualify, because the OBBBA deduction covers only loans on new, U.S.-assembled vehicles. Its own arithmetic then gave $5,335.60, but it submitted $6,224, which follows from none of its steps. The correct result is $5,903.11 on $50,868.68 of taxable income." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It deducted $950 of auto loan interest as if it were above the line. The OBBBA qualified vehicle loan interest deduction applies only to loans on new, U.S.-assembled vehicles, which these facts don't establish (the vehicle is worth $7,410). That cut taxable income to $49,919 instead of $50,868.68. As a result, the $468.68 that belongs in the 22% bracket above $50,400 was never taxed there, giving $5,742.28 instead of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted the ESI premium from wages and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It then applied an invalid bracket formula that produced $12,761.60. The $5,744 it submitted matches none of those steps. The correct result is $16,100 off $66,968.68, taxed at 10%/12%/22% with breakpoints at $12,400 and $50,400, which gives $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It guessed a 2026 standard deduction of $16,600 instead of $16,100 and computed $5,796. It then raised that to $6,196 with no computation to support the change. The correct taxable income of $50,868.68 puts $468.68 in the 22% bracket, giving $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning got the right answer: a $16,100 standard deduction, taxable income of $50,869, the 10%/12%/22% brackets at $12,400 and $50,400, and a tax of $5,903. It then discarded that result and submitted $6,027, calling it rounding to bracket estimates, which has no basis in its own computation." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived $5,903 from $50,869 of taxable income using the 2026 brackets of 10% to $12,400, 12% to $50,400 and 22% above. It then submitted $6,379, a figure unconnected to that derivation." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed a standard deduction of $15,700 instead of the OBBBA 2026 amount of $16,100. It also used the 2025 brackets (10% to $11,925 and 12% to $48,475) instead of the 2026 breakpoints of $12,400 and $50,400. That put $2,794 in the 22% bracket instead of $468.68, giving $6,193 instead of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached $50,869 of taxable income. It then built the brackets by inflating 2025 thresholds by 2% (about $12,164 and $49,445) instead of using the 2026 breakpoints of $12,400 and $50,400, and it gave an unexplained total of $6,154. The actual 2026 brackets put only $468.68 at 22%, for $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and brackets of 10% to $12,200 and 12% to $49,950. The correct 2026 figures are a $16,100 deduction and breakpoints of $12,400 and $50,400. Those errors raised taxable income to $51,319 and pushed $1,369 into the 22% bracket, giving $6,051.18 instead of $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, which cut AGI to $63,580. It also used a $15,400 standard deduction and a $12,225 top for the 10% bracket instead of the 2026 values of $16,100 and $12,400. Together these understated the tax at $5,537.10." +us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA sunsets in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% rates. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption, and rates of 10%, 12% and 22% at this income, giving $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset rules: an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the $16,100 deduction and the 10%/12%/22% rates permanent. It also subtracted the $3,389 ESI premium from the given wages." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It assumed a $15,300 standard deduction instead of $16,100 and gave no bracket computation. Its $3,939 is about $2,000 below the tax that its own taxable income would produce at 10%/12%/22%. The correct result is $1,240 + $4,560 + $103.11 = $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expires in 2026 and applied the pre-TCJA standard deduction, personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent. It also subtracted the ESI premium from wages to get an AGI of $63,580 instead of $66,968.68." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA sunset parameters: an $8,300 standard deduction, a $5,300 exemption and brackets of 10%, 15% and 25%. OBBBA kept the TCJA structure permanently, with a $16,100 deduction and 10%/12%/22% brackets. It also improperly subtracted the $3,389 ESI premium from wages." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no deduction or bracket figures, and $3,737 is about $2,200 below the correct amount. The correct result applies 10% to $12,400, 12% to $50,400 and 22% on $468.68 of the $50,868.68 left after the $16,100 standard deduction, for $5,903.11. Its number matches taxing only about $34,000 of income, which means it understated taxable income or used lower rates." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premium from wages that were already given as the taxable wage figure, which lowered taxable income to $47,480. That kept all income out of the 22% bracket, and a small arithmetic slip gave $5,459.10. With AGI equal to the $66,968.68 in wages, taxable income is $50,868.68 and the tax is $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $63,580 by subtracting the $3,389 ESI premium from the given wages, which cut taxable income to $47,480 and the tax to $5,449.60. The listed wages are the taxable wage base. The correct taxable income is $50,868.68, with $468.68 taxed at 22%." +us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used 2025 parameters for 2026: a $15,000 standard deduction and 12% bracket ending at $48,475. The 2026 figures are a $16,100 deduction and 12% bracket ending at $50,400. That put $3,494 in the 22% bracket instead of $468.68, giving $6,347.18." +us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and brackets ending at $11,600 and $47,150. The 2026 values are $16,100, $12,400 and $50,400. That overstated taxable income at $52,369 and put $5,219 in the 22% bracket, giving $6,574.18." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It subtracted a basic CTC or other nonrefundable credit, but this single filer has no children or dependents, so no CTC or credit for other dependents applies. The tax after nonrefundable credits equals the full regular tax of $5,903.11, not $3,657." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It used approximate standard deduction and bracket levels and gave no figures. Its $5,118 is about $785 below the tax the 2026 schedule produces on $50,868.68 of taxable income. Applying the $16,100 deduction and breakpoints at $12,400 and $50,400 gives $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction instead of $16,100. It also misapplied the brackets: even its own $50,969 of taxable income produces $5,925.18 at 10% to $12,400, 12% to $50,400 and 22% above, not the $5,793 it gave. The correct taxable income of $50,868.68 gives $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It got the correct taxable income of $50,869 but reported $5,985, which means it used wrong bracket thresholds. The 2026 single schedule taxes $12,400 at 10% ($1,240), $38,000 at 12% ($4,560) and only $468.68 at 22% ($103.11), for $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $3,389 of ESI premiums from wages that were already given as the taxable wage figure, which lowered taxable income to $47,480 and the tax to $5,449.60. With AGI equal to the $66,968.68 in wages, taxable income is $50,868.68 and the tax is $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,000 instead of the 2026 amount of $16,100, which gave taxable income of about $52,000. Its $5,850 doesn't even match the 2026 brackets applied to that income, which yield about $6,250. The correct result on $50,868.68 is $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA sunset law: an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses a $16,100 standard deduction, no exemption and 10%/12%/22% rates, giving $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset: an $8,522 standard deduction, a $5,435 restored exemption and 10%/15%/25% brackets. OBBBA made the $16,100 standard deduction and the 10%/12%/22% rate structure permanent for 2026." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 standard deduction and brackets of 10% to $12,225 and 12% to $49,725. The 2026 values are a $16,100 deduction and breakpoints of $12,400 and $50,400. That put $1,094 in the 22% bracket instead of $468.68, giving $5,963." +us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law, with an $8,372 standard deduction and 15%/25% brackets. OBBBA made the TCJA structure permanent, with a $16,100 deduction and 10%/12%/22% brackets. That overstated taxable income at $58,597 and the tax at $9,073." +us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It assumed TCJA expired and used an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the $16,100 deduction and 10%/12%/22% rates permanent. It also subtracted the $3,389 ESI premium from the given wages." +us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2023 standard deduction of $13,850 instead of $16,100 and computed $6,397.98. It then submitted $4,272, which contradicts its own arithmetic. The correct result on $50,868.68 of taxable income is $5,903.11." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction and 2025 brackets (breakpoints at $11,925 and $48,475) instead of the 2026 values of $16,100, $12,400 and $50,400, and computed $6,127.18. It then submitted $8,252, which contradicts its own computation." +us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It understated taxable income at $49,779 and then subtracted a nonrefundable child or dependent credit. This single filer has no children or dependents, so no CTC or credit for other dependents applies. The tax is the full $5,903.11 on $50,868.68 of taxable income." us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component." -us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $3,389 employer-sponsored insurance premium as a pre-tax payroll deduction and reduced FICA wages to $63,580. The prompt never identifies that premium as pre-tax, so Social Security and Medicare apply to the full $66,969 of wages." -us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $3,389 employer-sponsored insurance premium from gross wages before applying FICA. Because pre-tax treatment was not listed, the applicable Social Security and Medicare wage base is $66,969, not $63,580." -us,scenario_053,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model assumed the employer-sponsored insurance premium was excluded from FICA wages and calculated 7.65% on $63,580. The premium has no listed pre-tax status, so payroll taxes are calculated on all $66,969 of wages." -us,scenario_053,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced payroll-taxable wages by the $3,389 insurance premium without any fact establishing a pre-tax payroll deduction. Social Security and Medicare instead use the full $66,969 wage amount." -us,scenario_053,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly classified the employer-sponsored insurance premium as a pre-tax reduction of FICA wages. Under the supplied facts, the FICA base remains $66,969 rather than $63,580." -us,scenario_053,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model subtracted the $3,389 health premium from wages even though the prompt does not state that it was paid through a pre-tax arrangement. Applying Social Security and Medicare to the full $66,969 wage base produces the required payroll tax." -us,scenario_053,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model selected the correct full wage base and the correct 6.2% and 1.45% rates but computed their application incorrectly. Applying the combined 7.65% employee FICA rate to $66,969 yields the traced payroll tax after component-level calculation and rounding, not $5,124.74." -us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated the correct 6.2% Social Security and 1.45% Medicare rates and correctly excluded Additional Medicare Tax, but its arithmetic does not equal those rates applied to $66,969. A combined 7.65% calculation is approximately $5,123.1, not $5,329." -us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model assumed the $3,389 employer-sponsored insurance premium was a pre-tax FICA exclusion and reduced taxable payroll wages to $63,580. No pre-tax treatment is supplied, so both Social Security and Medicare apply to the full $66,969." +us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied the correct 6.2% Social Security and 1.45% Medicare rates to a base of $63,580. That base came from subtracting the $3,389 employer-sponsored insurance premium, which treats the premium as a pre-tax Section 125 exclusion the household facts never state. FICA applies to the full $66,969 of gross wages, and 7.65% of that is $5,123.10." +us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model explicitly deducted the $3,389 employer-sponsored insurance premium as a pre-tax payroll exclusion ($66,969 - $3,389 = $63,580) before applying 7.65%. The payroll-tax wage base is the full $66,969 of gross wages, so the correct total is $4,152.06 Social Security plus $971.05 Medicare, or $5,123.10." +us,scenario_053,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced FICA wages by the $3,389 employer-sponsored insurance premium, assuming a pre-tax cafeteria-plan arrangement that is not in the household facts. It then took 7.65% of $63,580 and rounded to $4,864. The employee FICA rates apply to the full $66,969 of gross wages, which gives $5,123.10." +us,scenario_053,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model computed Social Security ($3,941.96) and Medicare ($921.91) on $63,580 after subtracting the $3,389 employer-sponsored insurance premium as pre-tax. The premium does not reduce the payroll-tax base. On the full $66,969 of wages, Social Security is $4,152.06 and Medicare is $971.05, for $5,123.10 in total." +us,scenario_053,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model defined FICA-taxable wages as $66,969 minus $3,389 of 'pre-tax health insurance premiums', which is a Section 125 exclusion the facts never establish. Applying 6.2% and 1.45% to the full $66,969 of gross wages gives $5,123.10, not $4,863.87." +us,scenario_053,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model shrank the FICA wage base to $63,580 by excluding the $3,389 employer-sponsored insurance premium, which understated both Social Security and Medicare tax. Employee payroll tax is 7.65% of the full $66,969 of gross wages, which is $5,123.10." +us,scenario_053,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model used the correct base ($66,969 of gross wages) and the correct rates (6.2% Social Security plus 1.45% Medicare), but multiplied them out wrong. The correct figures are $4,152.06 plus $971.05, which is $5,123.10. The model reported $5,124.74, an arithmetic overstatement of $1.64." +us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model set up the right formula, 0.062×66,969 + 0.0145×66,969, but evaluated it as about $5,329, which is an effective rate of about 7.96% instead of 7.65%. Evaluated correctly, the formula gives $4,152.06 plus $971.05, which is $5,123.10, so the $206 overstatement is purely an arithmetic error." +us,scenario_053,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $63,580 'after pretax employer-plan premiums'. That subtracts the $3,389 employer-sponsored insurance premium as a cafeteria-plan exclusion the household facts do not state. FICA is levied on the full $66,969 of gross wages, which gives $5,123.10." +us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model defined taxable payroll wages as gross wages minus the $3,389 employer-sponsored insurance premium ($63,580), then applied 6.2% and 1.45% and rounded to $4,864. The premium is not excluded from the payroll-tax base. On the full $66,969 of wages, Social Security is $4,152.06 and Medicare is $971.05, for $5,123.10 in total." us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Applied the repealed 5.695% rate instead of Idaho's 2026 flat 5.3%, and discarded its own correct $50,869 taxable income for an invented $53,000 base built from a nonexistent $13,969 exemption. It never applied the $4,920 single-filer zero-tax bracket, which is what reduces the taxed base to $45,948.68." -us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Cut AGI to $66,019 by subtracting the $950 auto loan interest, which is nondeductible personal interest and leaves Idaho AGI at $66,968.68. It then taxed the entire $49,919 at 5.3% instead of exempting the first $4,920 under Idaho's single-filer zero-tax bracket, which drove the $210 overshoot." -us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used Idaho's pre-2023 graduated schedule (1%–7.15%), which the flat 5.3% rate replaced, together with a $7,200 standard deduction rather than the conformed federal $16,100. It compounded this by subtracting the $3,389 employer-plan premium from taxable income and applying a $59 personal exemption credit Idaho does not have." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used the 2025 standard deduction of $15,750 rather than the 2026 indexed $16,100, then abandoned its own 5.3% computation of $2,715 for an unsupported ~4.97% effective rate. It never applied the $4,920 zero-tax bracket, the step that turns $2,696 into $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Applied the repealed 5.695% rate and the stale $4,673 zero bracket, computed $2,631 from those inputs, then submitted $2,867 in contradiction of its own arithmetic. Its own correct $50,869 taxable income yields $2,435.28 under the 2026 flat 5.3% rate with the indexed $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Named the correct $16,100 standard deduction, 5.3% rate, and ~$4,900 exemption threshold but never subtracted the threshold, taxing the whole $50,869 to reach $2,696. It then netted the Idaho grocery credit against the result, but that credit is refundable and is excluded from tax before refundable credits." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Reduced wages by the $3,389 employer-plan premium, but Idaho AGI here is the full $66,968.68, and it used a $15,000 standard deduction instead of $16,100 plus the repealed 5.695% rate. Most of the $331 overshoot comes from taxing the entire base rather than only income above Idaho's $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Computed $2,960 from a $15,000 standard deduction and the repealed 5.695% rate, then submitted $3,706 with no arithmetic linking it to that figure. Idaho 2026 uses the conformed $16,100 standard deduction and 5.3% on taxable income above the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Invented a $15,650 standard deduction and a $4,000 personal exemption; Idaho conforms to the $16,100 federal standard deduction and allows no personal exemption. It also used a 5.59% rate rather than the 2026 flat 5.3% and taxed the entire base instead of only income above the $4,920 zero bracket." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Used the correct 5.3% rate but a $48,180 base, implying $18,789 of deductions against the $16,100 standard deduction Idaho actually allows. It then applied the rate to every dollar rather than only the $45,948.68 above the $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Subtracted an $8,300 standard deduction; Idaho conforms to the 2026 federal single standard deduction of $16,100, so taxable income is $50,868.68, not $58,669. It also omitted the $4,920 zero-tax bracket and taxed the full base at 5.3%." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied the repealed 5.695% rate rather than the 2026 flat 5.3%, and reached a $49,980 base by adding a personal exemption Idaho does not allow on top of the standard deduction. It also taxed the full base instead of only income above the $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave no bracket arithmetic; $2,652 corresponds to roughly 5.3% of the entire ~$50,000 taxable income, the signature of skipping Idaho's $4,920 zero-tax bracket. The correct computation is ($66,968.68 − $16,100 − $4,920) × 5.3% = $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Used a 5.692% rate that appears nowhere in Idaho law — the 2026 flat rate is 5.3% — on a $49,852 base rather than the correct $50,868.68. It also taxed all of that base instead of only the amount above the $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Cut AGI to $63,580 for the employer-plan premium and stacked a personal exemption on the standard deduction, neither of which applies; Idaho taxable income is $66,968.68 − $16,100 = $50,868.68. It then used the superseded 5.8% rate and omitted the $4,920 zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $3,450 is about 5.3% of $65,094, consistent with taxing AGI with essentially no standard deduction. Idaho subtracts the conformed $16,100 standard deduction and taxes only the $45,948.68 above the $4,920 zero bracket, giving $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Subtracted the $3,389 employer-plan premium to reach a $63,580 AGI, but Idaho AGI is the full $66,968.68, and it used the repealed 5.695% rate rather than 5.3%. It also taxed its whole $47,480 base instead of exempting the first $4,920 under Idaho's zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Applied the repealed 5.695% flat rate to a $47,470 base, which is $3,389 short of the correct $50,868.68 because it excluded the employer-plan premium from wages. It also taxed every dollar rather than only income above Idaho's $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the $15,000 standard deduction instead of the 2026 indexed $16,100 and the repealed 5.695% rate instead of 5.3%. It then applied that rate to the entire $51,969 base rather than to the income above Idaho's $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Fabricated a rule that Idaho allows $1,250 of deduction per $1,000 of federal taxable income, producing an $83,711 deduction that zeroes out AGI. Idaho's deduction is the greater of the conformed $16,100 federal standard deduction or itemized deductions of $377.35, leaving $50,868.68 taxable and $2,435.28 of tax." -us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Derived the correct $50,869 taxable income but taxed it at 5.6%; Idaho's 2026 flat rate is 5.3%. It also applied that rate to the whole base rather than only the $45,948.68 above the single filer's $4,920 zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Submitted an estimate with no rate, deduction, or bracket arithmetic; $2,050 implies a 4.46% effective rate on the $45,948.68 Idaho actually taxes. The 2026 computation is ($66,968.68 − $16,100 − $4,920) × 5.3% = $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Returned a bare guess with no derivation; $1,195 is 2.6% of the $45,948.68 that Idaho taxes, far below the 5.3% flat rate. Applying 5.3% to taxable income of $50,868.68 above the $4,920 zero bracket gives $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Reproduced Idaho's structure exactly — 5.3% on taxable income above the zero bracket — but plugged in un-indexed parameters: the 2025 standard deduction of $15,750 and the older $4,489 zero bracket instead of 2026's $16,100 and $4,920. That $781 of excess base at 5.3% is the entire $41 overshoot." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Derived the correct $50,869 taxable income and 5.3% rate but taxed all of it, omitting Idaho's $4,920 zero-tax bracket for single filers. Exempting that first $4,920 converts its $2,696.06 into the correct $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Applied the correct $16,100 standard deduction, 5.3% rate, and zero-bracket structure, missing only the indexed threshold amount: it used $4,946 where Idaho's 2026 single zero bracket is $4,920. That $26 of base at 5.3% accounts for the entire $1.36 miss." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Computed 5.3% on the full $50,869 of taxable income, skipping Idaho's $4,920 zero-tax bracket for single filers. Taxing only the $45,948.68 above that threshold produces $2,435.28 rather than $2,696." -us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Stacked a $5,000 health-insurance subtraction on top of the $16,100 standard deduction, but Idaho's deduction is the greater of the standard deduction or itemized deductions ($377.35 here), so taxable income stays $50,868.68. It also used a $4,811 zero-rate allowance rather than the indexed $4,920." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Used a 5.8% rate that Idaho replaced with the 5.3% flat rate for 2026, on a rounded ~$52,000 base rather than the correct $50,868.68. It also taxed the whole base instead of only the $45,948.68 above the $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted an $8,300 standard deduction; Idaho conforms to the 2026 federal single standard deduction of $16,100, so its $58,669 base overstates taxable income by $7,800. It compounded that with the repealed 5.695% rate and no $4,920 zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Asserted a 2026 federal standard deduction of $8,522; the conformed figure is $16,100, making taxable income $50,868.68 rather than $58,447. It also applied the repealed 5.695% rate and omitted the $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Used an implied standard deduction of roughly $8,372 against the conformed $16,100, and the superseded 5.8% rate rather than 5.3%. It also taxed its entire $58,597 base instead of only income above Idaho's $4,920 single zero-tax bracket." -us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Reached the correct $50,869 taxable income at the correct 5.3% rate but omitted Idaho's $4,920 zero-tax bracket, taxing the first $4,920 that Idaho exempts. That single omission overstates the tax by $261." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so no substantive computation reached scoring. The required figure is ($66,968.68 − $16,100 − $4,920) × 5.3% = $2,435.28." -us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Applied 5.3% to the full $50,869 of taxable income, omitting Idaho's $4,920 zero-tax bracket for single filers. Taxing only the $45,948.68 above that threshold yields $2,435.28 instead of $2,696.06." -us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Stated that no standard deduction adjustment applies and computed $66,969 × 5.695% = $3,814, then submitted $2,878, a figure its own reasoning never produces. Idaho subtracts the conformed $16,100 standard deduction and taxes the excess over the $4,920 zero bracket at 5.3%." -us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Computed $50,869 × 5.3% = $2,696.06 without exempting the first $4,920 under Idaho's single-filer zero-tax bracket, then subtracted $10 for the Permanent Building Fund, which is an additional Idaho tax rather than a nonrefundable credit." -us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Removed the $3,389 employer-plan premium from wages and applied a 5.8% rate that Idaho replaced with 5.3%, computing $3,688, then cut it to $2,702 without ever subtracting a stated standard deduction. The correct path is $66,968.68 − $16,100 = $50,868.68, less the $4,920 zero bracket, taxed at 5.3%." -us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Submitted a number with no rate, deduction, or threshold arithmetic; $1,588 is 3.5% of the $45,948.68 Idaho actually taxes, well below the 5.3% flat rate. Applying 5.3% above the $4,920 zero bracket to $50,868.68 of taxable income gives $2,435.28." -us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"Asserted that PolicyEngine gives Idaho's grocery credit nonrefundable/other-credit treatment and that $66,969 of wages exceeds income limits for refundable state credits. The grocery credit under Idaho Code 63-3024A carries no income test and is paid in full regardless of liability; PolicyEngine assigns id_grocery_credit directly to state_refundable_credits, giving $155 for 12 qualifying months." -us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Denied that Idaho has any refundable individual income tax credit for 2026 and invented an income threshold that $66,969 exceeds. Idaho's grocery credit is refundable and residency-based, not income-tested, so a single full-year resident under 65 qualifies for all 12 months at the 2026 per-person amount of $155." -us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"Applied a nonrefundable-first ordering, reasoning that Idaho tax on $66,969 exceeds the grocery credit so it offsets tax rather than being refunded. The grocery credit is not an offset against liability at all — it is claimed as a refundable credit for the full computed amount — and the model additionally used the superseded $120 per-person figure rather than the $155 in force for 2026." -us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"Treated the grocery credit as nonrefundable in effect via an income-tax-return offset, so it disappeared into liability for a $67k earner. Idaho pays the grocery credit out in full to full-year residents independent of tax liability, and the 2026 per-person amount is $155, not the $120 the model quoted." -us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,Stated the grocery credit is applied against liability so no refundable state credits remain at this income. The credit is refundable by statute with no liability-offset step and no income phase-out; 12 qualifying months of Idaho residency produce the full $155. -us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,Declared that Idaho has no refundable individual income tax credits and that the grocery credit is nonrefundable. Idaho Code 63-3024A makes the grocery credit refundable — it is claimable even by residents with no filing requirement — so the correct output is the full-year $155. -us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Ruled out any grocery credit refund beyond offset and conditioned refundable credits on dependents and income level. Idaho's grocery credit requires neither dependents nor income below a threshold and is refunded in full, so the single resident's 12 qualifying months yield $155." -us,scenario_053,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Invented a phase-out, claiming the refundable grocery credit is fully phased out at an AGI of $66,969. Idaho's grocery credit has no AGI phase-out — it is reduced only for months of nonresidency, incarceration, or SNAP receipt, none of which apply here — leaving the full $155." -us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Asserted the grocery credit has an income limit that this household fails. No income limit exists in Idaho Code 63-3024A; the credit is a flat per-person, per-month amount for full-year residents, producing $155 for all 12 months of 2026." -us,scenario_053,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Correctly identified a single refundable Idaho grocery credit for the head but used the superseded $120 per-person amount. Idaho's grocery credit was raised to $155 per person, and that is the 2026 base amount applied for all 12 qualifying months." -us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Got the structure right — a refundable per-resident grocery credit — but priced it at the stale $120 per person. The 2026 statutory per-person amount is $155, applied in full because the head qualifies for all 12 months." -us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Returned a bare denial that no state refundable credits apply, omitting Idaho's grocery credit from consideration entirely. Every full-year Idaho resident qualifies for it on residency alone, so the correct answer is 12 months at $155 per person." -us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly treated the grocery credit as refundable and correctly flagged age under 65 as the relevant tier, but used the outdated $120 per-person figure. The current per-person amount is $155, with no age-based reduction to the base amount for 2026." -us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used the long-superseded $100 per-person grocery credit from Idaho's pre-2022 schedule, which paired $100 for under-65 residents with $120 for those 65 and older. The 2026 amount is $155 per person for all 12 qualifying months." -us,scenario_053,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Asserted without analysis that no state refundable credits apply, never reaching Idaho's grocery credit. That credit is refundable and residency-based, so a full-year Idaho resident with 12 qualifying months receives $155." -us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Identified the refundable per-eligible-resident grocery credit but quoted the retired $120 rate. Idaho's per-person grocery credit stands at $155 for 2026, and the household qualifies for all 12 months with no reduction." -us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Applied the obsolete $100 under-65 tier of Idaho's grocery credit. Idaho replaced that two-tier $100/$120 structure with a single per-person amount of $155, which is what the head receives for 12 qualifying months." -us,scenario_053,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Priced the refundable grocery credit at the pre-2022 $100 per-resident amount. The 2026 per-person base is $155, awarded in full here because all 12 months qualify." -us,scenario_053,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"Claimed Idaho's grocery credit is nonrefundable and that no listed fact establishes eligibility for any refundable Idaho credit. Residency in Idaho for the full tax year is the eligibility fact, and the credit is refundable by statute, so the household receives $155." -us,scenario_053,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Explicitly labeled $120 the post-2025 amount for Idaho's refundable grocery credit, which is the earlier figure rather than the current one. The per-person amount for 2026 is $155, applied for all 12 qualifying months." -us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Concluded that no refundable Idaho credit is indicated by the household facts, looking for an affirmative triggering fact the grocery credit does not require. Idaho grants it on full-year residency alone, yielding $155 for this filer." -us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Searched only for child-related refundable state credits and returned zero when it found none. Idaho's refundable credit here is the grocery credit, which is per-person and available without children, giving $155 for 12 qualifying months." -us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,Correctly isolated the grocery credit as the sole refundable Idaho credit for one nonelderly resident but valued it at the superseded $120. The 2026 per-person amount is $155. -us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Held that the income is too high for the Idaho grocery credit. The grocery credit contains no income ceiling or phase-out; the only reductions are for nonresident months, incarceration, or SNAP months, so the full $155 is paid." -us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Reasoned correctly that the credit is refundable and the income below any phaseout, then used the outdated $120 per-eligible-resident amount. Idaho's 2026 per-person grocery credit is $155." -us,scenario_053,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Gave a bare no-state-refundable-credits answer with no consideration of Idaho's grocery credit. That credit is refundable and available to every full-year resident, producing $155 for 12 qualifying months." -us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Correctly identified the grocery credit as the only refundable Idaho credit but applied the retired $100 non-elderly tier. Idaho now uses a single per-person amount of $155 for filers of any age, which is the full-year value here." -us,scenario_053,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Used the obsolete $100 amount for a non-elderly adult under Idaho's former age-split grocery credit schedule. The current per-person amount is $155, awarded for all 12 qualifying months." -us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"Returned zero on the assertion that no state refundable credits apply, never evaluating the grocery credit. Full-year Idaho residency alone qualifies the head, yielding a refundable $155." -us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"Correctly counted one qualifying resident and correctly excluded other Idaho refundable credits, but estimated the grocery credit at about $120. The statutory per-person amount for 2026 is $155, applied for all 12 months." -us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_refundable_credits, so the required key was absent rather than substantively wrong. The correct derivation is Idaho's refundable grocery credit at $155 per person for 12 qualifying months, giving $155." -us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Applied the correct mechanics — one under-65 full-year resident, refundable, no SNAP-month or incarceration reduction — but used the superseded $120 per-person amount. Idaho's 2026 grocery credit is $155 per person, so the full-year value is $155." -us,scenario_053,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Limited its search to child-based and property-tax-based Idaho credits and concluded none apply. The applicable refundable credit is the grocery credit, which turns on residency rather than children or property tax, giving $155 for 12 qualifying months." -us,scenario_053,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Structured the calculation correctly as a refundable per-qualified-family-member grocery credit times one member, but used $120 as the 2026 per-member amount. Idaho's per-person grocery credit for 2026 is $155." -us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"Correctly noted Idaho has no state EITC or CTC, then wrongly inferred that no refundable credit exists for a childless filer. Idaho's refundable grocery credit is per-person and requires no children, so this resident receives $155 for 12 qualifying months." -us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"Asserted Idaho provides no refundable income tax credit for this household based on the listed facts. Those facts include Idaho residence for the full 2026 tax year, which alone qualifies the head for the refundable grocery credit of $155." +us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate instead of the 2026 flat rate of 5.3%. It also taxed a $53,000 base built from an invented $13,969 exemption instead of $66,969 − $16,100 = $50,869, and it never removed the $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to its entire taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also subtracted the $950 auto loan interest from AGI, so AGI came out as $66,019 instead of $66,969. The frozen reference's tax is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,state_local_rule,False,"The model used Idaho's repealed graduated brackets (1%–6.5%, plus an invented 7.15% tier) instead of the flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also used a $7,200 standard deduction instead of the $16,100 federal-conforming amount, subtracted the ESI premiums, and invented a $59 exemption credit." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,state_local_rule,False,"The model used the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100. It never applied the $4,920 zero-rate bracket, and it dropped the correct 5.3% rate for an arbitrary lower effective rate, landing on $2,548 instead of 0.053 × $45,949 = $2,435." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a stale $4,673 zero-bracket threshold instead of 5.3% above $4,811, the threshold the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. It then raised its own $2,631 intermediate result to $2,867 with no basis." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,state_local_rule,False,"The model named the 5.3% rate and a roughly $4,900 threshold but never subtracted the threshold. Its $2,650 equals 5.3% of about $50,000. It also claimed the grocery credit offsets tax as a nonrefundable credit, but that credit is refundable and plays no part in this output." +us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,state_local_rule,False,"The model used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. That taxed $109 more income at 5.3% than the frozen reference, $5.79 of tax." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate with no zero-rate bracket instead of 5.3% above the zero-rate threshold ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also subtracted the $3,389 ESI premiums from AGI and used a $15,000 standard deduction instead of $16,100 on the full $66,969 AGI." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a $15,000 standard deduction, which gave $2,960. It then raised that to $3,706 with no basis. The frozen reference's computation is 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by 0.053 × $4,920 = $260.78." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,state_local_rule,False,"The model invented a 5.59% rate and a $4,000 personal exemption, which Idaho does not allow, and used a $15,650 standard deduction. Idaho instead applies 5.3% to taxable income above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference) after the $16,100 standard deduction; the exclusion's corrected value also subtracts the $5,000 of premiums, assuming they are not paid pre-tax." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to its full $48,180 base with no $4,920 zero-rate bracket. That base implies it subtracted the $3,389 ESI premiums from AGI and used a $15,400 standard deduction instead of $16,100." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,state_local_rule,False,"The model used an $8,300 standard deduction instead of the $16,100 single standard deduction that Idaho conforms to. On top of that, it applied 5.3% to the full $58,669 without subtracting the $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to a $49,980 base with no zero-rate bracket. That base reflects subtracting the ESI premiums from AGI and adding a nonexistent personal exemption. The frozen reference's computation is 5.3% × ($50,869 − $4,920); the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,state_local_rule,False,"The model gave no derivation. Its $2,652 matches 5.3% applied to roughly $50,000 of taxable income with no $4,920 zero-rate bracket removed. The frozen reference's result is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,state_local_rule,False,"The model applied a superseded rate of about 5.69% to all of roughly $49,852 of taxable income instead of 5.3% on only the income above the zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference)." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,state_local_rule,False,"The model used Idaho's 2023 rate of 5.8% instead of the 2026 rate of 5.3%, applied it with no zero-rate bracket, and invented a personal exemption. It also reduced AGI to $63,580 by subtracting the ESI premiums, though AGI is the full $66,969 of wages." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,state_local_rule,False,"The model gave no derivation. Its $3,450 is about 6.8% of the frozen reference's $50,869 base, so it either used a rate above Idaho's flat 5.3% or skipped most of the $16,100 standard deduction. Either way, it applied no $4,920 zero-rate bracket; the frozen reference's tax is $2,435, and the exclusion's corrected value is $2,176.06." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to all $47,480 of its taxable income with no $4,920 zero-rate bracket. It also subtracted the $3,389 ESI premiums from AGI, though Idaho starts from the full $66,969 AGI." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,state_local_rule,False,"The model applied the superseded 5.695% rate to about $47,470 of taxable income, a base that reflects subtracting the ESI premiums from AGI. It also omitted the $4,920 zero-rate bracket; the frozen reference's computation is 5.3% × ($50,869 − $4,920), and the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,state_local_rule,False,"The model used the superseded 5.695% rate and a $15,000 standard deduction instead of 5.3% and $16,100. It also taxed all $51,969 without removing the $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,state_local_rule,False,"The model invented a '$1,250 per $1,000 of federal taxable income' deduction that wiped out taxable income. The frozen reference subtracts only the $16,100 standard deduction, leaving $50,869, which it taxes at 5.3% above the engine's projected $4,920 for $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,state_local_rule,False,"The model used a 5.6% rate instead of Idaho's 2026 flat rate of 5.3%. It also applied the rate to the full $50,869 without subtracting the $4,920 single zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,state_local_rule,False,"The model gave no derivation and cited nonexistent 'state brackets'. Its $2,050 is an effective rate of about 4.0% on the $50,869 base, well short of the frozen reference's 5.3% × $45,949 = $2,435 (above the engine's projected $4,920 zero bracket) and of the exclusion's corrected value, $2,176.06." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,state_local_rule,False,"The model gave no computation. Its $1,195 is an effective rate of about 2.3% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,state_local_rule,False,"The model used stale parameters: the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100, and a $4,489 zero-rate threshold instead of the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026 (the frozen reference uses the engine's projected $4,920). Both enlarged the income taxed at 5.3%, by $672 against the convention's threshold and $781 against the frozen reference." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,state_local_rule,False,"The model used a zero-rate threshold of $4,946, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $135 less income at 5.3% than the $4,811 threshold does, and $26 less than the frozen reference ($1.36 of tax)." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,state_local_rule,False,"The model subtracted the $5,000 of health insurance premiums from Idaho income, which the exclusion's corrected value subtracts under Idaho Code 63-3022P (assuming the premiums are not paid through a pre-tax salary reduction) and the frozen reference, AGI minus only the $16,100 standard deduction, leaves out. It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Together these taxed $4,891 less income than the frozen reference; its $2,176.07 is $0.013 above the exclusion's corrected value, $2,176.057." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,state_local_rule,False,"The model applied 'progressive' rates to the $50,869 taxable income, but Idaho has a single flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). Its effective rate of about 4.25% understated the tax." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,state_local_rule,False,"The model used a zero-rate threshold of about $4,800, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, taxing $46,069 at 5.3% instead of $46,058 under the convention or $45,949 in the frozen reference. That put its tax $6.38 above the frozen reference and $0.60 above the $2,441.06 that the $4,811 threshold gives before the premium subtraction." +us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,state_local_rule,False,"The model subtracted the $3,389 ESI premiums from AGI, which lowered taxable income to $47,480 instead of $50,869. It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,state_local_rule,False,"The model used Idaho's 2023 rate of 5.8% instead of the 2026 flat rate of 5.3%, applied it to about $52,000 with no $4,920 zero-rate bracket, and understated the $16,100 standard deduction." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,state_local_rule,False,"The model used an $8,300 standard deduction instead of the $16,100 single standard deduction that Idaho conforms to. It then applied the superseded 5.695% rate to all $58,669 with no $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,state_local_rule,False,"The model used an $8,522 standard deduction instead of $16,100. It then applied the superseded 5.695% rate to all $58,447 without removing the $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,state_local_rule,False,"The model used a $16,150 standard deduction instead of $16,100 and a $4,942 zero-rate threshold, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, understating the income taxed at 5.3% by $72 against the frozen reference and $181 against the convention. It then rounded down, landing $4.28 below the frozen reference's $2,435.28." +us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,state_local_rule,False,"The model used the obsolete 5.8% rate instead of 5.3% and applied it to all $58,597 of taxable income. That base implies a standard deduction of about $8,400 instead of $16,100, and the model omitted the $4,920 zero-rate bracket." +us,scenario_053,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the output is missing rather than substantively computed." +us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). That overstated the tax by $260.78." +us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,state_local_rule,False,"The model's reasoning skipped the $16,100 standard deduction entirely, applied the superseded 5.695% rate to all $66,969 of AGI, and used no zero bracket. It then submitted $2,878, which does not match its own $3,814 computation." +us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,state_local_rule,False,"The model applied 5.3% to the full $50,869 and omitted the $4,920 zero-rate bracket. It also subtracted the $10 Permanent Building Fund as a nonrefundable credit, but that fund is a separate $10 tax, not a credit, and it is not part of this liability." +us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,state_local_rule,False,"The model used the obsolete 5.8% rate and subtracted the ESI premiums from wages. It then arrived at $2,702 through an unexplained adjustment instead of computing the frozen reference's 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,state_local_rule,False,"The model gave no computation. Its $1,588 is an effective rate of only about 3.1% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06." +us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Stated that PolicyEngine treats Idaho's grocery credit as nonrefundable and that $66,969 wages exceed refundable-credit income limits. The grocery credit is modeled as a refundable state credit with no income limit, so the full $155 base belongs in state_refundable_credits." +us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Idaho has no refundable income tax credits at this income level, which overlooks the Idaho grocery credit entirely. That credit is refundable, has no income test, and pays $155 to this under-65 full-year resident." +us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Named the grocery credit but reasoned that it only offsets Idaho tax liability, so nothing counts as refundable. The grocery credit is classified as a refundable credit whatever the liability, so its full amount counts. The model also used the stale $120 figure instead of the 2026 amount of $155." +us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Called the $120 grocery credit 'nonrefundable in effect' because it offsets income tax and zeroed it out. The credit counts in full as a refundable state credit, and its 2026 value is $155, not $120." +us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the grocery credit as applied against liability, leaving no refundable credit. The Idaho grocery credit is a refundable credit counted in full in state_refundable_credits: $155 for this one-person household." +us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,Asserted that Idaho's grocery credit is nonrefundable and that Idaho has no refundable credits. The grocery credit is refundable and yields $155 for this under-65 full-year resident. +us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Ruled out any 'grocery credit refund beyond offset' and cited income level and no dependents. The grocery credit is a per-person refundable credit with no income or dependent requirement, and it counts in full at $155." +us,scenario_053,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,Correctly identified the refundable Idaho grocery credit for one person but used the outdated $120 per-person amount. The 2026 base for a filer under 65 is $155. +us,scenario_053,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Claimed the grocery credit is fully phased out at $66,969 AGI. The Idaho grocery credit has no income phase-out, so the full $155 base applies." +us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Zeroed the grocery credit by applying an income limit that does not exist. Every full-year Idaho resident gets the credit regardless of income, and it is $155 for this filer." +us,scenario_053,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,Recognized the refundable grocery credit for the head but used the pre-2026 $120 amount instead of the 2026 per-person base of $155. +us,scenario_053,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,Identified the under-65 grocery credit but used $150 per person. The 2026 base amount is $155. +us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,Estimated the refundable grocery credit at $120 per person for 2026. That is the outdated rate; the 2026 base is $155. +us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Concluded that no refundable state credits apply and never considered Idaho's refundable grocery credit. Every full-year resident receives that credit, and it is $155 here." +us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,Correctly treated the grocery credit as refundable but used the stale $120 under-65 amount instead of the 2026 amount of $155. +us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied a $100 per-person grocery credit, which is the pre-2023 rate. The 2026 base is $155." +us,scenario_053,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Said no state refundable credits apply and omitted Idaho's universal refundable grocery credit, which pays $155 to this under-65 full-year resident." +us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,Used $120 per eligible resident for the refundable grocery credit. The 2026 per-person base is $155. +us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,Used the old $100 under-65 grocery credit amount instead of the 2026 base of $155. +us,scenario_053,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Applied a $100 per-resident grocery credit, a figure superseded twice. The 2026 base amount is $155." +us,scenario_053,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,Asserted that Idaho's grocery credit is nonrefundable and returned 0. The grocery credit is a refundable state credit worth $155 for this one-person household. +us,scenario_053,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Labeled $120 the 'post-2025 amount' of the refundable grocery credit. The 2026 per-person base is actually $155, so the model applied the stale 2023-2025 rate." +us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded that the household facts indicate no refundable Idaho credits, which overlooks the grocery credit. That credit needs only full-year Idaho residency and pays $155 here." +us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Looked only for child-related refundable credits and returned 0, which misses Idaho's per-person refundable grocery credit of $155." +us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,Correctly identified the grocery credit as the only refundable Idaho credit but used $120 instead of the 2026 nonelderly amount of $155. +us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Claimed that income is too high for the Idaho grocery credit. The credit has no income test, so this full-year resident receives the full $155." +us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the grocery credit at the outdated $120 per person; the 2026 base is $155. Its reference to a phaseout threshold is also wrong, since the credit has no income phaseout." +us,scenario_053,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Returned 0 without considering Idaho's refundable grocery credit, which pays $155 to this under-65 full-year resident." +us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,Correctly treated the grocery credit as refundable but used the pre-2023 $100 non-elderly amount instead of the 2026 base of $155. +us,scenario_053,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,Paid a $100 grocery credit to a non-elderly adult. The 2026 per-person amount is $155. +us,scenario_053,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,Used $120 for the non-senior refundable grocery credit instead of the 2026 base of $155. +us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Concluded that no state refundable credits apply, which omits Idaho's universal refundable grocery credit of $155 for this resident." +us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,Applied the grocery credit at 'about $120' for one resident. The 2026 per-person base is $155. +us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_refundable_credits, so there is no answer to score. The correct derivation is the $155 Idaho grocery credit for one under-65 full-year resident." +us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly checked full-year eligibility and the SNAP and incarceration reductions, but used the stale $120 base. The 2026 under-65 amount is $155." +us,scenario_053,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,Considered only child-based credits and the property tax credit and never applied Idaho's refundable grocery credit. That credit pays $155 to every full-year resident regardless of children or rent. +us,scenario_053,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,Stated a 2026 grocery credit of $120 per qualified member. The 2026 per-person base is $155. +us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Checked only for a state EITC or child tax credit and concluded that no refundable credits apply. That misses Idaho's refundable grocery credit, worth $155 for this household." +us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Said Idaho provides no refundable credit for this household and overlooked the refundable grocery credit. Every full-year resident receives it, and it is $155 here." us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only a CHIP upper-income threshold and concluded that income below it establishes eligibility. It omitted the Medicaid-exclusion step: the child’s low income qualifies them for Medicaid under the OLDER_CHILD category, which makes CHIP eligibility false." us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated being below North Carolina’s CHIP income limit and lacking other coverage as sufficient. It failed to test Medicaid eligibility first; the 10-year-old is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP. us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from low income and the absence of employer-sponsored insurance. It omitted that low income places Child 1 in Medicaid’s OLDER_CHILD pathway, and Medicaid eligibility disqualifies the child from CHIP." @@ -4175,39 +4577,41 @@ us,scenario_054,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_elig us,scenario_054,child2_wic_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model placed the age-2 child in WIC's child category correctly, then asserted the household ""exceeds WIC eligibility limits"" without applying the actual screen: WIC tests gross income against 185% of the federal poverty guideline, roughly $49,000 for a three-person household in 2026, and this household's income is at most the head's $17,100 of self-employment income (and $0 in the engine's WIC income measure) — under a third of the limit. It also skipped adjunctive eligibility, which qualifies the child through the household's Medicaid/SNAP/TANF eligibility even before the income screen is reached, and it disregarded the head's listed WIC receipt, which itself establishes the household was certified under the 185% FPL test." us,scenario_054,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model answered from the absence of an explicit WIC-receipt flag on Child 2 rather than running WIC's eligibility test, applying the prompt's ""treat unlisted facts as false"" rule to a computed output instead of to inputs, in direct conflict with the instruction to report eligibility ""not whether they are currently enrolled."" Running the test yields eligibility: age 2 falls inside WIC's child category (ages 1 through 4), and household income far below 185% FPL (about $49,000 for three people) clears the income screen, with the head's own WIC receipt confirming household-level certification." us,scenario_054,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It computed a refundable CTC of $2,190 in its own reasoning and then submitted the EITC alone, dropping the ACTC out of the total entirely. Its EITC of $3,995 has no derivation either: with two qualifying children the household is still in the 40% phase-in, so the credit is 40% × $15,891.93 (self-employment income less the $1,208.07 half-SE-tax deduction) = $6,356.77." -us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Its own reasoning reached EITC $6,317 + ACTC $2,369 = $8,686 and it then discarded that arithmetic to submit $6,269 as a ""representative estimate."" Its component errors were small — using the 92.35% net-earnings figure $15,792 instead of $17,100 less the $1,208.07 half-SE-tax deduction, and omitting the $2,500 floor from the ACTC formula — so substituting an unexplained round number for its own computed sum is what cost it the answer of $6,356.77 + $2,008.79." -us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It asserted that earned income of $15,793 lies above the two-child EITC phase-in completion and guessed the credit at ""~$3,000""; the two-child phase-in runs to roughly $18,100 of earned income for 2026, so the full 40% rate applies and yields $6,356.77 on the correct base of $15,891.93. Its submitted $4,287 matches neither its own $3,000 + $1,994 nor the correct $6,356.77 + $2,008.79." -us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It described the EITC as reduced ""after phase-out region"" to about $4,700; with AGI near $15,892 the household is in the 40% two-child phase-in, far below the head-of-household phase-out start, so the credit is $6,356.77. It also used gross $17,100 rather than $17,100 less the $1,208.07 half-SE-tax deduction for the ACTC base, and submitted a round $6,900 that matches neither of its own stated components." -us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It identified the correct earned income (~$15,893) but then asserted an EITC ""near the maximum plateau"" of $6,164 instead of applying the 40% two-child phase-in rate, which gives $6,356.77 at that income. Its ACTC of $1,914 likewise understates 15% × ($15,891.93 − $2,500) = $2,008.79." -us,scenario_054,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used $15,791.85 — the 92.35% net-earnings-from-self-employment figure — as the earned income base for both credits; earned income here is self-employment income less the deductible half of SE tax, $17,100 − $1,208.07 = $15,891.93. The $100.08 base shortfall propagates to $40.03 of EITC and $15.01 of ACTC, the full $55.04 gap in its answer." -us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Its EITC is exactly right at 40% × $15,891.93 = $6,356.77, but it applied the pre-TCJA $3,000 ACTC earned-income floor instead of the $2,500 floor in force for 2026. That single substitution understates the refundable CTC by 15% × $500 = $75.00, the entire gap from $2,008.79." -us,scenario_054,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used gross self-employment income of $17,100 as earned income instead of $17,100 less the $1,208.07 deductible half of SE tax, and then capped the ACTC at the pre-TCJA $1,000-per-child level with a $3,000 floor. For 2026 the refundable CTC is 15% × ($15,891.93 − $2,500) = $2,008.79 against a $1,700-per-child cap that does not bind, and the EITC is 40% × $15,891.93 = $6,356.77." -us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It double-reduced the earned income base to $14,584 by applying the 0.9235 net-earnings factor and then also subtracting half of self-employment tax; only the half-SE-tax deduction applies, leaving $15,891.93. It compounded that with the repealed $3,000 ACTC floor rather than $2,500, so both components fall short of $6,356.77 and $2,008.79." -us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It asserted an EITC of $6,370 and an unexplained $497 residual of CTC without deriving either. The correct components are 40% × $15,891.93 = $6,356.77 of EITC and 15% × ($15,891.93 − $2,500) = $2,008.79 of ACTC; its total omits roughly three-quarters of the refundable CTC." -us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It derived the EITC exactly right at $6,356.77 from earned income of $15,891.92 but applied the repealed $3,000 ACTC earned-income floor instead of the $2,500 floor that governs 2026. The refundable CTC is 15% × ($15,891.93 − $2,500) = $2,008.79, making its $1,933.79 short by exactly $75.00." -us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its EITC of $6,356.80 is correct, but it computed the ACTC as 15% × ($15,892 − $3,000) using the pre-TCJA earned-income floor and cited a $1,000-per-child refundable cap. The 2026 floor is $2,500 and the refundable cap is $1,700 per child, giving 15% × ($15,891.93 − $2,500) = $2,008.79." -us,scenario_054,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It declared refundable credits to be zero with no computation at all. The EITC and the ACTC are payable in full with no tax liability, and $15,891.93 of earned income with two qualifying children produces $6,356.77 of EITC plus $2,008.79 of refundable CTC." -us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It offered no derivation, but $8,291 is the rounded value of 40% × $15,891.93 plus 15% × ($15,891.93 − $3,000) — the correct EITC combined with the pre-TCJA $3,000 ACTC earned-income floor. The 2026 floor is $2,500, so the refundable CTC is $2,008.79 rather than $1,933.79." -us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $8,290.56 is exactly 40% × $15,891.93 plus 15% × ($15,891.93 − $3,000), so it got the EITC and the earned-income base right and applied the repealed $3,000 ACTC floor. Using the 2026 floor of $2,500 raises the refundable CTC from $1,933.79 to $2,008.79." -us,scenario_054,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It gave no arithmetic, but $8,291 is the rounded result of the correct $6,356.77 EITC plus an ACTC computed as 15% of earned income over the pre-TCJA $3,000 floor. The floor for 2026 is $2,500, which makes the refundable CTC $2,008.79 and the total $8,365.56." -us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It computed only the refundable CTC and omitted the $6,356.77 EITC from the total, despite two qualifying children and $17,100 of self-employment earnings. Its $2,190 also applied the 15% formula to gross $17,100 instead of earned income of $15,891.93 (self-employment income less the $1,208.07 half-SE-tax deduction), for which the ACTC is $2,008.79." -us,scenario_054,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It treated the 2026 CTC as fully refundable at $2,300 per child; the credit is $2,200 per child with the refundable portion capped at $1,700 and, binding here, limited to 15% of earned income over $2,500 = $2,008.79. It also double-reduced earned income to $14,584 by applying the 0.9235 factor and the half-SE-tax deduction together, understating the EITC below the correct 40% × $15,891.93 = $6,356.77." -us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It submitted zero while acknowledging two qualifying children and self-employment income, treating the facts as insufficient to support a positive credit. Net earnings from self-employment are earned income for both credits, and $15,891.93 of earned income yields a 40% phase-in EITC of $6,356.77 plus an ACTC of 15% × ($15,891.93 − $2,500) = $2,008.79." -us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It declined to compute, asserting the facts lacked EITC or ACTC triggers; head-of-household status with two qualifying children aged 10 and 2 and $17,100 of self-employment income fully determines both credits. Earned income of $17,100 − $1,208.07 = $15,891.93 produces $6,356.77 of EITC and $2,008.79 of refundable CTC." -us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied both credit formulas to gross self-employment income of $17,100, failing to subtract the $1,208.07 deductible half of self-employment tax that reduces earned income to $15,891.93. On the correct base the EITC is $6,356.77 rather than $6,840 and the ACTC is $2,008.79 rather than $2,190." -us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used gross $17,100 as earned income for both the 40% EITC phase-in and the 15% ACTC formula, omitting the $1,208.07 deduction for half of self-employment tax. Earned income is $15,891.93, giving $6,356.77 of EITC and $2,008.79 of ACTC instead of $6,840 and $2,190." -us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $5,834 EITC and $1,813 ACTC both derive from an earned income of $14,584, which double-counts the reduction by applying the 0.9235 net-earnings factor and then subtracting half the SE tax on top of it. Only the half-SE-tax deduction applies, so earned income is $15,891.93 and the credits are $6,356.77 and $2,008.79." -us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It priced the EITC off ""one qualifying child and one young child""; both children, aged 10 and 2, are qualifying children, so the two-child 40% phase-in applies to $15,891.93 and yields $6,356.77. Its $5,978 total falls below the EITC alone and leaves no room for the $2,008.79 refundable CTC." -us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the 92.35% net-earnings figure $15,792 rather than $17,100 less the $1,208.07 half-SE-tax deduction, and applied the pre-TCJA $3,000 ACTC earned-income floor instead of $2,500. Correcting the base gives $6,356.77 of EITC and correcting the floor gives $2,008.79 of ACTC, in place of its $6,317 and $1,919." -us,scenario_054,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly applied the pre-TCJA ACTC formula, 15% of earned income over $3,000, when the 2026 floor is $2,500, and it based both credits on the 0.9235 net-earnings figure $15,792 rather than $17,100 less the $1,208.07 half-SE-tax deduction. The correct components are $6,356.77 of EITC on $15,891.93 and $2,008.79 of ACTC." -us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It asserted the CTC reverts to $1,000 per child for 2026 with a $3,000 ACTC floor, capping the refundable portion at $2,000; for 2026 the credit is $2,200 per child with $1,700 refundable and the refundable amount here is 15% × ($15,891.93 − $2,500) = $2,008.79. It also used gross $17,100 for the EITC instead of $15,891.93 after the $1,208.07 half-SE-tax deduction, inflating that credit to $6,840 from $6,356.77." -us,scenario_054,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the pre-TCJA $1,000-per-child refundable CTC for a flat $2,000 and ignored the 15%-of-earned-income-over-$2,500 limitation, which binds here at $2,008.79. It also ran the 40% EITC phase-in on gross $17,100 rather than on earned income of $15,891.93 after the $1,208.07 half-SE-tax deduction, overstating the EITC by $483.23." -us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for federal_refundable_credits and no explanation accompanied the submission, so the response failed the output contract rather than a tax rule. The required answer is 40% × $15,891.93 = $6,356.77 of EITC plus 15% × ($15,891.93 − $2,500) = $2,008.79 of refundable CTC." -us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"Its reasoning derived $4,532 of EITC plus $3,400 of ACTC, then broke off with ""let me reconsider"" and submitted $672, a number unrelated to any component it stated. Both components were also wrong: earned income is $15,891.93 rather than $13,093, and the ACTC is limited to 15% × ($15,891.93 − $2,500) = $2,008.79 rather than the full $1,700-per-child cap." -us,scenario_054,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied both credit formulas correctly — 40% phase-in and 15% over a $2,500 floor — but to gross self-employment income of $17,100, omitting the $1,208.07 deduction for half of self-employment tax that reduces earned income to $15,891.93. That single base error inflates the EITC by $483.23 and the ACTC by $181.21, exactly the $664.44 overstatement in its total." -us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed self-employment income does not qualify as earned income for the EITC and that SSTB status excludes it; net earnings from self-employment are earned income under IRC §32(c)(2), and the specified-service-trade-or-business designation affects only the §199A qualified business income deduction. It also treated zero tax liability as barring the refundable CTC, when the ACTC is precisely the portion payable without any liability, worth $2,008.79 here alongside $6,356.77 of EITC." -us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It limited the ACTC to $375 on the theory that the nonrefundable CTC first absorbed regular tax; with AGI of $15,891.93 against the head-of-household standard deduction, taxable income and tax before credits are zero, so no CTC is used nonrefundably and the full 15% × ($15,891.93 − $2,500) = $2,008.79 is refundable. Its EITC of $5,757 also came from an invented $15,522 base rather than 40% × $15,891.93 = $6,356.77." +us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model set the EITC at about $3,995, far below the two-child 40% phase-in amount of 40% x $15,891.93 = $6,356.77. It then left out the refundable CTC it had just computed ($2,190), reporting only the EITC as the total. Even that ACTC used gross $17,100 instead of earned income net of half the SE tax, so the correct refundable CTC is $2,008.79." +us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model used the 92.35% net-earnings figure ($15,792) as earned income instead of SE profit minus half the SE tax ($15,891.93). It also computed the ACTC as 15% of the whole amount without subtracting the $2,500 threshold. It then threw away its own $8,686 sum and submitted $6,269, which roughly equals the EITC alone and leaves out the $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model confused itself about where $15,793 falls on the EITC schedule and settled on an EITC of about $2,293 (its $4,287 total minus its $1,994 ACTC). Earned income of $15,891.93 is inside the 40% two-child phase-in, so the EITC is $6,356.77. It also misstated SE tax as $2,233 and used the 92.35% figure instead of profit minus half the SE tax." +us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household in the EITC phase-out region and cut the EITC to about $4,700. At $15,891.93 of earned income the credit is still phasing in at 40%, so it is $6,356.77. It also based the ACTC on gross $17,100 (about $2,190) instead of earned income net of half the SE tax, which gives $2,008.79." +us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly found earned income of about $15,893. It then treated the EITC as near the plateau and used $6,164, instead of the 40% phase-in amount of $6,356.77. Its ACTC of $1,914 also misapplies its own formula: 15% x ($15,891.93 - $2,500) = $2,008.79." +us,scenario_054,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used the 92.35% net-SE-earnings base ($15,791.85) as earned income. For the EITC and ACTC, self-employment earned income is net profit minus the deductible half of SE tax, $15,891.93. That understated the EITC ($6,316.74 vs $6,356.77) and the ACTC ($1,993.78 vs $2,008.79)." +us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model got earned income ($15,891.93) and the EITC ($6,356.77) exactly right. It then used the pre-TCJA $3,000 ACTC threshold instead of the permanent $2,500 threshold in effect for 2026. That makes the refundable CTC $1,933.79 instead of $2,008.79." +us,scenario_054,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income without subtracting the deductible half of SE tax, which inflated the EITC to $6,840 instead of $6,356.77. For the ACTC it used the expired $1,000-per-child credit and the $3,000 threshold. Under 2026 law the ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79, within the $1,700-per-child cap." +us,scenario_054,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model took the 92.35% SE-tax base ($15,791.85) as earned income instead of SE profit minus half the SE tax ($15,891.93). That understated the EITC by about $40 and the ACTC by about $15, giving $8,311 instead of $8,365.56." +us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted half the SE tax from the already-reduced 92.35% figure, arriving at $14,584. The correct earned income is $17,100 - $1,208.07 = $15,891.93, so this reduction was applied twice. It also used the obsolete $3,000 ACTC threshold instead of $2,500, understating both the EITC ($5,833.51 vs $6,356.77) and the ACTC ($1,737.57 vs $2,008.79)." +us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model's EITC of $6,370 is close to the correct $6,356.77. Its total implies a refundable CTC of only about $497, when the ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79. That undercounts the refundable CTC by about $1,500." +us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model computed earned income ($15,891.92) and the EITC ($6,356.77) correctly. It then applied the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold, getting $1,933.79 instead of $2,008.79." +us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's EITC is correct (40% x $15,892). It applied the expired pre-TCJA ACTC rules: a $3,000 threshold and a $1,000-per-child credit. Under the 2026 rules ($2,500 threshold, $1,700-per-child refundable cap) the ACTC is $2,008.79, not $1,933.80." +us,scenario_054,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model reported zero refundable credits. It missed that $17,100 of self-employment profit is earned income that qualifies a head of household with two qualifying children for a $6,356.77 EITC and a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $8,291 matches $6,356.77 EITC + $1,933.79 ACTC. So it got the correct earned income ($15,891.93) but used the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold, which gives $2,008.79." +us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $8,290.56 is exactly $6,356.77 EITC plus an ACTC of 15% x ($15,891.93 - $3,000). It used the obsolete $3,000 ACTC threshold instead of the 2026 $2,500 threshold, which gives $2,008.79." +us,scenario_054,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $8,291 matches the correct EITC of $6,356.77 plus an ACTC computed with the pre-TCJA $3,000 threshold ($1,933.79). The 2026 $2,500 threshold gives $2,008.79." +us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,other,False,"The model reported only the refundable CTC and left out the EITC entirely, even though the head has $15,891.93 of earned income and two qualifying children, worth $6,356.77. Its ACTC also used gross $17,100 instead of earned income net of half the SE tax, giving $2,190 instead of $2,008.79." +us,scenario_054,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model treated the entire OBBBA CTC ($2,300 per child) as refundable. The refundable portion is limited to 15% of earned income over $2,500 and capped at $1,700 per child, which gives $2,008.79 here. It also reduced earned income twice to $14,584, understating the EITC ($5,834 vs $6,356.77)." +us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model refused to credit any refundable amount even though the prompt says to assume filing. Self-employment profit of $17,100 with two qualifying children produces a $6,356.77 EITC and a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said it needed explicit eligibility indicators and returned 0, ignoring the instruction to assume filing and take-up. Self-employment earnings of $15,891.93 (net of half SE tax) with two qualifying children generate a $6,356.77 EITC and a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 of self-employment profit as earned income for both credits. Self-employment earned income excludes the deductible half of SE tax ($1,208.07), which gives $15,891.93. That overstated the EITC ($6,840 vs $6,356.77) and the ACTC ($2,190 vs $2,008.79)." +us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model based both credits on gross $17,100 instead of earned income after subtracting the deductible half of SE tax ($15,891.93). That inflated the EITC to $6,840 (correct: $6,356.77) and the refundable CTC to $2,190 (correct: $2,008.79)." +us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $5,834 EITC and $1,813 ACTC equal 40% and 15% x (E - $2,500) with E = $14,584. That base subtracts half the SE tax from the 92.35% figure, a double reduction. The correct earned income is $17,100 - $1,208.07 = $15,891.93, giving $6,356.77 + $2,008.79." +us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model called the household one qualifying child plus one young child, when both children (ages 10 and 2) are qualifying children for the EITC and CTC. Its $5,978 is less than the two-child 40% phase-in EITC alone. The correct answer is $6,356.77 EITC plus a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model knowingly applied the pre-TCJA $3,000 ACTC threshold instead of the 2026 $2,500 threshold. It also used the 92.35% figure ($15,792) as earned income instead of SE profit minus half the SE tax ($15,891.93). Both errors understate the credits: EITC $6,317 vs $6,356.77, ACTC $1,919 vs $2,008.79." +us,scenario_054,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model computed the ACTC as 15% of earned income over $3,000. The 2026 threshold is $2,500. It also used $15,792 (the 92.35% SE base) as earned income instead of $15,891.93 (profit minus half the SE tax), so both the EITC and the ACTC ($1,919 vs $2,008.79) came out too low." +us,scenario_054,federal_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income without subtracting half the SE tax, inflating the EITC to $6,840 instead of $6,356.77. It also assumed the CTC reverted to $1,000 per child with a $3,000 threshold in 2026. Under OBBBA's permanent rules, the refundable CTC is 15% x ($15,891.93 - $2,500) = $2,008.79." +us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model based the EITC on gross $17,100 instead of earned income net of half the SE tax ($15,891.93), overstating it at $6,840 vs $6,356.77. It also capped the ACTC at a reverted $1,000 per child with a $3,000 threshold. The 2026 rules ($2,500 threshold, $1,700-per-child cap) give $2,008.79." +us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model took gross $17,100 as earned income for the EITC ($6,840) instead of $15,891.93 net of half the SE tax ($6,356.77). It also set the refundable CTC at a flat $1,000 per child, when the 2026 ACTC is 15% x ($15,891.93 - $2,500) = $2,008.79." +us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so there was no answer to score." +us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model worked out $7,932, using an understated earned income of $13,093 and a full $3,400 ACTC that ignores the 15%-over-$2,500 earnings limit. It then abandoned that figure and submitted an unrelated $672. The correct total is $6,356.77 EITC plus $2,008.79 ACTC, both based on $15,891.93 of earned income." +us,scenario_054,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used gross $17,100 as earned income for both credits. The deductible half of SE tax ($1,208.07) must come off first, leaving $15,891.93. That overstated the EITC ($6,840 vs $6,356.77) and the refundable CTC ($2,190 vs $2,008.79)." +us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly claimed that SSTB self-employment income does not count as earned income for the EITC and ACTC. SSTB status matters only for the QBI deduction. Net SE earnings of $15,891.93 (after half the SE tax) qualify the household for a $6,356.77 EITC and a $2,008.79 refundable CTC." +us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"The model limited the ACTC to $375 on the premise that the CTC first offsets regular tax. After the head-of-household standard deduction, regular tax is $0, so the full 15% x ($15,891.93 - $2,500) = $2,008.79 is refundable. It also understated earned income at $15,522, and its $5,757 EITC does not even equal 40% of that base; the correct EITC is $6,356.77." us,scenario_054,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that household income exceeds the free-meal threshold, but $17,100 for a three-person household is 63% of the 2026 federal poverty guideline — far below the 130% FPG free-meal cutoff (roughly $35,000 for a household of three) and even below the 185% reduced-price line. It applied a nonexistent income ceiling and never checked the SNAP categorical-eligibility pathway, which independently confers free meals on the 10-year-old school-age child regardless of the income test." us,scenario_054,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model made free-meal eligibility contingent on an explicit eligibility flag or a positively stated SNAP amount in the household facts, when the benchmark required it to compute SNAP itself ($524.80/month here) and to apply the school-meal income test to the given $17,100 of self-employment income. It never evaluated either qualifying pathway — the 63%-of-FPG income share against the 130% free-meal threshold, or SNAP/TANF categorical eligibility for the 10-year-old K-12 student — and defaulted to 0 from the absence of a stated indicator rather than from a rule." us,scenario_054,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model classified the head as a ""non-disabled childless adult without dependents"" when the household explicitly contains the head's two children (ages 10 and 2), which places the head in the parent/caretaker-relative category the engine assigned. It compounded this by asserting North Carolina has not expanded Medicaid, when NC implemented expansion to 138% FPL on December 1, 2023 under S.L. 2023-7 and that coverage is in force for 2026." @@ -4216,21 +4620,22 @@ us,scenario_054,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligib us,scenario_054,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model asserted the head's income exceeds NC's adult Medicaid threshold, but $17,100 of self-employment income for a household of three is roughly $15,900 of MAGI after the one-half self-employment tax deduction — 0.58 x FPL, less than half of North Carolina's 138% FPL expansion limit and well under the parent/caretaker limit. Its answer is consistent with comparing gross earnings against a limit near or below 100% FPL for a single person rather than the three-person FPL." us,scenario_054,head_medicaid_eligible,minimax-m3,llm_error,other,False,"The model correctly derived the answer in its own explanation — NC covers adults to 138% FPL, ~$26,000 for a family of three, and $17,100 falls below it — and wrote ""Let me set to 1,"" then submitted value = 0. The submitted number contradicts its own completed income comparison, so the failure is in transcribing the derived result into the output field, not in the eligibility analysis." us,scenario_054,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model identified the head as a parent and computed income at roughly 65% FPL, then discarded that result by asserting North Carolina has not expanded Medicaid and that its parent pathway demands ""very low income"" — NC expanded to 138% FPL effective December 1, 2023 (S.L. 2023-7), and the parent/caretaker category the engine applied is satisfied at 0.58 x FPL. It abandoned a qualifying pathway it had already established on a false statement about NC's expansion status." -us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a household-level test it invented — ""WIC typically requires that at least one child in the household be under age 5"" — instead of the per-person categorical test, so it credited the 40-year-old head with Child 2's age-2 category. It compounded this by adding a nonexistent asset screen ($14,000 ""within typical asset limits""); WIC has no asset test, and the head fails at the categorical prong because pregnant, postpartum, and breastfeeding are all unlisted and therefore false." -us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model created a ""qualifying caretaker"" category from the presence of a 2-year-old, but WIC certifies only infants, children aged 1 through 4, and pregnant, postpartum, or breastfeeding women — a parent qualifies through her own pregnancy or postpartum status, never through her child's. With those statuses unlisted and thus false, the head's WIC category is NONE and the income test is never reached." -us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model recited the correct category list (pregnant, postpartum, breastfeeding, infant, or child under 5) and then failed to apply it to the head, substituting ""the household has a child under 5"" plus the receipt flag for a personal categorical status she does not hold. The 185% FPL income test it ran is the second prong; the head is excluded at the first, since she is 40 and no pregnancy, postpartum, or breastfeeding fact is listed." -us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model back-inferred a postpartum/breastfeeding status from the ""receives wic"" line (""as a postpartum/breastfeeding-type participant per stated receipt""), manufacturing a categorical qualifier the prompt directs to be false as an unlisted boolean. PolicyEngine assigns the head WIC category NONE on her age-40, non-pregnant, non-postpartum record, so the below-185% FPL income finding never applies to her." -us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the head as a ""categorically eligible caretaker/family member"" because a 2-year-old lives with her, and then used the receipt flag as ""adjunct eligibility"" — but WIC adjunctive eligibility (through SNAP, TANF, or Medicaid participation) substitutes only for the income test and never for the categorical test. The head holds no WIC category at age 40 with pregnancy, postpartum, and breastfeeding all unlisted, so she fails independently of the income finding." -us,scenario_054,head_wic_eligible,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model resolved eligibility on the income prong alone — receipt plus a 3-person household under 185% FPL — and skipped the categorical screen entirely. Income eligibility is necessary but not sufficient: the head is 40 and not pregnant, postpartum, breastfeeding, an infant, or a child under 5, so her WIC category is NONE." -us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model reasoned circularly from the input flag — ""Head receives WIC, so eligible"" — when the question asks for eligibility under PolicyEngine rules rather than current enrollment, and receipt is a takeup input that feeds no eligibility variable. Evaluating the actual rule, the head's WIC category is NONE at age 40 with no pregnancy, postpartum, or breastfeeding status listed." -us,scenario_054,head_wic_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model combined the receipt flag with an income check below the WIC limit and stopped, never testing whether the head occupies a WIC certification category. A 40-year-old woman with no pregnancy, postpartum, or breastfeeding fact listed carries category NONE, which fails eligibility before income is considered." -us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model equated the listed ""receives wic"" input with an eligibility determination (""confirming their eligibility""), directly contradicting the instruction to report eligibility under PolicyEngine rules and not enrollment. Applied to the rule itself, the head has no WIC category — she is 40, not pregnant, not postpartum, not breastfeeding — so eligibility is False." -us,scenario_054,head_wic_eligible,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model rested on the receipt flag plus income eligibility and never applied the categorical prong. WIC certifies only infants, children aged 1 through 4, and pregnant, postpartum, or breastfeeding women; the 40-year-old head matches none of these with those statuses unlisted and therefore false." -us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model identified the decisive gap — ""Head is pregnant/postpartum not stated"" — and then overrode it, treating the receipt flag and the presence of a young child as grounds to deem her eligible. The prompt's rule that unlisted statuses are false resolves that gap against eligibility: with no pregnancy, postpartum, or breastfeeding status, the head's WIC category is NONE." -us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model derived eligibility solely from the enrollment fact (""explicitly says Head receives WIC, so Head is eligible""), which the question expressly excludes as the basis for the answer. Under the rule, the head fails the categorical test at age 40 with pregnancy, postpartum, and breastfeeding all absent from the facts." -us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model restated the receipt flag as the whole justification and performed no categorical or income analysis. WIC eligibility requires a personal category — infant, child aged 1 through 4, pregnant, postpartum, or breastfeeding — and the 40-year-old head holds none, so PolicyEngine returns False for her while returning True only for the 2-year-old." -us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model treated current receipt as confirmation of eligibility, collapsing enrollment into the eligibility determination the question asked for. The categorical screen decides this case: the head is 40 with no pregnancy, postpartum, or breastfeeding status listed, giving her WIC category NONE regardless of the household's near-zero income." -us,scenario_054,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted as fact that ""the head is pregnant, postpartum, or breastfeeding for WIC purposes,"" fabricating the one status that would confer a category when the prompt directs unlisted booleans to be false. Without that status, the 40-year-old head's WIC category is NONE and she fails before the income limit she cited is applied." +us,scenario_054,head_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model counted having a child under 5 in the household as meeting the head's own WIC categorical requirement. In fact, an adult qualifies only if she is personally pregnant, postpartum or breastfeeding, and the head is none of these. The income test (185% of FPL) and asset test the model applied are irrelevant because the categorical test already fails, and WIC has no asset test at all." +us,scenario_054,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model called the head a 'qualifying caretaker' because Child 2 is age 2. WIC has no caretaker category: a parent applies on a child's behalf but is not a participant unless she is pregnant, postpartum or breastfeeding, and the head has none of those statuses." +us,scenario_054,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly listed the WIC categories (pregnant, postpartum, breastfeeding, infant, child under 5) but then credited the household's under-5 child to the head. The head is a 40-year-old with no pregnancy, postpartum or breastfeeding status, so she fits no category. The model also treated her reported WIC receipt as proof of eligibility." +us,scenario_054,head_wic_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model inferred a postpartum or breastfeeding status for the head from the 'receives wic' flag. The prompt sets every unlisted status input to false, so the head is neither postpartum nor breastfeeding and fails the WIC categorical test regardless of income below 185% of FPL." +us,scenario_054,head_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the head as a 'categorically eligible caretaker' of the 2-year-old and cited adjunctive eligibility. Adjunctive eligibility satisfies only the WIC income test, never the categorical test. An adult is categorically eligible only when pregnant, postpartum or breastfeeding, and the head is none of these." +us,scenario_054,head_wic_eligible,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model based eligibility only on WIC receipt and household income below 185% of FPL, and it skipped the categorical requirement. A 40-year-old head who is not pregnant, postpartum or breastfeeding has no WIC category, so income is never reached." +us,scenario_054,head_wic_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated the 'receives wic' flag as proof of eligibility. That flag records the household's WIC take-up, which runs through the 2-year-old child. The head herself fails the categorical test because she is not pregnant, postpartum or breastfeeding." +us,scenario_054,head_wic_eligible,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model relied on the reported WIC receipt and the income limit and never applied the categorical test. An adult qualifies only when pregnant, postpartum or breastfeeding, and the head has none of those statuses, so she is ineligible whatever her income." +us,scenario_054,head_wic_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model took the head's reported WIC receipt as confirmation of eligibility and skipped the individual categorical test. The head is 40 and not pregnant, postpartum or breastfeeding, so she is in no WIC participant category. Only the 2-year-old qualifies." +us,scenario_054,head_wic_eligible,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model combined reported WIC receipt with income eligibility and never checked whether the head fits a WIC category. She is not pregnant, postpartum or breastfeeding, so the categorical test fails before income is considered." +us,scenario_054,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model noted that no pregnancy or postpartum status was stated, which should have ended the analysis because unlisted statuses are false and an adult qualifies for WIC only in those statuses. It then overrode that finding with the receipt flag and the young child, and neither of those confers categorical eligibility on the head." +us,scenario_054,head_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model took 'Head receives WIC' as meaning the head is WIC-eligible. Eligibility requires the head herself to be pregnant, postpartum or breastfeeding, and she is none of these; the receipt comes through the eligible 2-year-old." +us,scenario_054,head_wic_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model asserted eligibility only because the head is listed as receiving WIC and applied no categorical test. A 40-year-old adult with no pregnancy, postpartum or breastfeeding status fits no WIC category." +us,scenario_054,head_wic_eligible,gpt-6-luna,llm_error,categorical_eligibility,False,"The model treated reported WIC receipt as proof of eligibility. WIC eligibility requires a personal categorical status, and the non-pregnant, non-postpartum, non-breastfeeding head does not have one." +us,scenario_054,head_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model said current WIC receipt confirms eligibility and never applied the categorical requirement. An adult qualifies only when pregnant, postpartum or breastfeeding, and the head has none of these statuses." +us,scenario_054,head_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model made up a pregnant, postpartum or breastfeeding status for the head, but the household facts contain no such input and the prompt sets unlisted statuses to false. Without that status the head has no WIC category and is ineligible." us,scenario_054,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages and that self-employment income is excluded from employee payroll tax, but then replaced the resulting zero with an invented $1,084 residual benchmark. No such residual employee payroll tax applies in North Carolina, so the stated computation yields $0." us,scenario_054,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the required payroll_tax output entirely. The derivation requires $0 because the household has no wages subject to employee Social Security, Medicare, Additional Medicare, or mandatory state employee payroll taxes." us,scenario_054,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the 185% reduced-price ceiling as sufficient and failed to evaluate the superior free-meals tier. At 63% of the federal poverty guideline and with SNAP/TANF categorical eligibility, the household receives free meals, so reduced-price eligibility is 0." @@ -4241,82 +4646,91 @@ us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,Fals us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's own formula produces approximately $2,416, but it submitted $1,596 without a valid computation connecting the two amounts. Excluding Additional Medicare Tax does not reduce the ordinary 12.4% Social Security plus 2.9% Medicare SECA liability, which equals $2,416.15." us,scenario_054,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly described a result near $2,416.18 but submitted the inconsistent amount $2,419.95. Exact arithmetic using the unrounded base gives $15,791.85 × 15.3% = $2,416.15305, rounded to $2,416.15." us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the deduction for one-half of self-employment tax from the SECA liability and then produced a final number inconsistent with its intermediate figures. That deduction reduces adjusted gross income for income-tax purposes; it does not reduce self-employment tax, which remains $17,100 × 92.35% × 15.3% = $2,416.15." -us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"It ran the deduction chain correctly to net income of ~$929 but invented a maximum allotment of ~$820/month by projecting a COLA off a misremembered FY2025 base of $768 (the actual FY2025 figure was $766). The real three-person maximum is $785 through September 2026 and $803 from October, so its $540 monthly benefit overstates the true $505.70/$525 split." -us,scenario_054,snap,claude-fable-5.1,llm_error,period_annualization,False,"Every input was right — $1,425 gross, $285 earned-income deduction, $209 standard deduction, net $931, $785 maximum allotment, $505.70 monthly benefit — but it multiplied that single month by twelve. Calendar 2026 straddles two federal fiscal years: the October 2026 COLA raises the three-person maximum to $803, lifting the last three months to about $525 each and the annual total to $6,125.69." -us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It disqualified the household on the $14,000 bank balance, but North Carolina operates broad-based categorical eligibility, which eliminates the SNAP resource test entirely, and this household is additionally categorically eligible through TANF non-cash assistance. With no asset test applying, the household receives $6,125.69." -us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted half of self-employment tax from the $17,100 before counting income, cutting gross monthly income from $1,425 to $1,324; SNAP counts gross self-employment receipts less actual allowable business expenses (zero here), never the income-tax deduction for half of SE tax. It then abandoned its own $6,456 arithmetic and submitted an unexplained $7,152 attributed to ""FY2026 COLA estimates.""" -us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"Its stated chain was essentially correct — net income ~$923, $785 maximum allotment, ~$508 monthly, $6,096 annual — and it correctly noted that NC broad-based categorical eligibility waives the asset test. It then discarded that result for an unsupported ""refined estimate"" of $707/month, submitting $8,484 in place of its own computation." -us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It asserted an $800/month three-person maximum and a $583 monthly benefit without showing the subtraction. Applying its own stated deductions gives net income of $931 and a benefit of $785 − $279.30 = $505.70 for January–September, rising to about $525 in October–December on the $803 FY2027 allotment." -us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a 40% automatic cost-of-doing-business deduction to the self-employment income, cutting countable gross from $1,425 to $855/month; SNAP allows only actual documented business expenses, which are zero here, so gross monthly income stays at $1,425 and net income at $931. Its own text then concluded $7,548 while it submitted $5,532, so the number bears no relation to either its stated derivation or the correct one." -us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"It reached roughly the right net income (~$900–950) and named a $768 maximum, then wrote that subtracting a $280 contribution leaves ""roughly $350/month"" — that subtraction yields $488, not $350. Using the correct $785 allotment and $279.30 contribution gives $505.70/month for January–September and ~$525 thereafter." -us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It removed $2,416 of self-employment tax from the $17,100 before applying SNAP deductions, dropping countable gross to $1,223.65/month and net income to $781 instead of $931; SNAP never deducts self-employment tax from countable income. It compounded this with a $775 maximum allotment rather than $785 rising to $803 in October." -us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $781 three-person maximum allotment and a $202 standard deduction instead of the FY2026 values of $785 and $209, then held that single allotment for all twelve months. The correct benefit is $505.70 for January–September and about $525 from October on the $803 FY2027 maximum." -us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its deduction chain was nearly exact (net $932 against the true $931) but it used a $784 maximum allotment and applied it to all twelve months. The October 2026 COLA raises the three-person maximum to $803, adding roughly $19/month for the final quarter that its $6,052.80 omits." -us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $766 and a $198 standard deduction, producing net income of $942 rather than $931. The FY2026 three-person maximum is $785, rising to $803 in October 2026, so the monthly benefit is $505.70 then ~$525, not $483.40." -us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It submitted $3,720 with no derivation at all, implying a $310 monthly benefit. The correct computation — $1,425 gross less the $285 earned-income deduction and $209 standard deduction gives net $931, and $785 less 30% of that gives $505.70 monthly, rising to ~$525 in October — is consistent with no step producing $310." -us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It carried the stale FY2025 three-person maximum of $766 and a $198 standard deduction, yielding net income of $942 and a $483.40 benefit. FY2026 sets the maximum at $785 with the standard deduction at $209, and the October 2026 COLA raises the maximum to $803 for the final quarter." -us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $747 three-person maximum allotment — below even the FY2025 value of $766 — and a $198 standard deduction that overstates net income at $942 against the correct $931. The FY2026 maximum is $785 and the FY2027 maximum from October 2026 is $803, so the monthly benefit is $505.70 then ~$525, not $464." -us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It zeroed the benefit on the claim that income exceeds the eligibility thresholds, but $1,425 monthly gross is 63–64% of the federal poverty guideline for three, far under the 130% gross test, and net income of $931 is 41–42% of the guideline, far under the 100% net test. The household clears every income screen and receives $6,125.69." -us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $7,548 implies a $629 monthly benefit, which is the $766 FY2025 maximum less 30% of a $456 net income — the figure produced only by applying a 40% self-employment expense allowance that SNAP does not grant when no business expenses are listed. Countable gross stays at $1,425, net income is $931, and the benefit is $505.70 monthly rising to ~$525 in October." -us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It submitted $5,820 without a derivation, implying a $485 monthly benefit consistent with a maximum allotment near $764. The FY2026 three-person maximum is $785, stepping to $803 in October 2026, so with net income of $931 the benefit is $505.70 for nine months and about $525 for three." -us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly applied NC broad-based categorical eligibility but submitted $5,844, a $487 monthly benefit implying a maximum allotment of roughly $766 held flat all year. With net income of $931 the benefit is $785 − $279.30 = $505.70 through September and about $525 from October, when the maximum rises to $803." -us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It used an annual standard deduction of $2,736 ($228/month) instead of $209/month, understating net income at $912, and an annual maximum allotment of $9,600 ($800/month) instead of the true $785 for January–September and $803 for October–December (about $9,474 for the year). Both parameter errors push its $6,317 above the correct $6,125.69." -us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Its deductions were exact — $285 earned-income, $209 standard, net income $931 — but it used a three-person maximum allotment of about $816. The applicable maximum is $785 through September 2026 and $803 from October, giving $505.70 then ~$525 monthly rather than its $537." -us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"It submitted $8,868 with no derivation, a $739 monthly benefit that sits just under the full three-person maximum and therefore omits nearly all of the household contribution. SNAP subtracts 30% of net income — $279.30 on net income of $931 — from the $785 maximum, giving $505.70 monthly and ~$525 once the October 2026 COLA raises the maximum to $803." -us,scenario_054,snap,gpt-5.4-nano,llm_error,other,False,"It declared that household size, income, and asset details were not supplied and defaulted to zero, when the prompt gives three household members, $17,100 of self-employment income, and $14,000 in bank assets. Those inputs fully determine the calculation: net income $931, benefit $505.70/month rising to ~$525 in October, $6,125.69 for the year." -us,scenario_054,snap,gpt-5.5,llm_error,other,False,"It named every correct input — $1,425 gross, $209 standard deduction, net $931, $785 maximum — then wrote that $785 less 30% of $931 is ""575 per month""; that subtraction is $785 − $279.30 = $505.70. The arithmetic slip inflates the annual figure by roughly $830 above the correct $6,125.69." -us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"It zeroed the benefit on the $14,000 bank balance exceeding a North Carolina resource limit, but NC's broad-based categorical eligibility eliminates the SNAP asset test, and the household is separately categorically eligible via TANF non-cash assistance. With no resource test, the benefit is $6,125.69." -us,scenario_054,snap,gpt-5.6-sol,llm_error,period_annualization,False,"It correctly waived the asset test under NC categorical eligibility and computed net income of $931 and a $506 monthly benefit, then multiplied by twelve. The three-person maximum allotment rises from $785 to $803 in October 2026, so the last three months pay about $525 each, bringing the year to $6,125.69." -us,scenario_054,snap,gpt-5.6-terra,llm_error,period_annualization,False,"It applied the FY2026 $785 maximum against net income of $931 for a $506 monthly benefit and annualized that single figure. Calendar 2026 spans two fiscal years: from October the maximum is $803, raising those three months to roughly $525 and the annual total to $6,125.69." -us,scenario_054,snap,gpt-6-astra,llm_error,period_annualization,False,"Its formula was exact for the first nine months — 12 × [785 − 0.30 × (1425 × 0.80 − 209)] = $505.70/month — but it held the FY2026 maximum allotment constant for all twelve. The October 2026 COLA raises the three-person maximum to $803, adding about $57 over the final quarter." -us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It zeroed the benefit claiming both income and assets exceed the limits for a three-person household, but NC broad-based categorical eligibility waives the asset test on the $14,000 balance and raises the gross screen to 200% of poverty, and gross income of $1,425/month is only 63–64% of the poverty guideline. The household is also categorically eligible through TANF non-cash assistance." -us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly identified NC BBCE but used an annual maximum allotment of $9,192 (the FY2025 $766/month) and a $2,376 standard deduction ($198/month) instead of $209. The applicable maximums are $785/month through September 2026 and $803 from October, about $9,474 for the year." -us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"Its eligibility reasoning was right but it used an annual maximum allotment of $9,216 ($768/month) and a $2,448 standard deduction ($204/month), understating both the allotment and net income. FY2026 sets the three-person maximum at $785 and the standard deduction at $209, with the maximum stepping to $803 in October 2026." -us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly applied NC BBCE and reached net income of about $935/month, but used a three-person maximum allotment of ~$811. The correct maximum is $785 for January–September 2026 and $803 for October–December, producing $505.70 and ~$525 monthly rather than its flat $531." -us,scenario_054,snap,inkling,llm_error,asset_resource,False,"It applied a countable-resource limit to the $14,000 bank balance, but North Carolina's broad-based categorical eligibility removes the SNAP asset test, and the household separately qualifies categorically through TANF non-cash assistance. With eligibility established, net income of $931 yields $6,125.69 for the year." -us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"It applied the federal resource limit for households without an elderly or disabled member to the $14,000 bank balance; North Carolina's broad-based categorical eligibility waives that resource test outright, and TANF non-cash assistance provides an independent categorical eligibility pathway. The household therefore receives the full $505.70–$525 monthly benefit." -us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly disregarded the $14,000 balance under categorical eligibility and reached net income of $931, but used a $783 maximum allotment and rounded the 30% contribution up to $280 from $279.30. The maximum is $785 through September 2026 and $803 from October, giving $505.70 and ~$525 rather than a flat $503." -us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It declared gross income of $17,100 to exceed 130% of the poverty line for three, which it itself estimated at ~$26,000 — its own figures show the household passing, not failing. Gross monthly income of $1,425 is 63–64% of the poverty guideline, so the household clears the gross test and receives $6,125.69." -us,scenario_054,snap,ox-alpha,llm_error,period_annualization,False,"It got every parameter right for the first three quarters — $1,425 gross, $285 earned-income deduction, $209 standard deduction, net $931, $785 maximum, $505.70 monthly — and then multiplied by twelve. The FY2027 COLA effective October 2026 raises the three-person maximum to $803, lifting the final three months to about $525 each." -us,scenario_054,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It awarded the full maximum allotment and set that maximum at ~$934/month, both wrong: the three-person maximum is $785 through September 2026 and $803 thereafter, and the benefit is reduced by 30% of net income ($279.30 on net income of $931). Skipping the household contribution and inflating the allotment nearly doubles the correct $6,125.69." -us,scenario_054,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It used a fabricated monthly self-employment figure of $1,579 (the actual $17,100/12 is $1,425) and concluded that net income exceeds the three-person limit. Net income after the $285 earned-income and $209 standard deductions is $931, which is 41–42% of the poverty guideline and far below the 100% net income test." +us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"Its net income (about $929) and $279 contribution were essentially right. It then projected a three-person maximum allotment of about $820 for FY2026 instead of the actual $785. That inflated the monthly benefit to about $540 ($6,480 a year) instead of $505." +us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"It used the correct $785 maximum, $209 standard deduction and $931 net income. It then subtracted an unrounded $279.30 contribution. SNAP rounds 30% of net income up to the next whole dollar ($280), so the allotment is $505 a month ($6,060), not $505.70." +us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It zeroed SNAP because the $14,000 bank balance exceeds a resource limit. It missed that North Carolina's broad-based categorical eligibility (TANF non-cash benefits) makes this household categorically eligible, so the balance does not disqualify it and it receives $505 a month." +us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It cut countable income by half the self-employment tax (to about $1,324 a month), which SNAP does not do: the full $1,425 counts. It also used a $224 standard deduction and a $789 maximum instead of $209 and $785. It then inflated its own $6,456 result to $7,152 with an unexplained 'COLA adjustment'." +us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"Its derivation reached about $508 a month ($6,096), close to the correct $505. It then dropped that result for an unsupported 'refined estimate' of $707 a month, which ignores the $280 expected contribution from $931 net income." +us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It assumed an $800 three-person maximum instead of $785. Its $583 monthly estimate also implies a contribution of only about $217, not the $280 that is 30% of $931 net income rounded up. Together these overstate the annual benefit at $7,000." +us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"It correctly derived $931 net income and the $785 maximum. It then subtracted an unrounded $279.30 contribution to get $505.70. SNAP rounds 30% of net income up to $280, which gives $505 a month and $6,060 a year." +us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a 40% cost-of-doing-business cut to self-employment income, reducing gross income to $855 a month. No business expenses are listed, so the full $1,425 counts. It also used a $228 standard deduction and a $766 maximum. Its derivation gave $7,548, yet it submitted $5,532, which none of its stated arithmetic supports." +us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"It stacked the 20% earned income deduction twice and used the FY2025 $768 maximum instead of $785. It then subtracted its roughly $280 contribution from $768 and got about $350 a month instead of $488. That arithmetic slip produced $4,200 instead of $6,060." +us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"It used the correct $785 maximum and $931 net income. It then took 30% as an unrounded $279.30 and got about $505.70 a month. SNAP rounds the contribution up to $280, which gives $505 a month and $6,060 a year, not $6,068." +us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted $2,416 of self-employment tax before counting income, which cut countable income to $1,223.65 a month. SNAP counts the full $1,425. It also used the outdated $198 standard deduction and a $775 maximum instead of $209 and $785, which inflated the benefit to $540.72 a month." +us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $781 maximum and a $202 standard deduction instead of the FY2026 values of $785 and $209. That raised net income to $938 and cut the allotment to $499 a month ($5,988) instead of $505." +us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,It used a $784 maximum and a $208 standard deduction instead of $785 and $209. It also left the 30% contribution unrounded at $279.60 rather than rounding it up to a whole dollar. The result was $504.40 a month instead of $505. +us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its $931 net income was correct, but it used a $787 maximum instead of $785. It also subtracted an unrounded $279.30 instead of rounding the contribution up to $280, which gave $507.70 a month ($6,092) instead of $505." +us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the FY2024 parameters: the $766 maximum, and net income of $942 from the $198 standard deduction. The FY2026 values are $785 and $209, which give $931 net income. The result was $483.40 a month instead of $505." +us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no derivation. Its $3,720 implies $310 a month, which requires an expected contribution of about $475 against the $785 maximum. The correct contribution is $280: 30% of $931 net income after the $285 earned income deduction and the $209 standard deduction." +us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $5,800.80 equals $483.40 a month, which is the FY2024 $766 maximum minus 30% of $942 net income (using the FY2024 $198 standard deduction). It did not use the FY2026 $785 maximum and $209 deduction, which give $505." +us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,It used a $747 three-person maximum and the outdated $198 standard deduction instead of the FY2026 values of $785 and $209. That gave $464 a month instead of $505. +us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It declared the household over the income limits, but $1,425 a month of gross income is 64% of the poverty guideline. $931 of net income is 42%. Both easily pass the gross and net income tests, so the household is owed $505 a month." +us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $7,548 ($629 a month) matches a derivation that cuts self-employment income by a 40% business-cost deduction to $855 a month and uses a $766 maximum. SNAP counts the full $1,425 because no expenses are listed, and the maximum is $785. The correct result is $505 a month." +us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,820 ($485 a month) matches outdated maximum allotment and standard deduction values of about $766–768 and $198–204. The FY2026 values are $785 and $209, which give a $280 contribution and $505 a month." +us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $5,844 ($487 a month) exactly matches the FY2025 parameters: the $768 maximum, minus a contribution rounded up to $281 on $936 net income after the $204 standard deduction. It did not apply the FY2026 $785 maximum and $209 deduction, which give $505 a month." +us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $228 monthly standard deduction ($2,736 a year) instead of $209, which understated the contribution at $273.60. It also used an $800 maximum ($9,600 a year) instead of $785. The benefit came out at $526 a month ($6,317) instead of $505." +us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Its $931 net income and roughly $279 contribution were right, but it used an $816 maximum instead of the FY2026 figure of $785. That inflated the benefit to $537 a month instead of $505." +us,scenario_054,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated countable income as 'very low' and awarded $739 a month, close to the full maximum. It never computed the $280 expected contribution that $931 of net income generates, which leaves the correct benefit at $505." +us,scenario_054,snap,gpt-5.4-nano,llm_error,other,False,"It claimed the facts did not settle eligibility and defaulted to $0. It ignored the listed $17,100 of self-employment income and the three-person household, which are all the formula needs to produce net income of $931 and an allotment of $505 a month." +us,scenario_054,snap,gpt-5.5,llm_error,other,False,"It set up the correct formula: the $785 maximum minus 30% of $931 net income. Then it miscalculated $785 − $279 as $575 instead of about $505. The correct figure, with the contribution rounded up to $280, is $505 a month ($6,060)." +us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It applied a North Carolina resource limit to the $14,000 bank balance and zeroed SNAP. It missed that the household is categorically eligible through TANF non-cash benefits under broad-based categorical eligibility, so the balance does not disqualify it." +us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"It correctly found $931 net income and eligibility under broad-based categorical eligibility. It then rounded the benefit to $506 instead of rounding the 30% contribution up to $280. $785 − $280 gives $505 a month ($6,060), not $506 ($6,072)." +us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but rounded $505.70 to a $506 monthly benefit. It should have rounded the $279.30 contribution up to $280, which gives $505 a month and $6,060 a year." +us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"Its formula, 12 × [785 − 0.30 × (1425 × 0.80 − 209)], left the 30% contribution unrounded at $279.30. SNAP rounds it up to the next whole dollar ($280), so the benefit is $505 a month and $6,060 a year, not $6,068.40." +us,scenario_054,snap,gpt-6-luna,llm_error,other,False,"It computed $785 minus 30% of $931 as $505.70 without rounding the contribution up to $280. The correct allotment is $505 a month, or $6,060 a year." +us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It claimed both income and assets exceed SNAP limits. Gross income is only 64% of the poverty guideline, and broad-based categorical eligibility in North Carolina means the $14,000 of assets does not disqualify the household. It is owed $505 a month." +us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It used FY2024 parameters: a $766 monthly maximum ($9,192 a year) and a $198 standard deduction ($2,376 a year). The FY2026 values are $785 and $209, which give a $280 contribution and $505 a month instead of its roughly $483." +us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"It used FY2025 parameters: a $768 maximum ($9,216 a year) and a $204 standard deduction ($2,448 a year). The FY2026 values are $785 and $209. It also left the contribution unrounded, and it got $487 a month instead of $505." +us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 values of $785 and $209. It also took an unrounded 30% contribution, which produced $5,846.40 instead of $6,060." +us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"Its contribution of about $280 was correct, but it assumed an $811 maximum instead of the FY2026 figure of $785. That inflated the benefit to $531 a month instead of $505." +us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"It zeroed SNAP because the $14,000 bank balance exceeds a resource limit. It ignored North Carolina's broad-based categorical eligibility through TANF non-cash benefits, which means the balance does not disqualify the household, so the full $505 monthly allotment applies." +us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the standard countable-resource limit for households without elderly or disabled members. The household is categorically eligible under North Carolina's broad-based categorical eligibility, so its $14,000 bank balance does not disqualify it." +us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"It correctly derived $931 net income and rounded the contribution up to $280. It then used a $783 three-person maximum instead of the FY2026 figure of $785, which gave $503 a month ($6,036) instead of $505." +us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It declared $17,100 of income over a 130% poverty limit of about $26,000, which is a mistaken comparison because $17,100 is below $26,000. Gross income is 64% of the poverty guideline, so the household passes the income tests and is owed $505 a month." +us,scenario_054,snap,ox-alpha,llm_error,other,False,"It used the correct $785 maximum and $931 net income, but subtracted an unrounded $279.30. SNAP rounds 30% of net income up to $280, so the benefit is $505 a month and $6,060 a year, not $505.70 a month." +us,scenario_054,snap,qwen-3.7-max,llm_error,other,False,"It awarded a 'maximum' benefit of $934 a month, which skips the 30% net-income contribution entirely and overstates the maximum, which is $785. The $931 of net income generates a $280 contribution, which leaves $505 a month." +us,scenario_054,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It misstated monthly earnings as $1,579 instead of $1,425. It also claimed net income exceeds the limit, but $931 of net income is only 42% of the $2,220.83 monthly poverty guideline, so the household passes the net income test and is owed $505 a month." us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly applied North Carolina's $19,125 head-of-household standard deduction against federal AGI of $15,892 and stated twice in its own reasoning that NC taxable income floors at $0 and ""NC tax = 0,"" then discarded that conclusion and submitted $245. No NC computation produces $245 from these facts: at the 2026 flat rate of 3.99% it implies $6,140 of taxable income, which the household does not have once the standard deduction is applied." us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly computing $0 NC taxable income using the head-of-household standard deduction of $19,125, the model reassigned the filer to single status and its $12,750 standard deduction despite two qualifying children in the household, yielding $3,142 of taxable income and $133.50 of tax. It then submitted $596, a figure tied to neither derivation and roughly 3.75% of gross self-employment income with no standard deduction at all." us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied the $12,750 single-filer standard deduction rather than the $19,125 head-of-household deduction the head qualifies for with two dependent children, and used the 2025 rate rather than the 2026 flat rate of 3.99%: $15,892 − $12,750 = $3,142 taxed at 4.5% is $141.39, matching its $142. Under the correct head-of-household deduction the base is zero and the tax is $0." us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model applied a rate directly to gross self-employment income with no standard deduction and no half-SE-tax adjustment: $342 is exactly 2% of the $17,100 SSTB figure. North Carolina's 2026 flat rate is 3.99% and it applies to taxable income after the $19,125 head-of-household standard deduction, which exceeds the $15,892 AGI and zeroes the base." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model nailed every hard step — 85% Social Security inclusion, the $18,150 standard deduction, and the $6,000 senior deduction phased down by 6% of the $81,276 MAGI excess to $1,123 — but omitted the OBBBA non-itemizer charitable deduction, which gives a single filer taking the standard deduction $1,000 of cash contributions on top of it starting in 2026. That single $1,000 of deduction left in taxable income is taxed at the 24% marginal rate, producing the $240 overstatement between its $25,479 and the reference $25,238.58." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a $20,550 + $3,200 = $23,750 standard deduction (the 2026 figures are $16,100 + $2,050 = $18,150) and applied 2023 bracket thresholds ($11,000/$44,725/$95,375) instead of the 2026 thresholds ($12,400/$50,400/$105,700), and it never applied the phased $1,123.45 senior deduction or the $1,000 non-itemizer charitable deduction. It then discarded its own $25,392 computation and submitted $14,829 with no supporting arithmetic — that figure follows from none of its stated steps." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model claimed the $6,000 OBBBA senior deduction was 'phaseout-free at this income,' but §151(d)(6) reduces it by 6% of MAGI over $75,000 for a single filer, so $81,275.86 of excess wipes out $4,876.55 and leaves $1,123.45 — a $4,877 over-deduction. It also used a fabricated $18,100 standard deduction and invented brackets, computed $24,431 from its own numbers, then reported $13,802 with no derivation connecting the two." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model stopped at a guessed $18,600 combined standard deduction and never applied either the phased $1,123.45 OBBBA senior deduction or the $1,000 non-itemizer charitable deduction, deducting $20,273.45 - $18,600 = $1,673 too little. It compounded that with an invented 22%-bracket ceiling of $107,500 instead of the statutory $105,700, pushing taxable income to $137,676 and tax to $25,604." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model included $1,181 of auto loan interest in itemized deductions, but the OBBBA deduction phases out at $200 per $1,000 of MAGI over $100,000, and $56,276 of excess eliminates the entire $1,181. It also stopped at a ~$17,750 deduction, omitting the phased $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction, and then reported $24,196 when its own $138,526 taxable income yields $25,844 on the 2026 brackets." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model guessed a $15,700 base standard deduction and a $2,000 age-65 additional; the 2026 statutory amounts are $16,100 and $2,050. More decisively, it treated the standard deduction as the whole deduction stack, omitting both the $1,123.45 phased-down OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, and used estimated brackets ($12,250/$49,800/$106,150) instead of $12,400/$50,400/$105,700." -us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $18,150 standard deduction and that the $6,000 senior deduction phases out, then botched the phaseout arithmetic — it wrote '~$8,175 additional,' subtracted '~$4,000,' and never computed the actual 6% × $81,275.86 = $4,876.55 reduction leaving $1,123.45. It then abandoned its own $24,788 result for $18,033 'after reconciling deduction phase-outs,' a figure supported by no stated calculation." -us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used a $15,500 base standard deduction plus $2,050 aged, $600 short of the 2026 $16,100 base, and omitted both the $1,123.45 phased senior deduction and the $1,000 non-itemizer charitable deduction, understating the deduction stack by $2,723.75. Its bracket thresholds ($12,075/$49,175/$104,875) are also fabricated rather than the statutory $12,400/$50,400/$105,700." -us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA sunset took effect in 2026, applying a $9,934 standard deduction, a $5,293 personal exemption, and pre-2018 10/15/25/28 rates. OBBBA made the TCJA rate schedule and the enlarged standard deduction permanent and kept the personal exemption at zero, so 2026 uses 10/12/22/24 with a $18,150 standard deduction plus the $1,000 charitable and $1,123.45 senior deductions; the 25%/28% rates it applied do not exist in 2026 law." -us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied a reinstated $5,400 personal exemption and pre-TCJA 10/15/25/28 brackets, both of which OBBBA eliminated for 2026 by making the zero personal exemption and the 10/12/22/24 schedule permanent. It also itemized $16,484 when the 2026 standard-deduction path totals $20,273.45, and its submitted $28,095.43 does not even match the $28,345.68 its own explanation computed." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model stated outright that 'the federal income tax system is estimated to return to pre-TCJA rules' in 2026, applying a $5,300 personal exemption and 10/15/25/28 rates. OBBBA made the TCJA rates and zero personal exemption permanent, so the correct path is the $20,273.45 standard-deduction stack against the 10/12/22/24 brackets, and the 28% top rate it used produced a $3,645 overstatement." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $157,137 gross income includes the full $5,742 of Social Security rather than the $4,880.71 that the 85% inclusion cap allows, and its $16,800 deduction omits the $2,050 aged additional, the $1,123.45 phased senior deduction, and the $1,000 non-itemizer charitable deduction. Its own stated inputs give taxable income of $140,337 and tax of $26,281, so the submitted $17,296 follows from nothing it wrote." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model claimed a personal exemption against taxable income; the personal exemption is permanently zero for 2026 under OBBBA. It also itemized $16,484 rather than taking the $20,273.45 standard-deduction stack ($18,150 + $1,000 charitable + $1,123.45 senior), and its $28,539 result reflects pre-TCJA 15/25/28 rates that 2026 law does not contain." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model explicitly applied 'post-TCJA expiration bracket rules' plus a $5,050 personal exemption, both of which OBBBA foreclosed by permanently extending the 10/12/22/24 schedule and the zero personal exemption. Its $134,742 taxable income also drops the $1,000 non-itemizer charitable deduction and the $1,123.45 senior deduction, and the 28% bracket it taxed into does not exist for 2026." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation beyond naming the income items. The correct 2026 path — $156,275.86 AGI less $20,273.45 of deductions, taxed at 10/12/22/24 — yields $25,238.58, while its $19,304 corresponds to roughly $111,275 of taxable income, implying about $45,000 of deductions where only $20,273.45 exists for a single filer with no itemized amounts exceeding the standard." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reached the correct $156,275.70 AGI but referred only to 'standard deduction and age 65+ allowance,' and its $26,105.17 backs out to about $16,663 of total deductions against the $20,273.45 actually allowed. That $3,611 shortfall is the omitted $1,123.45 phased OBBBA senior deduction plus the $1,000 non-itemizer charitable deduction plus most of the $2,050 age-65 additional standard deduction, taxed at 24%." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model applied only a base-plus-aged standard deduction of about $17,782 — its $25,836.43 backs out to $2,491 less deduction than the reference $20,273.45 at the 24% marginal rate. It never applied the $1,000 non-itemizer charitable deduction or the $6,000 OBBBA senior deduction that survives its 6%-of-excess-MAGI phaseout at $1,123.45, and its base standard deduction fell short of the statutory $16,100 + $2,050." -us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied 'pre-TCJA rules' with a reinstated ~$5,450 personal exemption and a 28% marginal bracket; OBBBA made the zero personal exemption and the 10/12/22/24 schedule permanent, capping this filer at 24%. It also itemized $16,484 instead of taking the $20,273.45 standard-deduction stack that includes the $1,000 non-itemizer charitable and $1,123.45 senior deductions." -us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $15,400 base standard deduction rather than the 2026 $16,100, and treated $17,450 as the entire deduction stack, omitting the $1,000 non-itemizer charitable deduction and the $6,000 senior deduction that phases down to $1,123.45 at $156,275.86 of MAGI. Its brackets ($12,250/$49,800/$106,150) are also estimates rather than the statutory $12,400/$50,400/$105,700." -us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model carried the 2025 $15,000 base standard deduction into 2026 instead of $16,100, and stopped at $17,050 of total deductions, omitting the $1,000 non-itemizer charitable deduction and the $6,000 OBBBA senior deduction reduced by 6% of the $81,275.86 MAGI excess to $1,123.45. Its taxable income of $139,226 exceeds the correct $136,002.41 by $3,224, taxed at 24%." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted a rule that does not exist — 'the standard deduction is reduced by excess itemized deductions only to the extent they exceed the standard deduction' — and never computed a deduction total, a taxable income, or a bracket walk. The correct path gives $136,002.41 of taxable income and $25,238.58 of tax; its $16,348 corresponds to about $98,345 of taxable income, i.e. roughly $58,000 of deductions against the $20,273.45 actually available." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that income was 'largely offset by standard deduction and nonrefundable itemization effects,' but the maximum 2026 deduction stack for this single filer is $20,273.45 against $156,275.86 of AGI, leaving $136,002.41 taxable and $25,238.58 of tax. There are no dependents, education, or dependent-care credits here, so nothing reduces that liability toward zero." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model declined to compute and submitted 0 as an admitted placeholder, so it applied no income aggregation, no deduction, and no bracket schedule. The required derivation — $156,275.86 AGI less the $20,273.45 deduction stack, taxed at 10/12/22/24 — is fully determined by the facts given and yields $25,238.58." -us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $24,379 backs out to about $23,855 of deductions, consistent with stacking the full $6,000 OBBBA senior deduction on the $18,150 standard deduction while ignoring the 6%-of-MAGI-over-$75,000 phaseout that cuts it to $1,123.45 at $156,275.86 of MAGI. It also omitted the $1,000 non-itemizer charitable deduction, so its over-deduction of about $3,582 understated tax by $859 at the 24% rate." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $157,137 gross income adds the entire $5,742 of Social Security rather than the $4,880.71 permitted by the 85% inclusion cap, and its $17,000 deduction omits the $1,123.45 phased senior deduction, the $1,000 non-itemizer charitable deduction, and part of the $18,150 standard deduction. Its own $140,137 taxable income produces $26,231 on the 2026 brackets, so the submitted $21,842 is unsupported by its stated steps." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a 'projected 2026 post-TCJA-expiration' standard deduction of ~$10,500, a ~$5,400 personal exemption, and pre-TCJA 10/15/25/28 rates. OBBBA permanently extended the TCJA rate schedule and the zero personal exemption, so 2026 uses 10/12/22/24 with an $18,150 standard deduction plus $1,000 charitable and $1,123.45 senior deductions, and its 28% top rate does not exist." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model stated 'under 2026 TCJA-sunset law' and applied a $5,300 personal exemption with 10/15/25/28 rates; OBBBA repealed that sunset, making the zero personal exemption and the 10/12/22/24 schedule permanent. Itemizing $16,484 also loses to the correct $20,273.45 standard-deduction stack, so the correct taxable income is $136,002.41, not its $134,492 taxed at pre-TCJA rates." -us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a 'projected 2026 standard deduction of 9,650' and post-TCJA brackets topping at 28%, both superseded by OBBBA's permanent extension of the $16,100 base standard deduction ($18,150 with the age-65 addition) and the 10/12/22/24 rate schedule. It also omitted the $1,000 non-itemizer charitable deduction and the $1,123.45 phased senior deduction, and its $139,792 taxable income taxed at 25%/28% overstated liability by $4,730." -us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied an 'approximate 2026 pre-TCJA standard deduction,' a ~$5,200 personal exemption, and 10/15/25/28 brackets, all of which OBBBA foreclosed by making the TCJA rates and zero personal exemption permanent. Under 2026 law the standard-deduction path totals $20,273.45 and the top rate reached is 24%, giving $25,238.58 rather than its $28,559." -us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model guessed a ~$17,400 combined standard deduction instead of the statutory $18,150 ($16,100 + $2,050), and treated that as the full deduction, omitting the $1,000 non-itemizer charitable deduction and the $6,000 senior deduction that phases down to $1,123.45 at this MAGI. Its $138,876 taxable income exceeds the correct $136,002.41 by $2,874, and its bracket thresholds ($12,225/$49,700/$105,950) are estimates rather than the statutory values." -us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model imputed ~$11,980 of mortgage interest from the $267,000 loan balance, but the prompt directs that unlisted numeric inputs are zero and no mortgage interest was given, so itemized deductions never reach the $20,273.45 standard-deduction stack. It then abandoned its own ~$18,500 figure and reported $0 by subtracting unnamed nonrefundable credits; this single filer with no dependents and no dependent-care or education expenses has zero nonrefundable credits, leaving $25,238.58." -us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model included only $2,045 of Social Security as taxable when provisional income of $154,266 far exceeds the $34,000 second threshold, forcing the 85% cap of $4,880.71 — a $2,836 understatement of AGI. It also used a $15,700 standard deduction with no age-65 addition, no $1,000 non-itemizer charitable deduction, and no $1,123.45 senior deduction, and invented bracket thresholds ($12,150/$49,275/$111,875) in place of $12,400/$50,400/$105,700." -us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model supplied no derivation at all, only a restatement of the task. Its $30,253 is consistent with taxing essentially the full $156,275.86 of gross income on the 2026 brackets with no deduction applied; the correct computation subtracts the $20,273.45 stack ($18,150 standard plus $1,000 non-itemizer charitable plus the $1,123.45 phased senior deduction) to reach $136,002.41 taxable and $25,238.58 of tax." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly applied the $18,150 standard deduction, the phased senior deduction of $1,123, and the 2026 brackets. It never took the OBBBA $1,000 charitable deduction for non-itemizers, even though it chose the standard deduction and had $15,647 of cash gifts. Its taxable income of $137,003 is $1,000 too high, which adds 24% x $1,000 = $240 to the tax." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a made-up standard deduction of $20,550 plus $3,200, applied 2023-era brackets (10% to $11,000, 22% to $95,375), and at first included all $5,742 of Social Security. It also omitted the $1,123.45 phased senior deduction and the $1,000 non-itemizer charitable deduction. It then submitted $14,829, which contradicts its own bracket result of about $25,392." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It took the full $6,000 OBBBA senior deduction as 'phaseout-free' and ignored the 6% reduction on MAGI above $75,000, which cuts it to $1,123.45 at $156,276 MAGI. It also omitted the $1,000 non-itemizer charitable deduction and used approximate brackets. It then replaced its own $24,431 bracket result with an unsupported $13,802." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It named the senior bonus deduction but never subtracted it, left out the $1,000 non-itemizer charitable deduction, and inflated the standard deduction to $18,600 instead of $18,150. That left taxable income at $137,676 instead of $136,002.41. It also used a guessed top of $107,500 for the 22% bracket instead of $105,700." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It deducted $1,181 of auto loan interest, but the OBBBA vehicle-loan interest deduction is fully phased out at this MAGI ($200 per $1,000 above $100,000). It also used about $17,750 of total deductions instead of $18,150 standard plus $1,123.45 senior plus the $1,000 non-itemizer charitable deduction. Its $24,196 is not even the 2026 bracket tax on its own $138,526 taxable income, which is about $25,844." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected 2026 parameters by inflating 2025 figures, giving a $17,700 standard deduction and a 22% bracket ending at $106,150. The actual 2026 figures are $16,100 plus $2,050 and $105,700. It also omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction, so its taxable income of $138,576 is about $2,574 too high." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It guessed a partial senior deduction of about $4,000 instead of computing $6,000 - 6% x ($156,276 - $75,000) = $1,123.45, and it omitted the $1,000 non-itemizer charitable deduction. It then cut its own $24,788 bracket result to an unsupported $18,033 by vaguely 'reconciling deduction phase-outs'." +us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got the $18,150 standard deduction, the $1,123 phased senior deduction, and the 2026 brackets right. It omitted the OBBBA $1,000 cash charitable deduction that standard-deduction filers take from 2026, even though it listed $15,647 of charity. Its taxable income of $137,002 is $1,000 high, which adds $240 of tax at 24%." +us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,500 base standard deduction instead of $16,100 and made-up brackets (10% on $12,075, 22% on $55,700) instead of the 2026 schedule of $12,400, $50,400 and $105,700. It also omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction, so its taxable income is $138,725.70 instead of $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law, with a $9,934 standard deduction, a $5,293 personal exemption, and 15/25/28% brackets. OBBBA made the TCJA standard deduction, the zero personal exemption, and the 10/12/22/24% brackets permanent. The correct path is the $18,150 standard deduction plus the $1,123.45 senior deduction plus the $1,000 non-itemizer charitable deduction, giving $136,002.41 taxable income taxed at 2026 rates." +us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA had expired: it itemized $16,484, subtracted a $5,400 personal exemption, and used 15/25/28% brackets. OBBBA kept the TCJA rate schedule and standard deduction permanent. It also submitted $28,095.43, which does not match its own computed $28,345.68." +us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It explicitly used 'pre-TCJA brackets' with a $5,419 personal exemption and 15/25/28% rates. OBBBA made the TCJA rates and the zero exemption permanent, so the tax is the 2026 10/12/22/24% schedule applied to $136,002.41, after the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It stated that 2026 returns to pre-TCJA rules, so it itemized $16,484, took a $5,300 personal exemption, and used 15/25/28% brackets. OBBBA made the TCJA structure permanent, so the standard deduction plus the senior and non-itemizer charitable deductions ($20,273.45) apply, taxed at the 2026 10/12/22/24% schedule." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted all $5,742 of Social Security as income (gross $157,137) instead of the 85% cap of $4,880.71. It used a $16,800 deduction, omitting the $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its $17,296 is far below the 2026 bracket tax on its own $140,337 taxable income, which is about $26,279." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It itemized $16,484 and took a personal exemption under the expired-TCJA framework. OBBBA made the TCJA standard deduction, the zero personal exemption, and the 10/12/22/24% brackets permanent. The correct path taxes $136,002.41 at 2026 rates to get $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied 'post-TCJA expiration' rules with a $5,050 personal exemption and pre-2018 brackets. OBBBA kept the TCJA rates and standard deduction permanently, so the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions apply, with the 2026 24% bracket starting at $105,700." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation beyond 'standard deductions and tax brackets'. Its $19,304 corresponds to taxable income of about $111,300 under the 2026 schedule, roughly $24,700 below the correct $136,002.41. The correct figure is AGI $156,275.86 minus the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its number reflects an understated bracket computation." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and the age-65 add-on, omitting the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. Its $26,105.17 implies taxable income of about $139,600 at 2026 rates, roughly $3,600 above the correct $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It took only the standard deduction for a single filer 65 or older and omitted the $1,123.45 phased senior deduction and the $1,000 non-itemizer charitable deduction. Its $25,836.43 implies taxable income of about $138,490, roughly $2,490 above $136,002.41." +us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied '2026 pre-TCJA rules' with a reinstated $5,450 personal exemption and a 28% marginal bracket. OBBBA made the TCJA structure permanent, so taxable income is $136,002.41 after the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions, taxed at a 24% top marginal rate." +us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It estimated a $15,400 base standard deduction instead of $16,100 and omitted both the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. That left taxable income at $138,825.70 instead of $136,002.41. It also taxed the result with projected brackets (22% to $106,150) instead of the actual 2026 schedule." +us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the 2025 base standard deduction of $15,000 instead of the 2026 figure of $16,100 and omitted the $1,123.45 senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $139,226 is $3,224 too high, which overstates the 24%-bracket tax." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no numbers. Its $16,348 corresponds to taxable income of about $98,300 under the 2026 schedule, about $37,700 below the correct $136,002.41, which is AGI of $156,275.86 minus the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and itemization offset the income and returned $0. Total deductions are only $20,273.45 against AGI of $156,275.86, leaving $136,002.41 taxable in the 24% bracket, and no nonrefundable credits apply." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"It declined to compute and submitted 0 as an explicit placeholder. The correct computation taxes $136,002.41 of taxable income at 2026 rates, giving $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a 'senior-enhanced standard deduction'. Its $24,379 is $859 below the reference, which at the 24% marginal rate means about $3,580 of excess deduction. That matches taking most of the unphased $6,000 senior deduction instead of the $1,123.45 left after the 6% phaseout above $75,000 MAGI." +us,scenario_055,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction, the aged add-on, and the phased senior deduction correctly but omitted the OBBBA $1,000 cash charitable deduction for non-itemizers. Its $25,479 is exactly $240 (24% x $1,000) above the correct $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It counted all $5,742 of Social Security (gross $157,137) instead of the 85% cap of $4,880.71 and used a $17,000 deduction, omitting the $1,123.45 senior and $1,000 non-itemizer charitable deductions. Its $21,842 is also far below the 2026 bracket tax on its own $140,137 taxable income, which is about $26,231." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-expiration' law, with a roughly $10,500 standard deduction, a $5,400 personal exemption, and 10/15/25/28% brackets. OBBBA made the TCJA standard deduction and 10/12/22/24% rates permanent, so the tax on $136,002.41 of taxable income is $25,238.58." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed 'TCJA-sunset law', itemizing $16,484 and taking a $5,300 personal exemption at 10/15/25/28% rates. OBBBA made the TCJA rules permanent, so the $18,150 standard, $1,123.45 senior and $1,000 non-itemizer charitable deductions apply, taxed at the 2026 24% bracket." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly spotted the $1,000 non-itemizer charitable deduction but subtracted it above the line to get a $155,276 AGI. That lowered the phaseout MAGI and inflated the senior deduction to $1,183 instead of $1,123.45. Its main error is bracket arithmetic: its own $135,943 taxable income produces $25,224 at 2026 rates, but it reported $25,314." +us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a projected post-TCJA standard deduction of $9,650 and 10/15/25/28% brackets. OBBBA made the TCJA standard deduction and 10/12/22/24% rates permanent. It also skipped the personal exemption it implied and reported taxable income of $139,792, which does not equal its AGI minus its $16,484 of itemized deductions ($139,792 is exactly that)." +us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It itemized $16,484 against a 'pre-TCJA standard deduction', subtracted a $5,200 personal exemption, and used 10/15/25/28% brackets. OBBBA made the TCJA structure permanent, so taxable income is $136,002.41 after the standard, senior and non-itemizer charitable deductions, taxed at the 2026 24% bracket." +us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It projected a roughly $17,400 standard deduction instead of $18,150 and omitted the $1,123.45 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $138,876 is $2,874 too high, and it used projected brackets (12% band ending at $49,700) instead of the 2026 schedule." +us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It invented about $11,980 of mortgage interest from the loan balance, even though no interest was listed. It then wiped out an estimated $18,500 of tax with nonrefundable credits that do not exist for this childless filer. No nonrefundable credits apply, so the tax before refundable credits is the full $25,238.58 on $136,002.41 of taxable income." +us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as $2,045 instead of the 85% cap of $4,880.71, which applies because provisional income far exceeds $34,000. It used a $15,700 standard deduction with no aged add-on and then itemized $16,524 including sub-floor medical. It never applied the $18,150 standard, $1,123.45 senior or $1,000 non-itemizer charitable deductions, and it used made-up brackets (22% to $111,875)." +us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $30,253 corresponds to taxable income of about $156,900 at 2026 rates, which is essentially all of AGI with no standard, senior or non-itemizer charitable deduction subtracted. The $20,273.45 of deductions brings taxable income down to $136,002.41." us,scenario_055,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model named the correct rule — 6.2% employee Social Security plus 1.45% employee Medicare on the $83,635 of wages, i.e. 7.65% × $83,635 — but never executed the multiplication, submitting a round $6,400 instead of the $6,398.08 its own stated formula produces. The error is purely arithmetic execution: it rounded the FICA product to the nearest hundred dollars rather than computing $5,185.37 + $1,212.71." us,scenario_055,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"The model identified the right base and rates (6.2% Social Security under the wage base plus 1.45% Medicare on $83,635, no Additional Medicare Tax, no Florida employee payroll tax), but mis-multiplied: 7.65% × $83,635 = $6,398.08, while it reported $6,437.70, an effective 7.697% rate that is $39.62 too high. Every structural judgment was right and only the arithmetic failed." us,scenario_055,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's explanation states the correct derivation — wages of $83,635 below the wage base, 6.2% plus 1.45% employee-side, no Additional Medicare Tax, no Florida payroll tax — which yields $6,398.08, yet it submitted $10,534, an effective 12.6% of wages. It abandoned the 7.65% employee share it had just described and emitted a figure only reachable by applying combined employer-plus-employee-level rates to the $83,635 wage." @@ -4344,136 +4758,153 @@ us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"T us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract." us,scenario_056,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the $400 threshold rule: $4,340 of self-employment income is above, not below, $400. Applying the 92.35% net-earnings adjustment and 15.3% combined rate yields $613.22." us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $4,340 below the $400 self-employment-tax threshold. Because the threshold is exceeded, the 92.35% net-earnings base is taxed at 15.3%, producing $613.22." -us,scenario_056,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added the $564.08/mo mortgage interest to the $995.75/mo real-estate-tax shelter charge and then applied a $744 excess-shelter cap, cutting net income to $462/mo instead of the actual $805.03/mo, where the uncapped excess shelter deduction is only $395.49. That inflated deduction plus the stale FY2025 $298 maximum allotment produced a $159/mo formula benefit, so it never reached the $63.18 formula result that New Jersey's $95/month state minimum benefit replaces." -us,scenario_056,snap,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It counted the full $18,722 of mortgage interest plus property taxes as shelter cost and applied an $8,928 annual shelter cap, giving net income of $5,535 instead of the actual $9,660 ($805.03/mo); the correct uncapped excess shelter deduction is $395.49/mo against a $995.75/mo shelter charge. Using $3,576 rather than the FY2026 $304.68/mo maximum allotment, it landed on $1,915.50 and never hit New Jersey's $95/month state minimum benefit, which is what sets the $1,140 answer." -us,scenario_056,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It declared $17,839 of gross income above the one-person SNAP limit when that gross is $1,486.54/mo, only 1.12 times the $1,330 monthly poverty guideline and inside both the 130% federal test and New Jersey's broad-based categorical eligibility limit. It compounded this by treating the $100,341 mortgage balance as disqualifying resources; the home is an excluded resource and the household passes the asset test, leaving a $63.18 formula benefit that New Jersey's $95/month minimum lifts to $1,140/year." -us,scenario_056,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It applied only the standard and 20% earned-income deductions and omitted the excess shelter deduction entirely, leaving net income at ~$1,400/mo instead of $805.03/mo; the $11,949 of real estate taxes generate a $395.49/mo excess shelter deduction after the 50%-of-adjusted-income offset. With net income overstated, 30% of net ($420) exceeded the maximum allotment, and it also skipped New Jersey's $95/month minimum benefit, which floors the true $63.18 formula result." -us,scenario_056,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"Its own figures show gross income of $1,486/mo against a stated $1,632/mo gross limit, yet it concluded the threshold was exceeded, then invoked assets implied by home ownership; the owner-occupied home is an excluded resource and the household passes the asset test. Net income after the $674.49 of deductions is $805.03/mo, giving a $63.18 formula benefit that New Jersey's $95/month state minimum raises to $1,140/year." -us,scenario_056,snap,claude-opus-5,llm_error,thresholds_rates,False,"It asserted that $17,839 of countable income exceeds both the gross and net limits for a one-person unit, when gross is 1.12 times the $1,330 monthly poverty guideline (limit 1.30) and net, after the $213.68 standard, $72.33 earned-income, and $395.49 excess shelter deductions, is $805.03/mo or 0.61 times the guideline (limit 1.00). Both tests pass, producing a $63.18 formula benefit that New Jersey's $95/month minimum benefit raises to $1,140 annually." -us,scenario_056,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It stopped after the standard and earned-income deductions, leaving net income at $1,216/mo and never applying the excess shelter deduction that the $11,949 of real estate taxes generates ($995.75/mo shelter less half of $1,200.53 adjusted income = $395.49), which brings net income to $805.03/mo. Having computed a negative formula result, it also treated the benefit as $0 instead of applying New Jersey's $95/month state minimum benefit, which floors the true $63.18 formula amount at $1,140/year." -us,scenario_056,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It ruled the household over 130% of the poverty line while also mislabeling the $12,000 of financial assistance as an asset-like resource; that assistance is countable unearned income, and total gross of $1,486.54/mo is 1.12 times the $1,330 guideline, comfortably inside the gross test. Net income of $805.03/mo passes the 100% net test and yields a $63.18 formula benefit that New Jersey's $95/month minimum benefit converts to $1,140 annually." -us,scenario_056,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It stopped deducting after the earned-income and standard deductions and concluded 30% of net income surpasses the one-person maximum allotment; adding the $395.49/mo excess shelter deduction from the $11,949 of real estate taxes brings net income to $805.03/mo, so the expected contribution is $241.50 against a $304.68 maximum allotment. The resulting $63.18 formula benefit is raised to $95/month by New Jersey's state-funded minimum SNAP benefit." -us,scenario_056,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It applied the ABAWD three-month time limit to zero out the entire year, but the head has $4,340 of self-employment earnings and the household passes the SNAP work-requirement test in the reference computation. With eligibility intact, net income of $805.03/mo gives a $63.18 formula benefit that New Jersey's $95/month state minimum lifts to $1,140 annually." -us,scenario_056,snap,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It capped the excess shelter deduction at $712 after counting mortgage interest as a shelter cost, producing net income of $498.25/mo rather than the actual $805.03/mo, where the uncapped excess shelter deduction is $395.49 against a $995.75/mo shelter charge. Combined with the FY2025 $292 maximum allotment instead of $304.68, it produced $142/mo and bypassed New Jersey's $95/month minimum benefit that sets the $1,140 reference." -us,scenario_056,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added an unlisted utility allowance and mortgage interest to shelter costs and then applied a $672 cap, giving net income of $544.25/mo instead of $805.03/mo; unlisted expenses are zero and the countable shelter charge is the $995.75/mo of real estate taxes, yielding an uncapped $395.49 excess shelter deduction. Its $291 allotment and inflated deduction produced $127.72/mo, skipping the $63.18 formula result that New Jersey's $95/month minimum benefit replaces." -us,scenario_056,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It asserted gross income exceeds SNAP thresholds, but gross of $1,486.54/mo is 1.12 times the $1,330 monthly poverty guideline, inside the 130% federal gross test and New Jersey's broad-based categorical eligibility limit. Net income of $805.03/mo passes the 100% net test, and the $63.18 formula benefit is raised to $95/month by New Jersey's state minimum SNAP benefit." -us,scenario_056,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied the maximum (capped) excess shelter deduction on shelter costs that included mortgage interest, driving net income to ~$489/mo instead of $805.03/mo; the correct deduction is the uncapped $395.49 computed from the $995.75/mo real-estate-tax shelter charge less half of $1,200.53 adjusted income. Its $158/mo result sits above the $63.18 formula benefit that New Jersey's $95/month state minimum benefit replaces." -us,scenario_056,snap,gemini-3.5-flash,llm_error,other,False,"It awarded the full one-person maximum allotment and omitted the 30%-of-net-income contribution step entirely; with net income of $805.03/mo the household owes a $241.50 expected contribution against the $304.68 FY2026 maximum, leaving a $63.18 formula benefit. That amount is then raised to New Jersey's $95/month state minimum, not to the $292/mo maximum it submitted." -us,scenario_056,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted zero with no derivation, a value consistent only with treating the household as income-ineligible; gross income of $1,486.54/mo is 1.12 times the $1,330 poverty guideline and net income of $805.03/mo is 0.61 times it, so both SNAP income tests pass. The correct chain gives a $63.18 formula benefit, which New Jersey's $95/month state minimum benefit raises to $1,140 annually." -us,scenario_056,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $140.72/mo result is consistent with a capped excess shelter deduction on shelter costs including mortgage interest, which drives net income to about $500/mo rather than the actual $805.03/mo produced by the uncapped $395.49 excess shelter deduction. It therefore never reached the $63.18 formula benefit that New Jersey's $95/month state minimum benefit replaces, which is what makes the answer $1,140." -us,scenario_056,snap,gemini-3.7-flash,llm_error,other,False,"It submitted 12 months of the one-person maximum allotment and skipped the benefit-reduction step: net income of $805.03/mo requires a $241.50 monthly expected contribution against the $304.68 FY2026 maximum. The resulting $63.18 formula benefit is floored at $95/month by New Jersey's state minimum SNAP benefit, giving $1,140 rather than $3,504." -us,scenario_056,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It disqualified the head as an ABAWD with zero work hours, but the head has $4,340 in self-employment earnings and the household passes the SNAP work-requirement test in the reference computation. With eligibility intact, net income of $805.03/mo yields a $63.18 formula benefit that New Jersey's $95/month state minimum benefit raises to $1,140 annually." -us,scenario_056,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"It counted $564.08/mo of mortgage interest as shelter cost on top of property taxes and then capped the excess shelter deduction at $733, giving net income of $471.25/mo instead of the actual $805.03/mo from the uncapped $395.49 deduction on a $995.75/mo shelter charge. Its $159.63/mo formula result therefore never fell to the $63.18 amount that New Jersey's $95/month state minimum benefit replaces." -us,scenario_056,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It excluded the $12,000 of financial assistance as TANF-type welfare, but SNAP counts it as unearned income, which is why gross income is $1,486.58/mo rather than the $487 it used. Dropping that income and the shelter deduction gave a net of ~$283/mo instead of $805.03/mo and a $207/mo benefit, far above the $63.18 formula result that New Jersey's $95/month minimum benefit replaces." -us,scenario_056,snap,gpt-5.4-mini,llm_error,other,False,"It declined to compute and defaulted to zero, but the listed facts fully determine the calculation: gross income of $1,486.54/mo, $674.49 of deductions, and net income of $805.03/mo, giving a $63.18 formula benefit. New Jersey's $95/month state minimum SNAP benefit then sets the annual amount at $1,140." -us,scenario_056,snap,gpt-5.4-nano,llm_error,other,False,"It treated the facts as unable to support a positive benefit and submitted zero, while the alimony, financial assistance, and self-employment income plus the $11,949 of real estate taxes yield gross income of $1,486.54/mo and net income of $805.03/mo, both within the SNAP limits. The $63.18 formula benefit is raised to $95/month by New Jersey's state minimum SNAP benefit, for $1,140 a year." -us,scenario_056,snap,gpt-5.5,llm_error,state_local_rule,False,"It correctly determined that the household passes both income tests and that the formula allotment falls below the minimum, then applied the federal $23/month one-person minimum instead of New Jersey's state-funded $95/month minimum SNAP benefit. Applying the New Jersey floor to the $63.18 formula result gives $95/mo, or $1,140 a year, not $276." -us,scenario_056,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It concluded net income leaves a zero allotment, which requires net income above $1,015/mo; the actual net is $805.03/mo because the $11,949 of real estate taxes produce a $395.49/mo excess shelter deduction on top of the $213.68 standard and $72.33 earned-income deductions. The expected contribution of $241.50 against the $304.68 maximum leaves $63.18, which New Jersey's $95/month state minimum benefit raises to $1,140 annually." -us,scenario_056,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the capped excess-shelter deduction, which drops net income to roughly $460-$500/mo instead of the actual $805.03/mo produced by the uncapped $395.49 deduction on a $995.75/mo shelter charge. Its $159/mo allotment therefore sits above the $63.18 formula result, and it never applied New Jersey's $95/month state minimum benefit that determines the $1,140 answer." -us,scenario_056,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the capped excess shelter deduction on shelter costs including the listed mortgage interest, giving net income near $490/mo rather than the actual $805.03/mo from the uncapped $395.49 deduction. That put its allotment at $158/mo instead of the $63.18 formula result, which New Jersey's $95/month state minimum SNAP benefit replaces to give $1,140." -us,scenario_056,snap,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied an $8,928 annual capped shelter deduction after adding mortgage interest to shelter costs, producing net income of $5,535 instead of the actual $9,660 ($805.03/mo) that the uncapped $4,745.88 annual excess shelter deduction yields. With the maximum allotment set at $3,576 rather than the FY2026 $304.68/mo, it reported $1,915.50 and skipped New Jersey's $95/month state minimum benefit that fixes the answer at $1,140." -us,scenario_056,snap,grok-4.3,llm_error,asset_resource,False,"It disqualified the household on income and assets, but the owner-occupied home and its mortgage are excluded resources and the household passes the asset test, while gross income of $1,486.54/mo is only 1.12 times the $1,330 poverty guideline. Net income of $805.03/mo leaves a $63.18 formula benefit that New Jersey's $95/month state minimum benefit raises to $1,140 annually." -us,scenario_056,snap,grok-4.5,llm_error,other,False,"It treated missing shelter and utility details as grounds for a zero benefit, while the prompt directs that unlisted amounts be treated as zero and the listed $11,949 of real estate taxes alone give a $995.75/mo shelter charge and a $395.49 excess shelter deduction. That produces net income of $805.03/mo and a $63.18 formula benefit, which New Jersey's $95/month state minimum benefit raises to $1,140." -us,scenario_056,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It concluded net income is too high for a positive allotment because no rent or utilities are listed, ignoring that the $11,949 of real estate taxes are the household's shelter cost and generate a $395.49/mo excess shelter deduction. With that deduction net income is $805.03/mo, the expected contribution is $241.50 against a $304.68 maximum, and the $63.18 remainder is floored at New Jersey's $95/month state minimum." -us,scenario_056,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted only $868 of earned-income disregard and a $3,492 standard deduction, omitting the $4,745.88 annual excess shelter deduction, so it computed net income of $13,479 instead of $9,660. Correct net income makes 30% of net $2,898 against a $3,656 annual maximum allotment, leaving $63.18/mo before New Jersey's $95/month state minimum benefit raises the total to $1,140." -us,scenario_056,snap,inkling,llm_error,taxable_income_or_deductions,False,"It applied a $712 shelter cap on costs that included mortgage interest, giving net income of $493/mo instead of the actual $805.03/mo produced by the uncapped $395.49 excess shelter deduction on the $995.75/mo real-estate-tax charge. Using the FY2025 $292 allotment as well, it reached $144/mo and never applied New Jersey's $95/month state minimum benefit, which is what replaces the $63.18 formula result." -us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for snap, so no substantive computation exists to evaluate. The required derivation gives net income of $805.03/mo, a $63.18 formula benefit, and $95/month under New Jersey's state minimum SNAP benefit, for $1,140 annually." -us,scenario_056,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"It added the $564.08/mo mortgage interest to the $995.75/mo property-tax shelter charge and applied a $744 cap, producing net income of $466.25/mo instead of the actual $805.03/mo from the uncapped $395.49 excess shelter deduction. With a $298 allotment rather than $304.68, it reported $158.13/mo, never reaching the $63.18 formula benefit that New Jersey's $95/month state minimum benefit replaces." -us,scenario_056,snap,minimax-m3,llm_error,thresholds_rates,False,"It asserted income exceeds the one-person SNAP limits, but gross income of $1,486.54/mo is 1.12 times the $1,330 monthly poverty guideline and net income of $805.03/mo is 0.61 times it, so both the 130% gross and 100% net tests pass. The resulting $63.18 formula benefit is raised to $95/month by New Jersey's state-funded minimum SNAP benefit." -us,scenario_056,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the excess-shelter cap to shelter costs that included $564/mo of mortgage interest, giving net income near $500/mo rather than the actual $805.03/mo from the uncapped $395.49 deduction on the $995.75/mo real-estate-tax charge. That produced roughly $150/mo instead of the $63.18 formula benefit, which New Jersey's $95/month state minimum benefit replaces to give $1,140." -us,scenario_056,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"Its own figures put $17,839 of income below the ~$19,000 gross limit it cited, yet it declared the limit exceeded and then asserted net income also fails; gross is 1.12 times the $1,330 monthly guideline and net, after the $213.68 standard, $72.33 earned-income, and $395.49 excess shelter deductions, is $805.03/mo or 0.61 times the guideline. Both tests pass, leaving a $63.18 formula benefit that New Jersey's $95/month state minimum benefit raises to $1,140 annually." -us,scenario_056,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed gross income exceeds 130% of the poverty line and net income exceeds the net limit, when gross of $1,486.54/mo is 1.12 times the $1,330 monthly guideline and net of $805.03/mo is 0.61 times it after the $674.49 of deductions, including the $395.49 excess shelter deduction it omitted. The household therefore qualifies with a $63.18 formula benefit, raised to $95/month by New Jersey's state minimum SNAP benefit." +us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added $564 a month of mortgage interest (plus an SUA with no utility costs listed) to the $996 in property taxes, then capped the excess shelter deduction at $744 to get $159 a month. Only real-estate taxes count, so the excess shelter deduction is $395.49 and net income is $805. The $63 formula benefit is then raised to NJ's $95 minimum." +us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated all $18,722 of mortgage interest plus property tax as shelter costs and took the capped $8,928 annual shelter deduction. Only the $11,949 in real-estate taxes counts, which gives a $395.49 monthly excess shelter deduction and a $63 formula benefit. NJ's $95 monthly minimum then applies." +us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 in gross income exceeds the one-person limit, but that is 112% of FPL, under NJ's BBCE gross limit. It also treated the mortgage balance as a countable asset, although the home is excluded and BBCE waives the asset test. It never applied the shelter deduction or NJ's $95 minimum." +us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75 a month in real-estate taxes and overstated net income at about $1,400. The correct net income is $805.03 after the $395.49 shelter deduction. The household is eligible, and its $63 formula benefit is raised to NJ's $95 minimum instead of $0." +us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Denied benefits because of assets implied by home ownership. The home is an excluded resource, and NJ BBCE waives the asset test. It also never worked out the property-tax shelter deduction or applied NJ's $95 minimum benefit that an eligible one-person household receives." +us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 in countable income exceeds the gross and net limits, but gross income is 112% of FPL (under the BBCE limit). After the standard, earned-income and $395.49 property-tax shelter deductions, net income is 61% of FPL. The eligible household gets NJ's $95 monthly minimum." +us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Counted only $487 a month of income, dropping the $12,000 of financial assistance that SNAP counts as unearned income. It also added mortgage interest to shelter costs, which drove net income to $0 and produced the maximum allotment." +us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for the $995.75 in monthly real-estate taxes, which put net income at $1,216 instead of $805. Having found the household eligible, it still returned $0 rather than the minimum allotment, which is $95 a month in NJ." +us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of FPL, but gross income of $1,486.54 a month is 112% of the $1,330 guideline and under NJ's BBCE limit. It also skipped all deductions, including the $395.49 property-tax shelter deduction, which lead to a $95 NJ minimum benefit." +us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $12,000 of financial assistance from income, although SNAP counts it as unearned income ($1,000 a month). It also added mortgage interest to shelter costs, so net income came out near zero and the answer was the maximum allotment instead of the $95 minimum." +us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,Applied only the earned-income and standard deductions and left out the $395.49 excess shelter deduction for real-estate taxes. It also ignored that an eligible one-person household in NJ receives at least the $95 state minimum. +us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the ABAWD time limit to cut the benefit to zero. In PolicyEngine this household passes the work-requirement test, so eligibility continues, and the benefit is NJ's $95 monthly minimum." +us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Included mortgage interest in shelter costs and took a capped $712 excess shelter deduction, which gave net income of $498 and $142 a month. Only real-estate taxes count, so the deduction is $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"A $0 answer from 30% of net income exceeding the maximum is what you get by leaving out the $395.49 property-tax shelter deduction. With it, net income is $805 and the formula benefit is $63, which is raised to NJ's $95 minimum." +us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built shelter costs from mortgage interest, property taxes and utilities that were never listed, and took a capped $672 shelter deduction. Only the $995.75 in property taxes counts, giving a $395.49 deduction, a $63 formula benefit and NJ's $95 minimum." +us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Said gross income exceeds the SNAP limits, but $1,486.54 a month is 112% of FPL, well under NJ's BBCE gross limit. The eligible household gets NJ's $95 monthly minimum." +us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum (capped) excess shelter deduction based on mortgage interest plus property taxes, which gave net income of $489. Mortgage interest does not count, so the deduction is $395.49 and net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted only self-employment income and alimony and left out the $1,000 a month of financial assistance that SNAP counts, then awarded the $292 maximum allotment. With that income counted, net income is $805 and the benefit is NJ's $95 minimum." +us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no derivation. The household is eligible at 112% of FPL gross, and after the $395.49 property-tax shelter deduction the $63 formula benefit is raised to NJ's $95 monthly minimum." +us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72 monthly benefit implies net income of about $500, which requires a capped shelter deduction that includes mortgage interest. Only property taxes count, so net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Awarded the full $292 maximum, which implies zero net income. That result requires dropping the $12,000 of financial assistance and/or counting mortgage interest as shelter cost. Correct net income is $805, which gives NJ's $95 minimum." +us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Disqualified the head as an ABAWD with no work. The household passes PolicyEngine's work-requirement test, so it stays eligible and receives NJ's $95 monthly minimum benefit." +us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08 of mortgage interest to $995.75 of property taxes and capped the excess shelter deduction at $733. Only property taxes count, so the deduction is $995.75 − $600.27 = $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 of financial assistance as welfare income, although SNAP counts it as unearned income. It also left out the property-tax shelter deduction entirely, which produced a $207 benefit instead of NJ's $95 minimum." +us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to zero without calculating anything. The household is income-eligible at 112% of FPL, and after the $395.49 property-tax shelter deduction the $63 formula benefit is raised to NJ's $95 monthly minimum." +us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Assumed zero without applying the SNAP formula. Gross income of $1,486.54 passes the BBCE test, and the deductions, including the $395.49 property-tax shelter deduction, give a $63 formula benefit, which is raised to NJ's $95 minimum." +us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly found that the formula benefit is below the minimum, but applied the federal $23 minimum allotment. New Jersey's state-funded minimum SNAP benefit of $95 a month applies instead, giving $1,140 a year." +us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,Concluded that net income is too high for any allotment by leaving out the $395.49 excess shelter deduction for real-estate taxes. It also ignored NJ's $95 minimum benefit for an eligible one-person household. +us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Took the capped excess shelter deduction by including mortgage interest in shelter costs, which gave $159 a month. With only property taxes counted, the deduction is $395.49 and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Explicitly built the capped shelter deduction from listed mortgage interest plus property taxes. Mortgage interest is not a counted shelter cost here, so the deduction is the uncapped $395.49, net income is $805, and the benefit is NJ's $95 minimum." +us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Applied the $8,928 annual shelter cap after adding mortgage interest to property taxes, which gave net income of $5,535. With only the $11,949 in property taxes counted, net income is $9,660 a year, and the $63 monthly formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,Found that 30% of net income exceeds the maximum by leaving out the $395.49 property-tax excess shelter deduction. It also skipped NJ's $95 minimum benefit for an eligible one-person household. +us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest as a shelter cost and applied the capped deduction, which gave net income of $5,535 a year. Only real-estate taxes count, which gives a $395.49 monthly deduction and a $63 formula benefit that NJ's $95 minimum replaces." +us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Called the household income- and asset-ineligible. Gross income is 112% of FPL, under NJ's BBCE limit, and BBCE waives the asset test, so the household is eligible and receives NJ's $95 minimum." +us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Acknowledged that income is within limits but returned zero, ignoring the $995.75 a month in listed real-estate taxes, which support a $395.49 excess shelter deduction. It also ignored NJ's $95 minimum for an eligible household." +us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Allowed no shelter deduction because rent and utilities were not listed. The $11,949 in real-estate taxes is a SNAP shelter cost and produces a $395.49 excess shelter deduction, so the benefit is NJ's $95 minimum." +us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance out of countable income, which cut net income to $210. It also set the shelter deduction to $0 despite listed real-estate taxes. With correct income and the $395.49 shelter deduction, the benefit is NJ's $95 minimum." +us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Left out the excess shelter deduction for real-estate taxes, which left net income at $13,479 a year. With the $395.49 monthly shelter deduction, net income is $805 a month, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied a capped $712 shelter deduction based on mortgage interest plus property taxes, which gave net income of $493. Only property taxes count, so the deduction is $395.49 and net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score against the $1,140 reference." +us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Included $564 of mortgage interest in $1,560.17 of monthly shelter costs and capped the excess at $744. With only the $995.75 in real-estate taxes, the excess shelter deduction is $395.49, net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Asserted that income exceeds one-person limits, but gross income is 112% of FPL (under NJ's BBCE limit) and net income after deductions is 61% of FPL. The eligible household receives NJ's $95 minimum." +us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Capped the excess shelter deduction using $564 of mortgage interest plus $996 of property taxes, which gave net income of about $500. Only property taxes count, so net income is $805, and the $63 formula benefit is raised to NJ's $95 minimum." +us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 exceeds a limit it put at $19,000, which contradicts its own comparison. It also denied benefits without applying the $395.49 property-tax shelter deduction or NJ's $95 minimum." +us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds 130% of FPL, but $1,486.54 a month is 112% of the $1,330 guideline. It also left out the shelter deduction, which brings net income to $805 and gives a benefit at NJ's $95 minimum." us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_056,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model built the correct NJ gross income base of $5,839 but never applied New Jersey's $10,000 minimum filing threshold for single filers, which exempts this household's entire liability and drives the tax to $0.00. It compounded this by inventing a ""$2,000 standard deduction"" that New Jersey's gross income tax does not provide, then taxed the residual $2,839 at 1.4% to reach $39.75." us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model reproduced PolicyEngine's intermediate NJ main income tax exactly — $5,838.53 of gross income less the $1,000 regular exemption, taxed at the 1.4% first-bracket rate — and submitted that $67.75 as the final answer. It omitted the last step: New Jersey's minimum filing threshold exempts a single filer whose gross income is at or below $10,000, so the $67.74 preliminary liability is fully offset to $0.00." us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model swept the $12,000 of financial assistance into NJ gross income to reach $17,839, which New Jersey does not tax, and then claimed a $1,428 charitable deduction that the New Jersey gross income tax does not allow at all. Its residual $3,462 taxed at 1.4% yields $48.47, and it never reached the dispositive rule — the $10,000 single-filer minimum filing threshold that zeroes the liability on the true $5,838.53 gross income base." us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs. This is a contract failure rather than a substantive modeling error: the correct derivation is $5,838.53 of NJ gross income less the $1,000 regular exemption at 1.4% = $67.74, fully exempted by New Jersey's $10,000 minimum filing threshold to $0.00." us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $12,000 of financial assistance as New Jersey gross income, inflating the base to $13,772.82 after the $1,000 exemption, when NJ gross income is only the $1,498.53 alimony plus $4,340.00 self-employment income. It then abandoned its own $192.82 arithmetic and submitted $299.43, a figure no step in its reasoning produces, and it never applied New Jersey's $10,000 single-filer minimum filing threshold, which exempts the entire $67.74 preliminary liability to $0.00." -us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model applied the federal childless-EITC minimum age of 25 to the state credit and concluded that a 20-year-old generates no NJEITC. New Jersey sets its own minimum age of 18 for childless filers and computes the credit from the federal formula the filer would have qualified for but for age, so the $4,033.39 of EITC earned income produces a $308.55 federal-formula credit and a $265.60 NJEITC under New Jersey's minimum benefit for filers without dependents." -us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly applied New Jersey's 18-24 childless expansion and correctly derived the federal-formula credit of $308.55 from earned income of $4,033.39, then stopped at the 40% federal match to report $123.42. It omitted New Jersey's minimum NJEITC benefit for filers with no dependents ($215 statutory, $265.60 at 2026 levels), which binds above the match and is the operative amount here." -us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that New Jersey has no state EITC or other refundable individual income tax credit for 2026. New Jersey has operated a refundable NJEITC since 2000, set at 40% of the federal credit with a childless-filer age floor of 18 and a minimum benefit for filers without dependents, which yields $265.60 for this 20-year-old with $4,033.39 of EITC earned income." -us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used an age-21 floor for the NJEITC (""under 21, no qualifying children"") to zero the credit. New Jersey lowered its childless-filer minimum age to 18 effective TY2021, so the 20-year-old qualifies, and the $4,033.39 of earned income supports a $265.60 credit under the state's minimum benefit for filers with no dependents." -us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model treated the $4,340 of SSTB self-employment income as not qualifying earned income and treated the filer's unearned income as disqualifying. Net self-employment earnings are EITC earned income — $4,340 less half of self-employment tax equals $4,033.39 — and the filer's AGI of -$5,882.05 is far below any phase-out threshold, so New Jersey's 18+ childless EITC pays $265.60." -us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model stated the NJEITC requires federal EITC eligibility and applied the federal age-25 childless floor. New Jersey explicitly decouples from that age test for childless filers aged 18-24 and 65+, computing the credit on the federal formula the filer would otherwise have received, which here yields $265.60." -us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model concluded the NJEITC requires federal eligibility and that the age 25-64 childless band therefore disqualifies the filer. New Jersey's own statute admits childless filers aged 18 and older, and with $4,033.39 of EITC earned income and negative AGI the state credit is $265.60, not zero." -us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model both imported the federal 25+ childless age requirement and speculated that the head is a dependent. The head is the only person listed and forms their own tax unit, and New Jersey's childless EITC age floor is 18, so the state credit phases in on $4,033.39 of earned income to $265.60." -us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model asserted that New Jersey's refundable working-families credit requires federal EITC eligibility, which the age-20 head lacks. The NJEITC is granted independently to childless filers aged 18-24 using the federal formula they would have qualified for but for age, producing $265.60 on $4,033.39 of net self-employment earnings." -us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model derived the state credit strictly as 40% of a $0 federal EITC. New Jersey's credit for childless filers aged 18-24 is computed on the hypothetical federal credit rather than the actual one, so the $308.55 federal-formula amount and the state minimum benefit for filers without dependents yield $265.60." -us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model correctly identified New Jersey's age-18 childless expansion but used gross self-employment income of $4,340 as EITC earned income (0.40 × 0.0765 × $4,340 = $132.80) instead of $4,033.39 after subtracting half of self-employment tax. It also stopped at the 40% match and omitted New Jersey's minimum NJEITC benefit for filers with no dependents, which sets the credit at $265.60." -us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model denied NJEITC eligibility on the ground that the household does not qualify for the federal EITC and has no children. New Jersey grants the credit to childless filers aged 18-24 on the federal formula they would have qualified for but for age, so the $4,033.39 of earned income produces a $265.60 state credit." -us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model gave a bare assertion that no state refundable credits apply. The correct derivation runs the NJEITC through New Jersey's 18+ childless rule on $4,033.39 of net self-employment earnings for $265.60; the $0 answer is consistent with silently importing the federal age-25 childless floor." -us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model correctly applied New Jersey's minimum age of 18 and the ~$309 federal-formula childless credit, then took 40% of it for $123. It omitted New Jersey's minimum NJEITC benefit for filers without dependents ($215 statutory, $265.60 at 2026 levels), which exceeds the 40% match and is the amount actually paid." -us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model applied the 18-24 state look-alike EITC but used $4,008 (self-employment income times 0.9235) as EITC earned income rather than $4,033.39 (self-employment income less half of self-employment tax), and then capped the result at the 40% federal match for $123. New Jersey's minimum benefit for childless filers sets the credit at $265.60, more than double the match figure." -us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model restated the $0 answer with no rule cited. New Jersey's childless EITC covers filers aged 18 and up, and $4,033.39 of EITC earned income with negative AGI yields a refundable $265.60 NJEITC." -us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model chained the state credit to $0 federal EITC eligibility and the absence of children. New Jersey computes the childless NJEITC for 18-24 year olds from the federal formula they would have qualified for but for age, giving $265.60 here." -us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model used an age-21 floor for New Jersey refundable credits. New Jersey's childless NJEITC floor is age 18 since TY2021, so the 20-year-old with $4,033.39 of net self-employment earnings receives $265.60." -us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model conditioned the state credit on federal EITC qualification and on having children under age 6 (the NJ Child Tax Credit test). The NJEITC has a separate childless pathway open at age 18, and it pays $265.60 on this filer's $4,033.39 of EITC earned income regardless of the child credit." -us,scenario_056,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model applied the federal 25-64 childless age band and treated federal ineligibility as controlling for the state credit. New Jersey admits childless filers at 18, computing the credit on the would-be federal amount of $308.55 and paying $265.60 under the state minimum benefit for filers with no dependents." -us,scenario_056,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model identified that New Jersey has its own childless age rule but set the floor at 21 rather than 18. New Jersey lowered the childless NJEITC minimum age to 18 effective TY2021, so this 20-year-old qualifies and receives $265.60." -us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model asserted that the facts indicate no New Jersey refundable credit without testing the NJEITC. The facts supply $4,340 of self-employment income in New Jersey and an age of 20, which trigger the state's 18+ childless EITC for $265.60." -us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model looked for children and explicit low-income indicators and, finding none, returned zero. The NJEITC's childless pathway keys on age 18+ and earned income, and $4,033.39 of net self-employment earnings phases the credit in to $308.55, paid out at $265.60." -us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model conditioned the credit on positive New Jersey income tax liability and on listed dependents. The NJEITC is refundable and pays out with zero liability, and its childless pathway opens at age 18, yielding $265.60 for this filer." -us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model denied that any qualifying New Jersey refundable credit exists for this filer. New Jersey's refundable EITC covers childless filers aged 18-24 and pays $265.60 on $4,033.39 of EITC earned income." -us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model substituted New Jersey's $50 property tax credit for the NJEITC. That $50 credit is available in lieu of the property-tax deduction and is refundable only for filers who are 65+, blind, or disabled, none of which applies to a 20-year-old; the credit that applies is the NJEITC's 18+ childless pathway, worth $265.60." -us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied New Jersey's age-18 childless rule but computed the federal-formula credit as $332 (7.65% of gross self-employment income of $4,340) instead of $308.55 on $4,033.39 net of half the self-employment tax, then took 40% for $132.80. New Jersey's minimum NJEITC benefit for filers without dependents sets the payout at $265.60 rather than the 40% match." -us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model correctly applied the age-18 expansion and the correct earned-income base of $4,033.39 net of half the self-employment tax, then reported the 40% match of $123.42. It omitted New Jersey's minimum NJEITC benefit for filers with no dependents, which binds above the match at $265.60." -us,scenario_056,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model gave a one-line denial with no rule applied. New Jersey's childless EITC covers filers aged 18 and older, and this filer's $4,033.39 of EITC earned income and negative AGI produce a $265.60 refundable state credit." -us,scenario_056,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model derived a zero NJEITC from a zero federal EITC. New Jersey computes the credit for childless filers aged 18-24 on the federal amount they would have received but for the age test, giving $308.55 phased in and $265.60 paid." -us,scenario_056,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model treated the absence of a federal EITC as dispositive for the state credit. The NJEITC's 18-24 childless pathway is independent of actual federal receipt and pays $265.60 on $4,033.39 of net self-employment earnings." -us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model computed the NJEITC as 40% of a $0 federal EITC and tied the conclusion to the property tax deduction. The property tax deduction has no bearing on the NJEITC, and New Jersey's 18+ childless rule generates a $265.60 refundable credit here." -us,scenario_056,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model zeroed the NJEITC because federal EITC eligibility fails and no dependents are present. New Jersey's childless EITC pathway opens at age 18 and uses the would-be federal credit of $308.55, paying $265.60." -us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for state_refundable_credits, so no substantive reasoning was produced. The requested derivation is the NJEITC on the 18+ childless pathway: $4,033.39 of EITC earned income phasing in at 7.65% to $308.55 and paid at $265.60." -us,scenario_056,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model applied the federal 25-64 childless age range to the NJEITC and correctly set aside the $50 property tax credit, landing on zero. New Jersey decoupled its childless EITC age floor to 18 effective TY2021, so this 20-year-old receives $265.60 based on $4,033.39 of net self-employment earnings." -us,scenario_056,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model attributed the zero to household composition. A single childless filer aged 20 is precisely who New Jersey's expanded 18-24 NJEITC pathway covers, and the $4,033.39 of EITC earned income produces a $265.60 refundable credit." -us,scenario_056,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model treated the filer as ""age-ineligible"" for the childless EITC and multiplied a $0 federal credit by New Jersey's match rate. New Jersey's own age floor for childless filers is 18, and the resulting NJEITC is $265.60 under the state minimum benefit for filers without dependents." -us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model concluded the filer lacks qualifying children and sufficient earned-income characteristics for the NJEITC. Net self-employment earnings of $4,033.39 are EITC earned income, and New Jersey's childless pathway at age 18+ phases the credit in to $308.55, paid at $265.60." -us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model required qualifying children and positive New Jersey tax liability for a refundable credit. The NJEITC is refundable without liability and reaches childless filers from age 18, producing $265.60 on this filer's $4,033.39 of EITC earned income." +us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied the federal 25-year minimum age for childless workers to the NJ credit and concluded that NJ EITC = $0. It missed that New Jersey extends its 40% EITC to childless filers aged 18+ who fail only the federal age test, which the engine values at $265.60." +us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model identified NJ's age-18 childless EITC and computed 40% × (7.65% × $4,033.39 = $308.55) = $123.42. The reference instead equals 40% of the full $664 childless maximum ($265.60), so the entire gap comes from the phase-in step." +us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model claimed New Jersey has no state EITC or other refundable credit. NJ has a refundable EITC equal to 40% of the federal credit and extends it to childless filers aged 18+, so this household is eligible." +us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used an under-21 cutoff for the childless NJ EITC. That was the old 2020 floor; New Jersey lowered the minimum age to 18, so the 20-year-old head qualifies and the engine awards $265.60." +us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model ruled out NJ refundable credits because the head is 20, has no children, and has unearned income. Alimony is not investment income for the EITC test, and New Jersey's childless EITC covers filers aged 18+, so the $4,033 of net self-employment earnings qualifies for $265.60." +us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,The model required federal EITC eligibility (age 25+) for the NJ EITC. It missed New Jersey's rule that childless filers aged 18–24 get the NJ credit as if they met the federal age test. +us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC to net earnings of $4,033.39 and took 40% of the $308.55 phased-in credit, giving $123.42. The reference equals 40% of the $664 childless maximum ($265.60), so the difference is only the phase-in treatment." +us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model stated that the NJEITC requires federal EITC eligibility and applied the federal 25–64 childless age range. New Jersey waives the federal age floor for childless filers aged 18+, so the 20-year-old head receives the NJ EITC ($265.60)." +us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model assumed the head could be a dependent and applied the federal 25+ childless-worker age requirement to the NJ EIC. The head files as a non-dependent, and New Jersey's childless EITC starts at age 18, producing $265.60." +us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model claimed the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65 or older, blind, or disabled, and the model also missed the NJ EITC for childless filers aged 18+, which is the only refundable credit here ($265.60)." +us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model said the NJ credit requires federal EITC eligibility, which the 20-year-old head lacks. New Jersey extends its EITC to childless filers aged 18+ who fail only the federal age test, so the credit is $265.60, not $0." +us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model computed the NJ EITC as 40% of a $0 federal EITC. It ignored New Jersey's separate pathway for childless filers aged 18–24, who get the credit computed as if federally age-eligible." +us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The model found the age-18 NJ pathway but applied 7.65% to the gross $4,340 of self-employment income. EITC earned income is net of the deductible half of self-employment tax ($4,033.39), and the engine's credit is $265.60, not 0.40 × 0.0765 × $4,340 = $132.80." +us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model tied the NJ EITC strictly to a $0 federal EITC because of the head's age. It missed that New Jersey allows childless filers aged 18+ to claim the credit, which yields $265.60." +us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model required federal EITC qualification for the NJ credit. New Jersey separately covers childless filers aged 18–24 who fail only the federal age test, so the head receives $265.60." +us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model answered $0 without analysis and never applied New Jersey's EITC for childless filers aged 18+. The head's $4,033 of net self-employment earnings qualifies for a $265.60 credit on that pathway." +us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model recognized NJ's age-18 childless EITC and took 40% of a phased-in federal-formula credit of about $309, giving $123. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is the phase-in step." +us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model applied NJ's age-18 look-alike EITC to about $4,008 of earnings and took 40% of a $307 phased-in credit, giving $123. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap comes from phasing in the credit." +us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,The model answered $0 with no derivation and missed New Jersey's EITC for childless filers aged 18+. The engine awards $265.60 on the head's self-employment earnings. +us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model concluded that $0 federal EITC and no children mean $0 NJ credits. New Jersey's EITC covers childless filers aged 18+ regardless of the federal age test, so the credit is $265.60." +us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model applied an age-21 floor to NJ refundable credits. New Jersey lowered the childless EITC minimum age to 18, so the 20-year-old head qualifies for $265.60." +us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"The model required federal EITC qualification for the NJ EITC and only considered child-based credits. It missed New Jersey's age-18+ childless EITC, which the engine values at $265.60." +us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model carried the federal 25–64 childless age limit over to the NJ EITC. New Jersey waives that limit for childless filers aged 18+, which gives $265.60." +us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model stated that the NJ childless EITC requires age 21+, which is the superseded 2020 threshold. The current NJ floor is 18, so the 20-year-old head qualifies and the engine awards $265.60." +us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model found no refundable NJ credit and missed the NJ EITC for childless filers aged 18+. The head's net self-employment earnings of $4,033.39 qualify for $265.60 on that pathway." +us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated the lack of children as ruling out every NJ refundable credit. New Jersey's EITC is available to childless filers aged 18+ with earned income, which yields $265.60 here." +us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model required positive NJ income tax liability and dependents before any refundable credit could apply. The NJ EITC is refundable, needs no liability, and is open to childless filers aged 18+, so it pays $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model asserted that no NJ EITC applies. New Jersey extends its EITC to childless filers aged 18+ who fail only the federal age test, so the 20-year-old head receives $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model claimed the $50 NJ property tax credit as refundable. That credit is refundable only for filers who are 65 or older, blind, or disabled, and the model missed the NJ childless EITC for filers aged 18+, which supplies the $265.60." +us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model found the age-18 NJ pathway but computed the federal-formula credit on gross self-employment income (7.65% × $4,340 = $332). EITC earned income is net of the deductible half of self-employment tax ($4,033.39), and the engine's credit is $265.60." +us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC to earnings net of half the self-employment tax and took 40% of the $308.55 phased-in credit, giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is only the phase-in step." +us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model concluded that the head does not qualify for the state EITC. New Jersey covers childless filers aged 18+, so the 20-year-old head with $4,033 of net self-employment earnings receives $265.60." +us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model identified NJ's childless EITC for a 20-year-old and took 40% of the $308.55 phased-in federal-formula credit, giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the gap is the phase-in step." +us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"The model applied NJ's age-18 childless EITC and took 40% of the credit phased in on net earned income ($308.55), giving $123.42. The reference equals 40% of the full $664 childless maximum ($265.60), so the difference is only the phase-in treatment." +us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,The model answered $0 with no analysis and missed New Jersey's EITC for childless filers aged 18+. The engine awards $265.60 on the head's self-employment earnings. +us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model treated the NJ EITC as purely 40% of a $0 federal EITC. It ignored New Jersey's age-18 childless pathway, which produces $265.60." +us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model concluded that no federal EITC means no NJ EITC. New Jersey computes the credit for childless filers aged 18–24 as if they met the federal age test, giving $265.60." +us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model correctly excluded the $50 property tax credit but zeroed out the NJ EITC because the federal EITC is $0. New Jersey extends its EITC to childless filers aged 18+, so the 20-year-old head receives $265.60." +us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model set the NJ EITC to 40% of a $0 federal EITC. It missed New Jersey's rule that childless filers aged 18+ qualify despite failing the federal age test, which yields $265.60." +us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model treated federal EITC ineligibility as disqualifying for the NJ EITC. New Jersey separately covers childless filers aged 18+, so the head receives $265.60." +us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there was no answer to score against the $265.60 NJ EITC." +us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model applied the federal 25–64 childless age range and set the NJ EITC to 40% of $0. New Jersey waives the federal age floor for childless filers aged 18+, so the credit is $265.60." +us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model ruled out NJ refundable credits based on household composition. It missed that the NJ EITC covers single childless filers aged 18+ with earned income, which pays $265.60 here." +us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model treated the NJ EITC as purely a percentage of the $0 federal EITC for an age-ineligible childless filer. New Jersey extends the credit to childless filers aged 18+, giving $265.60." +us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model required qualifying children or other characteristics for the NJ EITC. New Jersey grants its EITC to childless filers aged 18+ with earned income, and the head's $4,033 of net self-employment earnings qualifies for $265.60." +us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model required children and positive NJ tax liability for any refundable credit. The NJ EITC is refundable without liability and covers childless filers aged 18+, which yields $265.60." us,scenario_056,tanf,glm-5.3,llm_error,categorical_eligibility,False,"The model read the $12,000 `financial_assistance` input as if it were reported TANF receipt and echoed it back verbatim as the computed benefit, when the question asks what PolicyEngine computes for `tanf` from the household facts. It never applied TANF's categorical requirement that the assistance unit contain a dependent child — New Jersey's WorkFirst NJ TANF covers only units with a child under 18 (or 18 and in school full time), while childless adults fall to WFNJ General Assistance, which is not TANF — so this childless 20-year-old head yields $0. Its answer of exactly 12000 is arithmetically identical to the input line it copied, confirming no eligibility test or benefit computation was performed at all." us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no `tanf` entry in its `outputs` object and no explanation, so the required key was absent rather than substantively wrong. The correct derivation is that WorkFirst NJ TANF requires an assistance unit containing a dependent child, and this single 20-year-old head with zero children fails that categorical test, producing $0." -us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly identified the childless MFJ plateau and computed a maximum of roughly $659, then discarded its own figure and submitted $636 on the strength of an invented ""lookup-table rounding"" adjustment. The EITC table pays the full 2026 zero-child statutory maximum of $664 at $9,665 of earned income; the table's bracket rounding never reduces a plateau credit by $28." -us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated refundable credits as capped by tax liability — ""With no federal income tax liability, no refundable credits are available"" — when the EITC is paid out in full at zero taxable income, which is the entire point of refundability. Its secondary claim that income exceeds the EITC threshold is backwards: $9,665 of earned income is below the joint zero-child phaseout start (~$18,180), so the couple sits at the $664 maximum." -us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning ran the derivation correctly — net SE earnings above the phase-in ceiling, AGI below the joint phaseout start, head age 43 satisfying the 25-64 test — and concluded the credit equals the childless plateau maximum, which it estimated at $649 (the stale 2025 figure; 2026 is $664). It then submitted $397, a value no step of its own derivation produces." -us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It placed earned income of $9,665 ""in the phase-out region above the plateau"" and shaved the credit to $218. The 2026 zero-child phaseout for joint filers does not begin until about $18,180 of AGI, so this household is on the flat plateau and receives the full $664 maximum." -us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It located the plateau but applied an unexplained haircut to it, guessing a ""$690 maximum"" and then submitting $566 as a blend of plateau and phaseout. Once earned income clears the $8,690 phase-in ceiling and stays below the joint phaseout start, the credit is exactly the 2026 zero-child statutory maximum of $664 with no interpolation." -us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own derivation capped the credit at the childless maximum and estimated ""about $600,"" then it submitted $967 — a figure above every 2026 zero-child EITC value, since the credit cannot exceed the $664 plateau at any earned income. It also mislabeled the household as in the phase-out range while simultaneously saying the credit is near maximum." -us,scenario_057,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly hit the plateau and capped the phase-in, but used $650 as the 2026 zero-child maximum, a rounded carryover of the 2025 amount ($649). The indexed 2026 maximum is $664 ($8,690 earned-income amount x 7.65%); it also double-reduced earned income by applying the 92.35% SE factor and then subtracting half the SE tax." -us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It explicitly carried the 2025 zero-child maximum of $649 into 2026 without applying the annual inflation adjustment. The 2026 zero-child maximum is $664, which is what the plateau pays at $9,665 of earned income." -us,scenario_057,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"It applied the childless EITC age test to both spouses, concluding the 20-year-old spouse zeroes the credit. Section 32(c)(1)(A)(ii)(II) requires only that the taxpayer or the spouse be at least 25 and under 65 on a joint return, and the 43-year-old head satisfies it, so the couple receives the full $664." -us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It identified the plateau correctly but indexed from a wrong anchor, calling $600 the ""2024 max"" (that is the 2023 amount; 2024 was $632 and 2025 $649), landing at $630. Indexing from the actual 2025 maximum gives the 2026 zero-child maximum of $664." -us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It asserted the household meets neither the qualifying-child nor the earned-income requirements for the EITC. Net earnings from self-employment are earned income under section 32(c)(2)(A)(ii), the childless EITC needs no qualifying child, and $9,665 clears the $8,690 phase-in ceiling for the full $664 credit." -us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly placed the household in the maximum-credit plateau of the childless EITC but priced that plateau at $666. The 2026 zero-child maximum is $664 ($8,690 x 7.65%)." -us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached the right pathway — childless EITC on self-employment earnings — but guessed the 2026 maximum at $620 instead of computing it from the indexed parameters. The 2026 zero-child earned-income amount of $8,690 times the 7.65% phase-in rate gives the $664 maximum that applies here." -us,scenario_057,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It denied all refundable credits to a household whose income is self-employment plus disability benefits, treating that mix as disqualifying. Net earnings from self-employment are EITC-qualifying earned income, the head's age 43 satisfies the 25-64 test, and the $38 of disability benefits changes nothing, so the credit is the $664 childless maximum." -us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,It correctly derived a plateau childless EITC on the self-employment earnings but valued the 2026 maximum at $660. The indexed 2026 zero-child maximum is $664. -us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the 2025 zero-child maximum of $649 to tax year 2026 without indexing. The 2026 amount is $664, which the household receives in full since $9,665 of earned income is above the $8,690 phase-in ceiling and below the joint phaseout start." -us,scenario_057,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its derivation was exact — age test satisfied by the 43-year-old head, earned income $9,665.26, credit at the maximum — but it used the unindexed 2025 maximum of $649. The 2026 zero-child maximum is $664." -us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It fabricated the parameter set: a ""$1,231 maximum"" (no zero-child EITC maximum has ever approached that; 2026 is $664), earned income of $8,320 rather than $9,665, and then added 7.65% of earnings above $9,580 on top of a maximum, which double-counts phase-in past the plateau. The 7.65% rate applies only up to the $8,690 earned-income amount, after which the credit is flat." -us,scenario_057,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the 7.65% phase-in rate to the entire $10,400 of gross self-employment income without capping at the statutory maximum and without netting the deductible half of SE tax. Phase-in stops at the 2026 zero-child earned-income amount of $8,690, so the credit is the $664 plateau, not $795.60." -us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It required ""earned wages"" for the EITC and treated the absence of W-2 income as disqualifying. Section 32(c)(2)(A)(ii) counts net earnings from self-employment as earned income, so the $10,400 of SSTB self-employment income produces $9,665 of EITC earned income and the $664 childless maximum." -us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It declared without derivation that the facts trigger no refundable credit. The head's $10,400 of self-employment earnings is EITC-qualifying earned income and the 43-year-old head satisfies the childless age test on the joint return, producing the 2026 zero-child maximum of $664." -us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its structural reasoning was exactly right — one spouse aged 43 satisfying the age test, earnings above the phase-in range and below the joint phaseout — but it valued the 2026 childless maximum at $667. The indexed 2026 zero-child maximum is $664." -us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,It correctly awarded the childless-worker EITC at its maximum but used the 2025 amount of $649 as the 2026 maximum. The 2026 zero-child maximum after inflation indexing is $664. -us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It held that the 20-year-old spouse prevents the joint filers from meeting the childless EITC age test. On a joint return, section 32(c)(1)(A)(ii)(II) is satisfied if either spouse is at least 25 and under 65; the 43-year-old head qualifies the couple for the full $664." -us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It zeroed the credit on ""income phaseouts."" AGI of about $9,665 is roughly $8,500 below the 2026 joint zero-child phaseout start (~$18,180) and far below the ~$26,900 complete-phaseout point, so the household sits on the plateau at the $664 maximum." -us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its pathway was correct — head's age 43 satisfying the 25-64 rule, net SE earnings fully phasing in the credit below the joint phaseout threshold — but it projected the 2026 zero-child maximum as $667. The actual indexed maximum is $664." -us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly computed earned income of $9,665 on the plateau below the joint phaseout start but paid a maximum of $648, a 2025-level figure. The 2026 zero-child maximum is $664 ($8,690 earned-income amount x 7.65%)." -us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"It disqualified the household because the spouse is 20, below the 25-year minimum for the childless EITC. The joint-return age test is met when either spouse is 25 to 64, and the 43-year-old head meets it, so the couple receives the $664 maximum." -us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"It stated that ""both spouses filing jointly must be at least age 25"" for the childless EITC and zeroed the credit on the 20-year-old spouse. Section 32(c)(1)(A)(ii)(II) requires only the taxpayer or the spouse to fall in the 25-64 band, which the 43-year-old head does, leaving the full $664 credit." -us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It invented an eligibility bar from the SSTB label and claimed the self-employment income is ""above the EITC threshold."" Specified-service-trade-or-business status is a section 199A concept with no bearing on the EITC, and $9,665 of earned income is below the joint zero-child phaseout start, so the couple receives the $664 plateau maximum." -us,scenario_057,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It reasoned the plateau correctly but anchored on ""$611 for 2025"" — no such zero-child maximum exists; 2025 is $649 — and indexed that to $625. Indexing the real 2025 parameters gives the 2026 zero-child maximum of $664." -us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It zeroed the credit on the claim that disability benefits reduce EITC eligibility, and simultaneously said income ""falls below threshold."" The $38 of disability benefits is neither earned income nor large enough to approach the ~$18,180 joint phaseout start, there is no minimum-income floor below which the EITC vanishes, and $9,665 of earned income pays the $664 maximum." -us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It dismissed $10,400 as ""minimal self-employment income"" with ""no earned wage income,"" concluding no EITC. Self-employment net earnings are earned income for the EITC, and $9,665 is above the $8,690 phase-in ceiling, so the childless couple receives the full 2026 maximum of $664." +us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly placed earned income of about $9,604 on the childless-EITC plateau. It then used a guessed maximum of about $659 and cut it to $636 for 'lookup-table rounding'. On the plateau the credit equals the 2026 maximum of $664, and the EITC table does not reduce a plateau credit." +us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It made refundable credits depend on having positive income tax liability, and it claimed the income exceeds EITC thresholds. The EITC is refundable whatever the liability is, and about $9,665 of earned income is on the childless plateau, which gives $664." +us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning correctly found a joint-return age pass (head 43) and a plateau credit of about $649, but it submitted an unrelated $397 that no step of its reasoning supports. The $649 it did reason to is also the 2025 maximum rather than the 2026 maximum of $664." +us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It placed earned income of about $9,665 in the childless-EITC phase-out region and arrived at a small $218 credit. The joint phase-out does not begin until about $18,000 of AGI, so the household is on the plateau and gets the full 2026 maximum of $664." +us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It assumed a 2026 childless maximum of about $690 and then reduced it to $566 as if the household were partly phased out. Earned income of $10,400 gross (about $9,665 net) sits on the plateau below the joint phase-out start, so the credit is exactly the $664 maximum." +us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Its reasoning estimated a childless maximum of about $649 and a credit of about $600. It then submitted $967, which is above any childless-EITC maximum, apparently from treating both spouses as separate workers. A joint return gets one childless credit, capped at the 2026 maximum of $664." +us,scenario_057,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly found the phase-in fully complete and AGI below the phase-out start, but it capped the credit at an assumed 2026 maximum of $650. The 2026 statutory maximum for no qualifying children is $664." +us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used the 2025 maximum of $649 as the 2026 value. The inflation-adjusted 2026 maximum is $664. +us,scenario_057,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"It required both spouses to meet the 25-64 age test and denied the credit because the spouse is 20. Under IRC 32(c)(1)(A)(ii)(II), a joint return passes if either the taxpayer or the spouse is 25-64, and the 43-year-old head does, so the $664 plateau credit applies." +us,scenario_057,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $666. The statutory 2026 maximum for no qualifying children is $664. +us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,It correctly identified the plateau but inflated the wrong base: it treated $600 as the 2024 maximum (the actual figure was $632) and projected $630. The 2026 childless maximum is $664. +us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It denied the EITC for lack of a qualifying child and for failing an earned-income threshold. The no-qualifying-child EITC is available to a joint filer with one spouse aged 25-64, and about $9,665 of net SE earnings puts the household on the $664 plateau." +us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $666 instead of the statutory $664. +us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,It recognized childless-EITC eligibility but understated the 2026 maximum as $620. On the plateau the credit equals the full 2026 maximum of $664. +us,scenario_057,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It treated self-employment income and small disability benefits as disqualifying for the EITC. Net SE earnings are earned income for EITC purposes, disability benefits do not bar the credit, and the head's age of 43 satisfies the joint childless age test, which yields $664." +us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,It correctly identified the childless joint EITC but used a guessed maximum of $660 instead of the 2026 statutory maximum of $664. +us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly identified the childless joint EITC but submitted $649, which is the 2025 maximum. The 2026 inflation-adjusted maximum is $664." +us,scenario_057,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly computed earned income of $9,665.26 and passed the age test through the head, but it capped the credit at the 2025 maximum of $649 instead of the 2026 maximum of $664." +us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used invented childless-EITC parameters ($1,231 and $1,283 maxima, a $9,580 breakpoint) and a wrong earned-income figure of $8,320. The 2026 no-qualifying-child maximum is $664, and the household's about $9,665 of earned income sits on that plateau." +us,scenario_057,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the 7.65% phase-in rate to the full $10,400 with no cap and got $795.60. The phase-in stops at the earned-income amount (about $8,700 in 2026), which limits the credit to the $664 maximum. It also used gross SE income instead of net earnings after the half-SE-tax deduction." +us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It ruled out the EITC because the household has no wages or qualifying children. Net self-employment earnings count as earned income for the EITC, and the childless credit is available, so the household gets the $664 plateau amount." +us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted that no refundable credit is triggered and never evaluated the no-qualifying-child EITC. The head (43) meets the age test, and about $9,665 of net SE earnings puts the household on the $664 plateau." +us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly placed the household on the childless plateau through the head's age, but it used a projected 2026 maximum of $667 instead of the statutory $664." +us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,It correctly identified the plateau childless EITC but used the 2025 maximum of $649 instead of the 2026 maximum of $664. +us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It held that the 20-year-old spouse makes the joint return fail the childless-EITC age test. On a joint return, only one spouse has to be 25-64, and the 43-year-old head qualifies, so the $664 credit applies." +us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It claimed income phase-outs reduce the EITC to zero. AGI of about $9,665 is roughly half the joint childless phase-out start of about $18,000, so no phase-out applies and the full $664 is allowed." +us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly applied the age test through the head and placed the household on the plateau, but it used a projected 2026 maximum of $667 instead of the statutory $664." +us,scenario_057,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly passed the joint age test and placed the household below the phase-out, but it used a 2026 maximum of $666 instead of the statutory $664." +us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,It correctly placed the household on the childless plateau but used a 2026 maximum of $648 instead of the statutory $664. +us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"It denied the childless EITC because the spouse is 20. On a joint return the age test is met when either spouse is 25-64, and the 43-year-old head satisfies it, which yields the $664 plateau credit." +us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"It required both joint filers to be at least 25 for the childless EITC and zeroed the credit over the 20-year-old spouse. The statute requires only the taxpayer or the spouse to be 25-64, and the 43-year-old head qualifies, so the credit is $664." +us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It claimed the head's $10,400 of SSTB self-employment income exceeds an EITC threshold. SSTB status has no bearing on the EITC, and about $9,665 of net earnings sits on the childless plateau below the joint phase-out, which gives $664." +us,scenario_057,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly placed about $9,665 of earned income on the plateau but used a wrong 2025 maximum of $611 (the actual figure was $649) and indexed it to $625. The 2026 childless maximum is $664." +us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed income falls below an EITC threshold and that disability benefits reduce eligibility. The childless EITC phases in from the first dollar of earned income, disability benefits have no effect on it, and about $9,665 of net SE earnings yields the $664 maximum." +us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It denied the EITC because there is no wage income and no qualifying children. Net self-employment earnings count as EITC earned income, and the childless credit applies through the 43-year-old head, which gives $664." us,scenario_057,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that income exceeds Louisiana's Medicaid thresholds, but the household's MAGI is 0.45 x FPL — roughly one third of Louisiana's 138% FPL expansion-adult limit — so no income test excludes the head. It also never applied the SSI-recipient pathway: the head's blindness and disability with $10,400 of self-employment income yields $7,231 of SSI after the $20 general, $65 earned-income, and one-half exclusions, and Louisiana is a 1634 state where SSI receipt confers automatic Medicaid regardless of income." us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the disability flags and the $14 of disability benefits as the only disability-linked inputs and concluded no eligibility determinant was present, never computing SSI from those facts. Blindness plus disability with a non-dependent 43-year-old whose $10,400 self-employment income nets to low countable income under the SSI earned-income exclusions produces $7,231 of SSI, and SSI receipt is itself the Medicaid category (SSI_RECIPIENT) that grants automatic eligibility in Louisiana." us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model enumerated only the pregnancy, parent/caretaker, and aged-65+ pathways and declared none applied, omitting both the SSI-recipient categorical pathway and the fact that a 43-year-old childless adult is exactly who Louisiana's ACA expansion group covers. Its 'at this income level' qualifier inverts the arithmetic: MAGI is 0.45 x FPL, well under the 138% FPL expansion ceiling, and the head's $7,231 of SSI independently triggers automatic Medicaid under Louisiana's 1634 agreement." -us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated the boolean ""is disabled"" flag as sufficient for Medicare (""regardless of age""), skipping the statutory requirement of 24 months of entitlement to Social Security Title II disability benefits under 42 U.S.C. §426(b). PolicyEngine's is_medicare_eligible tests age >= 65 only, and the 43-year-old head fails it." -us,scenario_057,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model chained ""is disabled"" plus $14 of disability benefits into Medicare entitlement, treating any disability payment as SSDI. Only Title II disability entitlement sustained for 24 months opens the under-65 pathway; SSI and other disability income never do, and PolicyEngine's age >= 65 test returns False for a 43-year-old." -us,scenario_057,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated the 24-month SSDI entitlement rule and then abandoned it, substituting ""disabled and receiving disability benefits"" as an independent qualifying condition. Nothing in the facts establishes Title II entitlement or its 24-month duration — the instruction to treat unlisted facts as false rules it out — and PolicyEngine's is_medicare_eligible is the age >= 65 test the 43-year-old head fails." -us,scenario_057,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model conflated SSDI with SSI (""indicating SSDI/SSI receipt""), but SSI receipt confers no Medicare entitlement at any age and the disability-benefits line carries no Title II designation or entitlement history. It also asserted the 24-month waiting period was satisfied from a single-year fact set that establishes no prior entitlement, while PolicyEngine's rule is simply age >= 65." -us,scenario_057,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model invented a ""modeled disability pathway"" in PolicyEngine that does not exist — is_medicare_eligible is the single age >= 65 test — and additionally invoked blindness, which creates no Medicare eligibility route (statutory blindness matters for SSI/SSDI determinations, not Medicare entitlement). The head is 43, so the age test returns False." -us,scenario_057,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model applied a ""disability-beneficiary rule"" that treats mere receipt of a disability benefit as Medicare entitlement, ignoring that only Title II disability entitlement after the 24-month waiting period (or ESRD/ALS) qualifies someone under 65. PolicyEngine's is_medicare_eligible turns solely on age >= 65, which the 43-year-old head fails." -us,scenario_057,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model asserted a ""modeled Medicare disability eligibility pathway"" keyed to the is_disabled flag alone; PolicyEngine implements no such pathway, computing is_medicare_eligible as age >= 65. Even under statute, the disability route requires 24 months of Title II disability entitlement, which the fact set does not supply." -us,scenario_057,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model claimed PolicyEngine has an under-65 Medicare disability pathway triggered by the disabled flag; the variable is the age >= 65 test, so a 43-year-old head returns False. The real-world analogue it was reaching for requires 24 months of SSDI entitlement, a condition the household facts never establish." +us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated being disabled as enough for Medicare at any age. Under 65, Medicare requires 24 months of SSDI entitlement (or ESRD/ALS). The head is 43 and has zero months of Social Security disability receipt, so they fail both the age-65 test and the disability test." +us,scenario_057,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model read the $14 of 'disability benefits' as SSDI entitlement. That input is generic, non-Social Security disability income. The head has zero months of Social Security disability receipt, so they fail the 24-month SSDI requirement, and at 43 they are below the age-65 threshold." +us,scenario_057,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated the rule requiring 24 months of SSDI, then dropped it. It concluded that anyone disabled and 'receiving disability benefits' qualifies, and treated $14 of generic disability benefits as Social Security disability entitlement. No SSDI or months of SSDI receipt are listed, so the 24-month test fails and the 43-year-old head is not eligible." +us,scenario_057,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model used 'disabled and receiving disability benefits' as the Medicare test. PolicyEngine's under-65 pathway requires at least 24 months of Social Security disability receipt, and that input is 0 here. The $14 of generic disability benefits does not satisfy it, and at 43 the head fails the age-65 test." +us,scenario_057,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model stated the 24-month SSDI rule, then inferred SSDI or SSI receipt from $14 of generic disability benefits. SSI never confers Medicare, and the head has zero months of Social Security disability receipt, so the 24-month requirement is not met. The 43-year-old head is therefore not eligible." +us,scenario_057,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated disabled/blind status plus disability benefits as a Medicare pathway. Blindness and disability flags do not confer Medicare; the under-65 pathway requires 24 months of Social Security disability entitlement, which is 0 here. The head is 43, so they fail the age-65 test as well." +us,scenario_057,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model applied a 'disability-beneficiary rule' to the $14 of generic disability benefits. The rule requires at least 24 months of Social Security disability receipt, and the head has none. At 43 the head is also below 65, so they are not Medicare eligible." +us,scenario_057,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed PolicyEngine's disability pathway is triggered by the is_disabled flag. PolicyEngine actually requires 24 or more months of Social Security disability receipt, which is 0 for this head. At 43 the head also fails the age-65 test." +us,scenario_057,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model invoked a disability-based Medicare rule without its core condition: 24 months of Social Security disability entitlement. The head has no SSDI and zero months of receipt, only $14 of generic disability benefits. At 43 they are below 65, so they are ineligible." +us,scenario_057,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model treated disability status alone as meeting PolicyEngine's under-65 Medicare pathway. That pathway requires at least 24 months of Social Security disability receipt, and the input is 0. The 43-year-old head also fails the age-65 test, so they are not eligible." us,scenario_057,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_057,self_employment_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model inverted the §1402(b) de minimis rule, writing that $10,400 of self-employment earnings is ""below the $400 threshold"" when $400 is the floor above which SE tax is imposed and $10,400 exceeds it by a factor of 26. Applying the rule correctly yields net earnings of 0.9235 × $10,400 = $9,604.40 taxed at 15.3% for $1,469.47, not $0." us,scenario_057,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model chose the correct method — 92.35% base times 15.3% — but computed with a rounded 92% factor instead of 0.9235, since $10,400 × 0.92 × 0.153 = $1,463.90, which is exactly the ""about $1,464"" it submitted. The statutory factor is 1 − 0.0765 = 0.9235, giving a base of $9,604.40 and tax of $1,469.47." @@ -4482,143 +4913,161 @@ us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model computed the correct $1,469.46 from $9,604.40 × 15.3%, then added roughly $82.60 of 0.9% Additional Medicare Tax by treating $400 as the AMT threshold; the AMT threshold is $250,000 of combined earnings for joint filers, and the prompt explicitly excludes Additional Medicare Tax from this output. Its arithmetic is also self-contradictory — the stated components sum to $1,552.06, not the $1,531.79 submitted — where the correct answer is the unadjusted $1,469.47." us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model fabricated a ""$12,000 threshold for self-employment tax liability"" and zeroed the tax; no such filing floor exists, and the only threshold in §1402(b) is $400 of net earnings, which $9,604.40 clears. Applying the 15.3% rate to that base gives $1,469.47." us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model derived the correct liability — 0.153 × $9,604.40 = $1,469.47 — and then reported half of it, confusing the §164(f) above-the-line deduction for one-half of SE tax with the tax owed; that deduction reduces AGI and never reduces the §1401 liability itself. Its submitted $734.76 is not even the correct half ($734.74), and the requested output is the full $1,469.47." -us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Built gross income from only $866.67/month of self-employment earnings plus $3.17/month of disability benefits, omitting the $602.58/month SSI payment PolicyEngine imputes to the blind and disabled head, which SNAP counts in full as unearned income. Net income came out at $485 instead of $1,090.08 and the 30% contribution at $145 instead of $327, inflating the allotment from $219 to $380/month; it also used a $536-$540 maximum instead of the FY2026 two-person maximum of $546." -us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI at $1,099/month, the couple federal benefit rate, rather than PolicyEngine's $602.58 — the $995 individual FBR less $392.42 of countable income (half of $866.67 in earnings after the $20 general and $65 earned-income exclusions). That $496/month overstatement raised net income to $1,586 against the correct $1,090.08 and cut the benefit to $70/month instead of $219." -us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Established that the $912 in assets and the very low countable income clear the SNAP limits and then submitted $0, contradicting its own eligibility finding. Net income of $1,090.08 is 62% of the $1,762.50 poverty guideline and gross income of $1,472.42 is 84%, so the household passes both income tests and receives $546 − $327 = $219/month." -us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Omitted the $602.58/month imputed SSI from SNAP countable income and mixed periods by subtracting the $219 monthly standard deduction once from annual income, then submitted $6,240 after its own arithmetic produced $3,990. With SSI counted, annual net income is $13,081, the 30% contribution is $3,924 against a $6,552 annual maximum, and the benefit is $2,669.22." -us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Omitted the $602.58/month imputed SSI from countable income, then discarded its own $4,536 result and submitted $5,736 on the strength of 'disability/medical deductions' that do not apply — the household's $40 of annual OTC expenses is well under the $35-per-month excess-medical threshold for elderly/disabled members. Counting SSI gives net income of $1,090.08 and an allotment of $546 − $327 = $219/month." -us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Asserted net income near zero, ignoring both the $866.67/month of self-employment earnings (which alone leave $484 after the 20% earned and $209 standard deductions) and the $602.58/month imputed SSI that brings net income to $1,090.08. It compounded this with a $304/month maximum allotment where the FY2026 two-person maximum is $546, so the correct result is $546 − $327 = $219/month, not the maximum." -us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Omitted the $602.58/month imputed SSI and then applied a 40% cost-of-production offset to the $10,400 of self-employment income even though no business expenses are listed and unlisted inputs are zero; PolicyEngine counts the full $866.67/month as earned income. Its net income of $205.17/month against the correct $1,090.08 produced $473/month instead of $219." -us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Applied the federal one-half self-employment-tax deduction ($10,400 x 0.9235 x 0.5) to SNAP earned income, a deduction SNAP does not have, and omitted the $602.58/month imputed SSI, then asserted countable income 'roughly $2,000/month' while computing a $297/month benefit its own figures contradict. The correct chain is $866.67 earned + $602.58 SSI + $3.17 unearned, less $173.33 and $209, giving net $1,090.08 and $219/month." -us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Set monthly gross income at $869.84, omitting the $602.58 SSI payment that SNAP counts as unearned income, so net income was $488.51 rather than $1,090.08. The contribution fell to $146.55 instead of $327, and its $536 maximum (versus the FY2026 $546) yielded $389.45/month where the correct allotment is $219." -us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Counted only $866.67 earned and $3.17 unearned per month, omitting the $602.58 imputed SSI, and used the FY2025 parameters ($536 maximum, $204 standard deduction) instead of FY2026's $546 and $209. Net income of $489.34 versus $1,090.08 produced a $146.80 contribution instead of $327 and $389.20/month instead of $219." -us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted SSI at $1,067.17/month instead of PolicyEngine's $602.58 — the $995 individual federal benefit rate less $392.42 of countable income (half of $866.67 in earnings after the $20 general and $65 earned-income exclusions). The $465/month overstatement pushed net income to $1,594.50 against the correct $1,090.08 and cut the monthly benefit from $219 to $57.65." -us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted SSI at $13,122.96/year ($1,093.58/month) against PolicyEngine's $7,230.96, and claimed a $60 excess-medical deduction although the $40 of annual OTC expenses is below the $35-per-month threshold. With a $6,600 annual maximum rather than $546/month, its net income of $19,033 versus the correct $13,081 drove the benefit to $74/month instead of $219." -us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Asserted ineligibility with no computation. The household clears every test — net income of $1,090.08 is 62% of the $1,762.50 poverty guideline, gross income of $1,472.42 is 84%, assets are $911.76, and SSI receipt confers categorical eligibility — so the allotment is $546 − $327 = $219/month for most of the year." -us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Its $76/month benefit backs out to net countable income of about $1,567, which requires SSI near $1,079/month rather than PolicyEngine's $602.58. SSI is offset by $392.42 — half of the head's $866.67 in monthly earnings after the $20 general and $65 earned-income exclusions — leaving $995 − $392.42 = $602.58, net income of $1,090.08, and a $219/month allotment." -us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729 of annual unearned income (SSI at about $1,060.75/month plus disability benefits) where PolicyEngine counts $7,269 ($602.58 SSI + $3.17 disability per month). The roughly $455/month overstatement raised net income above $1,540 against the correct $1,090.08, cutting the benefit from $219/month to about $65." -us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Declared a zero benefit under 'Louisiana rules' without computing income or assets. Louisiana applies the federal tests, all of which this household passes: net income of $1,090.08 is 62% of the poverty guideline, gross income of $1,472.42 is 84%, and assets are $911.76, giving $546 − $327 = $219/month." -us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave no derivation; its $3,216 annual figure is $268/month, which backs out to net countable income of about $927 against the $546 FY2026 two-person maximum. That understates the $1,090.08 net income that results once the $602.58/month imputed SSI is added to the $866.67 of earnings, so the correct allotment is $219/month for nine months and $232.74 for three." -us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave no derivation; its $3,384 annual figure is $282/month, implying net countable income near $880 against the $546 maximum. Counting the $602.58/month imputed SSI as unearned income alongside $866.67 of earnings, less $173.33 and $209, gives net income of $1,090.08, a $327 contribution, and $219/month." -us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Set monthly gross income at $869.83, omitting the $602.58 SSI payment SNAP counts as unearned income, and used a $200 standard deduction instead of $209. Net income of $496.50 versus $1,090.08 left a $148.95 contribution instead of $327, so the $546 maximum produced $397.05/month rather than $219." -us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Refused to compute SNAP, treating it as unlisted benefit receipt to be zeroed; that instruction governs input facts, not the requested outputs, and snap was explicitly asked for. The household is eligible on the net income test (62% of the poverty guideline) and receives $219/month for nine months and $232.74 for three, $2,669.22 for the year." -us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted SSI at about $1,100/month instead of the income-reduced $602.58 and used a $230 standard deduction instead of $209, pushing net income to about $1,566 against the correct $1,090.08. That turned the $327 expected contribution into roughly $470 and the allotment into $66/month instead of $219." -us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Declared counted income too high without computing it. Gross income of $1,472.42 is 84% of the $1,762.50 poverty guideline and net income of $1,090.08 is 62%, so the household passes both the gross and net income tests, holds $911.76 in assets, and receives $546 − $327 = $219/month." -us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Declined to compute, stating no SNAP amount is provided by the facts, when the task is to derive the benefit from those facts. The derivation is $866.67 earnings + $602.58 imputed SSI + $3.17 disability, less $173.33 and $209, giving net $1,090.08 and an allotment of $219/month rising to $232.74 in the FY2027 months." -us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Used annual gross income of $10,438 and omitted the $602.58/month imputed SSI, so net income came to $5,862 rather than $13,081 for the year. Its $6,552 annual maximum was right, but the $1,758.60 contribution instead of $3,924 turned a $219/month benefit into $399.45." -us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Its $340/month figure backs out to net countable income of about $687, which requires leaving the $602.58/month imputed SSI out of SNAP unearned income. It also invoked 'allowable medical deductions' although the $40 of annual OTC expenses falls below the $35-per-month excess-medical threshold; with SSI counted, net income is $1,090.08 and the allotment is $219/month." -us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Its $70/month benefit off the $546 maximum implies net countable income of $1,586.67, which comes from carrying SSI at roughly $1,100/month — the couple federal benefit rate — instead of PolicyEngine's $602.58 ($995 individual rate less $392.42 of countable earnings). The correct net income of $1,090.08 gives a $327 contribution and $219/month." -us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI at roughly $1,100/month, the couple federal benefit rate, so net income reached $1,586.67 and the $546 maximum left $70/month. PolicyEngine pays the $995 individual rate less $392.42 of countable income (half of $866.67 in earnings after the $20 and $65 exclusions), giving SSI of $602.58, net income of $1,090.08, and $219/month." -us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Set annual SNAP income at $23,621, which embeds SSI of $13,183 ($1,098.58/month) — the $1,491 couple federal benefit rate less the $392.42 earnings offset — where PolicyEngine applies the $995 individual rate less that same offset for $602.58/month. The $496/month overstatement raised net income to $19,033 against $13,081 and cut the benefit from $219/month to $70.18." -us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that income or assets exceed the limits with no computation. Assets are $911.76, gross income of $1,472.42 is 84% of the $1,762.50 poverty guideline, and net income of $1,090.08 is 62%, so the household passes every test and receives $546 − $327 = $219/month for most of the year." -us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Its $1,928 monthly gross embeds SSI of about $1,058 rather than the income-reduced $602.58, so net income landed at $1,551 instead of $1,090.08 and the contribution at about $465 instead of $327. SSI here is the $995 individual federal benefit rate less $392.42 (half of $866.67 in earnings after the $20 general and $65 earned exclusions), and the correct allotment is $219/month." -us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Its $1,955 monthly gross embeds SSI near $1,085 instead of $602.58, and it used the FY2025 parameters ($536 maximum, $204 standard deduction) rather than FY2026's $546 and $209. Net income of $1,578 against the correct $1,090.08 dropped the benefit to $63/month where PolicyEngine pays $219." -us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Omitted the $602.58/month imputed SSI from countable income and subtracted the $209 monthly standard deduction once from annual income, using a $6,600 annual maximum instead of $6,552. Counting SSI gives annual net income of $13,081, a $3,924 contribution, and $2,669.22 rather than $4,155." -us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Set monthly gross income at about $870, omitting the $602.58 SSI payment SNAP counts as unearned income, so net income was $488 rather than $1,090.08. Its $546 maximum and $209 standard deduction were correct, but the $146 contribution instead of $327 produced $399/month where PolicyEngine pays $219." -us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value or explanation for snap, so no substantive computation reached the grader. The correct derivation is net income of $1,090.08 (gross $1,472.42 including $602.58 of SSI, less the $173.33 earned deduction and $209 standard deduction), giving $546 − $327 = $219/month for nine months and $558.24 − $325.50 = $232.74 for three, $2,669.22 annually." -us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple federal benefit rate to derive SSI of $1,098.58/month ($13,183/year) after the $392.42 countable-income offset, where PolicyEngine applies the $995 individual rate less that same offset for $602.58/month. Its $209 standard deduction and $546 maximum were correct, but the $496/month SSI overstatement raised net income to $1,586.08 instead of $1,090.08 and cut the benefit from $219 to $70.18/month." -us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Concluded that $912 in bank assets exceeds an asset limit it stated as about $4,500, a conclusion its own numbers contradict. The household passes the asset test with $911.76 and, with net income at 62% of the poverty guideline, receives $546 − $327 = $219/month for most of the year." -us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted $867 earned plus $3 unearned per month and omitted the $602.58 imputed SSI that SNAP treats as unearned income, so net income was about $488 rather than $1,090.08. Its $546 maximum and $209 standard deduction were correct, but the $146 contribution instead of $327 produced $400/month where the allotment is $219." -us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Excluded all household earned and unearned income on the theory that SNAP disregards the income of SSI-eligible individuals; SNAP counts SSI itself as unearned income and counts an SSI recipient's earnings subject only to the 20% earned-income deduction, with SSI receipt conferring categorical eligibility alone. It also used a $635/month maximum instead of $546, so it paid the full allotment where net income of $1,090.08 leaves $219/month." -us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that the household does not qualify for a positive benefit with no computation. Net income of $1,090.08 is 62% of the $1,762.50 poverty guideline, gross income of $1,472.42 is 84%, assets are $911.76, and SSI receipt confers categorical eligibility, so the allotment is $546 − $327 = $219/month for nine months and $232.74 for the last three." +us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Built gross income from only the $866.67/month of self-employment earnings and $3.17 of disability benefits and never counted the head's $602.58/month SSI as unearned income. That left net income at about $485 instead of $1,090.08. With SSI counted, the $546 allotment minus a $327 contribution gives $219/month, then $232.74 from October, not the ~$380/month it annualized." +us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,099/month, the couple rate that treats both spouses as SSI recipients. The household's SSI is the head's individual payment of $602.58/month ($994 FBR minus $391.42 countable). The inflated SSI pushed net income to $1,586 instead of $1,090.08, which cut the benefit to ~$70/month instead of $219/month (January–September) and $232.74/month (October–December)." +us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Asserted a $0 benefit because income was 'slightly exceeding the net income limit' but never computed net income, and it wrongly shrank self-employment income by the 92.35% SE-tax factor. Net income is $1,090.08, which is 62% of the $1,762.50 guideline, so the household passes and receives $546 − $327 = $219/month, rising to $232.74 in October–December." +us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of countable income and subtracted a monthly $219 standard deduction from an annual income figure. It then discarded its own $3,990 result and submitted $6,240 with no derivation. The correct monthly net income including SSI is $1,090.08, which yields $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Left the head's $602.58/month SSI out of gross income and inflated the $38 annual disability benefits to $456 by treating them as monthly. It then added about $1,200 for unspecified 'disability/medical deductions', even though the $40/year of OTC expenses never exceeds the $35/month medical threshold. Including SSI, net income is $1,090.08 and the benefit is $219/month, then $232.74." +us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Asserted net income near zero and used a two-person maximum of about $304, but the FY2026 two-person maximum is $546. The $866.67/month of self-employment earnings plus the head's $602.58/month SSI leave $1,090.08 of net income after the 20% earned-income and $209 standard deductions. That produces a $327 contribution and a $219/month benefit, not a near-maximum allotment." +us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of $1,098.58/month, the couple rate with both spouses as SSI recipients, instead of the head's individual SSI of $602.58/month ($994 FBR minus $391.42 countable income). That overstated net income at $1,586 instead of $1,090.08 and produced $70/month instead of $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Excluded the head's $602.58/month SSI from SNAP income. It also cut self-employment income with an invented 40% cost-of-production deduction, counting $6,240 instead of the full $10,400 ($866.67/month). The correct gross income of $1,472.42 and net income of $1,090.08 give $219/month, not its $473/month." +us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Reduced self-employment income by an income-tax-style SE-tax adjustment, treated the $38 annual disability benefits as monthly, and omitted the head's $602.58/month SSI. It then reported $297/month without any arithmetic connecting that figure to its stated ~$2,000 net income. The full $866.67/month of earnings plus $602.58 of SSI and $3.17 of benefits gives net income of $1,090.08 and a $219/month benefit." +us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,099/month at the couple rate instead of the head's individual SSI of $602.58/month. That raised net income to $1,586 instead of $1,090.08 and left a ~$70/month benefit instead of $546 − $327 = $219/month, plus $232.74/month for October–December." +us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Counted only $869.84/month of earnings and disability benefits and omitted the head's $602.58/month SSI. Net income came to $488.51 instead of $1,090.08, and the benefit to ~$389/month instead of $219 (January–September) and $232.74 (October–December). It also used a $536 maximum instead of the FY2026 two-person $546." +us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Built gross income of $870/month from earnings and disability benefits alone and never added the head's $602.58/month SSI. It also used a $204 standard deduction instead of $209 and a $536 maximum instead of $546. Its net income of $489.34 compares with the correct $1,090.08, which yields $219/month rather than $389.20." +us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted SSI of $1,067.17/month at the couple rate, as if both spouses were recipients, and treated the $38 annual disability benefits as monthly. The household's SSI is only the head's $602.58/month individual payment. The overstated net income of $1,594.50 (vs $1,090.08) cut the benefit to $57.65/month instead of $219." +us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Omitted the head's $602.58/month SSI from gross income and used a $204 standard deduction instead of $209. That gave net income of $492.50 instead of $1,090.08 and a $398/month benefit instead of $546 − $327 = $219/month (then $232.74 in October–December)." +us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted $13,122.96/year of SSI at the couple rate instead of the head's individual $602.58/month ($7,230.96/year). It also used a $198/month standard deduction instead of $209, and claimed a $60 medical deduction even though the $40/year of OTC expenses never exceeds the $35/month threshold. The correct net income of $1,090.08/month gives $219/month, not about $74." +us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Declared the household ineligible with no computation. Net income of $1,090.08 is 62% of the poverty guideline, gross income is 84%, and the $912 of assets is far under the limit. The household therefore receives $546 − $327 = $219/month, then $232.74, totaling $2,669.22." +us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Included SSI at the couple rate of about $1,099/month rather than the head's individual $602.58/month. Its $76/month benefit reflects net income near $1,567 instead of $1,090.08; the correct $327 contribution against the $546 maximum leaves $219/month." +us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729/year of unearned income, which is couple-rate SSI for both spouses plus $38 of disability benefits. The household's SSI is only the head's $602.58/month individual payment. The excess SSI drove the benefit down to about $65/month instead of $219/month (January–September) and $232.74/month (October–December)." +us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted a zero benefit without applying any test. The household passes the gross (84% of FPL), net (62% of FPL), and asset ($912) tests. Net income of $1,090.08 yields $546 − $327 = $219/month, then $232.74 from October." +us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave $268/month with no derivation; against the $546 maximum, that implies net income of about $927. The correct net income is $1,090.08: add the head's $602.58/month SSI to $866.67 of earnings and $3.17 of benefits, then subtract the 20% earned-income and $209 standard deductions. That gives $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave $282/month with no derivation; against the $546 maximum, that implies net income of about $880 instead of $1,090.08. It undercounted income by not including the head's full $602.58/month SSI alongside $866.67 of earnings and $3.17 of benefits. The correct $327 contribution leaves $219/month, then $232.74 in October–December." +us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Computed gross income as $869.83/month from self-employment and disability benefits only, omitting the head's $602.58/month SSI. It also used a $200 standard deduction instead of $209. Its net income of $496.50 compares with the correct $1,090.08, which yields $219/month rather than $397.05." +us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit-receipt input and set it to 0 instead of computing it, even though the prompt tells models to assume program take-up. Computing it gives net income of $1,090.08 and a benefit of $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted SSI of about $1,100/month at the couple rate instead of the head's individual $602.58/month. It also used a ~$230 standard deduction instead of $209 and a $536 maximum instead of $546. Its net income of ~$1,566 compares with the correct $1,090.08, which gives $219/month rather than ~$66." +us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed counted income was too high for SNAP without computing it. Gross income of $1,472.42 is 84% of the guideline and net income of $1,090.08 is 62%, so both tests pass. The benefit is $546 − $327 = $219/month, then $232.74 from October." +us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted benefit amount that defaults to 0 instead of computing the entitlement, even though take-up is assumed. The computed benefit from net income of $1,090.08 is $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Used annual gross income of $10,438 from self-employment and disability benefits only, omitting the head's $7,230.96/year ($602.58/month) of SSI. It also used a $208/month standard deduction instead of $209. Its net income of $488.50/month compares with the correct $1,090.08, which gives $219/month rather than ~$400." +us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Its $340/month benefit implies net income of about $687. It excluded the head's $602.58/month SSI and claimed an 'allowable medical deduction', even though the $40/year of OTC expenses falls below the $35/month threshold. Correct net income is $1,090.08, giving $219/month and then $232.74." +us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Its $70/month benefit reflects about $1,586 of net income, which comes from counting couple-rate SSI of about $1,099/month. The household's SSI is the head's individual $602.58/month, which gives net income of $1,090.08 and $546 − $327 = $219/month (then $232.74)." +us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI at the couple rate (~$1,099/month) instead of the head's individual $602.58/month, which produced a $70 monthly allotment. The correct net income of $1,090.08 yields $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Estimated SSI of about $13,183/year at the couple rate instead of the head's individual $7,230.96/year ($602.58/month). That produced annual SNAP income of $23,621 and net income of $19,033 instead of $1,090.08/month, which cut the benefit to $842.10 instead of $219/month plus $232.74/month for October–December." +us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Computed net income of $487.67/month from self-employment earnings and disability benefits alone, never adding the head's $602.58/month SSI. With SSI, net income is $1,090.08 and the $546 maximum less a $327 contribution gives $219/month, not ~$400." +us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Its $842.10 result reflects monthly net income of about $1,586, which comes from counting SSI at the couple rate of ~$1,099/month. The household's SSI is the head's individual $602.58/month, which gives net income of $1,090.08 and $219/month (then $232.74)." +us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Estimated SSI at the couple rate, giving countable income of $1,586.08. The household's SSI is only the head's $602.58/month individual payment ($994 FBR minus $391.42 countable), so net income is $1,090.08 and the benefit is $546 − $327 = $219/month, then $232.74 in October–December." +us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed income and assets exceed SNAP limits. Gross income is 84% of the guideline, net income is 62%, and assets are $912. All tests pass, and the benefit is $546 − $327 = $219/month, then $232.74." +us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Put monthly gross income at about $1,928 by including roughly $1,058/month of couple-rate SSI instead of the head's individual $602.58. It also used a $204 standard deduction instead of $209. Its net income of ~$1,551 compares with the correct $1,090.08, which gives $219/month rather than ~$71." +us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Counted about $1,085/month of SSI at the couple rate, putting gross income at $1,955 instead of $1,472.42. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546. The correct net income of $1,090.08 yields $219/month, not $63." +us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Counted $13,123/year of couple-rate SSI instead of the head's individual $7,230.96/year ($602.58/month), which drove net income to $18,973/year. The correct monthly net income of $1,090.08 gives $219/month for January–September and $232.74 for October–December." +us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Omitted the head's $602.58/month SSI and mixed periods by subtracting a single month's $209 standard deduction from annual income. It then took 30% of that $8,149 'annual net' against a $6,600 annual maximum. The correct monthly figures are gross income of $1,472.42 and net income of $1,090.08, giving $219/month." +us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Put gross income at about $870/month from self-employment and disability benefits and left out the head's $602.58/month SSI. Its net income of about $488 compares with the correct $1,090.08, which gives $219/month rather than $399." +us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so nothing was scorable. The correct derivation counts $866.67 of earnings, $3.17 of benefits, and $602.58 of SSI, giving net income of $1,090.08. That yields $219/month for January–September and $232.74 for October–December, totaling $2,669.22." +us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Estimated SSI at $13,183/year, the couple rate with both spouses as recipients, instead of the head's individual $7,230.96/year ($602.58/month). That produced monthly net income of $1,586.08 instead of $1,090.08 and a benefit of ~$70/month instead of $219, plus $232.74/month for October–December." +us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Claimed the $912 of bank assets exceeds the ~$4,500 disabled-household asset limit, but $912 is well below that limit, so the asset test passes. With net income of $1,090.08, the benefit is $546 − $327 = $219/month, then $232.74." +us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted only $867 of earnings and $3 of unearned income per month and omitted the head's $602.58/month SSI. Its net income of about $488 compares with the correct $1,090.08, which gives $219/month rather than ~$400." +us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Excluded all income of members it deemed SSI-eligible, but SNAP has no such exclusion. The head's $866.67/month of earnings and $602.58/month of SSI both count, giving net income of $1,090.08. It also used a $635 maximum instead of the FY2026 two-person maximum of $546." +us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted ineligibility without applying any test. The household passes the gross (84% of FPL), net (62% of FPL), and asset tests and is categorically eligible through SSI. The benefit is $546 − $327 = $219/month, then $232.74 from October." us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold." us,scenario_057,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that income exceeds Louisiana's Medicaid limits, but household MAGI is 0.45 x FPL — roughly $10,438 against a 138% FPL expansion cutoff near $29,200 for a household of two in 2026 — so it compared the $10,400 self-employment figure against a threshold far lower than the one the ACA adult expansion actually applies. It also gave the spouse's disability status weight while never running the only test that governs a 20-year-old in an expansion state: MAGI at or below 138% FPL under the ADULT category." us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid as requiring an affirmative qualifying attribute in the fact pattern and defaulted to 0 in its absence, missing that Louisiana's ACA adult expansion makes age 19–64 plus MAGI at or below 138% FPL sufficient on its own. The listed facts fully determine eligibility — spouse age 20, non-dependent, household MAGI at 0.45 x FPL — so the ADULT expansion category resolves to eligible without any additional detail." us,scenario_057,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model applied pre-ACA categorical requirements — pregnancy or parenthood of a minor child — as prerequisites for a childless adult, but Louisiana expanded Medicaid in 2016 and covers childless adults 19–64 at or below 138% FPL with no categorical link. Its added claim that expansion fails 'at this income level' inverts the test: household MAGI of 0.45 x FPL is roughly a third of the 138% expansion limit, which is precisely why the ADULT category grants eligibility." -us,scenario_057,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model invented a ""modeled Medicare eligibility rule"" under which disability confers Medicare regardless of age; PolicyEngine's `is_medicare_eligible` is a single age test (age >= 65) with no disability branch, so the spouse's `is_disabled` flag carries zero weight. At age 20 the spouse fails the only condition the variable evaluates, yielding 0." -us,scenario_057,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model imported the real-world Title II pathway (24 months of SSDI entitlement before Medicare attaches) and then manufactured satisfaction of the 24-month waiting period by stretching the prompt's ""treat facts as constant throughout the tax-benefit year"" instruction into a claim about pre-year entitlement history, which that instruction does not cover. PolicyEngine models Medicare eligibility solely as age >= 65, and the $24 annual disability-benefit amount is orders of magnitude below any SSDI entitlement, so the spouse's value is 0." -us,scenario_057,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model asserted a ""modeled disability pathway"" triggered by the `is_disabled` flag plus disability-benefit receipt; no such branch exists in PolicyEngine's `is_medicare_eligible`, which returns true only when age >= 65. The spouse is 20, so the age test fails and the correct value is 0." -us,scenario_057,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model applied a ""disability-beneficiary rule"" keyed to the listed $24 of disability benefits, treating any disability-benefit receipt as Title II SSDI entitlement that carries Medicare. PolicyEngine's Medicare eligibility variable tests age >= 65 only, and $24 per year is not an SSDI benefit level, so a 20-year-old spouse is not eligible and the value is 0." -us,scenario_057,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed a ""modeled Medicare disability eligibility pathway"" satisfied by the spouse's disability status alone, with no age condition. PolicyEngine's `is_medicare_eligible` implements only the age >= 65 requirement, so the 20-year-old spouse returns 0 regardless of disability." -us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model named a specific ""PolicyEngine under-65 Medicare disability eligibility pathway"" that does not exist in the model; `is_medicare_eligible` is computed as age >= 65 with no disability, ESRD, or ALS branch. Applying the actual rule to a 20-year-old spouse gives 0." +us,scenario_057,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model said disabled people qualify for Medicare regardless of age. Under-65 eligibility actually requires 24 months of SSDI entitlement (or ESRD/ALS), not just disabled status. The spouse has no SSDI and zero months of receipt, so they are not eligible." +us,scenario_057,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated the 24-month SSDI waiting-period rule. It then treated the $24 of generic disability benefits as SSDI and assumed the 24 months had been met. The prompt says unlisted numeric inputs are 0, so SSDI months are 0 and the disability pathway fails." +us,scenario_057,spouse_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model counted ""disabled and receiving disability benefits"" as Medicare eligibility. The $24 is non-SSDI disability income, and Medicare's disability pathway requires 24 months of SSDI entitlement, which the 20-year-old spouse does not have." +us,scenario_057,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model applied the disability pathway because the spouse is disabled and has disability benefits. That pathway requires 24 months of SSDI entitlement (or ESRD/ALS), and the $24 of generic disability benefits is none of these. The spouse fails both the age-65 and SSDI tests." +us,scenario_057,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model treated the $24 of listed disability benefits as meeting the ""disability-beneficiary rule."" That rule counts only Social Security disability entitlement lasting 24 months, and the spouse has zero SSDI months, so under-65 eligibility fails." +us,scenario_057,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model granted Medicare eligibility ""because the spouse is disabled."" Disabled status alone does not qualify anyone under 65; 24 months of SSDI receipt (or ESRD/ALS) is required. The spouse has none, and at 20 is far below the age-65 threshold." +us,scenario_057,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model invoked a disability-based Medicare rule for a 20-year-old without checking what that rule requires: 24 months of SSDI entitlement. The spouse has only generic disability benefits and zero SSDI months, so they are not eligible." +us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model said being disabled satisfies the under-65 Medicare pathway. That pathway requires 24 months of SSDI entitlement (or ESRD/ALS), not disabled status alone. The spouse has no SSDI months, so under-65 eligibility fails." us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility." us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present." -us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted an explicit placeholder (""Not calculated — placeholder before analysis"") instead of computing SSI, so no rule was applied at all. The required derivation is the eligible-individual FBR of $994/month ($11,928/year) less countable income of $4,697, which yields $7,231." -us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated the 20-year-old spouse as an SSI-eligible individual and applied the eligible-couple FBR of $17,892, but the spouse is not SSI aged/blind/disabled, so the head is an eligible individual with an ineligible spouse and the $11,928 individual FBR governs. It also pooled the spouse's $24 into the head's unearned income; only the head's own $14 counts, leaving $226 of the general exclusion to offset earnings and producing countable income of $4,697, not $4,709." -us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Applied only the $780 annual earned-income exclusion and omitted the 50% earned-income disregard entirely, reaching $9,620 of countable earnings and declaring the household over the income limit. The disregard halves earnings after exclusions to ($10,400 − $226 − $780)/2 = $4,697, far below the $11,928 individual FBR, so SSI is payable at $7,231." -us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Applied an eligible-couple FBR of ~$17,604 on the premise that ""Spouse age 20 and disabled qualifies,"" but the spouse is not SSI aged/blind/disabled and the head is an eligible individual with an ineligible spouse whose $24 of income falls below the couple/individual FBR differential, so nothing is deemed and the $11,928 individual FBR applies. Substituting the individual rate for its $5,175.50 countable-income figure collapses its $12,428 to roughly the reference $7,231." -us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Used an eligible-couple FBR of $18,456 rather than the $11,928 individual rate that applies to an eligible individual with an ineligible spouse, and counted the spouse's $24 in the head's unearned income. Its own subtraction gave $14,271.50 yet it submitted $13,628, so the number reported matches neither its arithmetic nor the correct $11,928 − $4,697 = $7,231." -us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Converted the $10,400 of self-employment income to ~$9,600 of net earnings and asserted it exceeds the couple FBR ""after exclusions,"" skipping the 50% earned-income disregard that is the dominant SSI exclusion. PolicyEngine counts the full $10,400 as earned income, subtracts the $226 residual general exclusion and $780 earned exclusion, and halves the remainder to $4,697, leaving $11,928 − $4,697 = $7,231." -us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Concluded the couple FBR applies because ""they are a married couple living together,"" but the spouse is not SSI aged/blind/disabled, so the head is an eligible individual with an ineligible spouse and the $11,928 individual FBR governs. It then abandoned its own correct general-exclusion carryover and used $4,828 of countable income; the correct figure counting only the head's $14 unearned is $4,697." -us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Derived a positive benefit of ~$12,590/year in its own reasoning and then submitted 0, so the reported value contradicts its stated computation. Its derivation was also built on the eligible-couple FBR of $1,450/month; the head is an eligible individual with an ineligible spouse, so $994/month less $391.42 of countable monthly income gives $602.58/month, or $7,231 annually." -us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Paid the spouse a near-full individual benefit of ~$991/month and added it to the head's payment, but the 20-year-old spouse is not SSI aged/blind/disabled and receives $0. Only the head's payment exists: $994/month less $391.42 countable, or $7,231 annually." -us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Its exclusion sequence was right in form but it applied the general exclusion against $3.17/month of pooled unearned income including the ineligible spouse's $24, and it used an eligible-couple FBR of $1,483/month. Only the head's $14/year ($1.17/month) is unearned income and the individual FBR of $994/month applies, giving $994 − $391.42 = $602.58/month, or $7,231." -us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Declared ""both spouses are SSI-eligible disabled/blind adults"" and applied the $1,486/month eligible-couple FBR, but the spouse is not SSI aged/blind/disabled, making the head an eligible individual at $994/month. It also dropped the residual general exclusion, using $418.83/month countable instead of $391.42; the correct payment is $602.58/month, or $7,231." -us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Applied the $17,832 eligible-couple FBR to a household where the spouse is not SSI aged/blind/disabled, so the $11,928 eligible-individual rate governs. It also double-counted the general exclusion, charging $18 of countable unearned income while simultaneously carrying the unused portion into the earned-income offset; the head's own $14 is fully excluded, leaving countable income of $4,697 and SSI of $7,231." -us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Asserted that ""the household's earned income disqualifies them from federal SSI"" without applying the $20 general exclusion, the $65 earned exclusion, and the 50% earned-income disregard. Those reduce $10,400 of earnings to $4,697 of countable income, which is below the $11,928 individual FBR and leaves $7,231 payable." -us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Subtracted countable income from ""the SSI couples maximum benefit,"" but the 20-year-old spouse is not SSI aged/blind/disabled, so the head is an eligible individual with an ineligible spouse and the $11,928 individual FBR applies. Using the individual rate with countable income of $4,697 gives $7,231, not $13,532." -us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Used a $17,400 eligible-couple FBR on the premise that ""both the head and spouse are disabled and eligible for SSI as a couple,"" while the spouse fails the SSI aged/blind/disabled test and receives $0. Its countable-income figure of $4,709 also folded in the spouse's $24; counting only the head's $14 unearned gives $4,697, and $11,928 − $4,697 = $7,231." -us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Claimed both income and resources exceed SSI limits, but the $912 bank balance is below the $2,000 individual resource limit and countable income after the $240 general exclusion, $780 earned exclusion, and 50% disregard is $4,697. That is below the $11,928 individual FBR, leaving $7,231 payable." -us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Treated the pair as qualifying for ""an annual SSI couple benefit"" although the spouse is not SSI aged/blind/disabled and receives $0. SSI here is the head's individual benefit alone: the $994/month FBR less $391.42 of countable income, or $7,231 annually." -us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Stated that both adults ""meet SSI financial eligibility requirements as an eligible couple"" and applied the ~$17,832 couple FBR, but the 20-year-old spouse fails the SSI aged/blind/disabled test. The head is an eligible individual with an ineligible spouse, so $11,928 − $4,697 = $7,231." -us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Declined to compute, asserting SSI ""is not simulated or reported for this household,"" when the engine pays the head an SSI benefit. The required computation is the $994/month individual FBR less $391.42 of countable monthly income, or $7,231 annually." -us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Grounded its zero on the absence of listed SSI receipt and a guess that income and resources exceed the limits, contradicting the instruction to assume take-up and ignoring the exclusion arithmetic. Resources of $912 clear the $2,000 individual limit and countable income of $4,697 is below the $11,928 individual FBR, so SSI is $7,231." -us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Its countable income of $390.83/month is essentially the correct $391.42, but it multiplied against a $1,490.56/month eligible-couple FBR when the spouse is not SSI aged/blind/disabled. The head is an eligible individual at $994/month, so the payment is $602.58/month, or $7,231 annually." -us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Asserted ""assets above the SSI asset limit,"" but $912 in bank assets is well under the $2,000 countable-resource limit for an individual, which the engine confirms as passed. It also never applied the 50% earned-income disregard that brings countable income to $4,697, below the $11,928 individual FBR, leaving $7,231." -us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SSI as an input that must be stated in the facts rather than a quantity to compute, submitting 0 because ""SSI eligibility/receipt"" was not specified. The head qualifies as blind and disabled, and the individual FBR of $11,928 less countable income of $4,697 gives $7,231." -us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491/month eligible-couple rate ($17,892/year) because ""both spouses are disabled,"" but the 20-year-old spouse is not SSI aged/blind/disabled, so the head is an eligible individual at $994/month. Its countable income of $4,709 also absorbed the spouse's $24; only the head's $14 counts, giving $4,697 and SSI of $7,231." -us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied ""the couple federal benefit rate"" on the assumption that both adults meet the disability criteria, while the spouse fails the SSI aged/blind/disabled test and receives $0. With the eligible-individual FBR of $11,928 and countable income of $4,697, the benefit is $7,231." -us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Reduced ""the 2026 couple maximum"" of $17,892 by countable income, but the spouse is not SSI aged/blind/disabled, so the head is an eligible individual with an ineligible spouse and the $11,928 individual FBR governs. It also counted the spouse's $24 as household unearned income; the correct countable total from the head's own $14 and $10,400 of earnings is $4,697, giving $7,231." -us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Used the eligible-couple FBR on the stated premise that ""both adults are disabled,"" but the 20-year-old spouse is not SSI aged/blind/disabled and gets $0. The head's individual computation is $11,928 less $4,697 of countable income, or $7,231." -us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Measured countable income against the ""$17892 annual couple maximum"" although the spouse is not SSI aged/blind/disabled, so the $11,928 eligible-individual FBR applies. Its exclusion formula also charged the spouse's $24 against the general exclusion; using only the head's $14 gives ($10,400 − $780 − $226)/2 = $4,697 and SSI of $7,231." -us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Concluded that countable self-employment income ""exceeds SSI federal benefit rate limits for couple"" without applying the 50% earned-income disregard, which halves post-exclusion earnings. Countable income is $4,697, below the $11,928 eligible-individual FBR, so SSI is $7,231." -us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Its monthly countable income of ~$393 nearly matches the correct $391.42, but it subtracted it from a projected couple FBR of $1,486 when the spouse is not SSI aged/blind/disabled. The head is an eligible individual at $994/month, yielding $602.58/month, or $7,231 annually rather than $13,123." -us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Applied the $17,832 couple FBR on the premise of a ""married couple living together,"" but SSI eligibility is per person and the spouse fails the aged/blind/disabled test, so the $11,928 individual rate applies. It also dropped the residual general exclusion, using $4,810 countable instead of $4,697; the correct benefit is $7,231." -us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Used a ~$17,835 couple FBR because ""both disabled with resources under the limit,"" while the 20-year-old spouse is not SSI aged/blind/disabled and receives $0. Against the $11,928 eligible-individual FBR with countable income of $4,697, SSI is $7,231." -us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Applied a ~$1,490/month couple rate rather than the $994/month eligible-individual FBR that governs an eligible individual with an ineligible spouse. It also understated countable earnings at ~$370/month against the correct $391.42, so the payment is $602.58/month, or $7,231 annually." -us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for ssi, so no substantive computation was submitted. The correct derivation is the $11,928 eligible-individual FBR less $4,697 of countable income, or $7,231." -us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $1,491/month couple benefit rate ($17,892/year) after concluding both spouses are disabled, but the spouse is not SSI aged/blind/disabled and the head is an eligible individual with an ineligible spouse. Its $4,709 countable figure also swept in the spouse's $24; counting only the head's $14 unearned gives $4,697 and SSI of $11,928 − $4,697 = $7,231." -us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Reasoned that the small $14 and $24 disability benefit amounts indicate the adults are not substantively disabled and therefore get nothing, but SSI eligibility rests on the stated blind/disabled status, not on the size of other disability payments. The head qualifies, and $11,928 less $4,697 of countable income gives $7,231." -us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Its exclusion work is correct to within a dollar — $390.83/month countable against the reference $391.42 — but it subtracted that from the $1,491/month eligible-couple FBR instead of the $994/month eligible-individual rate, since the spouse is not SSI aged/blind/disabled. The $497/month FBR differential is the entire gap: $994 − $390.83 gives roughly $603/month, or the reference $7,231." -us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Cycled through fabricated federal benefit rates ($979, $674, $706, $1,000, $1,034, $1,058 per month) and awarded the ineligible 20-year-old spouse a near-full individual payment of $8,040, then submitted $8,472, a figure matching none of its own computations. The correct rate is the $994/month eligible-individual FBR with the spouse at $0, giving $994 − $391.42 = $602.58/month, or $7,231." -us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that the listed income and resources produce no positive benefit without performing the exclusion arithmetic. Resources of $912 pass the $2,000 individual test, and $10,400 of earnings plus $14 of unearned income reduce to $4,697 of countable income after the $240 general exclusion, $780 earned exclusion, and 50% disregard, leaving $11,928 − $4,697 = $7,231." +us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted a placeholder 0 without doing any SSI calculation. The head is an eligible blind individual with $912 in resources, under the $2,000 limit, and has $391.42/month of countable income after the $20 general exclusion, the $65 earned-income exclusion, and the one-half earnings disregard. That gives $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated the spouse as SSI-disabled and applied the eligible-couple FBR of $17,892/year to pooled couple income of $4,709. The spouse is not aged/blind/disabled for SSI, so the head is paid from the $11,928 individual FBR. The spouse's $24/year is not deemed, and the head's own countable income is $4,697, which gives $7,231." +us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Left out the one-half earned-income disregard, counting $9,620 of earnings instead of about half that, and treated the $14 and $24 annual disability payments as monthly amounts. It then called the household ineligible even though its own $10,076 was below the couple rate it cited. The correct count is ($866.67 − $18.83 − $65) / 2 = $391.42/month against the $994 individual FBR, giving $7,231." +us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Counted the 20-year-old spouse as a qualifying disabled individual and subtracted pooled income from an assumed couple FBR of about $17,604. The spouse fails the SSI aged/blind/disabled test, so only the head is eligible, at the $994/month individual rate. With $391.42/month of countable income, the head receives $7,231/year." +us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Applied an assumed eligible-couple FBR of $1,538/month because it treated both spouses as SSI-disabled, then replaced its own $14,271.50 result with an unexplained $13,628. The spouse is not SSI aged/blind/disabled, so the head is paid $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Said countable self-employment income of about $9,600 exceeded the benefit rate, which means it never applied the one-half earned-income disregard. After the exclusions, the head's countable income is only $391.42/month, well below the $994 individual FBR, so SSI is $602.58/month, or $7,231/year." +us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the household as an eligible couple, used the $1,491 couple FBR, and pooled the spouse's $2/month into unearned income, giving $392.42 of countable income. The spouse is not SSI aged/blind/disabled and that $2/month is below the $497 deeming threshold. The head's countable income is therefore $391.42 against the $994 individual FBR, or $7,231/year." +us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Used an eligible-couple FBR of $1,486/month because it assumed the spouse was SSI-eligible. Its final pass also dropped the unused general exclusion and counted $18 of unearned income plus $4,810 of earned income. The head alone is eligible at the $994 individual rate with $4,697/year of countable income, giving $7,231." +us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Its own reasoning reached about $12,590 using a $1,450 couple FBR, but it submitted 0, which contradicts that work. It also treated the spouse as SSI-eligible. Only the head qualifies, at the $994 individual FBR less $391.42 of countable income, which is $7,231/year." +us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491/month eligible-couple rate because it treated the spouse as SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head is an eligible individual with an ineligible spouse. The head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Paid both spouses separate individual benefits, including about $991/month for the spouse, and added them together. The spouse is not SSI aged/blind/disabled and receives $0. The head's countable income is $391.42/month, not about $430, so the head alone receives $602.58/month, or $7,231/year." +us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Subtracted $392.42 of pooled countable income from an assumed couple FBR of $1,483 because it treated the spouse as SSI-eligible. The spouse is not aged/blind/disabled for SSI, so the head's own $391.42 is subtracted from the $994 individual FBR, giving $7,231/year." +us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Used a $1,486 couple FBR by treating both spouses as SSI-eligible. It also read the annual $38 of disability benefits as monthly and counted $18/month of unearned income, reaching $418.83. The head alone is eligible at $994/month with $391.42 of countable income, which is $7,231/year." +us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR and pooled the spouse's unearned income into the couple's countable income. The spouse fails the SSI aged/blind/disabled test, so the head receives $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Treated the household as a disabled couple and used a $1,486/month couple maximum with $4,709 of pooled countable income. The spouse is not SSI aged/blind/disabled, so the head's own $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231." +us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Said the $10,400 of self-employment earnings made the household ineligible, without applying the $65 earned-income exclusion and the one-half disregard. Those leave only $391.42/month of countable income against the $994 individual FBR, so SSI is $7,231/year." +us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Used the couple maximum benefit because it assumed both spouses qualified. The spouse is not SSI aged/blind/disabled, so the head is paid from the $994/month individual FBR less $391.42 of countable income, or $7,231/year." +us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Subtracted $4,709 of pooled countable income from a $17,400 couple FBR because it treated the spouse as SSI-eligible. Only the head qualifies, so the $11,928 individual FBR less the head's $4,697 of countable income gives $7,231." +us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Said income and resources exceeded the SSI limits. The $912 in resources is below the $2,000 individual limit, and after exclusions the head's countable income of $391.42/month is below the $994 FBR, so SSI is $7,231/year." +us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Calculated an eligible-couple benefit because it treated both adults as SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head receives the individual rate of $994/month less $391.42 of countable income, which is $7,231/year." +us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the household as an eligible couple, which implies a $1,486/month couple FBR with pooled income. Only the head is SSI aged/blind/disabled, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Said SSI was not simulated and returned 0 instead of calculating it. The head is an eligible blind individual with $391.42/month of countable income against the $994 individual FBR, so SSI is $7,231/year." +us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Assumed $10,400 of self-employment income and $912 of assets exceeded the SSI limits, and relied on the absence of listed SSI receipt even though take-up is assumed. The assets are under the $2,000 limit, and the exclusions bring countable income down to $391.42/month, which leaves a $7,231/year benefit." +us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Used a couple FBR of $1,490.56 because it treated the spouse as SSI-disabled. It also applied the full $85 of exclusions to earnings, even though part of the general exclusion is first used by unearned income. The head alone is eligible at $994/month with $391.42 of countable income, or $7,231/year." +us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Said the $912 in assets was above the SSI limit, but it is well under the $2,000 individual limit. It also said self-employment income ruled out eligibility, when the exclusions reduce countable income to $391.42/month and leave a $602.58/month benefit, or $7,231/year." +us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned 0 because the prompt listed no SSI receipt, ignoring the instruction to assume program take-up. The head qualifies as an eligible blind individual, and the benefit is $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied the $17,892 eligible-couple rate and pooled both spouses' disability income on the assumption that both were SSI-disabled. The spouse fails the SSI aged/blind/disabled test, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231." +us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied a couple federal benefit rate of about $1,450/month because it treated both adults as SSI-eligible. Only the head is SSI aged/blind/disabled, so the benefit is $994/month less $391.42 of countable income, or $7,231/year." +us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Reduced the 2026 couple maximum on the assumption that both spouses met the SSI disability criteria. The spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse and receives $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Calculated an eligible-couple benefit of $13,183 because it treated the spouse as SSI-disabled. The spouse fails the aged/blind/disabled test, so only the head is paid, from the $994/month individual FBR, which gives $7,231/year." +us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Used the $17,892 couple maximum and counted the spouse's $24 against the general exclusion. The spouse is not SSI aged/blind/disabled and that income is below the deeming threshold, so the head's $4,697 of countable income is subtracted from $11,928, giving $7,231." +us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Compared $392.42 of pooled countable income with the $1,491 couple FBR because it treated both spouses as SSI-disabled. Only the head qualifies, so the $994 individual FBR less $391.42 gives $602.58/month, or $7,231/year." +us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Started from the $1,491/month couple benefit, which assumes an eligible couple. The spouse fails the SSI aged/blind/disabled test, so the head's benefit is $994 − $391.42 = $602.58/month, or $7,231/year." +us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Subtracted $4,709 of pooled countable income from the $17,892 couple maximum because it treated both spouses as SSI-disabled. The spouse is not SSI-eligible, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231." +us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Said countable self-employment income exceeded the couple FBR, so it did not apply the $65 exclusion and the one-half earned-income disregard. The head's countable income is $391.42/month, below the $994 individual FBR, which leaves $7,231/year." +us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Used a projected $1,486 couple FBR because it treated the spouse as SSI-disabled. Only the head is SSI aged/blind/disabled, so the $994 individual FBR less $391.42 of countable income gives $602.58/month, or $7,231/year." +us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Applied the $17,832 couple FBR on the assumption that both spouses were eligible. It also left the unused $202 of the general exclusion off the earnings, counting $4,810 of income. The head alone is eligible, with $4,697 of countable income against the $11,928 individual FBR, which gives $7,231." +us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Applied the $1,486/month couple rate because it treated both spouses as disabled SSI claimants. The spouse fails the SSI aged/blind/disabled test, so the head's $4,697 of countable income is subtracted from the $11,928 individual FBR, giving $7,231." +us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Used a couple FBR of $17,835, and it counted the $20 general exclusion twice by fully excluding the $38 of unearned income while also subtracting the full $1,020 from earnings. Only the head qualifies, so the $11,928 individual FBR less $4,697 of countable income gives $7,231." +us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Used a $1,490/month couple rate because it treated the spouse as SSI-eligible, and it understated countable income at about $370/month. The head alone is eligible at $994/month with $391.42 of countable income, which gives $7,231/year." +us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SSI value and no explanation. The correct derivation uses the $994/month individual FBR less the head's $391.42 of countable income, which is $602.58/month, or $7,231/year." +us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $17,892 couple benefit rate because it treated both spouses as SSI-disabled. The spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse and receives $11,928 − $4,697 = $7,231." +us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Concluded from the small disability benefit amounts that no SSI was payable, without running the SSI income test. The head is an eligible blind individual with $391.42/month of countable income against the $994 FBR, which gives $7,231/year." +us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR because it treated both spouses as eligible, and it applied the full $85 of exclusions to earnings. Only the head qualifies, and the head's $1.17/month of unearned income uses part of the general exclusion, so countable income is $391.42 and SSI is $602.58/month, or $7,231/year." +us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Moved between made-up couple FBRs of $979–$1,058 and individual FBRs of $674–$706, treated the annual disability payments as monthly, and submitted $8,472, which none of its calculations produced. The actual 2026 individual FBR is $994/month and the head's countable income is $391.42/month, which gives $7,231/year." +us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded that income and resources produce no positive SSI benefit. The $912 in resources is below the $2,000 limit, and after the $20, $65, and one-half earned-income exclusions the head's countable income is $391.42/month, which leaves $602.58/month, or $7,231/year." us,scenario_057,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied Louisiana's correct post-reform 3% flat rate but subtracted only $9,000 as the ""MFJ standard deduction"" — that is the repealed $4,500-per-filer personal exemption amount, not the deduction. Under Act 11 of the 2024 Third Extraordinary Session, Louisiana's standard deduction for married filing jointly is $25,000 (indexed beginning 2026), which exceeds the household's ~$9,665 federal AGI, so Louisiana taxable income is zero. Its $665 base is arithmetic on the wrong subtraction, and 3% of a base that does not exist produced the $20." us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It reconstructed the pre-2025 Louisiana regime on two counts: it subtracted $4,500-per-filer personal exemptions ($9,000) and applied a graduated first-bracket rate, both eliminated effective tax year 2025 when Louisiana moved to a 3% flat tax with a $25,000 MFJ standard deduction. Its cited ""1.75% on the first $12,500"" matches neither the current flat rate nor the former 1.85% first bracket, whose MFJ width was $25,000. The $25,000 standard deduction alone wipes out the ~$9,665 federal AGI, leaving $0 rather than a $665 taxable base." us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the miss is a contract failure rather than a substantive rule error. The required derivation runs $10,400 of SSTB self-employment income to roughly $9,665 federal AGI after the deductible half of self-employment tax, then against Louisiana's $25,000 married-filing-jointly standard deduction under its 3% flat tax, giving $0." -us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model reached the correct structure mid-reasoning — Louisiana's refundable 5% EITC match, and the 43-year-old head satisfying the childless EITC's 25–64 age test — then discarded it by assuming a childless couple's federal EITC at ~$9,600 of net self-employment earnings is negligible. Earned income of $9,665 ($10,400 less the $735 deductible half of self-employment tax) sits on the plateau of the 2026 childless EITC schedule, well above the ~$8,700 phase-in ceiling and far below the ~$18,000 joint phase-out start, so the full $664 maximum applies and 5% of it is $33.20." -us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Louisiana's individual credits are predominantly nonrefundable, which erases the state's only relevant credit here: the Louisiana EITC under La. R.S. 47:297.8 is fully refundable at 5% of the federal EITC. With a federal EITC of $664, that refundable match is $33.20 regardless of the household's zero state liability." -us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model imposed a qualifying-child or child-care-expense requirement on every Louisiana refundable credit and separately claimed self-employment income does not produce a qualifying federal EITC. The childless federal EITC has no dependent requirement and net self-employment earnings are earned income for it, producing $664 federal and a 5% Louisiana refundable match of $33.20." -us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified Louisiana's 5%-of-federal refundable EITC but concluded the childless married-joint federal EITC at $10,400 of self-employment income 'rounds to zero.' The 2026 childless schedule phases in at 7.65% to a $664 maximum by roughly $8,700 of earned income and does not begin phasing out for joint filers until about $18,000, so this household's $9,665 of earned income yields the full $664 and a $33.20 state credit." -us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model characterized the Louisiana EITC as 'nonrefundable/limited.' La. R.S. 47:297.8 makes the credit refundable in full at 5% of the federal EITC, so the $664 federal credit produces a $33.20 refundable state credit even with zero Louisiana tax liability." -us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated Louisiana has no state EITC and that all its individual credits are nonrefundable. Louisiana has had a refundable EITC since 2008, set at 5% of the federal credit for tax years beginning on or after January 1, 2018, which here yields 5% × $664 = $33.20." -us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model denied that Louisiana offers any refundable individual credit reachable by this profile, treating its credits as generally nonrefundable or dependent-gated. The Louisiana refundable EITC is 5% of the federal EITC with no dependent requirement, and the household's $664 federal childless EITC produces $33.20." -us,scenario_057,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied the correct rule — 5% of the federal childless EITC — but plugged in $632, the tax-year-2024 maximum, instead of the 2026 indexed maximum of $664. Using the current-year parameter gives 5% × $664 = $33.20 rather than the $31.60 it computed and rounded to $32." -us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the correct 5% Louisiana match on a stale federal maximum, $649, which is the tax-year-2025 childless EITC cap rather than the 2026 inflation-indexed $664. The one-year-lagged parameter is the entire error: 5% × $664 = $33.20, not $32.45." -us,scenario_057,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model asserted a federal EITC of $0, a result consistent only with denying the childless EITC to this filer. The head is 43 and therefore inside the 25–64 age band, the household's $9,665 of earned income from self-employment is qualifying earned income, and investment income is zero, so the federal credit is the $664 maximum and Louisiana's 5% match is $33.20." -us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied Louisiana's correct 5% refundable match to a federal EITC of $630, an outdated childless maximum rather than the 2026 figure of $664. Substituting the current-year cap yields $33.20 instead of $31.50." -us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model asserted without derivation that no state refundable credits apply, missing Louisiana's refundable EITC equal to 5% of the federal credit under La. R.S. 47:297.8. The household's $664 federal childless EITC triggers a $33.20 Louisiana refundable credit." -us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model declared the household unqualified for any Louisiana refundable credit without testing the state EITC. Louisiana refunds 5% of the federal EITC with no dependent requirement, so the $664 federal childless credit produces $33.20." -us,scenario_057,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned zero by blanket assertion, never applying Louisiana's refundable 5%-of-federal EITC. The head's self-employment earnings put the household at the $664 federal childless EITC maximum, making the state credit $33.20." -us,scenario_057,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the right Louisiana rule and the wrong year's federal parameter, taking 5% of $649 — the tax-year-2025 childless EITC maximum — instead of the 2026 maximum of $664. The correct product is $33.20, not $32.45." -us,scenario_057,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied Louisiana's 5% refundable match to $649, the 2025 childless EITC cap, rather than the 2026 indexed cap of $664. Updating the federal maximum to the requested tax year gives $33.20." -us,scenario_057,state_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model disqualified the household from the EITC because the 20-year-old spouse is under 25. On a joint return the childless EITC age test is satisfied when either spouse is at least 25 and under 65, and the 43-year-old head satisfies it, producing a $664 federal credit and a $33.20 Louisiana refundable match." -us,scenario_057,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model computed the federal EITC as $795.60, which is exactly 7.65% of the full $10,400 of gross self-employment income — it ran the phase-in rate past the credit ceiling and never subtracted the deductible half of self-employment tax from earned income. The childless EITC is capped at $664 in 2026, so Louisiana's 5% match is $33.20, not $39.78." -us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model concluded no Louisiana refundable credit is triggered without evaluating the state EITC. Louisiana refunds 5% of the federal EITC, and this household's $664 federal childless credit makes that $33.20." -us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model returned zero by assertion, omitting Louisiana's refundable EITC entirely. With $9,665 of earned income the household is at the $664 federal childless EITC maximum, and the state's 5% match is $33.20." -us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model reasoned that a two-adult household without children qualifies for no Louisiana refundable credit. The Louisiana EITC is 5% of the federal EITC and the federal childless EITC covers filers aged 25–64 with no dependents, so the head's $664 credit yields $33.20 in state refundable credits." -us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model asserted the facts indicate no Louisiana refundable credit eligibility, skipping the state EITC test. Self-employment earnings of $10,400 place the household at the $664 federal childless EITC maximum, and Louisiana refunds 5% of that, $33.20." -us,scenario_057,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model derived the federal EITC of $664 correctly but applied a 3.5% Louisiana match, the pre-2018 rate. Act 2 of the 2018 Second Extraordinary Session raised the credit to 5% of the federal EITC for tax years beginning on or after January 1, 2018, giving $33.20 instead of $23.24." -us,scenario_057,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model asserted that no state refundable credits apply without evaluating Louisiana's refundable EITC. That credit is 5% of the federal EITC, and the household's $664 federal childless credit makes it $33.20." -us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model stated Louisiana provides no applicable refundable income tax credits. Louisiana's EITC under La. R.S. 47:297.8 is refundable at 5% of the federal credit, producing $33.20 on this household's $664 federal EITC." -us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model denied that Louisiana offers any refundable credit fitting this household, missing the state's refundable 5%-of-federal EITC. The head's $9,665 of earned income puts the federal childless credit at its $664 maximum, so the state credit is $33.20." -us,scenario_057,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model correctly identified the 5% Louisiana match but set the federal EITC to $0, a value consistent only with excluding this childless self-employed couple from the credit. The 43-year-old head meets the 25–64 age requirement, net self-employment earnings count as earned income, and $9,665 lands on the credit plateau, so the federal EITC is $664 and the state credit $33.20." -us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for state_refundable_credits, so no substantive computation was performed. The requested output is Louisiana's refundable EITC, 5% of the $664 federal EITC, or $33.20." -us,scenario_057,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model computed the federal EITC correctly at $664 but applied a 10% Louisiana match rate. Louisiana's refundable EITC has been 5% of the federal credit since tax year 2018 and was never 10%, so the credit is $33.20, not $66.40." -us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Louisiana imposes no individual income tax. Louisiana levies an individual income tax — a flat 3% rate beginning with tax year 2025 — and grants a refundable EITC of 5% of the federal credit, which here is 5% × $664 = $33.20." -us,scenario_057,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model conditioned Louisiana refundable credits on positive tax liability and on qualifying children or expenses. A refundable credit is paid out when liability is zero, and Louisiana's 5%-of-federal EITC has no dependent requirement, so the household's $664 federal childless EITC produces $33.20." -us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model asserted Louisiana has no state individual income tax and therefore no refundable individual credits. Louisiana taxes individual income at a flat 3% for 2026 and pays a refundable EITC equal to 5% of the federal credit, giving $33.20 on the $664 federal EITC." -us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model returned zero without applying Louisiana's refundable EITC. With $10,400 of self-employment income the household earns the $664 federal childless EITC maximum, and the state's 5% match is $33.20." -us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its own standard deduction figure ($36,600) already exceeded the correct AGI, but it replaced the Social Security worksheet with a guess of ""≈$16,400 taxable"" instead of applying the tier-2 formula: provisional income of $44,152 exceeds the $44,000 MFJ threshold by $152, so taxable Social Security is $6,000 + 0.85 × $152 = $6,129.20 and AGI is $31,729.20, below the deduction. It then submitted $1,177 while its own arithmetic produced $540, abandoning the derivation it wrote." -us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security ($6,129) and AGI ($31,729) exactly right, then used a standard deduction of $18,100 for MFJ with both spouses 65+; the 2026 amount is $32,200 plus two $1,650 age additions = $35,500, which alone drives taxable income below zero. It also omitted the OBBBA $6,000-per-senior deduction, worth another $12,000 for this couple." -us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA provisions lapsed after 2025, pairing a $19,900 standard deduction with $10,600 of personal exemptions; OBBBA made the larger standard deduction permanent and holds personal exemptions at zero, so the 2026 subtraction is $35,500 of standard deduction plus $12,000 of senior deduction against $31,729.20 of AGI. Taxable income is zero, not the $1,229.20 it reported." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied a single-filer-sized standard deduction ($15,875) to a joint return instead of the $32,200 MFJ amount, then added $10,125 of personal exemptions, which are zero under permanent law. Its $29,125 total subtraction falls short of the $35,500 standard deduction alone (before the $12,000 senior deduction), manufacturing $2,604.20 of taxable income where the correct AGI of $31,729.20 leaves none." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It taxed Social Security at the flat 85% maximum ($18,554) instead of running the worksheet, which limits inclusion to $6,000 + 0.85 × ($44,152 − $44,000) = $6,129.20 and yields AGI of $31,729.20, already below its own $32,600 standard deduction. It compounded this with an $842 credit for the elderly, a §22 credit reduced to zero here by the $30,974.80 of excluded Social Security and by half of AGI above $10,000." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of $31,729 and then subtracted a ""projected pre-TCJA standard deduction and personal exemptions"" of roughly $29,600, betting on a TCJA sunset that OBBBA canceled. Under 2026 law the subtraction is a $32,200 MFJ standard deduction plus $3,300 of age-65 additions plus the $12,000 senior deduction, leaving zero taxable income rather than $2,129." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used a single-filer-sized $15,300 standard deduction on a joint return and revived $10,000 of personal exemptions, which are permanently zero. The correct 2026 subtraction is $32,200 plus $3,300 in age-65 additions = $35,500, above its own correct AGI of $31,729, so the $3,329 of taxable income it taxed does not exist." -us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its taxable Social Security ($6,129.20) and AGI ($31,729.20) are exact, but it built a $29,900 subtraction from a reduced standard deduction plus personal exemptions, which are zero for 2026. The actual $35,500 standard deduction for MFJ with both spouses 65+, before the $12,000 senior deduction, eliminates AGI entirely, so there is no $1,829.20 of taxable income." -us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It included $12,129 of Social Security — the correct $6,129.20 plus an extra $6,000, counting the tier-1 ceiling twice on top of the tier-2 result — when the worksheet gives $6,000 + 0.85 × ($44,152 − $44,000) = $6,129.20. It then stacked a pre-TCJA $19,750 standard deduction with $10,600 of repealed personal exemptions instead of the current-law $35,500 deduction plus $12,000 senior deduction, which zeroes taxable income." -us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It summed Social Security as $37,504 instead of $37,104 and applied the flat 85% cap ($16,876) rather than the worksheet, which limits inclusion to $6,129.20 because provisional income exceeds $44,000 by only $152. It also built the standard deduction as $29,200 plus four $775 age additions; MFJ with two spouses 65+ gets two additions, and the 2026 figures are $32,200 + 2 × $1,650 = $35,500, which alone exceeds the correct AGI of $31,729.20." -us,scenario_059,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the 85% maximum to Social Security ($31,538) instead of the tier-2 result of $6,129.20, inflating AGI to $57,138 against the correct $31,729.20, and used a bare $32,200 standard deduction with neither the $3,300 of age-65 additions nor the $12,000 senior deduction. It then invented a $6,000 nonrefundable residential clean energy credit out of a $5,000 mortgage balance and reported $6,088.80 even though subtracting that credit from its own $2,343.80 tentative tax gives zero." +us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly noted that Louisiana has a 5% EITC match and that the 43-year-old head is in the eligible age range for the childless EITC. It then dropped that line of reasoning and submitted 0. The couple's earned income of about $9,665 after the half-SE-tax deduction puts the federal childless EITC at its $664 maximum, so the Louisiana match is $33.20." +us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed Louisiana has no meaningful refundable individual credits and that the couple needs dependents to get one. It missed Louisiana's refundable EITC, which is 5% of the federal EITC and applies to childless filers. Here that is 5% × the $664 maximum childless federal EITC = $33.20." +us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It wrongly treated the Louisiana EITC as requiring qualifying children and dismissed the self-employment income as too small. The childless federal EITC applies because the head is 43, and about $9,665 of net SE earned income reaches the $664 maximum. Louisiana's 5% match is therefore $33.20." +us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It correctly identified the Louisiana EITC as 5% of the federal EITC. It then assumed a childless couple with $10,400 of SE income gets only a minimal federal EITC and rounded the result to zero. Earned income of about $9,665 after the half-SE-tax deduction is past the phase-in, so the federal credit is the full $664 and the Louisiana credit is $33.20." +us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It acknowledged that Louisiana's EITC is 5% of the federal EITC but wrongly called it nonrefundable. The Louisiana EITC is refundable, so 5% × the $664 federal childless EITC = $33.20 counts as a state refundable credit." +us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly stated that Louisiana has no state EITC. Louisiana pays a refundable EITC equal to 5% of the federal EITC. This couple qualifies for the $664 maximum childless federal EITC, so the Louisiana credit is $33.20." +us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly claimed the Louisiana EITC is nonrefundable or requires dependents. The Louisiana EITC is a refundable 5% match of the federal EITC, and the federal credit here is the $664 childless maximum on about $9,665 of net SE earnings, so the Louisiana credit is $33.20." +us,scenario_057,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It gave no reasoning beyond saying no Louisiana credit applies, which means it left out the refundable Louisiana EITC. The correct figure is 5% × the $664 federal childless EITC = $33.20." +us,scenario_057,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match but used $632, the 2024 maximum childless federal EITC, instead of the 2026 maximum of $664. It then rounded $31.60 to $32; the correct figure is $33.20." +us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match but used $649, the 2025 maximum childless federal EITC, instead of the 2026 maximum of $664. That produced $32.45 instead of $33.20." +us,scenario_057,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It set the federal EITC to $0, which zeroed out the Louisiana match. The couple files jointly with a 43-year-old head, so they qualify for the childless EITC. About $9,665 of net SE earned income gives the $664 maximum, and 5% of that is $33.20." +us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the correct 5% Louisiana match to a federal EITC of $630. That is not the 2026 maximum childless credit of $664, which this couple receives at about $9,665 of earned income. It therefore submitted $31.50 instead of $33.20." +us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no reasoning and answered 0, which means it left out Louisiana's refundable EITC. That credit is 5% of the $664 federal childless EITC this couple earns, or $33.20." +us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It asserted without reasoning that no Louisiana refundable credit applies, so it missed the refundable Louisiana EITC. That credit is 5% × the $664 federal childless EITC = $33.20." +us,scenario_057,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It answered 0 without analysis and left out the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC generated by the head's net self-employment earnings, or $33.20." +us,scenario_057,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It used the correct 5% Louisiana match but applied it to $649, the 2025 maximum childless federal EITC, instead of the 2026 maximum of $664. That gave $32.45 instead of $33.20." +us,scenario_057,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct 5% Louisiana match but took the federal EITC to be $649, the 2025 childless maximum, instead of the 2026 maximum of $664. That gave $32.45 instead of $33.20." +us,scenario_057,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"It ruled the couple ineligible because the spouse is 20. The childless EITC age test for a joint return is met when either spouse is 25 to 64, and the head is 43. The couple therefore gets the $664 federal EITC and a $33.20 Louisiana EITC." +us,scenario_057,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Its federal EITC of $795.60 is the 7.65% phase-in rate applied to the full $10,400 of gross SE income. It never subtracted half of SE tax from earned income and never capped the credit at the $664 childless maximum. The correct Louisiana credit is 5% × $664 = $33.20." +us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no Louisiana refundable credit is triggered, missing the refundable Louisiana EITC. That credit equals 5% of the $664 federal childless EITC the couple receives on about $9,665 of net SE earnings, or $33.20." +us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It answered 0 without identifying any Louisiana credit, so it left out the refundable Louisiana EITC. The correct figure is 5% × the $664 federal childless EITC = $33.20." +us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It treated the absence of children as disqualifying for any Louisiana refundable credit. The Louisiana EITC matches the federal childless EITC, which this couple receives at the $664 maximum, so the credit is 5% × $664 = $33.20." +us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It found no Louisiana refundable credit eligibility and so missed the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC the couple earns, or $33.20." +us,scenario_057,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,It computed the federal EITC correctly at $664 but applied a 3.5% Louisiana match rate instead of the current 5%. That produced $23.24 instead of $33.20. +us,scenario_057,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,It answered 0 with no analysis and left out the refundable Louisiana EITC. That credit is 5% × the $664 federal childless EITC = $33.20. +us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It claimed Louisiana has no applicable refundable income-tax credit. Louisiana's refundable EITC equals 5% of the federal EITC, and this couple's federal childless EITC is $664, so the credit is $33.20." +us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It asserted that Louisiana offers no applicable refundable credit and so missed the refundable Louisiana EITC. That credit is 5% of the couple's $664 federal childless EITC, or $33.20." +us,scenario_057,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It correctly set up the Louisiana EITC as 5% of the federal EITC but wrongly took the federal EITC to be $0. A joint filer with a 43-year-old head and about $9,665 of net SE earned income gets the $664 childless maximum, so the Louisiana credit is $33.20." +us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so there is no parseable answer. The correct figure is 5% × the $664 federal childless EITC = $33.20." +us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,It computed the federal EITC correctly at $664 but applied a 10% Louisiana match rate instead of the 5% rate. That doubled the credit to $66.40 instead of $33.20. +us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It wrongly stated that Louisiana has no state individual income tax. Louisiana levies an income tax and pays a refundable EITC of 5% of the federal EITC, which here is 5% × $664 = $33.20." +us,scenario_057,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It reasoned that zero state liability and no children rule out refundable credits. The Louisiana EITC is refundable regardless of liability and matches the federal childless EITC, so the credit is 5% × $664 = $33.20." +us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly asserted that Louisiana has no state individual income tax. Louisiana has an income tax with a refundable EITC equal to 5% of the federal EITC, which is $33.20 on this couple's $664 federal childless credit." +us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It answered 0 without analysis and left out the refundable Louisiana EITC. That credit is 5% of the $664 federal childless EITC generated by the head's net self-employment earnings, or $33.20." +us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It guessed about $16,400 of taxable Social Security instead of working the tier-2 formula. Because provisional income is only $152 over the $44,000 adjusted base, the formula gives $6,000 + 85% x $152 = $6,129.20. It also left out the OBBBA $6,000-per-senior deduction ($12,000 for the couple), which together with the ~$35,500 standard deduction wipes out AGI. Its own $540 figure was then arbitrarily inflated to $1,177 with no computation behind it." +us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but used a TCJA-sunset standard deduction of $18,100. The 2026 MFJ standard deduction is $32,200, plus $1,650 per aged spouse, plus the $6,000-per-senior OBBBA deduction, for $47,500 in total. That brings taxable income to $0, not $13,629." +us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but assumed TCJA expired: a $19,900 pre-TCJA standard deduction plus $10,600 of personal exemptions. OBBBA made TCJA permanent, so personal exemptions stay at zero, and 2026 deductions are $32,200 + $3,300 aged + $12,000 senior = $47,500. Taxable income is $0, not $1,229.20." +us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but applied pre-TCJA sunset rules: a $16,000 standard deduction, $1,600 aged additions and $5,100 personal exemptions each, $29,400 in total. The permanent 2026 rules give $32,200 + $3,300 + the $12,000 OBBBA senior deduction, which leaves no taxable income." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729.20 correctly but assumed TCJA sunset: a $15,875 standard deduction, $10,125 of personal exemptions and $3,125 of senior additions, $29,125 in total. It missed that OBBBA kept the $32,200 MFJ standard deduction and added the $6,000-per-senior deduction. Total deductions of $47,500 zero out taxable income." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It skipped the provisional-income worksheet and included $18,554 of Social Security, overstating AGI at $44,154 instead of $31,729.20. It also left out the $12,000 OBBBA senior deduction. Then it applied an $842 elderly credit, which is fully eliminated here because the couple's nontaxable Social Security exceeds the $7,500 initial amount. With correct AGI and deductions, taxable income is $0." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but explicitly used a projected pre-TCJA standard deduction plus personal exemptions (about $29,600). OBBBA made the TCJA structure permanent: the $32,200 MFJ standard deduction plus $3,300 of aged additions plus the $12,000 senior deduction exceeds AGI, so tax is $0, not $213." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI of $31,729 correctly but used TCJA-sunset figures: a $15,300 standard deduction, $3,100 of elderly additions and $10,000 of personal exemptions. The actual 2026 amounts are $32,200 + $3,300 + the $12,000 OBBBA senior deduction, with no personal exemptions, which reduces taxable income to $0." +us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security ($6,129.20) and AGI ($31,729.20) correctly. It then subtracted a standard deduction plus personal exemptions totaling only $29,900, which reflects the expired-TCJA regime. OBBBA's permanent 2026 rules allow $32,200 + $3,300 aged + $12,000 senior = $47,500 with no exemptions, leaving taxable income of $0, not $1,829.20." +us,scenario_059,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It botched the Social Security worksheet, reporting $12,129 taxable instead of $6,000 + 85% x $152 = $6,129.20, so its AGI was $37,729 instead of $31,729.20. It also assumed pre-TCJA law ($19,750 standard deduction plus $10,600 of exemptions) instead of the permanent $32,200 + $3,300 + $12,000 senior deduction, which leaves no taxable income." +us,scenario_059,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It misstated gross Social Security as $37,504 and took a flat 85% share ($16,876) instead of running the provisional-income worksheet, which gives $6,129.20, so AGI came out at $42,476 instead of $31,729.20. It used an outdated $29,200 + $3,100 standard deduction and left out the $6,000-per-senior OBBBA deduction. The correct $47,500 of deductions exceeds AGI, so tax is $0." +us,scenario_059,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It capped taxable Social Security at the 85% maximum ($31,538) instead of applying the tier formula ($6,129.20). It used only the $32,200 base MFJ deduction, without the $3,300 aged additions or the $12,000 senior deduction. It then invented a residential clean energy credit from a mortgage balance and added that credit to tax instead of subtracting it. With correct AGI of $31,729.20 against $47,500 of deductions, tax is $0." us,scenario_059,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no payroll-taxable wages and therefore owed zero employee payroll tax, but then submitted $1,937 without any supporting computation. Its numeric output directly contradicts its own stated payroll-tax-base calculation." us,scenario_060,federal_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model correctly noted that the age-65 expansion applied only in 2021 but then applied the childless EITC phase-in rate anyway. At age 70, the head fails the under-65 age requirement for a taxpayer without qualifying children, so the $3,000 of wages generates no EITC rather than $229.50." us,scenario_060,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly excluded the $45,600 of veterans benefits when asserting that the head passed Texas's aged Medicaid income test. The head does not satisfy the aged pathway or any other Medicaid category, so low bank assets and MAGI of 1.08 times FPL do not establish eligibility." @@ -4629,27 +5078,31 @@ us,scenario_060,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Th us,scenario_060,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model stated 'Veterans benefits are generally excluded from SNAP income' and dropped the entire $45,600, cutting countable gross income from $62,775 to $17,175. Veterans' benefits are explicitly counted unearned income under 7 CFR 273.9(b)(2)(ii), which lists 'annuities; pensions; retirement, veteran's or disability benefits' among income that must be counted. With the $45,600 restored, half of adjusted income (~$23,800/year) exceeds the $26,400 rent only slightly, leaving net income near $45,000/year (~$3,760/month) — far above the one-person net limit — instead of the $0 net income that produced the model's full $292/month maximum allotment." us,scenario_060,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model asserted 'VA disability benefits are excluded' and computed countable income from only Social Security, wages, and interest. VA compensation and pension payments are counted unearned income for SNAP under 7 CFR 273.9(b)(2)(ii); the narrow VA exclusions concern aid-and-attendance-type payments that reimburse specific medical or attendant costs, not the benefit stream itself. Excluding $45,600 let the uncapped elderly shelter deduction drive net income to $0 and triggered the one-person maximum allotment ($300/month), whereas counting it leaves net income around $3,760/month, above the 100% FPL net income limit, for a $0 benefit." us,scenario_060,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model set gross countable income at $1,431.25/month — wages, Social Security, and interest only — omitting the $45,600 in veterans benefits that 7 CFR 273.9(b)(2)(ii) requires be counted as unearned income. That omission also inflated the shelter deduction: with the true income counted, half of adjusted income is roughly $1,987/month rather than $90, so excess shelter is about $213/month instead of $2,109.63. Net income is then roughly $3,760/month, failing the one-person net income test, so the allotment is $0 rather than the $291/month maximum the model's $0 net income produced." -us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added the standalone $28,800 survivor-benefits input to gross income, lifting provisional income to $55,786 and forcing the 85% Social Security inclusion ($18,115) instead of the $993 that the 50% tier produces at the actual provisional income of $26,986 (($4,793+$257+$11,280) + half of $21,312). It compounded this by filing the head as a qualifying surviving spouse on MFJ brackets and standard deduction, a status §2(a) grants only to a taxpayer with a dependent child, while PolicyEngine files this one-person unit as single. Countable income is $37,642, AGI is $17,323, and the single 65+ deduction stack (~$16,100 standard + $2,050 age addition + $6,000 OBBBA senior deduction) exceeds it, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $28,800 survivor-benefits input as taxable income, which drove provisional income to $55,786 and triggered the 85% Social Security inclusion of $18,115.20 rather than the $993 taxable under the 50% tier at the true $26,986 provisional income. It also stopped its deductions at $17,400 ($15,400 + $2,000), omitting the $6,000 OBBBA senior deduction available to a 65+ filer. On the correct $17,323 AGI, roughly $24,150 of deductions leaves zero taxable income and $0 of tax." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It built AGI of $63,245.20 by including the $28,800 survivor-benefits input, which pushed provisional income past the $34,000 breakpoint and produced $18,115.20 of taxable Social Security; excluding that input leaves $16,330 of other income, provisional income of $26,986, and only $993 of Social Security taxable. Its $17,425 standard deduction also omitted the $6,000 senior deduction. AGI of $17,323 against a ~$24,150 deduction stack yields $0 of taxable income and no tax." -us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly excluded the $28,800 survivor-benefits input and reached AGI of $17,323 with $993 of taxable Social Security, then applied a TCJA-sunset deduction stack — an $8,250 standard deduction plus a $5,250 personal exemption — that does not exist in 2026: OBBBA made the larger standard deduction permanent (about $16,100 for a single filer) and kept personal exemptions repealed. With the $2,050 age addition and the $6,000 senior deduction, deductions of roughly $24,150 exceed AGI, so taxable income is $0 rather than the $1,823 it taxed at 10%." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It made two compounding errors: it added the $28,800 survivor-benefits input to AGI, which forced the 85% Social Security inclusion of $18,115 in place of the $993 taxable at the actual $26,986 provisional income, and it assumed TCJA expiration, deducting an $8,000 standard deduction plus a $5,100 personal exemption and taxing at pre-TCJA 15%/25% rates. OBBBA made the ~$16,100 single standard deduction permanent, left exemptions repealed, and added a $6,000 senior deduction, so deductions of about $24,150 wipe out the correct $17,323 AGI and tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted both the $28,800 survivor-benefits input and the full $21,312 of Social Security as income, when the $28,800 never enters AGI and only $993 of the Social Security is taxable at a $26,986 provisional income. Its own $46,649 AGI figure is internally inconsistent with the items it lists, and the $178 it reports from $29,449 of claimed taxable income matches no bracket schedule (10% alone gives $2,945). The correct AGI of $17,323 sits below the ~$24,150 single 65+ deduction stack, so tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It correctly excluded the $28,800 survivor-benefits input and computed AGI of about $17,323 with $993 of taxable Social Security, then assumed TCJA provisions expire in 2026 and reinstated a personal exemption alongside a pre-TCJA standard deduction totaling only ~$15,800. OBBBA made the ~$16,100 single standard deduction permanent, kept exemptions repealed, and added a $6,000 deduction for filers 65+; with the $2,050 age addition, deductions of roughly $24,150 exceed AGI, leaving $0 of taxable income instead of the $1,523 it taxed." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It included the $28,800 survivor-benefits input in AGI, which raised provisional income to $55,786 and produced $18,115 of taxable Social Security instead of the $993 that the 50% tier gives at the actual $26,986 provisional income. Its ~$15,400 of deductions also omitted both the $2,050 age addition and the $6,000 senior deduction. Countable income is $37,642, AGI is $17,323, and roughly $24,150 of deductions leaves $0 of taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It added the $28,800 survivor-benefits input to gross income and applied the 85% Social Security inclusion ($18,115.20), when the excluded input leaves provisional income at $26,986 and only $993 of Social Security taxable. It then deducted a $5,300 personal exemption on top of an $8,300 standard deduction and taxed at 15%/25% pre-TCJA rates, all of which OBBBA superseded with a permanent ~$16,100 standard deduction, no exemption, and a $6,000 senior deduction. AGI of $17,323 against ~$24,150 of deductions gives $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation taxes survivor benefits alongside Social Security and subtracts a personal exemption, so it built AGI near $63,245 by including the $28,800 survivor-benefits input that PolicyEngine does not count and then deducted a personal exemption that OBBBA keeps repealed for 2026. The countable income is $37,642, provisional income is $26,986, only $993 of Social Security is taxable, and AGI of $17,323 falls below the single 65+ deduction stack of about $24,150, so tax before refundable credits is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It stated AGI of $63,245 including $18,115 of taxable Social Security, which requires counting the $28,800 survivor-benefits input as income; without it, non-Social-Security income is $16,330, provisional income is $26,986, and only $993 of Social Security is taxable under the 50% tier. Its 'standard deduction and aged addition' also omitted the $6,000 OBBBA senior deduction. AGI of $17,323 against roughly $24,150 of deductions produces $0 of taxable income and no tax." -us,scenario_062,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It reached the correct conclusion that taxable income and income tax before credits are zero, then subtracted a $1,031 credit for the elderly to report −$1,031; §26 limits nonrefundable credits to the tax liability, so the floor is $0. The §22 credit is itself $0 here because the $21,312 of nontaxable Social Security exceeds the $5,000 §22 initial amount for a single filer, reducing the credit base to nothing." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $4,328 backs out to about $38,100 of taxable income, i.e. AGI of $63,245 less a ~$24,150 deduction stack and a $959 QBI deduction, so it counted the $28,800 survivor-benefits input as income and used the resulting 85% Social Security inclusion. PolicyEngine's countable income is $37,642, provisional income is $26,986, and only $993 of Social Security is taxable, so AGI is $17,323 — below the standard, age, and senior deductions it otherwise applied correctly — and tax is $0." -us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It listed survivor benefits as taxable income alongside 85%-included Social Security, producing AGI of about $63,245 and roughly $38,100 of taxable income after the standard, age, senior, and QBI deductions. The $28,800 survivor-benefits input does not enter countable income, which is $37,642; at a $26,986 provisional income only $993 of Social Security is taxable, giving AGI of $17,323 against ~$24,150 of deductions and $0 of tax." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It combined two errors: including the $28,800 survivor-benefits input in AGI, which forced the 85% Social Security inclusion instead of the $993 taxable at the actual $26,986 provisional income, and assuming TCJA sunset — a ~$16,049 stack built from a pre-TCJA standard deduction plus a personal exemption, taxed at 10%/15% rates. OBBBA made the ~$16,100 single standard deduction and TCJA rate schedule permanent, repealed exemptions, and added a $6,000 senior deduction, so AGI of $17,323 against ~$24,150 of deductions yields $0." -us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It computed AGI of $63,245 by treating the $28,800 survivor-benefits input as income, which alone lifted provisional income to $55,786 and produced $18,115 of taxable Social Security in place of the $993 the 50% tier gives at $26,986. It then deducted a $10,612 standard deduction plus a $5,442 personal exemption and taxed at 15%, a pre-TCJA structure OBBBA replaced with a permanent ~$16,100 standard deduction, no exemption, and a $6,000 senior deduction. Correct AGI of $17,323 minus ~$24,150 of deductions is $0 of taxable income." -us,scenario_062,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It included the $28,800 survivor-benefits input in AGI and therefore applied the 85% Social Security inclusion of $18,115, when the correct non-Social-Security income of $16,330 gives provisional income of $26,986 and only $993 of taxable Social Security. Its $18,150 standard-plus-age deduction and $959 QBI deduction were close to right and by themselves exceed the true $17,323 AGI, so taxable income and federal tax are $0." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were submitted for this output, so the answer failed the response contract rather than any tax rule. The required derivation is countable income of $37,642, $993 of taxable Social Security at a $26,986 provisional income, AGI of $17,323, and a single 65+ deduction stack of about $24,150 that leaves $0 of tax." -us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly excluded the $28,800 survivor-benefits input and reached AGI of $17,323 with $993 of taxable Social Security, then applied a 'restored exemption' plus a pre-TCJA standard deduction totaling only about $15,500. Personal exemptions remain repealed for 2026 and the single standard deduction is roughly $16,100; with the $2,050 age addition and the $6,000 OBBBA senior deduction, deductions of about $24,150 exceed AGI, so taxable income is $0 rather than the $1,820 it taxed at 10%." -us,scenario_062,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It treated the $28,800 survivor-benefits input as fully taxable, and the resulting $55,786 provisional income drove the 85% Social Security inclusion of $18,115; countable income is $37,642, provisional income is $26,986, and only $993 of Social Security is taxable. Its deduction stack ($16,100 base + $1,650 age + $6,000 senior + $959 QBI) was otherwise on target and by itself exceeds the correct $17,323 AGI, so taxable income and tax are $0." -us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It settled on AGI of $63,245 by adding the $28,800 survivor-benefits input and applying the 85% Social Security inclusion, when that input is not countable income and provisional income of $26,986 makes only $993 of Social Security taxable. Its $16,550 deduction also omitted the $6,000 OBBBA senior deduction; the correct $17,323 AGI against roughly $24,150 of deductions gives $0 of taxable income and no tax before refundable credits." +us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $28,800 survivor-benefits input as taxable income. That pushed provisional income to $55,786 and made 85% ($18,115) of Social Security taxable, for an AGI of $63,245 instead of $17,323. It also used qualifying-surviving-spouse status, which requires a dependent child. The correct AGI of $17,323 is fully covered by the $16,100 standard deduction, the $2,050 aged addition and the $6,000 senior deduction, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits in gross income, which made $18,115 of Social Security taxable and raised AGI to $63,245 instead of $17,323. It also left out the $6,000 senior deduction. With AGI at $17,323, the single standard deduction plus the aged addition alone ($18,150) brings taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $28,800 survivor benefits as taxable, reaching AGI of $63,245 with 85% of Social Security taxable. Excluding that input, provisional income is $26,986, only $993 of Social Security is taxable, and AGI is $17,323. It also omitted the $6,000 senior deduction; the deductions available exceed that AGI, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the correct $17,323 AGI but applied TCJA-sunset law: an $8,250 standard deduction, a $2,000 aged addition and a $5,250 personal exemption. OBBBA made the TCJA structure permanent, giving a 2026 single deduction of $16,100 + $2,050 aged + $6,000 senior, which zeroes taxable income. Its submitted $157.30 also contradicts the $182.30 in its own explanation." +us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation lists survivor benefits as taxable income, so it taxed the $28,800 input and the resulting 85%-taxable Social Security, producing $5,135. AGI excluding that input is $17,323, and the $16,100 standard deduction plus the $2,050 aged addition and the $6,000 senior deduction eliminate taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $28,800 survivor benefits to AGI, giving $63,245 with $18,115 of taxable Social Security instead of $17,323 with $993 taxable. It then assumed TCJA had expired and used an $8,000 deduction, a personal exemption and a 15%/25% bracket schedule. Under 2026 OBBBA law, the $16,100 + $2,050 + $6,000 deductions exceed the correct AGI, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It put the full $21,312 of Social Security and the $28,800 survivor benefits into AGI and subtracted the farm rent instead of adding it, reaching $46,649. It then reported $178 of tax on $29,449 of taxable income, which does not follow from any bracket schedule. The correct AGI is $17,323 ($4,793 + $257 + $11,280 + $993 taxable Social Security), and that is below the $18,150 standard-plus-aged deduction." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It got AGI right at $17,323 but assumed TCJA expired in 2026, restoring a personal exemption and a pre-TCJA standard deduction that together total about $15,800. OBBBA kept the higher deduction: $16,100 single + $2,050 aged + the $6,000 senior deduction, or $24,150, which exceeds AGI and leaves $0 tax." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $28,800 survivor benefits and 85% of Social Security, giving AGI of $63,245 instead of $17,323. It also applied TCJA-sunset deductions and exemptions of about $15,400. The correct AGI against the $24,150 of 2026 OBBBA deductions yields zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits in gross income, which made $18,115 of Social Security taxable, for an AGI of $63,245. It also used TCJA-sunset rules: an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. Without that input, AGI is $17,323, and the OBBBA $16,100 + $2,050 + $6,000 deductions wipe out taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It taxed the survivor benefits and 85% of Social Security and applied a post-TCJA-sunset standard deduction plus personal exemption, producing $6,546.75. AGI excluding the $28,800 input is $17,323, and the 2026 single standard deduction ($16,100), aged addition ($2,050) and senior deduction ($6,000) reduce taxable income to zero." +us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It built AGI of $63,245 by including the $28,800 survivor benefits and $18,115 of taxable Social Security. Excluding that input, provisional income ($26,986) is just above the $25,000 single base, only $993 of Social Security is taxable, and AGI of $17,323 falls entirely within the standard, aged and senior deductions." +us,scenario_062,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly found taxable income below zero but then subtracted $1,031 of nonrefundable elderly/disabled credit to report a negative tax. Nonrefundable credits are capped at tax liability. The Schedule R credit is also $0 here because $20,319 of nontaxable Social Security exceeds the $5,000 initial amount. Tax before refundable credits is therefore $0, not -$1,031." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $28,800 survivor benefits as taxable. That drove 85% of Social Security into income and produced tax of $4,328 even after the senior and QBI deductions. Without that input, AGI is $17,323, which is below the $18,150 standard-plus-aged deduction before the senior deduction is even applied." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It included 'taxable survivor' income of $28,800 in AGI, which pushed provisional income past the $34,000 single threshold and made $18,115 of Social Security taxable. The correct AGI is $17,323 with only $993 of Social Security taxable, and the single standard, aged and senior deductions reduce it to zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $18,115 of taxable Social Security only by adding the $28,800 survivor-benefits input to gross income. Excluding it, taxable Social Security is $993 and AGI is $17,323, which the $16,100 standard deduction plus the $2,050 aged addition and the $6,000 senior deduction fully offset, so there is no tax and the $959 QBI deduction is moot." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly treated the $28,800 survivor payment as taxable non-Social-Security income, which inflated AGI to $63,245.20 and made 85% of Social Security taxable. With that input excluded, AGI is $17,323 against its own $24,150 of standard, aged and senior deductions, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse even though that status requires a dependent child; with no dependents, the head files single. It also taxed the $28,800 survivor benefits, which made $18,115 of Social Security taxable. The correct single AGI is $17,323, which the $16,100 + $2,050 + $6,000 deductions reduce to zero taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It included the $28,800 survivor benefits and 85% of Social Security in AGI ($63,245). It also applied TCJA-sunset rules: a roughly $16,049 standard deduction plus exemption, with 10%/15% brackets. OBBBA made the TCJA structure permanent, and the correct $17,323 AGI is fully offset by the $16,100 standard deduction, $2,050 aged addition and $6,000 senior deduction." +us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted $45,130 of 'other taxable income' by including the $28,800 survivor benefits, making $18,115 of Social Security taxable. It then used post-TCJA-sunset deductions ($10,612 plus a $5,442 exemption) and a 15% bracket. The correct AGI of $17,323 against the 2026 OBBBA deductions of $24,150 gives $0." +us,scenario_062,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $28,800 survivor benefits to AGI, giving $63,245 with 85% of Social Security taxable, and it also omitted the $6,000 senior deduction. Excluding that input, AGI is $17,323, which is already below its own $18,150 single-plus-aged standard deduction, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so the answer could not be scored." +us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It computed the correct $17,323 AGI with $993 of taxable Social Security but applied a 'restored exemption' and pre-TCJA standard deduction, leaving about $1,820 taxable. OBBBA kept the TCJA structure: the $16,100 single deduction, the $2,050 aged addition and the $6,000 senior deduction exceed AGI, so tax is $0." +us,scenario_062,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It labeled the $28,800 survivor benefits as taxable and added them to gross income, which made 85% ($18,115) of Social Security taxable and produced AGI of $63,245. Without that input, AGI is $17,323, and its own $23,750 of standard, aged and senior deductions eliminates taxable income." +us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It settled on taxing the $28,800 survivor benefits as SBP-type income and 85% of Social Security, for AGI of $63,245. It also used a $16,550 deduction and omitted the $6,000 senior deduction. The correct AGI is $17,323, and the 2026 single standard deduction ($16,100) plus the aged addition ($2,050) and the senior deduction reduce it to zero taxable income." us,scenario_062,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_062,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It computed $32,849 of countable income against the ~$15,060 poverty guideline and correctly flagged that $29,100 in bank assets blows through the $2,000 SSI-related resource limit, then discarded both of its own disqualifying findings by invoking Florida's medically needy share-of-cost program and spousal protections. The aged/disabled route to Medicaid attaches through SSI receipt, and SSI here is $0, so medicaid_category is NONE; medically needy spend-down confers no eligibility. It also omitted the $4,793 of farm rent from its countable total, understating income by that amount." us,scenario_062,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"It struck the $21,312 of Social Security survivor benefits from countable income as ""largely excluded,"" but Social Security is counted in full as unearned income under SSI-related rules past the $20 general disregard, raising countable income from its stated $16,330 to about $37,600 — roughly three times the ~$12,000 annual SSI federal benefit rate — and it never applied the $2,000 resource limit to $29,100 of bank assets. Because SSI is $0, no SSI-linked aged/disabled category attaches and medicaid_category is NONE, so its ""low countable income relative to the aged/disabled standard"" premise fails on both the income and resource tests." @@ -4660,115 +5113,133 @@ us,scenario_062,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligib us,scenario_062,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_062,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $4,793 of farm rent income as self-employment income. Farm rent does not create net earnings from self-employment here, so the correct tax base is zero rather than $4,793 and self-employment tax is $0." us,scenario_062,ssi,glm-5.3,llm_error,asset_resource,False,"The model counted only the $257 of taxable interest, excluding the $21,312 of Social Security survivor benefits, the $11,280 private pension, and the $4,793 of farm rent, all of which are fully countable unearned income for SSI under 20 CFR 416.1121, and it added the $20 general exclusion to income instead of subtracting it while inventing one-third in-kind support that no listed fact supports; actual countable unearned income of roughly $37,400 exceeds the annual federal benefit rate several times over. It also never applied the $2,000 individual countable-resource limit to the $29,100 bank account, which zeroes SSI on its own, and its $812.01 monthly rate matches no 2026 FBR, with Florida paying no optional state supplement." -us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It ran CHIP's income test on the child's own resources — ""with no income reported for Child 1"" — instead of on the tax unit's MAGI, which is the countable amount for a child claimed as a tax dependent. The tax unit's MAGI is about $130,700 ($97,295 wages + $20,000 taxable 401(k) distributions + $33,350 pass-through income, less the $16,800 farm loss and the $3,000 capital-loss cap), roughly 340% of the five-person poverty guideline and above Wisconsin's 306% BadgerCare Plus children's ceiling. It spent its entire reasoning on whether ESI blocks CHIP and never computed the FPL ratio that decides the case." -us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,health_coverage,False,"It asserted ""family income under 300% FPL"" without computing MAGI; 253% is what the head's $97,295 of wages alone produce against the ~$38,400 five-person guideline, and that is the only way to reach its conclusion. Counting the $20,000 taxable 401(k) distribution and the $33,350 of partnership/S-corp and self-employment partnership income, with the capital loss limited to $3,000, puts MAGI near $130,700, about 340% of the guideline and above Wisconsin's 306% children's ceiling." -us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"It cited the correct 306% BadgerCare Plus ceiling but never computed the numerator: staying under 306% requires MAGI below about $117,500 for the five-person unit, and the household's MAGI is roughly $130,700 once the $20,000 taxable 401(k) distribution and $33,350 of pass-through income are added to wages and the $18,235 capital loss is limited to the $3,000 annual cap. That is about 340% of the guideline, so the child fails the CHIP income test as well as the 156% FPL Medicaid limit for ages 6–18." -us,scenario_064,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was returned for child1_chip_eligible, so the model made no substantive determination to evaluate. The required answer is 0: the tax unit's MAGI of about $130,700 is roughly 340% of the five-person poverty guideline, above Wisconsin's 306% BadgerCare Plus children's ceiling and far above the 156% FPL Medicaid limit for a 12-year-old." -us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,health_coverage,False,"It correctly placed the child above Wisconsin's Medicaid ceiling for ages 6–18 but then asserted membership in the ""upper CHIP income band"" without computing MAGI against the poverty guideline. That band tops out at 306% FPL, and the tax unit's MAGI of about $130,700 — wages plus the $20,000 retirement distribution plus $33,350 of pass-through income, less the farm loss and the $3,000 capital-loss cap — is roughly 340% of the ~$38,400 five-person guideline, so the child overshoots the band rather than landing in it." -us,scenario_064,child1_chip_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its 297% FPL figure is exactly what results from deducting the $16,800 farm loss twice — once as ""farm income"" and once as ""farm operations income"" — which are two labels for the same loss ($150,645 of gross income less $33,600 of farm loss less the $3,000 capital-loss cap = $114,045, or 297% of the ~$38,400 five-person guideline). Deducting that loss once leaves MAGI at about $130,700, roughly 340% of the guideline and above the 306% BadgerCare Plus children's ceiling it otherwise cited correctly." -us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $106,225 MAGI understates countable income by about $24,500 because it wrote off the full $18,235 long-term capital loss rather than the $3,000 annual cap and treated Roth contributions and employer-sponsored premiums as income reductions, when only the ~$2,827 of traditional 401(k)/IRA contributions reduce AGI. Correctly counted, MAGI is about $130,700, roughly 340% of the five-person guideline and above the 306% ceiling; its ESI-affordability argument imports the ACA premium-tax-credit test, which plays no role in the CHIP income determination that already disqualifies the child." -us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The ""pre-tax contributions and allowable losses"" write-down it relied on does not exist at that scale: Roth 401(k)/IRA amounts never reduce AGI, the traditional contributions total only about $2,827, and the $18,235 capital loss is deductible against ordinary income only up to $3,000 per year. MAGI therefore stays near $130,700 — wages, the $20,000 taxable 401(k) distribution and $33,350 of pass-through income less the $16,800 farm loss — about 340% of the ~$38,400 five-person guideline and above Wisconsin's 306% BadgerCare Plus children's limit, so the ESI point it argued is moot." +us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"Called the child eligible because Child 1 has no income of their own, then never ran an income test on the household. CHIP measures the tax unit's Medicaid MAGI against the Wisconsin CHIP child limit, and this 5-person household's income level is above that limit. Its conclusion that ESI coverage does not matter has no bearing, because the child already fails the income test." +us,scenario_064,child1_chip_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Stated without any calculation that household income is within the Wisconsin CHIP limit. PolicyEngine's CHIP child test finds the tax unit's MAGI-to-FPG ratio above its Wisconsin CHIP child income limit, so Child 1 fails the income test even though the age test passes." +us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Assumed a 300% FPL BadgerCare Plus ceiling and placed the household below it. PolicyEngine compares the 5-person tax unit's Medicaid MAGI ratio with its Wisconsin CHIP child income limit, and this household is above that limit, so the child is not CHIP-eligible." +us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied a 306% FPL Wisconsin CHIP threshold and asserted the household was below it. Under PolicyEngine's Wisconsin CHIP child income limit, the tax unit's MAGI-based income level fails the test. Being under age 19 does not make up for failing the income test." +us,scenario_064,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child1_chip_eligible, so there is no answer to score. The correct output is 0: the 12-year-old passes CHIP's age test but fails PolicyEngine's Wisconsin CHIP income test and is also not Medicaid-eligible." +us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Correctly found the child above the Medicaid ceiling, but then placed the household inside Wisconsin's upper CHIP income band. The tax unit's MAGI-to-FPG ratio is above PolicyEngine's Wisconsin CHIP child income limit, so the child falls outside both Medicaid and CHIP." +us,scenario_064,child1_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"Computed household MAGI at about 297% FPL, compared it with a 306% FPL CHIP/BadgerCare Plus ceiling, and called the child eligible. PolicyEngine's Wisconsin CHIP child income limit is below this household's Medicaid income level, so the income test fails and Child 1 is not CHIP-eligible." +us,scenario_064,child1_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"Correctly used a five-person household above the child Medicaid limit, then treated 306% FPL as Wisconsin's CHIP ceiling and put the household under it. The household's MAGI-based income level is above PolicyEngine's Wisconsin CHIP child income limit, so the child fails CHIP's income criterion." +us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Put MAGI at $106,225 (273% FPL) against an assumed 300% Wisconsin limit and called the child eligible. PolicyEngine's CHIP child test finds the tax unit's Medicaid income level above its Wisconsin CHIP child income limit, so Child 1 fails the income test. Its argument that ESI is unaffordable has no effect on that result." +us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"Deducted pre-tax contributions and losses, then concluded that MAGI falls below a 300% FPL Wisconsin CHIP threshold for five people. The household's MAGI-to-FPG ratio is above PolicyEngine's Wisconsin CHIP child income limit, so the child fails the income criterion even though the age limit of 19 is met." us,scenario_064,child1_early_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,child1_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Asserted that Wisconsin makes every child under 19 ""categorically eligible"" for Medicaid at or below 300% FPL and never computed the household's MAGI against any limit. The 300%/306% FPL figure is the BadgerCare Plus CHIP ceiling, not the Medicaid MAGI limit for a child aged 6-18, which is 156% FPL; this household sits at 3.22x FPL, above both, so medicaid_category resolves to NONE." -us,scenario_064,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Used a 301% FPL children's limit — the combined BadgerCare Plus/CHIP ceiling — in place of the 156% FPL Medicaid MAGI limit that applies to a 12-year-old, then claimed the farm loss and capital losses pushed MAGI beneath it. The $18,235 long-term capital loss is limited to a $3,000 net capital loss deduction, and after the -$16,800 farm loss the household still stands at 3.22x FPL, above 156% and above 301%." -us,scenario_064,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Computed household MAGI at roughly $119,845 against its own ~$109,740 (300% FPL, family of five) line, correctly found it over the limit, and then discarded that result to answer eligible on the presumption that a 12-year-old with $0 personal income is treated as qualifying. Medicaid MAGI for a child is measured on the tax household, not the child's own income, and the governing Wisconsin limit for ages 6-18 is 156% FPL, so the income test its own arithmetic already failed is the decisive one." -us,scenario_064,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Treated the full -$18,235 long-term capital loss and -$16,800 farm loss as reducing countable MAGI, inflated the unit to six people when the tax household is five, and measured the result against a ~300% FPL line. Net capital losses are deductible only up to $3,000, leaving MAGI at 3.22x FPL, and the Wisconsin Medicaid MAGI limit for a child aged 6-18 is 156% FPL — the ~300% figure is the CHIP ceiling, which 3.22x FPL also exceeds." -us,scenario_064,child1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child1_medicaid_eligible, so no substantive determination was submitted. The correct derivation applies Wisconsin's 156% FPL Medicaid MAGI limit for children aged 6-18 to a household at 3.22x FPL, yielding medicaid_category = NONE and an answer of No." -us,scenario_064,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Applied Wisconsin's 306% FPL BadgerCare Plus ceiling — the CHIP limit — as the Medicaid limit for a 12-year-old, whose Medicaid MAGI limit is 156% FPL, and understated MAGI as $114,422 against the actual 3.22x FPL (about $124,600 for a family of five), a gap consistent with deducting the whole $18,235 capital loss rather than the $3,000 cap. Even measured against its own 306% line, the household's true MAGI clears it." -us,scenario_064,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child1_medicaid_eligible, so no substantive determination was submitted. The correct derivation applies Wisconsin's 156% FPL Medicaid MAGI limit for children aged 6-18 to a household at 3.22x FPL, yielding medicaid_category = NONE and an answer of No." +us,scenario_064,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It treated Wisconsin's ~300% FPL BadgerCare Plus ceiling as the Medicaid limit for children, but that upper band is CHIP-funded; the Medicaid limit for a 12-year-old is about 151% FPL. It also never calculated household MAGI and simply declared the child eligible. The household is at 3.22 × FPL, which fails even the 300% test the model itself cited." +us,scenario_064,child1_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It claimed the -$16,800 farm loss and the -$18,235 capital loss pull MAGI below 301% FPL, but only $3,000 of a net capital loss is deductible. With the wages, $19,350 of partnership income and $20,000 of 401(k) distributions, MAGI stays at 3.22 × FPL. It also used the 301% BadgerCare/CHIP ceiling instead of the much lower Medicaid limit for children aged 6–18." +us,scenario_064,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly found household MAGI (~$119,845) above 300% FPL for a family of five, then overrode that result by guessing at a child-only income test. Medicaid MAGI eligibility for children uses household income, so the child's own $0 income is irrelevant. It also used the ~300% BadgerCare figure, which is the CHIP-funded ceiling; the Medicaid limit for this age is about 151% FPL." +us,scenario_064,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It sized the household at 6 people when only 5 are listed (head, spouse, two dependents, child1), which inflated the FPL threshold to about $130,000. It also treated the full -$18,235 capital loss as reducing MAGI, although the deduction is capped at $3,000. Correctly computed, income is 3.22 × FPL, above both the ~300% CHIP-funded BadgerCare ceiling and the much lower Medicaid limit for children." +us,scenario_064,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It asserted household income is below 300% FPL without doing any calculation, but household MAGI is 3.22 × FPL. It also applied the ~300% BadgerCare/CHIP ceiling as the Medicaid limit, instead of Wisconsin's Medicaid limit for children aged 6–18 of about 151% FPL." +us,scenario_064,child1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child1_medicaid_eligible, so there is no answer to score." +us,scenario_064,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It tested MAGI against a ~306% FPL limit, which is the top of the CHIP-funded BadgerCare Plus band, not the Medicaid limit for children aged 6–18 (about 151% FPL). It also put MAGI below 306%, but the household's MAGI is 3.22 × FPL and fails even that ceiling." +us,scenario_064,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It compared its MAGI of $114,422 to a 306% FPL limit and then added another 5% disregard on top, even though 306% already includes the disregard. More fundamentally, 306% is the ceiling of the CHIP-funded BadgerCare Plus band. The Medicaid limit for a 12-year-old is about 151% FPL, and the engine's 3.22 × FPL MAGI exceeds every child pathway." +us,scenario_064,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child1_medicaid_eligible, so there is no answer to score." us,scenario_064,child1_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,child1_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value was returned for child1_wic_eligible: the model omitted the key from its submit_outputs `outputs` object despite the instruction to include every requested variable exactly once. This is a contract failure, not a substantive misreading of WIC's under-5 categorical rule or its 185% FPL income test." us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that a 12-year-old is ""in the WIC-eligible age range,"" but WIC's child category ends at the fifth birthday, so a 12-year-old has no WIC categorical status at all — the model appears to have imported a school-age or free-school-meals age frame into WIC. It also skipped the WIC income test entirely, which caps countable income at 185% of the federal poverty guideline and disqualifies this household's ~$98,810 of income for 5 people regardless of age." us,scenario_064,dependent1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model explicitly asserted that ""dependent income counting attributes the tax unit's income to the dependent's Medicaid income group,"" which inverts the MAGI household rule: under 42 CFR 435.603(f)(2)(i), an individual other than a spouse or a child under 19 who is claimed as a tax dependent is exempted from the tax-filer rule and has their household built under non-filer rules, so the 27-year-old's Medicaid household is himself alone. His own countable MAGI is $0 (0% FPL), which sits below Wisconsin's adult threshold in the ADULT MAGI category, so he is eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a resource test — the head's $56,000 bank account and $58,676 in stock — to a person evaluated under the MAGI-based ADULT category, where 42 CFR 435.603(g) prohibits any asset or resource test; it also attributed assets owned by the head to the dependent. With the tax-dependent exception placing the 27-year-old in a one-person non-filer household with $0 income (0% FPL), he qualifies under the adult category regardless of household assets." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model counted the head's ~$97,295 in wages plus 401(k) distributions and partnership income as the dependent's MAGI household income. The tax-dependent exception in 42 CFR 435.603(f)(2)(i) removes an adult tax dependent who is not a spouse or a child under 19 from the filer's household, leaving a household of one with $0 income (0% FPL), which clears the ADULT category threshold and makes him eligible." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model assumed the head's ~$97k-plus income flows to the dependent under either MAGI or disability-based limits. For a 27-year-old claimed as a tax dependent, MAGI household composition follows non-filer rules under the tax-dependent exception, so only his own income counts — $0, or 0% FPL — placing him inside the ADULT MAGI category that PolicyEngine returned as True." -us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The one-line rationale treats the dependent's income as the parents' household income. The MAGI tax-dependent exception gives an adult dependent who is not a spouse or a child under 19 a household of one, so the operative figure is his own $0 MAGI (0% FPL), which qualifies him under the ADULT expansion category rather than disqualifying him." -us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model summed the head's wages, partnership income, farm loss and 401(k) distributions into a single household MAGI and compared that total to 138% FPL for the whole family. MAGI eligibility is determined person by person: the 27-year-old adult tax dependent falls under the tax-dependent exception, receives a non-filer household of one, and his own MAGI of $0 (0% FPL) sits far below the ADULT category threshold." -us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model compared the parents' household income against the adult FPL limit rather than the dependent's own. As an adult tax dependent who is not a spouse or child under 19, his MAGI household is built under non-filer rules and contains only himself, so his income level is 0% FPL and he clears Wisconsin's adult BadgerCare income limit." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model asserted income exceeds Wisconsin's levels, which holds only if the head's earnings are attributed to the dependent. The MAGI tax-dependent exception isolates the 27-year-old into his own one-person household with $0 income (0% FPL), qualifying him under the ADULT category; his disability status is irrelevant because the MAGI pathway already covers him." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"The explanation restates the verdict without a derivation, and the submitted 0 is consistent only with charging the head's ~$100k tax-unit income to the 27-year-old dependent. The correct derivation applies the MAGI tax-dependent exception, gives him a non-filer household of one with $0 income (0% FPL), and returns eligible under the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model measured ""Dependent 1's household income"" as the parents' combined income against Wisconsin's adult threshold. Under 42 CFR 435.603(f)(2)(i) an adult tax dependent who is not a spouse or a child under 19 uses non-filer household rules, so his household is one person with $0 income — 0% FPL — and he is eligible in the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The stated ground, ""due to household income,"" attributes the head's earnings to the dependent. MAGI eligibility is computed per person with the tax-dependent exception applying to a 27-year-old claimed as a dependent, so his countable MAGI is his own $0 (0% FPL) and he qualifies under the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"The model applied a family-income test to an adult whose MAGI household, under the tax-dependent exception, contains only himself. His own income is $0 (0% FPL), which is below Wisconsin's adult limit, so the ADULT MAGI category returns eligible." -us,scenario_064,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model wrote that ""household income includes the tax unit's income"" for MAGI purposes and added employer coverage as a second disqualifier; both steps are wrong here. The tax-dependent exception gives the 27-year-old a one-person non-filer household with $0 MAGI (0% FPL), and Medicaid — unlike CHIP — imposes no bar on holding employer-sponsored insurance, so he is eligible in the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was returned for dependent1_medicaid_eligible, so the miss comes from the missing output rather than a substantive rule error. The correct derivation applies the MAGI tax-dependent exception to the 27-year-old, yielding a one-person household with $0 income (0% FPL) and eligibility under the ADULT category, value 1." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model reasoned from ""an adult in a high-income household,"" importing the head's income into the dependent's MAGI test. An adult claimed as a tax dependent who is not a spouse or a child under 19 is carved out by the tax-dependent exception and gets a household of one, so his 0% FPL income level makes him eligible under the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model defaulted to ineligible on the ground that household income is ""not low enough"" and that disability details were absent. Neither controls: the MAGI tax-dependent exception isolates the 27-year-old into a one-person household with $0 income (0% FPL), and the ADULT MAGI category alone establishes eligibility without any disability-based pathway." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model stated the family's annual income exceeds Wisconsin's adult Medicaid limit ""under the household MAGI calculation,"" applying the tax-filer household rule to an adult dependent. Because he is neither a spouse nor a child under 19, the tax-dependent exception routes him to non-filer rules — a household of one with $0 income, 0% FPL — and the ADULT category returns eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model invoked both income and resource rules to deny eligibility. The dependent's own MAGI is $0 (0% FPL) once the tax-dependent exception gives him a one-person non-filer household, and the MAGI-based ADULT category carries no resource test at all under 42 CFR 435.603(g), so both grounds fail and he is eligible." -us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model rested on ""the available income and assets,"" charging the head's earnings and the household's $56,000 bank and $58,676 stock holdings to the dependent. His MAGI household under the tax-dependent exception is himself alone with $0 income (0% FPL), and the ADULT MAGI category applies no asset test, so he qualifies." -us,scenario_064,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,"""Income exceeds Medicaid thresholds"" holds only against the parents' income, not the dependent's. The MAGI tax-dependent exception gives the 27-year-old a household of one with $0 countable income (0% FPL), which clears Wisconsin's adult threshold and makes him eligible under the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model ran a non-MAGI ABD resource test with a $2,000 limit against household assets he does not own, and then applied the MAGI adult limit to the parents' income. Eligibility here runs through the MAGI-based ADULT category, which has no resource test, and the tax-dependent exception makes his countable income $0 (0% FPL), so he is eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model correctly identified that MAGI adult rules govern a 27-year-old, then built the wrong household by counting him ""with the household"" for both income and resources. The tax-dependent exception in 42 CFR 435.603(f)(2)(i) gives an adult tax dependent a non-filer household of one, producing $0 income at 0% FPL, and the MAGI ADULT category applies no resource test — so he is eligible." -us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model computed a household MAGI of roughly 273% FPL for the whole family and applied it to the dependent, plus a $100k asset test drawn from ABD rules. His own MAGI household under the tax-dependent exception is one person with $0 income (0% FPL), and the MAGI ADULT category — the one PolicyEngine returned — carries no asset limit, so he qualifies." -us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for dependent1_medicaid_eligible, so the miss is a contract failure rather than a rule error. Applying the MAGI tax-dependent exception to the 27-year-old yields a one-person household with $0 income (0% FPL) and eligibility under the ADULT category, value 1." -us,scenario_064,dependent1_medicaid_eligible,minimax-m3,llm_error,household_unit_or_filing_status,False,"The explanation is a bare ""Not eligible,"" and the 0 is consistent only with testing the dependent against the parents' ~$100k tax-unit income. The correct derivation applies the MAGI tax-dependent exception, leaving a one-person household with $0 income (0% FPL) that qualifies under the ADULT category." -us,scenario_064,dependent1_medicaid_eligible,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model measured the dependent against a family income of ~350% FPL and against SSI-related resource tests using the head's $56,000 bank account and $58,676 in stock. Because he is an adult tax dependent who is not a spouse or a child under 19, non-filer household rules give him a household of one with $0 income (0% FPL), and the MAGI ADULT category that governs here imposes no resource test." -us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model built the dependent's MAGI from the head's $97,295 in wages plus partnership and other income. MAGI household composition is determined per person, and the tax-dependent exception places a 27-year-old claimed as a dependent in a one-person non-filer household with $0 income — 0% FPL — well under Wisconsin's adult limit, so he is eligible." -us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"""Income too high"" can only refer to the parents' income, since the dependent has no listed earnings and all unlisted inputs are 0. Under the MAGI tax-dependent exception his household is himself alone with $0 income (0% FPL), which places him in the ADULT category as eligible." -us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model asserted that ""under standard tax-benefit modeling, disabled individuals qualify for Medicare (SSDI-based eligibility),"" treating the is_disabled input as itself establishing SSDI entitlement. Medicare's under-65 pathway requires 24 months of actual SSDI entitlement or an ESRD/ALS diagnosis, and this household reports no Social Security disability income at all (unlisted inputs are 0), while Dependent 1 works 20 hours weekly with employer-sponsored insurance; PolicyEngine's is_medicare_eligible resolves on the age-65 threshold, which a 27-year-old fails." -us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model stated that disability ""qualifies for Medicare eligibility regardless of age,"" skipping the entitlement precondition entirely — Medicare under 65 requires 24 completed months of SSDI entitlement, or ESRD/ALS, none of which appear in the facts. With no Social Security disability income listed and every unlisted input set to 0, Dependent 1 has no SSDI entitlement, and PolicyEngine's age-65 test returns False for a 27-year-old." -us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model recited the correct rule — age 65+, or 24 months of SSDI entitlement, or ESRD/ALS — then waived its own 24-month condition by treating the instruction to hold disability status constant through the year as a substitute for SSDI receipt. Constancy of the is_disabled flag says nothing about SSDI entitlement, and the household lists no Social Security disability income (unlisted inputs are 0), so the 24-month clock never starts and the age-65 test that governs is_medicare_eligible fails for a 27-year-old." -us,scenario_064,dependent1_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for dependent1_medicare_eligible, so the submission never satisfied the contract requiring every requested key with a numeric value. The correct derivation is that a 27-year-old with no SSDI entitlement, no ESRD/ALS, and employer-sponsored coverage fails PolicyEngine's age-65 Medicare test, yielding 0." -us,scenario_064,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model wrote that being ""reported as disabled"" qualifies Dependent 1 for Medicare ""in this estimate,"" substituting the disability characteristic for the SSDI entitlement that the under-65 pathway actually requires (24 months of SSDI, or ESRD/ALS). No Social Security disability income is listed and unlisted inputs are 0, so no entitlement exists; PolicyEngine's is_medicare_eligible applies the age-65 threshold, which a 27-year-old does not meet." +us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly credited the head's and spouse's tax-unit MAGI to dependent1 as a claimed dependent and compared that total with a 100% FPL BadgerCare limit. The engine measures dependent1's own MAGI income level at 0.00 x FPL, which puts dependent1 in the ACA adult category as eligible." +us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It denied eligibility by applying an adult/disability asset limit to the household's $56,000 bank account and $58,676 in stock. Dependent1 qualifies through the MAGI-based ACA adult category, which has no asset test, and their own income is 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1 against the head's ~$97k wages plus 401(k) distributions and partnership income. Dependent1's MAGI income level is 0.00 x FPL, well under the adult-category threshold, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It tested dependent1 against the household's income (the head's ~$97k wages and other income) under both MAGI and disability limits. The engine rates dependent1 in the ACA adult category with a MAGI income level of 0.00 x FPL, which is eligible." +us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It held that the adult dependent's household income exceeds the Medicaid limits, which assigns the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which makes them eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It added the head's wages, partnership income, farm loss and 401(k) distributions into a household MAGI above 138% FPL and applied that total to dependent1. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 passes the adult expansion test." +us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It judged that household income is above the adult limit, crediting the parents' income to dependent1. Dependent1's MAGI income level is 0.00 x FPL, which qualifies them for the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It compared household income with a 100% FPL BadgerCare adult limit, which charges the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, below any adult threshold, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It declared that income exceeds Wisconsin's Medicaid levels even though dependent1 has $0 of income. The only income that exceeds the limit is the head's; dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no rule, only a bare 'not eligible'. That answer matches testing dependent1 against the high-income parents' household. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled that dependent1's household income exceeds the Wisconsin adult thresholds, which assigns the parents' income to dependent1. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 is eligible as an ACA expansion adult." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It denied eligibility 'due to household income', crediting the head's earnings to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which passes the ACA adult income test." +us,scenario_064,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied family income to dependent1's adult Medicaid test. Dependent1's MAGI income level is 0.00 x FPL, so they are eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the tax unit's ~$100k+ income (the head's wages, partnership income and 401(k) distributions) as dependent1's MAGI household income and also cited dependent1's employer-sponsored insurance. The engine measures dependent1's MAGI income level at 0.00 x FPL and returns eligible in the ACA adult category, with the ESI coverage in place." +us,scenario_064,dependent1_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for dependent1_medicaid_eligible. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible (1)." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled dependent1 ineligible because they live in a high-income household, which applies the parents' income to dependent1. Dependent1's own MAGI income level is 0.00 x FPL, which makes them eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It defaulted to ineligible because there were no disability-income details and because of household income, treating disability as the only relevant pathway. Dependent1 qualifies through the MAGI-based ACA adult category, which needs no disability determination, at a MAGI income level of 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly used the family's household MAGI for dependent1 as an adult tax dependent and found that total above Wisconsin's adult limit. The engine measures dependent1's MAGI income level at 0.00 x FPL, which is eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It applied income and resource rules as if dependent1 were being tested under a disability-based pathway. Dependent1 qualifies through the MAGI-based ACA adult category, which has no resource test, and their MAGI income level is 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It cited the available income and assets as blocking eligibility, which means testing the household's bank and stock holdings and the parents' income. The MAGI-based ACA adult category has no asset test, and dependent1's MAGI income level is 0.00 x FPL, which is eligible." +us,scenario_064,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed that income exceeds Medicaid thresholds even though dependent1 has $0 of income, so it counted the head's earnings. Dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category." +us,scenario_064,dependent1_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It failed dependent1 on the ABD $2,000 resource test and on the MAGI adult income limit, charging dependent1 with the household's assets and income. The ABD pathway is irrelevant because dependent1 passes the MAGI-based ACA adult category, which has no asset test, at a MAGI income level of 0.00 x FPL." +us,scenario_064,dependent1_medicaid_eligible,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly routed the 27-year-old to MAGI adult rules but counted household income against the adult limit and also applied ABD/SSI resource limits to household assets. Dependent1's own MAGI income level is 0.00 x FPL, and the adult category has no resource test, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It placed dependent1 in the adult category but tested household MAGI, which includes the head's ~$97k wages and other income, against the adult limit. The engine measures dependent1's MAGI income level at 0.00 x FPL, so dependent1 is eligible as an ACA expansion adult." +us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It assigned dependent1 a household MAGI of ~273% FPL built from the parents' income and applied an ABD asset limit to the family's $100k+ of assets. Dependent1's own MAGI income level is 0.00 x FPL, and the ACA adult category has no asset test, so dependent1 is eligible." +us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for dependent1_medicaid_eligible. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible (1)." +us,scenario_064,dependent1_medicaid_eligible,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It answered 'Not eligible' with no rule. That result matches testing dependent1 against the high-income parents' household rather than dependent1's own $0 income. The correct derivation puts dependent1's MAGI income level at 0.00 x FPL in the ACA adult category, which yields eligible." +us,scenario_064,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1 against family income of ~350% FPL and applied SSI-related tests to the head's $56k bank account and $58,676 in stock. Dependent1's MAGI income level is 0.00 x FPL, and the ACA adult category has no asset test, so dependent1 is eligible." +us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It placed the head's $97,295 wages plus partnership and other income in dependent1's household MAGI and compared the total with a ~100% FPL adult limit. Dependent1's MAGI income level is 0.00 x FPL, below that limit or any other adult limit, so they are eligible." +us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It stated that dependent1 has income too high to qualify, although dependent1's listed income is $0, so it imputed the parents' income to them. Dependent1's MAGI income level is 0.00 x FPL, which is eligible in the ACA adult category." +us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed 'SSDI-based eligibility' just because Dependent 1 is disabled, but the household lists no SSDI benefits, and unlisted amounts are 0. The under-65 disability path needs 24 months of actual SSDI entitlement, which Dependent 1 does not have, so the correct answer is not eligible (0)." +us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model said a disability alone qualifies someone for Medicare at any age. Under 65, Medicare instead requires 24 months of SSDI entitlement, or ESRD or ALS. Dependent 1 has no SSDI income, so a 27-year-old is not eligible." +us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly named the 24-month SSDI entitlement rule but then applied it without any SSDI receipt, using only the disability flag. The household has $0 in Social Security disability benefits and no months of receipt, so the SSDI path does not apply. A 27-year-old with no ESRD or ALS is not eligible." +us,scenario_064,dependent1_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for dependent1_medicare_eligible. The correct answer is 0: Dependent 1 is 27 with no SSDI entitlement and no ESRD or ALS, so no Medicare path applies." +us,scenario_064,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model counted being disabled under age 65 as enough for Medicare. Under-65 disability eligibility actually depends on 24 months of SSDI entitlement, and the household shows no SSDI benefits. Dependent 1 therefore does not qualify." +us,scenario_064,dependent1_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied a 'disability-based Medicare eligibility rule' that keys only on the disability flag. The real rule requires 24 months of SSDI entitlement (or ESRD or ALS), and Dependent 1 receives no SSDI, so a 27-year-old is not eligible." us,scenario_064,dependent1_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model reasoned 'Age 18, family income 299% FPL under Wisconsin BadgerCare Plus 300% limit,' accepting age 18 as inside the CHIP child band and reducing the question to a one-point income margin. PolicyEngine places age 18 above the age ceiling for the CHIP child category, so dependent2 has no CHIP category and carries Medicaid category NONE, and the 299%-vs-300% comparison never decides the output. Its FPL ratio also rests on an understated MAGI that gives no weight to the $20,000 taxable 401(k) distribution and $19,350 of partnership/S-corp income in the tax unit." -us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model applied the textbook 'under 19' CHIP age rule to an 18-year-old and then cleared them against Wisconsin's 306% FPL ceiling. PolicyEngine's CHIP child category does not extend to age 18, so dependent2 falls outside it entirely, and the engine records Medicaid category NONE for the same person, yielding is_chip_eligible = False regardless of where household MAGI lands relative to 306% FPL." -us,scenario_064,dependent2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for dependent2_chip_eligible, so no eligibility determination reached the submitted outputs object; this is a contract failure rather than a substantive misreading of Wisconsin CHIP rules. The required submission is 0: at age 18 dependent2 sits above PolicyEngine's CHIP child age ceiling and carries Medicaid category NONE." -us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,age_disability,False,"The model asserted that 'Dependent 2 is age 18, which is under the child CHIP age limit,' then declared the household 'treated as within Wisconsin's CHIP income limit' without computing MAGI at all. Both steps fail: PolicyEngine's CHIP child category excludes an 18-year-old, and the engine assigns dependent2 Medicaid category NONE, so is_chip_eligible = False on the age gate before any income test is reached." -us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model built its answer on a two-tier income band — 'above the Medicaid ceiling but within Wisconsin's upper CHIP income band' — after granting that dependent2 is 'under 19' and therefore inside the child pathway. PolicyEngine places age 18 above the CHIP child category's age ceiling, so dependent2 never enters the Medicaid-to-CHIP income band the model reasoned about; the trace shows Medicaid category NONE and is_chip_eligible False." -us,scenario_064,dependent2_chip_eligible,grok-4.6,llm_error,age_disability,False,"The model wrote 'Dependent 2 is 18, under 19' and then decided the case on income, placing household MAGI at about 297% FPL just under a 306% BadgerCare/CHIP ceiling. The age premise is the error: PolicyEngine's CHIP child category does not reach age 18, so dependent2 has no CHIP category and Medicaid category NONE, and the 297%-vs-306% margin does not determine the answer." -us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,age_disability,False,"The model kept dependent2 inside the child pathway on '18 years old (under 19)', cleared a $106,225 MAGI at 273% of FPL against a 300% Wisconsin ceiling, and added an affordability override ('ESI access does not disqualify given high premium relative to income'). PolicyEngine's CHIP child category excludes an 18-year-old, so the output is 0 on age alone; its $106,225 figure also understates a tax unit whose countable income includes $97,295 of wages, a $20,000 taxable 401(k) distribution, and $19,350 of partnership/S-corp income, and the CHIP model contains no unaffordable-employer-coverage exception of the kind it invoked." -us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for dependent2_chip_eligible, so the requested key is absent from its outputs and no CHIP determination was made. The required answer is 0, because at age 18 dependent2 is above PolicyEngine's CHIP child age ceiling and the engine assigns that person Medicaid category NONE." -us,scenario_064,dependent2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified the BadgerCare Plus children-under-19 pathway but never computed the household's MAGI-to-FPL ratio, instead assuming that the dependent's own zero income and the five-person MAGI household size would hold the family under the limit; MAGI Medicaid counts the entire MAGI household's income, not the child's own, and this household is at 3.22 x FPL against a 306% (301% plus the 5-point disregard) children's cap. Its stated 301% figure is itself the pre-disregard limit, and the household clears even the disregarded 306% ceiling, so the children's category fails on income." -us,scenario_064,dependent2_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was submitted for dependent2_medicaid_eligible, so no substantive reasoning exists to evaluate. The correct derivation compares the household's MAGI of 3.22 x FPL to Wisconsin's 306% FPL children-under-19 limit, finds it above, and — with SSI of 0 and no disability, pregnancy, or caretaker status in a non-expansion state — yields medicaid_category NONE and a value of 0." -us,scenario_064,dependent2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model's $114,422 MAGI omits the head's $14,000 self-employment partnership income; adding it back gives roughly $128,400, above the ~$118,400 threshold the model itself cited, which matches the engine's 3.22 x FPL against Wisconsin's 306% children's limit. Its threshold reasoning also double-counted the disregard by adding 5% on top of 306%, which already includes it." -us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for dependent2_medicaid_eligible, leaving no reasoning to assess. The correct derivation puts the household's MAGI at 3.22 x FPL, above Wisconsin's 306% FPL cap for the children-under-19 category — the only MAGI pathway open to an 18-year-old in a non-expansion state — and with SSI of 0 and no disability, pregnancy, or caretaker status, the answer is 0." +us,scenario_064,dependent2_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"The model applied only the under-19 age test and an income test, and never used the listed fact that Dependent 2 has employer-sponsored insurance. That coverage alone rules out CHIP. It also put household income at about 300% FPL when the engine computes 321.5%, which is above Wisconsin's 306% limit, so its income test passed when it should have failed." +us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,health_coverage,False,"The model ignored Dependent 2's employer-sponsored insurance, which disqualifies the child from CHIP. It also underestimated income at 299% FPL (the engine has 321.5%) and used a 300% limit instead of Wisconsin's 306%. Correct income would have failed either limit." +us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"The model declared eligibility from age under 19 and income below 306% FPL, and did not account for Dependent 2's employer-sponsored insurance, which bars CHIP eligibility. It also placed household income below 306% FPL when the engine's MAGI-based ratio is 321.5%, so the income test fails as well." +us,scenario_064,dependent2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for dependent2_chip_eligible, so no answer was scored. Its output is missing rather than substantively wrong." +us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,health_coverage,False,"The model confirmed only that Dependent 2 is under 19 and simply assumed the household was within Wisconsin's child CHIP income limit. It never considered Dependent 2's employer-sponsored insurance, which disqualifies the child from CHIP. It also missed that household income of 321.5% FPL exceeds the 306% limit." +us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,health_coverage,False,"The model placed Dependent 2 above the Medicaid ceiling but inside Wisconsin's upper CHIP income band. It ignored the child's employer-sponsored insurance, which excludes the child from CHIP. Household income is also 321.5% FPL, above the 306% CHIP ceiling, so the child is not in the CHIP band at all." +us,scenario_064,dependent2_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model computed household MAGI at about 297% FPL, but the engine's figure is 321.5%, above Wisconsin's 306% CHIP ceiling. It also skipped the fact that Dependent 2 is covered by employer-sponsored insurance, which on its own rules the child out of CHIP." +us,scenario_064,dependent2_chip_eligible,grok-4.7,llm_error,health_coverage,False,"The model placed the household's MAGI under Wisconsin's 306% FPL CHIP limit, when it is actually 321.5% FPL and therefore above the limit. It also ignored Dependent 2's employer-sponsored insurance, which disqualifies the child from CHIP regardless of income." +us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,health_coverage,False,"The model treated Dependent 2's employer-sponsored insurance as an offer that could be set aside as unaffordable. In fact the child is enrolled in ESI, and CHIP excludes a child already covered by group health insurance; no affordability exception applies. It also understated MAGI at about $106,225 (273% FPL) and used a 300% limit, when the engine's income is 321.5% FPL, above Wisconsin's 306% limit." +us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for dependent2_chip_eligible, so no answer was scored. Its output is missing rather than substantively wrong." +us,scenario_064,dependent2_medicaid_eligible,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It applied a 301% FPL 'children's MAGI threshold' to the 18-year-old and claimed, without doing the math, that the large family kept income under it. Household MAGI is 3.22× FPL, above the 301–306% BadgerCare/CHIP ceiling and far above Wisconsin's Medicaid older-child limit, so no Medicaid category applies." +us,scenario_064,dependent2_medicaid_eligible,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It asserted household income was below 300% FPL without computing MAGI. The five-person household's MAGI is 3.22× FPL, above every child income limit in Wisconsin, so the 18-year-old has no eligible category." +us,scenario_064,dependent2_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,"It gave no value or explanation for dependent2_medicaid_eligible, so its answer is missing rather than wrong on substance. The correct answer is 0: household MAGI of 3.22× FPL exceeds Wisconsin's child limits, and no disability, pregnancy or caretaker pathway applies." +us,scenario_064,dependent2_medicaid_eligible,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It estimated family MAGI below a roughly 306% FPL 'child Medicaid limit', but household MAGI is 3.22× FPL, above that ceiling. It also used the separate CHIP upper bound as the Medicaid limit for ages 6–18, which is much lower in Wisconsin, so the 18-year-old fails the income test." +us,scenario_064,dependent2_medicaid_eligible,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It computed household MAGI at $114,422, or about 296% FPL for five people, and compared it with a 306% cap plus the 5% disregard. The engine measures household MAGI at 3.22× FPL, which exceeds 306% and also the roughly 311% cap with the disregard. Grok also used the CHIP ceiling in place of Wisconsin's lower Medicaid limit for ages 6–18." +us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for dependent2_medicaid_eligible, so its answer is missing rather than wrong on substance. The correct answer is 0: household MAGI of 3.22× FPL exceeds Wisconsin's child limits, and no disability, pregnancy or caretaker pathway applies." us,scenario_064,dependent2_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent2_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own components — the $13,899 overtime deduction, a $510 QBI deduction, a ~$32,300 standard deduction, $7,630 of pre-credit tax, and $3,200 of child/other-dependent credits — produce $4,430, essentially the reference. It then submitted $5,959, subtracting only about $1,671 of the $3,200 in credits it had just enumerated; the derivation was right and the final credit subtraction was abandoned." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Added the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp figure that already contains it, took the $1,796 domestic production deduction above the line, and computed QBI as 20% of $16,550 = $3,310 instead of 20% of the $2,550 of partnership income remaining after the $16,800 farm loss ($510). Taxable income came to $77,217 against the correct $67,794.08." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Added the $13,899 FLSA overtime premium to wages as extra income when the 2026 qualified-overtime provision makes it a deduction of $13,899.29 from taxable income, and applied a $14,600 standard deduction instead of the $32,200 MFJ amount, reaching $98,486 of taxable income against the correct $67,794.08. It then cut its own $18,285 of pre-credit tax to $9,631 with $8,654 of credits it never identified; only $3,200 of nonrefundable child and other-dependent credits exist." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Omitted the $13,899.29 overtime deduction entirely and took a $989 half-self-employment-tax deduction although the $16,800 farm loss eliminates the $14,000 of self-employment partnership earnings so no SE tax arises. It also capped the nonrefundable child credit at $1,072 instead of the full $2,200 against ample liability, then added an unexplained $1,000 to its own $7,136 result." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Omitted the $13,899.29 overtime deduction, computed $1,978 of self-employment tax on the $14,000 partnership share despite the $16,800 farm loss wiping out net SE earnings, and folded that SE tax into an output defined to exclude it. Its own arithmetic yields $8,178 after credits; the submitted $14,959 adds $6,781 of tax no step generates." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Omitted the $13,899.29 overtime deduction, subtracted the $1,796 domestic production amount, and used a $3,900 QBI deduction instead of $510 (20% of the $2,550 left after the $16,800 farm loss). Its own figures give about $6,200 after credits; it submitted $11,200 by adding roughly $5,000 of 'additional tax on other income' that no provision produces at $114,403 of AGI with a net capital loss." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Never applied the 2026 deduction for the $13,899.29 FLSA overtime premium — worth $1,667.91 at the 12% bracket — and used a $30,000 standard deduction instead of $32,200. It further drove QBI to zero by subtracting a $1,560 self-employed health insurance deduction and a $989 half-SE-tax deduction that the $16,800 farm loss precludes; the correct QBI deduction is 20% × $2,550 = $510." -us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Omitted the $13,899.29 overtime deduction, deducted the $1,796 domestic production amount, and treated the child credit as producing no nonrefundable offset when $2,200 plus two $500 other-dependent credits reduce liability dollar for dollar. It then discarded its own $9,300 computation and submitted $12,500 for 'additional Medicare/NIIT effects,' neither of which applies at $114,403 of AGI with wages under $250,000 and a net capital loss." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Omitted the $13,899.29 overtime deduction, deducted $1,796 of domestic production, and claimed $4,000 of child tax credit for two qualifying children when only the 12-year-old qualifies at $2,200 and the 27-year-old and 18-year-old each draw the $500 other-dependent credit. Its $92,674 of taxable income exceeds the correct $67,794.08 by nearly $25,000." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied pre-TCJA 2026 parameters — a $15,900 standard deduction, five $5,200 personal exemptions, a 15% bracket, and a $1,000 child credit — when 2026 carries a $32,200 MFJ standard deduction, no personal exemptions, 10%/12% brackets, and $2,200 plus two $500 dependent credits. It also stripped $1,560 of health premiums out of wages and skipped both the $13,899.29 overtime deduction and the $510 QBI deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used pre-TCJA parameters — five personal exemptions and the $110,000 MFJ child-credit phaseout that cuts the credit to $150 — when 2026 grants no personal exemptions and phases the credit out only above $400,000, leaving the full $2,200 plus two $500 other-dependent credits. It also omitted the $13,899.29 overtime deduction, landing at $85,476 of taxable income against $67,794.08." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Reduced the head's wages by the $19,648 of employer-sponsored insurance premiums, which are already outside the $97,295 gross wage figure, then itemized $21,880 of medical expenses and claimed $25,250 of pre-TCJA personal exemptions with a $1,000 child credit. 2026 gives a $32,200 standard deduction, no personal exemptions, a $13,899.29 overtime deduction, a $510 QBI deduction, and $3,200 of child and other-dependent credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Submitted a figure with no derivation. The correct path takes AGI of $114,403.37 less the $32,200 standard deduction, the $13,899.29 overtime deduction and the $510 QBI deduction to $67,794.08 of taxable income, $7,639.29 of tax, and $4,439.29 after $3,200 of nonrefundable credits; its $11,090.45 corresponds to roughly $96,500 of taxable income with no credits — the arithmetic of dropping the overtime and QBI deductions and every child and dependent credit." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied pre-TCJA 2026 law (a $16,200 standard deduction plus ~$26,000 of personal exemptions and a $1,000 child credit), removed the employer-sponsored insurance premiums from wages, and added $2,000 of additional tax for early 401(k) distribution. The reference's $7,639.29 is exactly the bracket schedule applied to $67,794.08, so no early-distribution penalty enters this output, and 2026 provides a $32,200 standard deduction, a $13,899.29 overtime deduction, and $3,200 of nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Built AGI of $94,756 by stripping employer-sponsored insurance premiums out of the $97,295 of wages, then used pre-TCJA personal exemptions and a $1,000 child credit. The correct 2026 computation is AGI $114,403.37 less the $32,200 standard deduction, the $13,899.29 overtime deduction and the $510 QBI deduction, taxed to $7,639.29, less $2,200 of child credit and two $500 other-dependent credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Submitted a bare figure with no derivation. Taxable income of $67,794.08 yields $7,639.29 of tax and $4,439.29 after the $2,200 child credit and two $500 other-dependent credits; its $11,940 corresponds to taxing over $100,000 of income with no credits, the result of dropping the $13,899.29 overtime deduction, the $510 QBI deduction, and the full $3,200 of nonrefundable credits." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Applied the $32,200 standard deduction and the correct $3,200 of child and other-dependent credits but omitted the 2026 deduction for the $13,899.29 FLSA overtime premium. That deduction is worth $1,667.91 at the 12% bracket, and $4,439.29 + $1,667.91 = $6,107.20 — within $108 of its $6,215." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Landed $115.29 below the correct $4,439.29, the 12%-bracket value of about $961 of deductions beyond the $32,200 standard deduction, $13,899.29 overtime deduction and $510 QBI deduction that fix taxable income at $67,794.08. Its credit total matches the reference $3,200, so the entire error sits in an inflated deduction stack." -us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Computed the QBI deduction on the gross $19,350 of partnership income (20% = $3,870) rather than on the $2,550 that survives netting against the $16,800 farm loss (20% = $510). The extra $3,360 of deduction cuts tax by $403.20 from $4,439.29 to $4,036, matching its $4,057 to within $21." -us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so the key was absent from the submitted outputs object. This is a missing submission rather than a substantive tax error." -us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so the key was absent from the submitted outputs object. This is a missing submission rather than a substantive tax error." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted without computing that deductions and credits fully offset liability. After the $32,200 standard deduction, the $13,899.29 overtime deduction and the $510 QBI deduction, taxable income is still $67,794.08 producing $7,639.29 of tax, and only $3,200 of nonrefundable child and other-dependent credits are available, leaving $4,439.29." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Assumed the household's deductions and credits eliminate positive liability without performing the calculation. AGI of $114,403.37 less $46,609.29 of deductions leaves $67,794.08 of taxable income and $7,639.29 of tax, which the $2,200 child credit and two $500 other-dependent credits reduce only to $4,439.29." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used $108 of deductible traditional IRA contributions instead of the $126.22 that includes Dependent 1's $18 contribution, and rounded the overtime premium to $13,899 and the 401(k) deferral to $2,315 rather than $13,899.29 and $2,315.40. That left taxable income $18.92 high ($67,813 vs $67,794.08) and tax $2.27 high; every rule — standard deduction, overtime deduction, QBI, and the $3,200 of credits — was applied correctly." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,219 sits $2,220.29 below the correct $4,439.29, the 12%-bracket value of roughly $18,500 of excess deduction — the signature of deducting the entire $18,235.41 long-term capital loss instead of the $3,000 annual capital-loss limit the reference applies inside its $19,800 loss deduction. Its stated credit set (child credit plus two other-dependent credits) matches the reference, so the gap lies wholly in the capital-loss treatment." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Overshot by $45.71, the 12% tax on about $381 of taxable income: it dropped the $126.22 of deductible traditional IRA contributions and rounded the $13,899.29 overtime premium and $2,315.40 pre-tax 401(k) deferral. Every structural element — the $32,200 standard deduction, the overtime deduction, the $510 QBI deduction, and the $3,200 of nonrefundable credits — was correct." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied only the $2,200 qualifying-child credit for the 12-year-old and omitted the $500 other-dependent credit for the 27-year-old disabled dependent and the $500 for the 18-year-old, leaving $1,000 of nonrefundable credit unclaimed. $4,439.29 + $1,000 = $5,439.29, within $16 of its $5,455." -us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Reduced wages by the $19,648 of employer-sponsored insurance premiums and the $1,560 of other health premiums to reach AGI of $92,978, though employer-paid premiums are already excluded from the $97,295 gross wage figure and the $1,560 is not an above-the-line deduction. Correct AGI is $114,403.37; its resulting $46,369 of taxable income yields $5,068.28 of tax, and subtracting the same $3,200 of credits reproduces exactly its $1,868.28." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no computation and asserted a large liability from 'high household income.' The correct derivation nets AGI to $114,403.37, removes $46,609.29 of deductions ($32,200 standard, $13,899.29 overtime, $510 QBI) for $67,794.08 of taxable income and $7,639.29 of tax, then subtracts $3,200 of nonrefundable credits; its $13,500 exceeds even the uncredited pre-deduction tax and reflects neither the overtime deduction nor any child or dependent credit." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Explicitly applied 'TCJA-sunset' rules — a $17,000 standard deduction, $27,000 of personal exemptions, 10%/15% brackets, and a $1,000 child credit phased down to $750 above $110,000 of AGI. 2026 instead provides a $32,200 MFJ standard deduction, no personal exemptions, 10%/12% brackets, a $13,899.29 overtime deduction, a $510 QBI deduction, and $2,200 plus two $500 dependent credits with a $400,000 phaseout threshold." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Added the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp amount that already contains it, computed QBI as $3,310 on that un-netted business income instead of 20% of the $2,550 left after the $16,800 farm loss, and used a $30,700 standard deduction rather than $32,200. Taxable income came to $80,513 against the correct $67,794.08, even though it did take the overtime deduction and the right $3,200 of credits." -us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted $21,208 of health insurance premiums from income, double-counted the $14,000 self-employment partnership share already inside the $19,350, took a $989 half-SE-tax deduction that the $16,800 farm loss precludes, and omitted the $13,899.29 overtime deduction. It also applied a $2,000 child credit with no other-dependent credits instead of $2,200 plus $1,000." -us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"Added $2,000 of additional tax for early withdrawal of the $20,000 401(k) distribution; the reference's $7,639.29 is exactly the bracket schedule applied to $67,794.08, so no early-distribution penalty belongs in this output. It also took the $1,796 domestic production deduction while dropping the $510 QBI deduction, and its $97,723 AGI understates the correct $114,403.37." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Double-counted the $14,000 self-employment partnership income inside the $19,350, set QBI at 20% of $33,350 = $6,670 rather than 20% of the $2,550 remaining after the $16,800 farm loss, omitted the $13,899.29 overtime deduction, and used a $30,250 standard deduction with $2,500 of credits instead of $32,200 and $3,200. It then discarded its own $7,815 result and submitted $7,646." -us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Taxed the full $97,295 of wages without excluding the $2,315.40 traditional 401(k) deferral and dropped Dependent 1's $18 deductible IRA contribution, then substituted the $1,796 domestic production deduction for the $510 QBI deduction. The net is $68,842 of taxable income versus $67,794.08 and $125.75 of extra tax; the $13,899.29 overtime deduction, $32,200 standard deduction, and $3,200 of credits were all handled correctly." -us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Submitted zero with no derivation at all. Taxable income of $67,794.08 produces $7,639.29 of tax, and the only nonrefundable credits available are $2,200 for the 12-year-old plus $500 each for the 27-year-old and 18-year-old dependents, leaving $4,439.29." -us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Double-counted the $14,000 self-employment partnership income within the $19,350, took a $1,169 half-SE-tax deduction that the $16,800 farm loss precludes, deducted $1,796 of domestic production, set QBI at $3,310 instead of $510, and omitted the $13,899.29 overtime deduction. Taxable income came to $89,947 against the correct $67,794.08 even though its $3,200 of credits matched." -us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its own reasoning derives $7,629.40 of tax and it then submitted $14,629.40, a $7,000 addition no step supports. It also declared the household above the credit phaseout when the $400,000 MFJ threshold leaves the full $2,200 child credit and two $500 other-dependent credits available, and it omitted the $13,899.29 overtime deduction." -us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Derived $4,564.81 of regular tax and then added a nonexistent $12,150 'fully available child tax credit' instead of subtracting the actual $3,200 of nonrefundable credits, yielding $16,471.63. It also itemized $15,396 of medical expenses that the $32,200 standard deduction supersedes and deducted $11,827 of retirement contributions against the $2,441.62 that the $2,315.40 deferral and $126.22 of IRA contributions actually provide." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own chain reached about $4,430: AGI $114,422, less the $13,899 overtime and $510 QBI deductions, gives about $67,713 taxable and about $7,630 of tax, minus $2,200 CTC and $1,000 ODC. It then added an unsupported ~$1,529 of 'adjustments' and reported $5,959. No extra tax belongs in this measure; $7,639.29 − $3,200 = $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp income, even though it is part of that amount, and subtracted the $1,796 domestic production deduction, which was repealed after 2017. That raised AGI to $126,626 (vs $114,403.37) and QBI to $3,310 (vs 20% × $2,550 = $510), so taxable income was $77,217 instead of $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $13,899 FLSA overtime premium to wages that already include it instead of taking the 2026 qualified overtime deduction. It also deducted the full $18,235 capital loss instead of the $3,000 limit and used the $14,600 single-filer standard deduction instead of the $32,200 MFJ amount. Its $18,285 tax on $98,486 is far above the ~$11,300 the 2026 MFJ brackets give, and it reached $9,631 through unexplained credits rather than the $3,200 CTC/ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction of $13,899. It also charged SE tax and deducted half of it ($989) on the $14,000 partnership income, although the $16,800 farm loss makes net SE earnings negative. It then treated only $1,072 of the CTC as nonrefundable, when the full $2,200 CTC and $1,000 ODC offset $7,639 of tax, and added an unexplained $1,000." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction and added $1,978 of self-employment tax into income tax. SE tax is excluded from this measure, and it is zero here because the $16,800 farm loss exceeds the $14,000 SE partnership income. It never subtracted the $3,200 CTC/ODC, so its $14,959 is higher than its own ~$11,178 total before credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction. It deducted a half-SE-tax amount (no SE tax exists because the farm loss makes net SE earnings negative) and the repealed $1,796 DPAD, and it inflated QBI to ~$3,900 (correct: $510). It counted only $2,500 of CTC/ODC instead of $3,200, then added an unexplained 'additional tax', so its $11,200 is higher than its own $8,700 tax before credits." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed AGI of $114,422 by deducting only the head's $108 traditional IRA contribution. The engine's IRA adjustment is $126.22 because it also deducts Dependent 1's $18 contribution, and it uses unrounded wages ($94,979.60) and overtime ($13,899.29). Those differences bring taxable income down to $67,794.08 and explain the whole $2.27 gap; the standard deduction, overtime and QBI deductions, and $3,200 of credits all match." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction of $13,899 and used a $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 and $24,800 figures. It also took a $1,560 SE health-insurance deduction and a $989 half-SE-tax deduction, neither of which applies because the head has employer coverage and the farm loss makes net SE earnings negative. It used a $2,000 CTC instead of $2,200, ending at $81,873 taxable vs $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction, deducted the repealed $1,796 DPAD and a half-SE-tax amount that does not exist here, and treated the CTC as mostly refundable. In fact the $2,200 CTC and $1,000 ODC fully offset tax against this liability. It then raised its ~$9,300 tax to $12,500 by adding Additional Medicare Tax and NIIT effects, which are outside this measure and do not apply at this income." +us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of self-employment partnership income on top of the $19,350 partnership/S-corp income, which already contains it. That pushed AGI to ~$128.5k and QBI to ~$3.3k instead of $114,403 and $510. It also left out the $13,899 qualified overtime deduction, so taxable income was ~$92.9k instead of $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income twice (business net $16,550 instead of $2,550), subtracted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200, and left out the $13,899 qualified overtime deduction. That gave $92,674 taxable vs $67,794.08. It also claimed the $2,000 CTC for the 18-year-old, who is past the CTC's under-17 age limit and gets only the $500 ODC, so its credits totaled $4,500 instead of $3,200." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied the pre-TCJA sunset rules: a $15,900 standard deduction, five $5,200 personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has no personal exemptions, a $32,200 MFJ standard deduction, a 12% bracket, a $2,200 CTC plus $500 ODC per other dependent, and the $13,899 overtime deduction. It also subtracted the $1,560 of health premiums from wages." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-TCJA sunset rules: five personal exemptions and a $1,000 CTC phasing out above $110,000. OBBBA extended TCJA permanently, giving a $32,200 MFJ standard deduction, no exemptions, 10%/12% brackets, a $2,200 CTC plus two $500 ODCs with a $400,000 phase-out, and a $13,899 overtime deduction. As a result it applied $150 of credits against $11,628.90 of tax instead of $3,200 against $7,639.29." +us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $13,899 qualified overtime deduction, deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200, and applied a $2,000 CTC instead of the 2026 $2,200. Taxable income came to $81,366 rather than $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $19,648 of employer-sponsored insurance premiums from wages (taxable wages $75,332), although the stated wages are reduced only by the $2,315 traditional 401(k) deferral. It then itemized medical expenses and claimed $25,250 of personal exemptions and a $1,000 CTC under pre-TCJA rules. Under 2026 law there are no exemptions, the $32,200 standard deduction beats itemizing, the $13,899 overtime deduction applies, and credits are $2,200 CTC plus $1,000 ODC; its $4,717.20 is close to the correct figure only because the errors partly cancel." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation beyond mentioning itemized deductions. The correct path is AGI $114,403.37, less the $32,200 standard deduction (larger than any itemizable amount) and the $13,899.29 overtime and $510 QBI deductions, giving $67,794.08 taxable, $7,639.29 of tax, and $4,439.29 after $3,200 of CTC/ODC. Its $11,090.45 is exactly the bracket tax on about $96,554 of taxable income with no credits, which matches skipping the overtime deduction and the CTC/ODC and using small itemized deductions instead of the standard deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset (~$26,000 of personal exemptions, a $16,200 standard deduction, a $1,000 CTC), subtracted ESI premiums and the repealed DPAD from income, and added a $2,000 10% early-distribution penalty. OBBBA keeps the $32,200 MFJ standard deduction with no exemptions and adds the $13,899 overtime deduction, and the credits are a $2,200 CTC plus $1,000 ODC. The §72(t) additional tax is a Schedule 2 'other tax' and is not part of income tax before refundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $94,756 AGI is $19,647 below the correct $114,403.37, which matches the $19,648 of employer-sponsored insurance premiums; those premiums do not reduce the stated wages. It then applied pre-TCJA personal exemptions and a $1,000 CTC. 2026 law has no exemptions and gives the $32,200 standard deduction, the $13,899 overtime and $510 QBI deductions, and $3,200 of CTC/ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions gives $67,794.08 taxable and $7,639.29 of tax; after $2,200 CTC and $1,000 ODC the result is $4,439.29. Its $11,940 is higher than the $9,307 pre-credit tax even with the overtime deduction dropped, so it left out both the overtime deduction and the $3,200 of credits and also overstated income." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. After the $3,200 of CTC/ODC it names, $6,215 implies ~$9,415 of tax before credits and ~$82,600 of taxable income, about $14,800 above the correct $67,794.08. That gap is the size of the $13,899 qualified overtime deduction for the FLSA overtime premium, which it left out." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. With the $3,200 of CTC/ODC it names, $4,324 implies $7,524 of tax before credits and ~$66,830 of taxable income, about $960 below the correct $67,794.08. It therefore took about $960 more in deductions than AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI allows." +us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. With $3,200 of CTC/ODC, $4,057 implies $7,257 of tax before credits and ~$64,610 of taxable income, about $3,190 below the correct $67,794.08. It therefore deducted about $3,190 beyond the only allowable items: the $32,200 standard, $13,899.29 overtime and $510 QBI deductions from AGI $114,403.37." +us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and credits wipe out all tax. The only deductions are the $32,200 standard, $13,899.29 overtime and $510 QBI deductions; medical costs fall below the 7.5% AGI floor and never beat the standard deduction. That leaves $67,794.08 taxable and $7,639.29 of tax, of which the $3,200 CTC/ODC offsets only part." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It assumed deductions and credits eliminate all liability. After the $32,200 standard, $13,899.29 overtime and $510 QBI deductions, taxable income is still $67,794.08 with $7,639.29 of tax. The nonrefundable $2,200 CTC and two $500 ODCs remove only $3,200, leaving $4,439.29." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed AGI of $114,422 by deducting only the head's $108 IRA contribution. The engine's IRA adjustment is $126.22 because it also deducts Dependent 1's $18 contribution, and it carries unrounded wages ($94,979.60) and overtime ($13,899.29). That gives $67,794.08 taxable instead of $67,813 and explains the whole $2.27 gap; everything else matches." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"After its $3,200 of CTC/ODC, $2,219 implies $5,419 of tax before credits and ~$49,300 of taxable income, about $18,500 below the correct $67,794.08. It took ~$18,500 of deductions beyond the $32,200 standard, $13,899.29 overtime and $510 QBI deductions. The capital loss is capped at $3,000, and the $19,648 of ESI premiums does not reduce the stated wages." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,485 is $45.71 above $4,439.29, which equals ~$381 of extra taxable income at 12%. It used the right structure ($32,200 standard, overtime and QBI deductions, $3,200 of credits) but rounded its estimate instead of computing taxable income exactly as $114,403.37 − $32,200 − $13,899.29 − $510 = $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $2,200 CTC for Child 1 and left out the $500 credit for other dependents for Dependent 1 (27, disabled) and Dependent 2 (18, over the CTC age limit). Its tax before credits (~$7,655) was within ~$16 of $7,639.29, so the $1,015.71 overstatement comes almost entirely from the missing $1,000 ODC." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored insurance premiums from wages and took the repealed $1,796 domestic production deduction, bringing AGI down to $92,978 instead of $114,403.37. The stated wages are reduced only by the $2,315 traditional 401(k) deferral, so taxable income is $67,794.08, not $46,369." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $5,570 equals $8,770 of tax on $77,217 of taxable income minus the $3,200 CTC/ODC, which is $9,423 more taxable income than the correct $67,794.08. That total results from adding the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income that already contains it (raising QBI to $3,310) and subtracting the repealed $1,796 DPAD." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It reached $67,813 of taxable income by deducting only the head's $108 IRA contribution and rounded wage and overtime amounts. The engine's IRA adjustment is $126.22, which includes Dependent 1's $18 contribution, and its overtime deduction is $13,899.29. That gives $67,794.08 taxable and a result $2.27 lower; its deductions and $3,200 of credits are otherwise identical." +us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the repealed $1,796 domestic production activities deduction (AGI $112,626 instead of $114,403.37). It also claimed $1,500 of other-dependent credit (three $500 credits), but only Dependent 1 and Dependent 2 qualify, for $1,000. Together these cut taxable income to $66,017 and added $500 of credit that does not exist." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round $13,500 with no computation. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions gives $67,794.08 taxable and $7,639.29 of tax, and $4,439.29 after $3,200 of CTC/ODC. $13,500 is the tax before credits on about $109,400 of taxable income, so it skipped the overtime deduction and the CTC/ODC and overstated income by tens of thousands." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA sunset: a $17,000 standard deduction, $27,000 of personal exemptions, a 15% bracket and a $1,000 CTC phasing out above $110,000. OBBBA made TCJA permanent, so 2026 MFJ has a $32,200 standard deduction, no exemptions, 10%/12% brackets, a $2,200 CTC plus two $500 ODCs (phase-out at $400,000), and the $13,899 overtime deduction." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income that already includes it, which also inflated QBI to $3,310 (correct: 20% × $2,550 = $510). It also used a $30,700 standard deduction and a $24,150 top of the 10% bracket instead of 2026's $32,200 and $24,800. Taxable income came to $80,513 instead of $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It added the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income (inflating QBI to $3,310), giving $79,013 taxable vs $67,794.08. It then added a $2,000 10% early-distribution additional tax, which is a Schedule 2 'other tax' and not part of income tax before refundable credits." +us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $21,208 of ESI and health premiums from wages, counted the $14,000 of SE partnership income twice, and deducted half of an SE tax that does not exist (the farm loss makes net SE earnings negative). It also treated the repealed $1,796 DPAD as QBI, used a $30,750 standard deduction, and left out the $13,899 overtime deduction. On credits it applied only a $2,000 CTC, missing the 2026 $2,200 amount and both $500 ODCs." +us,scenario_064,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It added a $2,000 10% early-withdrawal additional tax on the 401(k) distribution, which is a Schedule 2 'other tax' and not part of income tax before refundable credits. It also brought AGI down to $97,723 by subtracting the overtime deduction plus an extra $2,800 net farm loss, and it replaced the $510 QBI deduction with the repealed $1,796 DPAD. That left $63,727 taxable instead of $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income on top of the $19,350 partnership/S-corp income and ignored the farm loss when computing QBI ($6,670 vs $510). It deducted Dependent 1's $386 401(k) deferral although that dependent has no wages, took a half-SE-tax deduction, and left out the $13,899 qualified overtime deduction. It also denied the $500 ODC for the 18-year-old, used a $2,000 CTC, and reached $7,646 through unexplained adjustments." +us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It used the full $97,295 of wages without removing the $2,315 traditional 401(k) deferral and deducted the repealed $1,796 DPAD instead of the $510 QBI deduction. Taxable income came to $68,842 instead of $67,794.08, making tax $125.71 too high." +us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It answered $0 with no reasoning, but the liability is positive. AGI $114,403.37 less $32,200 standard, $13,899.29 overtime and $510 QBI deductions leaves $67,794.08 taxable and $7,639.29 of tax, and the $3,200 of CTC/ODC offsets only part of it. The $0 answer treats deductions or credits as wiping out tax they do not cover." +us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 of SE partnership income twice (net business $16,550 instead of $2,550) and deducted half of an SE tax that does not arise because the farm loss makes net SE earnings negative. It also took the repealed $1,796 DPAD, inflated QBI to $3,310, and left out the $13,899 qualified overtime deduction, giving $89,947 taxable instead of $67,794.08." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the entire $18,235 capital loss instead of the $3,000 annual limit and left out the $13,899 overtime deduction. It claimed no CTC/ODC on the false premise that income is above the thresholds, when the MFJ phase-out starts at $400,000. It then reported $14,629.40, exactly $7,000 more than its own $7,629.40 bracket computation." +us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It added a made-up $12,150 'child tax credit' to $4,564.81 of tax instead of subtracting the actual $2,200 CTC and $1,000 ODC. Its taxable income was also wrong: it subtracted $11,827 of retirement contributions, took both itemized medical expenses and a $30,000 standard deduction, and left out the $13,899 overtime deduction." us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It equated the refundable credit with the full $2,000 CTC for Child 1, skipping the ACTC computation entirely — the CTC first offsets federal income tax liability and only the unused remainder becomes refundable, capped at $1,700 per child and at 15% of earned income over $2,500. With $97,295 of wages plus $20,000 of taxable 401(k) distributions and $19,350 of partnership income, MFJ tax before refundable credits far exceeds $2,000, so the entire CTC is consumed nonrefundably and nothing spills into the refundable column. Its own AGI estimate of ~$120,610 also double-counted, adding partnership income while ignoring that the capital loss deduction is capped at $3,000." us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning reached the correct conclusion — 'with significant federal tax owed, CTC is nonrefundable' — and it then submitted the $1,700 statutory ACTC ceiling anyway, converting a per-child maximum into an entitlement. The ACTC equals only the CTC remaining after the credit is applied against federal income tax liability; here liability on ~$98,810 of income absorbs the full $2,000 for the one under-17 child, so the refundable amount is $0." us,scenario_064,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It counted Dependent 2 (age 18) as a CTC qualifying child, but the credit requires age under 17 at year end, so that dependent yields only the $500 nonrefundable other-dependent credit rather than a $1,700 refundable share. It then paid out the per-child ACTC ceiling without applying the rule that only the CTC unused against tax liability is refundable; MFJ tax before refundable credits on ~$98,810 of income exceeds $2,000, so even the single qualifying child's credit is fully nonrefundable and the correct total is $0. Checking the $400,000 phaseout threshold was irrelevant — the binding constraint is the liability offset, not the income phaseout." @@ -4820,59 +5291,71 @@ us,scenario_064,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_ou us,scenario_064,spouse_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_064,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model read the listed fact ""is disabled"" on Dependent 1 as satisfying SSI's categorical disability criterion and paid the full 2026 federal benefit rate of $994/month for twelve months. SSI eligibility in the engine keys on `is_ssi_disabled`, a distinct input from the general disability flag that appears in the household facts; it is unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` for Dependent 1 and every other member (head 48, spouse 47, dependent2 18, child1 12 are all under 65 and not blind or SSI-disabled), and SSI is $0. The model never reached the income/deeming stage it argued about because the categorical gate closes first." -us,scenario_064,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model treated the household fact ""is disabled"" on the 27-year-old as establishing SSI disability and paid an uncounted-income FBR of about $991/month for twelve months. The engine's SSI categorical test reads the separate `is_ssi_disabled` input, which is unlisted and therefore false under the prompt's unlisted-equals-false rule, so `is_ssi_aged_blind_disabled=False` for all five members and SSI is $0. Its correct point that parental deeming stops at age 18 is irrelevant once the aged/blind/disabled gate fails." -us,scenario_064,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model granted Dependent 1 SSI on the strength of the listed ""is disabled"" fact, then reduced the $991 FBR by the one-third value-of-the-one-third rule for living in another's household. SSI eligibility in the engine depends on `is_ssi_disabled`, a separate input from the general disability flag, and it is unlisted and therefore false, making `is_ssi_aged_blind_disabled=False` for every member and SSI $0. The one-third reduction compounds the error by inferring unlisted in-kind food and shelter support the prompt forbids inferring." -us,scenario_064,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model paid Dependent 1 a $990 FBR cut by one third for in-kind support, having taken the listed ""is disabled"" fact as SSI's disability determination. The engine's SSI test uses the distinct `is_ssi_disabled` input, unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` across the household and SSI is $0. It also inferred unlisted household support to trigger the value-of-the-one-third reduction, which the prompt's do-not-infer rule excludes." -us,scenario_064,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model awarded Dependent 1 the projected 2026 maximum federal benefit ($990/month x 12 = $11,880) on the basis of the listed ""is disabled"" fact plus zero personal income and assets. SSI's categorical gate in the engine is `is_ssi_disabled`, a separate input from the general disability flag; unlisted, it is false, so `is_ssi_aged_blind_disabled=False` for all five members and SSI is $0. Head (48) and spouse (47) are also below the aged threshold of 65, leaving no qualifying member." -us,scenario_064,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model paid Dependent 1 a full federal benefit plus a Wisconsin state supplement, treating the listed ""is disabled"" fact as the SSI disability determination. SSI eligibility in the engine turns on the separate `is_ssi_disabled` input, which is unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` for every member and the SSI output is $0; a state supplement is payable only on top of an eligible federal SSI determination, which never arises here." -us,scenario_064,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model correctly ruled out the other four members on age and correctly chained the COLA to a 2026 FBR near $991/month, but it accepted the listed ""is disabled"" fact as satisfying SSI's disability criterion for the 27-year-old. The engine's categorical test reads `is_ssi_disabled`, a distinct input from the general disability flag, and it is unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` for Dependent 1 as well and SSI is $0." -us,scenario_064,ssi,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for `ssi`, so no substantive reasoning exists to evaluate. The required submission was $0, since `is_ssi_aged_blind_disabled=False` for all five members and the SSI categorical gate closes before any income test." -us,scenario_064,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model paid the 2026 individual federal benefit rate of $994/month for twelve months to the disabled adult dependent, arguing only that no personal countable income or resources were listed. It skipped the categorical gate: the engine's SSI disability determination is the separate `is_ssi_disabled` input, unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` for Dependent 1 and all other members and SSI is $0." -us,scenario_064,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model estimated $994/month x 12 for the 27-year-old on the strength of the listed ""is disabled"" fact and the absence of personal income or assets. SSI eligibility in the engine requires `is_ssi_disabled`, a distinct input from the general disability flag; unlisted, it is false, leaving `is_ssi_aged_blind_disabled=False` for all five members, so the SSI output is $0 on categorical grounds rather than on income." -us,scenario_064,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model reasoned about deeming and personal resources and paid $994/month x 12, taking the listed ""is disabled"" fact as the SSI disability finding for Dependent 1. The engine's SSI categorical test uses the separate `is_ssi_disabled` input, unlisted and therefore false under the prompt's unlisted-equals-false rule, so `is_ssi_aged_blind_disabled=False` household-wide and SSI is $0; the deeming analysis never becomes operative." -us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"The model paid Dependent 1 a one-third-reduced federal benefit plus an assumed Wisconsin supplement, treating the listed ""is disabled"" fact as the SSI disability determination. The engine's gate is the separate `is_ssi_disabled` input, unlisted and therefore false, so `is_ssi_aged_blind_disabled=False` for all five members and SSI is $0; the model additionally inferred unlisted in-kind food and shelter support to apply the value-of-the-one-third reduction and an unlisted state supplement, both barred by the do-not-infer instruction." -us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no `ssi` value and no explanation, so there is no substantive computation to fault. The required answer was $0, driven by `is_ssi_aged_blind_disabled=False` for head (48), spouse (47), dependent1 (27), dependent2 (18) and child1 (12)." -us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model correctly excluded head and spouse as neither 65+ nor blind/disabled but then declared Dependent 1 categorically SSI-eligible from the listed ""is disabled"" fact and paid the full 2026 individual FBR of $994/month x 12. SSI eligibility in the engine keys on `is_ssi_disabled`, a separate input from the general disability flag; unlisted, it is false, so `is_ssi_aged_blind_disabled=False` for Dependent 1 too and SSI is $0." -us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It computed $4,964 of bracket tax using the superseded Wisconsin schedule with 5.30% starting at $38,190 instead of the widened 4.40% bracket that runs to roughly $69,400 for 2026 married-joint filers, then cut $2,265 off that for an itemized-deduction credit and an unexplained 'capital loss/farm/standard deduction precision' adjustment. Wisconsin's itemized deduction credit is zero here: the $8,340 of deductible medical expenses falls below the 7.5%-of-AGI floor of $8,580 and no mortgage interest amount is listed, so the entire $1,907 shortfall below $4,606 comes from fabricated credits." -us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It built a Wisconsin base of $129,126 by adding the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp figure that already contains it and by adding back $2,500 of capital loss under a $500 state cap the engine does not apply, against the correct AGI of $114,403.37. It then used a $7,900 standard deduction instead of $8,759.12, producing taxable income of $117,700 versus the true $102,144.25 and a $794 overstatement." -us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the entire $18,235 long-term capital loss rather than the $3,000 federal limit, understating income by $15,235, and described Wisconsin's rate schedule as '5.3%–7.65%' when the 2026 married-joint schedule is 3.50% to about $20,300, 4.40% to about $69,400, then 5.30%. Its $4,512 is untethered from its own work: the $104,434 base it cites is not Wisconsin's $102,144.25 taxable income and no arithmetic connects that base to the submitted number." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It counted four $700 exemptions ($2,800) instead of the five people in the household ($3,500), used the superseded brackets with 5.30% beginning at $36,840, and then subtracted roughly $875 for a school property tax credit and itemized deduction credit. The school property tax credit requires rent or property tax, and this household lists neither, so both credits are zero and the $4,150 lands $456 under the $4,606 the correct schedule gives on $102,144.25." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its worked schedule produced $4,784 on a $100,000 base and it then said it would reduce that further for exemptions, yet it submitted $5,074 — a number its own arithmetic contradicts in the opposite direction. The schedule itself used a $38,190 second-bracket ceiling rather than the roughly $69,400 that Wisconsin's widened 4.40% bracket reaches in 2026, which by itself overstates tax on this return by about $292." -us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It never computed the sliding-scale standard deduction, which falls to $8,759.12 at an AGI of $114,403.37, or the exact bracket boundaries, reporting a rounded '$4,600' off an approximate '~114,000' income figure. Wisconsin taxable income is $102,144.25 and the 2026 married-joint schedule yields $4,606.01, so the $6 miss is a rounding artifact of an uncomputed estimate." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Its standard-deduction phase-out used a $22,010 base fully phased out at $134,000, producing $3,957 where Wisconsin's 2026 sliding scale leaves $8,759.12 at this AGI, and it added back $2,500 of capital loss the engine does not add. It then applied the superseded brackets with 5.30% from $36,840 and subtracted $520 of 'homestead/other small credits' — the homestead credit is refundable and requires rent or property tax, neither of which is listed — leaving it $131 over." -us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It stacked the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income that already includes it and subtracted the $1,796 domestic production deduction from the state base, reaching $126,626 against the correct $114,403.37. It then used a ~$14,000 standard deduction and four exemptions ($2,800) rather than $8,759.12 and $3,500, and submitted $4,200 — a figure the $109,826 base it reported would not produce under the rates it cited." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It invented a two-rate Wisconsin schedule — 4% on the first ~$23,000 and 5.3% above — where the 2026 married-joint schedule is 3.50% to about $20,300, 4.40% to about $69,400, and 5.30% above. Taxing roughly $47,000 of the 4.40% range at 5.30%, on a rounded $100,000 base instead of $102,144.25, put it $397 over $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied a flat $25,000 standard deduction where Wisconsin's sliding scale leaves only $8,759.12 at an AGI of $114,403.37, and also stripped the $1,796 domestic production deduction from the state base. Taxable income of $82,566 rather than $102,144.25, combined with a $39,000 second-bracket ceiling instead of about $69,400, left it $757 short." -us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reported Wisconsin taxable income of $119,774.69, $17,630 above the correct $102,144.25, which requires both double-counting the $14,000 self-employment partnership income already inside the $19,350 partnership/S-corp figure and denying the capital-loss and retirement-contribution subtractions the engine takes to reach AGI of $114,403.37. That inflated base at the 5.30% margin accounts for essentially the whole $1,224 excess." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its Wisconsin AGI of $114,582 is within $179 of the reference, but it used a $13,506 sliding-scale standard deduction where 2026 leaves $8,759.12 at that AGI — a $4,747 excess — and then subtracted $150 for 'non-refundable dependent credits' that Wisconsin does not have, since dependents enter only through the $700 exemptions it had already counted. Those two errors put it $101 under $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It supplied no derivation beyond asserting '2026 tax brackets and standard deductions,' and its $4,652.12 corresponds to a taxable income of $103,014 under the correct 3.50%/4.40%/5.30% schedule. Wisconsin's taxable income here is $102,144.25 — AGI of $114,403.37 less the $8,759.12 sliding-scale deduction and $3,500 of exemptions — which the model never computed." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $92,978, $21,425 below the correct $114,403.37, consistent with deducting the full $18,235 long-term capital loss instead of the $3,000 federal limit plus further unwarranted reductions to wages. Taxable income of $78,969 against the true $102,144.25 leaves it $934 short at the 5.30% margin." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reported a net taxable income of $82,256, $19,888 below Wisconsin's $102,144.25, which requires deductions far beyond the $8,759.12 sliding-scale standard deduction and $3,500 of exemptions available at this AGI. Its own base under the 2026 schedule produces about $3,553, so the submitted $4,356 does not follow from the figure it stated." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The bare $6,950 corresponds to taxing the full $114,403.37 AGI at roughly 6.1% — no sliding-scale standard deduction, no personal exemptions, and a rate above Wisconsin's 5.30% top applicable bracket rate. The correct base of $102,144.25 under the 3.50%/4.40%/5.30% schedule yields $4,606, so the answer overstates by $2,344." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $4,125 implies a taxable income of $93,069, $9,075 below Wisconsin's $102,144.25 — the gap of a standard deduction near $17,800 instead of the $8,759.12 the sliding scale leaves at an AGI of $114,403.37. No bracket arithmetic was performed at all." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $3,212 implies a taxable income of $75,843, consistent with applying a federal-size married-joint standard deduction of roughly $31,000 rather than Wisconsin's sliding-scale $8,759.12. Wisconsin's own deduction plus $3,500 of exemptions leaves $102,144.25 taxable and $4,606 of tax." -us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $2,490 implies a taxable income of $62,220, about $40,000 below Wisconsin's $102,144.25, consistent with a federal-size standard deduction stacked on top of deducting the full $18,235 capital loss instead of the $3,000 limit. Wisconsin allows only the $8,759.12 sliding-scale deduction and five $700 exemptions against an AGI of $114,403.37." -us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate. The required answer is the $4,606 that Wisconsin's 2026 married-joint schedule produces on taxable income of $102,144.25." -us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate. The required answer is the $4,606 that Wisconsin's 2026 married-joint schedule produces on taxable income of $102,144.25." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and retirement adjustments reduce the Wisconsin liability to zero, but the sliding-scale standard deduction at an AGI of $114,403.37 is only $8,759.12 and the five personal exemptions total $3,500, leaving $102,144.25 taxable. Wisconsin offers no deduction or nonrefundable credit that erases tax at this income — the married-couple credit is zero with a non-earning spouse — so the liability is $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed rather than computed that deductions and credits outweigh the state tax, submitting zero. Wisconsin's sliding-scale standard deduction shrinks to $8,759.12 at an AGI of $114,403.37 and the exemptions total $3,500, leaving $102,144.25 taxable and $4,606 of tax with no applicable nonrefundable credit." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $15,166 standard deduction against the $8,759.12 Wisconsin's sliding scale actually leaves at this AGI and counted only two $700 exemptions instead of the five people in the household ($3,500). It then subtracted a $472 itemized-deduction/medical credit that is zero here — deductible medical expenses of $8,340 fall below the 7.5%-of-AGI floor of $8,580 and no mortgage interest amount is given — leaving it $1,022 short." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The unexplained $2,800 implies a taxable income of $68,070, roughly $34,000 below Wisconsin's $102,144.25, which no combination of the $8,759.12 sliding-scale deduction and $3,500 of exemptions produces from an AGI of $114,403.37. No Wisconsin nonrefundable credit is available to this household to close that gap." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,872 is what Wisconsin's superseded rate schedule yields on essentially the correct $102,144.25 base, with 5.30% beginning near $38,000 of married-joint taxable income. Wisconsin's 2025 budget act widened the 4.40% bracket to roughly $69,400 for 2026, moving about $30,000 of this return from 5.30% to 4.40% and cutting $266 off the tax." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The unexplained $4,071 implies a taxable income of $92,050, $10,094 below Wisconsin's $102,144.25 — the signature of a standard deduction near $18,850 instead of the $8,759.12 the sliding scale leaves at an AGI of $114,403.37. It also described the $700 personal amounts as 'exemption credits' when Wisconsin subtracts them from taxable income, which the $3,500 in the reference already does." -us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied a $2,340 Wisconsin health-insurance subtraction that does not arise for employer-sponsored coverage and a $2,500 capital-loss addback the engine does not take, but the decisive error is its 'sliding standard deduction': its $3,254.36 implies a taxable income of $76,642 against the correct $102,144.25, meaning it deducted roughly $34,000 where Wisconsin allows $8,759.12 plus $3,500 of exemptions." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The one-line '$4,500' implies a taxable income of $100,144, $2,000 below Wisconsin's $102,144.25, with no computation of the $8,759.12 sliding-scale standard deduction or the five $700 exemptions against AGI of $114,403.37. The correct base under the 2026 married-joint schedule gives $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It reached the right starting point ($114,422 against $114,403.37) but used a $6,500 standard deduction instead of $8,759.12 and never subtracted the $3,500 of personal exemptions, leaving taxable income of $107,900 rather than $102,144.25. That $5,756 excess at the 5.30% margin, plus a second-bracket ceiling short of the roughly $69,400 Wisconsin's widened 4.40% bracket reaches, put it $294 over." -us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It stated the 2026 married-joint bracket boundaries as $29,350 and $58,700 when the schedule runs 3.50% to about $20,300 and 4.40% to about $69,400 — its bracket ceilings sum to $88,050 against the actual $89,738 — and it overstated the sliding-scale standard deduction by $317 ($9,076 versus $8,759.12). The two errors nearly cancel, leaving $4,610 against the correct $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 2024 brackets at 4.75% and 5.75%, rates that do not appear in Wisconsin's schedule of 3.50%, 4.40%, 5.30%, and 7.65%, to an AGI of $106,225 that is $8,178 below the correct $114,403.37, with a $13,500 standard deduction against the actual $8,759.12 and no personal exemptions. The inflated rates on a shrunken base landed $154 over." -us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It removed the $13,899 FLSA overtime premium from federal AGI to reach $97,723, but the federal overtime deduction is taken below the line and does not reduce AGI, so Wisconsin's starting point is the full $114,403.37. Even after adding the premium back it deducted roughly $21,600 where Wisconsin allows $8,759.12 plus $3,500, giving a $90,000 base against the true $102,144.25 and leaving it $806 short." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate. The required answer is the $4,606 that Wisconsin's 2026 married-joint schedule produces on taxable income of $102,144.25." -us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It taxed the top slice of income at 6.27%, Wisconsin's third-bracket rate before 2023 Act 19 cut it to 5.30%, and used a $5,481 standard deduction against the $8,759.12 the sliding scale leaves at an AGI of $114,403.37. The repealed rate applied to roughly $68,000 of income accounts for most of the $1,485 excess over $4,606." -us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It submitted zero with no derivation, which would require deductions equal to the entire $114,403.37 AGI; Wisconsin allows only the $8,759.12 sliding-scale standard deduction and $3,500 of exemptions, leaving $102,144.25 taxable and $4,606 of tax. No Wisconsin nonrefundable credit reaches this household — the married-couple credit is zero with a non-earning spouse, and no rent or property tax is listed for the school property tax credit." -us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used the federal married-joint standard deduction of about $31,300 in place of Wisconsin's sliding-scale $8,759.12, omitted the $3,500 of personal exemptions entirely, and inflated AGI to $127,253 by adding the $14,000 self-employment partnership income already contained in the $19,350 partnership/S-corp figure. It also invented bracket ceilings of $29,430 and $58,860 against the actual ~$20,300 and ~$69,400, and the offsetting errors left it $315 under." -us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded $13,899 of FLSA overtime premium from wages, cutting them to $83,396, but overtime pay is fully included in the wages that flow into Wisconsin's base — the federal overtime deduction is below the line and Wisconsin does not adopt it. It also used a 4.65% middle rate that Wisconsin replaced with 4.40% and a ~$13,500 standard deduction against $8,759.12, and its own arithmetic produced $3,767 while it submitted $4,221.67." -us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It subtracted $11,827 of 'retirement contributions' when the household's traditional 401(k) and IRA amounts total $2,827 ($2,315 + $108 + $386 + $18), started from an AGI of $101,610 instead of $114,403.37, and further deducted medical expenses that do not reduce Wisconsin taxable income. The resulting base is far below the $102,144.25 that yields $4,606 under the 2026 married-joint schedule." +us,scenario_064,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated Dependent 1's generic 'is disabled' flag as meeting SSI's disability requirement and paid the full 2026 federal benefit rate of $994 × 12 = $11,928. Under the engine's inputs, Dependent 1 does not meet SSI's aged/blind/disabled requirement (is_ssi_aged_blind_disabled=False), and no other member is 65+, blind or disabled, so SSI is $0. The model's conclusion that parental deeming does not apply is irrelevant because the disability test fails first." +us,scenario_064,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model counted Dependent 1 as SSI-disabled because of the 'is disabled' flag and paid the full federal benefit of $994 × 12 = $11,928. The engine's SSI aged/blind/disabled test is False for Dependent 1 and for every other member. SSI is therefore $0 regardless of deeming or Dependent 1's lack of income and resources." +us,scenario_064,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 (age 27, flagged disabled) as SSI-disabled and paid an estimated $991 × 12 = $11,892. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), and no other member qualifies, so the correct amount is $0. The model's income, resource and deeming analysis never becomes relevant." +us,scenario_064,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model accepted the generic disability flag as SSI disability for Dependent 1 and computed $991 × 2/3 × 12 = $7,928. Dependent 1 is not SSI aged/blind/disabled under the engine's inputs, so no member is eligible and SSI is $0. The model also applied a one-third in-kind-support reduction that no listed fact supports." +us,scenario_064,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled and paid a federal benefit rate of about $990, reduced by one-third, for $660 × 12 = $7,920. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), so SSI is $0. The one-third living-arrangement reduction is a second error on top of the eligibility mistake." +us,scenario_064,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled and subtracted an in-kind support (PMV) reduction of $350.33 (one-third of the $991 federal benefit rate plus $20), giving $640.67 × 12 = $7,688.04. Dependent 1 does not meet SSI's aged/blind/disabled requirement under the engine's inputs, and no other member does either, so SSI is $0." +us,scenario_064,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model concluded that Dependent 1's 'qualifying disability' made them eligible for the maximum federal SSI benefit and paid $990 × 12 = $11,880. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 and for all other members, so there is no eligible person and SSI is $0." +us,scenario_064,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated Dependent 1's disability flag as SSI eligibility and added a Wisconsin state supplement to reach $11,836. Dependent 1 does not meet SSI's aged/blind/disabled requirement under the engine's inputs, so neither the federal benefit nor any state supplement is paid and SSI is $0." +us,scenario_064,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as an 'adult SSI applicant' because of the disability flag, projected the federal benefit rate to $991 and paid $11,892. The engine's SSI aged/blind/disabled test is False for Dependent 1, just as the model found for the other four members, so SSI is $0. The model's conclusions on deeming and in-kind support are irrelevant." +us,scenario_064,ssi,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no SSI value and no explanation, so there is no answer to score. The correct derivation is $0: no member meets SSI's aged/blind/disabled requirement under the engine's inputs." +us,scenario_064,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated the disabled adult dependent as SSI-eligible because they had no income or resources of their own and paid the 2026 individual rate of $994 × 12 = $11,928. It skipped the disability test itself: Dependent 1 is not SSI aged/blind/disabled under the engine's inputs, and neither is any other member, so SSI is $0." +us,scenario_064,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model read the 'is disabled' flag as meeting SSI's disability requirement for the 27-year-old and paid the $994 monthly maximum, or $11,928. The engine's SSI aged/blind/disabled test is False for Dependent 1 and for every other member, so SSI is $0." +us,scenario_064,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model treated the 27-year-old as SSI-disabled, ruled out parental deeming and paid $994 × 12 = $11,928. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), so the deeming analysis is irrelevant and SSI is $0." +us,scenario_064,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model took Dependent 1's disability flag as SSI eligibility and paid the $994 monthly federal maximum, or $11,928. Under the engine's inputs, Dependent 1 does not meet SSI's aged/blind/disabled requirement and no other member does, so SSI is $0." +us,scenario_064,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model concluded that the disabled 27-year-old 'qualifies' for the 2026 individual rate of $994 × 12 = $11,928, based only on their lack of income and assets. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 and for every other member, so no one passes the SSI eligibility test and SSI is $0." +us,scenario_064,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model treated the disabled adult dependent as SSI-eligible, ruled out deeming of parental resources and paid the $994 monthly maximum, or $11,928. Dependent 1 does not meet SSI's aged/blind/disabled requirement under the engine's inputs, so the resource and deeming analysis never applies and SSI is $0." +us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled, applied the one-third reduction for living in another person's household ($644.67 × 12 ≈ $7,736) and added an estimated Wisconsin supplement to reach $7,952. Dependent 1 fails the engine's SSI aged/blind/disabled test (is_ssi_aged_blind_disabled=False), so no federal benefit or state supplement is paid and SSI is $0." +us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SSI value and no explanation, so there is no answer to score. The correct derivation is $0: no member meets SSI's aged/blind/disabled requirement under the engine's inputs." +us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model correctly excluded Head and Spouse as neither 65+ nor disabled, but wrongly declared Dependent 1 'categorically SSI-eligible' because of the disability flag and paid $994 × 12 = $11,928. The engine's SSI aged/blind/disabled test is also False for Dependent 1, so no member is eligible and SSI is $0." +us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It first overstated WI AGI at $118,248, adding a $2,500 capital-loss add-back and wrongly deducting about $989 of half-SE tax even though the -$16,800 farm loss wipes out net SE earnings. That led to about $4,964 of gross tax. It then cut this to $2,699 with invented nonrefundable credits (an itemized-deduction credit plus unexplained 'precision' adjustments), although no nonrefundable credit applies. The engine's result is $4,606, on $102,144 of taxable income." +us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 self-employment partnership income a second time on top of the $19,350 partnership/S-corp income, which already includes it, and also subtracted the $1,796 domestic production deduction. That gave a federal AGI of $126,626. It then added a $2,500 capital-loss add-back, so WI taxable income came out near $117,700 instead of the engine's $102,144." +us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from a guessed federal taxable income instead of WI AGI minus the $8,759 phased standard deduction and $3,500 of exemptions. It also described Wisconsin's rates as 5.3%–7.65%, when this income is actually taxed at 3.5%/4.4%/5.3%. As a result, $4,512 does not come from applying the 2026 schedule to $102,144." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It claimed only 4 exemptions ($2,800) instead of 5 ($3,500), and it started the 5.3% rate at $36,840 instead of at the top of the widened 4.4% bracket (about $50,500). It then subtracted about $875 for a school property tax credit and an itemized deduction credit. The household reports no property tax or rent, and its itemized deductions are below the standard deduction, so neither credit exists." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It computed $4,784 on about $100,000 of taxable income using a 4.4% bracket that ends at $38,190. It then raised the figure to $5,074 while saying it was 'reducing' tax for exemptions, which contradicts its own arithmetic. The engine applies 4.4% up to about $50,500 on $102,144 of taxable income, giving $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round-number estimate of $4,600 instead of running the brackets on the exact taxable income. The correct figure is $114,403 WI AGI minus the $8,759 phased standard deduction and $3,500 of exemptions, or $102,144, which produces $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a $2,500 capital-loss add-back, bringing WI AGI to $116,922, which is $2,519 above the engine's $114,403. It also used a standard deduction of about $8,000 instead of $8,759. Together these raised taxable income to $105,422 instead of $102,144, which pushed the tax up to $4,802." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It modeled the sliding-scale standard deduction with an invented $22,010 base and a phase-out ending at $134,000, which left only $3,957 instead of the engine's $8,759. It also added $2,500 to AGI for capital losses and started the 5.3% rate at $36,840, producing $5,257 of gross tax. It then subtracted an invented $520 of nonrefundable credits, although none apply." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income a second time on top of the $19,350 partnership total and subtracted the $1,796 domestic production deduction, reaching $126,626 of AGI. It then used only 4 exemptions and an overstated $14,000 standard deduction. Its stated $109,826 of taxable income would produce more than $5,000 of tax, so its final $4,200 does not follow from its own inputs." +us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It put WI taxable income at about $125,000, far above the engine's $102,144, and claimed only three $700 dependent exemptions, leaving out the two filer exemptions. Applying the rates to that inflated base produced $6,200 instead of $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used an invented schedule of 4% on the first $23,000 and 5.3% on everything above that. The actual rates are 3.5%, then 4.4% up to about $50,500, then 5.3%. It also left out the $3,500 of personal exemptions, which pushed the tax to $5,003." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $25,000 MFJ standard deduction without applying Wisconsin's sliding-scale phase-out, which cuts it to $8,759 at this income. It also subtracted the $1,796 domestic production deduction from WI AGI. Together these understated taxable income at $82,566 instead of $102,144." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its WI taxable income of $119,775 is about $17,600 above the engine's $102,144. That means it included income the engine does not count, such as the SE partnership income counted a second time, or it failed to net the farm loss. Taxing that inflated base gave $5,830." +us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It treated the 5 × $700 exemptions as a $3,500 credit subtracted from tax instead of a $3,500 deduction from income, which removed about $3,300 too much. It also used the outdated 3.54%/4.65% rates and an overstated $13,197 standard deduction." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It used a sliding-scale standard deduction of $13,506 instead of the engine's $8,759, which cut taxable income to $97,576 instead of $102,144. It then subtracted a $150 'dependent credit' that Wisconsin does not provide as a nonrefundable credit here." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,652 does not match the 2026 MFJ schedule applied to $102,144 of taxable income ($114,403 AGI − $8,759 standard deduction − $3,500 exemptions). Its number is consistent with slightly overstated taxable income or a narrower 4.4% bracket." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its starting AGI of $92,978 leaves out about $21,400 of income that the engine counts; the engine's WI AGI is $114,403. The largest omission is the $20,000 taxable 401(k) distribution. As a result, taxable income fell to $78,969 instead of $102,144." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It stated taxable income of $82,256, which means it took a standard deduction far larger than the phased $8,759. Its $4,356 of tax is also higher than what the 2026 schedule yields on $82,256, so both its deduction and its bracket math are wrong." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $6,950 overshoots the correct $4,606 by $2,344. That amount corresponds to taxing about $44,000 more than the correct $102,144 of taxable income, or to ignoring the phased standard deduction and exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and $4,125 is $481 below the tax the 2026 MFJ schedule yields on $102,144. That gap corresponds to about $9,000 of overstated deductions or phantom nonrefundable credits." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied unspecified nonrefundable credits, but none apply to this household: the spouse has no earnings, and itemized deductions are below the $8,759 standard deduction. Its $3,212 is $1,394 below the $4,606 that the schedule yields on $102,144 of taxable income." +us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted nonrefundable credits that do not apply to this household and/or overstated the standard deduction. The engine's standard deduction is phased to $8,759, taxable income is $102,144, and the tax is $4,606, not $2,490." +us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference." +us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that deductions and retirement adjustments eliminate Wisconsin tax. In fact, WI AGI is $114,403, the phased standard deduction is only $8,759, and exemptions are $3,500, which leaves $102,144 taxable and $4,606 of tax." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It assumed that deductions and credits wipe out the tax, but the household has $102,144 of Wisconsin taxable income after the $8,759 phased standard deduction and $3,500 of exemptions. No nonrefundable credit applies, so the tax is $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It overstated the phased MFJ standard deduction at $15,166 instead of $8,759 and claimed only two exemptions instead of five. It then subtracted a $472 itemized-deduction credit. That credit is zero because allowable itemized deductions (medical costs above 7.5% of AGI, with no mortgage interest input) are below the standard deduction." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $2,800 is $1,806 below the tax on $102,144 of taxable income, which means it applied phantom nonrefundable credits or a standard deduction far larger than the phased $8,759." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,872 is $266 above the correct tax on $102,144, which fits starting the 5.3% rate near $39,000–$40,000 instead of at the top of the widened 2026 4.4% bracket (about $50,500), combined with slightly overstated taxable income." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It applied 'exemption credits' against tax. Wisconsin's $700 × 5 exemptions are a $3,500 reduction of income, not a credit, so treating them as tax credits cut the result to $4,071 instead of $4,606." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It adjusted AGI with a $2,500 capital-loss add-back and a $2,340 health-insurance subtraction, then applied a sliding standard deduction and brackets that produced only $3,254. That is roughly $1,350 below the tax the 2026 schedule yields on the correct $102,144 of taxable income, so its deduction or bracket figures were far off the engine's $8,759 deduction and ~$50,500 top of the 4.4% bracket." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $5,654 is about $1,050 above the tax on $102,144 of taxable income, which fits leaving out most of the phased standard deduction and the exemptions, or taxing at 5.3% well below the ~$50,500 top of the 4.4% bracket." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its starting AGI (about $114,422) was close, but its final $4,200 implies deductions about $7,700 larger than the engine's $8,759 phased standard deduction plus $3,500 of exemptions. It also referred only to dependent exemptions, not to all five." +us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a health-premium deduction of about $4,100, bringing income to $110,286, whereas the engine's WI AGI is $114,403. It also used a $10,208 sliding standard deduction instead of $8,759. Both steps understated taxable income, which led to $4,472." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded $4,500 without computing the tax. The correct taxable income is $114,403 − $8,759 − $3,500 = $102,144, and the 2026 MFJ schedule gives $4,606 on that amount." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It understated the phased standard deduction at about $6,500 instead of $8,759 and never subtracted the $3,500 of personal exemptions. That left taxable income of about $107,900 instead of $102,144, which inflated the tax to $4,900." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $114,523 and a $9,076 standard deduction instead of $114,403 and $8,759. It also used invented brackets: 3.5% up to $29,350 and 4.4% up to $58,700, instead of 3.5% up to roughly $20,000 and 4.4% up to about $50,500. These errors partly cancel and leave the answer $4 off, at $4,610." +us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $102,274 is within $130 of the engine's $102,144, yet it computed $4,876, which is $270 too high. So it taxed far more income at 5.3% than the 2026 schedule does, where the 4.4% bracket for joint filers extends to about $50,500." +us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied invented 4.75%/5.75% rates instead of the actual 4.4%/5.3%. It also started from an understated AGI of $106,225, used an overstated $13,500 standard deduction, and left out the $3,500 of personal exemptions." +us,scenario_064,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It treated the federal overtime deduction as having reduced federal AGI to $97,723 and then added it back. It still ended up with about $90,000 of taxable income, well below the engine's $102,144, because it overstated the deductions taken from the engine's $114,403 WI AGI." +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable value for state_income_tax_before_refundable_credits, so nothing could be scored against the $4,606 reference." +us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 6.27% rate to its third bracket, but Wisconsin's third-bracket rate is 5.3%. It also understated the phased standard deduction at $5,481 instead of $8,759, which inflated the tax to $6,091." +us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned $0 with no reasoning, even though the household has $102,144 of Wisconsin taxable income after the $8,759 phased standard deduction and $3,500 of exemptions, which produces $4,606 of tax." +us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It took a $31,300 MFJ standard deduction without applying Wisconsin's sliding-scale phase-out, which reduces it to $8,759 at this income, and it left out the exemptions. It also overstated AGI at $127,253 and used invented bracket thresholds of $29,430 and $58,860." +us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the overtime premium out of wages, reducing them to $83,396, even though overtime stays in Wisconsin AGI. It also used an unphased standard deduction of about $13,500, and its final $4,221.67 does not follow from its own $3,767 bracket calculation." +us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $11,827 of retirement contributions a second time and also deducted medical expenses directly from income. The 401(k) deferral is already excluded from wages, and Wisconsin has no such medical deduction when the standard deduction is used. Both errors pushed taxable income far below $102,144." us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model enumerated every Wisconsin refundable credit — Homestead (income too high), the WI EITC (federal EITC is $0, and the WI credit is a straight percentage of it), Farmland Preservation, and the Veterans and Surviving Spouses Property Tax Credit — and stated that each one does not apply, then submitted $154 rather than the $0 its own enumeration produces. No credit it named yields $154: the Homestead Credit is additionally barred because no rent or property tax is listed, and the Married Couple and Itemized Deduction credits it correctly identified as nonrefundable are excluded from this output entirely." us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the state_refundable_credits key from its outputs object, so it submitted no value for a variable the prompt required exactly once. The correct derivation sums Wisconsin's refundable credits — WI EITC at $0 because the federal EITC is $0 on $97,295 of wages, Homestead at $0 under its household-income ceiling and with no rent or property tax listed, Farmland Preservation at $0 absent a preservation agreement — for a total of $0." us,scenario_064,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no state_refundable_credits entry, leaving the required key unanswered rather than making a substantive Wisconsin credit error. Working the rule set through yields $0: the WI EITC is a percentage of a federal EITC that is $0 at this income, the Homestead Credit fails both the income ceiling and the requirement that rent or property tax be paid, and the Farmland Preservation Credit requires an unlisted farmland preservation agreement." @@ -4896,138 +5379,138 @@ us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligib us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model incorrectly made other means-tested health coverage preclusive. It is not an exclusion from the Virginia ACA adult expansion category, under which the head qualifies based on age, status, and MAGI." us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit from non-MAGI Medicaid pathways to Virginia's MAGI-based ACA adult expansion category. That category has no asset test, so the $14,000 bank balance does not offset eligibility at 0.03 times FPL." us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,period_annualization,False,"The model improperly annualized $32 per hour at 40 hours per week into $66,560 despite the prompt explicitly defining $520 as the full-year gross wage total. It also wrongly treated other means-tested health coverage as disqualifying; using the stated annual income places the head at 0.03 times FPL and within Virginia's adult expansion category." -us,scenario_066,snap,claude-fable-5,llm_error,period_annualization,False,"It resolved eligibility and the benefit formula correctly — categorical eligibility waives the $14,000 resource test, and the 20% earned-income and standard deductions drive net income to $0 — and used the right FY2026 one-person maximum allotment of $298, then annualized as a flat 12 x $298 = $3,576. The calendar-2026 total blends the $298 allotment for nine months with the higher $304.68 allotment that governs the final three months of 2026, giving $3,596.04." -us,scenario_066,snap,claude-fable-5.1,llm_error,period_annualization,False,"It correctly zeroed net income under BBCE and used the correct FY2026 one-person maximum allotment of $298, but multiplied it flat across all 12 calendar months. The last three months of calendar 2026 carry the uprated $304.68 allotment, so the annual total is 9 x $298 + 3 x $304.68 = $3,596.04, not $3,576." -us,scenario_066,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It denied SNAP entirely on a $2,500/$3,750 countable-resource limit, ignoring that this household is categorically eligible through Virginia's broad-based categorical eligibility (TANF non-cash assistance), which waives the asset test outright so the $14,000 bank balance is irrelevant. It also treated the $43.33 monthly gross income as disqualifying when income that low passes the gross test and yields $0 net income, producing the full allotment of 9 x $298 + 3 x $304.68 = $3,596.04." -us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Its eligibility chain was right — BBCE waives the asset test, and deductions floor net income at $0 — but it plugged in a fabricated one-person maximum allotment of about $275/month. The FY2026 one-person maximum is $298, rising to $304.68 for the last three months of calendar 2026, so the correct annual total is $3,596.04." -us,scenario_066,snap,claude-opus-4.8,llm_error,other,False,"It derived the correct structure (net income essentially zero, $298/month maximum allotment, $3,576 for twelve months) and then subtracted an unexplained ""small income offset"" to submit $2,496, contradicting its own finding of zero net income. With $0 net income the 30% household contribution is $0, and the correct calendar-year total is 9 x $298 + 3 x $304.68 = $3,596.04." -us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,"It reached the right conclusion that near-zero net income yields the full one-person maximum allotment but used $283/month as that maximum. The FY2026 one-person maximum allotment is $298 per month, with $304.68 applying to the final three months of calendar 2026, totaling $3,596.04." -us,scenario_066,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It asserted that Virginia limits categorical eligibility to actual TANF/SSI recipients and therefore applied a $2,750 countable-resource limit to deny on the $14,000 bank balance. Virginia operates broad-based categorical eligibility through a TANF non-cash benefit at 200% of the poverty guideline, which confers categorical eligibility on this household and eliminates the asset test, leaving the full allotment of $3,596.04." -us,scenario_066,snap,claude-sonnet-5,llm_error,period_annualization,False,"It read the $520 gross wage figure as a weekly amount, inflating gross income to roughly $2,253/month and failing the 130% FPL gross income test. The prompt states that gross wage and salary amounts are annual totals, so monthly gross income is $43.33, which passes the gross test and leaves $0 net income and the full allotment of $3,596.04." -us,scenario_066,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly zeroed net income after the 20% earned-income and standard deductions but used $294 as the one-person maximum allotment. The FY2026 maximum is $298 per month, uprated to $304.68 for the last three months of calendar 2026, giving $3,596.04." -us,scenario_066,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It capped benefits at three months by applying the ABAWD time limit, despite the household fact of 40 usual weekly hours worked, which satisfies the 20-hour-per-week work requirement and removes any time limit. It compounded this with the stale FY2025 $292 allotment; the correct result is twelve months at $298 for nine months and $304.68 for three, or $3,596.04." -us,scenario_066,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It got eligibility, BBCE, and the $0 net income right but used the prior-year FY2025 one-person maximum allotment of $292. The FY2026 maximum is $298, and the final three months of calendar 2026 carry $304.68, producing $3,596.04 rather than $3,504." -us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its eligibility and net-income reasoning were correct, but it estimated the one-person maximum monthly allotment at $291. The FY2026 figure is $298, with $304.68 applying to October through December 2026, so the annual benefit is $3,596.04." -us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It submitted $0 on the assertion that the household's assets and circumstances produce no benefit, performing no allotment computation. Categorical eligibility through TANF non-cash assistance waives the resource test, and $0 net income yields the full maximum allotment of 9 x $298 + 3 x $304.68 = $3,596.04." -us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly identified BBCE and the maximum-allotment outcome but used $291 as the one-person monthly maximum. The FY2026 maximum allotment is $298, rising to $304.68 for the last three months of calendar 2026, for an annual total of $3,596.04." -us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached the right conclusion of a full maximum allotment under BBCE but used $291 per month as that maximum. The correct one-person maximum is $298 for nine months of calendar 2026 and $304.68 for the final three, totaling $3,596.04." -us,scenario_066,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted $3,504, which is exactly twelve times the stale FY2025 one-person maximum allotment of $292, with no allotment figure stated in its explanation. The FY2026 maximum is $298, and October through December 2026 carry $304.68, so the correct annual amount is $3,596.04." -us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly waived the asset test under BBCE and zeroed net income but applied the prior-year $292 one-person maximum allotment for all twelve months. FY2026 sets that maximum at $298, with $304.68 for the last three months of calendar 2026, giving $3,596.04." -us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It stated $292 per month as the one-person maximum allotment, which is the FY2025 value. The FY2026 maximum is $298 per month, uprated to $304.68 for October through December 2026, producing $3,596.04 instead of $3,504." -us,scenario_066,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its net-income derivation was correct, but it used $292 as the one-person maximum monthly allotment across the whole year. FY2026 sets the one-person maximum at $298, and the final three months of calendar 2026 use $304.68, for an annual total of $3,596.04." -us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"It worked the deduction chain correctly to $0 net income and full allotment, then rounded the FY2026 one-person maximum to ""approximately $300"" per month. The actual maximum is $298 for nine months of calendar 2026 and $304.68 for the final three, giving $3,596.04 rather than $3,600." -us,scenario_066,snap,glm-5.3,llm_error,thresholds_rates,False,"It correctly applied BBCE and the deduction chain to $0 net income but used about $297 per month as the one-person maximum allotment. The FY2026 maximum is $298, with $304.68 for the last three months of calendar 2026, totaling $3,596.04." -us,scenario_066,snap,gpt-5.4-mini,llm_error,asset_resource,False,"It denied benefits by applying a standard countable-resource limit to the $14,000 bank balance, never recognizing that categorical eligibility through Virginia's TANF non-cash assistance waives the asset test. With that test waived and net income at $0, the household receives the full allotment of $3,596.04." -us,scenario_066,snap,gpt-5.4-nano,llm_error,other,False,"It submitted $0 while stating that SNAP ""would typically be possible,"" substituting a blanket assumption that all outputs return zero for any computation of the allotment. The household passes the gross income test at $43.33/month, is categorically eligible, and has $0 net income, so it receives 9 x $298 + 3 x $304.68 = $3,596.04." -us,scenario_066,snap,gpt-5.5,llm_error,period_annualization,False,"It used the correct FY2026 one-person maximum allotment of $298 and correctly found countable income too low to reduce it, but multiplied by twelve flat months. October through December 2026 carry the uprated $304.68 allotment, so the annual total is $3,596.04, not $3,576." -us,scenario_066,snap,gpt-5.6-luna,llm_error,period_annualization,False,"It identified the correct $298 FY2026 one-person maximum and correct full-allotment outcome but applied that single monthly rate to all twelve months of calendar 2026. The last three months use $304.68, giving $3,596.04." -us,scenario_066,snap,gpt-5.6-sol,llm_error,period_annualization,False,"Its derivation — 20% earned-income deduction plus standard deduction to $0 net income, then the $298 one-person maximum — was correct through the monthly benefit, but it annualized at a constant $298. The blended calendar-2026 total is 9 x $298 + 3 x $304.68 = $3,596.04." -us,scenario_066,snap,gpt-5.6-terra,llm_error,asset_resource,False,"It denied SNAP solely because $14,000 in bank assets exceeds the nonelderly, nondisabled resource limit, missing that Virginia's broad-based categorical eligibility confers categorical eligibility on this household and eliminates the resource test. With no asset test and $0 net income, the annual benefit is $3,596.04." -us,scenario_066,snap,gpt-6-astra,llm_error,period_annualization,False,"It applied BBCE and the deduction chain correctly and used the right FY2026 one-person maximum of $298, then annualized it flat across twelve months. The final three months of calendar 2026 use the uprated $304.68 allotment, making the correct total $3,596.04." -us,scenario_066,snap,grok-4.3,llm_error,asset_resource,False,"It denied on a ""typical"" asset limit plus the presence of other means-tested health coverage, neither of which bars SNAP here: categorical eligibility waives the resource test in Virginia, and health coverage is not a SNAP eligibility factor. The household's $0 net income yields the full allotment of $3,596.04." -us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly waived the asset test under BBCE and computed $0 net income after deductions but used the FY2025 one-person maximum allotment of $292. FY2026 sets the maximum at $298, with $304.68 for the last three months of calendar 2026, totaling $3,596.04." -us,scenario_066,snap,grok-4.6,llm_error,thresholds_rates,False,"Its eligibility and deduction reasoning were correct, but it used $292 per month as the one-person maximum allotment. The FY2026 maximum is $298, uprated to $304.68 for October through December 2026, giving $3,596.04." -us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly derived $0 net income and full-allotment entitlement under BBCE but applied the stale $292 one-person maximum allotment for all twelve months. The correct figures are $298 for nine months of calendar 2026 and $304.68 for the final three, for $3,596.04." -us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"It approximated the one-person maximum monthly allotment as ""about $300"" and multiplied by twelve. The FY2026 maximum is $298 per month, with $304.68 applying to the last three months of calendar 2026, giving $3,596.04 rather than $3,600." -us,scenario_066,snap,kimi-k2.6,llm_error,asset_resource,False,"It denied on the resource limit because ""no categorical-eligibility waiver is listed"" among the household facts, treating categorical eligibility as an input flag rather than a Virginia state rule. Virginia's broad-based categorical eligibility applies automatically to households under its TANF non-cash gross income threshold, waiving the asset test and leaving the full allotment of $3,596.04." -us,scenario_066,snap,kimi-k3,llm_error,period_annualization,False,"Its full derivation — gross test passed, BBCE waiving the $14,000 asset balance, deductions to $0 net income, $298 FY2026 one-person maximum — was correct, and it then annualized as 12 x $298. The last three months of calendar 2026 carry $304.68, making the correct total $3,596.04." -us,scenario_066,snap,minimax-m3,llm_error,asset_resource,False,"It recognized that income is far below the SNAP threshold but denied on the $14,000 bank balance exceeding the asset limit. Categorical eligibility through Virginia's TANF non-cash assistance waives that asset test, so the household receives the full maximum allotment of $3,596.04." -us,scenario_066,snap,ox-alpha,llm_error,period_annualization,False,"It correctly applied Virginia's 200% FPL BBCE to waive the asset test, zeroed net income, and used the correct $298 FY2026 one-person maximum, then multiplied by twelve. October through December 2026 use the uprated $304.68 allotment, so the annual benefit is $3,596.04." -us,scenario_066,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After computing 12 x $292 = $3,504, it subtracted a phantom ""30% income contribution"" of about $872, a reduction inconsistent with $520 of annual gross wages. The 20% earned-income deduction ($8.67/month) and the standard deduction (about $204/month) reduce net income to $0, so the 30% contribution is $0, and the allotments are $298 and $304.68, totaling $3,596.04." -us,scenario_066,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It invented a $213 one-person maximum allotment and a $149-per-month minimum benefit floor, then submitted 12 x $149 = $1,788; the FY2026 minimum benefit is $24 per month and never applies here. With $0 net income the household receives the full maximum allotment — $298 for nine months of calendar 2026 and $304.68 for the last three — or $3,596.04." -us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly identified the Virginia refundable EITC election and the roughly $40 federal childless EITC, then priced the state credit at the superseded 15% rate rather than the 20% rate in force for tax year 2026. Its $6 is exactly 15% of $40; the correct computation is 20% x $39.78 = $7.96." -us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that Virginia has no refundable individual income tax credit and that its earned income benefit is only a nonrefundable credit or a subtraction, missing the refundable EITC option Virginia allows in lieu of the nonrefundable 20% credit and the low-income credit. That election pays 20% of the federal EITC of $39.78, or $7.96, as a refund even with zero Virginia liability." -us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It invented a minimum earned income threshold for the childless federal EITC and concluded the federal credit was essentially zero at $520 of wages; the childless credit phases in at 7.65% from the first dollar with no floor, giving $39.78. It also carried the outdated 15% state share, while Virginia's 2026 refundable election is 20% of the federal credit, yielding $7.96." -us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It dismissed the federal EITC as 'minimal' instead of applying the 7.65% childless phase-in to $520 of wages ($39.78), and then let the Virginia standard deduction zeroing liability drive the credit to zero, when a refundable credit is paid out at zero liability. Compounding this, it used the superseded 15% share; the 2026 Virginia refundable EITC is 20% of $39.78 = $7.96." -us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated Virginia's earned income benefit as liability-limited ('nonrefundable-limited here and tax is 0'), missing that Virginia permits the filer to elect a fully refundable EITC in lieu of the nonrefundable 20% credit and the low-income credit. With zero Virginia tax the refundable election is the optimal claim and pays 20% x $39.78 = $7.96 in cash." -us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It asserted the childless federal EITC requires earned income above roughly $600 and so set the federal credit to $0; the childless EITC phases in at 7.65% from the first dollar, making it $39.78 on $520 of wages. Its correct finding that Virginia taxable income and liability are $0 does not reduce a refundable credit, and Virginia's refundable EITC election pays 20% of the federal amount, $7.96." -us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It withheld the credit for want of an affirmative election, contradicting the prompt's instruction to assume filing and take-up; PolicyEngine claims the refundable EITC whenever it beats the nonrefundable credit and the low-income credit, which it does at zero liability. The election pays 20% of the $39.78 federal childless EITC, or $7.96." -us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It stated flatly that Virginia's earned income credit is nonrefundable, overlooking the separate refundable EITC option Virginia offers in lieu of the nonrefundable 20% credit and the low-income credit. That refundable option pays 20% of the household's $39.78 federal EITC, $7.96, regardless of liability." -us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It reproduced the federal childless EITC exactly ($39.78 = 7.65% x $520) and correctly treated Virginia's EITC as refundable, then applied the pre-2025 15% share. Virginia's refundable percentage for tax year 2026 is 20%, so the credit is $7.96, not $5.97." -us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It asserted with no derivation that no state refundable credit applies, missing Virginia's refundable EITC election. The head's federal childless EITC is 7.65% x $520 = $39.78 and Virginia refunds 20% of it, $7.96." -us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It identified Virginia's refundable EITC correctly but priced it at the superseded 15% rate and rounded the federal credit to $40. The 2026 refundable share is 20% and the federal EITC is 7.65% x $520 = $39.78, giving $7.96." -us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It derived the federal childless EITC exactly ($39.78) and correctly treated the Virginia credit as refundable, then multiplied by the pre-2025 15% rate. Virginia's refundable share is 20% for tax year 2026, making the credit $7.96." -us,scenario_066,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It declared that Virginia has no refundable individual income tax credit for this profile, missing the refundable EITC election available in lieu of the nonrefundable 20% credit and the low-income credit. The election pays 20% of the $39.78 federal childless EITC, or $7.96, even with zero Virginia liability." -us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed the federal childless EITC correctly at $39.78 and recognized Virginia's refundable EITC, but applied the repealed 15% share. The tax year 2026 rate is 20%, so the Virginia refundable credit is $7.96." -us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It had the structure and the federal base exactly right ($39.78 = 7.65% x $520) and failed only on the state percentage, using the pre-2025 15% rather than the 20% share in force for 2026. The correct product is $7.96." -us,scenario_066,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It conflated refundability with liability, reasoning that a household with no Virginia tax liability gets no refundable credit; a refundable credit is paid out precisely when liability is zero. Virginia's refundable EITC election pays 20% of the $39.78 federal childless EITC, $7.96." -us,scenario_066,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It claimed the Virginia EITC is nonrefundable and that zero liability settles the question, missing the refundable EITC election Virginia allows instead of the nonrefundable 20% credit and the low-income credit. That election is paid in full at zero liability: 20% x $39.78 = $7.96." -us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It asserted without derivation that the listed facts trigger no Virginia refundable credit, skipping the refundable EITC election that the head's $520 of wages qualifies for. The federal childless EITC is 7.65% x $520 = $39.78 and Virginia refunds 20% of it, $7.96." -us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It assumed no state refundable credit exists at this income level, when Virginia's refundable EITC is largest in relative terms exactly at low earnings and is not liability-limited. Twenty percent of the $39.78 federal childless EITC yields $7.96." -us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly framed the credit as a percentage of the federal childless EITC of about $40 but used the superseded 15% share. Virginia's refundable EITC is 20% of the federal credit for tax year 2026, and the federal credit is exactly $39.78, so the answer is $7.96." -us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It denied any Virginia refundable credit for a single adult with no dependents, but the state refundable EITC follows the federal childless EITC, which the 40-year-old head qualifies for under the 25-64 age rule. The federal credit is 7.65% x $520 = $39.78 and Virginia refunds 20% of it, $7.96." -us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated the childless household as categorically outside Virginia's refundable credits, missing that Virginia's refundable EITC tracks the federal childless EITC available to filers aged 25-64. That path gives $39.78 federally and 20% of it, $7.96, at the state level." -us,scenario_066,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It reasoned that zero tax liability precludes state refundable credits, inverting the definition of refundability. Virginia's refundable EITC election pays out at zero liability: 20% x the $39.78 federal childless EITC = $7.96." -us,scenario_066,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It asserted that no Virginia refundable credit applies to a single adult with $520 of income, overlooking the state's refundable EITC election tied to the federal childless credit. Federal EITC of 7.65% x $520 = $39.78 produces a Virginia refundable credit of 20%, or $7.96." -us,scenario_066,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly identified the Virginia refundable EITC and a roughly $40 federal credit, then applied the pre-2025 15% share. The tax year 2026 share is 20% and the exact federal credit is $39.78, giving $7.96." -us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It claimed Virginia has no state EITC at all, when Virginia has both a nonrefundable EITC set at 20% of the federal credit and a refundable EITC option elected in its place. Electing the refundable option on a $39.78 federal childless EITC produces $7.96." -us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached the correct mechanism, a refundable Virginia EITC on a $40 federal credit, but used the superseded 15% rate. Virginia refunds 20% of the federal EITC in 2026 and the federal credit is $39.78, so the credit is $7.96." -us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It denied the credit on two grounds that do not disqualify: having no dependents and having zero state tax liability. Virginia's refundable EITC follows the federal childless EITC ($39.78 at $520 of wages under the 7.65% phase-in) and is paid at 20%, $7.96, regardless of liability." -us,scenario_066,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It applied a stale sunset, holding that Virginia's refundable EITC covered only tax years 2022 through 2025; the refundable option remains in force for tax year 2026 and its share was raised to 20% of the federal credit. Applying it to the household's $39.78 federal childless EITC yields $7.96." -us,scenario_066,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It gave a bare denial that any Virginia refundable credit applies, bypassing the state's refundable EITC election. The head's $520 of wages generates a $39.78 federal childless EITC, 20% of which is Virginia's refundable credit of $7.96." -us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It stated that Virginia's credits require tax liability and that Virginia has no EITC paying beyond liability for a childless filer, which is exactly what the state's refundable EITC election does. On a $39.78 federal childless EITC the refundable 20% share pays $7.96 in cash against $0 of Virginia tax." -us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It ruled out Virginia refundable credits for a one-person household with no qualifying children, but the childless federal EITC applies to filers aged 25-64 and the 40-year-old head earns $39.78 of it at the 7.65% phase-in. Virginia's refundable EITC election pays 20% of that, $7.96." -us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model aggregated the parents' wages and pension into a single ~$87,300 three-person MAGI figure and applied it to the 23-year-old, while PolicyEngine scores this person's own Medicaid MAGI at 0.00 x FPL — his $10,800 disability benefits sit outside MAGI and he has no earnings. It then compounded the error by importing a resource test and parental deeming from the SSI-related pathway; the engine placed him in the ADULT category (Indiana's HIP expansion, ages 19-64 up to 138% FPL), which has no asset test at all." -us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model attributed the tax unit's ~$87,300 of wages and pension to the 23-year-old dependent and compared that aggregate against the 138% FPL adult limit. PolicyEngine evaluates this person's own MAGI at 0.00 x FPL, which places him squarely inside Indiana's ACA adult expansion group and yields eligibility." -us,scenario_067,dependent1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"The model equated the dependent's Medicaid MAGI unit income with the filing unit's $98,102 total, which additionally counts the $10,800 of disability benefits that never enter MAGI. PolicyEngine computes the 23-year-old's MAGI income level as 0.00 x FPL and places him in the ADULT expansion category, so the 138% FPL test is passed rather than failed." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model gave only the assertion that household income exceeds Indiana's Medicaid threshold, a conclusion consistent solely with charging the parents' $82,360 in wages and $4,942 pension to the 23-year-old. The engine's MAGI income level for this person is 0.00 x FPL, far below the 138% expansion limit, so the ADULT category applies and he is eligible." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model placed the 23-year-old in the parents' tax household and tested the combined ~$87,300 against the Medicaid limits. PolicyEngine measures his own MAGI at 0.00 x FPL and qualifies him under Indiana's ACA adult expansion group, independent of the parents' income and of his disability status." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model denied a childless-adult coverage pathway in Indiana, reasoning that adults without children do not qualify at this income level. Indiana adopted the ACA expansion through HIP, which covers non-elderly childless adults up to 138% FPL, and the engine placed this 23-year-old in that ADULT category at 0.00 x FPL." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model assigned the tax unit's $87,302 of wages plus pension to the dependent and compared it to Indiana's Medicaid income limits. The engine's MAGI income level for this 23-year-old is 0.00 x FPL, so he falls inside the ACA adult expansion group rather than above the limit." -us,scenario_067,dependent1_medicaid_eligible,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"The model built a three-person MAGI household around the joint-filing parents, computed $87,302 as roughly 317% of FPL, and denied expansion coverage on that basis. PolicyEngine evaluates the 23-year-old's own MAGI income level at 0.00 x FPL and returns eligibility under the ADULT expansion category." -us,scenario_067,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model summed $82,360 in wages, $4,942 of pension, and $10,800 of disability benefits into a $98,102 household figure and tested it against the 138% FPL expansion limit for three people. Disability benefits are excluded from MAGI and the parents' income is not charged to this person in the engine's calculation, which returns 0.00 x FPL and ADULT-category eligibility." -us,scenario_067,dependent1_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"The model routed the disabled 23-year-old exclusively through the SSI-linked disability pathway with its income and asset limits and stopped once that pathway failed. The engine assigns him the ADULT (ACA expansion) category, a MAGI pathway that ignores disability status and resources entirely; at 0.00 x FPL he clears the 138% threshold." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model abstained from applying any eligibility rule, stating that eligibility is not estimated absent SSI or qualification specifics, and defaulted the answer to 0. The determination requires only the ACA adult expansion test that Indiana has adopted — age under 65 and MAGI at or below 138% FPL — which this 23-year-old passes at 0.00 x FPL." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model charged the household's counted income to the 23-year-old, judged it far above the three-person MAGI limit, and additionally required SSI receipt or another categorical listing before granting eligibility. The engine measures this person's MAGI at 0.00 x FPL and grants eligibility through the ADULT expansion category, which requires neither SSI receipt nor any categorical add-on beyond age and income." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied the parents' wage and pension income to the dependent and concluded it exceeds Indiana's Medicaid income limits. The 23-year-old's own MAGI income level is 0.00 x FPL, placing him in the ACA adult expansion group that Indiana operates as HIP." -us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model tested the dependent against income-and-resource-tested pathways, counting the household's $30,200 in bank assets and $77,176 in vehicles against him. The ACA adult expansion category the engine applied is MAGI-based with no resource test, and his MAGI income level of 0.00 x FPL clears the 138% limit." -us,scenario_067,dependent1_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model counted the $10,800 of disability benefits as Medicaid-countable income and layered a household resource test on top of it. Disability benefits fall outside MAGI, the expansion category carries no asset test, and the engine's 0.00 x FPL income level yields eligibility under Indiana's ADULT expansion group." -us,scenario_067,dependent1_medicaid_eligible,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model correctly identified Indiana's HIP expansion at 138% FPL but then had the 23-year-old share a household MAGI of roughly $87,000 drawn from the parents' wages and pension. PolicyEngine assigns him a MAGI income level of 0.00 x FPL, so the very pathway the model named delivers eligibility." -us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model tested the 23-year-old against the parents' combined $87,302 of wages and pension and additionally described Indiana as lacking non-expansion pathways for disabled adults. Indiana operates the ACA expansion through HIP, and the engine's MAGI income level for this person is 0.00 x FPL, well inside the 138% limit." -us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that $10,800 in disability benefits exceeds Indiana's adult Medicaid income limit; 138% of the one-person federal poverty line is roughly $22,000 in 2026, so even that figure clears the test. Disability benefits are excluded from MAGI entirely, leaving the engine's income level at 0.00 x FPL and the ADULT expansion category satisfied." -us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Asserted a ""PolicyEngine disability-based rule"" for Medicare that the engine does not contain: is_medicare_eligible is the age-65 threshold test, and the is_disabled flag and disability_benefits amount feed SSI, EITC and medical-expense logic instead. Dependent 1 is 23, so the only test that governs this variable fails and the value is 0." -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Identified the 24-month SSDI Medicare waiting period and then discharged it by misreading the prompt's ""treat status facts as constant throughout the tax-benefit year"" instruction as proof the waiting period was already served; that instruction fixes within-year status and says nothing about a prior entitlement history. It also converted the generic $10,800 disability_benefits input ($900/month, an SSI-scale amount) into SSDI entitlement, while PolicyEngine's Medicare eligibility reads neither that income variable nor the disability flag and returns False for anyone under 65." -us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"Equated the generic disability_benefits input with SSDI, then stated the 24-month waiting period and waived it with no basis in the household facts, which specify no entitlement start date. PolicyEngine decides is_medicare_eligible on age alone against the 65 threshold, so the 23-year-old dependent returns 0 regardless of disability status or benefit receipt." -us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"Applied the statutory SSDI-to-Medicare disability route directly, treating ""disabled plus disability benefits"" as sufficient for under-65 entitlement and omitting the 24-month qualifying period entirely. PolicyEngine's is_medicare_eligible implements no disability route; it compares age to the 65 threshold, and Dependent 1 at 23 fails it." -us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"Attributed a ""disability-based Medicare under PolicyEngine rules"" pathway to the engine, which models Medicare eligibility purely as age at or above 65. The disability flag and $10,800 of disability benefits enter SSI and medical-expense computations, not the Medicare variable, so Dependent 1 at age 23 is 0." -us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Claimed a ""modeled disability-based Medicare pathway"" exists in the microsimulation and rested the answer on that claim alone, citing no age test and no waiting-period requirement. The modeled rule is the single age-65 threshold, which a 23-year-old fails, yielding 0." -us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"Stated that PolicyEngine treats disability plus disability-benefit receipt as qualifying for Medicare below age 65, fabricating engine behavior; is_medicare_eligible tests age against the 65 threshold and ignores both is_disabled and disability_benefits. Dependent 1 is 23, so the value is 0." -us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"Invoked real-world ""disability rules"" for Medicare with no mention of the 24-month SSDI qualifying period and no check of the age criterion PolicyEngine actually applies. The engine's Medicare eligibility is age at or above 65, and Dependent 1's age of 23 makes the answer 0." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted $1,845 of auto loan interest from income and used a $32,600 standard deduction instead of the 2026 MFJ figure of $32,200, driving its worksheet to $5,347. It then submitted $3,391, a number its own arithmetic never produces; the correct chain is $87,302 - $32,200 = $55,102 taxable, $6,116.24 of tax, less the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It had AGI, the $32,200 standard deduction, the $24,800 breakpoint, and the $500 other-dependent credit all correct, then additionally deducted $1,845 of auto loan interest, which is not taken here. That single unqualified deduction cut taxable income from $55,102 to $53,257 and the tax by exactly $221.40 (12% of $1,845), the whole of its error." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction, $3,700 below the 2026 amount of $32,200, pushing taxable income to $58,802 and pre-credit tax to roughly $7,080. It then subtracted approximately $1,266 of unnamed nonrefundable credits it never derived, when the only nonrefundable credit available is the $500 credit for other dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its worksheet reached $6,120 of pre-credit tax on approximately $55,002 of taxable income and applied the $500 other-dependent credit for roughly $5,620, within $4 of the correct $5,616.24. It then discarded that result and submitted $6,280, a figure that neither the $23,850/$96,950 brackets it cited nor the true $24,800 breakpoint produces and that silently restores the $500 credit it had already subtracted." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It imputed roughly $6,500 of mortgage interest from the $100,000 loan balance, although unlisted amounts are 0, and then declared its $23,121 itemized total larger than the $32,300 standard deduction it had just named one line earlier. Itemizing raised taxable income to $64,181 and tax to $7,222; the $32,200 standard deduction governs, giving $55,102 taxable and $6,116.24 before the $500 credit." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reproduced the reference derivation exactly: $55,102 taxable, $2,480 plus 12% of $30,302 = $6,116, and 'about 5,616' after the $500 other-dependent credit. It then submitted $6,104, discarding both the $500 credit and its own bracket arithmetic in favor of an unexplained 'bracket estimate'." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its income treatment and $500 other-dependent credit are right, but it anchored on a $30,000 MFJ standard deduction and a $24,500 10% breakpoint drawn from pre-OBBBA inflation projections; the 2026 values are $32,200 and $24,800. Overstating taxable income by $2,200 and mis-setting the breakpoint added $270 to the correct $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $6,068 of pre-credit tax on a $54,702 taxable income (a $32,600 standard deduction rather than $32,200) and identified the $500 other-dependent credit, then submitted $10,500 'after accounting for self-employment-like adjustments and rounding for benefit phase-outs'. The household has no self-employment income and no phase-out applies; its submitted figure exceeds even its own pre-credit tax by $4,432." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 disability benefit to the filers' AGI, producing $98,102; a dependent's own benefits stay on that person's return and never enter the joint return. It compounded this with a $30,600 standard deduction instead of $32,200 and denied the $500 credit for other dependents on the ground that the dependent is over 17, which is exactly the condition that credit exists to cover." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a TCJA-sunset regime for 2026: a $16,500 MFJ standard deduction, three $5,250 personal exemptions, and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction, no personal exemptions, and 12% above $24,800, and it also omitted the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed pre-TCJA 2026 law, subtracting three $5,300 personal exemptions and taxing at 15% above $24,400, and itemized $19,108 of medical expenses plus Indiana income tax. Personal exemptions are $0 in 2026, the $32,200 standard deduction exceeds that itemized total, and the rate above $24,800 is 12%, so taxable income is $55,102 with a $500 credit applied." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It stacked two errors: taxing 85% of the dependent's $10,800 benefit ($9,180) on the parents' return, and applying sunset parameters of a $15,900 standard deduction, $15,150 of personal exemptions, and a 15% bracket. The dependent's benefits are that person's own income, and 2026 provides a $32,200 standard deduction, no personal exemptions, and 12% above $24,800." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used a $31,200 standard deduction and then asserted a taxable income of about $60,400, which exceeds even its own AGI less that deduction ($56,102) by roughly $4,300, and it declared 'no significant nonrefundable credits'. The correct chain is $87,302 - $32,200 = $55,102, $6,116.24 of tax, less the $500 other-dependent credit the disabled 23-year-old generates." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed 'under the expiration of the TCJA', subtracting $16,050 of personal exemptions plus $19,064 of itemized deductions and taxing in a 15% bracket. For 2026 personal exemptions are repealed, the $32,200 standard deduction exceeds that itemized total, and the applicable rate above $24,800 is 12%, and it also dropped the $500 other-dependent credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules with $15,150 of personal exemptions and a 15% bracket, and manufactured $4,500 of mortgage interest and $2,571 of state tax that are not listed inputs. With exemptions repealed and the $32,200 standard deduction exceeding its $23,692 itemized total, taxable income is $55,102, not $48,460, and the $500 other-dependent credit applies." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $7,534 is the 2026 MFJ tax with no credit on roughly $66,900 of taxable income, i.e. household income of $98,102 including the dependent's $10,800 disability benefit less a standard deduction near $31,200. The dependent's benefits stay off the joint return and the standard deduction is $32,200, giving $55,102 taxable, $6,116.24 of tax, and $5,616.24 after the $500 credit for other dependents." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its income treatment and $500 other-dependent credit are exactly right; it used a $30,800 standard deduction and a $23,850 10% breakpoint instead of the 2026 values of $32,200 and $24,800. Those two parameter errors account for its entire $187 overstatement ($1,400 of extra taxable income at 12% plus $950 of bracket shift at 2 points)." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly took AGI of $87,302 and the $500 credit for other dependents, but its $5,912.24 implies a standard deduction near $29,800 rather than the 2026 MFJ $32,200. That understatement inflated taxable income by roughly $2,470 and the tax by $296 over the correct $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the $30,000 standard deduction and the $23,850 10% breakpoint instead of the 2026 amounts of $32,200 and $24,800, overstating taxable income by $2,200. Income treatment and the $500 other-dependent credit are correct, so those parameter values are the whole $283 error." -us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It settled on a $30,000 standard deduction and a $23,600 10% bracket ceiling, calling the latter a '2025 bracket proxy'; the 2026 values are $32,200 and $24,800. Taxable income therefore came out $57,302 instead of $55,102, and after its correct $500 other-dependent credit the tax overshot by $288." -us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It reached the correct $55,102 taxable income with the right $32,200 standard deduction, then used a $25,200 10% breakpoint rather than $24,800 and refused the $500 credit for other dependents by applying the qualifying-relative gross income test to the dependent's $10,800 of disability benefits. Those benefits are excluded from gross income for that test, and the credit is allowed for this one qualifying adult dependent." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It submitted $4,567 with no computation, asserting only that 'standard deduction and child/dependent credits do not create a larger nonrefundable offset'. The 2026 chain is $87,302 less the $32,200 standard deduction, $6,116.24 of tax on $55,102, less the $500 other-dependent credit for $5,616.24; its figure corresponds to no standard deduction, bracket set, or credit combination in that chain." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It produced no arithmetic and its $4,940 matches no step of the 2026 computation. It also invoked itemized deductions built on 'mortgage and medical context', when no mortgage interest amount is listed and the $16,621 of medical expenses above the 7.5% floor falls well short of the $32,200 standard deduction that yields $55,102 taxable and $5,616.24 after the $500 credit." -us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Every parameter it used is right, including the $32,200 standard deduction, the $24,800 breakpoint, and the $500 other-dependent credit, but it added $9,180 of 'taxable disability benefits' (85% of the dependent's $10,800) to the joint return. That benefit is the dependent's own income and stays out of the filers' AGI, so taxable income is $55,102 and the pre-credit tax $6,116.24, not $64,282 and $7,217.84." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It offered no computation beyond 'head wages and pension after standard deduction and credits' and submitted $8,500, which exceeds the entire pre-credit tax on $55,102 of taxable income ($6,116.24) by 39%. Applying the $32,200 standard deduction to $87,302 and the 2026 MFJ schedule, then the $500 other-dependent credit, gives $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a TCJA-sunset regime with $15,750 of personal exemptions and 'post-TCJA' joint rates while itemizing $19,192 of medical expenses and SALT. Personal exemptions are $0 in 2026, the $32,200 standard deduction exceeds that itemized total, and it also asserted zero nonrefundable credits when the $500 credit for other dependents applies." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $16,600 standard deduction, three $5,300 personal exemptions, and a 15% top rate, itemizing $16,621 of medical expenses because it believed that exceeded the standard deduction. The 2026 MFJ standard deduction is $32,200 with no personal exemptions, so taxable income is $55,102 taxed at 10%/12%, and the $500 other-dependent credit applies." -us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It reverted to pre-TCJA law, inflating the 2017 standard deduction by a 1.3615 factor to $17,291 and applying a 15% bracket above $25,119, yielding $70,011 of taxable income. The 2026 MFJ standard deduction is $32,200 and the rate above $24,800 is 12%, giving $55,102 taxable, and the $500 credit for other dependents applies to the disabled 23-year-old." -us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It used a standard deduction near $31,500 rather than $32,200 and stated 'no dependents claimed', forgoing the $500 credit for other dependents that the disabled 23-year-old generates. The credit accounts for most of its $590 overstatement; the deduction shortfall supplies the remaining $84." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Its income treatment and $500 other-dependent credit are correct, but it used a $30,800 standard deduction and a $24,400 10% breakpoint instead of the 2026 values of $32,200 and $24,800. Those two parameter errors produce exactly its $176 overstatement ($1,400 of extra taxable income at 12% plus $400 of bracket shift at 2 points)." -us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It made two adjustments that do not belong: it added $9,180 of 'taxable Social Security' derived from the dependent's $10,800 disability benefit, and it subtracted $1,845 of auto loan interest. The dependent's benefit is that person's own income and no auto loan interest deduction is taken here, so taxable income is $55,102 rather than $62,437, giving $6,116.24 before its correctly applied $500 credit." -us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It folded the dependent's $10,800 disability benefit into the filers' income and used a $30,000 standard deduction instead of $32,200, then claimed a $2,000 child tax credit for a 23-year-old. The CTC requires a child under 17, so only the $500 credit for other dependents is available; its inflated income and inflated credit nearly cancel, landing $29 below the correct $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It derived the pre-credit tax of $6,116.24 exactly, with the correct $32,200 standard deduction, $55,102 taxable income, and $24,800 breakpoint, then concluded 'no nonrefundable credits apply (dependent is 23 so no CTC)'. Failing the CTC age test is precisely what makes the dependent eligible for the $500 credit for other dependents, which brings the liability to $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 disability benefit as taxable unearned income on the joint return, raising AGI to $98,102, and used a $31,200 standard deduction instead of $32,200. The dependent's benefits belong on that person's own return, so taxable income is $55,102 rather than $66,902, and after its correctly applied $500 other-dependent credit the answer is $5,616.24." -us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It zeroed the liability by applying a fabricated $10,000 nonrefundable child tax credit against a $6,962 tentative tax. The 23-year-old is over 16 and generates no CTC at all; the only nonrefundable credit here is the $500 credit for other dependents, which reduces the $6,116.24 tax on $55,102 of taxable income to $5,616.24 rather than to $0." +us,scenario_066,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"Applied the $2,500/$3,750 SNAP resource limit to the $14,000 bank balance and denied eligibility. It missed that Virginia's TANF non-cash broad-based categorical eligibility confers eligibility, which this household meets. That pathway yields zero net income and the full $298/month maximum ($3,576/year)." +us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly applied BBCE and found net income of $0, but used about $275/month as the one-person maximum allotment. The FY2026 contiguous-US maximum is $298/month, so the benefit is $3,576/year, not $3,300." +us,scenario_066,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"Identified the correct $298/month maximum and near-zero net income, then subtracted an invented $1,080/year ($90/month) income offset. The earned income deduction and standard deduction reduce $43.33 of monthly earnings to $0 net income, so the 30% contribution is $0 and the benefit is the full $3,576." +us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,"Correctly concluded that net income is zero and the maximum allotment applies, but used an outdated maximum of $283/month. The FY2026 one-person maximum is $298/month, or $3,576/year." +us,scenario_066,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Wrongly asserted that Virginia limits categorical eligibility to TANF/SSI cash recipients, then denied SNAP under a $2,750 asset limit. Virginia's TANF non-cash BBCE makes this household categorically eligible despite its $14,000 in assets, and it receives the $298/month maximum." +us,scenario_066,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Misread the $520 annual wage total as weekly pay, inflating monthly income to about $2,253 and failing the 130% FPL gross income test. Actual monthly gross income is $43.33, which passes every income test and produces net income of $0 and the $298/month maximum." +us,scenario_066,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Correctly found $0 net income and the maximum allotment, but used $294/month instead of the FY2026 one-person maximum of $298/month. That gave $3,528 instead of $3,576." +us,scenario_066,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied the ABAWD three-month time limit on the claim that the head fails the 20-hour work requirement. The head's usual weekly hours are 40, so the work requirement is met and benefits run all 12 months. It also used a $292 maximum instead of $298, giving $876 instead of $3,576." +us,scenario_066,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Correctly applied BBCE and found net income of $0, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, so 12 months total $3,576, not $3,504." +us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and found net income of $0, but estimated the one-person maximum at $291/month. The FY2026 maximum is $298/month, or $3,576/year." +us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Treated very low income as grounds for no benefit, which inverts the SNAP formula; zero net income produces the maximum allotment. With BBCE removing the asset barrier and deductions reducing $43.33 of monthly earnings to $0 net income, the household receives $298/month ($3,576/year), not $0." +us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly applied BBCE and the maximum-benefit outcome, but used $291/month as the one-person maximum. The FY2026 figure is $298/month, which yields $3,576/year." +us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly reached the maximum benefit under BBCE, but used a $291/month maximum allotment. The FY2026 one-person contiguous-US maximum is $298/month, or $3,576/year." +us,scenario_066,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $3,504 answer equals 12 × $292, so it applied a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, and the household's zero net income yields $3,576/year." +us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and found net income of $0, but its $3,504 annual maximum reflects $292/month. The FY2026 one-person maximum is $298/month, or $3,576/year." +us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Correctly found net income of $0 and the maximum-allotment outcome, but used $292/month instead of the FY2026 one-person maximum of $298/month." +us,scenario_066,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Correctly zeroed out net income with the standard deduction, but applied a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, giving $3,576/year." +us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"Correctly applied BBCE and zeroed net income through the earned income, standard and shelter deductions, but rounded the maximum allotment up to $300/month. The FY2026 one-person maximum is $298/month, so the annual benefit is $3,576, not $3,600." +us,scenario_066,snap,glm-5.3,llm_error,thresholds_rates,False,"Correctly found net income of $0 under BBCE, but used $297/month as the one-person maximum. The FY2026 maximum is $298/month, which gives $3,576/year instead of $3,564." +us,scenario_066,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard resource limit. It missed that Virginia's TANF non-cash BBCE makes the household categorically eligible, after which zero net income yields the $298/month maximum ($3,576/year)." +us,scenario_066,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"Acknowledged the household could qualify but returned $0 with no computation. The correct derivation gives BBCE categorical eligibility and $0 net income after the earned income and standard deductions, so the benefit is the $298/month maximum, or $3,576/year." +us,scenario_066,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the federal resource limit for nonelderly, nondisabled households to the $14,000 bank balance and denied eligibility. Virginia's TANF non-cash BBCE confers categorical eligibility on this household, and it receives the $298/month maximum." +us,scenario_066,snap,grok-4.3,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard limit, and cited other health coverage as a factor. Health coverage has no bearing on SNAP, and Virginia's TANF non-cash BBCE makes the household categorically eligible for the $298/month maximum." +us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"Correctly applied BBCE and zeroed net income with the shelter deduction, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, so 12 months total $3,576." +us,scenario_066,snap,grok-4.6,llm_error,thresholds_rates,False,"Correctly found net income of $0 under BBCE, but applied a $292/month maximum. The FY2026 one-person maximum is $298/month, which yields $3,576/year instead of $3,504." +us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"Correctly waived the asset test under BBCE and zeroed net income, but used a $292/month maximum allotment. The FY2026 one-person maximum is $298/month, or $3,576/year." +us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"Correctly concluded that near-zero net income yields the maximum allotment, but rounded it to $300/month. The FY2026 one-person maximum is $298/month, so the annual benefit is $3,576, not $3,600." +us,scenario_066,snap,kimi-k2.6,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard resource limit and no categorical-eligibility waiver was listed. Virginia's TANF non-cash BBCE applies automatically from the household's income and assets without any listed benefit receipt, and the household receives $298/month." +us,scenario_066,snap,minimax-m3,llm_error,thresholds_rates,False,"Denied SNAP because the $14,000 in assets exceeds the standard asset limit. It missed Virginia's TANF non-cash broad-based categorical eligibility, which this household meets and which leads to the full $298/month allotment ($3,576/year)." +us,scenario_066,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"Stated that net income is far below the poverty line, then still subtracted an $872/year 30% income contribution. The earned income and standard deductions reduce net income to $0, so the contribution is $0. It also used a $292 maximum instead of $298, giving $2,632 instead of $3,576." +us,scenario_066,snap,qwen3.8-max,llm_error,thresholds_rates,False,"Used a wrong $213/month maximum allotment and subtracted 30% of the $520 annual gross income as if it were a monthly amount, skipping the deductions that reduce net income to $0. It then applied a nonexistent $149/month minimum benefit (the actual minimum is $24). The household receives the full $298/month, or $3,576/year." +us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,It correctly put the federal childless EITC at about $40 (7.65% phase-in) and knew Virginia pays a refundable EITC. But it used the outdated 15% rate and rounded to $6. The current 20% rate gives 0.20 × $39.78 = $7.96. +us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed Virginia has no refundable credits and that the EIC is federal-only. That misses Virginia's refundable EITC, which equals 20% of the federal EITC ($39.78 here) and is paid regardless of liability, giving $7.96." +us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It said a childless filer with $520 of wages gets essentially no federal EITC because of a minimum earned-income threshold. No such minimum exists: the credit phases in at 7.65% from the first dollar, giving $39.78. It also cited a 15% Virginia rate instead of 20%; the correct refundable credit is $7.96." +us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It identified Virginia's refundable EITC but dismissed it as negligible and rounded it to $0, while also citing the old 15% rate. The credit has no minimum amount and does not depend on liability. It is 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated Virginia's EITC/low-income credit as capped at tax liability, which is $0 here. It ignored the separate refundable EITC option, which pays 20% of the $39.78 federal EITC ($7.96) even with zero liability." +us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It claimed the federal childless EITC needs earned income above about $600 and is $0 at $520. That is wrong: the credit phases in at 7.65% of earnings from the first dollar, giving $39.78. Virginia's refundable EITC is 20% of that, or $7.96." +us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It zeroed the credit on the grounds that the refundable EITC option was never elected. The prompt says to assume take-up, so the household claims the refundable EITC: 20% of the $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It asserted that Virginia's earned income credit is only nonrefundable. That misses the refundable EITC, which pays 20% of the federal EITC; with a $39.78 federal credit, that is $7.96." +us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It computed the federal EITC exactly ($39.78) but used the outdated 15% Virginia rate, giving $5.97. The current refundable rate is 20%, which gives $7.96." +us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It said no state refundable credits apply and never recognized Virginia's refundable EITC. That credit is 20% of the $39.78 federal childless EITC, or $7.96." +us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the old 15% Virginia refundable EITC rate to a federal EITC it rounded to $40, getting $6. The correct calculation is 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly derived the $39.78 federal EITC but used the old 15% Virginia refundable rate, giving $5.97. The current rate is 20%, which gives $7.96." +us,scenario_066,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It said Virginia has no refundable individual credits for this profile. It missed the refundable EITC, which pays 20% of the $39.78 federal childless EITC ($7.96) to a worker with no children." +us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed $39.78 × 15% = $5.97 using the superseded 15% Virginia refundable EITC rate. The current rate is 20%, which gives $7.96." +us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly used the $39.78 federal EITC but applied the old 15% Virginia rate, giving $5.97. The current 20% refundable rate gives $7.96." +us,scenario_066,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,It reasoned that a household with no tax liability has no refundable credits. That inverts how refundability works: Virginia's refundable EITC pays 20% of the $39.78 federal EITC ($7.96) even with zero liability. +us,scenario_066,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It called the Virginia EITC nonrefundable and therefore useless against $0 tax. That misses the refundable EITC, which pays 20% of the $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It concluded that no Virginia refundable credit is triggered. That overlooks the refundable EITC, which is triggered by the $39.78 federal childless EITC on $520 of wages and pays 20% of it, or $7.96." +us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It assumed no state refundable credits at this income. Virginia's refundable EITC applies at exactly this low income: 20% of the $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly recognized Virginia's refundable EITC and a federal childless EITC of about $40. But it applied the outdated 15% rate to get $6 instead of the current 20% rate, which gives $7.96." +us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It ruled out any Virginia refundable credit because the filer is childless with $520 of wages. Childless workers do get the federal EITC ($39.78 here), and Virginia's refundable EITC pays 20% of it, or $7.96." +us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated having no children as disqualifying for any Virginia refundable credit. The childless federal EITC is $39.78, and Virginia's refundable EITC pays 20% of it, or $7.96." +us,scenario_066,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It said no refundable Virginia credit applies to a childless filer at this income. It missed that the $39.78 federal childless EITC carries through to Virginia's refundable EITC at 20%, giving $7.96." +us,scenario_066,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It reasoned that zero tax liability means no refundable credits. Refundable credits are paid regardless of liability, and Virginia's refundable EITC is 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It said no Virginia refundable credit applies to a single adult with $520 of income. It overlooked the refundable EITC, which pays 20% of that worker's $39.78 federal EITC, or $7.96." +us,scenario_066,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the outdated 15% Virginia refundable EITC rate to a federal EITC rounded to $40, getting $6. The correct calculation is 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It claimed Virginia's 15% refundable EITC election expired before tax year 2026. In fact the refundable EITC is still available in 2026, at 20% of the federal EITC, so it pays 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It asserted that Virginia has no state EITC. Virginia has a refundable EITC equal to 20% of the federal EITC, which here is 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the old 15% Virginia refundable EITC rate on a federal EITC rounded to $40, getting $6. The current 20% rate on the exact $39.78 gives $7.96." +us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,It tied Virginia refundable credits to having state tax liability and to having dependents. Virginia's refundable EITC pays a childless worker with zero liability 20% of the federal EITC: 20% × $39.78 = $7.96. +us,scenario_066,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It claimed Virginia's refundable EITC applied only to tax years 2022–2025 at 15%. In fact the refundable EITC still applies in 2026 at 20% of the federal EITC, so it pays 20% × $39.78 = $7.96." +us,scenario_066,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no Virginia refundable credits apply and never considered the refundable EITC. That credit pays 20% of the $39.78 federal childless EITC, or $7.96." +us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,It said Virginia's EITC requires tax liability and offers no refundable payment to a childless filer at this income. The refundable EITC option pays 20% of the $39.78 federal EITC ($7.96) whether or not there is liability. +us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It treated having no qualifying children as disqualifying for Virginia refundable credits. The childless federal EITC is $39.78, and Virginia's refundable EITC pays 20% of it, or $7.96." +us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It measured dependent1's ACA expansion eligibility on the parents' tax-unit MAGI (about $87,300 for 3 people) and also applied parental income and resources to the disability pathway. PolicyEngine puts dependent1 in the ADULT expansion category at 0.00×FPL, which is far under 138%, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It tested dependent1 against the 138% FPL adult limit using the whole tax unit's income. PolicyEngine measures dependent1's MAGI income level at 0.00×FPL in the ACA adult expansion category, so the parents' income does not disqualify dependent1." +us,scenario_067,dependent1_medicaid_eligible,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the whole filing unit as dependent1's MAGI household and counted $98,102, which combines parents' wages, pension and dependent1's disability benefits, against 138% FPL. PolicyEngine puts dependent1's MAGI income level at 0.00×FPL: the parents' income is not attributed and the disability benefits are not MAGI income. Dependent1 qualifies under the ACA adult expansion." +us,scenario_067,dependent1_medicaid_eligible,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted parental income toward the adult disabled tax dependent's Medicaid test. PolicyEngine evaluates dependent1 in the ACA adult expansion category at a MAGI income level of 0.00×FPL, with no parental income attributed, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It compared total household income with Indiana's Medicaid limit. PolicyEngine's test for dependent1 is the ACA adult expansion category at dependent1's own MAGI income level of 0.00×FPL, which is well under 138%." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It folded dependent1 into the parents' tax household and applied the combined income to the Medicaid limit. PolicyEngine puts dependent1 in the ADULT expansion category with a MAGI income level of 0.00×FPL, so the parents' income is irrelevant and dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It judged that dependent1's disability-benefit income puts a childless adult over Indiana's limit. Indiana's ACA expansion covers childless adults up to 138% FPL, and PolicyEngine counts dependent1's MAGI income level as 0.00×FPL because disability benefits are not MAGI income. Even the full $10,800 is under 138% FPL for one person." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied the tax unit's $87,302 total to dependent1's Medicaid test. PolicyEngine puts dependent1's MAGI income level at 0.00×FPL in the ACA adult expansion category, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It built a 3-person Medicaid household that includes the joint-filing parents and put dependent1 at about 317% FPL. PolicyEngine puts dependent1 in the ADULT expansion category at 0.00×FPL, under the 138% limit." +us,scenario_067,dependent1_medicaid_eligible,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added the parents' wages and pension to dependent1's disability benefits ($98,102) and tested the total against 138% FPL for 3 people. PolicyEngine counts neither the parents' income nor the disability benefits in dependent1's MAGI, which it puts at 0.00×FPL, so dependent1 qualifies under the ACA adult expansion." +us,scenario_067,dependent1_medicaid_eligible,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It checked only the SSI-linked disability pathway and never tested the ACA adult expansion category. PolicyEngine assigns dependent1 (age 23, under 65) to that category at a MAGI income level of 0.00×FPL, so dependent1 is eligible whether or not SSI criteria are met." +us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,It declined to evaluate eligibility and defaulted to 0 because no SSI or Medicaid qualification was listed. Applying Indiana's ACA adult expansion to a 23-year-old whose MAGI income level is 0.00×FPL gives eligibility. +us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It tested dependent1 against the adult expansion limit using the 3-person household's counted income, then found no SSI or other categorical pathway. PolicyEngine puts dependent1 in the ACA adult expansion category at 0.00×FPL, so the expansion pathway alone makes dependent1 eligible." +us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It used household income for dependent1's Indiana Medicaid test. PolicyEngine gives dependent1 a MAGI income level of 0.00×FPL in the ACA adult expansion category, which is well under 138%." +us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It applied household income and listed resources to dependent1's Medicaid pathways. The ACA adult expansion category has no resource test, and PolicyEngine measures dependent1's MAGI income level at 0.00×FPL, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It placed dependent1's income above the limit by measuring the household's income. PolicyEngine puts dependent1 in the ADULT expansion category at a MAGI income level of 0.00×FPL, under 138%." +us,scenario_067,dependent1_medicaid_eligible,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It counted dependent1's disability benefits as income and applied a household resource test. The ACA adult expansion category has no asset test, and PolicyEngine puts dependent1's MAGI income level at 0.00×FPL, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It chose the right program, Indiana HIP for adults up to 138% FPL, but tested dependent1 against the household's roughly $87K MAGI. PolicyEngine puts dependent1's own MAGI income level at 0.00×FPL, so dependent1 qualifies." +us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It wrongly treated Indiana as limited on expansion-type pathways and tested dependent1 against 3-person household MAGI of about $87,302. Indiana has adopted the ACA expansion, and PolicyEngine puts dependent1 in the ADULT category at 0.00×FPL, so dependent1 is eligible." +us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed dependent1's $10,800 in disability benefits exceed Indiana's adult Medicaid limit. The benefits are not MAGI income, so PolicyEngine puts dependent1 at 0.00×FPL, and $10,800 alone is still under 138% FPL for a single adult." +us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Treated the disabled flag plus generic disability benefits as satisfying a disability-based Medicare rule. Medicare's under-65 disability pathway requires 24 months of Social Security disability entitlement, and this household records neither SSDI entitlement nor any months of receipt." +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Relabeled the generic $10,800 'disability benefits' as SSDI and then declared the 24-month waiting period satisfied because the disabled status is constant for the year. A status that holds all year proves no prior 24 months of SSDI entitlement, and the prompt makes unlisted SSDI receipt and waiting-period months false, so the Medicare disability pathway fails." +us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"Assumed the disability benefits were SSDI and granted Medicare despite naming the 24-month waiting period itself. The household lists no SSDI entitlement and no months of receipt, so under the prompt's unlisted-is-false rule neither the SSDI condition nor the waiting period is met, and a 23-year-old is not Medicare-eligible." +us,scenario_067,dependent1_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Treated disability status alone as conferring Medicare eligibility. Under-65 Medicare requires 24 months of Social Security disability entitlement or ESRD, and neither appears for this 23-year-old." +us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"Read generic annual disability benefits as support for under-65 Medicare eligibility. It never checked the actual requirement of 24 months of Social Security disability entitlement, which the household does not record." +us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"Applied disability-based Medicare because the dependent is disabled and receives disability benefits. That pathway requires 24 months of SSDI entitlement, and the listed disability benefits are not SSDI entitlement with a completed waiting period." +us,scenario_067,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Invoked a 'modeled disability-based Medicare pathway' triggered by reported disability plus disability benefits. That pathway needs 24 months of Social Security disability entitlement, and this household records none, so the dependent does not qualify." +us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"Claimed that PolicyEngine treats a disabled person receiving disability benefits as Medicare-eligible under 65. Eligibility under 65 requires 24 months of Social Security disability entitlement or ESRD, and neither is present here." +us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"Concluded Medicare eligibility 'under disability rules' from the disabled flag and generic disability benefits. It skipped the required 24 months of Social Security disability entitlement, which the household does not have." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning reached $5,347 after using a $32,600 standard deduction (the 2026 figure is $32,200) and an auto-loan interest deduction that the facts do not support. It then submitted $3,391, a number no step of its own derivation produces." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Everything else matches the reference: $32,200 standard deduction, 10%/12% brackets at $24,800, and the $500 dependent credit. But it also subtracted the $1,845 OBBBA auto-loan interest deduction, though no qualified new, US-assembled vehicle loan is established. That cut taxable income to $53,257 and understated tax by exactly $221.40 (12% × $1,845)." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction instead of the 2026 $32,200, which gave taxable income of $58,802 instead of $55,102. It then subtracted an invented ~$1,266 of nonrefundable credits instead of the single $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its first pass, with a ~$32,300 deduction and a 10% bracket to ~$24,000, landed near the reference at ~$5,620. It then switched to 2025 bracket edges ($23,850) and submitted $6,280 with no arithmetic supporting that figure, instead of $55,102 taxed at the 2026 brackets less the $500 credit." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized using ~$6,500 of imputed mortgage interest (unlisted, so $0) and claimed its ~$23,121 itemized total beat a ~$32,300 standard deduction, which is false by its own numbers. It then replaced its computed $6,722 with an unsupported $5,594 instead of using the $32,200 standard deduction and taxable income of $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the reference path exactly: taxable income $55,102, tax ~$6,116, less the $500 other-dependent credit = ~$5,616. It then discarded that result for an unexplained 'bracket estimate' of $6,104." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It applied the correct $32,200 standard deduction, 2026 brackets, and $500 other-dependent credit, but also deducted the $1,845 auto-loan interest with no qualified vehicle loan established. That lowered taxable income to $53,257 and tax by $221.40." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 figure to a $30,000 standard deduction instead of the enacted 2026 $32,200 and used a guessed $24,500 10% bracket edge instead of $24,800. That overstated taxable income at $57,302 and gave $5,886 after the $500 credit." +us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"With a $32,600 standard deduction (the 2026 amount is $32,200) it derived $6,068 less the $500 other-dependent credit. It then submitted $10,500, citing 'self-employment-like adjustments' and 'benefit phase-outs' that do not apply to this all-wage household." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put the dependent's $10,800 disability benefits into the joint AGI ($98,102 instead of $87,302) and used a $30,600 standard deduction instead of $32,200. It also denied the $500 credit for other dependents because the dependent is 23, but a disabled dependent of any age qualifies for that credit." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used a $16,500 standard deduction, three $5,250 personal exemptions, and a 15% bracket. OBBBA made the $32,200 standard deduction, zero exemptions, and the 10%/12% brackets permanent. It also omitted the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: itemized medical plus Indiana tax, three $5,300 personal exemptions, and a 15% bracket. The correct rules are the permanent $32,200 standard deduction, no exemptions, and 12% above $24,800. It also dropped the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,486.84 matches this computation: $87,302 − $31,500 (the 2025 standard deduction) − $1,845 auto-loan interest = $53,957, taxed with a $24,400 10% bracket, less $500. The auto-loan deduction does not apply without a qualified vehicle loan, and the 2026 parameters are a $32,200 deduction and a $24,800 bracket edge." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It modeled a TCJA sunset: $15,150 of personal exemptions, a $15,900 standard deduction, and a 15% bracket. It also added 85% of the dependent's disability benefits ($9,180) to the parents' AGI and omitted the $500 other-dependent credit. The correct result uses AGI of $87,302 less the permanent $32,200 standard deduction." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reported taxable income of ~$60,400, though $87,302 minus its own $31,200 deduction is $56,102 and the correct figure is $55,102. It also treated the household as having no qualifying dependents, dropping the $500 credit for the disabled 23-year-old." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, subtracting $16,050 of personal exemptions plus itemized deductions and taxing in a 15% bracket. The correct rules are the permanent $32,200 standard deduction and 10%/12% brackets. It also omitted the $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules with $15,150 of personal exemptions and a 15% bracket. It itemized with $4,500 of mortgage interest the facts do not list (unlisted inputs are $0). The correct approach uses the $32,200 standard deduction and subtracts the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $7,534 is higher than even the reference's pre-credit tax of $6,116.24. It matches taxable income of about $66,900: the dependent's $10,800 benefits added to AGI ($98,102), minus a ~$31,200 standard deduction, with no $500 other-dependent credit. The correct base is $87,302 − $32,200 = $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 $32,200 and the 2025 $23,850 10% bracket edge instead of $24,800, overstating taxable income at $56,502. The $500 credit was applied correctly." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,912.24 matches taxable income of $57,302, from a $30,000 standard deduction instead of $32,200, taxed with a $23,200 10% bracket edge instead of $24,800, before the correct $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction instead of the 2026 $32,200 and the 2025 $23,850 10% bracket edge instead of $24,800. That gave taxable income of $57,302 and $5,899.24 after the correct $500 credit." +us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $30,000 standard deduction instead of the enacted 2026 $32,200 and used a $23,600 10% bracket edge instead of $24,800. That overstated taxable income at $57,302, though it correctly subtracted the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied any dependent credit by applying the qualifying-relative gross income test to the dependent's $10,800 of benefits. A permanently and totally disabled child has no age limit as a qualifying child and faces no gross income test, so the $500 other-dependent credit applies. It also used a $25,200 10% bracket edge instead of $24,800." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $4,567 is $1,049 below the reference. That equals removing about $8,700 of taxable income beyond the $32,200 standard deduction, and no deduction or credit in the facts supports it. The correct figure is $55,102 taxed at 10%/12% ($6,116.24) less only the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It stacked the standard deduction with itemized mortgage and medical deductions. Mortgage interest is unlisted (so $0), and the $16,621 of medical deductions is well below the $32,200 standard deduction. That understated taxable income by roughly $5,600 and gave $4,940." +us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated 85% of the dependent's $10,800 disability benefits ($9,180) as taxable Social Security on the parents' joint return. Those benefits belong to the dependent and stay out of the joint AGI. This raised taxable income to $64,282 and overstated tax by exactly $1,101.60 (12% × $9,180)." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its round $8,500 exceeds even the pre-credit tax of $6,116.24. It matches taxable income about $20,000 above the correct $55,102 ($87,302 − $32,200) with no $500 other-dependent credit subtracted." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules with $15,750 of personal exemptions, itemized medical plus SALT, and higher brackets, and it allowed no nonrefundable credits. Under permanent OBBBA law, the $32,200 standard deduction and 10%/12% brackets apply, less the $500 credit for the disabled dependent." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using a $16,600 standard deduction, three $5,300 personal exemptions, and a 15% bracket. The permanent 2026 rules are a $32,200 standard deduction, no exemptions, and 12% above $24,800. It also omitted the $500 other-dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law: $15,900 of personal exemptions, itemized deductions, and a 15% bracket. It also declared the other-dependent credit expired, but OBBBA made the $32,200 standard deduction, 10%/12% brackets, and $500 other-dependent credit permanent." +us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an inflated pre-TCJA $17,291 standard deduction and a 15% bracket, and it skipped the personal exemptions its own sunset framework requires. That gave taxable income of $70,011. The correct taxable income is $55,102, after the permanent $32,200 standard deduction, less the $500 dependent credit." +us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents and omitted the $500 credit for other dependents for the disabled 23-year-old, who is a qualifying child regardless of age. It also used the 2025 ~$31,500 standard deduction instead of 2026's $32,200." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 $32,200 and a $24,400 10% bracket edge instead of $24,800, overstating taxable income at $56,502. The $500 other-dependent credit was applied correctly." +us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It added $9,180 of the dependent's disability benefits to the joint AGI as taxable Social Security, though the benefits belong to the dependent. It also subtracted the $1,845 auto-loan interest deduction with no qualified vehicle loan established. That produced taxable income of $62,437 instead of $55,102." +us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed a $2,000 nonrefundable CTC for the 23-year-old disabled dependent, but the CTC requires a child under 17, so only the $500 credit for other dependents applies. It also added the dependent's $10,800 benefits to AGI and used a $30,000 standard deduction and $23,850 bracket edge instead of $32,200 and $24,800." +us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the pre-credit tax exactly at $6,116.24 on $55,102. It then applied no nonrefundable credit because the dependent is too old for the CTC, missing that the disabled 23-year-old is a qualifying child for the $500 credit for other dependents." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the dependent's $10,800 disability benefits as taxable income on the parents' return (AGI $98,102 instead of $87,302). It also used a projected $31,200 standard deduction and a $24,150 bracket edge instead of $32,200 and $24,800, overstating taxable income at $66,902." +us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a fabricated $10,000 nonrefundable CTC that wiped out the tax. The household has no child under 17, so the only nonrefundable credit is the $500 credit for the disabled adult dependent. Its taxable income of $58,150 also exceeds the correct $55,102." us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0." us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC." us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0." @@ -5037,87 +5520,96 @@ us,scenario_067,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,Fal us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule and scope — Social Security and Medicare on the head's $82,360 only, no Additional Medicare Tax — and then submitted a number that rule does not produce: $5,727.14 is 6.95% of $82,360, equivalently 7.65% of only $74,865, i.e. a base $7,495 below the wages it named. The combined 7.65% employee rate on $82,360 yields $5,106.32 + $1,194.22 = $6,300.54, so the answer understates by $573.40 through a slipped multiplication rather than any rule error." us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($82,360 × 0.062 = $5,106 and × 0.0145 = $1,194, summing to $6,300) and correctly ruled out Additional Medicare Tax, then discarded its own sum in favor of a fabricated bracket, writing ""total approximately 6300-6400, rounded to 6376."" That invented rounding overstates the exact $6,300.54 by $75.46; carrying its own arithmetic to the cent would have produced the reference value." us,scenario_067,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated the dependent's generic is_disabled flag as satisfying SSI's disability criterion, but SSI eligibility runs off the separate is_ssi_disabled determination, which is unlisted and therefore false, leaving is_ssi_aged_blind_disabled=False for the 23-year-old as well as the 60-year-old head and 58-year-old spouse. Having skipped the categorical gate, it spent its entire reasoning on deeming and the income test — $900/mo unearned income less the $20 general exclusion against a $994 FBR — producing $1,368 for a person who is not an SSI recipient at all." -us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model assumed the dependent's disability status establishes SSI aged/blind/disabled eligibility, when PolicyEngine requires the SSI-specific disability input, which is unlisted and therefore false for all three members (head 60, spouse 58 are also under 65). It then computed a resource test and an income test it should never have reached, subtracting $10,560 countable unearned income from a $11,894 annual FBR (itself an arithmetic slip: $991/mo × 12 = $11,892) to reach $1,334 instead of $0." -us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model went straight to the SSI income formula — $900/mo unearned income less the $20 general exclusion against a $991/mo FBR — without ever testing the categorical requirement that a recipient be 65+, blind, or disabled for SSI purposes. The dependent's listed is_disabled flag is not the SSI disability determination, which is unlisted and thus false, and the head (60) and spouse (58) are under 65, so is_ssi_aged_blind_disabled is False for every member and the correct answer is $0.00, not $1,332." -us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model inferred an unlisted fact — that the $10,800 in disability benefits is 'likely SSDI' — and used it plus the generic disability flag to grant SSI categorical eligibility, but SSI keys on its own is_ssi_disabled input, which is unlisted and therefore false, so no member satisfies the aged/blind/disabled test. Its $1,329 is the pure income-test residual ($11,889 assumed annual FBR minus $10,560 countable unearned income) computed for someone who never cleared the eligibility gate." -us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model never applied SSI's categorical aged/blind/disabled test, which is False for all three members because the SSI-specific disability input is unlisted (and the head at 60 and spouse at 58 are under 65), so the answer is $0.00 rather than any income-test residual. Its $824 compounds two further errors within the test it should not have run: it applied the $20 general income exclusion once annually ($10,800 − $20 = $10,780) instead of $20 per month ($240/year), and used $967/mo, the 2025 federal benefit rate, for a 2026 tax year." -us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model declared the 23-year-old 'the only SSI-eligible' member on the strength of the listed disability flag, but PolicyEngine's SSI eligibility requires the distinct is_ssi_disabled determination, unlisted and hence false, so is_ssi_aged_blind_disabled=False for all three people. It therefore ran the income test it should have skipped — $880 countable monthly unearned income against a $994 FBR — yielding $1,368 where the correct value is $0.00." -us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model computed only the SSI benefit formula ($11,928 annual FBR less $10,560 countable disability income after a $240 annual general exclusion) and never verified the categorical gate that a recipient be 65+, blind, or disabled for SSI purposes. The dependent's generic disability flag does not set SSI's own disability input, which is unlisted and therefore false, and the head (60) and spouse (58) fail the age test, so SSI is $0.00 rather than $1,368." -us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model's reasoning addressed only resources and deeming ('no listed own assets', 'parental income is not deemed to an adult') and assumed the dependent's disability status carries SSI eligibility, when SSI requires the separate is_ssi_disabled input that is unlisted and therefore false. With is_ssi_aged_blind_disabled=False for the 23-year-old, the 60-year-old head, and the 58-year-old spouse, the $11,928-minus-$10,560 income-test residual of $1,368 is moot and the correct amount is $0.00." -us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model checked resources ('under 2000 limit') and in-kind support but skipped the threshold aged/blind/disabled test, which fails for every member: the SSI disability input is unlisted and therefore false for the 23-year-old, and the head (60) and spouse (58) are under 65. Its $1,112 also misapplies the general income exclusion as $20 per year ($10,800 − $20 = $10,780) rather than $20 per month, on top of running an income test that never should have been reached." -us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"The model equated the dependent's listed is_disabled flag with SSI's disability criterion; PolicyEngine determines SSI eligibility from the separate is_ssi_disabled input, unlisted and therefore false, so is_ssi_aged_blind_disabled=False for all three members and no one reaches the benefit formula. Its $1,368 is exactly the $994 FBR minus $880 countable monthly income times twelve — the arithmetic is internally consistent but applied to an ineligible person, and the correct value is $0.00." -us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model devoted its reasoning to deeming ('parental income is not deemed after age 18') and the income test, assuming SSI categorical eligibility from the dependent's disability status rather than from SSI's own is_ssi_disabled determination, which is unlisted and therefore false. With the head at 60 and the spouse at 58 also failing the 65+ test, is_ssi_aged_blind_disabled is False for all three and SSI is $0.00, not the $994 − $880 = $114/mo it computed." -us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"The model correctly excluded the head and spouse on age but granted the 23-year-old SSI eligibility from the generic disability flag instead of SSI's own is_ssi_disabled input, which is unlisted and therefore false, so no member is aged/blind/disabled and SSI is $0.00. Its $12,156 additionally uses $1,913/month as the federal benefit rate — that figure is the SSI break-even monthly earned-income limit, roughly double the actual FBR of about $1,000/month — and reduces it dollar-for-dollar by the $10,800 unearned income with no $20 general exclusion." -us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Invented a $500 additional exemption for a disabled dependent; Indiana's $500 add-on belongs to a taxpayer or spouse age 65+ with federal AGI under $40,000, and no Indiana exemption attaches to a dependent's disability, so exemptions total $3,000, not $3,500. It then applied 2.90%, the rate that takes effect in 2027, instead of 2026's 2.95%, and finally discarded its own $2,430 arithmetic to submit $2,295 — an effective 2.74% of its own stated $83,802 base." -us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started Indiana's base from federal taxable income by subtracting the $28,700 federal MFJ standard deduction, when Indiana starts from federal AGI and grants no standard deduction — the base is $87,302 less $3,000 of $1,000-per-person exemptions = $84,302. It then fabricated Indiana health-insurance-premium and medical-expense credits and a $245 dependent exemption that do not exist, and submitted $4,168, a figure larger than its own $1,846 gross tax computation." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied a self-described 'projected' 3.12% rate when Indiana's statutory schedule fixes 2026 at 2.95%. It also swapped the dependent's $1,000 base exemption for the $1,500 dependent-child exemption, which requires a child under 19 or under 24 and a full-time student, so the 23-year-old non-student supports $1,000 and total exemptions are $3,000." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed the reference exactly — $87,302 less $3,000 of exemptions = $84,302 at 2.95% = $2,486.91 — then abandoned that result by adding a $1,500 disabled-dependent exemption Indiana does not provide, since the $1,500 add-on requires a dependent child under 19, or under 24 and a full-time student. It then submitted $2,620, which matches neither its $2,486.91 nor its revised $2,443, and corresponds to a 3.11% effective rate on the $84,302 base." -us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Used 2.90%, Indiana's 2027 rate, in place of the 2026 rate of 2.95%, and rounded the $84,302 base down to 'roughly 84,000'. Its own stated inputs yield $2,436, so the submitted $2,410 is below even its own derivation." -us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Deducted the entire $4,942 private pension under a nonexistent Indiana pension exclusion — the $16,000 deduction it invoked covers federal civil service annuities for annuitants age 62+, not private pensions, and the head is 60. It compounded that with a $1,500 disabled-dependent exemption limited to dependent children under 19 (or under 24 and full-time students) and taxed at 3.05%, the 2024 rate rather than 2026's 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Stacked $5,000 of exemptions by combining a $1,500 dependent exemption with a second $1,500 'qualifying child' amount, when Indiana allows $1,000 per person here for $3,000 total, and taxed at 3.05%, the 2024 rate. It then discarded its own $2,510 result and submitted $2,650 as an adjustment for 'county/local considerations' the question explicitly excludes." -us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Pulled the dependent's $10,800 of disability benefits into the joint filers' AGI, giving $98,102 when those benefits are the dependent's own income and never enter the filers' federal AGI of $87,302. It then subtracted a $4,000 Indiana MFJ standard deduction that does not exist — Indiana permits only the $1,000-per-exemption subtractions — and applied 3.00%, the 2025 rate." -us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Built the base exactly right at $87,302 less $3,000 of exemptions = $84,302, then applied 3.00%, Indiana's 2025 rate. The statutory step-down sets 2026 at 2.95%, which converts its $2,529.06 into $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Added a $1,500 additional exemption for a 'disabled child'; Indiana's $1,500 add-on requires a dependent child under 19, or under 24 and a full-time student, and attaches to no disability status, so exemptions are $3,000 and the base is $84,302. It also used 3.00%, the 2025 rate, rather than 2026's 2.95%, and the two errors offset to land $2.85 below the reference." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Cited 3.05%, Indiana's 2024 rate, and described applying it to AGI with no exemption subtraction, a method that produces $2,662.71 rather than the $2,489 it submitted. The 2026 computation is $87,302 less three $1,000 exemptions = $84,302 at 2.95% = $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Used the correct 2026 rate of 2.95% but inflated exemptions to $4,500 by adding a $1,500 dependent-child amount. That additional exemption requires a dependent child under 19, or under 24 and a full-time student, so this 23-year-old non-student carries only the $1,000 base exemption and the taxable base is $84,302." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Derived the base correctly as federal AGI of $87,302 less three $1,000 exemptions = $84,302, then taxed it at 3.05%, Indiana's 2024 rate. The 2026 rate under the statutory step-down is 2.95%, giving $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Submitted $5,382 with no computation behind it; that is 6.17% of the $87,302 AGI and 6.38% of the $84,302 Indiana base, more than double Indiana's 2.95% flat rate for 2026. The correct derivation, $84,302 at 2.95%, yields $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Derived the base exactly — $87,302 less $1,000 exemptions for head, spouse, and dependent = $84,302 — then taxed it at 3.00%, Indiana's 2025 rate. The 2026 rate is 2.95%, turning $2,529.06 into $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,529.06 is exactly $84,302 times 3.00%, so it carried Indiana's 2025 rate into 2026 on an otherwise correct base of federal AGI less three $1,000 exemptions. The 2026 rate is 2.95%, giving $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied 2.75%, a rate absent from Indiana's schedule of 3.05%/3.00%/2.95%/2.90% for 2024 through 2027, and applied it to the full $87,302 AGI without subtracting any exemptions. Indiana grants $1,000 each for head, spouse, and the dependent, so the base is $84,302 and the 2026 rate is 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Denied the dependent's $1,000 exemption on a gross-income test, but a permanently and totally disabled child is a qualifying child at any age with no gross-income limit, so all three $1,000 exemptions apply for $3,000 and the base is $84,302. It also used 2.90%, the 2027 rate, instead of 2026's 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Submitted $3,501 with no derivation, invoking a 'standard deduction' Indiana does not grant; that figure is 4.01% of the $87,302 AGI, far above Indiana's 2.95% flat rate for 2026. The computation is $87,302 less $3,000 of $1,000-per-person exemptions = $84,302 at 2.95% = $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Reduced the base with federal-style mortgage-interest and medical-expense deductions and unspecified Indiana nonrefundable credits, none of which exist in Indiana's individual income tax, where the sole subtraction is $1,000 per exemption. Its $2,183 is 2.50% of AGI, against the correct $84,302 at 2.95% = $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Applied the correct 2026 rate of 2.95% to a base understated by $1,500 because it claimed $4,500 of exemptions. Indiana's additional $1,500 dependent exemption requires a dependent child under 19, or under 24 and a full-time student, so the 23-year-old non-student supports only the $1,000 base exemption and exemptions total $3,000." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,619.06 is exactly 3.00% of the full $87,302 AGI, so it never subtracted the $3,000 of taxpayer, spouse, and dependent exemptions its own explanation claims to apply. It also used the 2025 rate; 2026 is 2.95% on the $84,302 base." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Used the correct 2.95% 2026 rate but subtracted $4,500 of exemptions by adding a 'qualifying-child' amount for a 23-year-old. Indiana's additional $1,500 exemption is limited to a dependent child under 19, or under 24 and a full-time student, so total exemptions are $3,000 and the base is $84,302." -us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Subtracted $4,500 of exemptions rather than $3,000 by adding the $1,500 dependent-child amount that requires a child under 19 or a full-time student under 24, and described excluding $9,180 of federally taxable disability benefits that were never in the filers' $87,302 federal AGI. At the correct base of $84,302, the 2.95% rate gives $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Submitted $2,200 with no computation; that is 2.52% of the $87,302 AGI and 2.61% of the $84,302 Indiana base, short of Indiana's 2.95% flat rate for 2026. Federal AGI less three $1,000 exemptions taxed at 2.95% produces $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Derived the $84,302 base correctly from federal AGI less $3,000 of personal exemptions, then taxed it at 3.05%, Indiana's 2024 rate. The statutory schedule drops to 2.95% for 2026, giving $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Derived the $84,302 base correctly — federal AGI less $1,000 exemptions for three people — but applied a 3.00% rate, which is Indiana's 2025 figure. The 2026 rate is 2.95%, producing $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Applied its rate to the full $87,302 of federal AGI, asserting 'no listed modifications', when Indiana subtracts $1,000 per exemption for the head, spouse, and dependent to reach a base of $84,302. It also used 2.90%, the 2027 rate, instead of 2026's 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Used 2.90%, Indiana's 2027 rate, rather than the 2026 rate of 2.95%, and counted only $2,000 of exemptions for the two filers, omitting the $1,000 Indiana grants for each dependent claimed federally. The correct base is $84,302 and the tax is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no substantive computation reached the answer contract. The required derivation is $87,302 of federal AGI less $3,000 of $1,000-per-person exemptions = $84,302, taxed at Indiana's 2026 rate of 2.95% for $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Applied the correct 2.95% rate but built a $5,000 exemption stack — $1,500 for the dependent plus another $1,500 for a 'disabled child'. Indiana gives this 23-year-old non-student dependent $1,000 and provides no disability-based exemption, so exemptions total $3,000 and the base is $84,302." -us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Added the dependent's $10,800 of disability benefits to the filers' AGI, which belongs to the dependent and never enters the joint federal AGI of $87,302, and miscomputed that sum as $97,202. It then subtracted a $3,800 Indiana MFJ standard deduction that does not exist and taxed at 3.05%, the 2024 rate rather than 2026's 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Claimed $5,000 of exemptions by giving the dependent $1,500 plus a $1,500 disabled-dependent add-on; Indiana's additional $1,500 exemption applies only to a dependent child under 19, or under 24 and a full-time student, and none attaches to disability, so exemptions total $3,000. It also used 2.90%, the 2027 rate, in place of 2026's 2.95%." -us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Reached the correct $84,302 base but taxed it at 3.05%, the 2024 rate, and then subtracted $474.42 of invented Indiana nonrefundable credits — a '$100 per dependent credit' and $374.42 of unspecified 'typical' credits — none of which exist for this household. At 2.95% with no credits, the tax is $2,486.91." -us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Included the dependent's $10,800 of disability benefits in Indiana AGI, which is the dependent's own income and stays out of the filers' $87,302 federal AGI, then subtracted $9,000 while describing it as $3,000 of exemptions ($98,102 less $3,000 is $95,102, not the $89,102 it used). It also applied 3.05%, the 2024 rate, instead of 2026's 2.95% on the correct $84,302 base." +us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as categorically SSI-eligible on disability and computed (994 − 880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, as it does for the head (60) and spouse (58), so no member passes the aged/blind/disabled gate and SSI is $0 before the income test." +us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the 23-year-old as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and for both adults, so SSI's categorical test fails for every member and the benefit is $0." +us,scenario_067,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Assumed Dependent 1 meets SSI's disability criterion and netted $10,560 of countable unearned income against an $11,928 annual FBR to get $1,368. The engine's is_ssi_aged_blind_disabled=False for Dependent 1 (and for the head and spouse, both under 65) stops SSI at the categorical step, giving $0." +us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability and subtracted $10,560 of countable income from an understated 2026 FBR of about $991/month, getting $1,334. The engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0 and the FBR and income steps never apply." +us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as SSI-disabled and computed (991 − 880) × 12 = $1,332, using an FBR below the 2026 rate of $994. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, the head (60) and the spouse (58), so no one clears SSI's categorical gate and SSI is $0." +us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as a disabled adult eligible for SSI and computed $11,889 − $10,560 = $1,329 from an assumed $990.75/month FBR. The engine evaluates is_ssi_aged_blind_disabled=False for all three members, so SSI is $0 before any unearned-income offset." +us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. Separately, it used the 2025 FBR of $967/month instead of the 2026 rate and subtracted the $20 general exclusion once a year instead of monthly ($240/year), which gives $11,604 − $10,780 = $824." +us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Declared the disabled 23-year-old the only SSI-eligible member and paid $114/month × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 too, so no member meets SSI's categorical requirement and SSI is $0." +us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and netted $10,560 of countable disability income against an $11,928 FBR to get $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 and both adults, so SSI stops at the categorical test with $0." +us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Treated the disabled adult dependent as categorically SSI-eligible with no parental deeming and computed $11,928 − $10,560 = $1,368. The engine's is_ssi_aged_blind_disabled is False for every member, so the aged/blind/disabled gate yields $0 SSI." +us,scenario_067,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as SSI-disabled and paid (994 − 880) = $114/month, or $1,368 a year. The engine evaluates is_ssi_aged_blind_disabled=False for Dependent 1, the head (60) and the spouse (58), so SSI is $0 before the income offset." +us,scenario_067,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability and reduced the $994 FBR by $880 of countable income to get $114/month ($1,368/year). The engine's is_ssi_aged_blind_disabled is False for all members, so no one passes SSI's categorical test and the benefit is $0." +us,scenario_067,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Named Dependent 1 the only categorically eligible member and computed $114/month × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 as well as for the head and spouse, so SSI is $0 at the eligibility gate." +us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. Separately, it subtracted the $20 general exclusion once a year instead of monthly ($240/year) and used an $11,892 annual FBR, which gives $11,892 − $10,780 = $1,112." +us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and computed ($994 − $880) × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and both adults, so SSI's categorical requirement fails for everyone and SSI is $0." +us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as an SSI-disabled adult with no parental deeming and paid (994 − 880) × 12 = $1,368. The engine evaluates is_ssi_aged_blind_disabled=False for every member, so SSI is $0 before any income or deeming step." +us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on disability, which fails at the engine's aged/blind/disabled gate (is_ssi_aged_blind_disabled=False for every member), so SSI is $0. On top of that, it used an invented $1,913/month FBR, well above the 2026 individual rate of $994 and the couple rate of about $1,491, and skipped the $20 general exclusion, which gives $22,956 − $10,800 = $12,156." +us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It invented a $500 extra exemption for the disabled dependent, which gave $3,500 of exemptions instead of $3,000. It then applied a guessed 2.9% rate instead of 2.95% and subtracted unspecified 'minor deductions' to reach $2,295. The correct figure is $84,302 × 2.95% = $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income after subtracting the $28,700 federal standard deduction, but Indiana starts from federal AGI and allows no standard deduction. It also used a 3.15% rate, invented health-premium and medical credits, and then inflated the result to $4,168 with no computation. It should have used $1,000 exemptions for three people and the 2.95% rate." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It gave the 23-year-old dependent a $1,500 exemption instead of the $1,000 dependent exemption, for $3,500 total. It then used a 'projected' 3.12% rate instead of Indiana's 2026 rate of 2.95%. The correct tax is 2.95% of $84,302 = $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"Its reasoning computed the correct $84,302 × 2.95% = $2,486.91. It then switched to a nonexistent $1,500 'disabled dependent' additional exemption, but that exemption requires a child under 19 or a full-time student under 24, so disability does not qualify. It finally submitted $2,620, which matches neither of its own calculations." +us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted unspecified 'additional dependent-related amounts' beyond the three $1,000 exemptions. It also applied a 2.9% rate instead of Indiana's 2026 flat rate of 2.95%, which produced $2,410 instead of $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the full $4,942 pension under an Indiana 'age 60+ pension deduction' that does not exist, and it added a $1,500 disabled-dependent exemption, for $4,500 of exemptions. It also used the old 3.05% rate. The correct calculation keeps the pension in Indiana AGI and taxes $84,302 at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It claimed a $1,500 dependent exemption plus a $1,500 additional qualifying-child exemption, for $5,000 total, but the 23-year-old non-student only gets the $1,000 dependent exemption. It used the old 3.05% rate and then pushed its own $2,510 up to $2,650 with no computation." +us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put the dependent's $10,800 of disability benefits into Indiana AGI ($98,102) and subtracted a fictitious $4,000 MFJ standard deduction; Indiana has no standard deduction. It then used a 3.0% rate instead of 2.95%. The correct base is $87,302 minus $3,000 of exemptions." +us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It derived the correct Indiana taxable income of $84,302 ($87,302 AGI minus three $1,000 exemptions). It then applied a 3.0% rate instead of Indiana's 2026 flat rate of 2.95%, overstating the tax by $42.15." +us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It cited the correct 2.95% rate but subtracted a 'standard deduction', which Indiana does not have. Its $2,226.48 implies a taxable base of about $75,474, or about $11,800 of deductions, instead of the $3,000 of personal and dependent exemptions that leave $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It added a $1,500 additional exemption for a 'disabled child'. Indiana's additional dependent-child exemption applies only to children under 19 or full-time students under 24, and disability does not substitute. It then applied 3.0% instead of 2.95% to $82,802. The two errors partly offset each other, giving $2,484.06 instead of $2,486.91." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied the outdated 3.05% rate. Its $2,489 implies a base of about $81,600, which matches no Indiana exemption structure. The correct calculation is 2.95% of $84,302 ($87,302 AGI minus three $1,000 exemptions)." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,age_disability,False,"It used the correct 2.95% rate but subtracted about $4,500 of exemptions. That total includes a $1,500 additional dependent exemption the 23-year-old non-student does not qualify for, because it requires a child under 19 or a full-time student under 24. Exemptions are $3,000, leaving $84,302 of taxable income." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed the correct $84,302 base ($87,302 minus three $1,000 exemptions) but applied the outdated 3.05% rate instead of Indiana's 2026 flat rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation, and its $5,382 is more than double the correct $2,486.91. That implies an effective rate of about 6.4% on the $84,302 base, far above Indiana's 2.95% flat state rate, which consistent with adding county tax that the question excludes." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It derived the correct $84,302 taxable income from $87,302 AGI minus three $1,000 exemptions. It then applied a 3.0% rate instead of Indiana's 2026 flat rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,529.06 is exactly 3.0% of the correct $84,302 base, so it applied a 3.0% flat rate instead of Indiana's 2026 rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It skipped all Indiana personal and dependent exemptions ($3,000) and taxed the full $87,302 AGI. It also used a 2.75% rate instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It dropped the dependent's $1,000 exemption on the theory that the 23-year-old's $10,800 of disability benefits disqualify them as a dependent. The household states this person is a dependent, so three exemptions apply. It also used 2.9% instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no computation and referred to a 'standard deduction' Indiana does not have. Its $3,501 is about 4.0% of total AGI, far above the 2.95% flat rate applied to $84,302 after three $1,000 exemptions." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted standard deductions plus mortgage-interest and medical deductions, but Indiana starts from federal AGI and allows neither a standard deduction nor federal itemized deductions. Its $2,183 implies a base of about $74,000 instead of $84,302 at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,age_disability,False,"It used the correct 2.95% rate but subtracted $4,500 of exemptions, which includes a $1,500 additional dependent exemption. That exemption requires a child under 19 or a full-time student under 24, which this 23-year-old non-student dependent is not. Exemptions are $3,000, so taxable income is $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It said it subtracted $3,000 of exemptions, but its $2,619.06 is exactly 3.0% of the full $87,302 AGI, so it applied no exemptions. It also used 3.0% instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,age_disability,False,"It added a $1,500 'qualifying-child' exemption for the 23-year-old disabled dependent, which brought exemptions to $4,500. Indiana's additional dependent-child exemption requires a child under 19 or a full-time student under 24, so only the $1,000 dependent exemption applies and taxable income is $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It correctly excluded the disability benefits and used 2.95%, but it subtracted $4,500 of exemptions. That includes the $1,500 additional dependent-child exemption, which is limited to children under 19 or full-time students under 24, and this 23-year-old non-student is neither. Exemptions are $3,000." +us,scenario_067,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct 2.95% rate but a taxable income of $83,802, which means $3,500 of exemptions. It gave the dependent $1,500 instead of the $1,000 dependent exemption Indiana allows a 23-year-old non-student." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no computation and cited unspecified 'deductions'. Its $2,200 implies a base of about $74,600 at 2.95%, which means roughly $12,700 of deductions. Indiana allows only $3,000 of exemptions here and no standard deduction, leaving $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It computed the correct $84,302 base (federal AGI minus three $1,000 exemptions) but applied the outdated 3.05% rate instead of Indiana's 2026 rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly subtracted three $1,000 exemptions from $87,302 but applied a 3% flat rate instead of Indiana's 2026 rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,grok-4.7,llm_error,age_disability,False,"It used the correct 2.95% rate but added a $1,500 additional dependent exemption, for $4,500 total. That exemption applies only to children under 19 or full-time students under 24, so the 23-year-old non-student dependent does not qualify and taxable income is $84,302." +us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied the rate to the full $87,302 AGI without subtracting Indiana's $1,000 personal exemptions for head, spouse, and dependent. It also used 2.9% instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It subtracted only the two filers' $1,000 exemptions, omitting the dependent's $1,000 exemption, and applied 2.9% instead of Indiana's 2026 rate of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against the $2,486.91 reference." +us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,age_disability,False,"It used 2.95% but claimed a $1,500 dependent exemption plus a $1,500 'additional disabled-child' exemption, for $5,000 total. Indiana gives this dependent only the $1,000 dependent exemption, because the additional exemption requires age under 19 or full-time student status under 24." +us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It started from a wrong AGI of $97,202, which pulls in the dependent's disability benefits (and even then mis-adds them). It subtracted a nonexistent $3,800 Indiana standard deduction and applied the outdated 3.05% rate. The correct base is $87,302 minus $3,000 of exemptions, taxed at 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,age_disability,False,"It claimed a $1,500 dependent exemption plus a $1,500 disabled-dependent additional exemption, for $5,000 total. The 23-year-old non-student gets only the $1,000 dependent exemption. It also applied 2.9% instead of 2.95%." +us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It reached the right $84,302 base, but only by calling the $3,000 a standard deduction. It then applied the outdated 3.05% rate and subtracted an invented $100 dependent credit and about $374 of fabricated nonrefundable credits. Indiana has no such credits for this household." +us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of disability benefits to Indiana AGI, for $98,102, but those benefits are not part of the filers' AGI. It also used the outdated 3.05% rate instead of 2.95% on the correct $84,302 base." us,scenario_067,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It read the $73,000 educational assistance as employer-provided IRC §127 tuition benefits and added $73,000 − $5,250 = $67,750 to wages, inflating AGI to $101,834; PolicyEngine's educational_assistance is student financial aid that never enters AGI, so AGI is the $34,083.53 of employment income and taxable income is $34,083.53 − $16,100 = $17,983.53. Its own bracket math then produced $13,573, yet it submitted $8,258, a figure no step in its derivation supports." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It drove the answer to zero by subtracting an EITC of roughly $852 against a quantity the prompt defines as excluding EITC and all refundable credits, and it compounded that with a stale $14,600 standard deduction and a fictitious $21,310 wage figure derived from 12 weeks unemployed even though the $34,084 is stated as the annual total. With the 2026 single standard deduction of $16,100, taxable income is $17,983.53 and tax is $1,240 + 12% × $5,583.53 = $1,910.02, all of it before any refundable credit." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It applied the IRC §127 $5,250 employer-tuition cap to the $73,000 educational assistance and added $67,750 to wages, producing AGI of $101,834 instead of the $34,083.53 of employment income; PolicyEngine's educational_assistance is student aid excluded from AGI entirely. It also used approximate brackets ($12,150/$49,350) rather than the 2026 thresholds of $12,400 and $50,400, so both the base and the rate schedule were wrong." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added $73,000 − $5,250 = $67,750 of ""taxable"" educational assistance to wages for AGI of $101,834, when educational assistance is excluded from AGI and AGI equals the $34,083.53 of employment income. Its bracket parameters ($16,100 standard deduction, 10% to $12,400, 12% to $50,400) were correct, so the entire $11,663 error comes from the educational-assistance inclusion." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived the correct answer inside its own reasoning — educational assistance excluded, $16,100 standard deduction, taxable income $17,984, tax = $1,240 + 12% × $5,584 ≈ $1,910 — and then submitted $2,372 instead. The $2,372 corresponds to no stated step; it is consistent only with taxing about $21,833 of income, i.e. a standard deduction near $12,250 that the model never invoked." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the §127 cap in gross income, raising AGI to $101,834 when educational assistance is excluded and AGI is $34,083.53. It layered on a guessed $15,750 standard deduction instead of the 2026 single amount of $16,100 and 2025-style brackets ($12,200/$46,600 instead of $12,400/$50,400), pushing income into a 22% bracket the household never reaches." -us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"After first adding $67,750 of educational assistance, it reversed course, correctly used wages of $34,084 less the $16,100 standard deduction, and computed $1,915 — then submitted $1,750 with no derivation. Its residual arithmetic error is the 10% bracket ceiling of $12,150 rather than the 2026 value of $12,400, but the submitted $1,750 matches neither of its own computations." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It treated the full $73,000 of educational assistance as taxable with no exclusion at all, giving AGI of $107,084 instead of $34,083.53, and used a $15,700 standard deduction rather than the 2026 single amount of $16,100. Excluding educational assistance leaves taxable income of $17,983.53, taxed entirely in the 10% and 12% brackets for $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made two AGI errors: it subtracted the $8,389 employer-sponsored insurance premium from wages, which is not an above-the-line adjustment to the stated $34,083.53 of employment income, and it added $67,750 of educational assistance that is excluded from AGI. With a further guessed $15,400 standard deduction, it taxed $78,045 when the correct taxable income is $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages and then applied a repealed pre-2018 structure — a $7,900 standard deduction plus a $5,050 personal exemption and 15%/25% brackets — none of which exist in 2026, where the single standard deduction is $16,100, personal exemptions are zero, and the rates above 10% start at 12%. The correct base of $17,983.53 sits entirely below the 12% bracket top of $50,400." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired for 2026 and used a $7,600 standard deduction plus a $5,100 personal exemption with a 15% second bracket; the 2026 law in force gives a $16,100 single standard deduction, no personal exemption, and a 12% second bracket starting at $12,400. It also subtracted the $8,389 ESI premium from wages, though AGI equals the full $34,083.53 of employment income." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used a $15,200 standard deduction instead of $16,100, applied a flat 10% to the entire $18,884 rather than the graduated 10%/12% schedule, and then subtracted roughly $1,749 of ""Child Tax Credit/other"" nonrefundable credits from a childless 31-year-old filer who qualifies for no CTC and no other nonrefundable credit. Removing that phantom credit and using the correct parameters yields $1,240 + 12% × $5,583.53 = $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed a 2026 TCJA sunset and used a combined $14,250 standard deduction plus personal exemption at a flat 10% rate; 2026 law provides a $16,100 standard deduction, no personal exemption, and 12% above $12,400. It also removed the $8,389 ESI premium from wages, but AGI is the full $34,083.53, giving taxable income of $17,983.53 and tax of $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It stacked three errors: adding the entire $73,000 of excluded educational assistance to income, subtracting the $8,389 ESI premium from wages, and applying a repealed pre-TCJA schedule ($8,300 standard deduction, $5,050 personal exemption, 15%/25% rates). The 2026 computation is $34,083.53 − $16,100 = $17,983.53 taxed at 10% to $12,400 and 12% above, for $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted $0 with no derivation. Wages of $34,083.53 exceed the 2026 single standard deduction of $16,100 by $17,983.53, and that taxable income produces $1,240 in the 10% bracket plus $670.02 in the 12% bracket; a zero answer is consistent only with treating the household as having no taxable income after deductions, which the $34,084 wage figure contradicts." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied a pre-TCJA 2026 schedule — an $8,300 standard deduction, a $5,100 personal exemption, a $11,150 10% bracket top and a 15% second rate — none of which apply in 2026, where the single standard deduction is $16,100, personal exemptions are zero, the 10% bracket runs to $12,400, and the next rate is 12%. Correct parameters give taxable income of $17,983.53 and tax of $1,910.02, not $20,684 and $2,545.10." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation, but $11,116 requires taxable income near $74,600 under the 2026 single brackets, which is reachable only by folding the $73,000 educational assistance into income. Educational assistance does not enter AGI, so AGI is $34,083.53, taxable income is $17,983.53, and the tax never leaves the 12% bracket, totaling $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its structure was right — wages only, standard deduction, graduated 10%/12% — but it used 2025 parameters: a $15,000 standard deduction and an $11,925 top of the 10% bracket instead of the 2026 values of $16,100 and $12,400. Those two parameter substitutions account for the entire $141.56 overstatement." -us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages for a base of $101,834, when educational assistance is excluded from AGI entirely, and then compounded that with 2024 parameters — a $14,600 standard deduction and $11,600/$47,150 bracket edges — instead of 2026's $16,100, $12,400 and $50,400. The correct taxable income is $17,983.53, yielding $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It got every structural step right — educational assistance excluded, AGI $34,084, standard deduction $16,100, taxable income $17,984 — but placed the top of the 10% bracket at $12,200 instead of the 2026 value of $12,400, shifting $200 from the 12% rate to the 10% rate. That single threshold error is the whole $4 gap between its $1,914 and the correct $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the income was too low to produce positive tax after the standard deduction, but $34,083.53 of wages exceeds the 2026 single standard deduction of $16,100 by $17,983.53. That taxable income is taxed at 10% on the first $12,400 and 12% on the remaining $5,583.53, giving $1,910.02 with no nonrefundable credits to offset it." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that wages of $34,084 less the standard deduction leave zero or negative liability, but the 2026 single standard deduction is $16,100, leaving $17,983.53 of taxable income. Applying the 10% and 12% brackets to that base gives $1,240 + $670.02 = $1,910.02, and no nonrefundable credit is available to a childless filer to reduce it." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a projected post-sunset 2026 regime — a combined standard deduction and personal exemption of about $13,650 with a 15% rate above $11,950 — when 2026 law provides a $16,100 single standard deduction, no personal exemption, and a 12% rate above $12,400. Correct parameters give taxable income of $17,983.53 and tax of $1,910.02, not $20,434 and $2,467.60." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the $5,250 §127 cap in taxable income, taking the base from $34,083.53 to $101,834; educational assistance is excluded from AGI, so taxable income is $17,983.53. Its $16,100 standard deduction and 2026 brackets were correct, so the entire $11,663.46 overstatement is the educational-assistance inclusion." -us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages for $107,084 of income, applying no exclusion at all, and then correctly used the $16,100 standard deduction and the 2026 single brackets. Because its parameters were right, the sole error is the inclusion: dropping the $73,000 leaves taxable income of $17,983.53 and tax of $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It correctly excluded the educational assistance and used the 10%/12% structure, but anchored on roughly $19,000 of taxable income, which corresponds to a $15,000 (2025) standard deduction rather than the 2026 single amount of $16,100, and to an $11,925 rather than $12,400 top of the 10% bracket. The correct base of $17,983.53 yields $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used a ""2025-like"" $15,000 standard deduction and the 2025 bracket edge of $11,925, producing taxable income of $19,084 instead of $17,983.53. Substituting the 2026 parameters — $16,100 standard deduction and a 10% bracket running to $12,400 — turns its $2,051.58 into $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed post-TCJA-sunset rules for 2026, using an $8,450 standard deduction plus a $5,300 personal exemption and a 15% second bracket; 2026 law gives a $16,100 single standard deduction, no personal exemption, and a 12% rate above $12,400. Its income treatment was right, so correcting the deduction and rate schedule alone moves $20,334 of taxable income down to $17,983.53 and the tax from $2,443 to $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from wages to reach AGI of $25,695, but AGI equals the full $34,083.53 of employment income with no premium adjustment, and it then applied a post-sunset $8,527 standard deduction with a 15% bracket instead of the 2026 $16,100 deduction and 12% bracket. The two errors partially offset, landing at $1,949 against the correct $1,910.02 on taxable income of $17,983.53." -us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive reasoning exists to evaluate. The required computation is $34,083.53 − $16,100 standard deduction = $17,983.53 taxable income, taxed at 10% to $12,400 plus 12% on $5,583.53, for $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It correctly excluded the $73,000 educational assistance but used a guessed $15,750 standard deduction (the 2025 figure) instead of the 2026 single amount of $16,100, and set the 10% bracket top at $12,200 rather than $12,400. Those two parameter substitutions raise taxable income to $18,334 and the tax to $1,956, against the correct $17,983.53 and $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Its income treatment was exactly right — educational assistance and ESI premiums excluded, taxable base from $34,084 of wages — but it used a $15,750 standard deduction and a $12,150 top of the 10% bracket instead of the 2026 values of $16,100 and $12,400. Correcting both parameters replaces its $18,334 of taxable income with $17,983.53 and its $1,957.08 with $1,910.02." -us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is zero ""after the standard deduction and EITC,"" combining two errors: $34,083.53 of wages exceeds the $16,100 standard deduction by $17,983.53, and the EITC is a refundable credit that the prompt explicitly excludes from this output and that never reduces taxable income in any case. The correct liability is $1,240 + 12% × $5,583.53 = $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $73,000 of educational assistance as employer §127 assistance and added the $67,750 above the $5,250 cap to wages, which pushed taxable income to $85,734 instead of $17,983.53. It then submitted $8,258, a number that matches none of its own arithmetic (it computed $13,573). With the correct wage-only base, its own $16,100 deduction and $12,400/$50,400 brackets give $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It used the 2024 $14,600 standard deduction, added $5,250 of educational assistance to income, and prorated wages over 40 weeks. It then zeroed the tax by subtracting the EITC, which is a refundable credit and so is excluded from this measure. A childless 31-year-old with $34,084 of earnings is also fully phased out of the EITC. The correct figure is $1,910.02 on $17,983.53 of taxable income." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 $5,250 cap to wages, making AGI $101,834 when it is really the $34,083.53 of wages alone. It also used stale bracket edges ($12,150/$49,350) in place of 2026's $12,400/$50,400." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Its 2026 parameters were correct ($16,100 deduction, $12,400/$50,400 brackets), but it turned the separately listed $73,000 of educational assistance into $67,750 of taxable wages. That set taxable income at $85,734 instead of $17,983.53 and produced $13,573 instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly excluded the educational assistance, applied the $16,100 deduction and the $12,400 10% bracket edge, and reached about $1,910 in its own reasoning. It then submitted an unsupported $2,372, which no step of its calculation produces." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages, making AGI $101,834 when it is really $34,083.53. It compounded this with a $15,750 standard deduction and invented bracket edges ($12,200/$46,600) in place of 2026's $16,100 and $12,400/$50,400." +us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It first taxed $67,750 of educational assistance, then switched to a wage-only base of $17,984 taxable income and computed $1,915 using a stale $12,150 10% bracket edge. It then reported an unexplained $1,750 that none of its calculations produce. The correct wage-only computation with the $12,400 edge is $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the entire $73,000 of educational assistance to wages, making AGI $107,084 when it is really $34,083.53. It also used a $15,700 deduction and $12,500 bracket edge instead of 2026's $16,100 and $12,400, and so taxed $91,384 in the 22% bracket in place of $17,983.53 of taxable income." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made two income errors: it subtracted the $8,389 ESI premium from the stated gross wages, and it added $67,750 of educational assistance above the §127 cap. That gave an AGI of $93,445 instead of $34,083.53. It also used a $15,400 deduction instead of $16,100." +us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance to wages, making AGI $101,834 when it is really $34,083.53. It then applied pre-TCJA law, with a $7,900 deduction, a $5,050 personal exemption and 15%/25% brackets. For 2026 the law is the permanent $16,100 deduction with 10%/12% brackets." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, giving AGI of $25,695 when it is really $34,083.53. It also assumed a 2026 TCJA sunset ($7,600 deduction + $5,100 exemption, 15% bracket), but the $16,100 standard deduction and 10%/12% brackets apply." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It used a $15,200 deduction and taxed everything at 10% ($1,888). It then cut the tax to $139 by applying a nonrefundable Child Tax Credit, but the single filer has no qualifying child, so no nonrefundable credit applies and tax stays at $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from gross wages (AGI $25,695 when it is really $34,083.53). It also assumed a TCJA sunset with about $14,250 of deduction plus exemption, but the $16,100 standard deduction applies, leaving $17,983.53 taxed at 10%/12%." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to income and subtracted the $8,389 ESI premium, giving AGI of $98,695 when it is really $34,083.53. It then applied sunset-era law ($8,300 deduction, $5,050 exemption, 15%/25% brackets) in place of the 2026 $16,100 deduction and 10%/12% brackets." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation for $0. Wages of $34,083.53 less the $16,100 standard deduction leave $17,983.53 of taxable income and $1,910.02 of tax with no nonrefundable credits. A zero answer means it treated this wage earner as having no positive liability, which the standard deduction alone never produces here." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied an $8,300 deduction, a $5,100 personal exemption and a 15% bracket. For 2026 the permanent $16,100 standard deduction, no exemption, and 10%/12% brackets apply, giving $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation lists the educational assistance among the tax inputs, and $11,116 is reachable only by taxing tens of thousands of dollars above the $34,084 of wages. It counted the educational assistance as taxable income, but only the $34,083.53 of wages is taxed: $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly taxed wages only but used stale pre-OBBBA 2025 parameters: a $15,000 standard deduction and an $11,925 10% bracket edge. The 2026 values are $16,100 and $12,400, so it overstated taxable income by $1,100 and reached $2,051.58." +us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to wages, making taxable wages $101,834 instead of $34,083.53. It also used 2024 parameters ($14,600 deduction, $11,600/$47,150 brackets) as a proxy for 2026." +us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the correct income base and $16,100 deduction but set the top of the 10% bracket at $12,200 instead of 2026's $12,400. That taxes $200 at 12% instead of 10% and gives $1,914 instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It claimed that $34,084 of wages is too low to owe tax after the standard deduction. The 2026 single deduction is only $16,100, which leaves $17,983.53 of taxable income and $1,910.02 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It declared the liability negative or zero after the standard deduction. The $16,100 deduction leaves $17,983.53 taxable, which produces $1,910.02 at the 10%/12% rates with no nonrefundable credits." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a projected post-sunset regime (about $13,650 of deduction plus personal exemption, a 15% second bracket). For 2026 the permanent $16,100 standard deduction and 10%/12% brackets apply, giving $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the $5,250 exclusion as taxable income. That pushed taxable income to about $85,734 and tax to $13,573.48, when only the $34,083.53 of wages is taxed ($1,910.02)." +us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, making income $107,084 when it is really $34,083.53. After the correct $16,100 deduction that taxed $90,984 instead of $17,983.53." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $19,000 implies a stale deduction of roughly $15,000 instead of the 2026 $16,100. That overstates taxable income by about $1,000 and gives $2,058 instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly used 2025-like parameters, a $15,000 standard deduction and an $11,925 10% bracket edge, instead of 2026's $16,100 and $12,400. That produced $19,084 of taxable income and $2,052 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules: an $8,450 deduction, a $5,300 personal exemption and a 15% bracket. For 2026 the permanent $16,100 standard deduction and 10%/12% brackets apply to the $34,083.53 of wages." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the correct base and $16,100 deduction but put the top of the 10% bracket at $12,200 instead of 2026's $12,400. That produced $1,914.08 instead of $1,910.02." +us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages (AGI $25,695 when it is really $34,083.53). It also used a projected post-TCJA $8,527 deduction with 15% brackets in place of the 2026 $16,100 deduction and 10%/12% brackets." +us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so no answer was available to score." +us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and a $12,200 10% bracket edge instead of 2026's $16,100 and $12,400. That overstated taxable income at $18,334 and gave $1,956." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and a $12,150 10% bracket edge instead of 2026's $16,100 and $12,400. That gave $18,334 of taxable income and $1,957.08 of tax." +us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It subtracted the EITC to reach $0 taxable income. The EITC is a refundable credit and never reduces taxable income or this pre-refundable-credit measure, and a childless filer with $34,084 of earnings is fully phased out of it. Wages less the $16,100 deduction leave $17,983.53 taxable and $1,910.02 of tax." us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly projected the temporary American Rescue Plan expansion of the childless EITC into 2026 and therefore used a phaseout endpoint near $33,000–$34,000. Under 2026 law, the temporary expansion is not in effect, and $34,084 exceeds the applicable childless EITC income limit, yielding zero EITC." us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated wages of $34,084 and a one-person household as qualifying for EITC without applying the childless single-filer phaseout endpoint. At that income the childless EITC is fully phased out, and no other refundable credit applies." us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract." @@ -5137,81 +5629,86 @@ us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The us,scenario_068,payroll_tax,inkling,llm_error,state_local_rule,False,"The model got both FICA components right on the full $34,084 wage base and then added a 0.45% Maryland FAMLI employee contribution of $153.38. Maryland's FAMLI employee contributions are not in effect for 2026 and are not part of PolicyEngine's payroll_tax, which comprises only employee Social Security, Medicare, and Additional Medicare Tax." us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive rule or computation was attempted. The required derivation is 6.2% plus 1.45% on the full $34,084 of wages, with no ESI offset, no educational assistance in the base, and no Maryland employee payroll contribution." us,scenario_068,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct base and rates but truncated each component to whole dollars ($2,113 Social Security and $494 Medicare) and then submitted $2,600 rather than the $2,607 those components sum to. The rule application was right; the reported total contradicts the model's own arithmetic by $7." -us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the IRC §127 $5,250 cap into Maryland AGI, inflating AGI from $34,083.53 to $101,834; educational assistance sits outside AGI here, so no gross-up applies. The inflated AGI then triggered a fictitious halving of the personal exemption to $1,600 and a $2,700 standard deduction, where the correct figures are the full $3,200 exemption and Maryland's 2026 flat $3,400 standard deduction." -us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It reproduced the entire correct structure but used a $3,350 Maryland standard deduction instead of the 2026 indexed value of $3,400, and $34,084 of income instead of $34,083.53. That $50.47 of extra taxable income times 4.75% is the entire $2.40 gap between its $1,255.37 and the reference $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer-sponsored insurance premium from AGI to reach $25,695, but the $34,084 wage figure is already the AGI-level amount and the premium is not a further subtraction. It then asserted that the remainder fell 'below the threshold' for Maryland tax; Maryland has no zero bracket, and even its own $25,695 less a $3,900 deduction and $3,200 exemption yields $18,595 of taxable income and about $831 of tax, not $0." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It grossed AGI up to $101,834 by treating educational assistance above $5,250 as Maryland-taxable, concluded 'best estimate ≈ $4,554,' and then submitted $1,029, a figure that matches neither that computation nor its own alternative full-exclusion figure of $1,336. The full-exclusion branch was the right one, but it paired it with the repealed $2,700 standard deduction cap rather than the 2026 flat $3,400, and its submitted value is unreconciled with either derivation." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It correctly kept educational assistance out of AGI but applied the repealed 15%-of-AGI standard deduction capped at $2,700 instead of Maryland's 2026 flat $3,400, producing taxable income of $28,184 rather than $27,483.53. It then computed $1,286 from that base and added an unexplained $22 'personal exemption credit adjustment' to reach $1,308; no such credit exists, the $3,200 exemption is already a subtraction from income." -us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a ~$2,700 standard deduction, the pre-2025 15%-of-AGI cap, instead of Maryland's 2026 flat $3,400, leaving taxable income of $28,184 against the correct $27,483.53. Its own bracket math on $28,184 gives $1,286.24; it reported $1,301, so it also fudged the schedule result upward." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the §127 cap to Maryland AGI, reaching $101,834 where the correct AGI is $34,083.53 with the assistance excluded entirely. It compounded this with a $2,500 standard deduction from the repealed 15%-of-AGI-capped formula rather than the 2026 flat $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded educational assistance correctly but applied the repealed 15%-of-AGI standard deduction capped at ~$2,550 instead of the 2026 flat $3,400, deriving taxable income of $28,334. Maryland's schedule on $28,334 is $90 + 4.75% × $25,334 = $1,293.37; it reported $1,050, a number its own bracket description does not produce." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It set Maryland AGI to $107,084 by adding the full $73,000 of educational assistance to wages with no exclusion at all, where the correct AGI is $34,083.53. That fabricated AGI dragged it into the exemption phase-out range above $100,000 and the 5% bracket, neither of which this household reaches." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $93,445 AGI is wages plus the full $73,000 educational assistance, less the $5,250 §127 exclusion and less the $8,389 ESI premium — two errors in opposite directions on top of one correct exclusion. Educational assistance does not enter AGI at all and the ESI premium is not an AGI subtraction, so AGI is $34,083.53, one-third of what it used." -us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the §127 cap in AGI ($101,834) when the assistance is entirely outside AGI. It also used a $2,400 standard deduction instead of $3,400 and omitted the $3,200 personal exemption altogether." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer-sponsored insurance premium from wages to get an AGI of $25,695; the $34,084 wage figure is already the AGI amount and the premium is not deducted again. It also used the repealed 15%-of-AGI standard deduction capped at $2,550 rather than the 2026 flat $3,400, so its $19,945 taxable base understates the correct $27,483.53 by $7,539." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no parameters, only a gesture at 'standard deductions and personal exemptions.' Its $1,133 back-solves to taxable income of $24,958, implying $9,126 of subtractions from the $34,083.53 AGI — $2,526 more than Maryland's actual $3,400 standard deduction plus $3,200 exemption." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It folded an 'estimated county tax rate around 3.0%' worth $592.35 into the state figure, but the requested output explicitly excludes local income taxes and Maryland's county piggyback tax is not part of state income tax before refundable credits. It separately reduced AGI to $25,695 by deducting the $8,389 ESI premium, which is not an AGI subtraction here." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $98,695 AGI is $34,084 plus the full $73,000 of educational assistance minus the $8,389 ESI premium — it both added income that is outside AGI and subtracted a premium that is not deductible from AGI. It also stated the bracket base as '$60 plus 4.75% over $3,000' when Maryland's 2%/3%/4% steps on the first $3,000 total $90." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted $997 with no derivation. That value back-solves to taxable income of $22,095, meaning $11,989 was removed from the $34,083.53 AGI — consistent with stripping out the $8,389 ESI premium plus roughly one deduction, where the only allowable subtractions are the $3,400 standard deduction and the $3,200 exemption that leave $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied the repealed 15%-of-AGI standard deduction at its old $2,550 cap instead of Maryland's 2026 flat $3,400, leaving taxable income of $28,334 against the correct $27,483.53. Its bracket arithmetic is otherwise exact, so the entire $40.40 error is that $850 of over-taxed income at 4.75%." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $4,786 with no derivation; that value back-solves to taxable income near $101,800, which is only reachable by adding the educational assistance to AGI. The assistance is excluded from AGI, so the correct base is $27,483.53 and the correct tax is $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $2,700 standard deduction — the old 15%-of-AGI cap — instead of Maryland's 2026 flat $3,400, giving taxable income of $28,184 rather than $27,483.53. Every other step, including the $90 on the first $3,000 and 4.75% on the remainder, is correct, so the $700 deduction shortfall at 4.75% is the whole $33.27 error." -us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $2,750 standard deduction and a $3,300 personal exemption instead of the correct $3,400 and $3,200. It then reported $1,453.28 on its own $28,034 taxable base, but Maryland's schedule on $28,034 is $90 + 4.75% × $25,034 = $1,279.12, so it also applied a rate schedule steeper than Maryland's top applicable 4.75% bracket." -us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It set the Maryland standard deduction at $3,200 instead of the 2026 value of $3,400, giving taxable income of $27,684 rather than $27,483.53. That $200 excess at 4.75% is the $9.52 gap, and its judgment that no nonrefundable credits apply is right." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed wage income was 'fully offset by available deductions/exemptions,' but Maryland's $3,400 standard deduction and $3,200 exemption total $6,600 against $34,083.53 of AGI, leaving $27,483.53 taxable. Maryland has no zero bracket and no exemption large enough to zero out a $34,000 wage earner, so the answer of $0 skips the computation entirely." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It floored the answer at zero on the unstated premise that deductions and unspecified nonrefundable credits wipe out the liability. Maryland's $3,400 standard deduction and $3,200 exemption leave $27,483.53 taxable, which produces $1,252.97 at the 2%/3%/4%/4.75% schedule, and no nonrefundable Maryland credit applies to this single childless filer." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It guessed a $2,900 Maryland standard deduction instead of the 2026 flat $3,400, yielding taxable income of $27,984 against the correct $27,483.53. Its bracket application ($20 + $30 + $40 + 4.75% of the excess) is otherwise exact, so the $500 deduction shortfall at 4.75% is the entire $23.77 error." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It set AGI to $101,834 by adding educational assistance above the §127 $5,250 cap to wages; the assistance is excluded from AGI, which is $34,083.53. Working from a base nearly triple the correct one, it landed at $4,478.99 instead of $1,252.97." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It projected a $2,850 maximum single standard deduction instead of Maryland's 2026 flat $3,400, leaving $28,034 of taxable income rather than $27,483.53. Everything else — the $3,200 exemption, the graduated rates, no nonrefundable credits — is right, so the $550 deduction shortfall at 4.75% is the whole $26.15 error." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 taxable income implies an AGI of $107,084, meaning it added the entire $73,000 of educational assistance to wages without even the $5,250 §127 exclusion. Educational assistance is outside Maryland AGI, so the base is $27,483.53 and the tax is $1,252.97, not $4,764.20." -us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used a $3,350 Maryland standard deduction instead of the 2026 indexed value of $3,400, giving taxable income of $27,534 rather than $27,483.53. The structure — excluded educational assistance, $3,200 exemption, graduated rates, no nonrefundable credits, county tax excluded — is otherwise exactly right, leaving only a $2.40 gap." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave a bare 'approximately 1400' with no deduction or exemption amounts. That value back-solves to taxable income of $30,578, implying only about $3,506 of subtractions from the $34,083.53 AGI — roughly one of the two allowable items rather than both the $3,400 standard deduction and the $3,200 personal exemption." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $2,350 Maryland standard deduction, a stale value from the old 15%-of-AGI formula, instead of the 2026 flat $3,400, so taxable income came out at $28,534 rather than $27,483.53. That $1,050 of extra income at 4.75% is the entire $50 overstatement." -us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It set the single standard-deduction maximum at ~$2,600 instead of Maryland's 2026 flat $3,400, giving taxable income of $28,284 rather than $27,483.53. The $3,200 exemption, the $90 on the first $3,000, and the 4.75% remainder rate are all correct, so the deduction parameter is the sole source of the $38 error." -us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $25,695 by subtracting the $8,389 employer-sponsored insurance premium, but the $34,084 wage figure is already AGI-level and the premium is not a further subtraction. It also used a $2,400 standard deduction rather than the 2026 flat $3,400, so its $20,095 taxable base falls $7,389 short of the correct $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It inverted the two parameters, using ~$2,650 as the standard deduction and ~$3,400 as the personal exemption when Maryland's 2026 values are a $3,400 standard deduction and a $3,200 exemption. The net $434 shortfall in subtractions left taxable income at ~$28,034 instead of $27,483.53." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It treated the full $73,000 of educational assistance as taxable Maryland income for lack of listed qualifying education expenses, producing a $107,084 gross income where the correct AGI is $34,083.53 with the assistance excluded. It then invented a '2%–6%' rate schedule: even on its own $101,184 base, Maryland's actual brackets yield $4,756.70, not the $5,598.54 it submitted." -us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used an 'indexed 2026 single standard-deduction cap' of $2,850 instead of Maryland's actual 2026 flat $3,400, leaving $28,034 taxable rather than $27,483.53. Its AGI, $3,200 exemption, and 2%-to-4.75% schedule are all correct, so the $550 deduction shortfall at 4.75% accounts for the entire $26.15 overstatement." -us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a $2,000 Maryland standard deduction, far below the 2026 flat $3,400, and omitted the $3,200 personal exemption entirely, leaving taxable income of $32,084 against the correct $27,483.53. That $4,600 of over-taxed income at 4.75% is the whole $218 error." -us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used $2,350 as the maximum single standard deduction, a figure from the repealed 15%-of-AGI formula, instead of Maryland's 2026 flat $3,400, so taxable income came to $28,534 rather than $27,483.53. Its exclusion of educational assistance, the $3,200 exemption, the bracket steps, and its exclusion of county tax are all correct." -us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"After computing $1,267.24 on a near-correct base, it subtracted a fabricated 'personal exemption credit' of $3,200 × 7.5% = $240 plus an unnamed 'standard deduction credit equivalent.' Maryland's $3,200 personal exemption is a subtraction from income that it had already taken, not a nonrefundable credit against tax, so no credit reduces this liability; it also used a $3,100 standard deduction instead of $3,400." -us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed Maryland taxable income is $0 'after the standard deduction and EITC,' conflating a credit with a deduction: Maryland's EITC is a credit against tax, and its refundable portion is excluded from this output by definition. The $3,400 standard deduction and $3,200 exemption reduce $34,083.53 of AGI to $27,483.53 of taxable income, which yields $1,252.97, not $0." +us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added $67,750 of educational assistance above the Section 127 limit to AGI, getting $101,834, but Maryland AGI is only the $34,083.53 of wages. It then used the repealed capped standard deduction ($2,700 instead of the flat $3,350), and its inflated AGI led it to cut the exemption to $1,600. With $34,083.53 of AGI, the $3,350 deduction and the full $3,200 exemption, the schedule gives $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer insurance premium from the listed gross wages and called taxable income 'minimal or zero' without applying the rate schedule. Maryland AGI is the full $34,083.53 of wages. Even its own $25,695 AGI minus $3,900 and $3,200 leaves about $18,600 of taxable income, which Maryland's 2%–4.75% schedule taxes at about $800, not $0." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Its reasoning produced $4,554 (after adding $67,750 of educational assistance to AGI) and $1,336 (wage-only AGI with the old $2,700 standard-deduction cap), but it submitted $1,029, which matches neither. The correct path uses wage-only AGI of $34,083.53, the flat $3,350 standard deduction and the $3,200 exemption, giving $27,533.53 of taxable income and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used the repealed 15%-of-income standard deduction with a $2,700 cap instead of Maryland's flat $3,350 single deduction, getting $28,184 of taxable income and $1,286 of tax. It then added an invented 'personal exemption credit adjustment' to reach $1,308. The $3,200 exemption is subtracted from income; it is not a separate credit." +us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the old capped standard deduction ($2,700) instead of the flat $3,350, which gave $28,184 of taxable income. Its $1,301 also misapplies the schedule, because $90 + 4.75% × $25,184 = $1,286.24. The correct taxable income of $27,533.53 is taxed at $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the $5,250 Section 127 limit in Maryland AGI ($101,834), but Maryland AGI is only the $34,083.53 of wages. It also used a $2,500 cap from the repealed 15% formula instead of the flat $3,350 standard deduction. Together these pushed taxable income to $96,134 and tax to $4,514." +us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It used a $2,550 capped standard deduction from the repealed 15%-of-income formula instead of the flat $3,350. It also misapplied its own schedule: $28,334 of taxable income gives $1,293.37 under the 2%/3%/4%/4.75% brackets, not $1,050." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the full $73,000 of educational assistance to wages, getting $107,084 of AGI, but Maryland AGI is only the $34,083.53 of wages. It also used a $2,400 standard-deduction cap and a phased-down $3,300 exemption instead of the flat $3,350 deduction and full $3,200 exemption, which pushed income into the 5% bracket." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It built a $93,445 AGI by adding $67,750 of educational assistance above the Section 127 cap and subtracting the $8,389 employer insurance premium from wages. Maryland AGI is just the $34,083.53 of listed wages, with neither adjustment. Its $5,980 of deductions is also below the $6,550 total of the flat $3,350 standard deduction and the $3,200 exemption." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It included $67,750 of educational assistance above the Section 127 limit in AGI ($101,834), used a $2,400 standard-deduction cap instead of the flat $3,350, and left out the $3,200 personal exemption entirely. The correct taxable income is $34,083.53 − $3,350 − $3,200 = $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its wage-only AGI and $3,200 exemption were correct, but it used a $2,800 standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $550 ($28,084 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It cut AGI to $25,695 by subtracting the $8,389 employer-sponsored insurance premium from the listed gross wages, but Maryland AGI is the full $34,083.53. It also used the repealed $2,550 capped standard deduction instead of the flat $3,350." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $1,133 matches Maryland taxable income of about $24,960, which is about $2,575 below the correct $27,533.53 ($34,083.53 of AGI minus the flat $3,350 standard deduction and the $3,200 exemption). So it overstated deductions by about $2,575." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It subtracted the $8,389 employer insurance premium from wages (AGI of $25,695 instead of $34,083.53) and used a $2,750 standard deduction instead of the flat $3,350. It then added about 3% of county income tax ($592.35) to the state figure, even though this output excludes local income tax." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It built a $98,695 AGI by adding the full $73,000 of educational assistance and subtracting the $8,389 insurance premium, but Maryland AGI is only the $34,083.53 of wages. It also used a $2,550 capped standard deduction instead of the flat $3,350, and a $60 base on the first $3,000 instead of $90." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $997 matches taxable income of about $22,095, which is about $5,440 below the correct $27,533.53. That gap fits an income base shrunk below the listed wages (for example, by subtracting the $8,389 insurance premium) instead of $34,083.53 of AGI minus the flat $3,350 standard deduction and the $3,200 exemption." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,550 'maximum' standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That left $28,334 of taxable income instead of $27,533.53 and tax of $1,293.37 instead of $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $4,786 matches taxable income of about $101,770 under Maryland's schedule (4.75% up to $100,000, then 5%), which is only reachable by counting the $73,000 of educational assistance as income. Only the $34,083.53 of wage AGI is taxable, giving $27,533.53 of taxable income and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,700 standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That produced $28,184 of taxable income instead of $27,533.53 and tax of $1,286.24 instead of $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $2,750 standard deduction and a $3,300 exemption instead of the flat $3,350 and $3,200, getting $28,034 of taxable income. It then misapplied the rate schedule: $90 + 4.75% × $25,034 = $1,279.12, not $1,453.28." +us,scenario_068,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a $3,200 standard deduction instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $150 ($27,684 instead of $27,533.53) and tax by about $7, and it then rounded to $1,262." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed deductions and exemptions fully offset the wages. Maryland's $3,350 standard deduction and $3,200 exemption total only $6,550 against $34,083.53 of AGI, leaving $27,533.53 taxable and $1,255.34 of tax." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It floored the tax at $0 without subtracting actual deductions or applying the rate schedule. $34,083.53 of AGI minus the $3,350 standard deduction and the $3,200 exemption leaves $27,533.53 of taxable income, taxed at $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It estimated a $2,900 standard deduction from the repealed capped formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $450 ($27,984 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It started from a $101,834 AGI that includes $67,750 of educational assistance above the Section 127 limit, but Maryland AGI is only the $34,083.53 of wages. That inflated the tax to $4,478.99." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It projected a $2,850 maximum standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $101,334 of taxable income equals $107,084 − $2,550 − $3,200, so it counted the full $73,000 of educational assistance as income. Maryland AGI is only the $34,083.53 of wages. It also used the repealed $2,550 cap instead of the flat $3,350 standard deduction." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It estimated a $2,900 standard deduction from the repealed capped formula instead of Maryland's flat $3,350 single deduction. That left $27,984 of taxable income instead of $27,533.53 and tax of $1,276.74 instead of $1,255.34." +us,scenario_068,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It indexed the standard deduction up to $3,450 for 2026 instead of using the flat $3,350 single amount. That understated taxable income by $100 ($27,434 instead of $27,533.53) and tax by about $4.75." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave only a rounded guess ('approximately 1400'). $1,400 matches taxable income of about $30,580, which implies only about $3,500 of total deductions and exemptions instead of the $6,550 from the flat $3,350 standard deduction plus the $3,200 exemption." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $2,350 standard deduction from the repealed 15%-of-income cap instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $1,000 ($28,534 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used a $2,600 standard-deduction maximum from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $750 ($28,284 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It treated the standard deduction as 15% of AGI capped at $2,850, but Maryland now allows a flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer insurance premium from the listed gross wages (AGI of $25,695 instead of $34,083.53) and used a $2,400 capped standard deduction instead of the flat $3,350. Together these understated taxable income by about $7,440." +us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used a $2,650 standard deduction instead of the flat $3,350 and a $3,400 exemption instead of $3,200. The net effect overstated taxable income by $500 ($28,034 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It counted the full $73,000 of educational assistance as taxable, getting $107,084 of income, but Maryland AGI is only the $34,083.53 of wages. It also used the repealed $2,700 cap instead of the flat $3,350 deduction. It then misapplied the schedule as well: Maryland's actual brackets tax $101,184 at $4,756.70, not $5,598.54." +us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It applied an indexed $2,850 standard-deduction cap from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $500 ($28,034 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used a $2,000 standard deduction instead of the flat $3,350 and left out the $3,200 personal exemption entirely. That gave $32,084 of taxable income instead of $27,533.53." +us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a $2,350 'maximum' standard deduction from the repealed 15%-of-income formula instead of Maryland's flat $3,350 single deduction. That overstated taxable income by $1,000 ($28,534 instead of $27,533.53)." +us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a $3,100 standard deduction instead of the flat $3,350, getting $1,267.24. It then subtracted an invented $240 'personal exemption credit' and a 'standard deduction credit equivalent', but Maryland has no such nonrefundable credits. Both the exemption and the deduction were already subtracted from income, so this counted them twice." +us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated the EITC as something that reduces taxable income to $0. The EITC is a credit, not a deduction, and a childless single filer earning $34,084 is above the childless EITC phase-out range anyway. Maryland taxable income is $34,083.53 − $3,350 − $3,200 = $27,533.53." us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the childless federal EITC phases out to zero at this income but then invented a $245 residual Maryland refundable credit. Applying Maryland’s percentage to a federal EITC of zero yields $0, not a minimal positive amount." us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated $34,084 as qualifying low income without applying the federal childless EITC phaseout ceiling, then assigned a Maryland EITC amount exceeding the credit pathway it cited. With no qualifying children, federal EITC is zero at this income, so the percentage-based Maryland refundable EITC and the total state refundable credits are both $0." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its derivation reproduced the reference chain exactly — AGI $36,269.67, $16,100 standard deduction, taxable income $20,169.67, pre-credit tax $2,172.36, Saver's Credit 10% × $1,924 = $192.40 — and its own text states the result 'approximately $1,980'. It then submitted 1697 in the value field, $283 below the number its explanation derives; the break is in the reported answer, not in the tax computation." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied a $14,600 single standard deduction — a 2023-era figure — instead of the 2026 amount of $16,100, and treated the $276 student loan interest as a second reduction of taxable income after the deduction rather than as part of AGI. It then abandoned bracket arithmetic and asserted '~$2,180' instead of computing 10% × $12,400 + 12% × $7,769.67 = $2,172.36, and omitted the $192.40 Saver's Credit that AGI $36,269.67 earns on $1,924 of qualified contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It reproduced the reference derivation exactly through $2,172.36 of pre-credit tax, then discarded its own arithmetic with an unexplained 'slight bracket estimate adjustment' that added $226. It also never applied the nonrefundable Saver's Credit of 10% × $1,924 ($1,485 401(k) + $69 traditional IRA + $262 Roth 401(k) + $108 Roth IRA) = $192.40, which is what turns $2,172.36 into $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own stated bracket components — $1,240 at 10% on the first $12,400 and $932 at 12% on the remaining $7,769.67 — sum to $2,172, yet it reported that sum as $2,549, a $377 arithmetic break within a single sentence. It then declared that no nonrefundable credits apply, missing the Saver's Credit of 10% × $1,924 = $192.40 available at AGI $36,269.67." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It used a $16,900 single standard deduction instead of the 2026 $16,100, then abandoned its own $2,076 subtotal for 2544 'after rounding of bracket parameters', a figure that follows from none of its stated inputs (the $16,100 deduction gives taxable income $20,169.67 and tax $2,172.36). It also asserted no nonrefundable credits, dropping the $192.40 Saver's Credit on $1,924 of qualified retirement contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After cycling through four standard-deduction guesses it settled on a projected $15,700 rather than the enacted 2026 single amount of $16,100, overstating taxable income by $400 and pre-credit tax by about $48. Its larger error is the flat conclusion that 'no nonrefundable credits' apply: the Saver's Credit pays 10% of $1,924 in 401(k), traditional IRA, Roth 401(k), and Roth IRA contributions = $192.40 at AGI $36,269.67." -us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It computed pre-credit tax of $2,172 correctly, then wrote that the Saver's Credit was phased out by income while nonetheless subtracting roughly $843 to reach 1329 — a credit amount that matches neither its own statement nor any rate band. AGI of $36,269.67 sits inside the 10% Saver's Credit band, so the credit is 10% × $1,924 = $192.40 and the tax is $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"AGI of $36,270 and the $192.40 Saver's Credit are exactly right; the whole $62 error is in the 2026 parameters, where it used a $15,600 standard deduction and a $12,300 top of the 10% bracket instead of $16,100 and $12,400. Those values raise taxable income to $20,670 and pre-credit tax to $2,234.40 against the correct $20,169.67 and $2,172.36." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages, but employer-plan premiums are already excluded from the $38,000 gross wage figure and are not a separate adjustment to income for a wage earner, and it used a $15,360 standard deduction rather than the 2026 $16,100. It also limited the Saver's Credit base to the $1,554 of traditional contributions, when the $262 Roth 401(k) and $108 Roth IRA contributions count as well, giving $1,924 × 10% = $192.40 rather than $155.40." -us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"AGI of $36,270 and the $192.40 Saver's Credit are correct; its single error is a $15,450 standard deduction instead of the 2026 single amount of $16,100. That overstates taxable income by $650 and pre-credit tax by $78, giving $2,252.90 in place of $2,172.36." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA expiration for 2026 and applied a $7,600 standard deduction, a $5,150 personal exemption, and a 15% second bracket. The 2026 parameters are a $16,100 single standard deduction, no personal exemption, and a 12% bracket above $12,400, so taxable income is $20,169.67 and pre-credit tax $2,172.36, not $23,520 and $2,948; its $192.40 Saver's Credit was the only correct element after AGI." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used a $15,500 standard deduction, never subtracted the $276 student loan interest from income, and reported 1494 with no bracket arithmetic; even its own inputs give taxable income of $21,046 and tax near $2,277. The correct chain is AGI $36,269.67, a $16,100 standard deduction, taxable income $20,169.67, tax $2,172.36, less the $192.40 Saver's Credit." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed AGI of $36,270 correctly and then subtracted a personal exemption alongside the standard deduction; no personal exemption exists for 2026, where the single standard deduction of $16,100 is the sole reduction. Its 2576 exceeds even the correct pre-credit tax of $2,172.36, so the Saver's Credit it names could not have been applied at the 10% rate on $1,924." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It deducted $600 of health insurance premiums that are already excluded from the $38,000 wage figure, then applied a 'reverted' post-sunset standard deduction plus a personal exemption to reach $22,370 of taxable income. 2026 uses a $16,100 single standard deduction with no personal exemption, giving taxable income of $20,169.67 and pre-credit tax of $2,172.36 rather than $2,775.50; its $192.40 Saver's Credit was correct." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It stopped at $36,615 of income and never subtracted the $69 deductible traditional IRA contribution or the $276 student loan interest, so its AGI was $345 too high, and it paired a ~$15,000 standard deduction with an $11,600 top of the 10% bracket instead of the 2026 $16,100 and $12,400. It also omitted the $192.40 Saver's Credit on $1,924 of qualified contributions." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"AGI of $36,270 and the $192.40 Saver's Credit — including the Roth 401(k) and Roth IRA amounts — are exactly right; it used a $15,700 standard deduction and the 2025 bracket break of $11,925 instead of the 2026 $16,100 and $12,400. That produced $2,229.90 of pre-credit tax against $2,172.36, the entire $58 gap." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its AGI of $36,270 and $192.40 Saver's Credit match the reference, but it applied a personal exemption on top of the standard deduction, which 2026 law does not provide. Its implied pre-credit tax of $2,780.50 corresponds to roughly $22,670 of taxable income taxed into a 15% bracket, whereas the $16,100 standard deduction leaves $20,169.67 taxed at 10% and 12% for $2,172.36." -us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It assumed TCJA sunset parameters for 2026 — an $8,300 standard deduction, a $5,300 personal exemption, and a 15% bracket above $12,300 — instead of the $16,100 standard deduction, no personal exemption, and a 12% bracket above $12,400. Its $192.40 Saver's Credit was correct, so the full $613 error comes from those deduction and rate assumptions, which inflated taxable income from $20,169.67 to $22,670." -us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"AGI of $36,270 and the $192.40 Saver's Credit — correctly built on $1,485 + $69 + $262 + $108 = $1,924 — are right; it used a $15,420 standard deduction and a $12,259 top of the 10% bracket rather than the 2026 $16,100 and $12,400. Those parameters produced $2,256.82 of pre-credit tax instead of $2,172.36, the whole $85 gap." -us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It reproduced the reference chain exactly — AGI $36,270, $16,100 standard deduction, taxable income $20,170, pre-credit tax $2,172 — then concluded 'no nonrefundable credits apply' after checking only children, dependents, and childless EITC. The Saver's Credit applies at 10% of $1,924 in 401(k), traditional IRA, Roth 401(k), and Roth IRA contributions = $192.40, which is precisely the amount by which its answer exceeds $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It zeroed the entire liability by invoking the CDCC or 'other credits' in the same sentence that concedes those are 'not present here'; with no dependents and no care expenses there is no such credit. Taxable income of $20,169.67 after the $16,100 standard deduction produces $2,172.36 of tax, and the only nonrefundable credit available is the $192.40 Saver's Credit, leaving $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It performed no bracket arithmetic and simply assumed unnamed nonrefundable credits reduce the liability to zero. The single filer has $20,169.67 of taxable income after the $16,100 standard deduction, producing $2,172.36 of tax, against which only the Saver's Credit of 10% × $1,924 = $192.40 applies." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'projected 2026 post-TCJA-sunset' parameters — a shrunken standard deduction plus a personal exemption and a 15% bracket — to reach $22,670 of taxable income. 2026 provides a $16,100 single standard deduction, no personal exemption, and a 12% bracket above $12,400, giving $20,169.67 of taxable income and $2,172.36 of tax; it also omitted the $192.40 Saver's Credit it asserted did not exist." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Every step matched the reference — $20,170 of taxable income after the $16,100 standard deduction and $2,172.40 of bracket tax — but it stated that no nonrefundable credits were available. The Saver's Credit is 10% of $1,924 in 401(k), traditional IRA, and Roth contributions = $192.40 at AGI $36,269.67, and its omission is the entire error." -us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It derived taxable income of $20,170 and pre-credit tax of $2,172.40 exactly, then submitted that figure as the final answer under 'no-credit' reasoning. The Retirement Savings Contributions Credit is nonrefundable and applies here at 10% of $1,924 in qualified contributions = $192.40, which reduces the liability to $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It supplied no derivation, and 1623 corresponds to no step of the correct computation: pre-credit tax is $2,172.36 and the post-credit amount is $1,979.96. Working backward, its figure requires a standard deduction near $20,700 — a head-of-household-sized allowance — rather than the $16,100 single amount that leaves $20,169.67 of taxable income." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset' parameters — an $8,300 standard deduction plus a $5,300 personal exemption and a 15% bracket above $11,900 — for $22,670 of taxable income. 2026 grants a $16,100 single standard deduction with no personal exemption and a 12% bracket, so taxable income is $20,169.67 and pre-credit tax $2,172.36; its $192 Saver's Credit was the only correct adjustment." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed post-TCJA-sunset parameters — a reduced standard deduction plus a personal exemption and a 15% bracket — giving $22,670 of taxable income and about $2,810 of tentative tax. The 2026 single standard deduction of $16,100 with no personal exemption leaves $20,169.67 taxed at 10% and 12% for $2,172.36, and its correctly sized $192 Saver's Credit then yields $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It removed $600 of health insurance premiums from wages, although employer-plan premiums are already excluded from the $38,000 gross wage figure and are not a wage earner's adjustment to income, and it used a $15,400 standard deduction instead of $16,100. It also computed the Saver's Credit on the traditional contributions alone for $155, when the $262 Roth 401(k) and $108 Roth IRA contributions belong in the base, giving $1,924 × 10% = $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed this taxpayer as head of household — a $21,900 standard deduction and the HoH 10% bracket running to $16,050 — although the household is one adult with no dependents, who files single with a $16,100 standard deduction and a 10% bracket ending at $12,400. It also excluded the $262 Roth 401(k) and $108 Roth IRA contributions from the Saver's Credit base, taking 10% of $1,554 rather than 10% of $1,924 = $192.40." -us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It computed roughly $2,108 of pre-credit tax on a $15,750 standard deduction and then wrote the liability down to zero using a '~$159' Saver's Credit plus unnamed 'other nonrefundable credits'. The Saver's Credit is $192.40 (10% of $1,924) and it is the only nonrefundable credit this household earns, so the correct pre-credit tax of $2,172.36 falls to $1,979.96, not to zero." -us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It matched the reference through AGI $36,270, the $16,100 standard deduction, and $2,172.40 of bracket tax, then applied the Saver's Credit at the 50% rate for $962 on the ground that AGI fell below the lowest single threshold. AGI of $36,269.67 sits far above the 50%-rate ceiling for single filers (about $25,500 for 2026) and inside the 10% band, making the credit $192.40 and the answer $1,979.96." -us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied a 50% Saver's Credit rate to the traditional contributions only, for $777, when AGI of $36,269.67 falls in the 10% rate band and the $262 Roth 401(k) and $108 Roth IRA contributions belong in the $1,924 base, making the credit $192.40. It also dropped the $276 student loan interest from AGI and used a $15,750 standard deduction instead of $16,100, and its submitted 1213.5 does not match the $1,475.52 its own explanation computes." -us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It asserted pre-credit tax of about $3,537, far above the $2,172.36 that $20,169.67 of taxable income yields at the 10% and 12% rates, then erased the whole amount with a Credit for Other Dependents even though the household contains no dependents. The only nonrefundable credit here is the $192.40 Saver's Credit on $1,924 of qualified retirement contributions." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning gets every step right: $2,172 of tax on $20,170 taxable income, less a 10% Saver's Credit of $192 on $1,924, which gives about $1,980. It then submitted $1,697, which contradicts its own derivation." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the outdated $14,600 single standard deduction instead of the 2026 amount of $16,100. It then guessed at the bracket tax, arriving at about $2,180. It never applied the nonrefundable 10% Saver's Credit of $192.40 on $1,924 of 401(k)/IRA contributions (AGI $36,270)." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It correctly computed $2,172 of tax on $20,170 taxable income, then replaced it with $2,398 under an unexplained 'bracket estimate adjustment'. It also never subtracted the 10% Saver's Credit of $192.40, which AGI of $36,270 qualifies for." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It computed $1,240 + $932 but reported the sum as $2,549 instead of $2,172, an arithmetic error. It also stated that no nonrefundable credits apply, missing the 10% Saver's Credit of $192.40 on $1,924 of retirement contributions." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $16,900 standard deduction instead of $16,100 and computed about $2,076. It then submitted an unrelated $2,544 without deriving it. It also left out the 10% Saver's Credit of $192.40, even though AGI of $36,270 falls in the 10% tier." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on a projected $15,700 standard deduction instead of the enacted 2026 figure of $16,100, which inflated taxable income to $20,570 and tax to $2,220. It also declared 'no nonrefundable credits', missing the $192.40 Saver's Credit (10% of $1,924 in 401(k)/IRA contributions, Roth included)." +us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It correctly reached $2,172 of pre-credit tax and noted the Saver's Credit, but its answer of $1,329 implies an $843 credit. AGI of $36,270 puts a single filer in the 10% tier, so the credit is $192.40 on $1,924. The $843 reduction matches no Saver's Credit rate." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used a $15,600 standard deduction and a $12,300 top for the 10% bracket. The 2026 figures are $16,100 and $12,400. That overstated taxable income at $20,670 and pre-credit tax at $2,234.40, when the correct figures are $20,170 and $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $600 of health insurance premiums from wages even though they are not pre-tax, and it used a $15,360 standard deduction with a $12,211 top for the 10% bracket instead of $16,100 and $12,400. It also computed the Saver's Credit on only the $1,554 of traditional contributions, leaving out the $370 of Roth 401(k)/IRA contributions from the $1,924 base." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It got AGI ($36,270) and the $192.40 Saver's Credit right, but used a $15,450 standard deduction instead of the 2026 amount of $16,100. That inflated taxable income to $20,820 and tax to $2,252.90." +us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $192.40 Saver's Credit, but ended the 10% bracket at $12,200 instead of the 2026 threshold of $12,400. That added $4 to pre-credit tax ($2,176.40) and produced $1,984 instead of $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026. It applied a $7,600 standard deduction plus a $5,150 personal exemption and a 15% second bracket, so tax before credits came out at $2,948. Under 2026 law the TCJA rates are permanent, with a $16,100 standard deduction, no personal exemption and a 12% bracket, so tax before credits is $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of AGI (using $36,546 instead of $36,270) and used a $15,500 standard deduction instead of $16,100. Its $1,494 answer does not follow from 10%/12% bracket tax on its own $21,046 of taxable income (about $2,270), and it never shows the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got AGI right at $36,270, but subtracted a personal exemption, which follows pre-TCJA sunset law. Its $2,576 answer is consistent with the reverted standard deduction, exemption and 15% bracket, rather than the permanent 2026 parameters: a $16,100 deduction, no exemption, and 10%/12% brackets." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a reverted post-TCJA-sunset standard deduction and personal exemption, which yield $22,370 of taxable income taxed in a 15% bracket for $2,775.50. The 2026 rules are a $16,100 deduction and 10%/12% brackets. It also wrongly subtracted $300 of health insurance premiums from AGI." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It left out the $69 traditional IRA and $276 student loan interest deductions ($36,615 AGI), and used the 2025 $15,000 standard deduction and 2024 $11,600 top for the 10% bracket. It also never applied the $192.40 Saver's Credit, ending at $2,362." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used a $15,700 standard deduction and the 2025 $11,925 top for the 10% bracket. The 2026 figures are $16,100 and $12,400. That overstated pre-credit tax at $2,229.90 instead of $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a personal exemption and reverted standard deduction on the assumption that the TCJA sunset in 2026. The implied pre-credit tax of $2,780.50 reflects the 15% bracket, while 2026 law keeps the $16,100 deduction and 10%/12% brackets for $2,172.36 before the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA expired. It used an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, producing $2,785.50 of pre-credit tax. The 2026 rules are a $16,100 standard deduction, no exemption and a 12% bracket, which give $2,172.36." +us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied the $192.40 Saver's Credit correctly, but used an estimated $15,420 standard deduction and a $12,259 top for the 10% bracket instead of the 2026 figures of $16,100 and $12,400. That inflated tax to $2,256.82." +us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It correctly computed $2,172 of tax, but concluded no nonrefundable credits apply. At AGI $36,270 the single filer qualifies for the 10% Saver's Credit on $1,924 of 401(k)/IRA contributions, which is $192.40." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed nonrefundable credits wipe out the regular tax while admitting no such credits are present. In fact $20,169.67 of taxable income produces $2,172.36 of tax, and the only offsetting credit is the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It assumed nonrefundable credits reduce the tax to zero without naming any. The household has no dependents, so the only nonrefundable credit is the $192.40 Saver's Credit against $2,172.36 of tax." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the projected post-TCJA-sunset standard deduction and personal exemption with a 15% bracket ($22,670 taxable income), instead of 2026 law with a $16,100 deduction and 10%/12% brackets. It also said no nonrefundable credits apply, missing the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It computed the pre-credit tax of $2,172.40 correctly, but stated that no nonrefundable credits are available. It missed the 10% Saver's Credit of $192.40 on $1,924 of traditional and Roth retirement contributions at AGI $36,270." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It computed $2,172.40 as a 'no-credit' tax and stopped there. It never subtracted the nonrefundable Saver's Credit of $192.40 (10% of $1,924 in 401(k)/IRA contributions) that AGI of $36,270 qualifies for." +us,scenario_070,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It computed the $2,172.40 bracket tax correctly, but asserted that no nonrefundable credits are used. That missed the $192.40 Saver's Credit, which is 10% of the $1,924 of traditional and Roth retirement contributions." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave no computation. The correct figure is $2,172.36 of 10%/12% tax on $20,169.67 of taxable income, less a $192.40 Saver's Credit. Its $1,623 answer sits $549 below the pre-credit tax, a reduction that no Saver's Credit rate on the $1,924 base produces (10% = $192.40, 20% = $384.80, 50% = $962), so it understated tax with an unsupported deduction or credit." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied 'current-law sunset rules': an $8,300 standard deduction, a $5,300 personal exemption and a 15% bracket, giving $2,806 of tax. 2026 law keeps the $16,100 standard deduction and 10%/12% brackets, which give $2,172.36 before the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed tax under post-TCJA-sunset rules, with a standard deduction plus personal exemption and 10%/15% brackets, giving about $2,810. It should have used the 2026 $16,100 standard deduction and 12% bracket, which give $2,172.36 before the $192.40 Saver's Credit." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used the correct $16,100 standard deduction and $192 Saver's Credit, but its $2,176 ordinary tax implies a 10% bracket ending at $12,200 instead of the 2026 threshold of $12,400. That $4 overstatement produced $1,984 instead of $1,979.96." +us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted health insurance premiums from AGI as if they were pre-tax (AGI $35,970) and used a $15,400 standard deduction instead of $16,100. It also computed a $155 Saver's Credit on only the traditional contributions, leaving the $370 of Roth 401(k)/IRA contributions out of the $1,924 base." +us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"It filed the childless head as head of household, using a $21,900 standard deduction and HoH brackets. With no qualifying person the head files single, with a $16,100 deduction. It also based the Saver's Credit on only the $1,554 of traditional contributions, leaving out the Roth 401(k)/IRA amounts that make up the $1,924 base." +us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"By its own figures it reached about $2,108 of tax less a $159 Saver's Credit, yet it submitted $0 on the claim that unnamed 'other' nonrefundable credits eliminate the tax. It also left the $100 of interest and the $276 student loan interest deduction out of taxable income, and used a $15,750 standard deduction instead of $16,100." +us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It computed the $2,172.40 pre-credit tax and the $1,924 contribution base correctly, but applied the 50% Saver's Credit rate, claiming AGI was below the lowest threshold. At AGI $36,270 a single filer is in the 10% tier, so the credit is $192.40, not $962." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It applied a 50% Saver's Credit rate to only the traditional contributions ($777), when AGI of about $36,270 puts a single filer in the 10% tier on the full $1,924 base including Roth amounts. It also left the $276 student loan interest deduction out of AGI and used a $15,750 standard deduction and a $12,150 bracket. It then submitted $1,213.50, which contradicts its own $1,475.52." +us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It overstated pre-credit tax at $3,537 and then zeroed it out with the Credit for Other Dependents, even though the single head has no dependents. The only nonrefundable credit is the $192.40 Saver's Credit against $2,172.36 of tax." us,scenario_070,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied the Additional Child Tax Credit earned-income formula despite the household having no qualifying child, so there was no Child Tax Credit balance available to refund. Its separate EITC rationale also misstates the rule: $100 of investment income does not exceed the limit; the childless EITC is zero because $38,000 of earnings exceeds its phaseout ceiling." us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model called eligibility borderline without calculating the stated household’s MAGI-to-FPL ratio. At 2.27 times FPL, the head exceeds Illinois’s 138% FPL ACA expansion limit, and age 57, low assets, and employer-sponsored insurance do not establish another Medicaid category." us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $300 health insurance premium as a pre-tax FICA exclusion even though no pre-tax treatment was specified. FICA applies to the full $38,000, yielding $2,356 of Social Security tax and $551 of Medicare tax." @@ -5219,40 +5716,41 @@ us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"T us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax." us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907." us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total." -us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model built the correct Illinois base income of $36,270 and applied the correct 4.95% flat rate, but subtracted a $2,850 personal exemption — Illinois's 2025 indexed amount — instead of the 2026 amount of $2,925. That $75 shortfall in the exemption is the entire error: $33,420 × 4.95% = $1,654.29 versus $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model never removed the $1,485 traditional 401(k) elective deferral from wages, so it started from $38,100 and reached a base of $37,755 instead of the correct AGI of $36,269.67. It then converted Illinois's personal exemption into a $120 nonrefundable credit ($2,425 × 4.95%) when the exemption is a $2,925 deduction from base income, and the value it submitted ($1,154) does not even match its own stated arithmetic of $1,869 − $120 = $1,749." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model derived the correct AGI of $36,270 and correctly applied the 4.95% rate, but used the 2025 Illinois personal exemption of $2,850 rather than the 2026 amount of $2,925, producing $1,654.29. It then discarded its own computed figure and submitted $1,659.83 as a 'rounding to exemption assumption' adjustment, moving further from the correct $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model reached the correct $36,270 base income but cycled through stale Illinois exemption vintages ($2,775 and $2,425) rather than the 2026 amount of $2,925, and then abandoned all of its own computations to submit $1,726, which corresponds to a fabricated taxable income of $34,869 — $1,524 above the correct $33,344.67. The correct chain is $36,269.67 − $2,925 = $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly identified federal AGI of $36,270 and the 4.95% flat rate but applied the 2025 exemption of $2,850 instead of the 2026 amount of $2,925, computing ≈$1,655. It then inflated that to $1,699 on an unstated 'higher base/exemption rounding' allowance, adding a $44 error on top of the $3.73 exemption-vintage error." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model added the $276 student loan interest deduction and the $69 traditional IRA deduction back into the Illinois base, but Illinois net income starts from federal AGI with no such addbacks, so both stay subtracted. It compounded this by excluding a $300 employer-sponsored health premium from wages (not an AGI adjustment here) and by using the $2,425 exemption instead of $2,925; the combined $545.33 overstatement of taxable income is exactly the $27.00 gap between its $1,677.56 and the correct $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model asserted that Illinois disallows the student loan interest deduction and therefore never subtracted the $276, leaving a base of $36,546 rather than $36,269.67; Illinois net income begins with federal AGI, in which that above-the-line deduction is already taken. It also used the 2025 exemption of $2,850 rather than $2,925, computed $1,668, and then fudged upward to $1,690 for a nonexistent 'exemption phase-in.'" -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model invented an Illinois standard deduction of $2,450 and stacked it on top of a $2,450 personal exemption, deducting $4,900 from the correct $36,270 base. Illinois grants no standard deduction — only the personal exemption, $2,925 for 2026 — so taxable income is $33,344.67, not $31,370, and the tax is $1,650.56 rather than $1,552.82." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $35,670, $600 below the correct $36,269.67, consistent with deducting the $300 employer-sponsored and $300 other health insurance premiums, neither of which reduces AGI on these facts. It also used an invented exemption of $2,842 instead of $2,925, and the two errors net to $32,828 of taxable income versus the correct $33,344.67." -us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model got the Illinois base income of $36,270 and the 4.95% rate exactly right but subtracted the 2025 personal exemption of $2,850 rather than the 2026 indexed amount of $2,925. The $75 exemption shortfall is the whole gap: $1,654.29 versus $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"After computing $1,658 using the stale $2,775 exemption instead of the 2026 amount of $2,925, the model subtracted a $135 'renter's credit' equal to 5% of one quarter of the $10,800 rent. Illinois's 5% property tax credit applies only to Illinois property tax actually paid on a principal residence the taxpayer owns; Illinois has no renter's credit, so no nonrefundable credit reduces the $1,650.56 liability." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model named the stale $2,775 exemption rather than the 2026 amount of $2,925, and its submitted $1,716 does not even follow from that: $36,270 − $2,775 = $33,495 × 4.95% = $1,658. The $1,716 figure implies a taxable base of about $34,667, roughly $1,322 above the correct $33,344.67." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model started from the correct federal AGI of $36,270 and the 4.95% flat rate but guessed the personal exemption at 'roughly $2,500' instead of the 2026 amount of $2,925. That $425 understatement of the exemption yields $33,770 of taxable income and $1,671 rather than $33,344.67 and $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $35,970, which is the correct $36,270 less a $300 health insurance premium that is not an AGI adjustment on these facts, and then subtracted the stale $2,775 exemption instead of the 2026 amount of $2,925. Removing both errors restores taxable income of $33,344.67 and tax of $1,650.56 rather than its $33,195 and $1,643.15." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave only the flat 4.95% rate and 'standard exemptions' with no figures, and its $1,683 answer corresponds to a taxable base of exactly $34,000 — $655 above the correct $33,344.67, consistent with an exemption of only about $2,270 against the $36,269.67 AGI. Illinois's 2026 personal exemption is $2,925, yielding $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly used base income of $36,270 and the 4.95% flat rate but applied the 2025 personal exemption of $2,850 instead of the 2026 indexed $2,925. The $75 exemption difference accounts for the entire $3.73 miss." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the 2024-vintage Illinois personal exemption of $2,775 to the correct federal AGI of $36,270; the 2026 exemption is $2,925. That $150 gap in the exemption is the whole error: $33,495 × 4.95% = $1,658 versus $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's base income of $36,270 and 4.95% rate are right, but it subtracted a $2,775 exemption — the 2024 amount — instead of the 2026 indexed $2,925. Correcting the exemption alone moves $1,658 to $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model stopped its AGI at $36,546 by never subtracting the $276 student loan interest deduction, which is an above-the-line federal deduction already embedded in the $36,269.67 AGI that Illinois adopts, and it used the $2,425 exemption instead of $2,925. Its own arithmetic produced $1,689, yet it submitted $1,626 — a value that follows from neither its stated base nor the correct one." -us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used the correct $36,270 base and 4.95% rate but subtracted the 2025 personal exemption of $2,850 rather than the 2026 amount of $2,925. That single stale parameter produces $1,654 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed that 'the standard exemption and retirement deductions' drive Illinois taxable income to zero, but Illinois offers no standard deduction and the retirement contributions here total only $1,554 of traditional 401(k) and IRA amounts already reflected in the $36,269.67 AGI. Subtracting the single $2,925 personal exemption leaves $33,344.67 taxed at 4.95%, or $1,650.56 — nowhere near zero." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that retirement contributions and student loan interest, plus unnamed nonrefundable credits, zero out the Illinois liability. Those adjustments are already inside the $36,269.67 federal AGI Illinois starts from, no Illinois nonrefundable credit applies to a renter with no dependents or education expenses, and $36,269.67 − $2,925 = $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model left the $276 student loan interest deduction out of its AGI, stopping at $36,546 instead of $36,269.67, and used an invented 2026 personal exemption of $1,824 rather than $2,925. The combined $2,478 overstatement of taxable income produced $1,718.72 against the correct $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model derived the correct $36,270 Illinois base and applied 4.95% correctly but subtracted the 2025 personal exemption of $2,850 instead of the 2026 amount of $2,925. That $75 parameter gap is the sole cause of the $3.73 miss." -us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no figures beyond 'flat tax of 4.95% after exemptions,' and its $1,777 answer implies a taxable base of about $35,899 — only about $371 below the $36,269.67 AGI, meaning it effectively taxed nearly the full AGI. Illinois's 2026 personal exemption of $2,925 leaves $33,344.67 and a tax of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly started from federal AGI of $36,270, correctly ruled out the property tax and K-12 education credits for a renter with no dependents, but subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. Using $2,925 turns its $1,658 into $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $35,970 — the correct $36,269.67 reduced by a $300 health insurance premium that is not an AGI adjustment on these facts — and paired it with the $2,425 exemption rather than the 2026 amount of $2,925. Its $33,545 taxable base overstates the correct $33,344.67, producing $1,660 instead of $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied the correct $36,270 base and 4.95% rate but used a $2,775 personal exemption, Illinois's 2024 figure, rather than the 2026 amount of $2,925. That $150 exemption shortfall fully explains $1,658 versus $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model stated that Illinois has 'no standard deduction or personal exemptions' and taxed the full $36,270 base at 4.95%. Illinois grants a personal exemption under 35 ILCS 5/204 — $2,925 for 2026 — so taxable income is $33,344.67 and the tax is $1,650.56, not $1,795." -us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly took Illinois base income as federal AGI of $36,270 with no additions or subtractions and correctly found no applicable nonrefundable credits, but subtracted a $2,775 personal exemption instead of the 2026 amount of $2,925. Substituting the correct exemption converts its $1,658.00 to $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model reduced the Illinois base to roughly $20,696, implying it subtracted a federal-style standard deduction of about $15,600 from the $36,270 AGI; Illinois allows no standard deduction, only the $2,925 personal exemption. Its submitted $1,106 also contradicts its own stated computation of $1,024, and the correct result is $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model got the structure exactly right — federal AGI of $36,270, one personal exemption, the 4.95% flat rate, and no property tax or K-12 credit for a renter — but rounded the exemption to 'about $2,900' instead of the actual 2026 amount of $2,925. That $25 rounding of a single parameter is the entire $1.26 miss." -us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model applied Illinois's personal exemption twice — once as a $2,800 deduction reducing the base to $33,738 and then again as a $2,800 nonrefundable credit subtracted from the $1,670 of tax, flooring the result at zero. The Illinois exemption allowance is a deduction from net income only, not a dollar-for-dollar credit, so with the correct 2026 exemption of $2,925 the liability is $33,344.67 × 4.95% = $1,650.56." -us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model named the right method — federal AGI minus the personal exemption at 4.95% — but supplied no figures, and its $1,658.66 implies a taxable base of about $33,508, roughly $164 above the correct $33,344.67 and consistent with an exemption near $2,761 rather than the 2026 amount of $2,925. With the correct exemption the tax is $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Used the correct $36,270 AGI base but subtracted the 2025 personal exemption of $2,850 instead of the 2026 indexed exemption of $2,925. That left taxable income of $33,420 instead of $33,345 and tax of $1,654.29 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Never excluded the $1,485 traditional 401(k) deferral from wages, so its base was $37,755 instead of $36,270. It also treated the exemption as a $120 credit based on an outdated $2,425 figure instead of deducting the $2,925 exemption from income. Its $1,154 answer does not even follow from its own $1,869 minus $120." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly derived $36,270 AGI but subtracted the 2025 exemption of $2,850 instead of the 2026 exemption of $2,925. It then inflated its own $1,654.29 result to $1,659.83 with an unexplained rounding adjustment, moving further from $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Computed the correct $36,270 AGI but used the 2024 exemption of $2,775, which gives $1,658, instead of $2,925. It then abandoned that figure for an unexplained taxable income of $34,870 (implying an exemption of only $1,400) and reported $1,726." +us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Used the $36,270 AGI with the 2025 exemption of about $2,850 instead of the 2026 exemption of $2,925, which gave about $1,655. It then raised that to $1,699 through unexplained base and rounding adjustments that have no basis in Illinois law." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Wrongly added the $69 IRA deduction and $276 student loan interest back to Illinois base income; Illinois base income starts from federal AGI and has no addition modification for either. It also subtracted a $300 health premium from wages that the case does not treat as a pre-tax exclusion. Finally it used the outdated $2,425 exemption instead of $2,925, which produced $33,890 of taxable income instead of $33,345." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of federal AGI, giving $36,546 instead of $36,270; Illinois inherits that deduction through AGI. It also used the 2025 exemption of $2,850 instead of $2,925, then added an arbitrary upward adjustment from $1,668 to $1,690." +us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Applied 4.95% to $36,270 AGI less the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925. That overstated taxable income by $75 and gave about $1,654 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Invented a $2,450 Illinois standard deduction on top of a $2,450 personal exemption. Illinois has no standard deduction, only a single personal exemption, which is $2,925 for 2026. It therefore subtracted $4,900 instead of $2,925 and understated the tax at $1,552.82." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $35,670 by deducting the $600 of listed health insurance premiums, which do not reduce federal AGI here; the correct figure is $36,270. It also used a $2,842 exemption instead of the 2026 exemption of $2,925, which gave $32,828 of taxable income and $1,624.99 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Subtracted the 2025 personal exemption of $2,850 from $36,270 AGI instead of the 2026 exemption of $2,925. That gave taxable income of $33,420 and tax of $1,654.29 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Applied a nonrefundable $135 renter's credit, treating 25% of rent as property tax, but Illinois has no such credit; its property tax credit requires property tax paid on a residence, and this household is a renter with no property tax. It also used the 2024 exemption of $2,775 instead of $2,925." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Cited a $2,775 exemption instead of the 2026 exemption of $2,925. Its $1,716 answer implies about $34,667 of taxable income, or roughly $37,442 of base income before the exemption, so it also failed to reduce income to the $36,270 federal AGI." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Started from the correct $36,270 AGI but subtracted an outdated exemption of about $2,500 instead of the 2026 indexed exemption of $2,925. That overstated taxable income by about $425 and gave $1,671." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,970, subtracting a $300 health premium that does not reduce AGI here; the correct figure is $36,270. It also used the 2024 exemption of $2,775 instead of $2,925, which gave $33,195 of taxable income and $1,643.15 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Gave no figures, but its $1,683 answer implies $34,000 of taxable income. Measured from the $36,270 AGI, that is an exemption of only about $2,270 instead of the 2026 exemption of $2,925, which overstates the base by about $655." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Subtracted the 2025 exemption of $2,850 from $36,270 AGI instead of the 2026 indexed exemption of $2,925. That gave $33,420 of taxable income and $1,654.29 of tax instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used the 2024 personal exemption of $2,775 instead of the 2026 exemption of $2,925. That overstated taxable income by $150 and gave $1,658 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Subtracted a $2,775 exemption, the 2024 value, from $36,270 AGI instead of the 2026 exemption of $2,925. That gave $33,495 of taxable income and $1,658 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Omitted the $276 student loan interest deduction from federal AGI, giving $36,546 instead of $36,270. It also used the outdated $2,425 exemption instead of $2,925, and then submitted $1,626, which contradicts its own computed $1,689." +us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Applied 4.95% to $36,270 AGI less the 2025 exemption of about $2,850 instead of the 2026 indexed exemption of $2,925. That gave about $1,654 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Claimed the exemption and retirement deductions reduce Illinois tax to zero. In fact the only reductions are the 401(k), IRA and student loan items already reflected in the $36,270 AGI, plus one $2,925 exemption, which leaves $33,345 taxable at 4.95% for $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that retirement contributions and student loan interest drive Illinois liability to zero. Those adjustments only reduce AGI to $36,270, and after the $2,925 exemption $33,345 remains taxable at the 4.95% flat rate." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Left the $276 student loan interest deduction out of AGI, giving $36,546 instead of $36,270. It also used a $1,824 exemption, far below the 2026 Illinois exemption of $2,925, which gave $34,722 of taxable income and $1,718.72 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Used the 2025 personal exemption of $2,850 instead of the 2026 exemption of $2,925. That gave $33,420 of base income and $1,654.29 of tax instead of $33,345 and $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no figures, but its $1,777 answer implies about $35,899 of taxable income, about $2,555 above the correct $33,345 ($36,270 AGI less the $2,925 exemption). It therefore failed to reduce wages to federal AGI and to apply the full 2026 exemption." +us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Correctly started from $36,270 AGI but subtracted the 2024 exemption of $2,775 instead of the 2026 exemption of $2,925. That gave $1,658 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Used an AGI of $35,970, deducting a $300 health premium that does not reduce AGI here, instead of $36,270. It also used the outdated $2,425 exemption instead of $2,925, which gave $33,545 of taxable income and about $1,660 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Subtracted the 2024 exemption of about $2,775 from $36,270 AGI instead of the 2026 indexed exemption of $2,925. That gave $33,495 of taxable income and $1,658 of tax." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Stated that Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a personal exemption of $2,925 for 2026, so taxable income is $33,345 and tax is $1,650.56, not $1,795." +us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Correctly found no additions or subtractions and used $36,270 as base income, but subtracted the 2024 exemption of $2,775 instead of the 2026 exemption of $2,925. That gave $1,658 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Applied a nonexistent Illinois standard deduction that cut taxable income to about $20,696, then reported $1,106 without a coherent base. Illinois taxable income is federal AGI of $36,270 less only the $2,925 exemption, which gives $33,345." +us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Used the correct $36,270 AGI but approximated the 2026 exemption as $2,900 instead of the indexed $2,925. That left taxable income $25 too high and gave $1,651.82 instead of $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Subtracted a $2,800 exemption from income, then applied the same $2,800 again as a dollar-for-dollar nonrefundable credit against tax, which zeroed out the liability. The Illinois exemption is only a $2,925 deduction from base income, not a credit, so tax is 4.95% of $33,345, or $1,650.56." +us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $1,658.66 answer implies about $33,508 of taxable income. That corresponds to an exemption of about $2,762, near the 2024 value of $2,775, subtracted from $36,270 AGI instead of the 2026 exemption of $2,925." us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0." us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE." us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE." @@ -5263,89 +5761,101 @@ us,scenario_071,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligib us,scenario_071,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model stated that the person’s income is below New York’s MAGI Medicaid threshold, but the traced MAGI income is 2.28 times FPL. It also failed to account for the absence of any qualifying Medicaid category." us,scenario_071,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated $240 of medical expenses as enough to establish spend-down eligibility even though subtracting them leaves $36,302, far above its cited $21,024 annual limit. The existence of a spend-down program and some medical expenses does not satisfy the spend-down, and the person qualifies through no Medicaid category." us,scenario_071,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage income by multiplying the listed 20 weekly work hours by an assumed New York minimum wage, despite the instruction that unlisted numeric inputs are zero. Social Security and Medicare employee taxes therefore apply to a zero wage base, not the model's imputed earnings, so payroll_tax is $0." -us,scenario_071,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly classified an estimated New York household credit as refundable. That credit is nonrefundable, so its asserted $63 belongs outside state_refundable_credits; the household qualifies for no refundable New York credit, yielding $0." +us,scenario_071,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It counted the NY household credit as a refundable credit and reported $63. That credit is nonrefundable under Tax Law §606(b), and NY tax here is zero once the $20,000 pension exclusion and $8,000 standard deduction apply, so the credit contributes nothing. It also interpolated $63 when the single-filer table gives a flat $60 for federal AGI of $5,001–$6,000. It correctly left out IT-214 but gave the wrong reason: the credit is ruled out because household gross income, which includes Social Security, is $36,542 against an $18,000 cap." +us,scenario_071,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It stated outright that the NY household credit is refundable and reported the $60 amount for the $5,000–$6,000 federal AGI band. The household credit is nonrefundable under Tax Law §606(b) and can only offset NY tax, which is zero here after the pension exclusion and standard deduction. No refundable NY credit (EITC, Empire State child credit, IT-214 real property tax credit) applies, so the correct total is $0." us,scenario_072,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated Medicare eligibility as an unlisted boolean that defaults to false instead of deriving it from age. The head is 80, exceeds the age-65 threshold, and is Medicare-eligible; enrollment status need not be provided." us,scenario_072,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly treated the Additional Medicare Tax threshold as an exemption from regular Medicare tax. Regular employee Medicare tax applies to all $20,000 of wages, adding $290 to the $1,240 Social Security tax for a total of $1,530." us,scenario_072,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $1,240 of Social Security tax plus $290 of Medicare tax and explicitly obtained $1,530, then replaced that exact result with an unsupported rounded value of $1,460. Payroll tax is not rounded from $1,530 to $1,460." us,scenario_072,payroll_tax,minimax-m3,llm_error,other,False,"The model's stated components correctly total $1,530, but it submitted $1,550. This is an arithmetic/output inconsistency: $1,240 plus $290 equals $1,530, with no Additional Medicare Tax or Michigan employee payroll tax to supply the extra $20." +us,scenario_072,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model treated the elderly household as exempt from any income screen and never compared its $73,974 gross income with Michigan's two-person gross income limit, which the household far exceeds, so SNAP is $0. It also reached its net-income step by counting both premium fields for each spouse, giving a $66,200 medical total and a $65,780 deduction. It then subtracted the $210 monthly standard deduction from annual income, which produced the spurious $332/month net income and the $446.40 monthly allotment." us,scenario_072,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model treated the absence of an explicit Medicare-status input as disqualifying and ignored the age-based eligibility rule. At age 77, the spouse exceeds the standard Medicare eligibility age of 65, so spouse_medicare_eligible equals Yes." -us,scenario_072,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It treated Michigan's senior relief as consisting only of the born-before-1946 interest/dividend subtraction plus two personal exemptions ($11,600), so $20,000 of wages survived and was taxed at 4.25%. Michigan's Tier 3 senior standard deduction of $40,000 joint applies against all income including wages for filers 67 and older, and it exceeds the $20,166 Michigan AGI, driving taxable income and tax to $0." -us,scenario_072,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied only two $5,700 personal exemptions to $20,166 of Michigan AGI and taxed the $8,766 remainder at 4.25%. It never applied the Michigan senior standard deduction of $40,000 for a joint return where both filers have reached 67, which absorbs the entire $20,166 and leaves $0 of Michigan taxable income." -us,scenario_072,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It invented a $3,200-per-person senior exemption on top of $6,000 regular exemptions (Michigan repealed the senior exemption in the 2011 reform and replaced it with the $20,000/$40,000 senior standard deduction) and applied the 4.05% rate that governed only tax year 2023 rather than the 4.25% statutory rate for 2026. The correct step is the $40,000 joint Tier 3 deduction against all income, which exceeds the $20,166 Michigan AGI and yields $0." -us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $850 is exactly 4.25% of $20,000, so despite asserting an adjustment for personal exemptions it taxed the full wage amount with no exemption and no deduction. Michigan grants filers 67 and older a $40,000 joint standard deduction against all income, which more than covers the $20,166 Michigan AGI and produces $0 of tax." -us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $1,638 is 4.25% of roughly $38,540, a base consistent with taxing the $20,000 of wages together with Social Security income that Michigan subtracts in full, and with taking no personal exemption and no senior deduction. Michigan AGI here is $20,166 after the full Social Security subtraction, and the $40,000 joint senior standard deduction for filers 67 and older wipes it out entirely." -us,scenario_072,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It correctly identified the $40,000 joint senior standard deduction but then reduced it by the household's $34,608 of gross Social Security benefits to $5,392 and discarded it in favor of the personal exemptions. Michigan reduces that deduction only by military pay and railroad/military retirement benefits; electing it forgoes the Social Security subtraction (worth $2,735 of taxable benefits here), so the full $40,000 still exceeds the $22,901 of federal AGI and produces $0 of tax." -us,scenario_072,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It built a $14,700 allowance from ~$5,850 personal exemptions plus a fabricated $1,500 senior exemption, which Michigan eliminated in the 2011 tax reform. The surviving provision is the Tier 3 senior standard deduction of $40,000 for a joint return where both spouses are 67 or older, applied against all income; it exceeds the $20,166 Michigan AGI, so taxable income is $0." -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It subtracted only two $5,900 personal exemptions from $20,166 and used the 4.05% rate that applied solely to tax year 2023 instead of the 4.25% rate in force for 2026. The determinative omission is the $40,000 joint senior standard deduction against all income for filers 67 and older, which exceeds Michigan AGI and yields $0 regardless of which rate is used." -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied only two estimated personal exemptions to the $20,166 remaining after excluding Social Security and veterans benefits, then taxed the balance at 4.25%. Michigan's senior standard deduction of $40,000 on a joint return where both filers have reached 67 applies to that same $20,166 of wage, dividend, and interest income and reduces Michigan taxable income to $0." -us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It described ""two senior personal exemptions"" leaving $7,566 of taxable income, conflating the repealed pre-2012 senior exemption with a modest bump to the ordinary personal exemption. Michigan's actual senior provision is the $20,000/$40,000 Tier 3 standard deduction against all income for filers 67 and older; at $40,000 joint it exceeds the $20,166 Michigan AGI and leaves $0 of tax." -us,scenario_072,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It performed the Michigan Social Security subtraction correctly ($22,901 − $2,735 = $20,166) but then allowed only two $5,600 personal exemptions and taxed $8,966 at 4.25%. It omitted the $40,000 joint senior standard deduction available to filers 67 and older against all income, which absorbs the full $20,166 and produces $0." -us,scenario_072,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It allowed only $10,000 of personal exemptions and applied the 4.05% rate that was in effect only for tax year 2023 rather than 4.25% for 2026. Both errors are subordinate to the omitted $40,000 joint senior standard deduction against all income for filers 67 and older, which exceeds the $20,166 Michigan AGI and drives the tax to $0." -us,scenario_072,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted taxable Social Security and about $12,000 of personal exemptions from federal AGI, then taxed the $8,166 balance at 4.25%. It never applied Michigan's Tier 3 senior standard deduction of $40,000 joint against all income for filers 67 and older, which covers the entire $20,166 of Michigan AGI and leaves $0 of tax." -us,scenario_072,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It stopped at two $6,100 personal exemptions against $20,166 of Michigan AGI and taxed $7,966 at 4.25%. The missed step is the $40,000 joint senior standard deduction against all income for taxpayers who have reached 67, fully available in 2026, which exceeds Michigan AGI and yields $0." -us,scenario_072,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It cycled through invented senior personal exemption amounts ($6,500, $7,900, $7,100) — Michigan repealed the senior exemption in 2011 and has a single indexed personal exemption — and combined them with the $166 interest/dividend subtraction, settling on $7,000 of taxable income at 4.25%. The controlling provision it never reached is the $40,000 joint Tier 3 senior standard deduction against all income for filers 67 and older, which exceeds the $20,166 Michigan AGI and makes the tax $0." -us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as proof that the head qualifies through an aged/disabled Medicaid pathway. SSDI receipt does not itself establish that categorical pathway, and the head qualifies through none of Michigan’s Medicaid categories." -us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the desired traditional 401(k) contribution from Medicaid MAGI despite the absence of wages from which that employee deferral could be made. The resulting MAGI is 1.60 times FPL, above Michigan’s expansion-adult limit." -us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the head’s Medicaid-countable income as low without applying the expansion-group income threshold. The engine’s MAGI calculation is 1.60 times FPL, which exceeds the Michigan expansion-adult limit, and no alternative category applies." -us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the expansion-group income condition was met, but the head’s MAGI is 1.60 times FPL and exceeds that pathway’s limit. The head therefore cannot qualify as a Michigan expansion adult, and no other category applies." -us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model excluded all nontaxable SSDI from Medicaid MAGI by treating MAGI as limited to taxable income. Medicaid MAGI adds nontaxable Social Security benefits, producing income of 1.60 times FPL and placing the head above the expansion-adult limit." -us,scenario_073,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model counted only the pension and excluded SSDI because no Social Security benefit was taxable for income-tax purposes. Medicaid MAGI includes nontaxable Social Security benefits, yielding 1.60 times FPL and disqualifying the head from the expansion-adult pathway." -us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model inferred Medicaid eligibility from the absence of wages and ignored the SSDI and pension income used in the Medicaid income calculation. Those sources produce MAGI of 1.60 times FPL, above the expansion-adult limit, and the head qualifies through no other category." -us,scenario_073,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model read the prompt's ""status constant throughout the tax-benefit year"" stipulation as proof that the 24-month SSDI qualifying period was already served, but that stipulation fixes status only within 2026 and says nothing about entitlement onset in prior years. PolicyEngine's is_medicare_eligible keys on the age-65 threshold and an explicit disability/ESRD determination flag, neither of which is met at age 57 with no disability status listed, so the answer is 0." -us,scenario_073,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model asserted that Social Security disability income ""confers Medicare eligibility under PolicyEngine rules,"" attributing to the model a pathway it does not implement: is_medicare_eligible tests the age-65 threshold and explicit disability determination inputs, not the ssdi income variable. At age 57 with every disability status input unlisted and therefore false, the flag is 0." -us,scenario_073,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model explicitly conditioned its answer on ""assuming established disability status"" — inventing prior-year SSDI entitlement that the household facts never supply and that the prompt's do-not-infer-unlisted-facts rule forbids. PolicyEngine resolves Medicare eligibility from the age-65 threshold, and 57 fails it, so the value is 0." -us,scenario_073,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model supplied the 24-month SSDI waiting period as satisfied from an entitlement history the facts do not contain — only a single benefit-year of $22,646 in disability income is given, with no onset date. PolicyEngine's is_medicare_eligible fires on age >= 65 or an explicit disability/ESRD flag, and the head at 57 with no such flag set returns 0." -us,scenario_073,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model skipped the 24-month statutory qualifying period entirely, treating bare receipt of Social Security disability benefits as immediately conferring Medicare ""regardless of being under 65."" PolicyEngine determines is_medicare_eligible from the age-65 threshold and explicit disability determination inputs; at age 57 with no disability status listed, the flag is 0." -us,scenario_073,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model converted ""disability income is listed as a full-year, ongoing benefit"" into 24 months of completed SSDI receipt, but a full-year annual amount for 2026 carries no information about entitlement before 2026. The reference computes is_medicare_eligible from the age-65 threshold, which the 57-year-old head fails, giving 0." -us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model stated the head is ""treated as having completed the required waiting period,"" an assumption sourced from nothing in the household facts, which give only a 2026 SSDI amount with no entitlement start date. PolicyEngine's Medicare flag turns on age >= 65 or an explicit disability determination input, so at age 57 with no such input the value is 0." -us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model applied a bare SSDI-recipient-equals-Medicare-beneficiary rule, omitting both the 24-month qualifying period and the fact that PolicyEngine's is_medicare_eligible reads age and disability determination flags rather than the disability income variable. Age 57 is below the 65 threshold and no disability status is listed, so the flag is 0." -us,scenario_073,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model treated the SSDI income line as directly qualifying for Medicare, collapsing a two-year entitlement requirement and an age test into a single income fact. PolicyEngine sets is_medicare_eligible from the age-65 threshold plus explicit disability/ESRD determination inputs; the 57-year-old head meets neither, so the answer is 0." -us,scenario_073,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model reasoned solely from ""Head receives SSDI, qualifying them for Medicare,"" never applying the age-65 threshold that governs PolicyEngine's is_medicare_eligible nor the 24-month entitlement period the facts do not establish. At age 57 with all disability status inputs unlisted and thus false, the flag is 0." -us,scenario_073,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model treated the presence of a ""Social Security disability income"" dollar amount as ""evidence of Medicare-eligible disability status,"" but an income line item is not the disability determination input PolicyEngine reads, and the prompt makes every unlisted status input false. With age 57 below the 65 threshold and no disability flag set, is_medicare_eligible is 0." -us,scenario_073,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model invoked a ""disability pathway"" triggered by full-year SSDI income, a pathway PolicyEngine's is_medicare_eligible does not implement — it evaluates the age-65 threshold and explicit disability/ESRD determination inputs. The head is 57 with no disability status listed, so the value is 0." -us,scenario_073,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model claimed SSDI receipt ""establishes disability-based Medicare eligibility under the applicable rules,"" skipping the 24-month entitlement requirement that the single-year facts do not satisfy and the age-65 test that PolicyEngine actually applies. At age 57 with no disability determination input set, the flag is 0." -us,scenario_073,head_medicare_eligible,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"The model asserted a disability pathway from the SSDI income figure alone, with no age test and no accounting for the 24-month qualifying period. PolicyEngine's is_medicare_eligible is satisfied by age >= 65 or an explicit disability/ESRD determination, and the 57-year-old head with all status inputs false returns 0." -us,scenario_073,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model routed the head through a ""disability-benefit pathway"" keyed to the presence of SSDI income, which is not how PolicyEngine computes is_medicare_eligible — that variable tests the age-65 threshold and explicit disability determination inputs. Age 57 fails the threshold and no disability status is listed, so the answer is 0." -us,scenario_073,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"The model both assumed the 24-month SSDI period was served for ""an ongoing disability at age 57"" — a prior-year fact the household never supplies — and read the input label ""health insurance premiums excluding Medicare Part B"" as corroborating Medicare enrollment, when that label is a standard premium field and the household's Medicare Part B premium is unlisted and therefore zero. PolicyEngine resolves is_medicare_eligible from the age-65 threshold and explicit disability determination inputs, so a 57-year-old with no such flag gets 0." -us,scenario_073,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model claimed that SSDI receipt ""under PolicyEngine rules confers Medicare eligibility,"" misattributing a real-world pathway to the model; PolicyEngine's is_medicare_eligible reads age against the 65 threshold and explicit disability/ESRD determination inputs, not the disability income variable. The head is 57 with every disability status input false, so the flag is 0." +us,scenario_072,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,age_disability,False,"It placed both spouses in the born-before-1946 tier, but ages 80 and 77 in 2026 mean births in 1946 and 1949, which is Tier 2. It then subtracted only the $166 of interest and dividends plus $11,600 of exemptions and concluded wages are not subtractible. Tier 2 filers past 67 get a $40,000 joint Michigan standard deduction against all income, wages included, which reduces the $20,166 base to $0." +us,scenario_072,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It subtracted only about $11,600 of personal exemptions and the interest/dividend subtraction, leaving $8,400 taxed at 4.25%. It never applied the $40,000 joint Michigan standard deduction for filers born 1946-1952 who are past 67. That deduction covers wages and takes the $20,166 base to zero." +us,scenario_072,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It correctly reached $20,166 of Michigan income after excluding Social Security, but then subtracted only two $5,700 personal exemptions and taxed $8,766. Both spouses were born in 1946-1952 and are past 67, so the $40,000 joint Michigan standard deduction applies against all income and eliminates the base, giving $0." +us,scenario_072,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It reduced $20,166 by $18,400 of regular plus senior exemptions and taxed the remaining $1,766 at 4.05%. It missed the $40,000 joint Michigan standard deduction for Tier 2 filers (born 1946-1952, past 67), which applies against wages and zeroes the tax." +us,scenario_072,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted $17,800 of personal-plus-senior exemptions from $20,166 and taxed the leftover $2,366. It never applied the $40,000 joint Michigan standard deduction available to the 1946 and 1949 birth cohort after age 67. That deduction exceeds the couple's taxable wages, interest, and dividends, so the tax is $0." +us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"$850 is exactly 4.25% of $20,000, so it taxed the full wages with no effective exemption or deduction. It missed the $40,000 joint Michigan standard deduction for filers born 1946-1952 who are past 67, which applies against wages and brings tax to $0." +us,scenario_072,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"$1,638 at the 4.25% rate implies about $38,500 of Michigan taxable income. That exceeds the couple's $20,166 of wages, interest, and dividends, so it could only come from adding Social Security or veterans benefits, which Michigan excludes. It also never applied the $40,000 joint Michigan standard deduction for the 1946-1952 cohort, which zeroes the tax." +us,scenario_072,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It correctly identified the $40,000 joint Michigan standard deduction for filers born 1946-1952. It then wrongly cut that deduction by the $34,608 of Social Security benefits, leaving $5,392, and fell back to $12,000 of personal exemptions. Social Security is a separate Michigan subtraction and does not offset the standard deduction, so the full $40,000 covers the $20,166 base and the tax is $0." +us,scenario_072,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It subtracted about $14,700 of personal and $1,500 senior exemptions from $20,166 and taxed the leftover $5,466 at 4.25%. It never applied the $40,000 joint Michigan standard deduction for Tier 2 filers past 67, which applies against all income and eliminates the tax." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It subtracted only two $5,900 personal exemptions from $20,166 and taxed the rest at 4.05%. It missed the $40,000 joint Michigan standard deduction for filers born 1946-1952 who are past 67, which applies against wages and takes Michigan taxable income to $0." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It correctly excluded Social Security and veterans benefits, but it offset the remaining $20,166 with only two personal exemptions, about $11,800, and taxed the rest at 4.25%. It omitted the $40,000 joint Michigan standard deduction for the 1946-1952 birth cohort past 67, which zeroes the base." +us,scenario_072,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It treated the couple's age benefit as two senior personal exemptions, about $6,300 each, leaving $7,566 taxed at 4.25%. The age-based rule it missed is the $40,000 joint Michigan standard deduction for filers born 1946-1952 after age 67. That deduction applies against all income and reduces the tax to $0." +us,scenario_072,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It subtracted two $5,600 personal exemptions from $20,166 and taxed $8,966 at 4.25%. It never applied the $40,000 joint Michigan standard deduction for Tier 2 seniors (born 1946 and 1949, both past 67), which exceeds the couple's wages, interest, and dividends and makes the tax $0." +us,scenario_072,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It subtracted only $10,000 of personal exemptions from $20,166 and taxed $10,166 at 4.05%. It missed the $40,000 joint Michigan standard deduction for filers born 1946-1952 who are past 67, which applies against wages and eliminates the tax." +us,scenario_072,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It subtracted two roughly $6,000 personal exemptions and taxed $8,166 at 4.25%. It omitted the $40,000 joint Michigan standard deduction for the 1946-1952 birth cohort past age 67, which applies against all income and brings Michigan taxable income to $0." +us,scenario_072,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It subtracted two $6,100 personal exemptions from $20,166 and taxed $7,966 at 4.25%. It never applied the $40,000 joint Michigan standard deduction for Tier 2 filers (born 1946-1952, past 67), which fully offsets the wages, interest, and dividends and produces $0 tax." +us,scenario_072,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It cycled through invented senior personal-exemption amounts ($6,500 and $7,900 each), subtracted interest and dividends, and settled on $7,000 taxed at 4.25%. It never identified the $40,000 joint Michigan standard deduction for filers born 1946-1952 who are past 67, which applies against wages and zeroes the tax." +us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly found MAGI of about $25,754, above the 138% FPL expansion limit. It then made up a disability route based on SSDI receipt and $3,600 of assets. PolicyEngine's disability route requires actual SSI receipt (SSI is $0 here) or a disability flag with income within Michigan's aged/disabled limit of 100% FPL. The head has no disability flag and income at 160% FPL, so no category applies." +us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the $5,557 traditional 401(k) contribution from MAGI. The head has no wages, and elective deferrals can only come out of wages, so nothing is deducted and MAGI stays at $25,754. It also used the outdated $15,060 FPL and suggested SSDI could be excluded, but Medicaid MAGI adds untaxed Social Security back. Income is 1.60 times FPL, above the 138% expansion limit." +us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model called the income 'low' without running the MAGI test. Medicaid MAGI counts the full $22,646 of SSDI plus the $3,108 pension, which is 1.60 times FPL and above Michigan's 138% FPL adult expansion limit. With no SSI receipt and no dependent children, the head fits no other category." +us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model said the head met the expansion income test with 'low Medicaid-countable income'. Medicaid MAGI adds back all nontaxable Social Security, so countable income is about $25,754, or 1.60 times FPL. That is above the 138% FPL expansion limit." +us,scenario_073,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model left nontaxable SSDI out of MAGI and counted only the $3,108 pension. Medicaid MAGI under 42 CFR 435.603(e) adds untaxed Social Security benefits back, so MAGI is $25,754, or 1.60 times FPL. That is above Michigan's 138% FPL adult expansion limit." +us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model said only taxable income counts for Medicaid MAGI and dropped the $22,646 of SSDI. Medicaid MAGI is AGI plus untaxed Social Security, tax-exempt interest and excluded foreign income, so the full SSDI counts. Total MAGI is $25,754, or 1.60 times FPL, above the 138% expansion limit." +us,scenario_073,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model applied the tax-law rule that Social Security counts only to the extent it is taxable. Medicaid MAGI instead adds all untaxed Social Security benefits back. MAGI is therefore $25,754, or 1.60 times FPL, not $3,108, which is above the 138% FPL expansion limit it cited." +us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the lack of wages as settling eligibility and never ran an income test. The $22,646 of SSDI and $3,108 pension give a Medicaid MAGI of 1.60 times FPL, above Michigan's 138% expansion limit. The head receives no SSI and has no dependent children, so no other category applies." +us,scenario_073,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"It named the 24-month SSDI waiting period correctly, then took the full-year, no-change assumption as proof the period was met. That assumption covers only the tax year, which is 12 months at most. The prompt's default makes the unlisted months of prior SSDI receipt 0, so the 24-month test fails and the head is not eligible." +us,scenario_073,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It said receiving Social Security disability income by itself makes the head Medicare eligible and never applied the 24-month waiting period. The months of SSDI receipt are unlisted and count as 0, so the disability route is not met and the 57-year-old head is not eligible." +us,scenario_073,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It stated the 24-month SSDI rule, then assumed disability status was already established. That assumption adds a fact the household does not list. Under the prompt's rule the months of SSDI receipt are 0, so the waiting period is not met and the head is not eligible." +us,scenario_073,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It noted that Medicare for people under 65 on SSDI starts only after 24 months, then treated the head as eligible anyway. Nothing in the household shows 24 months of receipt. The months of SSDI receipt default to 0, so the disability route fails and the answer should have been 0." +us,scenario_073,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"It treated current SSDI receipt as granting Medicare regardless of age and left out the 24-month waiting period. With the months of receipt unlisted and set to 0, the head is 57 and qualifies by neither age nor disability." +us,scenario_073,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It read the full-year, ongoing SSDI as meaning the 24-month waiting period was met. A benefit that is steady within the tax year shows at most 12 months of receipt. The prompt sets unlisted months of receipt to 0, so the head fails the 24-month test and is not eligible." +us,scenario_073,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It treated the head as having finished the SSDI waiting period, but no household fact says so. The prompt's default makes the months of SSDI receipt 0, so the 24-month requirement is not met and the 57-year-old head is not Medicare eligible." +us,scenario_073,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It counted being an SSDI recipient as enough for Medicare and skipped the 24-month waiting period. The months of receipt are unlisted and count as 0, so the head meets neither the age-65 test nor the disability test." +us,scenario_073,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It treated receiving SSDI as automatic Medicare eligibility and ignored the 24-month waiting period. With the months of receipt at their default of 0 and the head aged 57, no route to Medicare applies." +us,scenario_073,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It linked current SSDI receipt directly to Medicare without the 24-month waiting period. The household gives no duration of SSDI receipt, which defaults to 0, so the disability route is not met and the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It correctly said a 57-year-old needs a qualifying disability status, then treated the SSDI income as evidence that status was met. Disability-based Medicare requires 24 months of benefits, and the unlisted months of receipt count as 0, so the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"It took a full year of SSDI income as meeting the disability route. That route requires 24 months of benefits, and one year of income shows at most 12. The prompt's default of 0 for unlisted months of receipt means the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It said SSDI receipt establishes disability-based Medicare for someone under 65 and skipped the 24-month waiting period. The unlisted months of SSDI receipt count as 0, so the waiting period is not met and the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It treated receiving SSDI as automatically meeting the disability route to Medicare without the 24-month waiting period. The months of receipt are unlisted and set to 0, so the 57-year-old head qualifies by neither age nor disability." +us,scenario_073,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"It applied the disability route based only on current SSDI receipt and left out the 24-month waiting period. With months of receipt at the prompt's default of 0, the route fails and the head is not eligible." +us,scenario_073,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted SSDI receipt alone as enough for disability-based Medicare, skipping the rule that benefits must be received for 24 months first. Unlisted months of receipt count as 0, so the head is not Medicare eligible." +us,scenario_073,head_medicare_eligible,inkling,prompt_ambiguity,age_disability,False,"It assumed the 24-month SSDI waiting period was met for an ongoing disability, but the months of receipt are unlisted and count as 0. It also misread the input name 'health insurance premiums excluding Medicare Part B' as evidence of Medicare enrollment. That label only names the premium variable and says nothing about Part B coverage." +us,scenario_073,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"It said SSDI receipt gives Medicare eligibility under PolicyEngine rules and never applied the 24-month waiting period. With the months of receipt at their default of 0 and the head aged 57, the head is not eligible." us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no payroll-taxable wages and explicitly derived payroll tax of $0, but then submitted $2,846. Its numeric output contradicts its own payroll-tax-base reasoning and final stated derivation." -us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It computed net income of ~$1,757/mo essentially correctly — its $190 excess medical deduction (($2,600 + $100)/12 − $35) and ~$200 standard deduction reproduce the engine's $1,755.47 — and then disqualified the household on the 100% FPL net income test. Michigan's broad-based categorical eligibility as a TANF non-cash household waives both the net income and asset tests, and an eligible one-person household whose computed allotment is zero receives the minimum allotment of 8% of the $298 maximum, $23.84/mo." -us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It derived net income of $1,747/mo with the correct $190 excess medical deduction, then declared the household ineligible for exceeding the ~$1,305 one-person net income limit. BBCE categorical eligibility in Michigan removes the net income test, and the resulting zero computed allotment ($298 maximum minus a $526.50 expected contribution) is floored at the one/two-person minimum of $23.84/mo." -us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a 130% FPL gross income test and a $2,750 resource limit, neither of which governs here: the head's $22,646 of SSDI establishes a disabled household, exempt from the gross income test, and Michigan's BBCE raises the gross screen to 200% FPL while waiving the asset test entirely. Having declared ineligibility, it never reached the benefit step, where the zero computed allotment converts to the $23.84/mo one-person minimum." -us,scenario_073,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $5,789 employer-sponsored premium plus $2,600 of other premiums as deductible out-of-pocket medical expense, a ~$5,165 excess medical deduction that drove net income to zero and paid the full $298 maximum allotment for twelve months. PolicyEngine counts only the $2,600 of non-Medicare premiums and $100 of OTC expense, giving a $190/mo excess medical deduction and net income of $1,755.47, which produces a zero computed allotment and therefore the $23.84/mo minimum, not the maximum." -us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"It used a ~$1,696/mo gross limit and then asserted net income still exceeds the 100% FPL net limit, concluding ineligibility. Michigan's BBCE screens gross income at 200% FPL — the household is at 165% — and waives the net income and asset tests, so the household is eligible and receives the 8%-of-maximum minimum allotment of $23.84/mo even though its $526.50 expected contribution exceeds the $298 maximum." -us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"Its eligibility path was right — BBCE at 200% FPL with no asset test — but it used a maximum allotment of ~$587/mo, which is a two-person figure; the FY2026 one-person contiguous-US maximum is $298. Subtracting 30% of even its own understated net income from $298 yields a negative computed allotment, so the correct payment is the one-person minimum of $23.84/mo rather than a $214/mo residual; it also inflated the medical deduction by counting the $5,789 employer-sponsored premium, which is excluded." -us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It computed net income of $1,752/mo, matching the engine's $1,755.47, and then applied the 100% FPL net income test as a hard disqualifier. Michigan's TANF non-cash broad-based categorical eligibility waives that test, and the zero computed allotment that follows ($298 maximum minus $526.50 expected contribution) is floored at the one/two-person minimum of 8% of the maximum, $23.84/mo." -us,scenario_073,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It stopped at the observation that 30% of net income exceeds the maximum allotment and returned zero. Federal rules floor an eligible one- or two-person household's benefit at 8% of the one-person maximum allotment — $23.84/mo for FY2026, rising to $24.37 in October 2026 — which is exactly the payment issued here." -us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It found the correct pathway — disabled household, medical deduction, computed allotment of zero, minimum benefit — but valued the minimum at a rounded ~$25/mo. The minimum allotment is 8% of the one-person maximum: $23.84/mo for FY2026 and $24.37 from October 2026, totaling $287.68 across calendar 2026 rather than $300." -us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It failed the household on a 130% FPL gross income test of ~$19,578. The head's SSDI makes this a disabled household that is exempt from the gross income test, and Michigan's BBCE screens gross income at 200% FPL, so the household is eligible and receives the $23.84/mo one-person minimum allotment." -us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It disqualified the household on the one-person net income limit without computing the standard or excess medical deductions. Michigan's BBCE waives the net income test, and although net income of $1,755.47 produces a $526.50 expected contribution against the $298 maximum, the eligible one-person household still receives the 8% minimum allotment of $23.84/mo." -us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It identified the exact mechanism — BBCE eligibility below 200% FPL plus the minimum benefit when the computed allotment is zero — but priced the minimum at a flat $23/mo. The FY2026 minimum is 8% of the $298 maximum, $23.84, stepping to $24.37 for October through December 2026, which totals $287.68." -us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It asserted that countable income exceeds the one-person eligibility threshold, with no deduction or BBCE analysis. Gross income is 165% of the poverty guideline, inside Michigan's 200% BBCE screen, and categorical eligibility waives the net income and asset tests, so the household is eligible and paid the $23.84/mo minimum allotment." -us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly reasoned that with no shelter deduction 30% of net income exceeds the maximum allotment, then returned zero instead of the minimum benefit. Eligible one- and two-person households receive at least 8% of the one-person maximum allotment — $23.84/mo here — no matter how far the expected contribution exceeds the maximum." -us,scenario_073,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed net income of $1,541.50, found the $462.45 expected contribution exceeds the maximum allotment, and returned zero. The zero computed allotment matches the engine's result from net income of $1,755.47, but an eligible one-person household is floored at 8% of the $298 maximum, $23.84/mo, which it omitted." -us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no derivation and asserted that assets or income exceed SNAP limits. The $3,600 in bank and stock assets and gross income at 165% of the poverty guideline both clear Michigan's BBCE, which waives the asset and net income tests, and the eligible one-person household receives the $23.84/mo minimum allotment." -us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly reached BBCE categorical eligibility and the one-person minimum benefit but valued it at $23/mo for all twelve months. The minimum is 8% of the maximum allotment: $23.84 for January through September 2026 and $24.37 for October through December, yielding $287.68." -us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It returned zero on the ground that net income exceeds the maximum limit for a positive benefit. Michigan's TANF non-cash categorical eligibility removes the net income test, and the zero computed allotment that results from the $526.50 expected contribution is replaced by the one/two-person minimum allotment of $23.84/mo." -us,scenario_073,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It combined a net income test failure with the finding that 30% of net income exceeds the maximum allotment. Categorical eligibility under Michigan's BBCE waives the net income test, and the minimum allotment of 8% of the $298 one-person maximum, $23.84/mo, applies precisely when the computed benefit is zero." -us,scenario_073,snap,glm-5.2,llm_error,thresholds_rates,False,"It deducted $10,669 of medical expense by counting the $5,789 employer-sponsored premium and both $2,600 premium entries, then correctly derived a negative allotment and returned zero. The deductible medical expense is $190/mo, and the decisive omission is the minimum allotment: an eligible one-person household with a zero computed benefit receives 8% of the $298 maximum, $23.84/mo." -us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"It compared $25,754 of gross income directly to a ~$15,060 net income limit, skipping the standard deduction, the excess medical deduction, and Michigan's BBCE screen. Categorical eligibility waives the net income test, so the household is eligible and receives the $23.84/mo minimum allotment." -us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It asserted the income is well above one-person eligibility limits without applying any deduction or Michigan's BBCE screen. Gross income sits at 165% of the poverty guideline, inside the 200% categorical eligibility limit, and the eligible household is paid the 8%-of-maximum minimum allotment of $23.84/mo." -us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It refused to establish eligibility, citing absent housing costs and take-up indicators, though the prompt directs assuming take-up and treating unlisted facts as zero. With no shelter deduction, the $200.70 standard deduction and $190 excess medical deduction leave net income of $1,755.47, a zero computed allotment, and the $23.84/mo one-person minimum benefit." -us,scenario_073,snap,gpt-5.6-luna,llm_error,age_disability,False,"It stated that no qualifying elderly or disabled status is listed, but $22,646 of Social Security disability income establishes a disabled member, which exempts the household from the gross income test and unlocks the excess medical deduction. Michigan's BBCE independently waives the income tests, and the eligible one-person household receives the $23.84/mo minimum allotment." -us,scenario_073,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"It concluded the income is too high to produce a positive allotment after deductions and the 30% benefit reduction. That computation matches the engine — $298 maximum minus a $526.50 expected contribution — but the result is floored at the one/two-person minimum allotment of 8% of the maximum, $23.84/mo, not zero." -us,scenario_073,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied the standard and medical deductions and stopped at counted income exceeding the level that yields a benefit. The engine reaches the same zero computed allotment from net income of $1,755.47 and then pays the minimum allotment for an eligible one-person household, $23.84/mo rising to $24.37 in October 2026." -us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"It disqualified the household on the one-person net income limit with no deduction computation. Michigan's broad-based categorical eligibility as a TANF non-cash household waives the net income and asset tests, and the eligible household receives the 8%-of-maximum minimum allotment of $23.84/mo." -us,scenario_073,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly identified the disabled household, the asset clearance, and the net-income-only screen, then returned zero because the 30% expected contribution fully offsets the maximum allotment. That full offset is exactly the trigger for the minimum allotment rule: an eligible one- or two-person household receives 8% of the $298 one-person maximum, $23.84/mo." -us,scenario_073,snap,grok-4.6,llm_error,thresholds_rates,False,It found that with no shelter costs 30% of net income exceeds the one-person maximum allotment and returned zero. The minimum allotment floor for eligible one- and two-person households — $23.84/mo for FY2026 and $24.37 from October — applies in precisely that situation. -us,scenario_073,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It computed net income of ~$1,274 using an inflated $672 excess medical deduction that counts the $5,789 employer-sponsored premium, then returned zero because 30% of net exceeds a ~$310 maximum allotment. The deductible medical expense is $190/mo and the one-person maximum is $298, but the decisive error is skipping the $23.84/mo minimum allotment that applies whenever the computed benefit is zero." -us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"It reached the right structure — eligibility with assets under the disabled-household limit, a zero computed allotment, and the one/two-person minimum — but valued the minimum at $23/mo for all twelve months. The minimum is 8% of the maximum allotment: $23.84 for January through September 2026 and $24.37 for October through December, totaling $287.68." -us,scenario_073,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It stopped at the finding that 30% of net income exceeds the one-person maximum allotment and reported no benefit. The minimum allotment rule pays an eligible one-person household 8% of the maximum, $23.84/mo, exactly when the computed allotment is zero." -us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"It subtracted only the standard deduction, reaching net income of ~$1,942 and omitting the $190/mo excess medical deduction that the head's disability status unlocks, then disqualified the household on the 100% FPL net income test. Michigan's BBCE waives that test, and the eligible one-person household receives the $23.84/mo minimum allotment despite a zero computed benefit." -us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"It failed the household on both income limits and treated $1,800 as disqualifying countable resources. Gross income is 165% of the poverty guideline against Michigan's 200% BBCE screen, categorical eligibility waives the net income and asset tests, and the eligible one-person household is paid the $23.84/mo minimum allotment." -us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"It denied any medical deduction despite the head's SSDI-established disability — the $2,600 of non-Medicare premiums plus $100 of OTC expense give $190/mo above the $35 threshold — and then disqualified the household on the 100% FPL net income test. Michigan's BBCE removes that test, and the eligible one-person household receives the minimum allotment of $23.84/mo." -us,scenario_073,snap,qwen-3.7-max,llm_error,age_disability,False,"It explicitly ruled the head not disabled for SNAP purposes despite $22,646 of Social Security disability income, then applied the 130% FPL gross income test from which a disabled household is exempt. Michigan's BBCE screens gross income at 200% FPL, the household clears at 165%, and it receives the $23.84/mo one-person minimum allotment." -us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"It disqualified the household because $3,600 of combined bank and stock assets exceeds the SNAP resource limit. Michigan's broad-based categorical eligibility waives the asset test outright, and the standard limit for a household with a disabled member exceeds $3,600 in any case; the eligible household receives the $23.84/mo minimum allotment." +us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model acknowledged Michigan's BBCE 200% FPL gross test and computed net income of about $1,757, which is close to the correct $1,755. It then still applied the 100% FPL net income test, even though categorical eligibility waives it. Because it treated the household as ineligible, it never applied the $24 monthly minimum allotment for eligible one-person households." +us,scenario_073,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model routed the household through the elderly/disabled path, which skips only the gross test, and failed it on the 100% FPL net income limit using net income of $1,747. It missed that Michigan's TANF non-cash categorical eligibility waives the net income test entirely, so the eligible household receives the $24 monthly minimum allotment." +us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the standard 130% FPL gross income limit and a $2,750 asset limit and declared the household ineligible. Michigan's BBCE applies a 200% FPL gross limit, which the household passes at 165% of FPL, and categorical eligibility removes the asset limit, so the $3,600 in assets does not disqualify it. The eligible household receives the $24 monthly minimum, not $0." +us,scenario_073,snap,claude-opus-4.7,llm_error,period_annualization,False,"The model subtracted an annual medical expense figure (about $5,165) from monthly income. That drove net income to about zero and gave the full $298 maximum for all 12 months. With monthly figures, net income is $1,755, and the 30% contribution of $527 exceeds the $298 maximum. The benefit therefore falls to the $24 minimum, or $288 a year." +us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model compared income to the 130% FPL gross limit and then to the 100% FPL net income limit, and concluded the household was ineligible. It missed that Michigan's BBCE uses a 200% FPL gross limit and waives the net income test. The categorically eligible one-person household therefore receives the $24 monthly minimum allotment." +us,scenario_073,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used a one-person maximum allotment of about $587 a month instead of the FY2026 figure of $298. It also counted the $5,789 employer-sponsored insurance premiums as out-of-pocket medical costs, which understated net income at $1,242. Against the correct $298 maximum, 30% of net income wipes out the formula benefit, so only the $24 minimum applies, not $214 a month." +us,scenario_073,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model correctly computed net income of about $1,747 but treated the 100% FPL net income limit as binding for this disabled household and zeroed the benefit. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household stays eligible and receives the $24 monthly minimum allotment." +us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test: $1,752 against a limit it put at $1,255. From that, it declared the household ineligible. It missed that Michigan's BBCE confers categorical eligibility, which removes the net income test. The eligible one-person household therefore gets the $24 minimum allotment instead of $0." +us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model found that the 30% net-income contribution fully offsets the maximum benefit and also cited a $1,255 net income limit, then set SNAP to $0. It skipped the rule that eligible one- and two-person households receive the minimum allotment, $24 a month in FY2026, whenever the formula benefit is below it. It also missed that categorical eligibility waives the net income limit." +us,scenario_073,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model computed net income of about $1,750 and denied benefits because that exceeds the 100% FPL net limit for elderly/disabled households. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household remains eligible and receives the $24 monthly minimum allotment." +us,scenario_073,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly saw that the 30% contribution exceeds the maximum allotment and that the minimum benefit applies. However, it used about $25 a month instead of the FY2026 minimum of $24, which is 8% of the $298 one-person maximum, rounded. That gave $300 a year instead of $288." +us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the 130% FPL gross income limit and found the household ineligible. It missed that Michigan's BBCE raises the gross limit to 200% FPL, which the household passes at 165%, and that SSDI receipt exempts this disabled household from the standard gross test. The eligible household receives the $24 monthly minimum allotment." +us,scenario_073,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model denied SNAP because net income exceeds the one-person net income limit. Categorical eligibility through Michigan's TANF non-cash rules waives the net income test, so the household stays eligible and receives the $24 monthly minimum allotment rather than $0." +us,scenario_073,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model said gross and net income exceed SNAP limits and returned $0. The household passes Michigan's 200% FPL BBCE gross limit, and categorical eligibility waives the net income test, so it is eligible and receives the $24 monthly minimum allotment, or $288 a year." +us,scenario_073,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly found BBCE eligibility and applied the one-person minimum benefit, but it used the FY2025 minimum of $23 instead of the FY2026 minimum of $24. The FY2026 minimum is 8% of the $298 maximum, rounded, so the annual total is $288 rather than $276." +us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model said countable income exceeds the eligibility threshold and returned $0. Michigan's BBCE 200% FPL gross limit is met at 165% of FPL, and categorical eligibility waives the net income test. The household is therefore eligible, and the $24 monthly minimum allotment applies." +us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the maximum allotment but then paid $0. It missed that eligible one-person households receive the minimum allotment, $24 a month in FY2026, when the formula benefit falls below it." +us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model computed a 30% contribution of $462.45, more than the maximum benefit, and stopped at $0. It never applied the minimum allotment that every eligible one- or two-person household receives: $24 a month in FY2026, or $288 a year." +us,scenario_073,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that assets and income exceed SNAP limits without computing them. Michigan's TANF non-cash categorical eligibility waives the asset and net income tests, and the household passes the 200% FPL gross limit at 165%. It is therefore eligible for the $24 monthly minimum allotment." +us,scenario_073,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly identified BBCE categorical eligibility and the minimum-benefit outcome, but it used $276 a year, the outdated $23 monthly minimum. The FY2026 minimum is $24, which is 8% of the $298 one-person maximum, rounded, so the annual total is $288." +us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model zeroed the benefit because net income is too high for a positive allotment. It missed both that categorical eligibility waives the net income limit and that eligible one-person households receive the $24 monthly minimum allotment regardless of the formula result. +us,scenario_073,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model applied the net income eligibility threshold and the 30% offset to reach $0. Michigan's BBCE categorical eligibility waives the net income test, and eligible one-person households get the $24 monthly minimum allotment even when 30% of net income exceeds the maximum." +us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model found the household eligible under the net income test and computed a negative formula benefit, then paid $0. It omitted the minimum allotment for eligible one- and two-person households, $24 a month in FY2026. Its net income was also too low because it counted the employer-sponsored insurance premiums as medical costs, though the minimum applies either way." +us,scenario_073,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model compared total gross income of $25,754 directly to a 100% FPL net income limit, without taking any deductions. Categorical eligibility waives the net income test, and the household's $1,755 monthly net income leaves it eligible for the $24 minimum allotment, or $288 a year." +us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model called income well above SNAP limits. The household's gross income is 165% of FPL, under Michigan's 200% BBCE limit, and categorical eligibility waives the net income test. The eligible one-person household therefore receives the $24 monthly minimum allotment." +us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said eligibility was not established and returned $0, even though the prompt instructs it to assume take-up. The household is categorically eligible through Michigan's BBCE: gross income is 165% of FPL, under the 200% limit. The formula benefit is below zero, so the $24 monthly minimum allotment applies." +us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model said no elderly or disabled status was listed, overlooking that SSDI receipt makes the head disabled for SNAP, and failed the household on the gross income test. The household also passes Michigan's 200% FPL BBCE gross limit at 165% of FPL, which confers categorical eligibility and the $24 monthly minimum allotment." +us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly found that the 30% benefit reduction wipes out the formula allotment, but then returned $0. It skipped the minimum allotment rule that guarantees eligible one-person households $24 a month in FY2026." +us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model took the standard and medical deductions, found that the remaining income yields no formula benefit, and returned $0. It missed that this categorically eligible one-person household receives the $24 monthly minimum allotment whenever the formula falls below it." +us,scenario_073,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model concluded that no positive allotment remains without shelter costs, and returned $0. It omitted the minimum allotment, $24 a month in FY2026, which eligible one- and two-person households receive whenever the formula benefit falls below it." +us,scenario_073,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model noted about $2,146 of monthly income with no shelter costs and set SNAP to zero. The household is categorically eligible, so even though 30% of its $1,755 net income exceeds the $298 maximum, it receives the $24 monthly minimum allotment, or $288 a year." +us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model denied SNAP because income exceeds the one-person net income limit. Michigan's TANF non-cash categorical eligibility waives the net income test, so the household remains eligible and receives the $24 monthly minimum allotment." +us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,The model correctly placed the disabled household under the higher asset limit and found that the 30% reduction fully offsets the maximum allotment. It then paid $0 instead of applying the $24 monthly minimum allotment that eligible one-person households receive. +us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum allotment and set the benefit to $0. It missed the minimum allotment rule, under which an eligible one-person household receives $24 a month in FY2026 when the formula benefit falls below it." +us,scenario_073,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model denied SNAP because net income stays above 100% of poverty after the medical and standard deductions. Categorical eligibility through Michigan's BBCE waives the net income test, so the household is eligible and receives the $24 monthly minimum allotment." +us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model computed that 30% of net income, about $382, exceeds the maximum allotment and returned $0. It skipped the minimum benefit for eligible one- and two-person households, $24 a month in FY2026. It also counted the employer-sponsored insurance premiums as medical expenses, though the minimum applies either way." +us,scenario_073,snap,inkling,llm_error,thresholds_rates,False,"The model correctly applied the one-to-two-person minimum benefit, but it used about $23 a month, the FY2025 figure. The FY2026 minimum is $24, which is 8% of the $298 one-person maximum, rounded, so the annual total is $288 rather than $276." +us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model found that 30% of net income exceeds the one-person maximum and paid no benefit. It omitted the minimum allotment that every eligible one-person household receives, $24 a month in FY2026, or $288 a year." +us,scenario_073,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income limit to a disabled household and zeroed the benefit because 30% of net income exceeds $298. Categorical eligibility waives the net limit, and the $24 monthly minimum allotment applies. The model also skipped the medical deduction for the $2,600 in health premiums." +us,scenario_073,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model said income exceeds both the gross and net limits. The household's 165% FPL gross ratio passes Michigan's 200% BBCE gross limit, and categorical eligibility waives the net income test. The household is eligible for the $24 monthly minimum allotment." +us,scenario_073,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed that Michigan's BBCE retains a 100% FPL net income test and denied benefits on that basis. BBCE categorical eligibility removes the net income test, so the household is eligible and receives the $24 monthly minimum allotment. The model also wrongly dropped the elderly/disabled medical deduction for this SSDI recipient." +us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model denied disabled status despite the head's SSDI receipt. It then applied a 130% FPL gross limit computed at only about $1,311 a month and declared the household ineligible. Michigan's BBCE applies a 200% FPL gross limit, which the household passes at 165%, so it is categorically eligible for the $24 monthly minimum allotment." +us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model ruled the household ineligible because it treated $3,600 in bank and stock assets as over the resource limit. That amount is under the limit for a household with a disabled member. Michigan's TANF non-cash categorical eligibility also removes the asset test, so the household is eligible for the $24 monthly minimum allotment." us,scenario_074,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_074,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model stated that tax-exempt SSDI counts in MAGI and then computed the opposite, summing only the $384 taxable IRA distribution plus a ""taxable portion"" of Social Security — applying the §86 provisional-income taxability test instead of the §36B(d)(2)(B) MAGI definition, which adds back the entire Social Security benefit excluded from gross income. Counting the full $33,640 SSDI plus the $384 IRA distribution puts the head at 2.13 x FPL, roughly 213% of poverty and far above Louisiana's 138% FPL expansion-adult limit." @@ -5354,114 +5864,124 @@ us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eli us,scenario_074,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model excluded SSDI from MAGI on the reasoning that it is untaxable, reaching a MAGI of $3,410; Medicaid MAGI explicitly adds back Social Security benefits not included in gross income, so the $33,640 SSDI counts dollar-for-dollar. The correct MAGI of roughly $34,000 equals 2.13 x FPL, about ten times the model's figure and well above the ~$22,000 expansion-adult threshold it cited." us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model asserted ""very low countable income"" with no derivation, a conclusion consistent only with dropping the entire $33,640 SSDI from countable income as non-taxable. Medicaid MAGI adds back untaxed Social Security benefits in full, yielding 2.13 x FPL — above Louisiana's 138% FPL expansion limit — and with age 54, zero SSI, and no dependents there is no non-MAGI category, so medicaid_category is NONE." us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model restricted the MAGI base to ""the listed taxable income,"" which omits the untaxed SSDI that §36B(d)(2)(B) requires to be added back to AGI for Medicaid MAGI. Including the $33,640 SSDI with the $384 taxable IRA distribution gives 2.13 x FPL, so the head exceeds the 138% FPL expansion-adult limit and qualifies under no MAGI category." -us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"Treated the $33,640 Social Security disability income line as proof of SSDI entitlement and then declared the 24-month qualifying period satisfied because disability status is constant for the year. Within-year constancy establishes nothing about prior months of entitlement, and PolicyEngine's Medicare test is the age-65 threshold, which a 54-year-old fails." -us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Asserted a PolicyEngine rule under which Social Security disability income confers Medicare eligibility below age 65. PolicyEngine's Medicare eligibility variable turns on the age threshold of 65 alone, and the head's disability status is unlisted and therefore false, so a 54-year-old is ineligible." -us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Concluded that current SSDI receipt means the 24-month entitlement waiting period has already been met; receipt during the benchmark year carries no information about the 24 months preceding it, and the entitlement start date is an unlisted fact that defaults out. It never applied the age-65 threshold that actually decides eligibility for this 54-year-old head." -us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Applied the real-world SSDI-to-Medicare pathway and treated the instruction that facts hold constant across the tax-benefit year as supplying 24 months of prior entitlement. That clause covers the single benchmark year only; the disability status input is unlisted and false, and the governing eligibility condition is age at least 65." -us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Claimed SSDI receipt qualifies a person for Medicare ""regardless of age,"" discarding the age-65 threshold that is the operative test. The input is a dollar amount for social_security_disability income, not a disability status flag, and the unlisted disability status defaults to false, leaving the 54-year-old head ineligible." -us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"Invented a ""PolicyEngine disability-based rule"" granting Medicare to Social Security disability recipients under 65. No such pathway exists in the model: Medicare eligibility is the age-65 threshold, and the head is 54 with no listed disability status." -us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Explicitly noted that no SSDI start date is given and then assumed ongoing entitlement long enough to clear the 24-month wait. That assumption manufactures an unlisted fact the prompt requires be treated as false, and eligibility here rests on the age-65 threshold that age 54 fails." -us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Read the SSDI dollar amount as satisfaction of ""the disability waiting period,"" collapsing one year of listed income into a 24-month entitlement history. Medicare eligibility here is the age-65 threshold, and the head is 54." -us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Asserted a disability pathway to Medicare from the presence of Social Security disability income alone, without a disability status flag (unlisted, therefore false) and without the 24-month entitlement period. The operative test is age at least 65, which the 54-year-old head does not meet." -us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Equated SSDI recipiency with Medicare eligibility in a single step, omitting both the 24-month entitlement requirement and the age-65 threshold. The listed input is an income amount, not a disability status, and status inputs left unlisted are false." -us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Derived eligibility directly from ""receiving Social Security disability income"" with no eligibility rule applied. Medicare eligibility in this benchmark is the age-65 threshold, and the head is 54; the disability status input is unlisted and defaults to false." -us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Stated flatly that Social Security disability income qualifies the head for Medicare, skipping the age-65 threshold that governs the output. It also never accounted for the 24-month SSDI entitlement period, which a single year of income cannot establish." -us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Converted a disability income amount into a Medicare eligibility switch without invoking any age or entitlement-duration test. Eligibility here requires age 65 or older, and the head is 54 with disability status unlisted and therefore false." -us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the presence of SSDI income as itself conferring Medicare, applying neither the 24-month entitlement requirement nor the age-65 threshold. The head is 54, so the age test fails and the value is 0." -us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"Stated that ""under PolicyEngine rules"" SSDI recipients are Medicare eligible regardless of being under 65. PolicyEngine's Medicare variable tests only the age threshold of 65, and the head's disability status input is unlisted and false." -us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Correctly identified that age 54 gives no automatic eligibility, then reversed itself on an assumed rule treating disability income as qualifying. No such rule applies: eligibility is the age-65 threshold, and the unlisted disability status defaults to false." -us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied a ""disability pathway despite being under age 65"" triggered by a year of Social Security disability income. That pathway requires 24 months of SSDI entitlement, a duration fact not listed and not derivable from one year of income, and the governing test here is age at least 65." -us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"Attributed to PolicyEngine a treatment in which positive Social Security disability income switches on Medicare eligibility. The variable keys on age 65 or older, so the 54-year-old head returns 0." -us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that full-year SSDI receipt ""meets the 24-month waiting-period rule."" Twelve months of listed income cannot establish twenty-four months of entitlement, and Medicare eligibility here rests on the age-65 threshold that a 54-year-old fails." -us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"Treated receipt of disability income as establishing entitlement past the statutory waiting period for under-65 beneficiaries. The waiting-period start is an unlisted fact that defaults out, and the operative eligibility condition is age at least 65." -us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"Moved directly from ""receives Social Security disability income for the full year"" to ""disabled SSDI beneficiary entitled to Medicare,"" skipping both the disability status flag (unlisted, hence false) and the age-65 threshold. At age 54 the head fails the only test that applies." -us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"Acknowledged the 24-month waiting period and then assumed it was met from ongoing benefit receipt. That prior-entitlement fact is not listed and defaults to false, and the governing eligibility condition is age at least 65, which the 54-year-old head does not meet." -us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Social Security disability income as ""qualifying disability-based Medicare eligibility."" The input is an income amount rather than a disability status, unlisted status inputs are false, and Medicare eligibility here is the age-65 threshold that age 54 fails." -us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"Applied ""the usual Medicare rule"" for SSDI recipients and read the prompt's within-year constancy clause as satisfying the multi-year waiting period. That clause fixes facts across the single tax-benefit year only, and eligibility turns on the age-65 threshold that a 54-year-old head fails." +us,scenario_074,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"Stretched the instruction that disability status is constant for the full year into 24 months of SSDI entitlement. A full tax year covers at most 12 months, and the unlisted SSDI-duration input defaults to 0, so the disability pathway fails and the 54-year-old head fails the age-65 test." +us,scenario_074,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated positive Social Security disability income as enough for Medicare on its own. The disability pathway requires 24 months of SSDI receipt, a separate input that was not listed and is therefore 0, so only the age-65 test applies and the head, at 54, fails it." +us,scenario_074,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Reasoned that current SSDI receipt means the 24-month waiting period has already passed. The prompt forbids inferring unlisted status facts, and the months of SSDI receipt default to 0, so the head qualifies only by age and is ineligible at 54." +us,scenario_074,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Used year-long constant status to assume the 24-month SSDI waiting period was met. A constant status across one tax year supplies at most 12 months, and the unlisted receipt-duration input is 0, so the under-65 disability pathway fails and the head is not Medicare eligible." +us,scenario_074,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Correctly named the 24-months-of-SSDI rule, then assumed it was satisfied because SSDI income is constant during the year. The facts never establish 24 months of entitlement, the duration input defaults to 0, and the 54-year-old head fails the age-65 test." +us,scenario_074,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"Described PolicyEngine's disability rule as triggered by any SSDI receipt. That rule requires 24 months of SSDI receipt, recorded as a separate input that defaults to 0 here, so the head qualifies only through the age-65 test and fails it at 54." +us,scenario_074,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Took the missing SSDI start date to mean receipt was ongoing and long enough to clear the 24-month waiting period. The prompt instead says unlisted numeric and status inputs are 0 or false, so months of SSDI receipt are 0 and the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Explicitly assumed the 24-month SSDI waiting period was met, which contradicts the instruction not to infer unlisted facts. With SSDI receipt duration at the default 0, only the age-65 test applies, and the head, at 54, fails it." +us,scenario_074,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated SSDI income as proof that the disability waiting period was complete. The 24 months of entitlement are never established, the duration input defaults to 0, and the 54-year-old head does not meet the age-65 threshold." +us,scenario_074,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied the disability pathway based on SSDI receipt alone and skipped its 24-month entitlement requirement. That duration input is unlisted and therefore 0, so the pathway fails and the head, under 65, is ineligible." +us,scenario_074,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Claimed PolicyEngine makes any SSDI recipient Medicare eligible. The engine's disability pathway requires 24 months of SSDI receipt, which defaults to 0 because it was not listed, leaving only the age-65 test, which the 54-year-old head fails." +us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Equated being an SSDI recipient with being Medicare eligible and ignored the 24-month entitlement requirement. That requirement is unmet because receipt duration is unlisted and therefore 0, and at 54 the head fails the age-65 test." +us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Granted Medicare because the head receives Social Security disability income, with no check of the 24-month waiting period. Months of SSDI receipt default to 0, so the disability pathway fails and the 54-year-old head is ineligible by age." +us,scenario_074,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Treated SSDI receipt as qualifying for Medicare on its own. Under-65 eligibility requires 24 months of SSDI entitlement, which the facts do not supply (duration defaults to 0), so the 54-year-old head is not eligible." +us,scenario_074,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Concluded Medicare eligibility from SSDI income alone and omitted the 24-month entitlement condition. That condition fails because the unlisted months-of-receipt input is 0, and the head, at 54, is below the age-65 threshold." +us,scenario_074,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated SSDI receipt as making the head Medicare eligible without applying the 24-month entitlement requirement. The receipt duration was not listed and defaults to 0, so only the age-65 test counts, and a 54-year-old fails it." +us,scenario_074,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"Asserted that PolicyEngine treats SSDI recipients as Medicare eligible regardless of age. The engine gates the disability pathway on 24 months of SSDI receipt, which defaults to 0 here, so the head is eligible only by age and fails at 54." +us,scenario_074,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Correctly noted that age 54 fails the age test, then treated disability income as meeting the disability pathway. That pathway requires 24 months of SSDI entitlement, and the unlisted duration input is 0, so the head is ineligible." +us,scenario_074,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"Read SSDI receipt for this year as satisfying the disability pathway. One constant tax year gives at most 12 months, short of the 24 months of entitlement required, and the unlisted duration defaults to 0, so the 54-year-old head is not eligible." +us,scenario_074,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"Claimed PolicyEngine grants Medicare for any positive Social Security disability income. The engine requires 24 months of SSDI receipt, a separate input that defaults to 0 here, so the head fails both the disability pathway and the age-65 test." +us,scenario_074,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Keyed the PolicyEngine disability pathway on positive SSDI income, when the engine keys it on at least 24 months of SSDI receipt. That input was not listed and defaults to 0, so the 54-year-old head is ineligible." +us,scenario_074,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that SSDI receipt for the full year meets the 24-month waiting period. A single tax year supplies at most 12 months, and the prompt adds no prior receipt (duration defaults to 0), so the rule is unmet and the head, under 65, is ineligible." +us,scenario_074,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"Named the statutory waiting period but treated SSDI receipt as proof it had elapsed. The 24 months of entitlement are never given, the unlisted duration defaults to 0, and the 54-year-old head fails the age-65 test." +us,scenario_074,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"Treated full-year SSDI receipt as making the head a Medicare-eligible disabled beneficiary. Eligibility requires 24 months of SSDI entitlement, more than the one year the facts cover, and the duration input defaults to 0, so the head is ineligible at 54." +us,scenario_074,head_medicare_eligible,grok-build-0.1,prompt_ambiguity,age_disability,False,"Explicitly assumed the 24-month waiting period was met because disability benefits are ongoing, which contradicts the instruction to treat unlisted status facts as false. With months of SSDI receipt at 0, only the age-65 test applies, and the head fails it." +us,scenario_074,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"Treated Social Security disability income as automatically qualifying for disability-based Medicare and skipped the 24-month entitlement requirement. That requirement fails because the unlisted receipt duration is 0, and the head, at 54, is below age 65." +us,scenario_074,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disability,False,"Cited the SSDI waiting-period rule and then used the full-year constancy instruction to treat it as satisfied. Constancy covers only the 12 months of the tax year, not 24 months of entitlement, and the unlisted duration defaults to 0, so the 54-year-old head is not Medicare eligible." us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model applied employee payroll-tax rates to non-wage income despite acknowledging that the household's income came from Social Security disability. It failed to set unlisted wages to zero and therefore invented a covered-earnings base that the household facts do not contain. us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required output contract." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its worksheet is exactly correct — $16,100 standard deduction, $86,960 of ordinary income taxed at 10/12/22 for $13,843, plus $6,226 of preferential income at 15% for $934, totaling $14,777 — and it then submitted $12,523, a number its own derivation never produces. The submitted $12,523 is what results from deducting an extra $6,000 senior deduction, which requires age 65 rather than 62, and dropping the $933.86 preferential tax entirely." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"It granted a 62-year-old the age-65 additional standard deduction, using $24,000 instead of the 2026 single amount of $16,100; the additional standard deduction is available only for age 65+ or blindness, and disability alone confers none. It then subtracted a fabricated $5,252 'disability standard deduction adjustment' from its computed $9,698, and no such disability offset exists in the federal income tax." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the structure right — $16,100 deduction, $86,960 ordinary base, $6,226 of preferential income all at 15% for $934 — but used bracket edges of $12,150 and $49,400 instead of the 2026 values of $12,400 and $50,400. That narrowed the 10% and 12% bands and pushed an extra $1,000 into the 22% bracket, producing $13,948 of ordinary tax against the correct $13,843.28." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $15,000 single standard deduction rather than the 2026 amount of $16,100, leaving taxable income of $94,286 instead of $93,186.13, and paired it with estimated bracket edges of $12,250/$49,750 instead of $12,400/$50,400. Those two errors added $310 to ordinary tax ($14,153.20 versus $13,843.28); its stacking of $6,226 of preferential income entirely at 15% was correct." -us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It reached the reference figure exactly — $16,100 deduction, $86,960 of ordinary income taxed at 10/12/22 for $13,843.20, plus $6,226 at 15% for $933.90, totaling $14,777.10 — then discarded that result as 'high' and submitted $6,710 with no supporting arithmetic. No 2026 parameter set produces $6,710 on $93,186.13 of taxable income." -us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It inflated the 2026 single standard deduction to $19,000, where the actual amount is $16,100, cutting taxable income to $90,286, and used bracket edges of $12,200/$49,550 rather than $12,400/$50,400, yielding $13,294.20 of ordinary tax against the correct $13,843.28. Its explanation states a total of $14,228.10 while the submitted value is $14,988, so the graded number matches neither its own worksheet nor the correct computation." -us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a pre-TCJA regime — an $8,600 standard deduction plus a $5,400 personal exemption and a 10/15/25 rate schedule — none of which governs 2026, where personal exemptions remain zero, the standard deduction is $16,100, and the rates over this range are 12% and 22%. Taxing $89,060 of ordinary income at 15% and 25% produced $16,605 instead of the correct $13,843.28, an overstatement of about $2,760." -us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed a TCJA sunset, using an $8,300 standard deduction plus a $5,300 personal exemption and 15%/25% brackets; for 2026 the single standard deduction is $16,100, personal exemptions are zero, and the applicable marginal rates are 12% and 22%. That inflated ordinary tax to $16,778.75 against the correct $13,843.28 on $86,960.37 of ordinary income." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It combined a $13,600 standard-deduction-plus-personal-exemption allowance with projected 10/15/25 brackets, both pre-TCJA-sunset assumptions that do not govern 2026; the correct figures are a $16,100 standard deduction, no personal exemption, and 10/12/22 rates. Its $17,315 of ordinary tax on $89,460 exceeds the correct $13,843.28 on $86,960.37 by roughly $3,470, while its $933.90 of preferential tax matched." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $14,930.50 is exactly what the correct 2026 rate schedule and the correct $933.86 of preferential tax produce from a standard deduction of about $15,400 rather than the actual $16,100. The $700 shortfall in the deduction left an extra $700 of ordinary income in the 22% bracket, adding $154 to the correct $14,777.15." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It states outright that it assumed expiration of the TCJA standard deduction and reinstatement of personal exemptions for 2026; neither occurs, so the single standard deduction is $16,100 with no exemption and the top applicable rate is 22%, not 25%. Applying the reverted schedule to a smaller allowance drove its answer $2,987 above the correct $14,777.15." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used an $8,300 standard deduction, a $5,150 personal exemption, and reverted 10/15/25 brackets for 2026, producing $17,122 of ordinary tax on $89,610. The governing 2026 parameters are a $16,100 standard deduction, no personal exemption, and 10/12/22 rates, which give $13,843.28 on $86,960.37 of ordinary income." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no computation, and its $11,843 is what the correct 2026 schedule yields only when ordinary taxable income is about $73,600 rather than the actual $86,960.37 — roughly $13,300 of the household's income is absent from its base, about the size of the $12,656 in wages. The correct derivation subtracts only the $16,100 standard deduction from $109,286.13 of gross income and yields $13,843.28 of ordinary tax plus $933.86 of preferential tax." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied post-sunset parameters — an $8,300 standard deduction, a $5,050 personal exemption, and 10/15/25 brackets — where 2026 provides a $16,100 standard deduction, zero exemptions, and 10/12/22 rates. Its ordinary tax of $17,405 on $89,710 exceeds the correct $13,843.28 by roughly $3,560, while its $933.90 of preferential tax matched." -us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used an entire set of stale parameters — a $15,000 standard deduction and the 2025 bracket edges of $11,925/$48,475 — instead of the 2026 $16,100 deduction and $12,400/$50,400 edges. That left $1,100 more income in the base and narrowed the low-rate bands, giving $14,287.20 of ordinary tax against the correct $13,843.28." -us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It estimated the 2026 single standard deduction at $15,400 instead of $16,100, leaving $87,660 of ordinary income rather than $86,960.37, and used bracket edges of $12,250/$49,800 instead of $12,400/$50,400. The $700 deduction shortfall added $154 at the 22% rate and the bracket drift added another $63, producing $14,060.20 of ordinary tax against the correct $13,843.28." -us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no arithmetic, and $9,468 is what the correct 2026 schedule produces only from about $62,800 of ordinary taxable income rather than the actual $86,960.37 — roughly $24,000 of the $109,286.13 of gross income is missing from its base. The correct computation subtracts only the $16,100 standard deduction and yields $13,843.28 of ordinary tax plus $933.86 on the $6,225.76 of preferential income." -us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed a reduction from itemized medical expenses, but the $500 of other medical plus $100 of over-the-counter expenses total $600, far below the 7.5%-of-AGI floor of $8,196, so the $16,100 standard deduction governs and no medical deduction exists. Its $1,446 corresponds to taxing about $14,100 at the 10% and 12% rates, meaning the $90,020 of taxable private pension is absent from its base, whereas the correct taxable income is $93,186.13." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It showed no computation, and $13,361 is what the correct 2026 rate schedule plus the correct $933.86 of preferential tax produce only from about $80,520 of ordinary income rather than $86,960.37 — an extra deduction of roughly $6,440 on top of the $16,100, matching the $6,000 senior deduction that requires age 65 and is unavailable to this 62-year-old head. With only the $16,100 standard deduction, ordinary tax is $13,843.28 and the total is $14,777.15." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly assumed a 'current law TCJA sunset' for 2026, applying a $5,322 personal exemption plus an $8,344 standard deduction and 10/15/25 brackets. The 2026 rules retain the $16,100 single standard deduction with no personal exemption and 10/12/22 rates, so its $16,749 of ordinary tax overstates the correct $13,843.28 by about $2,900." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a post-sunset $8,300 standard deduction plus a $5,300 personal exemption and taxed $89,460 of ordinary income at 10/15/25 for $16,895. For 2026 the single standard deduction is $16,100, personal exemptions are zero, and the applicable rates are 10/12/22, giving $13,843.28 on $86,960.37 of ordinary income." -us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed post-TCJA-sunset rules with an $8,522 standard deduction and a 10/15/25 rate schedule, taxing $94,538 of ordinary income for $17,916. The 2026 single standard deduction is $16,100 and the rates over this range are 10%, 12%, and 22%, giving $13,843.28 on $86,960.37 of ordinary income, so its answer overstates the total by more than $4,000." -us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs object. This is a missing submission rather than a substantive computational error." -us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It double-counted the preferential income by leaving the $5,794 of long-term gain and $432 of qualified dividends inside a $103,536 ordinary base and then taxing the same $6,226 again at 15%, and it used a $15,750 standard deduction instead of $16,100. It then subtracted roughly $18,608 of invented senior and disability nonrefundable credits; the Credit for the Elderly or Disabled is fully phased out at $109,286.13 of income and no other nonrefundable credit applies, leaving the liability at $14,777.15." -us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 bracket edges of $11,925 and $48,475 and a $15,750 standard deduction rather than the 2026 edges of $12,400 and $50,400 and the $16,100 deduction, leaving $87,310 of ordinary income taxed for $14,122.20 against the correct $13,843.28 on $86,960.37. Its stacking of the $6,226 of preferential income entirely at 15% for $933.90 was correct, so the whole $279 gap comes from the stale deduction and bracket parameters." -us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It added the $2,240 of tax-exempt private pension into AGI, which is excluded from gross income, and applied a $20,350 age-65 standard deduction to a 62-year-old head whose 2026 deduction is $16,100 with no age addition. It then reported zero tax after computing $20,146 of liability, wiping the whole amount out with the Credit for the Elderly or Disabled, a credit fully phased out at this income and capped far below that liability in any event." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning uses the correct $16,100 standard deduction and 2026 brackets. It gets $13,843 of ordinary tax plus $934 of capital gains tax, for $14,777 total. It then submitted 12,523, a number that nothing in its own derivation supports." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"It gave a 62-year-old a $24,000 standard deduction by adding a $9,000 age/blindness amount. That add-on requires age 65+ or blindness, and disability alone does not qualify. It also started from the outdated $15,000 base instead of $16,100. It then subtracted an invented $5,252 'disability standard deduction adjustment', which no provision of the tax code supports." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the correct $16,100 deduction and $934 of preferential-rate tax, but estimated 2026 single bracket thresholds too low (12% band ending at $49,400 instead of $50,400, 10% band ending at $12,150 instead of $12,400). That pushed $1,000 more income into the 22% band and gave $13,948 of ordinary tax instead of $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied the pre-OBBBA $15,000 standard deduction instead of the 2026 amount of $16,100, which overstated taxable income by $1,100 ($94,286). It also taxed ordinary income on guessed inflation-adjusted brackets (12,250/49,750) instead of the actual 2026 thresholds (12,400/50,400)." +us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its reasoning computed exactly $14,777.10: $16,100 deduction, $13,843.20 of ordinary tax on $86,960, and $933.90 at 15%. It then discarded that figure as 'too high' and submitted an unsupported $6,710." +us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $19,000 single standard deduction instead of $16,100, which understated taxable income at $90,286. It also used bracket thresholds of 12,200/49,550 instead of 12,400/50,400, and its explanation derives $14,228.10. The submitted $14,988 matches neither that derivation nor the correct $14,777." +us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset for 2026: an $8,600 standard deduction, a $5,400 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA structure permanent, so the law is a $16,100 deduction, no exemption, and 10/12/22% brackets. Taxing $38,660 at 25% instead of 22% inflated ordinary tax to $16,605." +us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied post-sunset pre-TCJA law: an $8,300 deduction, a $5,300 personal exemption, and 10/15/25% brackets. The 2026 law is the permanent TCJA structure with a $16,100 deduction, no exemption, and 12% and 22% brackets. This gave $16,778.75 of ordinary tax instead of $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed a TCJA sunset: a $13,600 combined standard deduction and personal exemption, with 10/15/25% brackets. 2026 law has a $16,100 standard deduction, no personal exemption, and 10/12/22% brackets. The sunset assumption inflated ordinary tax to about $17,315 instead of $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no numeric derivation. The correct path subtracts the $16,100 deduction to reach $93,186, taxes $86,960 of ordinary income on 2026 brackets (10% to $12,400, 12% to $50,400, then 22%) for $13,843, and adds $934 on $6,226 at 15%. Its $14,930.50 is $153 too high, which matches taxing ordinary income on thresholds or a deduction lower than the 2026 values." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA standard deduction expired and personal exemptions returned for 2026. OBBBA made the TCJA $16,100 standard deduction and 10/12/22% brackets permanent with no personal exemption. That produced the inflated $17,764." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: an $8,300 standard deduction, a $5,150 personal exemption, and 10/15/25% brackets, taxing $89,610 of ordinary income for $17,122. Under the 2026 law ($16,100 deduction, no exemption, 10/12/22% brackets), ordinary income is $86,960 and its tax is $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation taxes $93,186 of taxable income: $13,843 on $86,960 of ordinary income on 2026 brackets, plus $934 on $6,226 at 15%. Its $11,843 is $2,934 too low, which equals roughly $13,300 of 22%-bracket income escaping tax. That points to an overstated deduction or omitted income." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed a post-TCJA sunset: an $8,300 standard deduction, a $5,050 personal exemption, and 10/15/25% brackets, taxing $89,710 of ordinary income for $17,405. 2026 law keeps the TCJA structure: a $16,100 deduction, no exemption, and 10/12/22% brackets, giving $13,843 on $86,960." +us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 $15,000 standard deduction and the 2025 bracket thresholds (11,925/48,475) instead of 2026's $16,100 deduction and 12,400/50,400 thresholds. That overstated taxable income by $1,100 and pushed extra income into the 22% band, giving $15,221.10." +us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It guessed a $15,400 standard deduction instead of the 2026 amount of $16,100, and bracket thresholds of 12,250/49,800 instead of 12,400/50,400. Ordinary taxable income became $87,660 with $14,060 of tax, instead of $86,960 with $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation taxes $93,186 of taxable income ($109,286 AGI less the $16,100 deduction): $13,843 on ordinary income plus $934 at 15%. Its $9,468 is $5,309 short, which at the 22% marginal rate means about $24,000 of taxable income dropped out of its base." +us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed itemized medical deductions. The $600 of medical and OTC expenses is far below the 7.5%-of-AGI floor ($8,196), so the deductible medical amount is $0 and the $16,100 standard deduction applies. Its $1,446 also implies it taxed only about $14,000 of income, leaving most of the $90,020 taxable pension untaxed." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no numeric derivation. The correct computation is $13,843 of ordinary tax on $86,960 (after the $16,100 deduction, on 2026 10/12/22% brackets) plus $934 at 15% on $6,226. Its $13,361 is $1,416 short, which equals about $6,400 of 22%-bracket income escaping tax, for example through an extra deduction the 62-year-old does not qualify for." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset: about $8,344 of standard deduction plus a $5,322 personal exemption, taxing $89,394 on 10/15/25% brackets for $16,749. For 2026 the TCJA structure is permanent: a $16,100 deduction, no exemption, and 10/12/22% brackets." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law: an $8,300 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets, giving $16,895 on $89,460 of ordinary income. 2026 law is a $16,100 deduction, no exemption, and 10/12/22% brackets, giving $13,843 on $86,960." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated a $15,400 standard deduction instead of the 2026 amount of $16,100, and bracket thresholds of 12,250/49,775 instead of 12,400/50,400. That taxed $87,660 of ordinary income for $14,062.70 instead of $86,960 for $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA-sunset law with an $8,522 standard deduction and 10/15/25% brackets. 2026 law uses a $16,100 deduction and 10/12/22% brackets. It also miscomputed taxable income as $100,764; $109,286 less $8,522 is $100,764 before removing nothing, yet it then taxed $94,538 of ordinary income at 25% above $50,928." +us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $14,777.15 derived from $93,186 of taxable income." +us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It invented about $18,608 of nonrefundable credits for 'age 62 senior credit and disability considerations'. The Credit for the Elderly or Disabled is capped at 15% of a $5,000 base, and that base is reduced by the $2,240 nontaxable pension and by half of AGI above $7,500, so it is fully eliminated at $109,286 AGI. It also used the 2025 $15,750 deduction instead of $16,100, and it mis-subtracted to get $103,536 of taxable income instead of $93,536." +us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 $15,750 standard deduction and the 2025 bracket thresholds (11,925/48,475) instead of 2026's $16,100 deduction and 12,400/50,400 thresholds. That taxed $87,310 of ordinary income for $14,122.20 instead of $86,960 for $13,843." +us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It computed $20,146 of tax and stated that the Credit for the Elderly or Disabled yields no usable credit, yet it submitted $0. Tax before refundable credits equals the pre-credit tax when no nonrefundable credit applies. Along the way it also put the $2,240 tax-exempt pension into AGI, misadded AGI as $131,866, and gave a 62-year-old the over-65 standard deduction." us,scenario_075,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the listed disability as sufficient to activate a Florida disabled-adult Medicaid pathway without applying that pathway's complete eligibility requirements. The head receives no SSI and the engine assigns medicaid_category NONE, so no categorical pathway exists and Medicaid eligibility is 0." -us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model asserted that disabled persons ""qualify for Medicare via SSDI-based eligibility,"" but the household lists no Social Security disability income (unlisted inputs are 0), so the head has zero months of SSDI entitlement against the 24-month requirement of 42 U.S.C. § 426(b). It substituted the raw disability flag for the entitlement-duration test that the disability pathway actually requires, while PolicyEngine's Medicare eligibility variable applies the age-65 threshold that a 62-year-old fails." -us,scenario_075,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model claimed ""under PolicyEngine rules disabled individuals are treated as Medicare eligible,"" a rule PolicyEngine does not implement — its Medicare eligibility test is the age-65 threshold, which a 62-year-old fails. It never checked the statutory disability pathway's precondition of 24 months of SSDI entitlement, which is unmet here because Social Security disability income is $0." -us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model correctly recited the 24-month SSDI-entitlement requirement and then discarded it, declaring that the bare ""is disabled"" fact should be ""treated as"" satisfying it even though the household reports $0 Social Security disability income under the rule that unlisted inputs are 0. Its second ground is also wrong: claiming early retirement benefits at 62 lead to Medicare eligibility, when OASI early retirement confers no Medicare coverage before age 65." -us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model reduced the question to ""disabled individuals qualify for Medicare,"" skipping the fact that the under-65 pathway requires 24 months of entitlement to Social Security disability benefits (or ESRD/ALS), none of which this household establishes since disability income is $0. PolicyEngine's Medicare eligibility resolves on age >= 65, so the head's age of 62 produces False." -us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model attributed to PolicyEngine a rule it does not have — that disabled individuals are Medicare eligible — when the implemented test is the age-65 threshold. It applied no SSDI-entitlement check, which is where the claim collapses: the head draws $90,020 of private pension and $12,656 of wages with $0 Social Security disability benefits, so the 24-month entitlement clock never starts." -us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented an ""age threshold (62) for disability-based Medicare eligibility""; no such threshold exists — the disability pathway is age-independent and gated on 24 months of SSDI entitlement, and the only Medicare age threshold is 65. Its fallback that the head ""is approaching age eligibility"" is not an eligibility rule at all, and PolicyEngine evaluates age 62 against 65 to return False." -us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model invoked ""the disability pathway"" on the strength of the disability flag alone, omitting its operative condition: 24 months of entitlement to Social Security disability insurance benefits, which is impossible here because the household's Social Security disability income is $0. PolicyEngine's Medicare eligibility test is age >= 65, which the 62-year-old head fails." -us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model asserted that ""at age 62, the head is eligible for Medicare under these PolicyEngine rules,"" fabricating an age-62 trigger; PolicyEngine sets the Medicare age threshold at 65, and 62 < 65 yields False. It confused the Social Security early-retirement age of 62 with the Medicare entitlement age and offered no disability-entitlement analysis at all." -us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model stated as fact that PolicyEngine treats a disabled person as Medicare eligible below age 65; PolicyEngine implements only the age-65 threshold for Medicare eligibility, so the 62-year-old head returns False. The real disability route requires 24 months of SSDI entitlement or an ESRD/ALS diagnosis, and this household reports $0 Social Security disability income and no such diagnosis." -us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model treated the listed disability status as directly qualifying the head for Medicare under age 65, skipping the 24-month SSDI-entitlement precondition that makes the under-65 pathway operative; the head receives $0 Social Security disability benefits and $102,676 of wage and pension income instead. The governing computation is the age-65 comparison, which a 62-year-old fails." -us,scenario_075,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed eligibility under ""the modeled Medicare disability pathway,"" but no disability pathway is modeled — PolicyEngine determines Medicare eligibility from the age-65 threshold alone, returning False for a 62-year-old. Even the statutory pathway would fail here, since it requires 24 months of SSDI entitlement and the household's Social Security disability income is $0." -us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model stated the rule as ""age 65+ or disabled,"" adding a disjunct PolicyEngine does not contain; the implemented condition is the age-65 threshold, which the 62-year-old head fails. The disability disjunct it assumed also has no statutory basis in this form — the under-65 route demands 24 months of entitlement to Social Security disability benefits, and the head receives none." -us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model concluded that being 62 and disabled ""makes them eligible for Medicare,"" collapsing the disability flag into entitlement without applying the 24-month SSDI-entitlement requirement that the under-65 pathway rests on; disability income here is $0. PolicyEngine's Medicare eligibility resolves against the age-65 threshold, so age 62 yields False." +us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"It treated the ""is disabled"" flag as automatic Medicare eligibility. The disability pathway requires 24 months of SSDI entitlement, and this head gets no Social Security disability benefits and has zero months of SSDI receipt. At 62, the head also fails the age-65 test, so the answer is not eligible." +us,scenario_075,head_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"It assumed disabled people under 65 are Medicare-eligible by default. Under-65 Medicare requires 24 months of SSDI entitlement or ESRD, and the head has neither: no Social Security disability income and no ESRD. With the head below the age-65 threshold, eligibility is 0." +us,scenario_075,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"It correctly named the 24-month SSDI requirement, then assumed the ""is disabled"" flag meets it, even though no Social Security disability benefits are listed. It also wrongly claimed that early retirement eligibility at 62 leads to Medicare; Social Security retirement at 62 never confers Medicare, and the Medicare age test is 65." +us,scenario_075,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"It equated being disabled with qualifying for Medicare. It skipped the requirement of 24 months of SSDI entitlement (or ESRD) for anyone under 65. The head receives no SSDI and is 62, so no pathway applies." +us,scenario_075,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"It asserted that PolicyEngine treats disabled individuals as Medicare-eligible. The rule actually requires age 65+, at least 24 months of SSDI receipt, or ESRD. This head has zero SSDI months and no ESRD, and is 62." +us,scenario_075,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"It invented an age-62 threshold for disability-based Medicare and treated ""approaching"" 65 as qualifying. Medicare requires age 65 or older, or 24 months of SSDI entitlement or ESRD; there is no age-62 pathway, and the head has no SSDI." +us,scenario_075,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"It applied the disability pathway based only on the ""is disabled"" flag. That pathway requires 24 months of Social Security disability benefit entitlement, and the head receives no SSDI. At 62, the head fails the age-65 test, so they are not eligible." +us,scenario_075,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"It claimed the head is Medicare-eligible at age 62, but the age threshold is 65. It also cited no valid alternative: the head has no months of SSDI receipt and no ESRD, so eligibility is 0." +us,scenario_075,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"It stated that anyone disabled is Medicare-eligible under 65. Under-65 eligibility actually requires at least 24 months of SSDI receipt or ESRD. The head has no Social Security disability benefits, so the disability flag alone does not qualify them." +us,scenario_075,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"It treated disability status as qualifying for Medicare under 65 and ignored the requirement of 24 months of SSDI entitlement. The head receives no SSDI, has no ESRD, and is 62, so they are not eligible." +us,scenario_075,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"It invoked the Medicare disability pathway based only on the ""is disabled"" flag. That pathway is triggered by 24 or more months of SSDI receipt, and this head has none. With no ESRD and age 62, the head is not eligible." +us,scenario_075,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"It claimed disability alone qualifies for Medicare. The modeled rule requires age 65+, at least 24 months of SSDI receipt, or ESRD, and the head meets none of these: age 62, no SSDI income, no ESRD." +us,scenario_075,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"It applied a disability-based Medicare rule keyed to the ""is disabled"" flag. The actual pathway requires 24 months of SSDI entitlement, and the head has zero SSDI months. With no ESRD and age below 65, eligibility is 0." +us,scenario_075,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"It stated the rule as ""age 65+ or disabled"", dropping the requirement that under-65 disability eligibility rest on 24 months of SSDI receipt. The head receives no Social Security disability benefits, so only the age test applies, and it fails at 62." +us,scenario_075,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"It concluded that being 62 and disabled makes the head Medicare-eligible. Medicare needs age 65+, or 24 months of SSDI entitlement or ESRD for younger people. The head has no SSDI and no ESRD, so they are not eligible." us,scenario_075,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model named the correct 6.2% Social Security and 1.45% Medicare rates but miscalculated their combined application to wages. The PolicyEngine component amounts are $784.69 and $183.52, which sum to $968.21 rather than $966." us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model applied the stated FICA rates incorrectly: 6.2% plus 1.45% on $12,656 does not produce $1,054. The component calculation yields $784.69 of Social Security tax and $183.52 of Medicare tax, totaling $968.21." us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model's own component calculation produced $968.18 before engine-level precision, but it then changed that result to $989 under the label of rounding. Rounding cannot turn approximately $968.2 into $989; the PolicyEngine components total $968.21." us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely extended Idaho WIC eligibility through age 8 even though WIC child eligibility ends at the fifth birthday. It also divided or expressed the income ratio incorrectly: using its own $23,030 poverty guideline, $173,820 is about 755% of poverty and exceeds the 185% limit by a wide margin; medical expenses do not establish WIC eligibility or cure either failure." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It reached the correct structure — $164,220 AGI, an $800 QBI deduction, and its own tentative tax of $24,321.80 — then discarded that figure and subtracted the $4,600 of credits from an unsourced $29,226 tentative tax, a $4,904 fabrication that is the whole of its error. Its own chain would have landed at $19,721.80, within $110 of the correct $19,611.80, the residual coming from a $24,300 standard deduction where 2026 HoH is $24,150." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It reproduced every element of the correct chain — $164,220 AGI, $24,150 standard deduction, $800 QBI deduction, $139,270 taxable income, $4,400 CTC and $200 CDCC — then priced the income on the 2025 HoH schedule (10% to $17,000, 12% to $64,850, 22% to $103,350) instead of the 2026 thresholds of $17,700 / $67,450 / $105,700. That single substitution inflated pre-credit tax by $321, exactly its $19,932.80 versus $19,611.80 gap." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It invented a HoH schedule whose 22% band begins at $78,100 and priced $141,290 of taxable income at $22,863, against $24,211.80 on the correct $139,270. It compounded that by adding the $9,600 of child support to gross income (child support received is excluded), subtracting the $6,480 ESI premium as an AGI adjustment, using a $26,050 standard deduction instead of $24,150, omitting the $800 QBI deduction, and claiming $2,000 per child with no CDCC instead of $4,400 plus $200." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed roughly $29,400 of mortgage interest from the $420,000 loan balance to itemize $37,900; no mortgage-interest amount is listed and unlisted numeric inputs are zero, so itemized deductions fall below the $24,150 HoH standard deduction the return takes. It then abandoned its own $17,033 result and submitted $21,889 with no supporting computation." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It imputed about $25,200 of mortgage interest from the $420,000 balance plus $8,000 of state tax to itemize $33,200, when no interest payment is listed and the return uses the $24,150 HoH standard deduction. It then discarded its own $14,306 figure and submitted $21,625 as an unexplained adjustment." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated $1,700 per child of the $2,200 CTC as refundable and allowed only about $1,000 against liability; the CTC is nonrefundable up to tax liability, and the refundable ACTC reaches only the excess over that liability, which is zero against $24,211.80 of tax, so the full $4,400 offsets tax before refundable credits. Its submitted $24,160 also fails to follow its own $24,449 less $1,200 arithmetic." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It itemized $37,300 built from $27,300 of mortgage interest imputed at 6.5% on the $420,000 balance plus a full $10,000 SALT deduction, neither of which is a listed input, and it priced the return on single brackets with a single-filer standard deduction. The head is unmarried with two qualifying children and files head of household with the $24,150 standard deduction, giving $139,270 of taxable income and $24,211.80 of tax." -us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It filed the return as single with a $16,100 standard deduction and single brackets, when the head is unmarried with two qualifying children and files head of household at $24,150 with HoH brackets. It also excluded the $6,480 ESI premium from wages, giving $157,740 instead of the $164,220 AGI, and used $2,000-per-child CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It set the nonrefundable CTC to zero on the assertion that the credit is fully refundable in 2026; the $2,200-per-child credit offsets liability first, so all $4,400 reduces tax before refundable credits. It also used a $23,000 standard deduction instead of $24,150 and never took the $800 QBI deduction on the rental income." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,480 employee ESI premium from the $160,000 wage figure, which is already the taxable wage amount, so AGI is $164,220 rather than its $157,740. It further used a $23,200 standard deduction instead of $24,150, omitted the $800 QBI deduction, and applied $2,000 per child instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied a TCJA-sunset regime — a $12,155 HoH standard deduction plus three $5,265 personal exemptions and a fully phased-out CTC — when 2026 keeps the $24,150 standard deduction, repealed personal exemptions, and a $2,200-per-child CTC unreduced below the $200,000 HoH threshold. Its submitted $26,346.25 additionally contradicts the $24,775.75 its own explanation derives." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expiration for 2026, using an $11,687 standard deduction, three $5,062 personal exemptions, and a $1,000-per-child CTC; 2026 provides a $24,150 HoH standard deduction, no personal exemptions, and $2,200 per child for $4,400 of nonrefundable credit. It also removed the $6,480 ESI premium from wages and skipped the $800 QBI deduction, so its $130,867 taxable income understates the correct $139,270." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It started from the correct $164,220 AGI but applied a $4,000 CTC rather than the 2026 $2,200-per-child amount of $4,400, and dropped the $200 CDCC on the $1,000 of childcare expenses entirely. It also never took the $800 QBI deduction on the rental income; the full chain is $139,270 taxable income, $24,211.80 of tax, less $4,600 of nonrefundable credits." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It priced the return under pre-TCJA law with a $12,090 standard deduction and $15,900 of personal exemptions, and allowed no CTC at all; 2026 gives a $24,150 standard deduction, no exemptions, an $800 QBI deduction, and $4,400 of nonrefundable CTC. It also cut wages by the $6,480 ESI premium, so its $157,740 AGI understates the $164,220 base." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used a sunset regime — a pre-TCJA standard deduction with personal exemptions and a CTC declared fully phased out at the pre-2018 $75,000 HoH threshold — while 2026 supplies a $24,150 standard deduction, no exemptions, and a $4,400 CTC unreduced below $200,000 of AGI. It also removed the $6,480 ESI premium from wages, producing $157,740 instead of the $164,220 AGI." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted a bare number with no derivation. Its $19,535 sits $76.80 below the correct chain of $139,270 taxable income → $24,211.80 of tax → less $4,400 CTC and $200 CDCC, a gap that corresponds to pricing that same taxable income on bracket thresholds wider than the 2026 HoH schedule of $17,700 / $67,450 / $105,700." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 employee ESI premium to build its income base; that premium is not an AGI adjustment and AGI is the unadjusted $164,220. With no bracket or credit figures offered, its $21,743 matches no step of the correct chain — $24,150 standard deduction and $800 QBI deduction give $139,270 of taxable income, $24,211.80 of tax, less $4,400 CTC and $200 CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It zeroed the CTC as phased out at its stated $157,740 AGI; the 2026 HoH phaseout begins at $200,000 and cuts $50 per $1,000 above it, so both children's $2,200 credits survive in full for $4,400 of nonrefundable credit. It also subtracted the $6,480 ESI premium from wages, when AGI is the unadjusted $164,220." -us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It applied the 2025 parameter set to a 2026 return: the $22,500 HoH standard deduction instead of $24,150, the 2025 brackets of $17,000 / $64,850 / $103,350, and $2,000-per-child CTC instead of $2,200, while omitting the $800 QBI deduction. Those exact choices generate its $141,720 taxable income, $25,120.80 tentative tax, and $4,200 of credits." -us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,480 ESI premium from wages for a $157,740 AGI when AGI is $164,220, then used a $23,100 standard deduction instead of $24,150 and an invented schedule whose 22% band ends near $100,350 rather than $105,700. It also capped the CTC at $2,000 per child instead of $2,200, taking $4,200 of credits rather than $4,600." -us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It took a $23,625 HoH standard deduction and a bracket schedule producing roughly $24,965 of tax on $139,795 of taxable income, against $24,150 and $24,211.80 on $139,270 under the 2026 HoH thresholds of $17,700 / $67,450 / $105,700. It also inflated the CTC to $2,250 per child, where 2026 sets it at $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no computation and invoked a 'standard/itemized' deduction choice; its $14,143 falls $5,469 below the correct answer, equal to roughly $22,800 of extra deduction — the scale of mortgage interest a 6-7% rate on the $420,000 balance would produce, and no interest amount is listed, so that deduction is zero. The correct chain is $164,220 AGI less $24,150 and $800 QBI, $24,211.80 of tax, less $4,400 CTC and $200 CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It explicitly treated the $9,600 of child support received as taxable income, which is excluded from gross income, so its base overstates the $164,220 AGI. It then applied no CTC and no CDCC, where the return absorbs $4,400 of nonrefundable CTC and a $200 CDCC against $24,211.80 of tax." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 employee ESI premium to reach a $157,740 AGI, when AGI is the unadjusted $164,220, and its $22,198 tentative tax accordingly runs $2,013.80 below the $24,211.80 due on $139,270 of taxable income. It also used $2,000-per-child CTC instead of the 2026 $2,200, taking $4,200 of credits rather than $4,600." -us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It filed the return as married filing jointly: $164,220 less the $32,200 MFJ standard deduction gives $132,020, taxed at $18,468.40 on MFJ brackets, less $4,600 of credits, reproducing its $13,868.40 exactly. The head is unmarried with two dependents and files head of household — $24,150 standard deduction plus the $800 QBI deduction, $139,270 taxable income, $24,211.80 of tax." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its only substantive statement — taxable income after 'deductions and exemptions' — places the return in the repealed personal-exemption regime, while 2026 allows no exemptions and a $24,150 HoH standard deduction with an $800 QBI deduction. Its $23,250 corresponds to subtracting only the $200 CDCC from a tentative tax and dropping the entire $4,400 of nonrefundable CTC." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It understated the 2026 HoH standard deduction as $23,063 instead of $24,150, leaving $140,357 of taxable income against the correct $139,270, and capped the CTC at $2,000 per child instead of $2,200. Its QBI deduction and $200 CDCC were right; those two parameter errors account for the entire $758 overstatement." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It assumed TCJA sunset for 2026 — a $12,450 HoH standard deduction, three $5,300 personal exemptions, and a CTC fully phased out above $75,000 — producing $135,870 of taxable income and $26,533 of pre-credit tax. 2026 applies a $24,150 standard deduction, no personal exemptions, an $800 QBI deduction, and a $2,200-per-child CTC unreduced below $200,000, worth $4,400 here." -us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It removed the $6,480 ESI premium from wages, when AGI is the unadjusted $164,220, and layered a sunset regime on top: a $12,585 standard deduction plus $16,353 of personal exemptions and no CTC. 2026 gives a $24,150 standard deduction, an $800 QBI deduction, no exemptions, and $4,400 of nonrefundable CTC alongside the $200 CDCC it did capture." -us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It allowed only about $1,000 of the CTC as nonrefundable; the $2,200-per-child credit offsets liability first, and against $24,211.80 of tax the entire $4,400 is used before any refundable portion arises. Its roughly $24,500 tentative tax also overshoots the $24,211.80 the 2026 HoH schedule yields on $139,270 of taxable income." -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so the required key carried no submission at all; the failure is a missing output rather than a substantive computation error. The chain it needed to submit is $164,220 AGI less the $24,150 HoH standard deduction and $800 QBI deduction, $24,211.80 of tax on $139,270, less $4,400 of CTC and $200 of CDCC." -us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It matched every input — $164,220 AGI, $24,150 standard deduction, $800 QBI deduction, $139,270 taxable income, $4,400 CTC, $200 CDCC — and erred only on the rate schedule, pricing that income at $24,332.80 against $24,211.80. Its upper bracket thresholds are too narrow, with the 22% band ending below the 2026 HoH figure of $105,700, worth $121 of excess tax." -us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It invented $1,815 of rental depreciation that is not a listed input, cutting rental income to $2,185, AGI to $162,405, and the QBI deduction to $437 instead of $800 on the full $4,000 of rental income. It also used a $23,200 standard deduction instead of $24,150 and $2,000-per-child CTC instead of $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used the correct $24,150 standard deduction and the exact 2026 HoH schedule but split the $4,400 CTC into $3,400 refundable and $1,000 nonrefundable; the credit offsets liability first, and with $24,211.80 of tax none of it is refundable, so all $4,400 applies before refundable credits. It also omitted the $800 QBI deduction, which accounts for the remaining $192 of its $3,592 gap." -us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed the return as married filing jointly with a $16,600 standard deduction and MFJ brackets on $147,620 of taxable income; the head is unmarried with two qualifying children and files head of household, taking $24,150 plus the $800 QBI deduction for $139,270 and $24,211.80 of tax. It also applied $2,000-per-child CTC instead of the 2026 $2,200." -us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It put the $9,600 of child support into AGI for $169,820, although child support received is excluded from gross income, invented a $1,700-per-dependent deduction, and declared the CTC fully phased out for AGI above $400,000 when AGI is $164,220 and the HoH phaseout starts at $200,000, leaving the full $4,400 available. Its submitted $18,779.30 matches neither the $14,152.40 its own explanation computes nor any step of the correct chain." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It set up AGI $164,220, the $800 QBI deduction and taxable income of about $139,120 correctly, and its own bracket computation gave about $24,321. It then replaced that with an unexplained ""tentative tax of $29,226"" (roughly $4,900 higher) before subtracting the $4,600 of credits. The correct pre-credit tax is $24,211.80 at the 2026 HoH brackets, so the result should have been $19,611.80." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its taxable income ($139,270) and credits ($4,400 CTC + $200 CDCC) are exactly right, but its bracket tax of $24,532.80 uses the 2025 HoH thresholds (10% to $17,000, 12% to $64,850, 22% to $103,350). The inflation-indexed 2026 thresholds are $17,700 / $67,450 / $105,700, which give $24,211.80, so it overstated tax by $321." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $9,600 of child support to gross income, although child support is not taxable. It also subtracted the $6,480 ESI premium as an above-the-line deduction and used an invented $26,050 HoH standard deduction with fabricated brackets (10% to $20,550, 12% to $78,100). On credits, it applied $2,000 per child instead of $2,200, dropped the $200 CDCC, and never took the $800 QBI deduction; offsetting errors produced $18,863." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It itemized $37,900 by imputing $29,400 of mortgage interest from the $420,000 balance and $8,500 of state tax. No mortgage interest is listed, so it counts as $0, and the $24,150 HoH standard deduction applies. It also used 2025-era brackets and a $2,000-per-child CTC. It then discarded its own computed $17,033 for an unsupported $21,889." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It itemized about $33,200 by imputing $25,200 of mortgage interest from the $420,000 loan balance plus $8,000 of SALT. Unlisted mortgage interest is $0, so the $24,150 HoH standard deduction governs. It also used a $2,000-per-child CTC and miscomputed the 22% band. It then abandoned its own $14,306 result for an arbitrary $21,625." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated most of the $4,400 CTC as refundable and counted only about $1,000 as nonrefundable. Because pre-credit tax (about $24,000) far exceeds the credit, the entire $4,400 CTC is nonrefundable and none is refundable ACTC. It also used a $24,500 standard deduction instead of $24,150 and 2025 bracket thresholds instead of the 2026 HoH brackets." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $800 Section 199A QBI deduction on the $4,000 of rental income, leaving taxable income at $140,070 instead of $139,270. That adds $192 of tax at the 24% rate ($24,403.80 vs $24,211.80). Its standard deduction, 2026 brackets and $4,600 of credits were otherwise correct." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It filed the unmarried parent of two qualifying children as single instead of head of household. It also itemized $37,300 by imputing $27,300 of mortgage interest from the loan balance plus $10,000 SALT, although no mortgage interest is an input and the $24,150 HoH standard deduction applies. It then taxed the result at 2025 single brackets and used a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It filed as single with the single standard deduction ($16,100) and single brackets instead of head of household ($24,150 and HoH brackets). It also cut wages by the $6,480 ESI premium, although AGI includes the full $160,000. It used a $2,000-per-child CTC and then rounded $22,315 to $22,300." +us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its bracket tax of about $24,725 on $140,070 matches 2025 HoH thresholds ($17,000 / $64,850 / $103,350) rather than the 2026 thresholds ($17,700 / $67,450 / $105,700). It also omitted the $800 QBI deduction on rental income, so it taxed $140,070 instead of $139,270." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It declared the CTC ""fully refundable in 2026"" and subtracted $0 of it. The CTC first offsets income tax as a nonrefundable credit, so the full $4,400 reduces this $24,211.80 liability. It also used a $23,000 standard deduction, omitted the $800 QBI deduction and overstated bracket tax at $27,259." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 ESI premium to get AGI of $157,740, but AGI includes the full $160,000 of wages, for $164,220. It also used a $23,200 standard deduction instead of $24,150, omitted the $800 rental QBI deduction, and applied a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law to 2026, using a $12,155 HoH standard deduction, three personal exemptions and a CTC it called fully phased out. The OBBBA made the TCJA structure permanent: a $24,150 HoH standard deduction, no personal exemptions and a $2,200 CTC unreduced below $200,000 AGI. It also subtracted the ESI premium from wages. Its submitted $26,346.25 does not even match its own stated $24,775.75." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, so it used an $11,687 standard deduction plus three $5,062 exemptions and a $1,000-per-child CTC. 2026 law instead provides a $24,150 HoH standard deduction, no exemptions and a $2,200-per-child CTC. It also cut wages by the ESI premium and omitted the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It started from the correct $164,220 AGI but subtracted a $4,000 CTC ($2,000 per child) instead of the 2026 $2,200-per-child $4,400. It never subtracted the $200 CDCC. Its implied pre-credit tax of $22,985 also understates the $24,211.80 from the 2026 HoH brackets on $139,270 of taxable income." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expiration, applying a $12,090 standard deduction, $15,900 of personal exemptions and pre-TCJA brackets, and it gave no CTC at all. 2026 law keeps the $24,150 HoH standard deduction, the 10/12/22/24 brackets and a nonrefundable $2,200-per-child CTC worth $4,400 here. It also subtracted the ESI premium from wages." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied ""2026 sunset"" law, with personal exemptions and a smaller standard deduction, and declared the CTC ""fully phased out due to high income"" under the old $75,000 threshold. Under 2026 law the CTC phase-out starts at $200,000, so the full $4,400 applies against tax computed with the $24,150 HoH standard deduction. It also cut AGI to $157,740 by subtracting the ESI premium." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. The correct path is taxable income of $139,270 ($164,220 - $24,150 - $800), tax of $24,211.80 at 2026 HoH brackets, less $4,600 of CTC and CDCC. Its $19,535 implies pre-credit tax of $24,135 after the same $4,600 of credits, a $76.80 bracket-arithmetic understatement." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It computed tax on ""gross income after pre-tax health insurance deductions,"" subtracting the $6,480 ESI premium from wages. AGI includes the full $160,000 of wages ($164,220 total). Its $21,743 is $2,131 above the correct $19,611.80, consistent with a pre-credit tax overstatement or undercounted nonrefundable credits on top of the wage error." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It treated the CTC as phased out, a pre-TCJA-sunset assumption, and subtracted only the $200 CDCC. Under 2026 law the $2,200-per-child CTC phases out only above $200,000 AGI, so the full $4,400 is nonrefundable here. It also computed tax on an AGI of $157,740 that wrongly subtracts the $6,480 ESI premium." +us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $22,500 HoH standard deduction instead of the 2026 $24,150 and omitted the $800 QBI deduction on rental income, overstating taxable income at $141,720 instead of $139,270. It also applied a $2,000-per-child CTC ($4,000) instead of $2,200 ($4,400)." +us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,480 ESI premium from wages to get AGI of $157,740 instead of $164,220. It also used a $23,100 standard deduction instead of $24,150, approximate brackets instead of the 2026 HoH thresholds, and a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2025 HoH standard deduction ($23,625) instead of the 2026 $24,150. It overstated bracket tax at $24,965 on $139,795 (2026 HoH brackets give $24,337.80 on that amount). It also applied an invented $2,250-per-child CTC instead of the $2,200 statutory amount." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its reasoning invokes a ""standard/itemized deduction"" choice. No mortgage interest or other itemizable expense is an input, so the $24,150 HoH standard deduction applies, giving tax of $24,211.80 less $4,600 of credits. Its $14,143 is $5,469 too low, which requires roughly $22,800 of extra deductions at the 24% rate, consistent with itemizing deductions absent from the inputs." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It explicitly treated the $9,600 of child support received as taxable income, although child support is excluded from gross income. It also referenced mortgage-interest and state-tax adjustments that have no inputs. It never subtracted the $4,400 CTC and $200 CDCC against pre-credit tax of $24,211.80, so it landed at $28,930." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,480 employee ESI premium, giving AGI of $157,740 instead of $164,220, which understated taxable income and tax. It also applied a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 ($4,400)." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It computed tax as married filing jointly with the $32,200 MFJ standard deduction and MFJ brackets. The head is unmarried with two qualifying children and files head of household ($24,150 standard deduction, HoH brackets). It also omitted the $800 QBI deduction: $132,020 at MFJ brackets gives $18,468.40, less $4,600, which is exactly its $13,868.40." +us,scenario_076,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It followed the correct structure (HoH, standard deduction, nonrefundable CTC and CDCC) but gave no figures. Its $19,606 implies pre-credit tax of $24,206 instead of $24,211.80 on $139,270 of taxable income, a $5.80 bracket-arithmetic shortfall." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It computed tax after ""deductions and exemptions,"" applying pre-TCJA personal exemptions that do not exist in 2026. The correct base is the $24,150 HoH standard deduction plus $800 QBI with no exemptions, then 2026 brackets and a full $4,400 CTC. Its $23,250 is $3,638 too high, consistent with losing most of the CTC under sunset-era rules." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used an understated $23,063 HoH standard deduction instead of $24,150 and a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 ($4,400). Its $24,570 bracket tax also exceeds what the 2026 HoH thresholds produce on its own $140,357." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: a $12,450 standard deduction, three $5,300 exemptions, pre-TCJA brackets and a CTC ""fully phased out above 75000 AGI."" 2026 law keeps the $24,150 HoH standard deduction, 10/12/22/24 brackets, and a $2,200-per-child CTC phasing out only above $200,000. It also omitted the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed ""post-TCJA-expiration law,"" using three $5,300 exemptions, a $12,250 standard deduction, 10/15/25 brackets and a $1,000 CTC phased out above $75,000. The TCJA structure is permanent in 2026: a $24,150 HoH standard deduction and a $2,200-per-child CTC fully available at $164,220 AGI. It also omitted the $800 QBI deduction." +us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied TCJA-expiration rules (a $12,585 standard deduction, $16,353 of personal exemptions, inflated 2017 brackets) and allowed no CTC. 2026 law gives a $24,150 HoH standard deduction, no exemptions and a nonrefundable $4,400 CTC. It also cut AGI by subtracting the $6,480 ESI premium from wages." +us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It subtracted only ""about $1,000 nonrefundable CTC portion"" for two children, treating the rest as refundable. With about $24,200 of pre-credit tax, the entire $4,400 CTC is used as a nonrefundable credit. That leaves $19,611.80 rather than its $23,300." +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_income_tax_before_refundable_credits, so no answer could be scored." +us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its AGI, $24,150 standard deduction, $800 QBI deduction, $139,270 taxable income and $4,600 of credits are all correct. Its bracket tax of $24,332.80 misapplies the 2026 HoH thresholds (10% to $17,700, 12% to $67,450, 22% to $105,700, 24% above), which give $24,211.80, so it overstated tax by $121." +us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It invented $1,815 of rental depreciation from the $49,921 unadjusted basis of qualified property, shrinking rental income to $2,185 and the QBI deduction to $437. That basis only feeds the 199A UBIA limit; the $4,000 rental income stands and the QBI deduction is $800. It also used a $23,200 standard deduction instead of $24,150 and a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It split the $4,400 CTC into $3,400 refundable and $1,000 nonrefundable. The refundable ACTC applies only to CTC exceeding tax liability, and here liability far exceeds the credit, so all $4,400 is nonrefundable. It also omitted the $800 QBI deduction, taxing $140,070 instead of $139,270." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed as married filing jointly (with a wrong $16,600 standard deduction) and MFJ brackets for an unmarried parent who files head of household with a $24,150 standard deduction and HoH brackets. It also omitted the $800 QBI deduction and used a $2,000-per-child CTC instead of $2,200." +us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It added the $9,600 of nontaxable child support to AGI and treated the children as $1,700 other-dependent credits. It also declared the CTC phased out above $400,000 even though its own AGI was $169,820. It then submitted $18,779.30, which matches neither its computed $14,152.40 nor the correct $24,211.80 - $4,400 CTC - $200 CDCC = $19,611.80." us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied the married-filing-jointly $400,000 phase-out threshold and a $2,000-per-child CTC instead of the head-of-household $200,000 threshold and 2026's $2,200 per child, then — after correctly stating that ACTC equals the CTC in excess of tax liability — ignored that offset even though pre-credit tax above $25,000 absorbs the entire $4,400 CTC nonrefundably. Its $2,400 is not the output of any rule it named; it is an unexplained reduction of the $3,400 ACTC ceiling, while the excess-over-liability computation yields $0." us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It stated the governing rule — the refundable ACTC is 'claimed to the extent CTC exceeds tax used' — and then contradicted it by reporting the $1,700-per-child cap of $3,400 as the refundable amount. With taxable income near $140,000 and pre-credit federal tax over $25,000, the full $4,400 CTC is consumed as a nonrefundable credit, leaving zero excess to convert into refundable_ctc." us,scenario_076,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It treated the entire Child Tax Credit as refundable, calling it a 'refundable child tax credit of 2,000 per child,' which errs twice: the 2026 per-child credit is $2,200, and only the portion unused against tax liability is refundable, capped at $1,700 per child. Because this filer's federal tax before credits exceeds $25,000, all $4,400 of CTC is applied nonrefundably and the refundable component is $0, not $4,000." @@ -5482,109 +6002,120 @@ us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $4,000 of rental income as earnings from self-employment and invented a 30% expense deduction that the prompt did not provide. Rental income is not subject to self-employment tax here, leaving a zero tax base and $0 liability." us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no self_employment_tax output or explanation, violating the required structured-output contract." -us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Got the 5.3% rate, the $24,150 head-of-household standard deduction and the $800 QBI deduction right, then taxed the full $139,270 instead of only the $128,430 above Idaho's $9,840 head-of-household zero-rate bracket, and never took Idaho's $1,000 household and dependent care expense deduction. It also subtracted $410 of Idaho child tax credits that this output does not net out, and after computing $6,971 it submitted an unexplained $7,099." -us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Correctly applied the 5.3% rate, the $24,150 standard deduction and Idaho's $1,000 household and dependent care deduction, but taxed the entire $139,070 rather than only income above the $9,840 head-of-household zero-rate bracket, and dropped the $800 §199A deduction on the $4,000 of rental income. It then subtracted $410 of Idaho child tax credits, which this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Treated the $9,600 of child support as taxable income, which is excluded from AGI, subtracted the $6,480 of employer-sponsored insurance premiums that are already excluded from taxable wages, used the single $15,000 standard deduction instead of the $24,150 head-of-household amount, and then invented a $9,380 federal-state conformity add-back. Applying a pre-2024 rate to that fabricated $161,720 base rather than 5.3% to the $128,430 above the $9,840 zero-rate bracket produced $9,400." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used Idaho's repealed 5.695% rate instead of the 5.3% rate in effect for 2026, a $23,625 standard deduction rather than $24,150, and took neither Idaho's $1,000 household and dependent care deduction nor the $800 QBI deduction. It then abandoned its own zero-bracket calculation of $7,741, recomputed $8,007 with no bracket at all, and submitted $8,204 with no derivation." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Applied the repealed 5.695% rate to a $141,720 base built from a $22,500 head-of-household standard deduction (the 2026 amount is $24,150), with no Idaho household and dependent care deduction and no $800 QBI deduction, and used a $3,420 zero-rate threshold instead of the indexed $9,840. Its own arithmetic yielded $7,876, but it submitted $9,112." -us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Had the correct 5.3% rate but used a $24,500 standard deduction, skipped Idaho's $1,000 household and dependent care expense deduction and the $800 QBI deduction, and taxed the whole base instead of only the amount above the $9,840 head-of-household zero-rate bracket. It computed $7,405 and then submitted $7,480 after unspecified 'typical adjustments'." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Filed the head as single and invented three $4,714 Idaho personal exemptions; Idaho conformed to the federal repeal of personal exemptions and allows none. It also used the repealed 5.695% rate, replaced Idaho's $1,000 household and dependent care expense deduction with a fabricated $100 credit equal to 50% of the federal CDCC, and never applied the $9,840 head-of-household zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Used the repealed 5.695% rate and a $16,000 standard deduction instead of 5.3% and $24,150, and omitted Idaho's $1,000 household and dependent care deduction, the $800 QBI deduction and the $9,840 head-of-household zero-rate bracket. Its own arithmetic gave $8,443, and it submitted $9,400 after 'rounding for local conformity nuances'." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied Idaho's 2023 rate of 5.8%, superseded by the 2024 cut to 5.695% and the 2025 cut to 5.3%, to a $144,320 base built on a $19,900 head-of-household standard deduction instead of $24,150. It also omitted the $1,000 household and dependent care deduction, the $800 QBI deduction and the $9,840 zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Subtracted the $6,480 of employer-sponsored insurance premiums from AGI to reach $157,740; those premiums are already excluded from taxable wages and Idaho's health-insurance premium deduction excludes amounts paid through an employer plan. It then used a $23,200 standard deduction and the repealed 5.8% rate on the entire base, skipping the $9,840 zero-rate bracket, the $1,000 household and dependent care deduction and the $800 QBI deduction." -us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $157,740 by subtracting the $6,480 of pre-tax employer-sponsored insurance premiums, used a $12,155 standard deduction rather than the $24,150 head-of-household amount, and applied the repealed 5.695% rate with no $9,840 zero-rate bracket, no $1,000 household and dependent care deduction and no $800 QBI deduction. Its explanation states $8,291.07 while the submitted value is $8,247.54." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied the repealed 5.8% rate rather than 2026's 5.3% to a $130,867 base that is $7,403 below the correct $138,270 of Idaho taxable income, and never applied the $9,840 head-of-household zero-rate bracket. It then subtracted $410 of Idaho child tax credits, which this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Gave no derivation. The correct computation is 5.3% × ($138,270 − $9,840) = $6,806.79; its $8,100 corresponds to a pre-2025 rate near 5.7–5.8% applied to roughly AGI less a standard deduction, with the $9,840 head-of-household zero-rate bracket, the $1,000 household and dependent care deduction and the $800 QBI deduction all omitted." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied Idaho's repealed 5.695% rate instead of the 5.3% rate effective for 2026 and taxed all $129,750 of its assumed base rather than only income above the $9,840 head-of-household zero-rate bracket. Its base is also $8,520 below the correct $138,270, because it took neither the $1,000 household and dependent care deduction nor the full $24,150 standard deduction plus $800 QBI deduction." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used an 'approx. 5.8%' rate, which is Idaho's 2023 rate cut to 5.695% for 2024 and 5.3% from 2025 onward, and never applied the $9,840 head-of-household zero-rate bracket. It then subtracted $205-per-child Idaho child tax credits that this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $7,322 equals 5.3% of $138,151, essentially the correct $138,270 of Idaho taxable income taxed from the first dollar. It had the right rate and deductions but never applied Idaho's $9,840 head-of-household zero-rate bracket, which removes $521.52 of tax." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Gave no derivation; the correct computation is 5.3% × ($138,270 − $9,840) = $6,806.79. Its $6,985 is 5.3% of $131,792, a base matching neither Idaho taxable income nor the amount above the $9,840 head-of-household zero-rate bracket, so it applied the rate to an ad hoc figure rather than AGI less the $24,150 standard deduction, the $1,000 household and dependent care deduction and the $800 QBI deduction, and it further netted nonrefundable credits this output does not subtract." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Gave no derivation; at Idaho's 5.3% rate its $7,665 implies $144,623 of taxed income, $16,193 more than the $128,430 actually taxed after the $24,150 head-of-household standard deduction, the $1,000 household and dependent care deduction, the $800 QBI deduction and the $9,840 zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Asserted that Idaho's income tax is not modeled and returned $0 rather than computing anything. Idaho's individual income tax applies a 5.3% rate to head-of-household taxable income above $9,840, giving $6,806.79 on this household's $138,270 of Idaho taxable income." -us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied the repealed 5.695% rate to a $141,720 base built on a $22,500 head-of-household standard deduction instead of $24,150, and omitted Idaho's $1,000 household and dependent care expense deduction, the $800 QBI deduction and the $9,840 zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Double-counted the standard deduction: it started from federal taxable income of about $139,795, already net of the federal standard deduction, then subtracted a $23,625 'Idaho head-of-household standard deduction' again, cutting the base to $116,170 instead of the correct $138,270. It also subtracted $410 of Idaho child tax credits that this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Gave no derivation. Idaho taxable income is $164,220 AGI less the $24,150 head-of-household standard deduction, the $1,000 household and dependent care deduction and the $800 QBI deduction, or $138,270, taxed at 5.3% above the $9,840 zero-rate bracket for $6,806.79; its $6,494 corresponds to only $122,528 of taxed income, $5,902 short of the $128,430 actually taxed." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Built its base on 'itemized medical assumptions', but this household takes the $24,150 head-of-household standard deduction and neither the $1,000 other medical nor the $500 over-the-counter expenses reduce Idaho taxable income. Its $7,450 is 5.3% of $140,566, showing it also skipped Idaho's $1,000 household and dependent care deduction, the $800 QBI deduction and the $9,840 zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used a $23,350 head-of-household standard deduction instead of $24,150, omitted Idaho's $1,000 household and dependent care expense deduction and the $800 QBI deduction, and taxed the full $140,870 rather than only income above the $9,840 zero-rate bracket. It then subtracted two $173 child credits; Idaho's child tax credit is $205 per child and none is netted out of this output." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Applied 5.3% to a $132,020 base, implying $32,200 of deductions against $164,220 of AGI rather than the correct $24,150 standard deduction plus $1,000 household and dependent care deduction plus $800 QBI deduction, and taxed that base from the first dollar instead of only above the $9,840 head-of-household zero-rate bracket. It also subtracted a $440 child credit that matches neither Idaho's $205-per-child credit nor this output's treatment, which nets out no credit." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Reached the correct $139,270 base after the $24,150 standard deduction and $800 QBI deduction but missed Idaho's $1,000 household and dependent care expense deduction, then applied the single-filer zero-rate threshold of about $4,930 instead of the doubled $9,840 head-of-household amount. It further subtracted $410 of Idaho child tax credits, which this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied the correct 5.3% rate to the correct post-standard-deduction, post-QBI figure of $139,270 but taxed it from the first dollar and never took Idaho's $1,000 household and dependent care expense deduction. Its $574.52 overstatement is exactly 5.3% of the omitted $9,840 head-of-household zero-rate bracket plus that $1,000 deduction." -us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Deducted the $6,480 of employer-sponsored insurance premiums to reach $131,790; Idaho's health-insurance premium deduction excludes premiums paid pre-tax through an employer plan, so the correct base is $138,270. It also used a $9,960 zero-rate threshold instead of the $9,840 head-of-household amount and subtracted $410 of child credits that this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Applied an invented '~4% effective' rate instead of Idaho's 5.3% flat rate and performed no base computation at all. The correct result is 5.3% × ($138,270 − $9,840) = $6,806.79." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Explicitly denied Idaho's QBI conformity, but Idaho starts from federal taxable income and therefore allows the $800 §199A deduction on the $4,000 of rental income. It also used the repealed 5.695% rate rather than 5.3%, skipped Idaho's $1,000 household and dependent care deduction, and taxed all $141,157 instead of only income above the $9,840 head-of-household zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Used a $12,450 standard deduction, roughly half the $24,150 head-of-household amount for 2026, together with the repealed 5.695% rate, and taxed all $151,770 rather than only income above the $9,840 zero-rate bracket. It also omitted the $1,000 household and dependent care deduction and the $800 QBI deduction." -us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $6,480 of pre-tax employer-sponsored insurance premiums from AGI to get $157,740 and used a $12,585 standard deduction instead of $24,150. It then applied the repealed 5.8% rate with no $9,840 zero-rate bracket, no $1,000 household and dependent care deduction and no $800 QBI deduction." -us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Used the correct 5.3% rate but taxed roughly $135,850 from the first dollar, never applying Idaho's $9,840 head-of-household zero-rate bracket, and took neither the $1,000 household and dependent care expense deduction nor the $800 QBI deduction that bring Idaho taxable income to $138,270." -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_income_tax_before_refundable_credits, so no substantive computation was recorded. The required answer is 5.3% × ($138,270 − $9,840) = $6,806.79." -us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Applied the single-filer, un-indexed $2,500 zero-rate threshold rather than the indexed $9,840 head-of-household amount, and stated that no Idaho QBI subtraction or household and dependent care deduction applies, when Idaho conforms to federal taxable income (the $800 §199A deduction) and allows a $1,000 household and dependent care expense deduction. It also subtracted $410 of Idaho child tax credits that this output does not net out." -us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Used a $3,500 'head-of-household standard deduction' instead of $24,150 and an unexplained $162,405 AGI, and applied the repealed 5.695% rate. It then subtracted $2,000 by treating the federal $1,000-per-child CTC as an Idaho child tax credit; Idaho's credit is $205 per child and none is netted out of this output." -us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Invented two $1,500 per-dependent deductions; Idaho grants no per-dependent deduction and instead allows the $1,000 household and dependent care expense deduction, which it never took, along with the $800 QBI deduction. It also taxed the whole $137,070 at 5.305% rather than applying the rate only to income above the $9,840 head-of-household zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"Applied the married-filing-jointly $30,000 standard deduction to a single parent who files head of household with a $24,150 standard deduction, and used the repealed 5.8% rate rather than 5.3%. Its $96,448 taxable income does not follow from its own $164,220 AGI, and it never applied the $9,840 head-of-household zero-rate bracket." -us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Used Idaho's pre-2023 graduated 1%/3%/5.5% bracket schedule, which was replaced by a flat rate (5.8% in 2023, 5.695% in 2024, 5.3% from 2025), and a $2,400 'Idaho standard deduction' in place of the $24,150 federal head-of-household standard deduction Idaho conforms to. Its stated arithmetic also yields $5,456.85 while it submitted $5,456.20." -us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model got the structure of Idaho's grocery credit exactly right — refundable, three exemptions, no senior increment, no income test — but priced it at the stale $120 per person that applied for tax years 2022 through 2024. Idaho HB 231 (2025) raised the §63-3024A base amount to $155 per person effective for tax years beginning in 2025, so 3 × $155 = $465, not 3 × $120 = $360." -us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Idaho provides no state-level refundable individual income tax credits for 2026, which erases Idaho Code §63-3024A: the grocery credit is refundable and is claimed even by residents with no filing requirement (on Form 24). It compounded the error by treating the household's $160,000 wage income as disqualifying, when the grocery credit has no income test at all; 3 residents × $155 = $465." -us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model identified the right credit and the right per-person structure, then abandoned its own $120 figure for an invented $170 per-person rate to reach $510. The statutory 2026 amount under Idaho Code §63-3024A as amended by HB 231 (2025) is $155 per person, giving 3 × $155 = $465." -us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model inverted the grocery credit's defining feature, calling it a nonrefundable offset that 'only becomes refundable in limited circumstances.' Idaho Code §63-3024A makes the credit fully refundable for every full-year resident regardless of tax liability or income, so the three household members generate 3 × $155 = $465." -us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model classified Idaho's grocery credit as nonrefundable and conditioned it on income, both of which are wrong: §63-3024A is refundable to any full-year resident with no income test or phaseout. It was right that Idaho's $205-per-child credit is nonrefundable, but that does not zero out the grocery credit's 3 × $155 = $465." -us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated flatly that Idaho's main credits, 'such as the grocery credit,' are nonrefundable. The grocery credit under Idaho Code §63-3024A is Idaho's signature refundable credit — claimable on Form 24 by residents owing no tax — and the head plus two children each earn 12 qualifying months at $155, totaling $465." -us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model recognized mid-reasoning that the Idaho grocery credit 'is actually refundable,' then discarded it by asserting the household's $160,000 income was 'far too high.' Idaho Code §63-3024A carries no income limit and no phaseout, so the correct result is 3 residents × $155 = $465." -us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model treated Idaho's $205-per-child credit under Idaho Code §63-3029L as refundable; it is a nonrefundable credit limited to tax liability, and it is not part of state_refundable_credits. It also omitted the credit that actually applies — the refundable grocery credit at $155 for each of the three household members, $465." -us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model labeled Idaho's $205-per-qualifying-child credit refundable, but §63-3029L makes it nonrefundable and capped at tax liability, so it contributes nothing to this output. The refundable credit it skipped is the grocery credit, $155 per person for the head and both children, totaling $465." -us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly picked the refundable grocery credit and the correct three-member base but used the superseded $120 per-person amount from tax years 2022–2024. HB 231 (2025) set the rate at $155 per person for 2025 forward, making the total 3 × $155 = $465." -us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the pre-2025 $120 per-person grocery credit rate to three residents. The 2026 statutory amount under Idaho Code §63-3024A, as amended by HB 231 (2025), is $155 per person, yielding $465 rather than $360." -us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model denied any Idaho refundable credit 'for this income level,' importing an income phaseout that Idaho's grocery credit does not have. Every full-year Idaho resident, including this household's head and two children, receives $155 under §63-3024A regardless of income, totaling $465." -us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model conditioned Idaho refundable credits on 'income level and configuration,' but the grocery credit under §63-3024A is a flat per-resident refundable amount with no income or household-configuration test. Three residents at $155 for all 12 months produce $465." -us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the $100 per-person grocery credit rate that was repealed by HB 509 in 2022 and superseded again by HB 231 in 2025. The 2026 rate is $155 per household member, so the three-person household receives $465, not $300." -us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model returned zero with a bare restatement and no rule cited, missing Idaho's refundable grocery credit entirely. Idaho Code §63-3024A grants $155 to each of the three full-year residents, all qualifying for 12 months, for a total of $465." -us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model asserted the household does not qualify for refundable Idaho credits, overlooking that the grocery credit is granted to every full-year resident with no eligibility screen beyond residency and non-incarceration. The head and both children each qualify for all 12 months at $155, giving $465." -us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model selected the right credit and the right three-person base but applied the obsolete $120 per-person amount from the 2022–2024 rate schedule. Idaho's 2026 grocery credit is $155 per person, so the correct total is $465." -us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model claimed no refundable state income tax credits apply in Idaho, contradicting Idaho Code §63-3024A, which makes the grocery credit refundable to all full-year residents. Three qualifying members at $155 each across 36 qualifying months yield $465." -us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly counted three household members and correctly treated the grocery credit as refundable, but priced it at the pre-2022 $100 per person. The 2026 amount set by HB 231 (2025) is $155 per person, making the total $465." -us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,The model applied a $100 per-person grocery credit — the rate in effect before HB 509 (2022) and two increases out of date for 2026. At the current $155 statutory rate the head and two children generate $465. -us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model concluded no Idaho refundable credit was 'indicated from the provided facts,' but the grocery credit requires only Idaho residency and the number of exemptions, both of which the facts supply. Three residents × $155 = $465." -us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invoked 'typical Idaho refundable credit thresholds' to disqualify the household on its $160,000 income. Idaho's grocery credit has no income threshold whatsoever; the head and both children each receive the full $155 for 12 months, totaling $465." -us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model built the calculation correctly — refundable grocery credit, three household members, no senior increment — but used the stale $120 per-person rate from tax years 2022–2024. The 2026 rate is $155, giving $465." -us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model asserted no refundable Idaho individual income tax credit applies, missing the grocery credit under §63-3024A, which is refundable and claimed by every Idaho resident filer for themselves and each dependent. Three members at $155 each produce $465." -us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the refundable grocery credit and the correct three-person count but held the per-person amount at the superseded $120. HB 231 (2025) raised it to $155 per person for 2025 and later years, so the total is $465." -us,scenario_076,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model returned zero with no rule identified, omitting Idaho's refundable grocery credit. Idaho Code §63-3024A grants $155 per full-year resident to the head and both children, totaling $465 for the household." -us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied the grocery credit to the head plus two dependents but used a $100 per-person rate that predates HB 509 (2022). The 2026 amount is $155 per person, so the credit totals $465." -us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model claimed $100 per resident is the Idaho grocery credit 'under permanent law for 2026,' but the $155 rate enacted by HB 231 (2025) is itself permanent law, not a temporary bump reverting to a $100 baseline. Three residents at $155 give $465." -us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model asserted the Idaho grocery credit is 'fully phased out at this income level.' The credit under §63-3024A has no phaseout and no income ceiling — it is a flat $155 per resident for every full-year Idaho resident — so this household receives 3 × $155 = $465. -us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model had the credit, the refundability, and the three-member count right but priced each member at the obsolete $100 rate. The 2026 per-person grocery credit is $155, making the household total $465." -us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for state_refundable_credits, so the key was absent from its outputs rather than answered incorrectly. The required derivation is Idaho's refundable grocery credit at $155 per full-year resident for the head and two children, $465." -us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model reasoned correctly that all three members qualify and none is 65+, then applied the outdated $120 per-person rate from tax years 2022–2024. Idaho's 2026 grocery credit is $155 per person, so 3 × $155 = $465." -us,scenario_076,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model dismissed Idaho refundable credits as 'minimal at this income level and not triggered by listed facts,' but the grocery credit is triggered by Idaho residency and exemption count alone and does not shrink with income. The listed state (ID) and three household members produce the full $465." -us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly ruled out the aged supplement and correctly counted three members, but used a $100 per-person grocery credit rate that is two statutory increases out of date. At the 2026 rate of $155 per person the credit is $465." -us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model looked only for an Idaho earned income credit and, finding none, returned zero — Idaho has no state EITC, but it does have the refundable grocery credit under §63-3024A. That credit pays $155 for each of the head and two children, totaling $465." -us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model asserted no Idaho refundable credits apply, missing the grocery credit that every full-year Idaho resident claims for themselves and each dependent. Three members qualifying all 12 months at $155 each yield $465." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its stated chain is exactly the reference's: AGI $23,006, the $16,100 single standard deduction, taxable income $6,906, and 10% tax of $691. It then submitted $951, a figure none of its own steps produce, discarding a correct derivation at the reporting step." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It subtracted the $8,389 employer-sponsored insurance premium from wages that are already net of pre-tax premiums, cutting AGI to $14,617 instead of $23,006, and paired that with the 2024 $14,600 standard deduction rather than the 2026 single amount of $16,100. After computing $1.70 down that path it submitted $1,402, a number none of its calculations yield; the correct taxable income is $6,906 for $690.62 of tax." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It reached AGI $23,006, the $16,100 standard deduction, taxable income $6,906, and 10% tax of $691, then reported $1,080 as a 'rounded estimate reflecting slightly higher taxable income.' The 2026 10% bracket runs to $12,400, so all of $6,906 is taxed at 10% and its own arithmetic gives $690.62 with no upward adjustment available." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It manufactured the 2026 single standard deduction by inflating $15,000 by ~2.3% to $15,350 instead of using the actual $16,100, overstating taxable income by $750 and tax by $75. Every other step — the $3,000 capital-loss cap, AGI $23,006, standard over $7,583 of itemized deductions, the 10% bracket, no nonrefundable credits — matches the reference." -us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It derived taxable income of $6,906 and tax of $691, then submitted $0 by invoking a QBI deduction and unspecified 'other adjustments' that do not exist for a wage-only single filer with no business income. Nonrefundable credits are zero for a childless filer with no care or education expenses, so its own $690.62 stands as the answer." -us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 single standard deduction for 2026 instead of $16,100, leaving taxable income of $7,606 rather than $6,906 and tax $70 too high. The $3,000 capital-loss limitation, AGI of $23,006, and 10% bracket application are all correct." -us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the 2025 single standard deduction of $15,000 to tax year 2026 instead of the indexed $16,100, inflating taxable income to $8,006 and tax to $800.60. Its capital-loss treatment and AGI of $23,006 match the reference exactly, so the entire $110 gap is the deduction parameter." -us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It set the 2026 single standard deduction at $15,350 rather than $16,100, producing taxable income of $7,656 and tax of $765.60. The $750 deduction shortfall accounts for the full $75 overstatement; every other step matches the reference." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It treated the $8,389 employer-sponsored insurance premium as a further reduction of wages that already exclude pre-tax premiums, and then applied a repealed $5,300 personal exemption alongside a pre-TCJA $8,300 standard deduction instead of the single $16,100 standard deduction in force for 2026. Together those errors cut taxable income to $1,017 against the correct $6,906." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the 2025 $15,000 standard deduction instead of $16,100 and then zeroed out its own $8,006 of taxable income by asserting 'available credits,' when a childless single filer with no education, care, or retirement-saver expenses has no nonrefundable credit at all. Its stated taxable income implies positive tax, so the submitted $0 contradicts its own arithmetic; the correct figure is $6,906 taxed at 10% for $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It removed $8,389 of ESI premiums from already-post-premium wages and then assumed a TCJA sunset for 2026, combining an ~$8,300 standard deduction with a personal exemption. The 2026 single standard deduction is $16,100 with no personal exemption, and wages stay at $26,006, so taxable income is $6,906, not $1,117." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It double-counted the $8,389 employer-sponsored insurance premium against wages that already exclude it and applied a post-TCJA-sunset $8,300 standard deduction plus a $5,200 personal exemption. For 2026 the single standard deduction is $16,100 with no personal exemption and AGI is $23,006, giving taxable income of $6,906 rather than $1,117." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted $0 with no computation. Wages of $26,006 less the $3,000 capital-loss deduction give AGI of $23,006, which exceeds the 2026 single standard deduction of $16,100 by $6,906; that amount sits in the 10% bracket for $690.62, so $0 is consistent only with assuming the standard deduction covers all of AGI." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It reached the correct AGI of $23,006 but never named a standard deduction, and its $965.60 answer implies subtracting only about $13,350 — roughly $2,750 short of the 2026 single standard deduction of $16,100. With $16,100 the taxable income is $6,906 and the 10% bracket yields $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It subtracted $8,389 of ESI premiums from wages already reported net of pre-tax premiums, dropping AGI to $14,617 instead of $23,006, and then applied a personal exemption that no longer exists. Its explanation states $102 while the submitted value is $67, and neither follows from its steps; the 2026 single standard deduction of $16,100 leaves $6,906 of taxable income and $690.62 of tax." -us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied a $15,000 standard deduction to 2026 instead of the indexed $16,100, leaving taxable income of $8,006 and tax of $800.60. Its $3,000 capital-loss cap on the $7,968 net loss and AGI of $23,006 are correct, so the $1,100 deduction shortfall is the entire error." -us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at about $15,450 instead of $16,100, overstating taxable income by $650 and tax by $65. Its AGI of $23,006, capital-loss limitation, bracket placement, and conclusion of no nonrefundable credits all match the reference." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that $26,006 of wage income falls below the standard deduction, when the 2026 single standard deduction is $16,100 and AGI after the $3,000 capital-loss deduction is $23,006 — $6,906 above it. Its zero-tax conclusion rests on that magnitude error, not on any credit; $6,906 at 10% is $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It declared the household's wage income low relative to the standard deduction without computing either figure; AGI is $23,006 and the 2026 single standard deduction is $16,100, leaving $6,906 of taxable income. No nonrefundable credit exists for this childless filer, so the tax before refundable credits is $690.62, not zero." -us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used an $8,150 standard deduction — a pre-TCJA-era figure — instead of the 2026 single amount of $16,100, and compounded that by taxing income above $11,925 at 15%, a rate and breakpoint that do not exist for 2026 (the 10% bracket runs to $12,400, then 12%). Correct taxable income is $6,906, entirely at 10%, for $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is negative or zero after the standard deduction and capital-loss offset. AGI is $23,006 after the $3,000 §1211(b) limit and the 2026 single standard deduction is $16,100, so reaching zero requires $6,906 more deduction than the law provides; the tax is $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA sunset parameters for 2026 — an $8,300 standard deduction plus a $5,300 personal exemption — but the 2026 single standard deduction is $16,100 and the personal exemption remains zero. Its $13,600 of allowances is $2,500 short, raising taxable income to $9,406 instead of $6,906 and tax by exactly $250." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its AGI of $23,006 and the $3,000 cap on the $7,968 net capital loss are correct, but it applied a sunset-regime $8,300 standard deduction plus a $5,300 personal exemption for 2026 rather than the $16,100 single standard deduction with no personal exemption. That $2,500 allowance shortfall is the whole gap between its $941 and the correct $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It made two offsetting errors: subtracting $8,389 of employer-sponsored insurance premiums from wages that already exclude pre-tax premiums, and using a post-sunset $8,300 standard deduction instead of the 2026 single amount of $16,100. AGI is $23,006 and taxable income $6,906, so the tax is $690.62 rather than the $632 its $6,317 figure produced." -us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so the submission carries no substantive computation to evaluate. The contract required every requested key to be present with a numeric value." -us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It deducted the entire $7,968 of capital losses against wages instead of the $3,000 annual limit under IRC §1211(b), giving AGI of $18,038 rather than $23,006, and used a $15,750 standard deduction instead of $16,100. It then submitted $0 while its own text computes roughly $230, invoking 'personal exemption credits' that were repealed; the correct taxable income is $6,906 for $690.62." -us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It carried the 2025 single standard deduction of $15,750 into tax year 2026 instead of the indexed $16,100, leaving taxable income of $7,256 rather than $6,906. That $350 excess at 10% is the entire $35 overstatement; its capital-loss cap, AGI, and no-credit conclusion are correct." -us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It concluded there is 'no regular tax after nonrefundable credits' without computing taxable income; AGI is $23,006 after the $3,000 capital-loss deduction and the 2026 single standard deduction is $16,100, leaving $6,906 taxed at 10%. No nonrefundable credit reaches a childless single filer with no care, education, or retirement-contribution expenses, so nothing offsets the $690.62." +us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 and never subtracted the $9,622 head-of-household zero-rate amount. It also missed the $1,000 Idaho dependent care subtraction and took off a $410 child credit that the reference does not apply. It then moved the result to an unexplained $7,099." +us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It got the $1,000 dependent care subtraction right but left out the $800 QBI deduction, so its taxable income was $139,070 instead of $138,270. It then taxed that whole amount at 5.3% without subtracting the $9,622 head-of-household zero-rate amount, and took off a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted $9,600 of non-taxable child support as income and subtracted the $6,480 ESI premiums. It used the $15,000 single standard deduction instead of the $24,150 head-of-household amount, and then applied made-up graduated brackets instead of Idaho's flat 5.3% above $9,622. These errors produced an arbitrary $9,400." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It used the old 5.695% rate instead of the 2026 rate of 5.3%, and a $23,625 standard deduction. It dropped its own $4,673 threshold, so the $9,622 head-of-household zero-rate amount was never applied. It also missed the dependent care and QBI subtractions and ended on an unexplained $8,204." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used the old 5.695% rate, an understated $22,500 head-of-household standard deduction, and a $3,420 threshold instead of $9,622. After working out $7,876, it reported $9,112, a number its own arithmetic does not support." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It applied 5.3% to $139,720 using a made-up $24,500 standard deduction, with no zero-rate amount, and skipped the $1,000 dependent care and $800 QBI subtractions. It also said the credits lower the tax, yet reported $7,480, which is above its own $7,405 pre-credit figure." +us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its taxable income of $140,070 leaves out Idaho's $1,000 household and dependent care subtraction and the $800 QBI deduction. It used $9,346 as the zero-rate amount instead of $9,622. It then subtracted a $410 child credit that the reference computation does not include, which left it about $300 low." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It filed the head as single, using a $15,000 deduction, even though the head qualifies as head of household with a $24,150 deduction. It made up Idaho personal exemptions of $4,714 each, although Idaho has none, and used the repealed 5.695% rate. It also claimed a nonexistent $100 Idaho child care credit instead of the $1,000 dependent care subtraction, and never applied the $9,622 zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It used the old 5.695% rate and a $16,000 standard deduction instead of the $24,150 head-of-household amount, with no zero-rate amount. It then raised its own $8,443 figure to $9,400 with no computation behind the change." +us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It taxed all of $140,070 at 5.3% without subtracting the $9,622 head-of-household zero-rate amount. It also missed the $1,000 dependent care subtraction and the $800 QBI deduction, and took off a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It used the repealed 5.8% rate instead of 5.3%, and an understated $19,900 standard deduction instead of $24,150. It never applied the $9,622 zero-rate amount or the dependent care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It lowered AGI to $157,740 by subtracting the $6,480 ESI premiums, which the reference does not take out of AGI. It used a $23,200 deduction and the repealed 5.8% rate, and never applied the $9,622 zero-rate amount or the dependent care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted the $6,480 ESI premiums from AGI and used a $12,155 head-of-household deduction, about half the correct $24,150. It applied the old 5.695% rate with no zero-rate amount, and reported $8,247.54 even though its own arithmetic gave $8,291.07." +us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used a 5.0% rate instead of Idaho's 2026 rate of 5.3%, and did not subtract the $9,622 zero-rate amount. It also skipped the $1,000 dependent care and $800 QBI subtractions, and took off a $410 child credit plus a $40 dependent care credit, neither of which the reference applies." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used the repealed 5.8% rate on an unexplained $130,867 taxable income, with no $9,622 zero-rate amount. It then subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation. Its $8,100 answer matches applying an outdated rate of about 5.7–5.8% to roughly $140,000 without the $9,622 zero-rate amount, while the correct result is 5.3% × ($138,270 − $9,622) = $6,818.34." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to $129,750 of taxable income with no $9,622 zero-rate amount. The taxable income itself is understated and has no stated derivation, while the correct base is $138,270 taxed at 5.3%." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It used an outdated 5.8% rate instead of 5.3% and never subtracted the $9,622 head-of-household zero-rate amount. It also subtracted a child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. Its $7,322 is about what 5.3% on the full $138,270 comes to ($7,328), so it left out the $9,622 head-of-household zero-rate amount, which is worth $510 of tax." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $6,985 matches 5.3% on about $139,300 of taxable income, minus a $410 child credit. That means no $9,622 zero-rate amount, a missing $1,000 dependent care subtraction, and a child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation. Its $7,665 is $847 above the correct $6,818.34, which matches taxing a base of about $140,000 or more (5.3% gives about $7,400) with no $9,622 zero-rate amount and without the dependent care and QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It wrongly said Idaho income tax is not modeled and answered $0. Idaho levies a 5.3% flat tax on taxable income above $9,622 for head-of-household filers, which here comes to $6,818.34." +us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to all of $141,720, using an understated $22,500 standard deduction. It had no $9,622 zero-rate amount and no $1,000 dependent care or $800 QBI subtractions." +us,scenario_076,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It took the standard deduction twice: it started from federal taxable income of $139,795, which already reflects the deduction, and then subtracted about $23,625 more, giving $116,170. It also left out the zero-rate amount and subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave no derivation. Its $6,494 is $324 below the correct figure, which matches subtracting a $410 child credit that the reference does not include. The correct result is just 5.3% × ($138,270 − $9,622) = $6,818.34." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It based its estimate on 'itemized medical assumptions' instead of the $24,150 head-of-household standard deduction. Its $7,450 matches taxing roughly all of taxable income at 5.3% without the $9,622 zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It taxed all of $140,870 at 5.3%, using an understated $23,350 deduction, with no $9,622 zero-rate amount. It left out the dependent care and QBI subtractions and took off $346 in child credits that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed all of an understated $132,020 at 5.3% without the $9,622 zero-rate amount. It then subtracted a made-up $440 child credit, which the reference computation does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used a $4,930 zero-rate amount, which is the single-filer level, instead of the $9,622 head-of-household amount. It worked from $139,270 without the $1,000 dependent care subtraction and took off a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 and never subtracted the $9,622 head-of-household zero-rate amount. It also missed Idaho's $1,000 household and dependent care subtraction." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It wrongly subtracted the $6,480 employer insurance premiums, which dropped taxable income to $131,790 instead of $138,270. It used $9,960 as the zero-rate amount instead of $9,622 and subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied 5.3% to all of $139,270 with no $9,622 head-of-household zero-rate amount and missed the $1,000 dependent care subtraction. It then took off a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It taxed all of $139,270 at 5.3% without subtracting the $9,622 zero-rate amount and without the $1,000 dependent care subtraction. It then subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It got taxable income right at $138,270 but used a $4,980 single-level zero-rate amount instead of the $9,622 head-of-household amount. It then subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It guessed a 4% effective rate instead of working the calculation. The correct figure comes from applying Idaho's flat 5.3% to $138,270 minus the $9,622 head-of-household zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It wrongly said Idaho does not follow the federal QBI deduction; Idaho allows the $800 QBI deduction here. It also missed the $1,000 dependent care subtraction, used the repealed 5.695% rate, and left out the $9,622 zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used a $12,450 head-of-household standard deduction instead of $24,150, which pushed taxable income up to $151,770. It applied the repealed 5.695% rate with no $9,622 zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to all of $136,070, instead of 5.3% on the amount above the $9,622 head-of-household zero-rate threshold." +us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It lowered AGI to $157,740 by subtracting the ESI premiums and used a $12,585 deduction instead of $24,150. It then applied the repealed 5.8% rate to the whole amount with no zero-rate amount." +us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to a taxable income of $135,000–$140,000 with no $9,622 head-of-household zero-rate amount, which overstates the tax by roughly $400 or more compared with the $6,818.34 correct figure." +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to grade." +us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It used a made-up $2,500 zero-rate amount instead of $9,622. It explicitly refused the Idaho QBI deduction and left out the $1,000 dependent care subtraction, giving $140,070. It also subtracted a $410 child credit that the reference does not include." +us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It used an unexplained AGI of $162,405 and a $3,500 head-of-household deduction instead of $24,150, then applied the repealed 5.695% rate. It subtracted a made-up $1,000-per-child Idaho credit, and the offsetting errors happened to land at $7,049." +us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It made up $1,500-per-dependent deductions and used a 5.305% rate. It never applied the $9,622 zero-rate amount, the $1,000 dependent care subtraction, or the $800 QBI deduction." +us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used a married-filing-jointly deduction of about $30,000 for a head-of-household filer and arrived at a taxable income of $96,448 that does not follow from its own figures. It then applied the repealed 5.8% rate." +us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It applied Idaho's old graduated brackets (1%/3%/5.5%), although since 2023 Idaho has used a single flat rate, 5.3% in 2026, above $9,622 for head of household. It also started from a federal taxable income of $121,620 and subtracted a made-up $2,400 Idaho standard deduction." +us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It identified the grocery credit and the 3 exemptions correctly but used the superseded $120 per-person amount. For 2026 the amount is $155, so 3 x $155 = $465, not $360." +us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It asserted that Idaho has no refundable state credits and pointed to the household's high income, missing the refundable grocery credit. That credit has no income limit and pays $155 for each of the 3 members, totaling $465." +us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It recognized the grocery credit for 3 people but replaced its own $120 figure with an invented $170 per person, giving $510. The 2026 amount is $155 per person, which yields $465." +us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It called Idaho's grocery credit nonrefundable except in limited cases and zeroed it out at this income. The grocery credit is fully refundable with no income test, worth $155 x 3 = $465." +us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It treated the grocery credit as nonrefundable or not applicable at this income and answered $0. The grocery credit is refundable regardless of income and pays $155 for each of the 3 members, $465 in total." +us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,It stated that Idaho's grocery credit is nonrefundable and so found no refundable credits. The grocery credit is Idaho's refundable credit: $155 per member x 3 = $465. +us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,It conceded that the grocery credit is refundable but then excluded it as 'already reflected elsewhere' and answered $0. The refundable grocery credit is exactly what state_refundable_credits captures: $155 x 3 = $465. +us,scenario_076,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,It applied the outdated $120 per-person grocery credit to 3 people ($360) instead of the 2026 amount of $155 per person ($465). +us,scenario_076,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It counted Idaho's $205-per-child child tax credit as refundable ($410). That credit is nonrefundable, and the model omitted the refundable grocery credit of $155 x 3 members = $465." +us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It reported Idaho's $205-per-child child tax credit as refundable ($410). That credit is nonrefundable, so it belongs in pre-refundable state tax. The only refundable state credit is the grocery credit of $155 x 3 = $465." +us,scenario_076,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly applied the refundable grocery credit to 3 members but used the old $120 amount instead of the 2026 amount of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used $120 per person for the grocery credit, which is the pre-2026 level. The 2026 credit is $155 per person, so 3 people yield $465, not $360." +us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the superseded $120 per-person grocery credit to 3 members ($360). The 2026 per-person amount is $155, which gives $465." +us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It assumed income excludes the household from Idaho refundable credits. The grocery credit has no income test and pays $155 for each of the 3 members, $465 in total." +us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It claimed that no Idaho refundable credit applies at this income level, missing the income-independent refundable grocery credit of $155 per member x 3 = $465." +us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used the pre-2023 grocery credit of $100 per person ($300). The 2026 amount is $155 per person, so 3 members give $465." +us,scenario_076,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Its bare $0 shows it never applied Idaho's refundable grocery credit. That credit is worth $155 for each of the head and two children, $465 in total." +us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It concluded that the household qualifies for no refundable Idaho credit, overlooking the grocery credit. Every resident member receives it regardless of income: $155 x 3 = $465." +us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,It applied the old $120 per-person grocery credit to 3 members ($360) instead of the 2026 amount of $155 per person ($465). +us,scenario_076,state_refundable_credits,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It asserted that PolicyEngine models no Idaho refundable credits. PolicyEngine computes id_refundable_credits from the refundable grocery credit, at $155 x 3 members = $465." +us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the pre-2023 grocery credit of $100 per household member ($300). The 2026 amount is $155 per member, which yields $465 for 3 members." +us,scenario_076,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,It correctly treated the grocery credit as the only refundable Idaho credit but used the outdated $100 per-person amount. The 2026 amount of $155 per person gives $465. +us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It found no Idaho refundable credit in the listed facts, missing that the grocery credit needs only Idaho residency. It pays $155 for each of the 3 members, $465 in total." +us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,It assumed Idaho's refundable credits have income thresholds that this household exceeds. The grocery credit has no income limit and pays $155 x 3 = $465. +us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly applied the refundable grocery credit to 3 non-senior members but used the superseded $120 amount. At the 2026 amount of $155, the total is $465." +us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It stated that no refundable Idaho credit applies, omitting the refundable grocery credit of $155 per member for the head and two children ($465)." +us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied a $120 per-person grocery credit to 3 members ($360), using the pre-2026 amount instead of the 2026 amount of $155 per person ($465)." +us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It declared that no state refundable credits apply, missing Idaho's refundable grocery credit of $155 for each of the 3 household members ($465)." +us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used the outdated $100 per-person grocery credit ($300). The 2026 amount is $155 per person, which gives $465 for the head and two dependents." +us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It treated $100 per resident as the permanent-law grocery credit for 2026. The 2026 amount is $155 per person, so 3 members yield $465, not $300." +us,scenario_076,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly identified the grocery credit as the sole refundable credit but used $120 per member instead of the 2026 amount of $155, giving $360 instead of $465." +us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,It claimed that the grocery credit phases out fully at this income level. Idaho's grocery credit has no income phase-out and pays $155 x 3 members = $465. +us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,It applied the pre-2023 grocery credit of $100 per household member ($300) instead of the 2026 amount of $155 per member ($465). +us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so no answer was scored. The correct derivation is Idaho's grocery credit of $155 x 3 members = $465." +us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the grocery credit to the head and two non-senior children at the superseded $120 amount ($360). For 2026 the amount is $155 per person, giving $465." +us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,It dismissed the grocery credit as not triggered at this income level. The credit applies to every resident member regardless of income and pays $155 x 3 = $465. +us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It correctly found no aged supplement but used the pre-2023 $100 per-member grocery credit ($300). The 2026 amount is $155 per member, which gives $465." +us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It looked only for an earned income credit and found no refundable Idaho credit, missing the refundable grocery credit of $155 for each of the 3 members ($465)." +us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It stated that no Idaho refundable credits apply, omitting the grocery credit. Idaho refunds it at $155 per household member, $465 for this 3-person household." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference: AGI $23,006, a $16,100 standard deduction, taxable income $6,906, and 10% tax of $691. It then submitted 951, a number its own derivation does not support. This is a transcription error on the final value." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, although the reference AGI starts from the full $26,006. It also used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100, which gave $1.70 of tax. It then submitted an unexplained $1,402 that none of its steps produce." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived AGI $23,006, a $16,100 standard deduction, taxable income $6,906 and 10% tax of about $691. It then replaced that with an arbitrary $1,080, citing 'slightly higher taxable income', and nothing in its computation supports the adjustment." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took the pre-OBBBA 2025 standard deduction of $15,000 and indexed it to $15,350 for 2026. The OBBBA-set 2026 figure is $16,100. That overstated taxable income at $7,656 instead of $6,906, giving $766 instead of $691." +us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly computed taxable income of $6,906 and tax of about $691. It then zeroed the liability by invoking a QBI deduction and other credits that a childless W-2 wage earner with no business income cannot claim. No nonrefundable credit applies, so the $691 stands." +us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 single standard deduction instead of the 2026 figure of $16,100. That gave taxable income of $7,606 instead of $6,906 and tax of $760.60 instead of $690.62." +us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $15,000 instead of the 2026 figure of $16,100. That overstated taxable income at $8,006 and tax at $800.60." +us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used an inflation-indexed $15,350 standard deduction built from the pre-OBBBA $15,000 base, not the 2026 figure of $16,100. That gave taxable income of $7,656 and tax of $765.60." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, which cut AGI to $14,617 instead of $23,006. It also applied TCJA-sunset rules, an $8,300 standard deduction plus a $5,300 personal exemption, instead of the permanent $16,100 standard deduction with no exemption." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the outdated $15,000 standard deduction and reached positive taxable income of $8,006. It then declared the tax fully offset by 'available credits', but a childless single wage earner has no nonrefundable credit, so the correct $16,100 deduction leaves $691 of tax." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated wages, which gave AGI of $14,617 instead of $23,006. It then assumed the TCJA sunset, with about $13,500 of combined standard deduction and personal exemption, instead of the 2026 law's $16,100 standard deduction and no exemption." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $8,389 ESI premium, which gave AGI of $14,617 instead of $23,006. It then applied the TCJA-sunset $8,300 standard deduction plus a $5,200 personal exemption instead of the $16,100 standard deduction in force for 2026." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It asserted $0 on the grounds that the standard deduction and capital-loss limit eliminate taxable income. In fact AGI of $23,006 less the $16,100 standard deduction leaves $6,906 taxable at 10%, which is $691." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its AGI of $23,006 is correct. Its $965.60 implies taxable income of $9,656, which means it subtracted only about $13,350 in deductions (a sunset-style standard deduction plus an exemption) instead of the $16,100 standard deduction." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated wages, which gave AGI of $14,617 instead of $23,006. It then applied a sunset-era standard deduction plus a personal exemption. On top of that, it submitted $67 while its explanation states $102." +us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 for 2026 instead of $16,100. That gave taxable income of $8,006 and tax of $800.60 instead of $690.62." +us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction at $15,450 instead of the enacted $16,100. That gave taxable income of $7,556 and tax of $755.60." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed $26,006 of wages falls below the standard deduction, but the 2026 single standard deduction is $16,100. AGI of $23,006 after the $3,000 capital-loss deduction therefore leaves $6,906 of taxable income and $691 of tax." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the wage income sits below the standard deduction and produces no tax. In fact $23,006 of AGI less the $16,100 standard deduction leaves $6,906 taxed at 10%, and no nonrefundable credit offsets that $691." +us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a sunset-era $8,150 standard deduction instead of $16,100, which inflated taxable income to $14,856. It then taxed that income with a sunset 15% bracket above $11,925, instead of the 2026 schedule where all $6,906 of taxable income falls in the 10% bracket." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is zero or negative after the standard deduction and capital-loss offset. In fact $26,006 minus the $3,000 loss limit minus the $16,100 standard deduction leaves $6,906 taxable, which is $691 of tax." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset parameters, an $8,300 standard deduction plus a $5,300 personal exemption, instead of the permanent 2026 standard deduction of $16,100 with no personal exemption. That raised taxable income from $6,906 to $9,406." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and subtracted an $8,300 standard deduction plus a $5,300 personal exemption. Current 2026 law allows a $16,100 standard deduction and no exemption, so taxable income is $6,906, not $9,406." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules, an $8,300 standard deduction and a $5,300 personal exemption, instead of the 2026 standard deduction of $16,100 with personal exemptions still zeroed. That produced $9,406 of taxable income instead of $6,906." +us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 ESI premium from the stated gross wages, which gave AGI of $14,617 instead of $23,006. It also used the sunset-era $8,300 standard deduction instead of $16,100, and the two errors partly offset to give $632." +us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this output, so no answer was available to score against the $690.62 reference." +us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It deducted the full $7,968 of capital losses against wages without the $3,000 annual limit and used the 2025 $15,750 standard deduction instead of $16,100. It then zeroed the result with nonexistent 'personal exemption credits' instead of taxing $6,906 at 10%." +us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 OBBBA single standard deduction of $15,750 without the 2026 inflation adjustment to $16,100. That gave taxable income of $7,256 and tax of $725.60." +us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It correctly netted the capital losses to the $3,000 limit but then asserted there is no regular tax. In fact $23,006 of AGI less the $16,100 standard deduction leaves $6,906 taxed at 10%, and no nonrefundable credit applies." us,scenario_077,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly determined that the greater-of-earned-income-or-AGI rule uses $26,006 and fully phases out the childless EITC, then submitted $56 despite its own zero-credit computation. No other refundable credit supplies the submitted amount." us,scenario_077,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly left $9 of childless EITC after acknowledging $26,006 of wages. EITC phaseout uses the greater of earned income or AGI, so the capital-loss deduction does not prevent the credit from being fully phased out." us,scenario_077,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model reduced $26,006 of wages by the $8,389 employer-sponsored insurance premium and treated $17,617 as EITC earned income. Employer premiums do not reduce EITC earned income here, so the phaseout must use $26,006 and eliminates the childless credit." @@ -5624,89 +6155,103 @@ us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model’s explanation correctly states that the one-person household exceeds the SNAP net-income limit and explicitly concludes “value = 0,” but it submitted 276. The numeric output contradicts its own eligibility conclusion and should have been 0." us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person SNAP net-income and benefit calculations to its $26,006 of wages. After the allowable earned-income and standard deductions, the household does not generate an eligible allotment, so the asserted $2,400 benefit has no valid computation." us,scenario_077,snap,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly awarded the $23 minimum allotment after determining that the regular formula produced zero. Broad-based categorical eligibility does not entitle a household failing the applicable net-income or benefit test to the minimum allotment; the minimum applies only to an otherwise eligible one- or two-person household. -us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Froze Louisiana's flat-tax standard deduction at the enacted $12,500 rather than the 2026 inflation-indexed $12,835, then discarded its own $315.18 arithmetic and reported $496 ""after accounting for the deduction structure"" with no supporting computation. The correct chain is $23,006.18 AGI − $12,835 = $10,171.18 taxed at the flat 3% = $305.14; $496 corresponds to no Louisiana rate-and-deduction combination." -us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Built the base correctly — federal AGI of $23,006 (wages less the $3,000 net capital-loss limitation) at Louisiana's flat 3% — but used the unindexed $12,500 combined standard deduction. Louisiana indexes that deduction for inflation beginning in tax year 2026, where the single amount is $12,835, leaving $10,171.18 taxable and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that Louisiana levies no individual income tax, confusing it with the no-income-tax states; Louisiana's 2025 reform kept the tax and converted it to a flat 3% rate. On $23,006.18 of federal AGI less the 2026 single standard deduction of $12,835, the base is $10,171.18 and the tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly identified the post-reform flat 3% rate and the repeal of the separate personal-exemption credit, but applied the statutory $12,500 standard deduction without the 2026 inflation indexation that raises it to $12,835. That $335 shortfall in the deduction overstates taxable income at $10,506 instead of $10,171.18 and the tax at $315 instead of $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Reached the right AGI of $23,006 and the right flat 3% rate but subtracted a $12,500 combined deduction/exemption, missing the 2026 inflation adjustment to $12,835. Taxable income is $10,171.18, not $10,506, so the tax is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Derived $315 from a $12,500 deduction and then inflated the answer to $446, attributing the difference to ""capital-loss treatment and rounding in PolicyEngine""; no such adjustment exists — the $3,000 capital-loss cap is already embedded in the $23,006 AGI. The only correction its derivation needed was the 2026 indexed deduction of $12,835, which yields $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Reconstructed Louisiana's repealed pre-2022 base, subtracting a $4,500 personal exemption and a deduction for federal income tax paid ($801), and never applied the combined standard deduction at all. Louisiana's flat-tax regime allows one $12,835 single deduction for 2026 and no federal-tax deduction, so the base is $10,171.18 rather than $17,705 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Computed $315 from the unindexed $12,500 deduction and then raised the submitted value to $448 for ""standard deductions and personal exemption credits"" — deductions and credits reduce liability, so no such adjustment raises the tax. Using the 2026 indexed deduction of $12,835 gives $10,171.18 taxable at 3% = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied the correct 3% flat rate to the correct $23,006 AGI but subtracted an unstated deduction of $15,400 to reach its $7,606 base. Louisiana's 2026 single standard deduction is $12,835, leaving $10,171.18 taxable and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Used the unindexed $12,500 standard deduction against the correct $23,006 AGI at the correct 3% flat rate. Louisiana indexes the combined deduction from 2026, where it is $12,835, so taxable income is $10,171.18 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied Louisiana's repealed graduated 1.85%/3.5% brackets on top of the pre-reform $4,500 standard deduction, $1,000 personal exemption, and federal income tax deduction of $765.60. For 2026 Louisiana taxes federal AGI less a single $12,835 combined deduction at one flat 3% rate, giving $10,171.18 taxable and $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $8,389 employer-sponsored insurance premium from wages to reach a $14,617 AGI; that premium never reduces Louisiana AGI, which is federal AGI of $23,006 (wages less the $3,000 capital-loss cap). It compounded this by using the repealed $4,500 deduction and 1.85% bottom bracket instead of the $12,835 deduction and flat 3%, which produce $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Submitted $158 with no deduction amount, rate, or base stated; the figure matches neither Louisiana's flat 3% rate nor its $12,835 standard deduction, and corresponds to the repealed 1.85% bottom-bracket rate applied to roughly $8,540 of income. The correct computation is $23,006.18 − $12,835 = $10,171.18 at 3% = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Netted the $8,389 ESI premium out of wages to get a $14,617 AGI, then stacked a $4,500 standard deduction with a separate $4,500 personal exemption and taxed the remainder at the repealed 1.85% rate. Louisiana AGI is $23,006.18, the exemption is folded into a single $12,835 combined deduction for 2026, and the rate is a flat 3%, giving $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $14,617 by deducting the $8,389 ESI premium from the $26,006 of wages, which Louisiana does not allow, and applied the repealed 1.85% bracket to income above a $4,500 deduction. Louisiana AGI is $23,006.18, the 2026 combined deduction $12,835, and the rate a flat 3%, yielding $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Asserted without computation that deductions and exemptions zero out the Louisiana base; the single 2026 combined deduction is $12,835 against $23,006.18 of federal AGI, leaving $10,171.18 fully taxable at the 3% flat rate for $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Reported $315.18, the exact result of taxing $23,006 less an unindexed $12,500 deduction at 3%. Louisiana's combined deduction is inflation-indexed from 2026 to $12,835, so the base is $10,171.18 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Its explanation states $130 while it submitted $152, and neither number is tied to any stated deduction, base, or rate. Louisiana's 2026 tax here is $23,006.18 of AGI less the $12,835 single standard deduction, or $10,171.18, at the flat 3% rate = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Claimed Louisiana's individual income tax is not modeled in PolicyEngine and returned zero; PolicyEngine implements Louisiana's flat 3% rate and its indexed $12,835 single standard deduction. On $23,006.18 of AGI that produces $10,171.18 of taxable income and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Derived the correct $23,006 AGI including the $3,000 capital-loss limitation and the correct 3% flat rate, but used the unindexed $12,500 standard deduction. The 2026 indexed amount is $12,835, giving $10,171.18 taxable and $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Made two parameter errors: the unindexed $12,500 deduction instead of 2026's $12,835, and Louisiana's repealed 1.85% bottom-bracket rate instead of the single 3% flat rate that replaced the graduated schedule. Applying 3% to $10,171.18 gives $305.14, not 1.85% of $10,506." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Claimed taxable income falls below the lowest-bracket threshold; Louisiana's flat 3% structure has no zero-rate bracket above the standard deduction, so every dollar past the deduction is taxed. AGI of $23,006.18 less the $12,835 single deduction leaves $10,171.18 taxable and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Returned zero on the assumption that low wage income plus standard treatment leaves no liability; $26,006 of wages less the $3,000 capital-loss cap gives $23,006.18 of AGI, well above Louisiana's $12,835 single deduction. The remaining $10,171.18 is taxed at the flat 3% with no nonrefundable credit to offset it, for $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Got the flat 3% rate and the $23,006 AGI right but subtracted the unindexed $12,500 standard deduction. Louisiana indexes that deduction beginning in 2026 to $12,835, making the base $10,171.18 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Applied the $3,000 capital-loss limitation and the 3% flat rate correctly but held the standard deduction at $12,500. The 2026 inflation-indexed single deduction is $12,835, yielding $10,171.18 of taxable income and $305.14 of tax rather than $315.18." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used $12,500 as the Louisiana standard deduction against the correct $23,006 AGI and correct 3% flat rate. The deduction is indexed to $12,835 for 2026, so the base is $10,171.18 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Stacked a $4,500 personal exemption on top of a $12,500 standard deduction; Louisiana's flat-tax reform folded the personal exemption into a single combined standard deduction, which is $12,835 for a single filer in 2026. Only that one deduction applies, leaving $10,171.18 taxable at 3% = $305.14 rather than 3% of $6,006." -us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Correctly recognized that Louisiana's flat-tax standard deduction is inflation-indexed for 2026 and applied the flat 3% rate to the correct $23,006 AGI, but used $12,875 as the indexed amount instead of $12,835. The $40 overstatement of the deduction understates the base by $40 and the tax by $1.21." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Asserted the Louisiana base is zero or negative after deductions and exemptions; $23,006.18 of federal AGI less the single $12,835 combined deduction leaves $10,171.18. Taxed at Louisiana's flat 3%, that is $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Applied Louisiana's repealed graduated schedule — 1.85% on the first $12,500 and 3.5% above — to AGI reduced only by a $4,500 personal exemption. For 2026 Louisiana has one 3% rate and one $12,835 combined standard deduction, so the base is $10,171.18 rather than $18,506 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Identified the flat 3% rate, the $23,006 AGI, and the absence of Louisiana nonrefundable credits, but used the unindexed $12,500 standard deduction. The 2026 indexed single deduction of $12,835 leaves $10,171.18 taxable, for $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Removed the $8,389 ESI premium from wages to reach a $14,617 Louisiana AGI, a deduction Louisiana does not permit, and then used the pre-reform $4,500 standard deduction with the repealed 1.85% rate. Louisiana AGI is $23,006.18, the 2026 deduction is $12,835, and the flat rate is 3%, giving $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Applied the right structure (federal AGI $23,006, flat 3%) with the unindexed $12,500 standard deduction. Louisiana's combined deduction is indexed to $12,835 for 2026, producing $10,171.18 of taxable income and $305.14 of tax." -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no Louisiana computation was submitted to evaluate. The required answer is $23,006.18 of AGI less the 2026 single standard deduction of $12,835, taxed at Louisiana's flat 3% = $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Carried the correct $23,006 AGI into Louisiana and applied the correct flat 3% rate, but subtracted $12,500 as the single standard deduction. The 2026 inflation-indexed amount is $12,835, so the base is $10,171.18 and the tax $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Stated flatly that Louisiana has no state income tax; Louisiana imposes a 3% flat individual income tax for 2026 on federal AGI above a $12,835 single standard deduction. That gives $10,171.18 of taxable income and $305.14 of tax here." -us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Substituted the federal standard deduction of $16,100 plus a $4,500 Louisiana personal exemption for Louisiana's own deduction; the federal standard deduction never enters the Louisiana base, and the exemption is folded into the state's combined $12,835 deduction. The base is $10,171.18, not $2,406, so 3% gives $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Dropped the $3,000 federal capital-loss limitation and deducted the full −$7,968 of net losses to reach an $18,038 AGI, then subtracted a $4,500 personal exemption and applied the repealed 1.85%/3.5% graduated brackets. Louisiana starts from federal AGI of $23,006.18, allows one $12,835 combined deduction for 2026, and taxes the $10,171.18 remainder at a flat 3% for $305.14." -us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Submitted $347.24 without naming a deduction amount or rate; at Louisiana's 3% flat rate that figure implies a taxable base of $11,575 and therefore a deduction near $11,431, which matches neither the enacted $12,500 nor the 2026 indexed $12,835. The correct base is $23,006.18 − $12,835 = $10,171.18, giving $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly worked out 3% × ($23,006 − $12,500) = $315.18, but it used the unindexed $12,500 deduction instead of the 2026 indexed $12,835. It then replaced that result with an unexplained $496 'after accounting for the deduction structure', and no Louisiana rule supports that figure." +us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used Louisiana's statutory $12,500 single standard deduction without the inflation indexing that begins in 2026. The correct figure is 3% × ($23,006.18 − $12,835) = $305.14, not 3% × $10,506 = $315.18." +us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It said Louisiana has no state individual income tax. Louisiana levies a flat 3% income tax on AGI minus a $12,835 standard deduction in 2026, which produces $305.14 here." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It correctly identified the post-reform flat 3% rate and that there is no separate personal credit. However, it subtracted the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835, which overstated taxable income by $335 and gave $315 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied the flat 3% rate to $23,006 − $12,500. It missed that Louisiana indexes the $12,500 single deduction for inflation starting in 2026, making it $12,835, so its taxable income of $10,506 should be $10,171.18 and the tax $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It computed about $315 using the unindexed $12,500 deduction, then raised the answer to $446 citing capital-loss treatment and rounding, which no rule supports. The capital loss is already capped at $3,000 inside AGI, and the correct deduction is the indexed $12,835, which gives $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the $12,500 standard deduction from 2025 instead of the 2026 inflation-indexed $12,835. That left $10,506 of taxable income instead of $10,171.18 and $315.18 of tax instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used Louisiana's repealed pre-2025 structure: a $4,500 personal exemption plus a deduction for federal income tax paid, which gave $17,705 of taxable income. From 2025 both were replaced by a single standard deduction ($12,835 in 2026), leaving $10,171.18 taxable at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It reached about $315 using the unindexed $12,500 deduction and then raised it to $448 by citing 'federal-based deductions' and 'personal exemption credits'. Louisiana repealed both of these for 2025+, and they would lower the tax, not raise it. The correct base is $23,006.18 − $12,835 at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It explicitly assumed a $12,500 standard deduction and ignored Louisiana's inflation indexing from 2026, which makes the single deduction $12,835. That overstated the tax at $315.18 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its taxable base of $7,606 means it subtracted $15,400 from the $23,006 AGI, which is roughly a federal-style standard deduction. Louisiana's 2026 single standard deduction is $12,835, which leaves $10,171.18 taxable at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. As a result it taxed $10,506 instead of $10,171.18 at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It applied Louisiana's repealed pre-2025 law: a deduction for federal income tax, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. For 2026 Louisiana taxes AGI minus the $12,835 standard deduction at a flat 3%." +us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used $12,500 as the single standard deduction and missed the 2026 inflation adjustment to $12,835. The result was $315.18 instead of 3% × $10,171.18 = $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,389 employer-sponsored insurance premiums from the stated gross wages, cutting AGI to $14,617 instead of $23,006.18. It then applied the repealed $4,500 exemption and the old 1.85% bottom bracket instead of the $12,835 deduction and the flat 3% rate." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no derivation. $158 matches the repealed 1.85% bottom-bracket rate on about $8,500 of income. The correct calculation is the flat 3% rate on $23,006.18 − $12,835 = $10,171.18, which gives $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It removed the $8,389 ESI premiums from gross wages, giving an AGI of $14,617. It then subtracted both a $4,500 standard deduction and a $4,500 personal exemption and taxed the rest at the repealed 1.85% rate. The 2026 law is a single $12,835 deduction from the $23,006.18 AGI and a flat 3% rate." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $14,617 by subtracting the ESI premiums from the stated gross wages. It then used the repealed $4,500 combined exemption and the 1.85% bracket, instead of the $12,835 standard deduction and flat 3% rate applied to the $23,006.18 AGI." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It said deductions and exemptions wipe out the tax. Louisiana's only 2026 deduction is the $12,835 standard deduction, which leaves $10,171.18 of taxable income and $305.14 of tax at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $315.18 comes from subtracting the unindexed $12,500 deduction from $23,006. The 2026 inflation-adjusted Louisiana standard deduction is $12,835, which gives $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It subtracted a separate standard deduction and personal exemption, which is the pre-2025 structure. Its explanation also states $130 while the submitted value is $152, and both are well below the correct 3% × ($23,006.18 − $12,835) = $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It claimed PolicyEngine does not model Louisiana income tax and returned $0. The engine applies Louisiana's flat 3% tax to AGI minus the $12,835 standard deduction, which gives $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly capped the capital loss at $3,000 and used the flat 3% rate. However, it used the unindexed $12,500 deduction instead of the 2026 inflation-adjusted $12,835, which overstated taxable income by $335." +us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the repealed 1.85% bottom-bracket rate instead of Louisiana's flat 3% rate. It also used the unindexed $12,500 deduction instead of $12,835." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed taxable income after the standard deduction is below a threshold. AGI of $23,006.18 minus the $12,835 deduction leaves $10,171.18, and Louisiana's flat 3% rate has no zero bracket, so the tax is $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed zero tax because wages are low and never applied the Louisiana calculation. AGI of $23,006.18 exceeds the $12,835 standard deduction by $10,171.18, which is taxed at 3% for $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without Louisiana's 2026 inflation indexing, which makes it $12,835. That gave $315.18 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 deduction instead of the 2026 indexed $12,835. That left $10,506 instead of $10,171.18 taxable at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the 2025 statutory $12,500 deduction to 2026 and missed the inflation adjustment to $12,835. This overstated the tax by about $10." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It subtracted the repealed $4,500 personal exemption on top of the $12,500 standard deduction. Louisiana's 2025 reform replaced the exemption with a single standard deduction, which is $12,835 in 2026, so taxable income is $10,171.18, not $6,006." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It correctly inflation-indexed the standard deduction but overstated it as $12,875 instead of $12,835. It also rounded AGI down to $23,006, which understated taxable income at $10,131 instead of $10,171.18 and gave $303.93 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction. The 2026 Louisiana single deduction is $12,835, which gives 3% × $10,171.18 = $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It applied $12,500 as the single-filer deduction and missed the 2026 inflation indexing to $12,835. The result was $315.18 instead of $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used Louisiana's $12,500 base-year standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated taxable income by $335 and the tax by about $10." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed deductions and exemptions reduce taxable income to zero. The only 2026 Louisiana deduction is the $12,835 standard deduction, which leaves $10,171.18 of the $23,006.18 AGI taxable at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It applied the repealed pre-2025 law: a $4,500 exemption and graduated 1.85%/3.5% brackets. For 2026 Louisiana uses a $12,835 standard deduction and a flat 3% rate." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 standard deduction instead of the 2026 inflation-adjusted $12,835. It then rounded 3% × $10,506 to $315 instead of computing 3% × $10,171.18 = $305.14." +us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It correctly noted that the federal-tax deduction was repealed and used the flat 3% rate. However, it used the $12,500 deduction without the 2026 inflation adjustment to $12,835." +us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premiums from gross wages, giving an AGI of $14,617 instead of $23,006.18. It then used the repealed $4,500 deduction and the 1.85% bracket instead of the $12,835 standard deduction and the flat 3% rate." +us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the flat 3% rate after the unindexed $12,500 deduction. The 2026 single deduction is $12,835, which leaves $10,171.18 taxable and $305.14 of tax." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_income_tax_before_refundable_credits, so no substantive answer exists to grade against the $305.14 from 3% × ($23,006.18 − $12,835)." +us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835, so it taxed $10,506 instead of $10,171.18 at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said Louisiana has no state income tax. Louisiana imposes a flat 3% individual income tax on AGI minus the $12,835 standard deduction, which gives $305.14 here." +us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $16,100 standard deduction plus a repealed $4,500 Louisiana personal exemption. Louisiana does not use the federal standard deduction; it allows only its own $12,835 standard deduction in 2026, which leaves $10,171.18 taxable at 3%." +us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It deducted the full $7,968 net capital loss instead of capping it at $3,000, which understated AGI at $18,038. It then applied the repealed $4,500 exemption and graduated 1.85%/3.5% brackets instead of the $12,835 standard deduction and flat 3% rate." +us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $347.24 implies about $11,575 of taxable income at 3%, which means a deduction of only about $11,430 from the $23,006.18 AGI. The correct 2026 Louisiana single standard deduction is $12,835, which leaves $10,171.18 taxable and $305.14 of tax." us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero." us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero." us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25." us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract." us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted only the listed real estate taxes ($26,460) as SALT and never deducted the household's own Maryland state-plus-county income tax liability of $9,773.77 that completes the engine's $36,233.77 SALT deduction, and it added the $1,632 state tax refund to AGI, producing taxable income of $145,467 instead of $134,061.22. It then misreported its own bracket sum: the components it listed (1,240 + 4,560 + 12,166 + 9,544.08) total $27,510.08, not the $33,499 it submitted." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It estimated the Maryland income tax component of SALT at about $6,905, roughly $2,869 short of the $9,773.77 state-plus-county liability the engine deducts, and it added the $1,632 state tax refund to AGI. Those two errors lifted taxable income from $134,061.22 to $138,562, and the $4,500.78 excess taxed at 24% is exactly the $1,080.19 by which its $25,852.88 exceeds $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted $2,606 of unreimbursed employee business expenses from AGI, a deduction the section 67(g) suspension of miscellaneous itemized deductions eliminates entirely, added the $1,632 state tax refund to AGI, and limited SALT to the $26,460 of real estate taxes while omitting the $9,773.77 Maryland state and county income tax. Its resulting $142,861 taxable income overstates the engine's $134,061.22, and $25,779 falls below even the $26,884.64 that its own taxable figure yields under 2026 single brackets." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It applied the pre-OBBBA $10,000 SALT cap; for 2026 the cap is $40,400 and phases down only above $505,000 of MAGI, so this filer's full $36,233.77 of real estate and Maryland income taxes is deductible. Capping SALT at $10,000 and adding the $1,632 refund inflated taxable income by $27,865.77 over the engine's $134,061.22, which at 24% is the entire $6,652 gap between $24,772.69 and its $31,425." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used the repealed $10,000 SALT cap instead of the 2026 cap of $40,400, deducting $38,210 rather than $64,443.77, and it added the $1,632 refund to AGI. It then discarded its own arithmetic result of $31,424 and submitted $36,757 with no computation supporting it." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000, a limit OBBBA replaced with $40,400 for 2026 subject to phase-down only above $505,000 of MAGI, so it deducted $38,210 instead of the engine's $64,443.77 and left $9,773.77 of Maryland state and county income tax undeducted. Adding the $1,632 refund to AGI compounded it, giving taxable income of $161,927 against the correct $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap that no longer exists for 2026, where the cap is $40,400 and this filer's $36,233.77 of real estate plus Maryland state and county income taxes is fully deductible, and it added the $1,632 refund to AGI. It also treated 2026 brackets as 2025 brackets nudged 2.8% rather than the published 10%/12%/22%/24% thresholds of $12,400, $50,400, $105,700 and $201,775." -us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000 rather than the 2026 limit of $40,400, dropping $26,233.77 of deductible Maryland state, county and property taxes and reaching taxable income of $160,295. It then abandoned its own $31,069 result and submitted $42,800 by adding an unspecified adjustment for Medicare and Additional Medicare surtax, which are payroll taxes and form no part of income tax before refundable credits." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap to the $26,460 of property taxes when the 2026 cap is $40,400, and it never deducted the $9,773.77 Maryland state and county income tax liability, so its itemized total was $38,210 instead of $64,443.77. Adding the $1,632 refund to AGI put taxable income at $161,927 rather than $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Its itemized total of $64,380 nearly matches the engine's $64,443.77, but it assumed the TCJA provisions expired: it subtracted a $5,063 personal exemption, which remains permanently repealed for 2026, and applied pre-TCJA 10/15/25/28 rates instead of the 2026 schedule of 10% to $12,400, 12% to $50,400, 22% to $105,700 and 24% to $201,775. The sunset assumption alone accounts for the entire $3,049 overstatement." -us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed tax on ""2026 pre-TCJA single brackets"" with a $5,300 personal exemption, but OBBBA made the 10/12/22/24 rate schedule permanent and personal exemptions stay repealed for 2026. It also understated the Maryland income tax in SALT at $7,472.43 versus the engine's $9,773.77 and added the $1,632 refund to AGI." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It explicitly computed ""following the expiration of the TCJA,"" claiming a $5,150 personal exemption and 10/15/25/28 rates; for 2026 the personal exemption is zero and the brackets are 10%/12%/22%/24% at $12,400, $50,400, $105,700 and $201,775. Its deduction total of $65,964 sits close to the engine's $64,443.77, so the obsolete rate schedule and phantom exemption produce the entire $2,254 error." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It took a $15,000 standard deduction and never compared it against the $64,443.77 of itemizable mortgage interest and SALT, discarding $48,343.77 of deductions; the 2026 single standard deduction is also $16,100, not $15,000. Its submitted $31,317 is consistent with a $10,000-capped SALT itemization rather than the $183,505 taxable income its own reasoning states, which under 2026 brackets would yield $36,639." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It asserted the restoration of personal exemptions and an AMT liability for 2026; personal exemptions remain repealed, and with a 2026 AMT exemption of $90,100 for single filers phasing out only above $500,000, tentative minimum tax on this household falls far below the regular tax, so AMT adds nothing. Those two invented adjustments push its $28,446 above the $24,772.69 that the 2026 rate schedule produces on $134,061.22 of taxable income." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It computed ""assuming TCJA sunset provisions,"" subtracting a $5,150 personal exemption that does not exist for 2026 and applying pre-TCJA rates; on its own $131,799 of taxable income the 2026 schedule yields $24,230, not the $28,554 it reported. It also set the Maryland income tax at about $6,886 rather than the $9,773.77 state-plus-county liability the engine deducts." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It applied only the standard deduction to $198,505 of wages, discarding the $64,443.77 of itemized deductions ($28,210 mortgage interest plus $36,233.77 SALT) that beat the $16,100 standard deduction by $48,343.77. Ignoring roughly $48,000 of deductions is what carries its $33,020 above the correct $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It subtracted a $5,050 personal exemption and applied post-sunset rates, but for 2026 the personal exemption is zero and the bracket schedule is 10%/12%/22%/24% at $12,400/$50,400/$105,700/$201,775. Its SALT of $37,999.06 and AGI of $200,137 (which wrongly includes the $1,632 refund) leave taxable income near the engine's, so the phantom exemption and stale rates drive the $2,349 overstatement." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $31,316.32 corresponds to taxable income near $161,300, which is what a $10,000 SALT cap plus the $1,632 refund produces; the 2026 SALT cap is $40,400, so the full $36,233.77 of Maryland state, county and property taxes is deductible and taxable income is $134,061.22. It withheld the $26,233.77 of SALT the engine allows, and 24% of that shortfall is the bulk of its $6,544 overstatement." -us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It stated it was ""claiming itemized deductions for mortgage interest and real estate taxes following TCJA expiration rules,"" so it both applied the repealed pre-TCJA rate schedule with a personal exemption and left the $9,773.77 Maryland state and county income tax out of SALT. The 2026 schedule with the engine's $64,443.77 of itemized deductions gives $24,772.69, not $26,742." -us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It took a $15,000 standard deduction, ignoring the $64,443.77 itemized total, then subtracted a $12,000 nonrefundable child and dependent care credit for a child this household does not have; the household is a single 19-year-old with no dependents, and the CDCC is in any case limited to $6,000 of qualifying expenses for two or more children. Removing the fabricated credit would have left it at $42,393, nearly $18,000 above the correct $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap, which OBBBA replaced with $40,400 for 2026 with phase-down only above $505,000 of MAGI, so it deducted $38,210 rather than the engine's $64,443.77 and omitted the entire $9,773.77 Maryland state and county income tax. Adding the $1,632 refund to AGI took taxable income to $161,927 instead of $134,061.22." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It stated that ""itemized deductions are capped by SALT at 10000"" when the 2026 cap is $40,400, leaving $26,233.77 of deductible Maryland and property taxes unused. Its $42,034 does not even follow from that: the $10,000-cap path yields taxable income of $161,927 and about $31,460 of tax, so it submitted a figure roughly $10,500 above its own described derivation." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $28,347 implies taxable income near $148,950, about $15,000 above the engine's $134,061.22, which is what itemizing only the $28,210 mortgage interest and $26,460 real estate taxes produces while omitting the $9,773.77 Maryland state and county income tax from SALT. The correct 2026 itemized total is $64,443.77 against the $16,100 standard deduction, giving $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It limited SALT to the $26,460 of real estate taxes and never deducted the $9,773.77 Maryland state and county income tax liability, and it added the $1,632 refund to AGI, reaching $145,467 of taxable income instead of $134,061.22. Its own taxable figure yields $27,510.08 under the 2026 single brackets, so the $29,576.96 it submitted adds a further $2,066.88 of bracket arithmetic error." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $25,947.40 corresponds to taxable income of $138,956, which is what adding the $1,632 refund to AGI and taking a SALT deduction about $3,263 short of the engine's $36,233.77 produces; it understated the Maryland county income tax layered on top of state tax. The correct base is $198,505 of AGI less $64,443.77 of itemized deductions, or $134,061.22, giving $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It itemized only $54,670 of mortgage interest and real estate taxes, omitting the $9,773.77 of Maryland state and county income tax that PolicyEngine deducts as part of the $36,233.77 SALT total, and it added the $1,632 refund to AGI. Those two moves overstate taxable income by $11,405.78, and 24% of that is exactly the $2,737.39 gap between $24,772.69 and its $27,510.08." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It deducted only the $28,210 mortgage interest and $26,460 real-estate taxes, leaving out the $9,773.77 Maryland state and county income tax that fits under the 2026 $40,400 SALT cap, so its deduction total was $54,670 against the engine's $64,443.77. That $9,773.77 shortfall, plus adding the $1,632 refund to AGI, is what carries $24,772.69 up to its rounded $27,500." -us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It wrote that ""no other deductible tax payments are reported,"" but the SALT deduction includes the filer's own computed Maryland state and county income tax liability of $9,773.77, not merely the itemized facts listed, giving $36,233.77 of SALT rather than $26,460. Combined with adding the $1,632 refund to AGI, it taxed $145,467 instead of $134,061.22, and 24% of the $11,405.78 excess is the whole $2,737.39 error." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $41,500 corresponds to applying the 2026 brackets to roughly undeducted gross income near $204,000, taking neither the $16,100 standard deduction nor the $64,443.77 itemized total of mortgage interest and SALT. The correct base is $134,061.22 of taxable income, which yields $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied ""2026 post-TCJA-sunset brackets"" with a $5,427 personal exemption, but the 10/12/22/24 schedule is permanent for 2026 and personal exemptions are repealed. It also limited SALT to the $26,460 of real estate taxes, omitting the $9,773.77 Maryland state and county income tax, and added the $1,632 refund to AGI." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used ""restored 10/15/25/28 percent brackets"" and a $5,300 personal exemption, neither of which applies in 2026, where the schedule is 10%/12%/22%/24% at $12,400, $50,400, $105,700 and $201,775 with no personal exemption. It also set the Maryland income tax at $6,936 rather than the $9,773.77 state-plus-county liability and added the $1,632 refund to AGI." -us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Its itemized total of $64,556, including $9,886 of Maryland income tax, essentially reproduces the engine's $64,443.77, so its only error is computing tax ""using 2026 pre-TCJA brackets"" with a $5,434 personal exemption. Under the actual 2026 schedule and no personal exemption its own deduction figures give about $25,137, close to $24,772.69, instead of the $27,025 it reported." -us,scenario_078,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly applied the raised 2026 SALT cap but counted only the $26,460 of property taxes as SALT, omitting the $9,773.77 of Maryland state and county income tax that brings the engine's SALT deduction to $36,233.77, and it added the $1,632 refund to AGI. That gives $145,467 of taxable income against the correct $134,061.22, and its $27,490 is its own $27,510.08 rounded." -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for federal_income_tax_before_refundable_credits and no reasoning accompanied the submission, so nothing substantive was evaluated. The requested computation is $198,505 of AGI less $64,443.77 of itemized deductions, taxed under the 2026 single brackets for $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly identified the 2026 SALT cap of $40,400 but filled it with only the $26,460 of real estate taxes, never adding the $9,773.77 of Maryland state and county income tax that PolicyEngine deducts, and it added the $1,632 refund to AGI. Its $54,670 of itemized deductions falls $9,773.77 short of $64,443.77, and the resulting $145,467 taxable income produces $27,510.08 instead of $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted zero tax with no derivation for a single filer with $198,505 of wages, $64,443.77 of itemized deductions and $134,061.22 of taxable income, which the 2026 single brackets tax at $24,772.69. A zero liability would require taxable income at or below $0, which no reading of these facts supports." -us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It recognized the 2026 $40,000-class SALT cap but treated SALT as only the $26,460 of real estate taxes, leaving out the $9,773.77 Maryland state and county income tax that completes the engine's $36,233.77 SALT deduction, and it added the $1,632 refund to AGI. Taxing $145,467 rather than $134,061.22 at 24% on the difference is precisely the $2,737.31 gap between its $27,510 and $24,772.69." -us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000 on the reasoning that ""the SALT cap is $10,000 total,"" when for 2026 the cap is $40,400 and the filer's $36,233.77 of state, county and property taxes is fully deductible. It then explicitly computed with 2024 brackets ($11,600/$47,150/$100,525/$191,950) instead of the 2026 thresholds of $12,400/$50,400/$105,700/$201,775, compounding the deduction error." -us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It computed tax on the $16,100 standard deduction base and then manufactured an AMT liability using a fabricated ""$2,527.46 AMT foreign-tax-equivalent offset"" and a ""$1,127.51"" rounding add-on; with the 2026 AMT exemption of $90,100 for single filers, tentative minimum tax here is far below regular tax and AMT is zero. It also should have deducted the full $64,443.77 of mortgage interest and SALT directly, giving $134,061.22 of taxable income and $24,772.69 of tax." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax. Those taxes push SALT to the $40,400 cap, which lowers taxable income to $131,526.91. It then mis-added its own bracket sum: $1,240 + $4,560 + $12,166 + $9,544 is $27,510, but it wrote $33,510 and submitted $33,499." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT figure was real estate tax plus about $6,905 of Maryland state income tax, or $33,365, and it left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap. Its $61,575 of itemized deductions is short of the correct $68,610, so it taxed $138,562 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $2,606 of unreimbursed employee business expenses from AGI. Under the permanent TCJA/OBBBA rules these are suspended miscellaneous itemized deductions, not an above-the-line adjustment. It also limited SALT to real estate tax, dropping state and county income tax that brings SALT to the $40,400 cap. Finally, it used outdated 2024 figures (a $14,600 standard deduction) instead of the 2026 brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the OBBBA 2026 cap of $40,400, which cut $30,400 from itemized deductions and gave taxable income of $161,927 instead of $131,526.91. It also used $107,450 as the top of the 22% bracket; the 2026 figure is $105,700." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired $10,000 SALT cap instead of the 2026 $40,400 cap and computed $31,424. It then discarded that figure without explanation and submitted $36,757, which matches the tax under the $16,100 standard deduction ($36,766.86), not itemizing." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions by $30,400 and put taxable income at $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It set SALT at $33,396, which is real estate tax plus $6,936 of Maryland state income tax, and left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap. That makes itemized deductions $68,610 and taxable income $131,526.91, not $138,531." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the pre-OBBBA $10,000 SALT cap instead of $40,400, which gave taxable income of $161,927. It also made up inflation-adjusted brackets (for example, 22% up to $106,150) instead of using the 2026 schedule, where the 22% bracket ends at $105,700." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of $40,400 and left the $1,631.91 state tax refund out of income, arriving at about $31,069. It then raised the answer to $42,800 by citing Medicare surtax interactions, but payroll Medicare taxes are not part of federal income tax before refundable credits." +us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT was real estate tax plus about $7,000 of Maryland state income tax, and it left out Maryland county income tax. Including county tax takes SALT to the $40,400 cap, so itemized deductions are $68,610, not about $61,700." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the old $10,000 cap instead of the 2026 cap of $40,400, which gave taxable income of $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026, so it applied a $5,063 personal exemption, the pre-TCJA 10/15/25/28% brackets and an uncapped SALT deduction. OBBBA made the TCJA brackets permanent, kept the personal exemption at zero and set a $40,400 SALT cap." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law in 2026, with a $5,300 personal exemption and pre-TCJA single brackets. OBBBA made the TCJA brackets permanent with no personal exemption, and SALT is capped at $40,400, not uncapped." +us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap. As a result it taxed $145,467 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as a post-TCJA-sunset year and applied a $5,150 personal exemption, uncapped SALT and the 10/15/25/28% brackets. OBBBA kept the TCJA brackets, repealed the exemption and capped SALT at $40,400. It also left the $1,631.91 taxable state refund out of income." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing. Itemized deductions total $68,610 ($40,400 capped SALT plus $28,210 mortgage interest), far more than the standard deduction. It also left the $1,631.91 state refund out of income." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: restored personal exemptions, no SALT cap, and an AMT triggered by the uncapped SALT. OBBBA made the TCJA brackets and zero exemption permanent with a $40,400 SALT cap. Under those rules, regular tax on $131,526.91 is $24,164.46, and AMT does not apply." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset, deducting a $5,150 personal exemption and using pre-TCJA bracket rates. OBBBA made the TCJA single brackets permanent with no personal exemption. It also left out Maryland county income tax, which takes SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction instead of itemizing $68,610 ($40,400 capped SALT plus $28,210 mortgage interest). That left taxable income far above the correct $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,050 personal exemption, uncapped SALT and pre-TCJA brackets. OBBBA made the TCJA 10/12/22/24% brackets permanent and repealed the personal exemption. SALT is limited to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 matches about $161,900 of taxable income, which is what the old $10,000 SALT cap produces. The correct SALT deduction is the 2026 OBBBA cap of $40,400, which gives taxable income of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-expiration rules to 2026, even though OBBBA made the TCJA brackets and zero personal exemption permanent. It also itemized only mortgage interest and real estate tax, leaving out the Maryland state and county income tax that takes SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing the $68,610, and it left the state refund out of income. It then subtracted a made-up $12,000 child and dependent care credit, even though the household has no child and no care expenses." +us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the expired $10,000 limit instead of the 2026 cap of $40,400, which gave taxable income of $161,927 instead of $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It stated a $10,000 SALT cap instead of the 2026 cap of $40,400. Its $42,034 is higher than the tax even under the standard deduction ($36,766.86), so it also misapplied the 2026 single brackets." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It then overstated the tax on even that base, since 2026 brackets on $145,467 give $27,510.08." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT was only the $26,460 real estate tax, leaving out Maryland state and county income tax that bring SALT to the $40,400 cap. It then miscomputed the brackets: 2026 single rates on its own $145,467 base give $27,510.08, not $29,576.96." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 corresponds to taxable income of about $138,956, so its SALT was roughly $33,000: real estate tax plus Maryland state income tax only. It left out Maryland county income tax, which pushes SALT to the $40,400 cap and taxable income down to $131,526.91." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($54,670) and did not treat Maryland state and county income tax as deductible SALT. Those taxes bring SALT to the $40,400 cap and itemized deductions to $68,610." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It limited SALT to the $26,460 real estate tax and left out Maryland state and county income tax, which take SALT to the $40,400 cap. That overstated taxable income by $13,940." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It stated that no other deductible tax payments were reported. It overlooked that the household's own Maryland state and county income tax liability counts toward the SALT deduction. That liability brings SALT to the $40,400 cap, so itemized deductions are $68,610, not $54,670." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $26,460 of real estate tax as SALT and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap, so taxable income is $131,526.91, not $145,467." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added about $6,936 of Maryland state income tax to real estate tax but left out Maryland county income tax. With county tax included, SALT reaches the $40,400 cap, cutting taxable income to $131,526.91 from its $138,530.66." +us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It included estimated Maryland state income tax in SALT but left out Maryland county income tax, so its SALT stayed near $33,400. With county tax included, SALT hits the $40,400 cap and itemized deductions total $68,610." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no reasoning, and its $41,500 is higher than the tax under the standard deduction ($36,766.86). The correct calculation itemizes $68,610 (the $40,400 SALT cap plus $28,210 mortgage interest) and applies 2026 single brackets to $131,526.91 for $24,164.46. Its answer is consistent with skipping itemization and also overstating the bracket tax." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a $5,427 personal exemption and 2017 brackets indexed forward. OBBBA made the TCJA 10/12/22/24% brackets permanent with no exemption. It also limited SALT to real estate tax, leaving out the Maryland state and county income tax that takes SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It restored the 10/15/25/28% brackets and a $5,300 personal exemption as if TCJA had sunset. OBBBA made the TCJA brackets permanent with no personal exemption. It also left out Maryland county income tax, so its SALT stayed below the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It also used lower 10% and 12% bracket thresholds than the 2026 schedule ($12,400 and $50,400)." +us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax under pre-TCJA brackets with a $5,434 personal exemption and uncapped SALT. OBBBA made the TCJA single brackets permanent, eliminated the exemption and capped SALT at $40,400." +us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $26,460 real estate tax as the whole SALT deduction and left out Maryland state and county income tax. Those taxes bring SALT to the $40,400 cap, so taxable income is $131,526.91, not $145,467." +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits and no reasoning, so no substantive answer exists to score." +us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the correct $40,400 cap but counted only real estate tax as SALT, leaving out Maryland state and county income tax. With those included, SALT reaches the cap and itemized deductions are $68,610, not $54,670." +us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It submitted $0 with no derivation. A single filer with $200,136.91 of AGI and $131,526.91 of taxable income owes $24,164.46 under the 2026 brackets, and no nonrefundable credit offsets it." +us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It used the correct 2026 SALT cap but counted only the $26,460 real estate tax toward it. It left out Maryland state and county income tax, which take SALT to the $40,400 cap." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400, which gave taxable income of $161,927. It then applied 2024 brackets as a stand-in for 2026." +us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 32% rate below its 2026 threshold of $201,775. It then invented an AMT computation with no exemption and a made-up foreign-tax offset. With the $90,100 AMT exemption, tentative minimum tax is far below the regular tax of $24,164.46 on $131,526.91, so AMT does not apply." us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household." us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the correct answer twice in its own reasoning — $184,500 wage base x 6.2% = $11,439 plus $198,505 x 1.45% = $2,878.32, total $14,317.32 — and then submitted $12,240.44, a figure its stated arithmetic never produces. The error is purely in the transfer of its own computed total to the value field, not in the payroll tax rules." @@ -5735,177 +6280,198 @@ us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals us,scenario_078,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with the bare explanation 'value = 0' for a wage earner with $198,505 of covered wages, which carry mandatory 6.2% OASDI up to the $184,500 base and uncapped 1.45% Medicare. No exemption in the household facts removes FICA from these wages; the correct total is $11,439 + $2,878.32 = $14,317.32." us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model projected a 2026 wage base of about $174,900 by inflating the 2024 base of $168,600, skipping the actual 2025 base of $176,100 and the 2026 base of $184,500. That understated OASDI at $10,843.80 instead of $11,439, and the resulting $13,722.12 falls short by exactly that $595.20." us,scenario_078,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated Social Security tax of $10,453.20, which is 6.2% of the 2024 wage base of $168,600 rather than the 2026 base of $184,500, and its stated components sum to $13,331.52. It nonetheless submitted $15,622.81, a value $2,291.29 above its own arithmetic and above the correct $14,317.32." -us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It applied a ~$2,700 standard deduction and an $800 exemption instead of the $54,670 Maryland itemized deduction ($28,210 mortgage interest + $26,460 real estate taxes), which beats the $3,400 standard-deduction cap by more than 16x, leaving taxable income of $195,005 rather than $143,835. It also granted an $800 exemption that Maryland phases to $0 for single filers with FAGI above $150,000, then double-counted the deduction by subtracting it again from its own $9,735 bracket result to reach $9,327." -us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It executed the entire Maryland derivation correctly — refund subtraction to $198,505 AGI, $54,670 itemized deduction, and the 2%/3%/4%/4.75%/5%/5.25% schedule — but subtracted a $600 personal exemption, giving taxable income of $143,235 instead of $143,835. Maryland's exemption table sets the exemption to $0 for single filers with FAGI over $150,000, and the $600 overstatement at the 5.25% marginal rate is exactly the $31.50 shortfall between $6,904.84 and $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invoked the federal $10,000 SALT cap to disallow the property-tax deduction and took the $3,200 Maryland standard deduction, arriving at taxable income of $195,305; Maryland's itemized deduction is federal Schedule A less state and local income taxes, and the 2026 federal SALT cap is $40,000, so the full $26,460 of real estate taxes plus $28,210 of mortgage interest deduct for $54,670 and taxable income is $143,835. Its submitted $7,558 does not even follow from its own $195,305 base, which the bracket schedule maps to $9,712." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning reached the correct answer — $198,505 MD AGI, $54,670 itemized deduction, $0 exemption, taxable income $143,835, and bracket tax of $6,936.59 — and then discarded it in an unexplained ""recomputing more carefully"" step. The submitted $9,755 corresponds to applying the brackets to roughly $195,400 of taxable income, i.e. income reduced only by a standard deduction, contradicting the itemized computation it had just completed." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It asserted that Maryland itemized deductions are limited by the federal $10,000 SALT cap and guessed ""~$30,000"" of deductions, when Maryland only adds back state and local income taxes and the 2026 federal SALT cap of $40,000 leaves the full $26,460 of property taxes deductible, so the deduction is $54,670 and taxable income is $143,835, not $168,505. It then submitted $9,876 — a no-deduction figure — against its own computed $8,277.78." -us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It capped the real estate tax deduction at $10,000, reporting Maryland itemized deductions of ""~$38,210"" instead of the full $28,210 + $26,460 = $54,670 that Maryland allows after adding back only state and local income taxes, inflating taxable income to ~$161,900 from the correct $143,835. It compounded this by stating Maryland's rate tops out at 5.0% between $150k and $175k, when the applicable bracket for $143,835 is 5.25% on the $125,000–$150,000 band." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never considered itemizing, subtracting a $2,400 standard deduction and a $3,200 personal exemption from federal AGI of $200,137 to get taxable income of $194,537; the $54,670 Maryland itemized deduction applies, the exemption is $0 above $150,000 FAGI, and the $1,632 state tax refund is subtracted in reaching MD AGI, so taxable income is $143,835. Its bracket arithmetic on the wrong base was internally consistent, which is why $9,709.49 lands almost exactly $2,773 above the reference in the 5.5% band." -us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly identified that Maryland allows the uncapped $54,670 itemized deduction and derived taxable income of $143,835, then abandoned that base and submitted $9,200 on the claim that Maryland's schedule runs ""up to 4.75% at top plus scaled brackets, effective near 5.0%."" The actual bracket sum on $143,835 is $6,936.34; $9,200 corresponds to roughly $185,300 of taxable income, a figure its own reasoning never produced." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped property taxes at $10,000 for Maryland purposes (deduction $38,210 instead of $54,670), kept federal AGI at $200,137 without subtracting the $1,632 state tax refund, and applied a $3,200 personal exemption that Maryland zeroes out above $150,000 FAGI. Those three errors compound to taxable income of $158,727 versus the correct $143,835, and $7,739.99 versus $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took the standard-deduction path only — $200,137 less a $2,400 standard deduction and a $3,200 exemption — ignoring that $28,210 of mortgage interest plus $26,460 of fully deductible real estate taxes give a $54,670 Maryland itemized deduction. It also failed to subtract the $1,632 state tax refund and applied an exemption Maryland phases to $0 above $150,000, producing taxable income of $194,537 instead of $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It handled the refund subtraction and the $0 exemption correctly but applied a $10,027 itemized-deduction limitation, reporting ""itemized deduction after limitation"" of $44,642.60 against the allowable $54,670. Maryland's only reduction of itemized deductions for 2026 is 2% of federal AGI in excess of $200,000, and this filer's AGI of $198,505 is below that threshold, so no limitation applies; the phantom haircut pushed taxable income to $153,862.40 and the tax into the 5.5% band." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation beyond ""federal AGI with adjustments"" and progressive rates, and its $9,070 corresponds to applying the Maryland brackets to roughly $182,900 of taxable income. The correct base is $143,835 after the $54,670 itemized deduction for mortgage interest and real estate taxes, so the answer reflects taxing income essentially undiminished by any deduction." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It explicitly added ""typical county/local income tax rates"" to the state figure, when the requested output is state individual income tax excluding local income taxes. The $11,406 exceeds the correct $6,936.34 by $4,469.66, which is a 3.1% county rate on the $143,835 of Maryland taxable income." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It got the derivation right — gross income less the $54,670 of itemized deductions with the exemption phased to $0, taxable income $143,835 — and then mis-summed the bracket schedule, reporting $6,886 where $20 + $30 + $40 + 4.75%×$97,000 + 5%×$25,000 + 5.25%×$18,835 = $6,936.34. The $50.34 shortfall is a bracket-boundary arithmetic slip, not a rule error." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It named only ""standard deductions/exemptions and brackets,"" and $9,876 corresponds to Maryland brackets applied to roughly $197,500 of taxable income. Itemizing is mandatory here on the numbers — $28,210 of mortgage interest plus $26,460 of real estate taxes give $54,670 against a $3,400 standard-deduction cap — so the correct base is $143,835 and the tax stops in the 5.25% bracket rather than reaching the 5.5% band." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It computed the state tax exactly — $198,505 MD AGI less $54,670 itemized deductions, taxable income $143,835, tax $6,936.34 — and then added Maryland county income tax at 3.2% ($4,602.72) into the state total. The requested output is state individual income tax excluding local income and payroll taxes, so the county piece must not be included." -us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly applied the itemized deduction for mortgage interest and real estate taxes with a zero personal exemption, reaching the right $143,835 base, but summed the bracket schedule to $6,889 instead of $6,936.34. The $47.34 gap is an arithmetic slip in the 4.75%/5%/5.25% tiers, not a misapplied rule." -us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It kept Maryland AGI at the federal $200,137 rather than subtracting the $1,632 state and local tax refund that Maryland excludes, so its taxable income was $145,467 instead of $143,835. Even on that base the bracket schedule yields $7,022.02, so its $7,300.69 also overstates the bracket sum; the correct figure on $143,835 is $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It limited federal itemized deductions to $38,210 by applying a $10,000 SALT cap to the $26,460 of real estate taxes, then solved a circular equation subtracting its own Maryland tax from the deduction. Maryland's itemized deduction is federal Schedule A less state and local income taxes only, and the 2026 federal SALT cap is $40,000, so the deduction is the full $54,670 with no circularity; it also collapsed the rate schedule to ""effectively 4.75% above $5,000,"" ignoring the 5% and 5.25% brackets that actually apply at $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It described only wages less ""Maryland deductions and standard allowance,"" and $9,720 corresponds to brackets applied to roughly $194,700 of taxable income — income reduced by a standard deduction of a few thousand dollars. The $54,670 itemized deduction from $28,210 of mortgage interest and $26,460 of real estate taxes takes taxable income to $143,835 and the tax to $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed to account for itemized deductions, but $10,120 corresponds to Maryland brackets on roughly $202,000 of taxable income — more than the filer's entire $200,137 of federal AGI. The correct derivation subtracts $54,670 of Maryland itemized deductions from $198,505 of MD AGI for a $143,835 base and $6,936.34 of tax." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It cited ""allowable itemized deductions and the personal exemption,"" and $6,511.18 corresponds to taxable income of about $135,700 — roughly $8,100 below the correct $143,835. Maryland grants no personal exemption at $198,505 of FAGI and allows exactly $54,670 of itemized deductions, so nothing beyond that $54,670 comes out of the $198,505 base." -us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the correct $54,670 itemized deduction but started from federal AGI of $200,137 without Maryland's subtraction of the $1,632 state and local tax refund, leaving taxable income of about $145,500 rather than $143,835. Taxing that extra $1,632 at the 5.25% marginal rate accounts for essentially the entire $89 gap between $7,025 and $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It supplied no reasoning, and $9,200 corresponds to Maryland brackets on roughly $185,300 of taxable income, consistent with taxing wages reduced only by a small standard deduction. The correct derivation itemizes $28,210 of mortgage interest and $26,460 of real estate taxes for a $54,670 deduction, giving taxable income of $143,835 and tax of $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It elected the $2,400 Maryland standard deduction over the $54,670 itemized deduction and kept the $1,632 state tax refund in Maryland AGI, producing taxable income of $197,737 instead of $143,835. The itemized total exceeds the standard deduction by more than $52,000, so itemizing is mandatory on these facts, and the difference alone accounts for the $2,950 overstatement." -us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It cut the $54,670 itemized deduction by ""3% of AGI over $100,000"" to $51,666 and revived an 82%-phased $3,200 exemption worth $576. Maryland imposes no Pease-style 3% haircut — its only itemized reduction is 2% of federal AGI above $200,000, which this $198,505 AGI does not trigger — and the exemption is a flat $0 above $150,000 FAGI, not proportionally phased; it also used $200,137 without subtracting the $1,632 refund, leaving taxable income of $147,895 rather than $143,835." -us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for state_income_tax_before_refundable_credits and no reasoning accompanied the submission, so the key is absent from the outputs object rather than substantively wrong. The graded answer is $6,936.34, from $143,835 of Maryland taxable income after the $54,670 itemized deduction." -us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with the bare text ""value = 0,"" asserting no Maryland liability for a single filer with $198,505 of Maryland AGI and $143,835 of taxable income after the $54,670 itemized deduction. Zero is reachable only if Maryland taxable income were nil, which requires deductions exceeding wages by a factor of three; the bracket schedule on $143,835 yields $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a truncated Maryland rate schedule — 2%/3%/4% on the first $3,000 and a flat 4.75% on everything above — omitting the 5% bracket on $100,000–$125,000 and the 5.25% bracket on $125,000–$150,000 that apply to this taxable income. It also kept the $1,632 state tax refund in Maryland AGI, giving a $145,467 base; the two errors partly offset, leaving $6,857 against the correct $6,936.34." -us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It stated there were ""no listed deductions or adjustments specific to Maryland"" and ran the brackets on the full $198,505 of wages, ignoring the $28,210 of mortgage interest and $26,460 of real estate taxes that form a $54,670 Maryland itemized deduction. It then raised its own no-deduction result of $9,927.78 to $10,933, a figure implying about $216,800 of taxable income — more than the household's entire income." -us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started Maryland's computation from federal taxable income of $184,037 (federal AGI less the federal standard deduction) when Maryland starts from federal AGI and then applies its own $54,670 itemized deduction, giving $143,835. It also invented a bracket schedule with 4.75% on only $2,000, 5% on $95,000 and 5.25% thereafter, and subtracted a $736 Maryland nonrefundable credit for which no qualifying fact exists in the household." +us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took the ~$2,700 Maryland standard deduction and an $800 exemption instead of itemizing $54,670 of mortgage interest and property tax, which put about $195,000 into the 5.5% bracket. It then subtracted those same allowances again from its $9,735 bracket result to reach $9,327. The correct base is $198,505 minus $54,659.73 in itemized deductions with no exemption above $150,000 AGI." +us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the refund and itemized $54,670. It then applied a $600 personal exemption, but Maryland's single-filer exemption is $0 above $150,000 AGI. It also skipped the $10.27 phase-out (7.5% of federal AGI over $200,000), so its taxable income of $143,235 is $610.27 too low, a $32.04 shortfall at 5.25%." +us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It took a $3,200 Maryland standard deduction, leaving about $195,305 of taxable income, and never itemized the $54,670 of mortgage interest and property tax. Its $7,558 also does not match its own base: the Maryland brackets on $195,305 give about $9,776." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached $6,936.59 on taxable income of about $143,835 by itemizing correctly, then dropped that result for an unsupported $9,755. That figure matches taxing about $195,000, i.e. the standard-deduction path with no itemizing. It also skipped the $10.27 itemized phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It capped property tax at a $10,000 SALT limit, using about $30,000 of itemized deductions and $168,505 of taxable income. The full $26,460 is deductible because it sits under the 2026 $40,400 cap. It then submitted $9,876, which contradicts its own $8,277.78 bracket calculation." +us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $200,137 without subtracting the $1,632 state refund and itemized only about $38,210, capping property tax at $10,000. The full $26,460 property tax is deductible. It also used a wrong 5.0% rate for the $150,000-$175,000 range, which produced its $161,900 base and $7,800 tax." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from unadjusted federal AGI of $200,137, giving $194,537. It never itemized the $54,670 of mortgage interest and property tax, never subtracted the state refund, and gave an exemption that is $0 above $150,000 AGI." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly reached about $143,835 of Maryland taxable income after the $54,670 itemized deduction, but it never ran the brackets on that base. It guessed $9,200, an average rate of about 6.4%, even though the top marginal rate at this income is 5.25%. The brackets give about $6,936." +us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used Maryland AGI of $200,137 without subtracting the $1,632 state tax refund, which Maryland exempts, and skipped the $10.27 phase-out. That gave taxable income of $145,467 and tax of $7,022 instead of $143,845.27 and $6,936.88." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped property tax at $10,000 (itemized $38,210 instead of $54,670), skipped the refund subtraction, and took a $3,200 exemption that is $0 above $150,000 AGI. That inflated taxable income to $158,727. Its bracket math on that base is correct." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption from $200,137, giving $194,537. It never itemized the $54,670 of mortgage interest and property tax and never subtracted the state refund." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It got Maryland AGI ($198,505) and the $0 exemption right but cut itemized deductions to $44,642.60, a reduction of $10,027. Maryland's phase-out is 7.5% of federal AGI over $200,000, which removes only $10.27 here. The over-cut raised taxable income to $153,862 and the tax to $7,472." +us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It skipped the refund subtraction (used $200,137) and reduced itemized deductions by a federal-style 3% of AGI over $200,000 ($4.11) instead of Maryland's 7.5% ($10.27). It also took a $1,600 exemption that is $0 above $150,000 AGI, so taxable income came to $143,871.11 instead of $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $9,070 matches taxing about $183,000 at Maryland's 5.5% top bracket, i.e. income reduced only by a standard-type deduction. It never subtracted the $54,670 of mortgage interest and property tax that bring taxable income down to $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It says it included county/local income tax, which the requested output excludes. Its $11,406 is the $6,936.88 state tax plus roughly $4,470 of county tax at about 3.1% of taxable income." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached taxable income of $143,835 by itemizing with no exemption, but misapplied the brackets: 2%/3%/4%/4.75%/5%/5.25% on $143,835 gives $6,936.34, not $6,886. It also skipped the $10.27 phase-out, which raises taxable income to $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It says it used standard deductions and exemptions, and its $9,876 matches taxing about $197,500 in the 5.5% bracket. It never itemized the $54,670 of mortgage interest and property tax, which cut taxable income to $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly computed the state tax as $6,936.34 on $143,835, then added a 3.2% county tax of $4,602.72, which this output excludes. It also skipped the $10.27 itemized phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It took the right path (itemized mortgage interest and property tax, zero exemption) but its $6,889 is $47 below the bracket tax on that base ($6,936.34 on $143,835), a bracket arithmetic error. It also skipped the $10.27 phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It skipped the refund subtraction, so taxable income was $145,467. It then misapplied the brackets: 2%/3%/4%/4.75%/5%/5.25% on that base gives $7,022.02, not $7,300.69. Its claim that the SALT cap expires in 2026 is also wrong; the 2026 cap is $40,400." +us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used $38,210 of itemized deductions, capping property tax at $10,000, instead of the full $26,460. It then set up a circular add-back of state income tax that no one paid and taxed the result at a flat 4.75%, skipping the 5% and 5.25% brackets." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It cites a standard allowance, and its $9,720 matches taxing about $194,500 ($200,137 minus a $2,400 standard deduction and a $3,200 exemption). It never itemized the $54,670 of mortgage interest and property tax and never subtracted the state refund." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $10,120 is higher than the Maryland bracket tax on the full $200,137 with no deductions at all (about $10,018). So it applied neither the $54,659.73 itemized deduction nor the correct 2%-5.5% schedule." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $6,511.18 corresponds to taxable income of about $135,700, roughly $8,100 below the correct $143,845.27. It took deductions beyond the $54,659.73 of mortgage interest and property tax (after the phase-out), including a personal exemption that is $0 for a single filer above $150,000 AGI." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,025 matches taxing about $145,500, i.e. federal AGI of $200,137 minus $54,670 of itemized deductions. It skipped Maryland's subtraction of the $1,632 state tax refund and the $10.27 phase-out." +us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used taxable income of $145,467, starting from $200,137 without subtracting the $1,632 state tax refund that Maryland exempts. It also skipped the $10.27 phase-out, so its $7,022.02 is $85 too high." +us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. Its $9,200 matches taxing about $185,000 in the 5.5% bracket, roughly AGI minus a standard deduction. It never subtracted the $54,670 of mortgage interest and property tax, which bring taxable income to $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction from $200,137, giving $197,737 taxed up to 5.5%. It never itemized the $54,670 of mortgage interest and property tax and never subtracted the $1,632 state tax refund." +us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It cut itemized deductions by 3% of AGI over $100,000 ($3,004) instead of Maryland's 7.5% of federal AGI over $200,000 ($10.27). It also kept a $576 partial exemption that is $0 above $150,000 and skipped the refund subtraction, so taxable income came to $147,895 instead of $143,845.27." +us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so no answer was scored." +us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 with no reasoning, as if no Maryland income tax were owed. With $198,505 of wages and $143,845.27 of taxable income after itemizing, the Maryland brackets give $6,936.88." +us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It taxed everything above $3,000 at a flat 4.75%, skipping the 5% bracket above $100,000 and the 5.25% bracket above $125,000. It also left the $1,632 state refund in Maryland AGI, which gave taxable income of $145,467." +us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took only a ~$2,550 standard deduction and never itemized the $54,670 of mortgage interest and property tax. Its $10,933 is also higher than its own bracket computation on $195,955 (about $9,788) and even its $9,927.78 on full wages." +us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income of $184,037, which is after the federal standard deduction, instead of Maryland AGI of $198,505 less $54,659.73 of Maryland itemized deductions. It used a made-up schedule (5% on $95,000, then 5.25% on the rest) and subtracted a nonrefundable credit when none applies." us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_079,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model tested only Arizona's income-based adult (MAGI) Medicaid rule and declared the head failed it, never applying the SSI-recipient category: the head's blindness, disability, and $7,560 of SSDI produce $5,286 of SSI, and Arizona is a 1634 state where SSI approval automatically confers AHCCCS/Medicaid with no separate income or resource test. Its own stated route also cannot yield 'No' — countable income at 0.35 x FPL sits far below Arizona's 138% FPL expansion limit — so the model both skipped the operative categorical pathway and misapplied the fallback one it invoked." us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed pathway or coverage input and returned 0 for lack of one, but SSI receipt is computed from the facts given (age 58, blind, disabled, $7,560 SSDI, $1,045 in resources, no other income), and that SSI receipt is itself the categorical Medicaid pathway in Arizona, which auto-enrolls SSI recipients. It compounded this by reading Indian Health Service coverage at interview as the head's coverage source; IHS is not minimum essential coverage and never displaces or disqualifies Medicaid eligibility." -us,scenario_079,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model applied the SSDI-to-Medicare disability pathway, but PolicyEngine's Medicare eligibility variable keys solely on age >= 65 and Head is 58, so the flag is False. It then manufactured the missing qualifying period by treating the prompt's ""status constant throughout the tax-benefit year"" instruction as proof that the 24-month SSDI entitlement period had already elapsed, when months of prior SSDI receipt are an unlisted numeric input the prompt directs to be taken as 0." -us,scenario_079,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated bare receipt of Social Security disability income as conferring Medicare entitlement, dropping the 24-month prior-entitlement requirement entirely. PolicyEngine determines Medicare eligibility from age >= 65 alone, and Head is 58, so the correct value is 0." -us,scenario_079,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model correctly recited that Medicare disability entitlement requires 24 months of SSDI receipt, then supplied that duration itself from ""assuming take-up and constant status,"" although the household lists no months of prior SSDI receipt and the prompt sets unlisted numeric inputs to 0. PolicyEngine's Medicare eligibility test is age >= 65, which Head fails at 58." -us,scenario_079,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model acknowledged the age-65 threshold and the ""typically after the waiting period"" qualifier, then discarded both and resolved eligibility from the disability flag plus SSDI receipt. PolicyEngine's Medicare eligibility depends on age >= 65 only; at 58, Head is not eligible, and the 24-month SSDI entitlement period is an unlisted fact that resolves to zero months." -us,scenario_079,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model asserted a one-step SSDI-implies-Medicare rule with no age test and no 24-month qualifying period. PolicyEngine computes Medicare eligibility from age >= 65, so Head at 58 returns False." -us,scenario_079,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model stated as fact that Head ""received SSDI for at least 24 months"" when the household lists only an annual SSDI amount and no receipt duration, which the prompt's unlisted-input rule fixes at 0. It also treated the standalone ""is disabled"" flag as an independent Medicare pathway; Medicare disability entitlement flows only from 24 months of SSDI entitlement, and PolicyEngine's variable turns on age >= 65, which Head at 58 fails." -us,scenario_079,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model invoked ""disability-based eligibility"" from the disability flag and SSDI receipt, skipping the 24-month prior-entitlement condition that the household facts never establish. PolicyEngine resolves Medicare eligibility on the age >= 65 test alone, and Head is 58." -us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model explicitly assumed Head ""met the standard 24-month waiting period,"" importing an unlisted duration that the prompt instructs to treat as 0. Without that assumption the disability pathway never opens, and PolicyEngine's Medicare eligibility test is age >= 65, which Head fails at 58." -us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare eligibility in a single step, omitting both the 24-month entitlement period and the age test. PolicyEngine's Medicare eligibility flag is driven by age >= 65, so Head at 58 is not eligible." -us,scenario_079,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model derived eligibility purely from the presence of Social Security disability income, applying no age threshold and no 24-month qualifying period. PolicyEngine returns False because Head is 58 and its Medicare eligibility rule requires age >= 65." -us,scenario_079,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model combined the ""is disabled"" boolean with SSDI receipt to declare Medicare eligibility, treating a disability flag as a Medicare pathway. Medicare disability entitlement requires 24 months of SSDI entitlement, a duration absent from the facts and therefore 0, and PolicyEngine's eligibility test is age >= 65, which Head fails at 58." -us,scenario_079,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model attributed to PolicyEngine a disability-based Medicare rule that the engine does not implement; its Medicare eligibility variable resolves on age >= 65. Head is 58, so the value is 0 regardless of the disability flag or SSDI receipt." -us,scenario_079,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model made the standalone ""is disabled"" flag sufficient for Medicare, citing no SSDI entitlement duration at all. Medicare's under-65 pathway runs through 24 months of SSDI entitlement, not a disability status flag, and PolicyEngine keys the variable on age >= 65, which Head at 58 fails." -us,scenario_079,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model asserted that disability plus SSDI receipt makes Head Medicare eligible ""under PolicyEngine rules,"" but PolicyEngine's Medicare eligibility test is the age >= 65 threshold with no disability branch. At age 58 the engine returns False." -us,scenario_079,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as automatically conferring Medicare eligibility on disability grounds, omitting the 24-month entitlement requirement that the facts do not supply. PolicyEngine's Medicare eligibility depends on age >= 65, and Head is 58." -us,scenario_079,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model invoked ""the model's disability-based Medicare eligibility rule,"" a rule PolicyEngine does not contain; its Medicare eligibility variable is the age >= 65 test. Head is 58, so the engine returns 0." -us,scenario_079,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model granted Medicare on SSDI receipt ""after the standard 24-month waiting period,"" supplying that elapsed period itself when the household states no prior receipt duration, which the prompt fixes at 0. PolicyEngine's Medicare eligibility flag requires age >= 65, and Head is 58." -us,scenario_079,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,The model claimed PolicyEngine makes a disabled SSDI recipient Medicare eligible under 65; PolicyEngine's Medicare eligibility variable applies only the age >= 65 threshold. Head at 58 therefore returns False. -us,scenario_079,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model asserted that Head had completed the 24-month disability entitlement period, a duration the household facts never list and the prompt sets to 0. PolicyEngine resolves Medicare eligibility from age >= 65, so Head at 58 is not eligible." -us,scenario_079,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model flagged its own gap by writing that 24 months of SSDI receipt was ""(assumed),"" then submitted 1 on that self-declared assumption; prior months of receipt are an unlisted input the prompt sets to 0. PolicyEngine's Medicare eligibility turns on age >= 65, which Head fails at 58." -us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read the instruction to hold facts constant across the tax-benefit year as establishing 24 months of prior SSDI entitlement, which it is not; the qualifying-period length is unlisted and therefore 0. PolicyEngine determines Medicare eligibility from the age >= 65 threshold, and Head is 58, giving False." +us,scenario_079,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model named the SSDI waiting period but treated 'constant for the full year' as proof that it was met. Constant receipt through 2026 adds up to only 12 months. The months-of-receipt input is unlisted and so defaults to 0, which means the 24-month requirement fails and age 58 is below the 65 threshold." +us,scenario_079,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model said SSDI receipt alone gives Medicare and skipped the 24-month entitlement requirement. No months of prior receipt are listed, so the count defaults to 0 and Head (58) qualifies by neither disability nor age." +us,scenario_079,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model stated the 24-month SSDI rule correctly but then assumed Head had met it, even though no listed fact supports that. The prompt's defaults set the unlisted SSDI duration to 0, so the disability route fails and Head at 58 is under the age-65 route." +us,scenario_079,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model noted that SSDI Medicare 'typically' comes after a waiting period but granted eligibility without confirming that period. SSDI months are unlisted and default to 0, so the 24-month requirement is not met and Head is ineligible." +us,scenario_079,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt as enough for disability-based Medicare and never applied the 24-month entitlement requirement. With 0 months of receipt on record and Head aged 58, Head qualifies by neither disability nor age." +us,scenario_079,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model cited the 'at least 24 months' SSDI condition but concluded Head met it from the annual SSDI amount and the disability flag, neither of which shows how long Head has received benefits. The unlisted duration defaults to 0, so the condition fails." +us,scenario_079,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model counted being disabled plus current SSDI receipt as disability-based Medicare eligibility, leaving out the required 24 months of SSDI entitlement. No months of receipt are listed (so 0), and age 58 is below 65." +us,scenario_079,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model assumed Medicare eligibility from being disabled and receiving SSDI without testing the 24-month SSDI duration condition. That duration input is unlisted and defaults to 0, so Head does not qualify." +us,scenario_079,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model explicitly assumed Head had completed the 24-month waiting period, which is an unlisted fact the prompt says to treat as 0. Once that assumption is removed, the disability route fails and Head (58) is under the age-65 threshold." +us,scenario_079,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model equated SSDI receipt with Medicare eligibility and skipped the 24-month entitlement requirement. SSDI months default to 0 and Head is 58, so neither route applies." +us,scenario_079,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model gave Medicare purely because Head receives SSDI and never applied the 24-month SSDI waiting period. With no months of receipt listed (defaulting to 0) and Head aged 58, Head is ineligible." +us,scenario_079,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model treated the disability flag plus SSDI income as qualifying on their own, leaving out the requirement of 24 months of SSDI entitlement. The unlisted months input is 0, so the disability route fails." +us,scenario_079,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model said PolicyEngine grants Medicare to any disabled SSDI recipient under 65, but PolicyEngine's disability route also requires at least 24 months of SSDI receipt. That months input defaults to 0 here, so Head is ineligible." +us,scenario_079,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model based eligibility on Head being 58 and disabled, as if the disability flag alone gave Medicare. The rule requires 24 months of SSDI entitlement, and Head's months of receipt are unlisted and default to 0." +us,scenario_079,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt 'for the year' as meeting PolicyEngine's disability route, but that route requires at least 24 months of SSDI receipt. A full year gives only 12 months, and prior months default to 0." +us,scenario_079,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model concluded that receiving SSDI makes Head Medicare-eligible on the basis of disability, without applying the 24-month SSDI entitlement requirement. Head's months of receipt default to 0, so the requirement is not met." +us,scenario_079,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model cited a 'disability-based Medicare eligibility rule' but left out that rule's condition of 24 months of SSDI receipt. That condition fails because Head's months of receipt are unlisted and default to 0, and Head (58) is under 65." +us,scenario_079,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model treated being disabled and receiving SSDI as enough for disability-based Medicare, skipping the required 24-month entitlement period. With SSDI duration defaulting to 0 and Head aged 58, Head is ineligible." +us,scenario_079,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model applied PolicyEngine's disability-based Medicare route without its condition of at least 24 months of SSDI receipt. The months input is unlisted and defaults to 0, so the route fails, and age 58 fails the age-65 test." +us,scenario_079,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model took for granted that Head had passed the standard 24-month waiting period, but no months of SSDI receipt are listed. Under the prompt's defaults the duration is 0, so the waiting period is not satisfied." +us,scenario_079,head_medicare_eligible,kimi-k2.6,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine makes any disabled SSDI recipient under 65 Medicare-eligible, leaving out the 24-month SSDI receipt requirement. With 0 months of receipt, Head does not qualify." +us,scenario_079,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model stated the 24-month entitlement rule and then applied it as if it were already met. Nothing in the facts establishes 24 months of receipt, and the unlisted duration defaults to 0, so Head is ineligible." +us,scenario_079,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model explicitly assumed 'at least 24 months' of SSDI receipt, which the prompt forbids because unlisted numeric inputs are 0. Without that assumption, Head (58) meets neither the disability route nor the age-65 route." +us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model read the 'constant throughout the year' instruction as satisfying the 24-month SSDI requirement, but constant receipt within 2026 gives only 12 months. Prior months are unlisted and default to 0, so Head does not qualify." us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Treated $630/month of SSDI as the household's entire SNAP gross income and never computed the $881/month of SSI the couple qualifies for (2026 couple FBR less $610 of countable SSDI), so it worked from $630 instead of $1,511 of monthly gross income. It also used the older $536 two-person maximum instead of the $546/$558.24 allotments in force during tax year 2026, and its submitted $4,560 abandons its own stated arithmetic ($536 − $118.50 = $417.50/month = $5,010) with an unexplained downward revision." -us,scenario_079,snap,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Built the correct $1,511 monthly gross ($630 SSDI + $881 SSI) but allowed only $235 of monthly deductions ($209 standard + $26 excess medical) against the $354 PolicyEngine applies, leaving net income of $1,276 instead of $1,157 and cutting roughly $36/month from the allotment. It also held the maximum allotment flat at $546 for all twelve months, missing the October 2026 uprating to $558.24 that raises the last three months to $212.64." -us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"Denied eligibility outright by counting home equity behind the $455,000 mortgage and $9,700 of vehicle equity as countable resources; SNAP excludes the household's own home entirely regardless of equity, and PolicyEngine counts only the $1,045.29 bank balance, well under the limit. It also missed that SSI receipt plus TANF non-cash eligibility make this household categorically eligible, which waives the resource and gross-income tests, and that the correct benefit is $546 − 30% of $1,157 net income = $198.90/month." -us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Omitted the $881/month SSI payable to this blind/disabled couple and computed from $630/month of SSDI alone, so its net income of $385 sits far below the reference's $1,157. It compounded this by revising its own computed $420.50/month down to $351.50 with no rule cited, and by using a $536 maximum allotment rather than the $546 and $558.24 that apply across tax year 2026." -us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Used only $630/month of SSDI as gross income, never computing the $881/month SSI that PolicyEngine pays and counts as SNAP unearned income, so gross was $1,511 not $630. It then dropped net income from its own $413 figure to 'near $300' by double-applying the medical expenses, whereas the actual deduction total is $354/month against $1,511, giving net income of $1,157 and a $198.90 monthly allotment." -us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630/month SSDI and omitted the $881/month SSI that is countable SNAP unearned income, producing net income near $400 instead of $1,157. Its submitted $283/month is not even consistent with its own stated inputs ($536 − 0.30 × $400 = $416), and it used a $536 allotment rather than the $546/$558.24 in effect during 2026." -us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Ran an otherwise clean derivation on the wrong income base: it declared the household's only income to be $7,560 of SSDI and never computed the $881/month of SSI these two blind and disabled adults receive, so its $400 net income understates the reference's $1,157 by $757. It also used the FY2025 $535 maximum allotment instead of $546 rising to $558.24 in October 2026, and $230 of deductions instead of $354." -us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Inferred an uncapped excess shelter deduction large enough to zero out net income from the $455,000 mortgage balance alone; SNAP's shelter deduction runs on actual monthly housing payments, and none are listed, so PolicyEngine allows no shelter deduction and the deduction total is $354/month. It also omitted the $881/month SSI from countable income and used a $374 maximum allotment instead of $546/$558.24, so it paid a fabricated 'maximum' rather than $546 − $347.10 = $198.90." -us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Correctly counted SSI as SNAP unearned income but set it at $876/month instead of $881 and allowed only $254 of deductions ($228 standard + $26 medical) against PolicyEngine's $354, leaving net income of $1,252 rather than $1,157. It also applied a flat $549 allotment for all twelve months instead of $546 for January–September and $558.24 from October 2026." -us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Failed the household on the resource test by counting $5,050 of vehicle equity after a $4,650 exclusion; PolicyEngine excludes the household vehicle here and counts only the $1,045.29 bank balance, and SSI receipt plus TANF non-cash eligibility confer categorical eligibility that waives the asset limit altogether. The household therefore qualifies and receives $546 − 30% of $1,157 net income = $198.90/month, rising to $212.64 in the last quarter." -us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Set SSI at $10,464/year ($872/month) rather than the $881/month PolicyEngine pays, and allowed only $228 of monthly deductions ($202 standard + $26 medical) against the $354 applied, leaving net income of $1,274 instead of $1,157. It also used a single $548 allotment for the whole year, missing both the $546 FY2026 figure and the October 2026 step to $558.24." -us,scenario_079,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Computed SSI at $805/month instead of $881 (the 2026 couple FBR less $610 of countable SSDI after the $20 exclusion) and allowed only $224 of deductions against PolicyEngine's $354, so its $1,211 net income and the reference's $1,157 diverge in offsetting ways. It then used a $535 maximum allotment rather than $546 for January–September and $558.24 thereafter, landing at $171.70/month instead of $198.90/$212.64." -us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"Asserted a zero benefit because no rent or utility costs are reported, treating shelter expenses as a precondition for any allotment. The SNAP benefit is the maximum allotment minus 30% of net income and requires no shelter costs at all: $546 − 30% × $1,157 = $198.90/month for this household, with no excess shelter deduction applied." -us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Claimed 30% of net income exceeds the maximum allotment and fell back to the $23/month minimum allotment; with $1,511 gross less $354 of deductions, net income is $1,157 and 30% of it is $347.10, comfortably below the $546 two-person maximum. The correct allotment is $198.90/month, roughly nine times the minimum-benefit figure it submitted." -us,scenario_079,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Allowed only $2,712 of annual deductions ($2,400 standard + $312 medical) against the roughly $4,248 PolicyEngine applies ($354/month), inflating net income to $15,360 versus $13,884. It also used a $6,720 annual maximum ($560/month) instead of $546 rising to $558.24 in October 2026, so its $2,112 mixes an overstated allotment with an overstated income contribution." -us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Asserted disqualification on unnamed asset and income rules with no derivation; countable resources are $1,045.29, gross income is 86% of the federal poverty guideline and net income 66%, and SSI receipt with TANF non-cash eligibility makes the household categorically eligible. The household qualifies and receives $198.90/month rising to $212.64 in October 2026." -us,scenario_079,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Got gross income roughly right at $18,096/year (SSDI plus SSI) but its $2,136 answer implies about $281 of monthly deductions and a single flat allotment, whereas PolicyEngine deducts $354/month for net income of $1,157. Applying the correct deductions and the October 2026 allotment step yields $198.90 for nine months and $212.64 for three, not a flat $178." -us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $5,640 equals $470/month, which implies net income of about $253 — a figure reachable only from the $630 SSDI alone, meaning it omitted the $881/month of SSI that PolicyEngine pays and counts as SNAP unearned income. With gross income of $1,511 and $354 of deductions, net income is $1,157 and the allotment is $198.90/month." -us,scenario_079,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Set SSI at $10,080/year ($840/month) instead of $10,572 ($881/month), and computed the medical deduction as $432 minus the $420 annual floor = $12, discarding the $300 of over-the-counter expenses that belong in the elderly/disabled medical total of $732. It also used a $6,432 annual maximum ($536/month) instead of $546 rising to $558.24, so its $1,878 falls short of $2,428.02." -us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for snap, so nothing substantive was scored. The required derivation is $1,511 gross less $354 of deductions = $1,157 net, giving $546 − $347.10 = $198.90/month for nine months and $558.24 − $345.60 = $212.64 for three, totaling $2,428.02." -us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"Declared gross annual income to be the $7,560 of SSDI alone, omitting the $10,572/year of SSI these two blind and disabled adults receive and which counts in full as SNAP unearned income. Its $400/month net income should be $1,157, so its $431/month allotment overstates the $198.90 the correct formula produces." -us,scenario_079,snap,gpt-5.4-mini,llm_error,other,False,"Concluded a zero benefit on the grounds that no shelter costs or dependents are listed, treating those as prerequisites for an allotment. SNAP pays the maximum allotment less 30% of net income regardless of shelter costs or children: with $1,511 of gross income (SSDI plus $881 of SSI) and $354 of deductions, the household receives $198.90/month." -us,scenario_079,snap,gpt-5.4-nano,llm_error,other,False,"Declined to compute, asserting the facts are insufficient for a SNAP calculation, and submitted zero. Every input needed is present: $630/month SSDI generates $881/month of SSI, gross income is $1,511, deductions are $354, net income is $1,157, and the allotment is $546 − $347.10 = $198.90/month." -us,scenario_079,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Reached the exact reference gross income of $1,511/month ($630 SSDI + $881 SSI) but allowed only $230 of deductions ($204 standard + $26 excess medical) instead of the $354 PolicyEngine applies, so its net income of $1,281 exceeds the correct $1,157 and its $161/month falls $38 short. It also used $546 for all twelve months, missing the October 2026 rise to $558.24 that lifts the final quarter to $212.64." -us,scenario_079,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Paid the full $546/month maximum allotment for twelve months, skipping the expected-contribution step entirely. A household with $1,157 of net income contributes 30% of it — $347.10 — so the allotment is $198.90/month, not the maximum; only a household with essentially zero net income receives the full $546." -us,scenario_079,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $163/month is the result of applying only the standard deduction plus a $26 excess-medical figure (about $230) to $1,511 of gross income, where PolicyEngine deducts $354 and reaches net income of $1,157. Correcting the deduction total and adding the October 2026 allotment increase to $558.24 gives $198.90 for nine months and $212.64 for three." -us,scenario_079,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $165/month allotment corresponds to roughly $230 of monthly deductions against $1,511 of gross income, understating PolicyEngine's $354 total and leaving net income near $1,270 instead of $1,157. It also applied one flat allotment across the tax year rather than $546 through September and $558.24 from October 2026." -us,scenario_079,snap,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Wrote the formula explicitly as 12 × [546 − 0.30 × (1511 − 209 − 26)], getting gross income exactly right but allowing only $235 of deductions where PolicyEngine allows $354, so net income came out $1,276 instead of $1,157. Holding the allotment at $546 for all twelve months also missed the October 2026 uprating to $558.24." -us,scenario_079,snap,grok-4.3,llm_error,categorical_eligibility,False,"Zeroed the household on the assertion that SSDI-based income exceeds SNAP limits after deductions for a two-person unit. Gross income of $1,511 is 86% of the federal poverty guideline and net income of $1,157 is 66%, both far under the applicable tests, and SSI receipt with TANF non-cash eligibility confers categorical eligibility; the benefit is $198.90/month rising to $212.64." -us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Used $7,560 of SSDI as the household's entire unearned income and never added the $10,572/year of SSI PolicyEngine pays and counts for SNAP, so its net income of $4,728 should be $13,884. It also used a ~$6,600 annual maximum instead of $546/month rising to $558.24, producing $5,182 where the formula gives $2,428.02." -us,scenario_079,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"Counted SSI but at about $876/month instead of $881 and allowed only $235 of deductions ($209 standard + $26 medical) against PolicyEngine's $354, leaving net income of $1,271 rather than $1,157. Its flat $549 allotment also replaces the $546 that applies through September 2026 and the $558.24 that applies from October." -us,scenario_079,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Priced SSI off the 2025 couple FBR of $1,450, producing $840/month instead of the $881 the 2026 couple rate yields after subtracting $610 of countable SSDI, and allowed only $230 of deductions against PolicyEngine's $354. Net income therefore came out $1,240 rather than $1,157, and the $536 allotment it used is neither the $546 nor the $558.24 in force during tax year 2026." -us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"Built its entire calculation on $630/month of gross SSDI and omitted the $881/month of SSI that these two blind and disabled adults receive and that counts as SNAP unearned income, so net income came to $395 instead of $1,157. Its $429/month allotment therefore more than doubles the correct $198.90, and the $548 maximum it used replaces the $546/$558.24 pair." -us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no snap value and no explanation, so no substantive reasoning was submitted. The correct derivation is gross income $1,511 less $354 of deductions = $1,157 net, giving allotments of $198.90 for nine months and $212.64 for three, or $2,428.02 for tax year 2026." -us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"Took monthly gross income to be the $630 of SSDI alone and never computed the $881/month of SSI that PolicyEngine pays this couple and counts in full for SNAP, so its $395 net income falls $762 short of the correct $1,157. Its $427.50/month allotment consequently overstates the $198.90 that $546 minus 30% of net income produces." -us,scenario_079,snap,minimax-m3,llm_error,other,False,"Acknowledged that gross income falls below the two-person SNAP thresholds yet submitted zero, reading the instruction not to infer unlisted benefit receipt as a bar on computing a requested benefit output. The prompt directs assuming take-up, and PolicyEngine's calculation gives $546 − 30% × $1,157 = $198.90/month, rising to $212.64 from October 2026." -us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"Passed the household on the gross-income test using $630/month of SSDI and then used that same $630 as the base for net income, omitting the $881/month of SSI that PolicyEngine pays and counts as SNAP unearned income. Correct net income is $1,157, not $399, so the allotment is $198.90/month rather than the $427 it computed." -us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Omitted the $10,572/year of SSI from countable income and applied the $35 medical threshold annually instead of monthly, deducting $697 of medical expenses where the monthly $35 floor allows far less against the $732 total. Combined with a $598/month maximum allotment in place of $546/$558.24, this produced $5,830 instead of the $2,428.02 that $1,157 of net income yields." -us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,Asserted that SSI receipt bars SNAP for all household members and submitted zero. SSI receipt is a pathway to SNAP categorical eligibility rather than a disqualification — it is exactly what waives the asset and gross-income tests here — and Arizona operates no SSI cash-out; the household receives $198.90/month through September 2026 and $212.64 thereafter. +us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted only the head's $630/month SSDI and left out the $881/month of SSI that the blind/disabled couple receives, which is countable unearned income for SNAP. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Its final $380/month doesn't even match its own $417.50 arithmetic; with $1,511 gross and $1,157 net income, the benefit is $546 − $348 = $198/month." +us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI, the $209 standard deduction, and the $546 maximum allotment. But it deducted only the $26 actual medical excess ($61 − $35) instead of Arizona's $145 standard medical deduction for elderly/disabled households with over $35 of medical costs. That left net income at $1,276 instead of $1,157 and cut the benefit to $163 instead of $198/month." +us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It ruled the household ineligible on resources by counting home and vehicle equity. The home is always an excluded resource and Arizona SNAP does not count vehicles, so countable assets are only the $1,045 bank balance, which passes. It never counted the $881/month SSI or computed the benefit, which is $198/month on $1,157 of net income." +us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $630/month SSDI as all of the household's income and left out the couple's $881/month SSI. It also used a $219 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction. Its final $351.50/month is an unexplained adjustment away from its own $420.50 arithmetic; correct inputs give $1,157 net income and $198/month." +us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted only the $630/month SSDI and left out the $881/month SSI, then guessed net income 'near $300' instead of applying the $209 standard deduction and Arizona's $145 standard medical deduction to $1,511 of gross income. Its $303/month benefit is about $105 above the correct $198." +us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the couple's $881/month SSI, so gross income was $630 instead of $1,511, and it did not apply Arizona's $145 standard medical deduction. Its $283/month answer doesn't follow from its own $536 − 30% × ~$400 arithmetic. Correct net income is $1,157, giving $198/month." +us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI and used the $209 standard deduction and $546 maximum. But it deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. That overstated net income by $119 ($1,276 vs $1,157) and understated the benefit at $163 instead of $198/month." +us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It acknowledged that the members receive SSI/SSDI, but then counted only the $630/month SSDI and left out the $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, plus outdated $204 standard-deduction and $535 maximum-allotment figures. Correct inputs give $1,157 net income and $198/month, not $415." +us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left out the $881/month SSI and inferred large shelter costs from the $455,000 mortgage balance, even though no mortgage payment, interest, or housing cost was listed. It then used $374 as the two-person maximum instead of $546. With no shelter deduction and Arizona's $145 standard medical deduction, net income is $1,157 and the benefit is $198/month." +us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It treated the $630/month SSDI as the only income and left out the couple's $881/month SSI, which is countable unearned income for SNAP. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. With $1,511 gross and $1,157 net income, the benefit is $198/month, not $427.50." +us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $228 standard deduction (the correct figure is $209), a $549 maximum (correct is $546), and SSI of $876 (correct is $881). It also skipped rounding the $375.60 contribution up to a whole dollar. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month." +us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"It applied the federal $4,650 vehicle fair-market-value exclusion and counted $5,050 of vehicle equity. Arizona SNAP does not count vehicles, so countable resources are only the $1,045 bank balance and the household passes the asset test. With SSI counted and Arizona's $145 standard medical deduction, the benefit is $198/month." +us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It counted SSI (at $872 instead of $881) but deducted only the $26 actual medical excess ($732/yr − $420/yr) instead of Arizona's $145 standard medical deduction. It also used a $202 standard deduction and a $548 maximum instead of $209 and $546. Correct net income is $1,157, giving $198/month, not $165.80." +us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed net income from the $630 SSDI alone and left out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct gross income is $1,511 and net income $1,157, so the benefit is $198/month, not $427." +us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"It counted SSI, though at $805 instead of $881/month, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $198 standard deduction and a $535 maximum instead of $209 and $546. Correct net income is $1,157 and the benefit is $198/month." +us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It concluded SNAP is zero because no rent or utilities were reported. Shelter costs only drive the optional excess shelter deduction; a household with no housing costs still gets the maximum allotment minus 30% of net income. Net income of $1,157 gives $546 − $348 = $198/month." +us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It claimed that 30% of net income exceeds the maximum allotment and paid only a $23/month minimum benefit. After the $209 standard deduction and Arizona's $145 standard medical deduction, net income is $1,157, and 30% of that ($348) is well below the $546 maximum. The benefit is therefore $198/month, not the minimum." +us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"It included SSI but deducted only the $312/year actual medical excess instead of Arizona's $1,740/year ($145/month) standard medical deduction. It also used a $2,400 standard deduction and a $6,720 maximum instead of $2,508 and $6,552. Correct annual net income is $13,884, giving $2,376." +us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"It asserted asset and income disqualification without computing either. Countable assets are only the $1,045 bank balance because the home and vehicles are excluded, and net income of $1,157 is below the $1,762.50 poverty line. The household qualifies for $198/month." +us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"It counted SSDI plus SSI ($18,096/yr) but, per its own description, used the actual medical excess ($26/month) instead of Arizona's $145 standard medical deduction for elderly/disabled households. That overstated net income by about $119/month and gave $178 instead of $198/month." +us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $470/month answer implies net income of about $250, which is only reachable by counting the $630 SSDI alone and leaving out the couple's $881/month SSI. Counting $1,511 of gross income, the $209 standard deduction, and the $145 Arizona medical deduction gives $1,157 net income and $198/month." +us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted SSI, though at the 2025-based $840/month instead of $881. But it deducted only $12/year of medical costs by counting just the $432 of other medical expenses (dropping the $300 of over-the-counter costs), instead of Arizona's $145/month standard medical deduction. It also used $204 and $536 instead of the $209 standard deduction and $546 maximum." +us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"It returned no SNAP value or explanation, so there was no substantive answer to score. The correct derivation is $1,511 gross income minus $209 standard and $145 Arizona medical deductions, giving $1,157 net and $198/month ($2,376/year)." +us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed net income from the $7,560 of SSDI alone and left out the couple's $10,572/year of SSI. It also used the $312/year actual medical excess instead of Arizona's $1,740/year standard medical deduction, and a $6,612 maximum instead of $6,552. Correct annual net income is $13,884, so SNAP is $2,376, not $5,172." +us,scenario_079,snap,gpt-5.4-mini,llm_error,other,False,"It set SNAP to zero because no dependents or shelter costs were listed. SNAP doesn't require either: a two-person disabled household gets the $546 maximum minus 30% of net income. With $1,157 of net income, the benefit is $198/month." +us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned zero on the claim that the listed facts don't support a SNAP computation. The facts fully determine the benefit: $630 SSDI plus $881 SSI, minus the $209 standard deduction and the $145 Arizona standard medical deduction, gives $1,157 net income and $198/month." +us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI and used the $546 maximum, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction instead of $209. That gave net income of $1,281 instead of $1,157 and $161 instead of $198/month." +us,scenario_079,snap,gpt-5.6-luna,llm_error,other,False,"It paid the full $546 maximum allotment every month and never subtracted the expected contribution of 30% of net income. With $1,511 of gross income and $1,157 of net income, that contribution is $348, leaving $198/month." +us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163/month answer matches $1,511 gross income minus the $209 standard deduction and only the $26 actual medical excess. It missed Arizona's $145 standard medical deduction, which lowers net income to $1,157 and raises the benefit to $198/month." +us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"It counted SSDI plus SSI but applied only the actual 'allowable' medical excess (about $26/month) instead of Arizona's $145 standard medical deduction for elderly/disabled households. That produced $165 instead of the $198/month that $1,157 of net income yields." +us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"It correctly used $1,511 of income, the $209 standard deduction, and the $546 maximum, but subtracted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also skipped rounding the contribution up to a whole dollar. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month." +us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $395 net income comes from the $630 SSDI alone, leaving out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct net income is $1,157, so the benefit is $198/month, not $427." +us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI with the $209 standard deduction and $546 maximum, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. That gave net income of $1,276 instead of $1,157 and $163 instead of $198/month." +us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"It used the right $1,511 of income, $209 standard deduction, and $546 maximum, but subtracted the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also left the 30% contribution unrounded. Correct net income is $1,157, the contribution is $348, and the benefit is $198/month ($2,376/year)." +us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"It asserted that the SSDI income exceeds SNAP limits for a household of two. Even with SSI included, gross income of $1,511 is 86% of the $1,762.50 poverty guideline, and net income of $1,157 easily passes the net income test. The household gets $198/month." +us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It counted only $7,560 of SSDI and left out the couple's roughly $10,572/year of SSI. It also used the $312/year actual medical excess instead of Arizona's $145/month standard medical deduction, with a $2,520 standard deduction and a $6,600 maximum instead of $2,508 and $6,552. The correct annual benefit is $2,376, not $5,182." +us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,"It included SSI ($1,506 total, close to the correct $1,511) and the $209 standard deduction, but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $549 maximum instead of $546. Correct net income is $1,157, giving $198/month rather than $168." +us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"It correctly counted $1,511 of SSDI plus SSI with the $209 standard deduction and $546 maximum, but applied only $26 of actual excess medical costs instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also left the contribution unrounded. Correct net income is $1,157 and the benefit is $198/month." +us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"It counted SSI at the 2025 rate ($840 instead of $881/month) and deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction and a $536 maximum instead of $209 and $546. Correct net income is $1,157, giving $198/month rather than $164." +us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"It treated $630/month of SSDI as the household's only income and left out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and a $548 maximum instead of $546. With $1,511 gross and $1,157 net income, the benefit is $198/month, not $429." +us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SNAP value or explanation, so there was no substantive answer to score. The correct derivation is $1,511 gross income minus $209 standard and $145 Arizona medical deductions, giving $1,157 net and $198/month ($2,376/year)." +us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed net income as $630 SSDI − $209 − $26 = $395, leaving out the couple's $881/month SSI and using the $26 actual medical excess instead of Arizona's $145 standard medical deduction. Correct net income is $1,157, and $546 − $348 = $198/month." +us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,"It returned zero on the grounds that benefit receipt shouldn't be inferred, even though the prompt tells it to assume program take-up and it had itself found income below the SNAP thresholds. Computing the benefit gives $1,511 gross, $1,157 net, and $198/month." +us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted only the $630/month SSDI and left out the couple's $881/month SSI. It also used a ~$205 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction. Correct net income is $1,157, so the benefit is $198/month, not ~$427." +us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left out the couple's SSI and used $7,560 of SSDI as total income. It applied the $35 medical threshold per year instead of per month and never used Arizona's $145/month standard medical deduction. It also used a $598 maximum and a $198 standard deduction instead of $546 and $209. Correct annual SNAP is 12 × ($546 − $348) = $2,376." +us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated SSI receipt as ruling out separate SNAP, applying the old California SSI cash-out rule. Arizona SSI recipients are SNAP-eligible, and SSI receipt actually confers categorical eligibility. With SSI counted as income, the household receives $198/month." us,scenario_079,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that the spouse's ""income and circumstances"" fail Arizona's adult Medicaid rules, inverting the test: the household's MAGI is 0.35 × FPL, far below Arizona's 138% FPL expansion-adult limit, so low income qualifies rather than disqualifies. It also skipped the pathway the engine used — the SSI-recipient category, which grants automatic Medicaid in Arizona (a 1634 state) to the blind/disabled spouse whose $0 countable income and $1,045 in resources (under the $2,000 individual limit) produce $5,286 of SSI — and treated an income screen as the sole route to adult coverage." us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to 0 on the ground that ""no additional Medicaid-eligibility inputs"" beyond disability and IHS coverage were supplied, treating Medicaid eligibility as requiring an explicit enrollment or income input rather than deriving it from the facts given. Blindness plus disability plus $0 earned and unearned income and $1,045 in countable resources produce SSI of $5,286, and SSI receipt in Arizona triggers the SSI-recipient category's automatic Medicaid; separately, 0.35 × FPL MAGI clears the 138% FPL expansion-adult test, and IHS coverage at interview is not minimum essential coverage that blocks eligibility." -us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model invoked ""SSDI-linked disability eligibility"" for a spouse who receives no Social Security disability income — the household's $7,560 SSDI is the head's — so there is no SSDI entitlement from which Medicare's disability pathway could flow. It treated the boolean ""is disabled"" as itself conferring Medicare, ignoring that the pathway requires 24 months of actual disability-benefit entitlement, and that PolicyEngine's Medicare eligibility test is age 65+ only." -us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model correctly stated the governing rule — Medicare via SSDI requires 24 months of SSDI receipt — and then overrode its own rule with ""Spouse is disabled, treated as eligible under disability."" It also invented ""SSDI-equivalent status"" for a spouse with $0 listed Social Security disability income, when the instructions direct that unlisted income be treated as zero, so the 24-month clock never begins and the age-65 threshold is unmet at 55." -us,scenario_079,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model asserted a bare ""disability-based Medicare eligibility rule"" with no derivation, mapping the disabled flag directly to eligibility. The correct derivation requires age 65+ (spouse is 55) or 24 months of SSDI entitlement (spouse receives no Social Security disability income); the model's answer is consistent with reading the disability and blindness booleans as a standalone eligibility trigger, which is the SSI/SSDI disability definition, not a Medicare entitlement condition." -us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"It built the 2026 couple rate as $967 × 1.5 = $1,450.50, but that product is the 2025 couple FBR, and then labeled the same figure ""indexed ~2.8%"" without ever applying the 2026 COLA, which raises the couple rate to $1,491/month. Its $610 monthly countable income and resource test were correct, so the entire $483.60 shortfall is the skipped indexation step." -us,scenario_079,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It treated the federal benefit rate as a hard income cliff and compared the head's $7,560 annual SSDI against a $943/month individual rate it itself restated as $11,316/year, declaring ineligibility even though $7,560 is below that figure. SSI pays FBR minus countable income: $630/month SSDI less the $20 general exclusion leaves $610 against the 2026 couple FBR of $1,491, so $881/month ($10,572/year) is payable, and it never applied the couple rate at all." -us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"It derived the correct structure ($610/month countable, resources under the $3,000 couple limit) and then discarded it in favor of an invented $1,940/month couple FBR — roughly two individual rates rather than the 1.5× couple rate of $1,491/month for 2026. Substituting the real couple rate into its own subtraction gives $17,892 − $7,320 = $10,572." -us,scenario_079,ssi,claude-opus-4.8,llm_error,other,False,"It used a fabricated $1,538/month couple FBR (the 2026 rate is $1,491) to derive $11,136, then submitted $7,212, a number none of its own arithmetic produces. The correct computation is ($1,491 − $610) × 12 = $10,572." -us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"It set the couple FBR at $1,489/month instead of $1,491 and then truncated mid-sentence, submitting $6,072 when its own stated inputs ($17,868 − $7,320) yield $10,548. The correct figure is the $1,491 couple rate less $610 countable, $881/month or $10,572 annually." -us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It anchored on $1,415 as the 2025 couple FBR — that is the 2024 rate, 2025 is $1,450 — and applied a 2.5% bump that landed it back on $1,450, leaving its 2026 rate a full COLA behind the actual $1,491/month. Its resource analysis and $610 countable income were correct, so the $492 annual shortfall is entirely the stale benefit rate." -us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"It invented a $1,927/month couple FBR, derived $15,804 after concluding the resource and income tests both pass, and then submitted $0 — the opposite of its own conclusion. With the 2026 couple rate of $1,491 and $610 countable, the payment is $881/month, $10,572 for the year." -us,scenario_079,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Every step matched the reference except the couple federal benefit rate: it used $1,486/month rather than the 2026 rate of $1,491 ($1,450 indexed by the 2.8% COLA). That $5/month rate gap produces $876 instead of $881 monthly and the full $60 annual shortfall." -us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used $1,485/month for the 2026 couple FBR instead of $1,491, yielding $875 rather than $881 monthly. The $610 countable income, $240 annual exclusion, and $3,000 couple resource test were all handled correctly, so the $72 annual gap is purely the understated rate." -us,scenario_079,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It stated the 2026 couple rate as $1,482/month when the indexed rate is $1,491, producing $872 instead of $881 monthly. Its income exclusion and resource findings were correct, so the $108 annual shortfall is entirely the $9/month rate error." -us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used $1,415/month — the 2024 couple FBR — as the 2026 rate, skipping both the 2025 increase to $1,450 and the 2.8% 2026 COLA to $1,491. Its $610 countable income was right, so the $76/month rate gap accounts for the entire $912 annual shortfall." -us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It asserted that countable income exceeds the couple federal benefit rate; countable income is $610/month ($630 SSDI less the $20 general exclusion) against a $1,491/month couple rate, so a payment of $881/month is due. It also treated the FBR comparison as an eligibility cliff instead of the subtraction that determines the benefit." -us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used $16,980/year ($1,415/month) as the couple maximum — the 2024 FBR — instead of the 2026 rate of $1,491/month ($17,892/year). Its $7,320 countable income was exact, so $17,892 − $7,320 = $10,572 and the $912 error is entirely the two missed COLAs." -us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"It projected the 2026 couple FBR at $17,832/year ($1,486/month) rather than the actual $17,892 ($1,491/month). With countable income correctly at $7,320, the entire $60 shortfall is the $5/month understatement of the benefit rate." -us,scenario_079,ssi,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It asserted a zero result with no computation. The couple clears the resource test with $1,045.29 countable against the $3,000 limit and has only $610/month countable income against the $1,491 couple FBR, so $881/month — $10,572 annually — is payable." -us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $10,536 implies $878/month, i.e. a couple FBR of $1,488 rather than the 2026 rate of $1,491 ($1,450 indexed 2.8%). It applied the $240 annual exclusion and the $7,320 countable income correctly, so the $36 gap is the understated benefit rate alone." -us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $10,488 implies $874/month, i.e. a couple FBR of $1,484 instead of the 2026 rate of $1,491. The $20 monthly general exclusion and the couple eligibility and asset findings were correct, so the $84 shortfall is entirely the $7/month rate error." -us,scenario_079,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 couple FBR of $1,450/month ($17,400/year) as the 2026 rate. The 2.8% 2026 COLA raises it to $1,491/month ($17,892/year), and $17,892 − $7,320 = $10,572, so the whole $492 error is the missing indexation." -us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"No ssi value and no explanation were returned, so the requested output was absent rather than substantively wrong. The correct value is the 2026 couple FBR of $1,491 less $610 countable income, $881/month or $10,572 annually." -us,scenario_079,ssi,glm-5.3,llm_error,thresholds_rates,False,"It rounded the indexed 2026 couple rate to ""about $1,490""/month instead of the $1,491 that applies, giving $880 rather than $881 monthly. Every other step — the $20 general exclusion, $610 countable income, and the resource test — matched, so the $1/month rounding is the entire $12 annual shortfall." -us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"It denied SSI on the resource test, claiming the bank account plus vehicle value exceed the limit; one vehicle used for transportation is excluded in full regardless of value and the home is excluded, leaving $1,045.29 countable against the $3,000 couple limit. The resource test passes and the couple is paid $1,491 − $610 = $881/month, $10,572 annually." -us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It declined to compute because SSI receipt was not listed among the facts, ignoring the instruction to assume program take-up and the requirement to compute eligibility from the stated disability and income inputs. Both spouses meet the aged/blind/disabled criterion, and the $1,491 couple FBR less $610 countable income yields $10,572." -us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"It projected the 2026 couple maximum at $1,487/month instead of $1,491, producing $877 rather than $881 monthly. Its treatment of the $20 general exclusion against the $630 monthly SSDI was correct, so the $48 annual shortfall is the $4/month rate error." -us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $1,486/month couple payment standard rather than the 2026 rate of $1,491 ($1,450 indexed by the 2.8% COLA), giving $876 instead of $881 monthly. Every other element — $630 SSDI, $20 exclusion, couple eligibility — was right, so the $60 gap is the rate alone." -us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"It asserted without computing that SSDI exceeds the couple's SSI income limits. The $630/month SSDI less the $20 general exclusion leaves $610 countable against a $1,491/month couple FBR, and the couple clears the $3,000 resource limit with $1,045.29, so $881/month — $10,572 a year — is payable." -us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"It used a 2026 couple FBR of about $1,486/month instead of the actual $1,491, producing $876 rather than $881 monthly. The $20 disregard, $610 countable unearned income, and resource test were all correct, so the $60 annual shortfall is entirely the understated rate." -us,scenario_079,ssi,grok-4.6,llm_error,thresholds_rates,False,"It set the 2026 couple FBR near $1,486/month rather than $1,491, yielding $876 instead of $881 monthly. Its exclusions ($20 general), countable income ($610), resource treatment ($1,045 with home and one vehicle excluded), and the absence of an Arizona supplement were all correct, leaving the $60 gap as pure rate error." -us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly applied the ""2025 federal SSI couple rate $1,450/month"" to a 2026 question, skipping the 2.8% COLA that sets the 2026 couple FBR at $1,491. Its $610 countable income and resource findings were correct, so the missing indexation is the whole $492 shortfall." -us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"No ssi value and no explanation were returned, so the failure is an absent output rather than a substantive computation error. The correct value is ($1,491 couple FBR − $610 countable) × 12 = $10,572." -us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"It returned zero because no SSI receipt was listed, conflating the instruction against inferring unlisted benefit receipt with the instruction to assume take-up when computing a requested benefit. Both spouses are blind and disabled, countable income is $610/month against the $1,491 couple FBR, and the computed benefit is $10,572." -us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"After reaching the correct $7,320 countable income, it used $1,450/month ($17,400/year) — the 2025 couple FBR — as the 2026 rate. The 2.8% COLA puts the 2026 couple rate at $1,491/month ($17,892/year), so the payment is $17,892 − $7,320 = $10,572 and the $492 error is the missing indexation." -us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It split the $7,560 SSDI in half under spousal deeming but then counted only one spouse's $3,780 as the household's combined countable income instead of the $7,320 that both halves produce, and subtracted it from a fabricated combined benefit rate of about $1,895/month. The correct pairing is the $1,491/month couple FBR against $610/month countable income, $881/month or $10,572 annually." -us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model identified Arizona's increased excise tax credit but invented an income floor for it, writing that it ""requires filing with income"" and that the couple's $7,560 of nontaxable SSDI disqualifies them. A.R.S. § 43-1072.01 imposes only a ceiling — federal AGI at or below $25,000 for married filing jointly — plus Arizona residency and non-dependent status, with no earned-income or tax-liability requirement, so the credit pays $25 per person for the head and spouse, $50." -us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Arizona has no state-level refundable individual income tax credit program at all, which erases the increased excise tax credit (A.R.S. § 43-1072.01), a fully refundable per-person sales-tax offset claimed on Form 140. Because it is refundable, the household's zero Arizona taxable income is irrelevant; the credit pays $25 each for the head and spouse, $50." -us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model reached the correct answer inside its own reasoning — ""The Increased Excise Tax Credit is $25/person up to $100 max — MFJ with 0 dependents = $50"" — and then submitted $200 as a ""best estimate"" with no rule supporting it. $200 is double the statutory $100 household cap and corresponds to eight claimed persons at $25 each; the two-person tax unit yields $50." -us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model dismissed the increased excise tax credit on the ground that ""income tax filing thresholds [are] not met here,"" adding a minimum-income gate the credit does not contain. The credit's only income test is the $25,000 married-filing-jointly federal AGI ceiling, and Arizona pays it as a refund to filers with no liability, giving $25 each for the head and spouse, $50." -us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model conditioned all Arizona refundable credits on earnings or qualifying dependents, importing federal EITC/CTC structure into a state credit that has neither test. The increased excise tax credit is computed per person in the tax unit — $25 for the head plus $25 for the spouse — subject only to the $25,000 joint AGI ceiling, so the answer is $50." -us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model named the increased excise tax credit and then classified it with the family income tax credit as ""non-refundable or [with] specific eligibility requirements not met here."" The increased excise tax credit is the refundable one under A.R.S. § 43-1072.01 — excess over liability is refunded — and its requirements are Arizona residency and joint AGI under $25,000, both satisfied, yielding $25 per spouse for $50." -us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model searched only for the Form 140PTC property tax credit, correctly ruled it out on the age-65/SSI test, and then declared no other refundable Arizona credit exists. It never reached the increased excise tax credit, which pays $25 per tax unit member to Arizona residents with joint AGI at or below $25,000 regardless of age or liability, giving $50 for this two-adult unit." -us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model gated Arizona refundable credits on positive income and qualifying children, neither of which the increased excise tax credit requires. The credit is a per-person sales-tax rebate of $25 for each member of the tax unit, capped at $100, with an upper AGI limit of $25,000 for joint filers, so the head and spouse produce $50." -us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model considered only dependent-based credits and property tax credits and stopped there. Arizona's refundable increased excise tax credit is neither: it grants $25 per person in the tax unit to residents with joint federal AGI under $25,000, so the childless couple still collects $50." -us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model issued a blanket denial that any Arizona refundable credit applies, without testing the increased excise tax credit. That credit's conditions — Arizona residence, joint federal AGI at or below $25,000, not claimable as a dependent — are all met, and it pays $25 each for the head and spouse, $50." -us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model rejected all state refundable credits on ""income and demographics"" without identifying any Arizona credit by name. The increased excise tax credit is triggered by exactly those facts: Arizona residence and joint AGI far below the $25,000 ceiling, paying $25 per person for the two-adult unit, $50." -us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model asserted non-qualification with no rule cited, skipping Arizona's increased excise tax credit. That credit requires only Arizona residency and joint federal AGI at or below $25,000 and is refundable against zero liability, so the married couple receives $25 each, $50." -us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model declared the household qualifies for no refundable Arizona credit without evaluating the increased excise tax credit. Its $25-per-tax-unit-member structure, $100 household cap, and $25,000 joint AGI ceiling all apply here, producing $50 for the head and spouse." -us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model made the credit contingent on ""state income tax liability or qualifying earnings,"" which inverts what refundable means. The increased excise tax credit is paid as a refund when liability is zero and has no earnings test, delivering $25 per person for the head and spouse, $50." -us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stated a bare conclusion of no qualifying Arizona refundable credit, bypassing the increased excise tax credit. With Arizona residence, married-filing-jointly status, and $7,560 of income against a $25,000 joint AGI ceiling, the two-person unit earns $25 each, $50." -us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model denied Arizona refundable credits without naming or testing any. The increased excise tax credit applies on residency plus the $25,000 joint AGI ceiling alone and is computed at $25 per tax unit member, giving $50 for the head and spouse." -us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model invented a PolicyEngine rule that ""childless married couples with no wage income have no refundable state tax credits"" in Arizona. PolicyEngine's az_increased_excise_tax_credit keys off state, filing status, and tax unit size with an AGI ceiling — no wage or dependent test — so it returns $25 per adult, $50." -us,scenario_079,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model conditioned Arizona refundable credits on positive taxable income, which is the opposite of the increased excise tax credit's design. The credit offsets sales tax borne by low-income residents, is refunded in full when liability is zero, and pays $25 for each of the head and spouse, $50." -us,scenario_079,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model claimed Arizona's refundable family tax credit is a percentage of the federal EITC and zeroed it out. Arizona's family income tax credit is a nonrefundable per-exemption credit unrelated to the EITC, and the state's refundable credit is the increased excise tax credit, which pays $25 per tax unit member under a $25,000 joint AGI ceiling, $50 here." -us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model concluded no Arizona refundable credit is triggered by the listed facts, missing that state = AZ, married filing jointly, and $7,560 of income are precisely the facts that trigger the increased excise tax credit. At $25 per tax unit member under the $25,000 joint AGI ceiling, the two adults yield $50." -us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model required earned income or children for any state refundable credit, tests the increased excise tax credit does not impose. The credit is per-person — $25 for the head, $25 for the spouse — capped at $100 and limited only by the $25,000 joint AGI ceiling, so the answer is $50." -us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model required ""qualifying tax liability, earned income, or qualifying children"" for an Arizona refundable credit. The increased excise tax credit requires none of the three; it needs Arizona residency and joint federal AGI at or below $25,000, and pays $25 per person for the head and spouse, $50." -us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model found no fact generating an Arizona refundable credit, overlooking that residency plus low AGI is the entire eligibility test for the increased excise tax credit. It pays $25 for each member of the tax unit up to $100, so this married couple receives $50." -us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model conditioned the result on earnings, children, or other qualifying circumstances, none of which the increased excise tax credit uses. Its tests are Arizona residency, non-dependent status, and joint federal AGI at or below $25,000, and it pays $25 per tax unit member, giving $50." -us,scenario_079,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model tied the credit to state tax liability, but the increased excise tax credit is refundable and paid in full when liability is zero. Arizona residency and joint AGI of $7,560 against the $25,000 ceiling qualify the couple for $25 each, $50." -us,scenario_079,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model required qualifying children or positive liability as the basis for any Arizona refundable credit. The increased excise tax credit counts adults as well as dependents at $25 apiece and is refunded against zero liability, so the head and spouse produce $50." -us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted Arizona has a state EITC equal to a percentage of the federal EITC and set it to zero. Arizona has no state EITC; its refundable credit is the increased excise tax credit, which pays $25 per tax unit member — $50 for this couple — under a $25,000 joint AGI ceiling with no earned-income requirement." -us,scenario_079,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model derived $0 from zero state tax liability, which does not bind a refundable credit. Arizona's increased excise tax credit is paid out as a refund to residents with joint AGI at or below $25,000 at $25 per person, so the head and spouse yield $50." -us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model required either listed rent/property tax payments or earned income for any Arizona refundable credit. The increased excise tax credit is a flat per-person sales-tax offset needing neither — $25 each for the head and spouse under the $25,000 joint AGI ceiling — for $50." -us,scenario_079,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model stated that Arizona offers no refundable income tax credits, contradicting A.R.S. § 43-1072.01's increased excise tax credit. That credit is refundable, is not conditioned on children or liability, and pays $25 for each of the two adults, $50." -us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Arizona has no state income tax, confusing it with a no-income-tax state. Arizona levies a 2.5% flat individual income tax and grants the refundable increased excise tax credit of $25 per tax unit member to residents with joint AGI at or below $25,000, giving this couple $50." -us,scenario_079,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model tested only the Form 140PTC property tax credit's age-65 requirement and then concluded that zero liability rules out everything else. The increased excise tax credit is a separate refundable credit with no age test and no liability requirement, paying $25 per tax unit member for a $50 total here." -us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model required Arizona taxable income or qualifying dependents for a refundable state credit. The increased excise tax credit is claimed on the return by residents with joint federal AGI under $25,000 regardless of taxable income, and counts the taxpayer and spouse at $25 each, $50." -us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model gave a bare denial that any Arizona refundable credit applies, never evaluating the increased excise tax credit. Its eligibility is Arizona residency plus joint AGI at or below $25,000, and its amount is $25 per tax unit member capped at $100, producing $50 for this two-adult household." +us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disabled flag as enough for Medicare under 65, citing 'SSDI-linked disability eligibility'. However, the spouse gets no Social Security disability income; the $7,560 SSDI is the head's. No months of SSDI receipt are listed, so the 24-month SSDI entitlement requirement is not met and the spouse is not Medicare eligible." +us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model correctly stated that disability-based Medicare requires 24 months of SSDI receipt, then set that requirement aside and marked the spouse eligible for being disabled. It also wrongly gave the spouse 'SSDI-equivalent status' when only the head has Social Security disability income. With zero SSDI and zero months of receipt, the spouse fails the 24-month test and, at 55, the age-65 test." +us,scenario_079,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model cited a 'disability-based Medicare eligibility rule' but applied it to the disabled flag alone. The disability pathway requires 24 months of SSDI entitlement (or ESRD/ALS). The spouse has no SSDI income, no months of receipt, and no ESRD, and at 55 is under 65, so the correct answer is not eligible." +us,scenario_079,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model claimed the spouse qualifies under disability-based Medicare rules, which equates being disabled with Medicare entitlement. Those rules require at least 24 months of SSDI receipt or ESRD/ALS. The spouse receives no Social Security disability benefits (the SSDI belongs to the head) and is 55, so the spouse is not eligible." +us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"It computed countable income correctly ($630 SSDI - $20 = $610/mo). For the couple rate it used a projected $1,450.60/mo, which is essentially the 2025 couple FBR, instead of the 2026 couple FBR of $1,491/mo. As a result, 12 x ($1,450.60 - $610) = $10,088.40 understates the correct 12 x $881 = $10,572." +us,scenario_079,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied an individual income-limit test and claimed that $7,560/yr of SSDI exceeds a $11,316/yr limit, which is false on its own numbers. It also ignored that both spouses are blind and disabled and form an eligible couple under the $1,491/mo couple FBR. Countable income is only $610/mo, which leaves an $881/mo benefit, not $0." +us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"It first computed $10,080 using a $1,450/mo couple FBR. It then replaced that with an invented $1,940/mo couple FBR ($23,280/yr), producing $15,960. The 2026 couple FBR is $1,491/mo, so the correct benefit is ($1,491 - $610) x 12 = $10,572." +us,scenario_079,ssi,claude-opus-4.8,llm_error,thresholds_rates,False,"It overstated the 2026 couple FBR at $1,538/mo (actual: $1,491) and derived $11,136 in its explanation. It then submitted $7,212, which matches neither its own derivation nor the rules. Against its $7,320 of countable income, that figure implies a couple FBR of only about $1,211/mo. The correct result is ($1,491 - $610) x 12 = $10,572." +us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"It set up the calculation almost correctly: a $1,489/mo couple FBR ($17,868/yr) minus $7,320 of countable income. It then subtracted wrong: $17,868 - $7,320 is $10,548, not the $6,072 it submitted. With the correct 2026 couple FBR of $1,491/mo, the answer is $10,572." +us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It took $1,415/mo as the 2025 couple FBR (that is the 2024 rate) and inflated it 2.5% to $1,450/mo, which is the 2025 rate. The 2026 couple FBR is $1,491/mo. With the correct $610 of countable income, it got $840/mo instead of $881/mo, or $10,080 instead of $10,572." +us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"Its explanation finds the couple eligible and computes $15,804/yr using an inflated $1,927/mo couple FBR, yet it submitted 0, which contradicts its own conclusion. The 2026 couple FBR is $1,491/mo, so the correct benefit is ($1,491 - $610) x 12 = $10,572." +us,scenario_079,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The method is correct: couple FBR minus $610 of countable SSDI. It used a $1,486/mo couple FBR instead of the 2026 value of $1,491/mo, so it got $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly applied the couple computation with $610/mo of countable income. It used a $1,485/mo couple FBR instead of the 2026 value of $1,491/mo, which gives $875/mo ($10,500/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly subtracted $610/mo of countable SSDI from the couple rate. The rate it used was $1,482/mo instead of the 2026 couple FBR of $1,491/mo, so it got $872/mo ($10,464/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly computed $610/mo of countable income. It used a $1,486.25/mo couple FBR instead of the 2026 value of $1,491/mo, which yields $876.25/mo ($10,515/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used $1,415/mo as the couple SSI rate, which is the 2024 couple FBR, instead of the 2026 value of $1,491/mo. With the correct $610 of countable income, that gives $805/mo ($9,660/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It asserted that the SSDI exceeds the couple federal benefit rate. In fact, countable SSDI is $610/mo after the $20 exclusion, far below the 2026 couple FBR of $1,491/mo. The couple is therefore eligible for $881/mo, or $10,572/yr, not $0." +us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly computed $7,320 of annual countable income. It subtracted that from a $16,980 couple maximum, which is the 2024 rate of $1,415/mo. The 2026 couple FBR is $17,892/yr ($1,491/mo), which gives $10,572." +us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a projected couple FBR of $17,832/yr ($1,486/mo) instead of the 2026 value of $17,892/yr ($1,491/mo). Subtracting the correct $7,320 of countable income gave $10,512 instead of $10,572." +us,scenario_079,ssi,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave $0 without any computation. Both spouses are blind and disabled, so they form an eligible couple. Countable resources of $1,045 are under the $3,000 couple limit, and countable income of $610/mo is under the $1,491/mo couple FBR, which yields $881/mo, or $10,572/yr." +us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly subtracted $7,320 of countable SSDI. Its $10,536 answer implies a couple FBR of $17,856/yr ($1,488/mo) instead of the 2026 value of $17,892/yr ($1,491/mo), which gives $10,572." +us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the $20/mo exclusion correctly, leaving $7,320/yr of countable income. Its $10,488 answer implies a couple FBR of $17,808/yr ($1,484/mo) instead of the 2026 value of $1,491/mo, which gives $10,572." +us,scenario_079,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a couple FBR of $1,450/mo ($17,400/yr), which is the 2025 rate, instead of the 2026 rate of $1,491/mo ($17,892/yr). Subtracting the correct $7,320 of countable income gave $10,080 instead of $10,572." +us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"It returned no SSI value and no explanation, so no answer was scored. The correct derivation is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, times 12, which is $10,572." +us,scenario_079,ssi,glm-5.3,llm_error,thresholds_rates,False,"It correctly derived $610/mo of countable income. It used an approximate couple FBR of $1,490/mo instead of the 2026 value of $1,491/mo, so it got $880/mo ($10,560/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"It counted the vehicle toward the resource limit and concluded the couple failed the resource test. SSI excludes one automobile regardless of value, so countable resources are only the $1,045 bank account, well under the $3,000 couple limit. The couple is eligible for ($1,491 - $610) x 12 = $10,572." +us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned $0 because no SSI receipt was listed, instead of computing eligibility with take-up assumed, as the prompt directs. Blindness and disability make both spouses an eligible couple. The benefit is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, or $10,572/yr." +us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly reduced the couple rate by $610 of countable SSDI. It used a projected $1,487/mo couple maximum instead of the 2026 value of $1,491/mo, which gives $877/mo ($10,524/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"It correctly computed $610/mo of countable income. It used a $1,486/mo couple payment standard instead of the 2026 couple FBR of $1,491/mo, which gives $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"It asserted that the SSDI exceeds the couple's SSI limits without computing anything. Countable SSDI is $630 - $20 = $610/mo, which is well below the 2026 couple FBR of $1,491/mo. That leaves an $881/mo benefit, or $10,572/yr, not $0." +us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"It correctly took $610/mo of countable SSDI. It used a couple FBR of about $1,486/mo instead of the 2026 value of $1,491/mo, so it got $876/mo ($10,512/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,grok-4.6,llm_error,thresholds_rates,False,"Its resource and income analysis is correct ($1,045 of countable resources and $610/mo of countable income). It used a couple FBR of about $1,486/mo instead of the 2026 value of $1,491/mo, which gives $10,512 instead of $10,572." +us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly applied the 2025 couple SSI rate of $1,450/mo to a 2026 case instead of the 2026 couple FBR of $1,491/mo. With the correct $610 of countable income, it got $840/mo ($10,080/yr) instead of $881/mo ($10,572/yr)." +us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no SSI value and no explanation, so no answer was scored. The correct derivation is the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, times 12, which is $10,572." +us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"It read the rule against inferring unlisted benefit receipt as a reason to report $0 SSI. The prompt says to assume program take-up, and SSI is the output to compute. The blind and disabled couple is eligible for the 2026 couple FBR of $1,491/mo minus $610 of countable SSDI, or $10,572/yr." +us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"It estimated the 2026 couple FBR at $1,450/mo ($17,400/yr), which is the 2025 rate, instead of the 2026 rate of $1,491/mo ($17,892/yr). Subtracting the correct $7,320 of countable income gave $10,080 instead of $10,572." +us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It treated each spouse as an individual claimant and summed individual rates instead of applying the $1,491/mo couple FBR. It also counted only $3,780 of combined income, which is half the SSDI. Under couple computation, the full $7,560 counts against the couple ($610/mo after the $20 exclusion), which leaves $881/mo, or $10,572/yr, not $18,963.60." +us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said the increased excise tax credit requires filing with income and wrote the couple off because they have only nontaxable SSDI. The credit is refundable and goes to filers with federal AGI at or below $12,500 (MFJ), so this couple's $0 AGI qualifies them for $25 x 2 = $50." +us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Arizona has no refundable individual income tax credit and treated the absence of taxable income as disqualifying. It missed the refundable increased excise tax credit, which is keyed to low federal AGI rather than taxable income and pays $25 each to head and spouse, $50 in total." +us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly worked out the increased excise tax credit at $25 per person, $50 for an MFJ couple with no dependents, and ruled out the property tax credit. It then submitted $200 with no computation behind it. That figure contradicts the model's own $50 derivation and even exceeds the credit's $100 cap." +us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model claimed the increased excise tax credit requires meeting income tax filing thresholds, and it tied refundable credits to taxable wages. The credit is refundable and is designed for filers whose federal AGI is below $12,500 (MFJ), so this $0-AGI couple receives $25 x 2 = $50." +us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model conditioned Arizona refundable credits on earnings or qualifying dependents. The increased excise tax credit requires neither: it pays $25 for each of head and spouse ($50) to an MFJ unit with federal AGI at or below $12,500." +us,scenario_079,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model concluded that no Arizona refundable credit applies and overlooked the increased excise tax credit. That credit pays $25 per taxpayer, $50 for head and spouse, because the couple's federal AGI of $0 is below the $12,500 MFJ ceiling." +us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model labeled the increased excise tax credit nonrefundable and required Arizona taxable income or wages for any refundable credit. The credit is refundable and depends only on federal AGI of $12,500 or less for MFJ filers, which yields $25 x 2 = $50 here." +us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model checked only the Property Tax Credit (Form 140PTC) and then dismissed every other refundable credit. It missed the increased excise tax credit, which pays $25 per person ($50) to an MFJ household with federal AGI under $12,500 regardless of age or SSI status." +us,scenario_079,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model treated the absence of tax liability as ruling out a refundable credit. Arizona's increased excise tax credit is fully refundable and pays this zero-liability, zero-AGI MFJ couple $25 each, $50 in total." +us,scenario_079,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model treated zero income and no qualifying children as disqualifying. Zero federal AGI is exactly what qualifies the couple for the increased excise tax credit, which counts head and spouse at $25 each, $50 in total, under the $12,500 MFJ AGI ceiling." +us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model looked only for child-based and property tax credits. It missed the increased excise tax credit, which is refundable, counts the filer and spouse at $25 each and pays $50 because federal AGI is under $12,500." +us,scenario_079,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model asserted that no Arizona refundable credit applies without checking the increased excise tax credit. That credit pays $25 per taxpayer, $50 for this MFJ couple, because their federal AGI of $0 is under the $12,500 limit." +us,scenario_079,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model made refundable credits depend on having state income tax liability. The increased excise tax credit is refundable regardless of liability and pays $25 x 2 = $50 to an MFJ unit with federal AGI at or below $12,500." +us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model concluded from the low income and demographics that no state refundable credit applies. Low income is precisely the trigger for Arizona's refundable increased excise tax credit, which pays $25 for each of head and spouse, $50 in total, at federal AGI of $12,500 or less." +us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model stated that the household qualifies for no state refundable credit and never applied the increased excise tax credit. That credit gives $25 per filer to MFJ units with federal AGI of $12,500 or less, which is $50 for this $0-AGI couple." +us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stated that no refundable Arizona credit applies and omitted the increased excise tax credit. That credit is refundable, is available at federal AGI of $12,500 or less for MFJ filers and pays $25 x 2 = $50 here." +us,scenario_079,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model required state income tax liability or qualifying earnings for a refundable credit. The increased excise tax credit needs neither: it pays $25 each to head and spouse, $50 in total, because federal AGI is $0, under the $12,500 MFJ limit." +us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model concluded that the household qualifies for no refundable Arizona credit and missed the increased excise tax credit. That credit pays $25 per person, $50 for this MFJ couple, because their federal AGI is $0, below the $12,500 ceiling." +us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model denied any refundable Arizona credit and never applied the increased excise tax credit. That credit gives $25 for each of head and spouse ($50) to MFJ filers with federal AGI of $12,500 or less." +us,scenario_079,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model asserted that childless married couples without wages get no refundable Arizona credits. The increased excise tax credit pays such couples $25 per spouse, $50 in total, when federal AGI is at or below $12,500, with no wage or child requirement." +us,scenario_079,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated zero taxable income as ruling out Arizona refundable credits. The increased excise tax credit is refundable, turns on federal AGI of $12,500 or less (MFJ) rather than taxable income, and pays $25 x 2 = $50 here." +us,scenario_079,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model considered only a family credit tied to the federal EITC and asserted that Arizona has no other refundable credits. It missed the increased excise tax credit, which pays $25 each to head and spouse, $50 in total, because federal AGI is $0, under the $12,500 MFJ limit." +us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model found no refundable Arizona credit triggered by the facts and overlooked the increased excise tax credit. That credit is triggered by federal AGI of $12,500 or less for MFJ filers and pays $25 per spouse, $50 in total." +us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model required earned income or children for a state refundable credit. Arizona's increased excise tax credit requires neither: it pays $25 for each of head and spouse ($50) to MFJ filers with federal AGI of $12,500 or less." +us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model conditioned refundable Arizona credits on tax liability, earned income or qualifying children. The increased excise tax credit is refundable without any of these and pays $25 x 2 = $50 because federal AGI is under $12,500 (MFJ)." +us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded that no reported fact generates an Arizona refundable credit. The facts do generate one: federal AGI of $0 with MFJ status and two adults qualifies the couple for the increased excise tax credit of $25 per person, $50 in total." +us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model required qualifying earnings or children for an Arizona refundable credit. The increased excise tax credit pays $25 to each of head and spouse, $50 in total, to MFJ filers with federal AGI at or below $12,500, without earnings or children." +us,scenario_079,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model required taxable income or dependents for a refundable Arizona credit. The increased excise tax credit is keyed to federal AGI of $12,500 or less (MFJ) and counts the filer and spouse at $25 each, $50 in total." +us,scenario_079,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,The model conditioned Arizona refundable credits on earnings or dependent children. The increased excise tax credit needs neither and pays this $0-AGI MFJ couple $25 x 2 = $50. +us,scenario_079,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model treated the absence of state tax liability as eliminating refundable credits. The increased excise tax credit is refundable regardless of liability and pays $50 ($25 per spouse) to MFJ filers with federal AGI of $12,500 or less." +us,scenario_079,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model looked for qualifying children and tax liability as the basis for Arizona refundable credits. The increased excise tax credit is based on per-person counts of filer and spouse at $25 each, gated only by federal AGI of $12,500 or less (MFJ), and yields $50." +us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model assumed Arizona has a state EITC set at a percentage of the federal EITC, which Arizona does not have, and then declared that no other refundable credit applies. It missed the increased excise tax credit, which pays $25 x 2 = $50 because federal AGI is $0, under the $12,500 MFJ limit." +us,scenario_079,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model tied refundable credits to state tax liability and found none applicable. The increased excise tax credit is refundable and pays this zero-liability MFJ couple $25 each, $50 in total, because federal AGI is under $12,500." +us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked only the property tax refund, which needs rent or property tax, and assumed every other refundable credit needs earned income. The increased excise tax credit needs neither and pays $25 for each of head and spouse ($50) at federal AGI of $12,500 or less (MFJ)." +us,scenario_079,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model treated being childless with zero liability as disqualifying. The increased excise tax credit counts the filer and spouse themselves at $25 each, $50 in total, and is refundable at federal AGI of $12,500 or less for MFJ filers." +us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model asserted that Arizona has no state income tax. Arizona levies a flat individual income tax and offers the refundable increased excise tax credit, which pays this $0-AGI MFJ couple $25 x 2 = $50." +us,scenario_079,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model correctly ruled out the property tax credit on its age-65 test, then assumed no other refundable credit applies at $0 liability. It missed the increased excise tax credit, which has no age test and pays $25 per spouse, $50 in total, to MFJ filers with federal AGI of $12,500 or less." +us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model required Arizona taxable income or qualifying dependents for a refundable credit. The increased excise tax credit is gated by federal AGI of $12,500 or less (MFJ), counts head and spouse at $25 each and yields $50." +us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model asserted that no Arizona refundable credit applies and never applied the increased excise tax credit. That credit pays $25 for each of head and spouse ($50) because the couple's federal AGI is $0, below the $12,500 MFJ ceiling." us,scenario_080,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. +us,scenario_080,head_chip_eligible,claude-sonnet-5.5,llm_error,other,False,"Its explanation says the head is not eligible and ends with 'value = 0', which matches the correct result, but its scored answer records Yes. The submission breaks the contract that the numeric value must match the explanation. Its reasoning relied on age alone (CHIP is for children). The rule the engine applies is that the head is Medicaid-eligible under the PA adult expansion category, and that excludes CHIP." +us,scenario_080,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"GLM gave no value and no explanation for head_chip_eligible, so there was nothing to score. The correct derivation is that the 41-year-old head, with about $940 of income, is Medicaid-eligible under Pennsylvania's ACA adult expansion category. That excludes her from CHIP, so the answer is 0." us,scenario_080,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Because the head is disabled, the model routed him into the aged/blind/disabled pathway and demanded SSI-linked categorical status plus an asset screen, but Pennsylvania adopted the ACA adult expansion, whose MAGI group covers every non-elderly adult 19-64 at or below 138% FPL regardless of disability and independent of any SSI receipt. Disability status does not remove an adult from the MAGI adult group, and MAGI-based groups take no resource test, so the $19,828 bank balance is not part of the determination. Head's MAGI of $940 is 0.06 x FPL, clearing the 138% FPL limit by a wide margin." us,scenario_080,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model stated the two premises that decide the case - Pennsylvania expanded Medicaid, and the financial assistance and investment income fall below the MAGI thresholds - then submitted 0, contradicting its own reasoning. Applying the threshold it had already named yields eligibility: $940 of MAGI is 0.06 x FPL against the 138% FPL adult expansion limit for a 41-year-old. It deferred the conclusion to 'PolicyEngine evaluates it' instead of completing the comparison it had set up." us,scenario_080,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for head_medicaid_eligible, so the submission carried no substantive determination to evaluate. The correct derivation is a 41-year-old in expansion-state Pennsylvania with MAGI of $940, or 0.06 x FPL, inside the 138% FPL ACA adult expansion limit, giving eligible." @@ -5924,45 +6490,52 @@ us,scenario_080,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F us,scenario_080,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_080,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_080,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model built the correct structure — PA BBCE waiving the asset test, the LIHEAP-triggered standard utility allowance, and the uncapped elderly/disabled excess shelter deduction zeroing net income — then priced the one-person maximum allotment at $293/month. The FY2026 maximum is $298/month and the October 1, 2026 uprating raises it to $304.68, so calendar 2026 is 9 × $298 + 3 × $304.68 = $3,596.04, not $293 × 12." -us,scenario_080,snap,claude-fable-5.1,llm_error,period_annualization,False,"The model identified the correct FY2026 one-person maximum allotment of $298/month and correctly zeroed net income, but annualized by flat multiplication of $298 × 12. Calendar year 2026 spans two allotment schedules — $298 for January through September and $304.68 for October through December after the FY2027 uprating — giving $3,596.04." -us,scenario_080,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model added the $3,600 financial assistance (an income stream, not a resource) to the $19,828 bank balance and disqualified the household against a $2,750 resource limit. Pennsylvania's broad-based categorical eligibility through TANF non-cash services waives the SNAP resource test outright, so the household qualifies and receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly recognized that PA's BBCE eliminates the asset test and that net income falls to $0, then used a stale $293/month maximum allotment for a one-person household. The correct calendar-2026 figure blends the FY2026 $298 for nine months with the post-October $304.68 for three months, yielding $3,596.04." -us,scenario_080,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model conceded the household's income qualified, then disqualified it by applying the ~$4,500 elderly/disabled federal resource limit to the $19,828 bank balance. Pennsylvania grants categorical eligibility through TANF non-cash services, which eliminates the resource test, so the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model never applied the heating standard utility allowance triggered by the $2,000 LIHEAP-type energy subsidy, whose excess shelter cost is deductible without cap for a disabled household and drives net income to exactly $0. It instead assumed a partial allotment and selected $218.50/month from a self-generated $218–$300 range — a reduction of about $80/month that corresponds to 30% of roughly $265 of net income, far more than even its own stated $78/month of countable income." -us,scenario_080,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly derived the SUA-based uncapped shelter deduction, $0 net income, and BBCE waiver of the asset test, but priced the allotment at the FY2025 value of $292/month and multiplied by 12. The FY2026 maximum is $298/month, rising to $304.68 in October 2026, giving $3,596.04." -us,scenario_080,snap,claude-sonnet-5,llm_error,asset_resource,False,"After correctly concluding that net income falls to $0 or near $0, the model reversed itself and disqualified the household by applying the $4,250 elderly/disabled federal resource limit to the $19,828 bank balance. Pennsylvania's TANF non-cash categorical eligibility waives the resource test, so the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model counted the $924 long-term capital gain as SNAP countable income (it is not; countable gross income is $301.33/month from the $300 assistance plus $1.33 dividends) and omitted the shelter deduction entirely, leaving $170.33 of net income and shaving 30% of it off the allotment. The $2,000 energy subsidy triggers the heating SUA, whose excess shelter cost is uncapped for a disabled household and zeroes net income, so the full allotment applies." -us,scenario_080,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model explicitly wrote ""no shelter or medical deductions,"" ignoring that the $2,000 LIHEAP-type energy subsidy entitles the household to the heating standard utility allowance and that the resulting excess shelter deduction is uncapped for a disabled household. It also counted the $924 capital gain as countable income and used the stale $292 maximum, producing a 30% net-income reduction where the correct net income is $0 and the allotment is the full $3,596.04." -us,scenario_080,snap,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model included the $924 long-term capital gain in countable income and applied only the standard deduction, stating no medical or shelter deductions, then subtracted 30% of $168.33 net income from the allotment. The LIHEAP-triggered heating SUA gives an uncapped excess shelter deduction for this disabled household, driving net income to $0 and the benefit to the full $3,596.04." -us,scenario_080,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model reached $0 net income and the maximum allotment by the right route (SUA from the energy subsidy), though its stated excess medical deduction is wrong — $112/year is $9.33/month, below the $35/month medical floor. Its numeric miss is the allotment rate: it used $291/month rather than the FY2026 $298 blended with $304.68 from October 2026, which gives $3,596.04." -us,scenario_080,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model gave no derivation and concluded that the household's assets plus its lack of reported income produce no benefit. Both halves are backwards: PA's TANF non-cash categorical eligibility waives the resource test, and near-zero income produces the maximum allotment rather than zero, so the correct answer is $3,596.04." -us,scenario_080,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified that the SPM energy subsidy grants the standard utility allowance and that the resulting excess shelter deduction zeroes net income, qualifying the household for the maximum single-person allotment. It then valued that maximum at the FY2025 $292/month × 12; the correct calendar-2026 blend of $298 (nine months) and $304.68 (three months) is $3,596.04." -us,scenario_080,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied PA BBCE, the SUA from the energy subsidy, and the uncapped shelter deduction to reach $0 net income, then multiplied the stale FY2025 maximum of $292/month by 12. The FY2026 maximum is $298/month and rises to $304.68 in October 2026, giving $3,596.04." -us,scenario_080,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"The model asserted without computation that the financial assistance and assets zero out the benefit. Pennsylvania's categorical eligibility waives the resource test, and $301.33/month of countable income against the standard deduction plus the uncapped SUA shelter deduction leaves $0 net income, so the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model got the entire eligibility and deduction chain right — BBCE, heating SUA from the energy assistance, $0 net monthly income, maximum allotment — but priced that maximum at $292/month. Calendar 2026 blends the FY2026 $298 for January–September with $304.68 for October–December, giving $3,596.04." -us,scenario_080,snap,gemini-3.7-flash,llm_error,period_annualization,False,"The model used the correct FY2026 one-person maximum of $298/month but annualized it as a flat $298 × 12 = $3,576. The October 1, 2026 uprating raises the maximum to $304.68 for the final quarter of the calendar year, so the annual total is 9 × $298 + 3 × $304.68 = $3,596.04." -us,scenario_080,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly derived categorical eligibility and the HCSUA-driven excess shelter deduction that zeroes net income, then applied a $292/month maximum allotment. That is the FY2025 figure; FY2026 is $298 and the October 2026 uprating gives $304.68, producing $3,596.04 for calendar 2026." -us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for snap, so no substantive SNAP reasoning was submitted to evaluate. The required output was the full one-person allotment of $3,596.04." -us,scenario_080,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the energy subsidy from income and recognized that categorical eligibility overrides the bank assets, but then stopped at the standard deduction and reduced the allotment by 30% of about $170 of net income. It missed that the LIHEAP payment itself entitles the household to the heating SUA, whose excess shelter cost is deductible without cap for a disabled household — net income is $0 and the full allotment of $3,596.04 applies." -us,scenario_080,snap,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model claimed the household facts lacked the income and composition details needed to compute SNAP and defaulted to zero. The facts fully specify a one-person Pennsylvania SNAP unit with $301.33/month of countable income and a $2,000 LIHEAP energy subsidy, which yields $0 net income and the maximum one-person allotment of $3,596.04." -us,scenario_080,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model treated the absence of earned income and of additional household members as missing information and returned zero. SNAP does not require earned income, and the stated facts define a complete one-person unit whose net income is $0 after the standard deduction and the SUA-based uncapped shelter deduction, giving the full allotment of $3,596.04." -us,scenario_080,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model computed countable income correctly at $300/month and applied the standard deduction, but explicitly assumed ""no shelter deduction"" and therefore subtracted 30% of $96 net income from the $298 maximum. The $2,000 SPM-unit energy subsidy is a LIHEAP payment that triggers the heating standard utility allowance, whose excess shelter cost is uncapped for a disabled household, so net income is $0 and no benefit reduction applies." -us,scenario_080,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model built countable income of $4,540 by including the $924 long-term capital gain, which is not SNAP countable income (the correct figure is $301.33/month), and stated ""no reported shelter deduction,"" leaving positive net income and a reduced allotment of $247.20/month. The LIHEAP-triggered SUA supplies an uncapped excess shelter deduction for this disabled household, so net income is $0 and the full $3,596.04 is payable." -us,scenario_080,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied only the one-person standard deduction to the assistance and dividend income and reduced the allotment by 30% of the remaining net income, arriving at $270/month. It omitted the heating standard utility allowance conferred by the $2,000 energy subsidy and the uncapped excess shelter deduction for a disabled household, which zero net income and leave the maximum allotment of $3,596.04." -us,scenario_080,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model disqualified the household solely on the $19,828 bank balance exceeding the elderly/disabled SNAP resource limit. Pennsylvania confers categorical eligibility through TANF non-cash services, which eliminates the resource test entirely, so the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,gpt-6-astra,llm_error,period_annualization,False,"The model reasoned correctly through categorical eligibility, the heating utility allowance from the energy subsidy, $0 net income, and the FY2026 $298 monthly maximum, then annualized as $298 × 12. Calendar 2026 carries $298 for only nine months; the October 2026 uprating to $304.68 for the final three months brings the total to $3,596.04." -us,scenario_080,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the absence of listed shelter, utility, and dependent-care expenses as producing a $0 benefit, inverting the SNAP formula: deductions reduce net income, and lower net income raises the benefit toward the maximum. The $2,000 SPM-unit energy subsidy is itself a LIHEAP payment that entitles the household to the heating standard utility allowance, and with $0 net income the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied PA BBCE, the SUA from the energy subsidy, and the unlimited excess shelter deduction for a disabled household to reach $0 net income and the full allotment, then valued that allotment at $292/month. The FY2026 one-person maximum is $298, rising to $304.68 in October 2026, so calendar 2026 totals $3,596.04." -us,scenario_080,snap,grok-4.6,llm_error,thresholds_rates,False,"The model derived the correct chain — BBCE with no asset test, HCSUA from the $2,000 energy subsidy, $0 net income — and then multiplied the FY2025 maximum of $292/month by 12. The correct calendar-2026 blend is nine months at the FY2026 $298 plus three months at $304.68, giving $3,596.04." -us,scenario_080,snap,grok-build-0.1,llm_error,asset_resource,False,"The model applied the $4,250 federal resource limit for households with a disabled member to the $19,828 bank balance and declared the household ineligible. Pennsylvania's broad-based categorical eligibility through TANF non-cash services waives the SNAP asset test, so the household qualifies and receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,inkling,llm_error,asset_resource,False,"The model disqualified the household on countable resources against a PA resource limit for disabled-member households and additionally invoked the absence of school-age children, which is irrelevant to SNAP. Pennsylvania's categorical eligibility eliminates the resource test, and this one-person unit's $0 net income yields the full allotment of $3,596.04." -us,scenario_080,snap,kimi-k2.6,llm_error,asset_resource,False,"The model applied the federal resource limit for households containing a disabled member to the $19,828 bank balance and found the household ineligible. That federal limit is superseded in Pennsylvania by broad-based categorical eligibility through TANF non-cash services, which removes the asset test, leaving the household with the full one-person allotment of $3,596.04." -us,scenario_080,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly waived the gross income test for the disabled member, applied PA BBCE to skip the asset test, excluded the energy subsidy from income, and reached $0 net income and the maximum allotment. It then used $292/month, the FY2025 rate; FY2026 is $298 and the October 2026 uprating gives $304.68, producing $3,596.04." -us,scenario_080,snap,minimax-m3,llm_error,asset_resource,False,"The model rejected eligibility on the combination of $19,828 in assets and $3,600 of assistance without running the income or deduction tests. Pennsylvania's TANF non-cash categorical eligibility waives the asset test, and $301.33/month of countable income against the standard deduction plus the uncapped SUA shelter deduction leaves $0 net income and the full allotment of $3,596.04." -us,scenario_080,snap,ox-alpha,llm_error,asset_resource,False,"The model recognized the household's very low income but failed it on the ~$4,500 SNAP asset limit for households containing a disabled member. Pennsylvania grants categorical eligibility through TANF non-cash services, which eliminates the resource test entirely, so the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model applied a $2,750 asset limit to the $19,828 bank balance and called the household ""categorically ineligible"" — inverting the doctrine, since categorical eligibility in Pennsylvania is precisely what removes the asset test. With no resource test and $0 net income after the standard and SUA-based shelter deductions, the household receives the full one-person allotment of $3,596.04." -us,scenario_080,snap,qwen3.8-max,llm_error,asset_resource,False,"The model denied the benefit on countable resources above ""the applicable limit"" and on the absence of earned income. SNAP requires no earned income, and Pennsylvania's TANF non-cash categorical eligibility waives the resource limit, so this one-person unit's $0 net income yields the full allotment of $3,596.04." +us,scenario_080,snap,claude-fable-5,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE, the energy-subsidy SUA and the uncapped disabled shelter deduction to reach zero net income. It then paid a made-up $293/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also counted the $924 capital gains, which are not SNAP income, though that did not change the zero net income." +us,scenario_080,snap,claude-fable-5.1,reference_engine_defect,categorical_eligibility,False,"Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum. But it reached zero net income the wrong way: it excluded the $300/month financial assistance, which is countable, and counted capital gains, instead of using the $857 heating SUA triggered by the energy subsidy." +us,scenario_080,snap,claude-haiku-4.5,reference_engine_defect,categorical_eligibility,False,"It denied SNAP under a $2,750 resource limit and counted the $3,600 financial assistance as a resource. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test. The frozen reference reaches zero net income after the $857 SUA and pays the maximum allotment, $3,596.04, using the engine's projected FY2027 $304.68 for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 all year, which on that path gives $298 × 12 = $3,576." +us,scenario_080,snap,claude-opus-4.7,reference_engine_defect,categorical_eligibility,False,"It paid a $293/month maximum for all 12 months instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also excluded the $300/month financial assistance, which is countable SNAP income, and counted capital gains, so it reached zero net income through the standard deduction alone rather than the energy-subsidy SUA." +us,scenario_080,snap,claude-opus-4.8,reference_engine_defect,categorical_eligibility,False,"It applied the roughly $4,500 elderly/disabled asset limit to the $19,828 bank balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash removes the asset test for this low-income household." +us,scenario_080,snap,claude-opus-5,reference_engine_defect,categorical_eligibility,False,"Its own reasoning says there is essentially no countable income, yet it submitted about $218.50/month instead of the maximum allotment. It never applied the $857 heating SUA from the energy subsidy that zeroes net income, and never used the FY2026 $298 maximum, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,claude-opus-5.5,reference_engine_defect,categorical_eligibility,False,"It paid $298 for all 12 months and knowingly left out any October 2026 cost-of-living update, which is what the release's SNAP convention (c_snap_hold_fy2026) does: October–December 2026 hold the FY2026 schedule because USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze. The frozen reference instead uses the engine's projected FY2027 maximum of $304.68 from October, giving 298×9 + 304.68×3 = $3,596.04. Its route to zero net income is wrong, though: $301.33 of countable income less the $209 standard deduction leaves $92.33, which the frozen reference zeroes only with the $857 heating SUA." +us,scenario_080,snap,claude-sonnet-4.6,reference_engine_defect,categorical_eligibility,False,"It correctly applied BBCE, the energy-subsidy SUA and the uncapped disabled shelter deduction to reach zero net income. It then used the FY2025 $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also wrongly counted the $924 capital gains as SNAP income." +us,scenario_080,snap,claude-sonnet-5,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 in liquid assets exceeds the $4,250 disabled-household limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test entirely." +us,scenario_080,snap,claude-sonnet-5.5,reference_engine_defect,categorical_eligibility,False,"It said there were no shelter costs, missing that the $2,000 energy subsidy triggers Pennsylvania's $857 heating SUA through heat-and-eat. With the uncapped disabled excess shelter deduction, that SUA zeroes net income. It also counted the $924 capital gains, which pushed its net income up to $169." +us,scenario_080,snap,deepseek-v4-flash-0731,reference_engine_defect,categorical_eligibility,False,"It stopped at the standard deduction and never applied the $857 heating SUA from the energy subsidy, which zeroes net income through the uncapped disabled shelter deduction. It also counted the $924 capital gains and used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December." +us,scenario_080,snap,deepseek-v4-pro,reference_engine_defect,categorical_eligibility,False,"It excluded the energy subsidy from income but missed that the subsidy triggers the $857 heating SUA under heat-and-eat, which zeroes net income. It also counted capital gains and used a stale $292 maximum and $204 standard deduction." +us,scenario_080,snap,deepseek-v4-pro-0813,reference_engine_defect,categorical_eligibility,False,"It applied no shelter deduction and missed the $857 heating SUA that the $2,000 energy subsidy triggers, which with the uncapped disabled excess shelter deduction reduces net income to $0. It also counted the $924 capital gains, and it used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December." +us,scenario_080,snap,deepseek-v4.1-flash,reference_engine_defect,categorical_eligibility,False,"It applied the disabled-household asset limit to the $19,828 balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test." +us,scenario_080,snap,gemini-3-flash-preview,reference_engine_defect,categorical_eligibility,False,"It correctly used the energy-subsidy SUA to reach zero net income, then paid the outdated $291/month one-person maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,gemini-3.1-flash-lite-preview,reference_engine_defect,categorical_eligibility,False,"It zeroed SNAP because of the household's assets, missing that Pennsylvania's broad-based categorical eligibility waives the asset test. The household has $301.33/month of countable income, and with the $857 SUA it gets the maximum allotment." +us,scenario_080,snap,gemini-3.1-pro-preview,reference_engine_defect,categorical_eligibility,False,"It correctly used the SUA from the energy subsidy to zero net income, then used the FY2025 $292/month maximum for all 12 months. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,gemini-3.5-flash,reference_engine_defect,categorical_eligibility,False,"It reached zero net income through BBCE and the energy-subsidy SUA, but used a stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also used an understated $198 standard deduction and a $600 SUA instead of $209 and $857, and it counted capital gains as income." +us,scenario_080,snap,gemini-3.5-flash-lite,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because of the household's assets and financial assistance. It missed that Pennsylvania's broad-based categorical eligibility removes the asset test, and that $301.33/month of income less the standard deduction and $857 SUA leaves zero net income and the maximum allotment." +us,scenario_080,snap,gemini-3.6-flash,reference_engine_defect,categorical_eligibility,False,"It correctly applied BBCE and the heating SUA from the energy subsidy to zero net income, then paid the FY2025 $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,gemini-3.7-flash,reference_engine_defect,categorical_eligibility,False,"It paid the FY2026 $298 maximum for all 12 months, which matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference instead uses the engine's projected FY2027 increase to $304.68 for October–December 2026." +us,scenario_080,snap,gemini-3.8-flash,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the energy-assistance heating SUA to zero net income, then paid the stale $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there was no answer to score against the $3,596.04 reference." +us,scenario_080,snap,glm-5.3,reference_engine_defect,categorical_eligibility,False,"It noted the energy subsidy but used it only for categorical eligibility. It never applied the $857 heating SUA the subsidy triggers, which with the uncapped disabled shelter deduction zeroes net income. Instead it subtracted 30% of about $170 from a $290 maximum, after also counting the $924 capital gains." +us,scenario_080,snap,gpt-5.4-mini,reference_engine_defect,categorical_eligibility,False,"It wrongly claimed the income and household details were insufficient and defaulted to $0. The facts fully define a one-person unit with $300/month financial assistance and $1.33 dividends, which under BBCE with the $857 SUA gets the maximum allotment: $3,596.04 in the frozen reference, which uses the engine's projected FY2027 $304.68 for October–December, or $298 × 12 = $3,576 under the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 all year." +us,scenario_080,snap,gpt-5.4-nano,reference_engine_defect,categorical_eligibility,False,"It treated the absence of earned income as grounds for zero SNAP. Unearned income does not bar SNAP: a one-person unit with $301.33/month of unearned income qualifies under Pennsylvania BBCE, and the $857 SUA zeroes net income, giving the maximum allotment." +us,scenario_080,snap,gpt-5.5,reference_engine_defect,categorical_eligibility,False,"It applied no shelter deduction and computed $298 minus 30% of $96, about $269/month. The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 × 12 = $3,240, $12 above the model because it used $96 of net income rather than $92. Its $298 maximum for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum." +us,scenario_080,snap,gpt-5.6-luna,reference_engine_defect,categorical_eligibility,False,"It computed $3,576 minus 30% of ($4,540 − $2,508) with no shelter deduction, so it never applied the $857 heating SUA from the energy subsidy, which zeroes net income. It also counted the $924 capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum." +us,scenario_080,snap,gpt-5.6-sol,reference_engine_defect,categorical_eligibility,False,"Its $270/month equals $298 minus 30% of the $92.33 left after the standard deduction, which is the result with no shelter deduction. Its $3,240 equals the exclusion's corrected value, which withholds the heat-and-eat SUA from a household without an elderly or disabled member; the model reached it by applying no SUA although it called the household disabled, a reading under which the record says the energy subsidy would still confer the SUA. Holding $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026). The frozen reference instead applies the $857 heating SUA the energy subsidy triggers, which zeroes net income, and uses the engine's projected FY2027 $304.68 for October–December." +us,scenario_080,snap,gpt-5.6-terra,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 bank balance exceeds the disabled-household resource limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test." +us,scenario_080,snap,gpt-6-astra,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the heating SUA to zero net income, then paid $298 for all 12 months. That matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December 2026; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum." +us,scenario_080,snap,gpt-6-luna,reference_engine_defect,categorical_eligibility,False,"It applied only the standard deduction and paid $269/month. The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 a month ($3,240), $1 above the model's $269. Keeping one maximum for all 12 months is consistent with the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum." +us,scenario_080,snap,gpt-6-sol,reference_engine_defect,categorical_eligibility,False,"It counted the $940 of investment income, including $924 of capital gains that are not SNAP income, and applied only the $209 standard deduction. It never applied the $857 heating SUA from the energy subsidy, which zeroes net income and yields the maximum allotment." +us,scenario_080,snap,gpt-6.1-sol,reference_engine_defect,categorical_eligibility,False,"It subtracted 30% of $2,032 net income from $3,576 with no shelter deduction, so it never applied the $857 heating SUA from the energy subsidy, which zeroes net income. It also counted capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum." +us,scenario_080,snap,grok-4.3,reference_engine_defect,categorical_eligibility,False,"It wrongly treated the lack of shelter and dependent expenses as making SNAP zero, when missing expenses only reduce deductions. It also missed that the $2,000 energy subsidy triggers the $857 heating SUA, which zeroes net income and gives the maximum allotment." +us,scenario_080,snap,grok-4.5,reference_engine_defect,categorical_eligibility,False,"It correctly used BBCE and the energy-subsidy SUA with the uncapped disabled shelter deduction, then paid the stale $292/month maximum. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,grok-4.6,reference_engine_defect,categorical_eligibility,False,"It correctly reached zero net income through BBCE and the heating SUA, then multiplied the FY2025 $292 maximum by 12. The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum." +us,scenario_080,snap,grok-4.7,reference_engine_defect,categorical_eligibility,False,"Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum. But it counted only the $940 of investment income, wrongly excluding the $300/month financial assistance and wrongly including capital gains, instead of zeroing net income through the $857 SUA." +us,scenario_080,snap,grok-build-0.1,reference_engine_defect,categorical_eligibility,False,"It denied SNAP under the $4,250 disabled-household asset limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test." +us,scenario_080,snap,inkling,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the bank balance plus vehicle value exceed the disabled-household resource limit. It missed that Pennsylvania's broad-based categorical eligibility removes the resource test, and it wrongly treated school-age children as relevant to SNAP." +us,scenario_080,snap,kimi-k2.6,reference_engine_defect,categorical_eligibility,False,"It applied the federal disabled-household resource limit to the $19,828 balance and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test." +us,scenario_080,snap,kimi-k3,reference_engine_defect,categorical_eligibility,False,"It paid the stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December. It also reached zero net income by treating investment income as the only countable income, missing the $300/month financial assistance and the $857 heating SUA from the energy subsidy that actually zeroes net income." +us,scenario_080,snap,minimax-m3,reference_engine_defect,categorical_eligibility,False,"It treated the $19,828 bank balance plus the $3,600 financial assistance as disqualifying resources. It missed that Pennsylvania's broad-based categorical eligibility waives the SNAP asset test, and that the assistance is income, not a resource." +us,scenario_080,snap,ox-alpha,reference_engine_defect,categorical_eligibility,False,"It denied SNAP because the $19,828 balance exceeds the roughly $4,500 disabled-household asset limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash removes the asset test." +us,scenario_080,snap,qwen-3.7-max,reference_engine_defect,categorical_eligibility,False,"It applied a $2,750 asset limit, which is also the wrong limit for a disabled household, and denied SNAP. It missed that Pennsylvania's broad-based categorical eligibility waives the asset test entirely." +us,scenario_080,snap,qwen3.8-max,reference_engine_defect,categorical_eligibility,False,It denied SNAP because resources exceed the limit. It missed that Pennsylvania's broad-based categorical eligibility through TANF non-cash waives the asset test for this one-person unit with $301.33/month of income. us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, violating the required structured-output contract." us,scenario_080,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the prompt's general disability fact as establishing SSI disability, despite the applicable engine input is_ssi_aged_blind_disabled=False. Because the 41-year-old is neither SSI-aged, blind, nor SSI-disabled, the federal base-rate and unearned-income calculation does not apply and SSI is $0." us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived 100% Pennsylvania Tax Forgiveness and explicitly calculated that it reduces the $28.86 preliminary tax to $0, but then submitted the unreduced $28.86. It failed to carry its own nonrefundable-credit calculation into the output." @@ -5976,117 +6549,134 @@ us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_erro us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable output for state_refundable_credits, violating the required submission contract." us,scenario_080,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model misclassified Pennsylvania Schedule SP tax forgiveness as a refundable credit that pays out the $28.86 liability. Schedule SP is nonrefundable and only reduces state income tax, so it contributes $0 to state_refundable_credits." us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It explicitly set the Massachusetts income tax component of SALT to zero (""MA income tax withholding treated as 0""), itemizing $22,543 instead of $31,028.65 and omitting the $8,484.41 of deductible MA income tax, which pushed taxable income to $151,551 versus the actual $143,059.47. It then abandoned its own bracket arithmetic of $28,759 and reported $29,385 with no derivation." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the law right — itemize under the $40,400 SALT cap, 2026 brackets, $1 QBI — but estimated the deductible Massachusetts income tax at about $8,230 instead of $8,484.41 and added the $6 of REIT/PTP income into AGI, which PolicyEngine treats as a QBI input only. Those two slips raised taxable income by $261 over $143,059.47, and at 24% that is the entire $62.73 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the 2024 standard deduction of $14,600 and asserted that $16,900 of mortgage interest was ""below the standard deduction threshold for itemization,"" discarding $31,028.65 of itemized deductions, and it also deducted the $1,782 of unreimbursed employee business expenses above the line even though miscellaneous itemized deductions remain suspended. Its final $29,825 is not the $30,956.92 its own 2024-bracket arithmetic produced, so the submitted number is untethered from its stated computation." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000, a cap that does not apply for 2026 where the limit is $40,400, so it itemized $26,900 rather than $31,028.65 and lost $4,127.41 of deduction. It then reported $31,369 after its own bracket walk produced $27,887, a $3,482 jump it never justified." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap instead of the 2026 $40,400 cap, itemizing $26,900 against the actual $31,028.65, and priced the result on 2025 brackets ($11,925/$48,475/$103,350) rather than 2026's ($12,400/$50,400/$105,700). It computed $28,172 and then submitted $31,379, a $3,207 unexplained adjustment." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It itemized only $22,543 of mortgage interest plus real estate tax, never counting the $8,484.41 of Massachusetts income tax that is deductible in full under the $40,400 SALT cap, leaving taxable income at $151,544 instead of $143,059.47. It also discarded its own $28,968 bracket result and submitted $31,650." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated real estate tax as the only SALT item and never added state income tax to the SALT total, itemizing $22,543 rather than $31,028.65 and omitting $8,484.41 of deductible MA income tax. It compounded that with 2025 bracket breakpoints ($11,925/$48,475/$103,350) instead of 2026's, producing $29,219 on a taxable income $8,492 too high." -us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000 rather than the 2026 limit of $40,400, itemizing $26,900 instead of $31,028.65, and then wrote ""tax ≈ 33,700"" without any bracket walk. Its own taxable income of $147,194 yields $27,925 under the 2026 single schedule, so the submitted $33,700 is $5,775 above what it claimed to compute." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It correctly identified that state income tax belongs in SALT but capped the combined total at $10,000, a cap superseded for 2026 by the $40,400 limit, so it deducted $26,900 rather than $31,028.65 and set taxable income at $147,194 instead of $143,059.47. The $4,135 of lost deduction at the 24% marginal rate is essentially the entire $1,047 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It ran a TCJA-sunset 2026: a $5,400 personal exemption and pre-2018 10/15/25/28 brackets, neither of which applies since the 10/12/22/24 schedule and the repeal of personal exemptions are permanent. It also subtracted the $7,389 employer-sponsored insurance premium from the $175,002 of wages, which already exclude it, and left the $8,484.41 of MA income tax out of SALT entirely." -us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA single brackets plus a $5,312 personal exemption for 2026, when the correct schedule is 10/12/22/24 with no personal exemption, and it understated SALT at $12,754.55 against the actual $14,127.41 ($5,643 property tax plus $8,484.41 MA income tax). Its explanation states $29,794.41 while the submitted value is $27,594.14, so the answer does not match its own derivation." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its deduction work was nearly right — $30,823.10 of itemized deductions including state income tax — but it subtracted a $5,050 personal exemption and applied restored 10/15/25/28 rates, both of which are gone for 2026 under the permanent TCJA schedule. Applying the actual 2026 brackets to even its own $138,220.90 taxable income gives $25,771, not the $29,982.35 its sunset rates produced." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It took a $15,000 standard deduction and never itemized, even though $16,900 of mortgage interest alone exceeds the $16,100 2026 single standard deduction and the full itemized total is $31,028.65 once the $5,643 property tax and $8,484.41 MA income tax are added. Dropping $16,029 of deduction put taxable income near $159,094 instead of $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed with ""TCJA-expired tax brackets for 2026"" and a personal exemption, but the 10/12/22/24 schedule with no personal exemption governs 2026. It also used AGI of $166,705, which subtracts the $7,389 employer-sponsored insurance premium a second time from wages that already exclude it; the correct AGI is $174,088.12." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed ""TCJA provisions expired"" for 2026, applying a $5,300 personal exemption and pre-2018 rates instead of the permanent 10/12/22/24 schedule with no exemption. It also cut wages to $167,613 by removing the $7,389 ESI premium already excluded from the stated $175,002, and itemized only $22,543 by leaving the $8,484.41 of MA income tax out of SALT." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of $169,008 implies roughly $5,000 of total deductions against $174,088.12 of AGI, whereas itemizing mortgage interest and the full uncapped $14,127.41 of SALT gives $31,028.65 and taxable income of $143,059.47. Its own $169,008 base yields $33,160 under the 2026 single brackets, so the submitted $24,197 is $8,963 below even its stated derivation." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It made two offsetting errors: it removed the $7,389 ESI premium from wages that already exclude it, cutting AGI to $166,694 instead of $174,088.12, and it itemized only $22,543 by counting real estate taxes as the sole SALT item while omitting $8,484.41 of MA income tax. Those errors partially cancel, leaving taxable income of $144,151 and $27,858 rather than $143,059.47 and $26,932.27." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The correct derivation is AGI $174,088.12 less $31,028.65 of itemized deductions, giving taxable income $143,059.47 and tax $26,932.27 under the 2026 brackets. Its $24,907 corresponds to taxable income of about $134,621 — roughly $8,438 low — the signature of subtracting a TCJA-sunset personal exemption on top of the itemized total; it supplied no bracket arithmetic to support any other reading." -us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It nailed AGI at $174,088 and even identified $8,238.40 of MA income tax, then capped SALT at $10,000 — the repealed TCJA cap — instead of the 2026 $40,400 limit, deducting $26,900 rather than $31,028.65. That single cap error raised taxable income to $147,187 and produced $28,050 instead of $26,932.27." -us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly deducted uncapped SALT and reached $31,248 of itemized deductions, but then subtracted a $5,150 personal exemption and taxed the result at restored 10/15/25/28 brackets, none of which exist for 2026 under the permanent TCJA schedule. It also applied pre-TCJA floors (10% medical, 2% miscellaneous) rather than the 7.5% floor and the suspension of miscellaneous deductions." -us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It estimated the deductible Massachusetts income tax at $8,388.70 rather than $8,484.41 and added the $6 of REIT/PTP income into AGI, leaving taxable income $102 above $143,059.47. It then mis-added its own bracket amounts — $1,240 + $4,560 + $12,166 + $8,990.66 is $26,956.66, not the $26,916.66 it submitted." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It capped the SALT deduction at $10,000 for 2026, when the cap is $40,400 and the full $14,127.41 of real estate and MA income tax is deductible, cutting itemized deductions from $31,028.65 to $26,900. Even that base yields $27,925 under the 2026 single brackets, so its submitted $22,450 is $5,475 below its own described computation." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Taxing the full $174,088.12 of AGI with no deduction at all gives only $34,379 under the 2026 single schedule, so its $41,495 exceeds the ceiling of any deduction-based derivation and cannot come from the $31,028.65 of itemized deductions it claims to have applied. It supplied no brackets, rates, or taxable income figure to connect the number to the household's facts." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It reached the right itemized total of about $30,933 and then subtracted an ""estimated 2026 personal exemption,"" which does not exist, and taxed the remainder at sunset 10/15/25/28 rates instead of 10/12/22/24. With the correct 2026 schedule and no exemption, its own deduction figure gives about $26,956, not $29,417.08." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It reduced wages by ""the listed employer-sponsored insurance premium"" of $7,389, which is already excluded from the $175,002 of stated wages, so its AGI fell about $7,389 below the correct $174,088.12. Its $24,920 implies taxable income near $134,675 against the actual $143,059.47, an $8,384 understatement that tracks that double subtraction." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the right structure — itemize mortgage interest plus uncapped Massachusetts income and property tax, 2026 brackets, no nonrefundable credits — but its taxable income ran $251 above $143,059.47, the arithmetic signature of adding the $6 REIT/PTP amount to AGI and estimating the deductible MA income tax roughly $245 short of $8,484.41. At the 24% marginal rate that is the entire $60.32 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its method was correct — itemize $16,900 of mortgage interest plus the uncapped Massachusetts income and property tax, land in the 24% bracket, apply no nonrefundable credits — but the deductible Massachusetts income tax it used fell about $141 short of $8,484.41. That $141 of missing SALT at 24% is the whole $33.94 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its only error is the size of the state income tax inside SALT: it used $30,773.10 of itemized deductions, implying $8,230.10 of Massachusetts income tax against the actual $8,484.41, so total itemized deductions came in $255.55 low. That leaves taxable income at $143,313.70 instead of $143,059.47, and at 24% produces exactly the $61.02 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of ""about $151,500"" corresponds to itemizing only $22,543 of mortgage interest and real estate tax, omitting the $8,484.41 of Massachusetts income tax that is fully deductible under the 2026 SALT cap of $40,400. The correct taxable income is $143,059.47, and the $8,492 of missing deduction at 24% accounts for the $2,018 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It handled SALT correctly, deducting uncapped property and Massachusetts income tax for $30,939 of itemized deductions, but then assumed ""2026 current-law TCJA sunset,"" subtracting a $5,447 personal exemption and applying restored 15/25/28 rates. The 2026 schedule is 10/12/22/24 with no personal exemption, so its taxable income of $137,708 and its rate table are both wrong." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It reached essentially the right itemized total of $30,932 including uncapped Massachusetts income tax, then applied a ""restored personal exemption"" of $5,300 and the pre-TCJA 10/15/25/28 brackets, neither of which applies for 2026. Removing the exemption and using the 2026 schedule on its own numbers gives about $26,956, not $29,479." -us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It made both structural errors at once: it itemized only $22,543 of mortgage interest and real estate tax, omitting $8,484.41 of deductible Massachusetts income tax, and it taxed the result on ""pre-TCJA reverted brackets"" of 10/15/25/28 with breakpoints at $12,491/$50,834/$123,100. The 2026 rates are 10/12/22/24 breaking at $12,400/$50,400/$105,700, and the correct taxable income is $143,059.47." -us,scenario_081,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It set up the calculation correctly but estimated the Massachusetts income tax inside SALT at about $8,335 rather than $8,484.41 and added the $6 REIT/PTP amount to AGI, putting taxable income near $143,215 against $143,059.47. Its own numbers give $26,970 under the 2026 brackets, and it rounded that up to the submitted $27,056." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so no substantive computation exists to evaluate. The required derivation is AGI $174,088.12 less $31,028.65 of itemized deductions, giving taxable income $143,059.47 and tax $26,932.27." -us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed AGI exactly right at $174,088.00 and applied the correct 2026 brackets and the $1.20 REIT QBI deduction, but itemized only $22,543 from mortgage interest and real estate taxes, leaving out the $8,484.41 of Massachusetts income tax that is deductible in full under the $40,400 SALT cap. That single omission raised taxable income to $151,543.80 and produced the entire $2,036 overstatement." -us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 with the explanation ""value=0"" for a single filer with $175,002 of wages whose 2026 taxable income after $31,028.65 of itemized deductions is $143,059.47, well into the 24% bracket. A zero liability requires deductions exceeding AGI, which no fact in the household supports." -us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the correct 2026 structure — itemize under the $40,400 SALT cap, brackets breaking at $12,400/$50,400/$105,700 — but overstated the Massachusetts income tax inside SALT at $8,538 versus the actual $8,484.41, giving $31,081 of itemized deductions instead of $31,028.65. Net of adding the $6 REIT/PTP amount to AGI, taxable income landed $47.67 low, which at 24% is the $11.27 shortfall." -us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It claimed the $15,000 standard deduction was larger than $22,543 of itemized deductions — false on its own figures, and the true itemized total is $31,028.65 once the $8,484.41 of Massachusetts income tax joins the $5,643 property tax under the $40,400 SALT cap. It then computed $31,281.32 from its own bracket walk and submitted $39,827.88, a figure that matches nothing in its derivation." -us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It took the standard deduction on the claim that itemized deductions of $22,543 were lower, but $22,543 already exceeds the $16,100 2026 single standard deduction and the actual itemized total is $31,028.65 including $8,484.41 of deductible Massachusetts income tax. Taxable income of $160,402 instead of $143,059.47 is a $17,343 overstatement, and even that base yields $31,094 under the 2026 brackets rather than the $30,041 submitted." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted only the $5,643 real estate tax as SALT and set Massachusetts income tax to zero. The $8,484.41 of MA income tax is deductible under the 2026 $40,400 cap, so its itemized total fell from $31,027 to $22,543 and taxable income was about $8,490 too high. It then used a wrong $17,754.75 base for the 24% bracket (the correct figure is $17,966) and added an unexplained ~$626 on top, giving $29,385." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Its method is correct, but it put deductible Massachusetts income tax at about $8,230 instead of $8,484.41. It also added the $6 REIT/PTP income on top of dividends, making AGI $174,094. Together these overstate taxable income by about $261 (143,320 vs 143,059.47), which is $63 of tax at 24%." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It took a 2024-level $14,600 standard deduction and claimed the $16,900 of mortgage interest was below that threshold. In fact mortgage interest plus $14,127.41 of SALT gives $31,027 of itemized deductions, far above the $16,100 2026 standard deduction. It also subtracted the $1,782 of unreimbursed employee business expenses, which are nondeductible, used 2024 brackets, and then submitted $29,825, which differs from its own $30,956.92 result." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000, but OBBBA sets the 2026 cap at $40,400, so the full $14,127.41 (real estate tax plus MA income tax) is deductible. That cut about $4,127 from itemized deductions and pushed taxable income to $147,187. It then dropped its own $27,887 bracket result and submitted an unsupported $31,369." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It applied the expired $10,000 SALT cap instead of the 2026 $40,400 cap, losing $4,127 of deductible state and property taxes. It also used 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 ones (12,400/50,400/105,700). Its own computation reached $28,172, but it submitted an unexplained $31,379." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($22,543) and left out the $8,484.41 of Massachusetts income tax, which is deductible SALT under the $40,400 cap. That put taxable income at $151,544 instead of $143,059.47. Its correct 2026 bracket math gave $28,968, which it then inflated without basis to $31,650." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It put Massachusetts income tax at $8,238.70, which is 5% of income after additional FICA and rent subtractions. The engine deducts $8,484.41, which is 5% of $169,688.12. It also counted dividends as $116 by adding the $6 REIT income. Together these raised taxable income by $251.63 to $143,311.10 and tax by $60.39." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It counted only the $5,643 real estate tax toward SALT and left out the $8,484.41 of Massachusetts income tax, so it itemized $22,543 instead of $31,027 and reached taxable income of $151,551. It then applied 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 ones (12,400/50,400/105,700), giving $29,219." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 $40,400 cap, understating itemized deductions by $4,127. On its own $147,194 of taxable income, the 2026 single brackets give about $27,925. Its $33,700 does not follow from any bracket computation it showed." +us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Its approach is correct, but it put Massachusetts income tax at about $8,284 instead of the $8,484.41 in the engine's SALT deduction. It also added the $6 REIT income to AGI. Taxable income came out about $207 too high (143,266 vs 143,059.47), which is about $50 of extra tax." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied the $10,000 SALT cap even though OBBBA raises the 2026 cap to $40,400. That dropped about $4,127 of deductible Massachusetts income and property tax and left taxable income at $147,194 instead of $143,059.47. It also overshot the correct 2026 bracket result on that income ($27,925) by about $54." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It modeled a TCJA sunset: pre-TCJA 10/15/25/28% brackets and a $5,400 personal exemption. OBBBA made the TCJA rate schedule and the $0 exemption permanent for 2026. It also subtracted the $7,389 ESI premium from the $175,002 wages, which the engine counts in full, and left the $8,484.41 of Massachusetts income tax out of SALT." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law, with a $5,312 personal exemption and the 10/15/25/28% schedule, even though OBBBA made the TCJA brackets and the zero exemption permanent for 2026. It also understated SALT at $12,754.55 instead of $14,127.41. Its submitted $27,594.14 contradicts the $29,794.41 in its own explanation." +us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Its structure is correct, but it put deductible Massachusetts income tax at $7,539.70, which is $944.71 below the $8,484.41 in the engine's SALT deduction. It also added the $6 REIT income to AGI. Taxable income came out $950.63 too high ($144,010.10), which adds $228.15 of tax at 24%." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA had expired for 2026, so it subtracted a $5,050 personal exemption and taxed the income under 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 personal exemption permanent. Its itemized deductions ($30,823) were close; the error is the rate schedule and the exemption." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing $16,900 of mortgage interest plus $14,127.41 of SALT ($31,027 total), overstating taxable income by about $16,000. Even on its own $159,094 of taxable income, the 2026 brackets give $30,781, not the $31,317 it submitted." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-expired brackets and a personal exemption, although OBBBA made the TCJA 10/12/22/24% schedule and the zero exemption permanent for 2026. It also reduced AGI to $166,705 by subtracting the $7,389 ESI premium from wages, while the engine counts the full $175,002 as employment income." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed TCJA had expired, with a $5,300 personal exemption and pre-TCJA brackets, although OBBBA keeps the 10/12/22/24% schedule and the $0 exemption for 2026. It also subtracted the $7,389 ESI premium from the $175,002 wages and deducted only the real estate tax, omitting the $8,484.41 of Massachusetts income tax from SALT." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $169,008 of taxable income is only about $5,000 below AGI, so it left out most of the $31,027 of itemized mortgage interest and SALT that bring taxable income to $143,059.47. Its $24,197 also does not follow from 2026 single brackets on $169,008, which give about $33,160." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,389 ESI premium from the $175,002 wages, which the engine counts in full. It also dropped the $8,484.41 of Massachusetts income tax from SALT, itemizing only $22,543. These two errors offset each other into taxable income of $144,151, on which the 2026 brackets give $27,194; it then misapplied the brackets and reported $27,858." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The correct derivation is taxable income of $143,059.47 taxed at 24% above $105,700, which yields $26,932.27. Its $24,907 implies taxable income of about $134,620, which is roughly $8,440 of excess deductions. That is consistent with subtracting the $7,389 ESI premium from wages and over-deducting on top of the correct $31,027 of itemized deductions." +us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It correctly estimated $8,238 of Massachusetts income tax, but then capped total SALT at $10,000. OBBBA's 2026 cap is $40,400, so the full $14,127.41 is deductible. That understated itemized deductions by about $4,127 and gave taxable income of $147,187, and it then overshot the 2026 bracket tax on that income ($27,923) by $127." +us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It treated 2026 as post-TCJA-sunset law, subtracting a $5,150 personal exemption and applying 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 exemption permanent. Its uncapped SALT treatment was close in effect, since the actual $40,400 cap does not bind, but the wrong rate schedule raised tax to $29,110." +us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It put Massachusetts income tax at $8,388.70 instead of $8,484.41 and added the $6 REIT income to AGI, leaving taxable income at $143,161.10. On that figure its own bracket components sum to $26,956.66. It mis-added them and reported $26,916.66, $40 below its own components." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 $40,400 cap, cutting $4,127 of deductible Massachusetts income and property tax. With that cap, taxable income is about $147,187 and 2026 tax is about $27,923. Its $22,450 is roughly $5,500 below even that, so it also misapplied the single rate schedule." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $41,495 exceeds the $34,379 that the 2026 single brackets produce on the full $174,088 AGI with no deduction at all. So it neither subtracted the $31,027 of itemized deductions nor applied the 10/12/22/24% schedule, whose 24% bracket runs to $201,775." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its itemized deductions (about $30,933) were close to the correct $31,027. But it added a 2026 personal exemption and taxed the result under the pre-TCJA rate schedule, while OBBBA permanently sets the exemption to $0 and keeps the 10/12/22/24% brackets. That pushed tax to $29,417 instead of $26,932." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $7,389 employer-sponsored insurance premium, but the engine treats the full $175,002 as taxable employment income. That removes about $7,400 of taxable income and roughly $1,770–$2,000 of tax at 24%, producing $24,920 instead of $26,932.27." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its itemization, 2026 brackets and QBI treatment match the engine, but it estimated deductible Massachusetts income tax about $250 below the $8,484.41 in the SALT deduction. That overstated taxable income by about $251 and tax by $60.32." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its approach is correct: itemize mortgage interest plus MA income and property tax, then apply the 2026 24% bracket. It understated deductible Massachusetts income tax by about $141 relative to the $8,484.41 in the engine's SALT figure, which adds $33.94 of tax." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Its $30,773.10 of itemized deductions implies Massachusetts income tax of $8,230.10, which is 5% of $164,602. The engine's SALT deduction uses $8,484.41, which is 5% of $169,688.12, so the model took about $5,086 of extra Massachusetts subtractions. That understated itemized deductions by $254.31 and overstated tax by $61.02." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Its method is correct, but its estimated taxable income of about $143,260 is $200 above the correct $143,059.47. That comes from putting deductible Massachusetts income tax about $200 below the $8,484.41 in the engine's $14,127.41 SALT deduction, and it adds $48 of tax." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It deducted $8,330.10 of Massachusetts income tax instead of the $8,484.41 in the engine's SALT deduction, subtracting about $3,086 more Massachusetts income than the engine does. Taxable income came out $154.23 too high ($143,213.70), which adds $37.02 of tax." +us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Its $30,773.10 of itemized deductions implies Massachusetts income tax of $8,230.10, which is 5% of $164,602, instead of the engine's $8,484.41, which is 5% of $169,688.12. That understated SALT by $254.31 and raised taxable income to $143,313.70, overstating tax by $61.02." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $151,500 is roughly $8,440 above the correct $143,059.47. That matches itemizing only mortgage interest and real estate tax and leaving the $8,484.41 of Massachusetts income tax out of SALT, which is deductible under the 2026 $40,400 cap." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed a 2026 TCJA sunset, subtracting a $5,447 personal exemption and applying restored 10/15/25/28% brackets. OBBBA made the TCJA 10/12/22/24% schedule and the $0 exemption permanent. Its itemized deductions ($30,939) were close to the correct $31,027, so the error is the wrong rate schedule and the phantom exemption." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It modeled a TCJA sunset, with a restored $5,300 personal exemption and 10/15/25/28% brackets, although OBBBA made the TCJA rates and the zero exemption permanent for 2026. Its $30,932 of itemized deductions was nearly correct, but the pre-TCJA schedule pushed tax to $29,479." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law, with a $5,350 personal exemption and restored 10/15/25/28% brackets, although OBBBA keeps the 10/12/22/24% schedule and the $0 exemption for 2026. Its $30,827.70 of itemized deductions was close to the correct $31,027.41, so the rate-schedule error drives its $29,389." +us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied reverted pre-TCJA brackets (10% to $12,491, 15%, 25%, 28%) instead of the permanent OBBBA 10/12/22/24% schedule. It also itemized only $22,543, omitting the $8,484.41 of Massachusetts income tax from SALT, so taxable income was $151,551 instead of $143,059.47." +us,scenario_081,federal_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It estimated Massachusetts income tax at about $8,335 instead of $8,484.41, giving taxable income of about $143,215. On that figure the 2026 brackets give $26,970, and it then misapplied the 24% bracket, adding about $86 more to reach $27,056." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It itemized only $16,900 of mortgage interest and $5,643 of real estate tax, leaving the $8,484.41 of Massachusetts income tax out of SALT even though the 2026 $40,400 cap allows it. Taxable income was $151,543.80 instead of $143,059.47, and its otherwise-correct 2026 bracket math gave $28,968.51." +us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 with no derivation. A single filer with $174,088 of AGI and $143,059.47 of taxable income owes $26,932.27 under the 2026 brackets, with no nonrefundable credit to offset it." +us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It estimated Massachusetts income tax at $8,538, which is 5% of income without netting the $1,080 short-term loss. The engine's SALT deduction uses $8,484.41, which is net of that loss. The excess $53.59 of SALT, partly offset by adding $6 of REIT income to AGI, left taxable income $47 low at about $143,012, understating tax by about $11." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It claimed the $22,543 of itemized deductions was lower than a $15,000 standard deduction, which is false, and it also omitted the $8,484.41 of Massachusetts income tax. The correct itemized total of $31,027 far exceeds the $16,100 2026 standard deduction. It then used non-2026 brackets and submitted $39,827.88, which contradicts its own $31,281.32 calculation." +us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It chose the standard deduction on the claim that itemized deductions were only $22,543. It omitted the $8,484.41 of Massachusetts income tax, and with it the itemized total is $31,027, well above the $16,100 2026 standard deduction. That left taxable income at $160,402, and even there the 2026 brackets give $31,094, not the $30,041 it reported." us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model computed every component correctly in its own text — SS $10,850.12, Medicare $2,537.53, and MA PFML at 0.46% = $805.01 — then discarded the PFML component on the false premise that PolicyEngine omits state payroll taxes, and submitted $13,268.70, a figure matching neither its FICA-only subtotal of $13,387.65 nor the correct $14,192.66. MA's Paid Family and Medical Leave employee contribution is a mandatory employee-side state payroll tax and is included in spm_unit_payroll_tax." -us,scenario_081,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model computed federal FICA correctly ($10,850.12 + $2,537.53 = $13,387.65) but never applied the Massachusetts Paid Family and Medical Leave employee contribution of 0.46% on $175,002 = $805.01, which is a mandatory employee state payroll tax counted in the requested output. Its answer is short by exactly that PFML component." -us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model asserted that Massachusetts has no mandatory employee payroll tax distinct from income tax, omitting the MA PFML employee contribution of 0.46% × $175,002 = $805.01, and it also churned through a stale $168,600 Social Security wage base before reverting to full-wage FICA. Its submitted $13,383.41 is $4.24 below even its own 7.65% × $175,002 arithmetic, on top of the missing $805.01 state component." -us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model reached the correct FICA subtotal of $13,387.65 but then set the MA PFML employee contribution to approximately $0 after confusing the employer-size split, dropping the $805.01 that 0.46% of $175,002 produces. It then submitted $12,321.73, a number disconnected from every computation it performed, including its own stated $13,387.65." -us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model identified the MA PFML employee contribution as applicable but zeroed it out for lack of a remembered rate, discarding the $805.01 that the 0.46% employee rate yields on $175,002. It then submitted $12,717.62, $670.03 below its own stated FICA-only total of $13,387.65." -us,scenario_081,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model set the Massachusetts PFML employee contribution to approximately zero, omitting the mandatory 0.46% × $175,002 = $805.01 employee contribution that PolicyEngine adds to FICA. It also submitted $13,385 against its own stated FICA subtotal of $13,388, compounding the $805.01 shortfall with a $3 rounding discrepancy." -us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model derived the exact reference figure of $14,192.66 using the 0.46% MA PFML employee rate, then overrode it with a constructed 0.604% employee share (full 0.42% medical plus 40% of the 0.46% family portion) for 25+ employee employers, producing $1,057.01 of PFML instead of $805.01. The MA PFML employee contribution rate applied to these wages is 0.46%, so the substitution added $252.00 of tax that does not exist." -us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model declared that Massachusetts levies no mandatory employee-side state payroll tax and classified PFML as employer-side and therefore excluded, dropping the $805.01 employee contribution (0.46% × $175,002). It then submitted $12,318.65, $1,069.00 below its own stated FICA total of $13,387.65." -us,scenario_081,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model stated flatly that MA has no employee state payroll tax and stopped at FICA of $13,387.65, omitting the Massachusetts PFML employee contribution of 0.46% × $175,002 = $805.01. Its shortfall equals that omitted component exactly." -us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model correctly included a MA PFML component but applied a 0.38% employee rate instead of the 0.46% rate, producing $665.01 rather than $805.01 of paid-leave contribution. That $140.00 rate error, plus rounding the total to whole dollars, accounts for the entire miss." -us,scenario_081,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model used a 0.375% MA PFML employee rate, yielding $656.26 instead of the $805.01 produced by the 0.46% employee rate on $175,002. It compounded that with a submission of $13,593.41 that is $450.50 below the $14,043.91 total its own explanation computed." -us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model got both FICA components exactly right but applied a 0.318% MA PFML employee rate for $556.51 instead of the 0.46% rate that yields $805.01 on $175,002. The $248.50 PFML rate shortfall is the entire error." -us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model described only 6.2% Social Security and 1.45% Medicare, omitting the MA PFML employee contribution of $805.01, and then submitted $11,843 — $1,544.65 below the $13,387.65 that its own described 7.65% calculation on $175,002 produces. Both the omitted state component and the unexplained arithmetic shortfall are errors." -us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model computed FICA on $167,613 — gross wages of $175,002 less the $7,389 employer-sponsored insurance premium — treating the ESI premium as a Section 125 pre-tax exclusion from the Social Security and Medicare wage base. PolicyEngine assesses employee Social Security and Medicare on the full $175,002 of employment income, so excluding $7,389 understated FICA by $565.26, and its estimated PFML fell short of the $805.01 that 0.46% produces." -us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model applied both FICA rates to a reduced base of $167,613, subtracting the $7,389 ESI premium from taxable wages, and then applied a 0.24% PFML rate for $402. Employee Social Security and Medicare apply to the full $175,002 of employment income, and the MA PFML employee rate is 0.46% ($805.01), so both the base reduction and the halved state rate drove the $968.66 shortfall." -us,scenario_081,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model reported Social Security of $10,830 rather than the $10,850.12 that 6.2% × $175,002 produces, and used a PFML contribution of $765.01 instead of the $805.01 that the 0.46% employee rate yields. Those two shortfalls, $20.12 and $40.00, plus rounding to whole dollars, account for its $59.66 miss." -us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model explicitly reduced the FICA base to $167,613 by subtracting the $7,389 employer-sponsored insurance premium as a pre-tax exclusion, and omitted the MA PFML employee contribution entirely. Employee Social Security and Medicare apply to the full $175,002 of employment income, so the base cut cost $565.25 and the missing 0.46% state paid-leave contribution cost another $805.01." -us,scenario_081,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"The model named the right three components but showed no arithmetic, and its $13,444 total is the correct FICA figure of $13,387.65 plus only about $56 of MA PFML. The Massachusetts PFML employee contribution on $175,002 at the 0.46% employee rate is $805.01, so the model understated that component by roughly $749." -us,scenario_081,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"The model computed both FICA components exactly right and stopped there, never adding the Massachusetts PFML employee contribution of 0.46% × $175,002 = $805.01. The miss equals that omitted mandatory state payroll tax to the cent." -us,scenario_081,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model built a 0.532% MA PFML employee rate by adding a 0.352% medical-leave employee share to a 0.18% family-leave share, producing $931 of contribution instead of the $805.01 that the 0.46% employee rate yields on $175,002. That $126 rate overstatement, plus rounding SS and Medicare to whole dollars, produced the $126.34 overshoot." -us,scenario_081,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model asserted there is no Massachusetts mandatory employee payroll tax and returned FICA alone at $13,387.65, omitting the MA PFML employee contribution of 0.46% × $175,002 = $805.01. The MA Paid Family and Medical Leave employee contribution is withheld from wages and belongs in employee-side payroll tax." -us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model described only Social Security and Medicare, omitting the $805.01 MA PFML employee contribution, and its $13,389.37 also exceeds the $13,387.65 that 7.65% × $175,002 produces by $1.72. The missing mandatory state paid-leave contribution is the substantive error." -us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model claimed wages exceed the $200,000 Additional Medicare Tax threshold when $175,002 falls $24,998 below it, and then submitted $5,420 — less than half of the $13,387.65 that its own described 6.2% plus 1.45% rates produce on $175,002. It also omitted the $805.01 MA PFML employee contribution." -us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model had FICA exactly right at $13,387.65 but withheld $962.51 of MA PFML, a 0.55% rate, instead of the $805.01 that the 0.46% employee rate produces on $175,002. The $157.50 rate overstatement is the whole error." -us,scenario_081,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model applied the combined employer-plus-employee Medicare rate of 2.9% instead of the employee-only 1.45%, adding $5,075.06 of Medicare rather than $2,537.53 and inflating the total by $2,537.53. It also omitted the $805.01 MA PFML employee contribution, leaving a net $1,733.34 overstatement." -us,scenario_081,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model included a MA PFML component but priced it at 0.318% for $556 instead of the 0.46% employee rate that yields $805.01 on $175,002. That $249 rate shortfall, plus whole-dollar rounding of SS and Medicare, is the entire miss." -us,scenario_081,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model stated there is no state payroll tax and returned 7.65% × $175,002 rounded to $13,388, omitting the Massachusetts PFML employee contribution of 0.46% × $175,002 = $805.01. The MA paid-leave contribution is a mandatory employee-side withholding included in this output." -us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the model produced nothing to score. The required derivation is $10,850.12 Social Security plus $2,537.53 Medicare plus $805.01 MA PFML at 0.46%, totaling $14,192.66." -us,scenario_081,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"The model applied a 0.256% MA PFML employee rate, producing $448.01 instead of the $805.01 that the 0.46% employee rate yields on $175,002. Its FICA components were exact, so the $357.00 PFML rate shortfall is the whole error." -us,scenario_081,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with no derivation, which is consistent only with a household having no earnings. Wages of $175,002 generate mandatory employee Social Security of $10,850.12, Medicare of $2,537.53, and MA PFML of $805.01, for $14,192.66." -us,scenario_081,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model stated that Massachusetts levies no mandatory employee payroll tax and stopped at FICA of $13,387.65, omitting the MA Paid Family and Medical Leave employee contribution of 0.46% × $175,002 = $805.01. That withheld state contribution is the exact amount of its shortfall." -us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model declared that no mandatory state payroll tax applies in Massachusetts, dropping the $805.01 MA PFML employee contribution, and then submitted $14,525.24 while its explanation computed $13,387.65. Both the omitted state paid-leave contribution and the $1,137.59 gap between its stated total and its submitted value are errors." -us,scenario_081,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied the 0.9% Additional Medicare Tax even though wages of $175,002 fall below the $200,000 single-filer threshold, adding roughly $534 of phantom tax to its $13,387.65 FICA base to reach $13,922. It simultaneously omitted the $805.01 MA PFML employee contribution that is actually owed." +us,scenario_081,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It correctly computed the MA PFML employee contribution as 0.46% x $175,002 = $805.01. It then dropped that contribution, assuming PolicyEngine counts only SS and Medicare. It also submitted $13,268.70, which does not match even its own $13,387.65 subtotal; the correct total adds the $805.01 to reach $14,192.66." +us,scenario_081,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed SS ($10,850.12) and Medicare ($2,537.53) correctly. It left out the mandatory MA PFML employee contribution of 0.46% of wages ($805.01), which the payroll_tax definition includes as a mandatory employee state payroll tax." +us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It said Massachusetts has no separate mandatory employee payroll tax, missing the PFML employee contribution of 0.46% x $175,002 = $805.01. It also capped SS at the outdated $168,600 wage base, even though the full $175,002 is below the 2026 base. It then submitted $13,383.41, which matches neither of its own subtotals." +us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It estimated the MA PFML employee contribution at about $0 by treating the medical share as employer-paid. The actual employee rate is 0.46% of wages, which gives $805.01. It also submitted $12,321.73, which contradicts its own $13,387.65 subtotal of SS plus Medicare on full wages." +us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It set MA PFML to 0 because it had no rate, omitting the 0.46% employee contribution of $805.01. It then submitted $12,717.62, which is not the $13,387.65 sum of the SS and Medicare figures it had just computed." +us,scenario_081,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It explicitly set the MA PFML employee contribution to about $0 instead of 0.46% x $175,002 = $805.01. It also rounded SS plus Medicare down to $13,385 rather than $13,387.65." +us,scenario_081,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It identified the MA PFML employee contribution of about $805 at 0.46%, then excluded it on the assumption that the model leaves it out. The payroll_tax definition covers mandatory employee state payroll taxes, so the $805.01 belongs in the $14,192.66 total." +us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It first reached the correct $14,192.66 using the 0.46% PFML employee rate. It then switched to 0.604%, built on an inverted split: employees paying all of a 0.42% medical portion plus 40% of a 0.46% family portion. The MA employee share is 0.46% of wages, so its PFML came out at $1,057.01 instead of $805.01." +us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It treated MA PFML as employer-side only and excluded it, missing the mandatory employee contribution of 0.46% x $175,002 = $805.01. It then submitted $12,318.65, which contradicts its own $13,387.65 sum of SS and Medicare." +us,scenario_081,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It stated outright that it did not include the MA PFML contribution. That dropped the mandatory 0.46% employee contribution ($805.01) from an otherwise correct SS plus Medicare total of $13,387.65." +us,scenario_081,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It asserted that Massachusetts has no employee state payroll tax, overlooking the mandatory PFML employee contribution of 0.46% of wages ($805.01) that brings the total to $14,192.66." +us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"It correctly included MA PFML but applied a 0.38% employee rate instead of 0.46%. That understated PFML at about $665 versus $805.01, and it then rounded the total to $14,053." +us,scenario_081,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a 0.375% MA PFML employee rate ($656.26) instead of the 0.46% rate ($805.01). It also submitted $13,593.41, which differs from the $14,043.91 total stated in its own explanation." +us,scenario_081,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a 0.44% MA PFML employee rate ($770.01) instead of 0.46% ($805.01), which put the total $35 short of $14,192.66." +us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It included MA PFML at an estimated 0.318% of wages ($556.51). The employee rate is 0.46% ($805.01), so the total fell $248.50 short." +us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only 6.2% SS plus 1.45% Medicare and omitted the 0.46% MA PFML employee contribution ($805.01). Its $11,843 does not even equal 7.65% x $175,002 = $13,387.65, the result of the formula it described." +us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It applied SS and Medicare to $167,613, having subtracted the $7,389 employer-sponsored insurance premiums, when the payroll tax base is the full $175,002. It also used an estimated PFML of about $580 instead of 0.46% x $175,002 = $805.01." +us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It cut the FICA base to $167,613 by subtracting the $7,389 ESI premiums, when SS and Medicare apply to the full $175,002 ($10,850.12 and $2,537.53). It also used a 0.24% PFML rate ($402) instead of 0.46% ($805.01)." +us,scenario_081,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It included MA PFML at $765.01 instead of 0.46% x $175,002 = $805.01. It also miscomputed SS as $10,830 rather than 6.2% x $175,002 = $10,850.12, so the total came up $60 short." +us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages to $167,613 by subtracting the $7,389 ESI premiums as pre-tax, when the payroll tax base is the full $175,002. It also omitted the 0.46% MA PFML employee contribution ($805.01) entirely." +us,scenario_081,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"It named the right three components (SS, Medicare and MA PFML), but its $13,444 total is only $56 above SS plus Medicare on full wages ($13,387.65). That means it counted about $56 of PFML instead of the $805.01 produced by the 0.46% employee rate." +us,scenario_081,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It stopped at SS plus Medicare ($13,387.65) and never added the mandatory 0.46% MA PFML employee contribution of $805.01." +us,scenario_081,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"It built a 0.532% PFML employee rate by taking 40% of the entire 0.88% combined rate (0.352%) as the medical share and adding the 0.18% family share. The employee's 40% applies only to the 0.70% medical portion, giving 0.28% + 0.18% = 0.46%. It therefore overstated PFML at $931 instead of $805.01." +us,scenario_081,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It asserted that Massachusetts has no mandatory employee payroll tax and omitted the PFML employee contribution of 0.46% x $175,002 = $805.01." +us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It counted only SS and Medicare, which it also mis-added as $13,389.37 instead of $13,387.65. It omitted the mandatory MA PFML employee contribution of $805.01 at 0.46% of wages." +us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"It wrongly claimed the wages exceed the $200,000 Additional Medicare Tax threshold ($175,002 does not). It also submitted $5,420, less than half the $10,850.12 SS tax alone, and omitted the $805.01 MA PFML contribution." +us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its SS plus Medicare total of $13,387.65 is correct, but it estimated MA PFML at $962.51, which is 0.55% of wages. The employee rate is 0.46%, giving $805.01." +us,scenario_081,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It counted only 6.2% SS and 1.45% Medicare (about $13,388) and omitted the mandatory MA PFML employee contribution of 0.46% of wages ($805.01)." +us,scenario_081,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"It applied the combined 2.9% employer-plus-employee Medicare rate instead of the 1.45% employee rate, doubling Medicare to about $5,075. It also omitted the 0.46% MA PFML employee contribution ($805.01)." +us,scenario_081,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It included MA PFML at 0.318% of wages ($556) instead of the 0.46% employee rate ($805.01), and it rounded the components to whole dollars." +us,scenario_081,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It assumed there is no state payroll tax and applied only 7.65% FICA. That omitted the mandatory MA PFML employee contribution of 0.46% x $175,002 = $805.01." +us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The correct derivation is SS $10,850.12 + Medicare $2,537.53 + MA PFML $805.01 = $14,192.66." +us,scenario_081,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"It applied a 0.256% MA PFML employee rate ($448.01) instead of 0.46% ($805.01), which put the total $357 short." +us,scenario_081,payroll_tax,minimax-m3,llm_error,other,False,"It returned $0 with no reasoning, as though the head owed no employee payroll tax at all. On $175,002 of wages the correct derivation gives SS $10,850.12 + Medicare $2,537.53 + MA PFML $805.01 = $14,192.66." +us,scenario_081,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that Massachusetts levies no mandatory employee payroll tax and stopped at $13,387.65. That omitted the PFML employee contribution of 0.46% x $175,002 = $805.01." +us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It asserted that Massachusetts has no mandatory state payroll tax, missing the 0.46% PFML employee contribution ($805.01). It then submitted an unexplained $14,525.24 that contradicts its own $13,387.65 subtotal." +us,scenario_081,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Its listed components (SS, Medicare and Additional Medicare Tax) leave out the MA PFML employee contribution. By its own formula the total is 7.65% x $175,002 = $13,387.65 with no Additional Medicare Tax, not $13,922. The correct total adds $805.01 of PFML at 0.46% to reach $14,192.66." us,scenario_081,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It built the correct Massachusetts structure — wages less the $2,000 payroll-tax deduction, the $4,000 rental deduction and the $4,400 exemption at 5%, plus 5% on the $166 of interest and dividends — and its own reasoning reached ""about 8,238,"" matching the reference to the dollar. It then discarded that result and submitted 8,380 on a fabricated ""adding back rounding of components"" adjustment worth $141.60 that no Massachusetts rule generates." -us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the right deduction set ($2,000 payroll-tax, $4,000 rental, $4,400 exemption) but removed the $56 of interest and $110 of dividends from the base entirely by offsetting them with the $1,080 short-term capital loss. The interest stays in Part B and the $110.12 dividend is taxed in Part A at 5% with no capital-loss offset, so its 8,230 is short by the $8.41 of tax on that investment income." -us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It took no Massachusetts Part B deductions — neither the $4,000 rental deduction nor the $2,000 Social Security/Medicare deduction — and instead subtracted an invented $3,835 of ""nonrefundable credits for property taxes and personal exemptions."" The personal exemption is a $4,400 deduction rather than a credit, and the senior circuit-breaker property-tax credit requires age 65+, so that fabricated credit block is what cut its 8,525 gross figure to 4,690." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted $11,287 of actual FICA from Part B income; Massachusetts caps the Social Security/Medicare deduction at $2,000. It also denied the rental deduction on the theory that the filer owns the home, ignoring the $44,400 rent input that fills the $4,000 cap, and then abandoned its own 7,966 arithmetic for an unexplained 8,730." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It subtracted only the $4,400 personal exemption, omitting both Massachusetts Part B deductions — the $2,000 Social Security/Medicare deduction and the $4,000 rental deduction (50% of the $44,400 rent, capped). Those $6,000 of missed deductions overstate the 5% base by exactly $6,000 and the tax by $300, which is essentially the whole gap between its 8,538 and the reference." -us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It applied the $2,000 payroll-tax deduction and the $4,400 exemption but never took the $4,000 rental deduction generated by the $44,400 of listed rent, worth $200 of tax. It then inflated its own 8,438 arithmetic to 8,620 with an unspecified ""small adjustments"" fudge." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It capped the rental deduction at the superseded $3,000 limit rather than the $4,000 cap in force for 2026, and omitted the $2,000 Social Security/Medicare deduction entirely. Those two errors leave $3,000 of extra income in the 5% base, putting its 8,388.70 exactly $150.29 above the reference." -us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It declared the filer a homeowner and dropped the rental deduction despite the $44,400 of pre-subsidy rent that fills the $4,000 cap, and it skipped the $2,000 payroll-tax deduction, computing 8,539 from income less only the $4,400 exemption. It then discarded that figure for 8,200 on ""part-year adjustments"" that contradict the stated full-year facts." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $4,800 personal exemption instead of the Massachusetts single exemption of $4,400 and took no Part B deductions at all, ignoring the $4,000 rental deduction and the $2,000 Social Security/Medicare deduction. Missing $6,000 of deductions net of the $400 exemption overstatement leaves its 8,465 about $227 above the reference." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It carried $22,543 of federal itemized deductions (the $16,900 mortgage interest and $5,643 real estate taxes) and a $5,000 exemption into the Massachusetts computation. Massachusetts allows neither: the Part B reductions are the $2,000 payroll-tax deduction, the $4,000 rental deduction and the $4,400 exemption, so its 139,162 taxable figure understates the correct $164,658 by more than $25,000." -us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It stacked $22,543 of federal itemized mortgage-interest and real-estate-tax deductions into its Massachusetts deduction total, which Massachusetts disallows, driving taxable income to 142,231 against the correct $164,658. It compounded this by submitting 7,519.53 while its own explanation computed 7,111.55." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It capped the rental deduction at $3,000 instead of the $4,000 cap applicable for 2026 and built the Part B base from wages alone, leaving out the $56 of taxable interest and the $110.12 Part A dividend. The extra $1,000 of taxed income less the untaxed investment income puts its 8,280.10 precisely $41.69 above the reference." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It multiplied 5% by an undeducted $175,000 wage figure, applying neither the $4,400 personal exemption nor the $4,000 rental and $2,000 Social Security/Medicare deductions. Its 8,750 is exactly the no-deduction ceiling, $511.59 above the tax on the correct $164,658 Part B base plus the $110.12 Part A dividend." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It named the correct reduction set — $4,400 exemption, $4,000 rental deduction, $2,000 FICA deduction — but then reported taxable Part B income of 157,219 when 175,002 less those $10,400 is 164,602. That $7,383 subtraction error drops its tax about $377 below the reference." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from $167,613 of wages, which is the $175,002 gross figure less the $7,389 employer-sponsored insurance premium that is already excluded from that figure, so it removed the premium twice. It then took only the $4,400 exemption, skipping the $4,000 rental and $2,000 payroll-tax deductions, landing at 8,161." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It reported 8,750 with no derivation, which is 5% of the full $175,000 wage figure with no reductions. The Massachusetts computation subtracts the $2,000 Social Security/Medicare deduction, the $4,000 rental deduction and the $4,400 exemption to reach a $164,658 Part B base, and adds 5% on the $110.12 Part A dividend, for 8,238.41." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its deduction set ($4,400 exemption, $4,000 rental deduction, $2,000 FICA deduction) matches the reference, but it started from $167,613 of wages after stripping the $7,389 employer-sponsored insurance premium that the $175,002 wage figure already excludes. That double exclusion, plus the omitted $56 of interest and $110.12 dividend, accounts for the entire $377.76 shortfall." -us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its 7,869 corresponds to a Part B base near 157,380, which is the $175,002 of wages reduced by the $7,389 employer-sponsored insurance premium already excluded from that amount plus the correct $4,400/$4,000/$2,000 reductions. Leaving the premium in the wage figure gives a $164,658 base and $8,232.90, plus $5.51 on the Part A dividend." -us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It applied the federal $3,000 capital-loss-against-ordinary-income rule to net the $1,080 short-term loss into the Massachusetts base, capped the rental deduction at $3,000 instead of $4,000, and dropped both the $4,400 personal exemption and the $2,000 Social Security/Medicare deduction. Its 171,094 taxable figure exceeds the correct $164,658 by $6,436, producing the $316.29 overstatement." -us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the superseded $3,000 rental-deduction cap in place of $4,000 and omitted the $2,000 Social Security/Medicare deduction. Those $3,000 of missing deductions put its base at 167,774 against the correct $164,658 and leave 8,388.70 exactly $150.29 high." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It submitted 6,440 with no derivation, implying a taxable base near $128,800 — roughly $36,000 below the $164,658 that the $2,000 payroll-tax deduction, $4,000 rental deduction and $4,400 exemption produce from $175,058 of Part B gross income. No Massachusetts deduction stack reaches that base." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its 13,951 implies an effective rate near 8% on this household's income. Massachusetts taxes Part B income at a flat 5% with the 4% surtax starting above roughly $1.08 million, so 5% of the $164,658 base plus 5% of the $110.12 Part A dividend gives 8,238.41." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied all three Massachusetts reductions correctly to reach 164,602, then let the $1,080 short-term capital loss erase the $166 of interest and dividends. The $56 of interest remains in Part B and the $110.12 dividend is taxed in Part A at 5% with no offset from that loss, so its 8,230.10 misses by exactly the $8.31 of tax on that income." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave 7,915 with no arithmetic, implying a base near $158,300 — about $6,400 below the correct $164,658. Massachusetts allows only the $4,400 exemption, the $4,000 rental cap and the $2,000 Social Security/Medicare cap against the $175,058 of Part B gross income, so it stacked reductions beyond what the state permits." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It took the $4,400 exemption and the full $4,000 rental deduction from a $175,174 all-income base but never applied the $2,000 Social Security/Medicare deduction. That single omission puts $2,116 of extra income in the 5% base and accounts for its $100.29 excess at 8,338.70." -us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It removed the $56 of taxable interest from the base by invoking the Massachusetts bank-interest exclusion, which does not reach this household's generic taxable interest, and offset the $110.12 dividend with the short-term capital loss. Both amounts are taxed at 5% — the interest in Part B and the dividend in Part A — so its otherwise correct 164,602 computation lands $8.31 short." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied 5% to the full $174,094 income with no reductions whatsoever — no $4,400 personal exemption, no $4,000 rental deduction, no $2,000 Social Security/Medicare deduction. Those omissions are the entire $466.59 by which 8,705 overshoots the reference." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It deducted $1,782 of unreimbursed employee business expenses, which Massachusetts Schedule Y restricts to specific occupational categories and does not grant this ordinary employee, while omitting the $4,000 rental deduction and $2,000 payroll-tax deduction it was entitled to. The net $3,254 of overstated taxable income puts its 8,396 about $158 above the reference." -us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It invented a $1,000 Massachusetts capital-loss allowance against ordinary income and skipped the $4,000 rental deduction generated by the $44,400 of rent. The forgone rental deduction adds $200 of tax and the phantom loss allowance removes $50, netting the $150.59 by which its 8,389 exceeds the reference." -us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It asserted that Massachusetts allows no deductions beyond the personal exemption, dropping the $4,000 rental deduction (50% of $44,400, capped) and the $2,000 Social Security/Medicare deduction, and it also netted the $1,080 short-term capital loss into ordinary income. The $6,000 of missed Part B deductions dominate, leaving 8,485 about $247 high." -us,scenario_081,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the $3,000 rental cap instead of the $4,000 cap and took no $2,000 Social Security/Medicare deduction, while starting from a base that already netted the $1,080 short-term capital loss against ordinary income. Its 166,694 taxable figure exceeds the correct $164,658, producing the $96.59 overstatement at 8,335." -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing was submitted against the $8,238.41 reference. The required derivation is 5% of a $164,658 Part B base (wages $175,002 plus $56 interest, less the $2,000 payroll-tax deduction, $4,000 rental deduction and $4,400 exemption) plus 5% of the $110.12 Part A dividend." -us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It stated there is ""no separate rent expense input"" even though the household lists $44,400 of pre-subsidy rent, forfeiting the $4,000 rental deduction worth $200 of tax, and it netted the $1,080 short-term capital loss into its income base. Those choices raise its taxable figure to 167,688 against the correct $164,658 and account for the full $145.99 gap." -us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It reported zero state income tax for a Massachusetts resident with $175,002 of wages. Massachusetts applies its 5% Part B tax after only the $2,000 Social Security/Medicare deduction, the $4,000 rental deduction and the $4,400 personal exemption, giving $164,658 of taxable income and $8,232.90, plus $5.51 on the $110.12 Part A dividend." -us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted only the $4,400 personal exemption from wages, skipping the $2,000 Social Security/Medicare deduction and the $4,000 rental deduction on the $44,400 of rent, and it explicitly considered only the age-65 circuit breaker before concluding no further relief existed. Those $6,000 of missed Part B deductions are exactly the $300 by which its 8,538 overshoots." -us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It stated that Massachusetts allows no deductions for this household's expenses and applied 5% to income less only the $4,400 exemption, forfeiting the $4,000 rental deduction and the $2,000 Social Security/Medicare deduction. It also netted the $1,080 short-term capital loss against ordinary income, which Massachusetts does not permit against Part B wages." -us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported 7,800 with no derivation, implying a taxable base near $156,000 — $8,658 below the $164,658 that the $2,000 payroll-tax deduction, $4,000 rental deduction and $4,400 exemption produce. Applying the flat 5% to the correct base and adding 5% on the $110.12 Part A dividend gives 8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached about $8,238: $164,602 of Part B at 5% ($8,230.10) plus $166 of interest and dividends at 5% ($8.30), which matches the correct total. It then added an unexplained ~$142 'rounding of components' and submitted $8,380. No Massachusetts rule supports that addition." +us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $4,400 exemption, the $4,000 rent deduction and the $2,000 payroll deduction correctly. However, it used the $1,080 short-term loss to wipe out the $166 of interest and dividends. The $56 of interest belongs in Part B income ($175,058) and the $110 of dividends stays taxable in Part A. Its answer is therefore missing $2.80 plus $5.51 of tax: $8,232.90 + $5.51 = $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted an invented $4,550 deduction and skipped the $2,000 payroll deduction, the $4,000 rent deduction and the $4,400 personal exemption. It then took off $3,835 of made-up nonrefundable credits (a dependent exemption credit and a property tax credit), even though there are no dependents and no Massachusetts nonrefundable credit applies. The correct figure is 5% of $164,658 in Part B plus 5% of the $110 of dividends." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full ~$11,287 of FICA, but the Massachusetts Social Security/Medicare deduction is capped at $2,000. It also refused the rent deduction on the grounds that the head owns a home, even though $44,400 of rent is listed, which gives the capped $4,000 deduction. It then inflated its own $7,966 to $8,730 with no basis." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from $175,168 of income. It left out the $2,000 Social Security/Medicare deduction and the $4,000 rent deduction (50% of $44,400, capped). Those two deductions bring Part B taxable income down to $164,658." +us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $4,400 exemption and the $2,000 payroll deduction but left out the $4,000 rent deduction for the $44,400 of rent. It then pushed its own $8,438 up to $8,620 with unexplained 'small adjustments'." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at $3,000 instead of the current $4,000. It also left out the $2,000 Social Security/Medicare deduction entirely and added the $6 of REIT income to the 5% base. Together these overstated taxable income by $3,006 ($167,774 against the correct $164,658 plus $110)." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It treated the head as a homeowner and denied the rent deduction, even though $44,400 of rent gives the capped $4,000 deduction. It also skipped the $2,000 payroll-tax deduction, which produced $8,539. It then cut that to $8,200 with an unexplained 'exemption differences' adjustment." +us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It started from $174,088, which nets the $1,080 short-term capital loss against wage income; Massachusetts does not allow that loss against Part B income. It also left out the $2,000 Social Security/Medicare deduction, which gave $165,688 of taxable income instead of $164,658 in Part B plus $110 in Part A." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used federal AGI ($174,094, which nets the capital loss) as the base and a $4,800 exemption instead of the $4,400 single exemption. It left out both the $4,000 rent deduction and the $2,000 payroll-tax deduction." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $22,543 of federal-style itemized deductions (mortgage interest and taxes), which Massachusetts does not allow. It also used a wrong AGI of $166,705 and a $5,000 exemption instead of $4,400, and it never applied the $4,000 rent or $2,000 payroll deductions correctly against $175,058 of Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It correctly applied the rent, payroll and exemption deductions, but it also subtracted $22,543 of federal itemized deductions (mortgage interest and real estate taxes), which Massachusetts does not allow. That undercounted taxable income by more than $20,000. Its explanation reports $7,111.55, which does not match the $7,519.53 it submitted." +us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $16,900 of mortgage interest and $2,000 of real estate taxes as if they were federal itemized deductions, which Massachusetts does not allow. It also left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction that make up the correct Part B base." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It capped the rent deduction at $3,000 instead of $4,000, which overstates tax by $50. It also left the $56 of interest out of Part B and never taxed the $110 of Part A dividends. The correct result is $164,658 × 5% + $5.51 = $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to about $175,000 of gross income and took no deductions. It skipped the $4,400 personal exemption, the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, which together bring Part B taxable income down to $164,658." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It took the $7,389 of employer-sponsored insurance premiums out of wages. The input is gross wages, so Part B income stays at $175,002 plus $56 of interest. As a result its Part B base of $157,219 is about $7,400 too low, even though the exemption, rent and payroll deductions were right." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by subtracting the $7,389 of ESI premiums from the listed gross wages, which lowered Part B income. It then left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, applying only the $4,400 exemption." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $8,750 equals 5% of $175,000 of gross wages with no deductions, so it skipped the $4,400 exemption, the $4,000 rent deduction and the $2,000 payroll deduction. With those applied, Part B taxable income is $164,658, and the correct tax including $110 of Part A dividends is $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $167,613 by taking the $7,389 of ESI premiums out of the listed gross wages, which understated Part B income by $7,389. It also dropped the $56 of Part B interest and the $110 of Part A dividends. Its $4,400, $4,000 and $2,000 deductions were right." +us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,869 equals 5% of about $157,380: wages reduced by the $7,389 of ESI premiums, less $4,400, $4,000 and $2,000, plus $166 of interest and dividends. Taking the premiums out of the listed gross wages understated Part B income by $7,389." +us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $4,400 single personal exemption and the $2,000 Social Security/Medicare deduction, and it used the old $3,000 rent cap instead of $4,000. It also netted the $1,080 short-term loss against ordinary income. The result was $171,094 of taxable income against the correct $164,658 plus $110." +us,scenario_081,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It used the old $3,000 rent-deduction cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction entirely. It also added the $6 of REIT income to the base, which overstated taxable income by $3,006." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $6,440 implies about $128,800 of taxable income, roughly $36,000 below the correct $164,768. That gap matches subtracting federal-style itemized deductions (the $16,900 of mortgage interest and state and local taxes), which Massachusetts does not allow. Only the $2,000 payroll, $4,000 rent and $4,400 exemption deductions reduce the $175,058 of Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Its $13,951 is about 8.5% of the roughly $164,000 taxable base, which is Massachusetts's short-term capital gains rate. Part B wage income and Part A dividends are taxed at the flat 5%, which gives $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $2,000 payroll, $4,000 rent and $4,400 exemption deductions correctly. However, it let the $1,080 short-term loss offset the $166 of interest and dividends. The $56 of interest belongs in Part B income ($175,058) and the $110 of dividends stays taxable at 5% in Part A, which adds $2.80 + $5.51 to reach $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It cited only the exemption and the rent deduction and left out the $2,000 Social Security/Medicare deduction. On $175,002 of wages those two deductions alone give $8,330, so its $7,915 includes about $8,300 of unsupported extra deductions and also leaves out the Part A dividend tax. The correct base is $175,058 − $2,000 − $4,000 − $4,400 = $164,658 in Part B plus $110 in Part A." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,338.70 is 5% of $166,774: $175,174 less $4,400 and $4,000. It left out the $2,000 Social Security/Medicare deduction and added the $6 of REIT income to the base." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $56 of interest with a bank-interest exemption and used the short-term capital loss to cancel the $110 of dividends. The interest stays in Part B income ($175,058) and the dividends are taxed at 5% in Part A ($5.51). With its correct $2,000, $4,000 and $4,400 deductions, the result is $8,232.90 + $5.51 = $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $8,284 is 5% of about $165,688: income net of the $1,080 short-term loss, less the $4,400 exemption and the $4,000 rent deduction. It left out the $2,000 Social Security/Medicare deduction and wrongly netted the short-term capital loss against Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $2,000 Social Security/Medicare deduction, which overstates tax by $100. It also used the short-term loss to cancel the $56 of interest, which belongs in Part B, and the $110 of dividends, which Part A taxes at 5%." +us,scenario_081,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It treated the $56 of bank interest as exempt and let the capital loss absorb the $110 of dividends. The interest stays in Part B income ($175,058) and the dividends are taxed at 5% in Part A ($5.51). With its correct $2,000, $4,000 and $4,400 deductions, the result is $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It applied 5% to the full $174,094 of federal AGI. It skipped the $4,400 personal exemption, the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it netted the capital loss against wage income." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the $1,782 of unreimbursed employee business expenses, which PolicyEngine's Part B base does not include. It left out the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it netted the $1,080 short-term loss against ordinary income." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $4,000 rent deduction (50% of $44,400, capped) and subtracted a made-up $1,000 of capital loss from ordinary income. It also added the $6 of REIT income, which gave $167,774 of taxable income instead of $164,658 in Part B plus $110 in Part A." +us,scenario_081,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It started from federal AGI of $174,094, which nets the $1,080 short-term loss against wages, and subtracted only the $4,400 exemption and the $4,000 rent deduction. It left out the $2,000 Social Security/Medicare deduction that Massachusetts allows against Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only the $4,400 exemption from federal AGI of $174,094. It left out the $4,000 rent deduction for the $44,400 of rent and the $2,000 Social Security/Medicare deduction." +us,scenario_081,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the old $3,000 rent-deduction cap instead of $4,000 and left out the $2,000 Social Security/Medicare deduction. It also started from federal AGI, which nets the capital loss against wage income." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this output, so there is no answer to score. The correct figure is 5% of $164,658 of Part B taxable income plus 5% of $110 of Part A dividends, which is $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It said there was 'no separate rent expense input', but the facts list $44,400 of pre-subsidy rent, which gives the capped $4,000 rent deduction. It also started from $174,088, which nets the $1,080 short-term loss against Part B income." +us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned zero Massachusetts tax with no reasoning, even though a single filer with $175,002 of wages owes the 5% flat tax. The correct figure is 5% × ($175,058 − $2,000 − $4,000 − $4,400) + 5% × $110 = $8,238.41." +us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It reduced $175,002 of wages only by the $4,400 exemption. It left out the $4,000 rent deduction for the $44,400 of rent and the $2,000 Social Security/Medicare deduction, which overstated Part B taxable income by $6,000." +us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It said Massachusetts allows no deductions other than the personal exemption and applied 5% to $174,094 less $4,400. That misses the $4,000 rent deduction and the $2,000 Social Security/Medicare deduction, and it nets the short-term loss against wages." +us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its $7,800 implies about $156,000 of taxable income, roughly $8,800 below the correct $164,768 ($164,658 in Part B plus $110 in Part A). That means it subtracted items beyond the only allowed deductions: the $2,000 payroll deduction, the $4,000 rent deduction and the $4,400 exemption, taken from $175,058." us,scenario_081,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 1 as within the applicable CHIP age range. The trace applies CHIP's age criterion against the child and returns false, while Medicaid eligibility also returns NONE." us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested child1_chip_eligible output and supplied no explanation, violating the required submission contract." @@ -6106,45 +6696,52 @@ us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,health_coverage,Fa us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household." us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its worksheet ended at $10,353 ($12,953 of tax less $2,600 of credits) and it then submitted $14,748, a figure no step in its own reasoning produces. That worksheet was already short by omitting the 2026 non-itemizer charitable deduction for the $700 cash gift and by using a $2,000 CTC instead of the 2026 $2,200 amount." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It correctly took the $700 non-itemizer charitable deduction, the $2,200 CTC, and the $600 CDCC, but added the $68 state and local tax refund to income, which the §111 tax-benefit rule excludes for a filer who takes the standard deduction. Its bracket application then produced $10,444.60 of ordinary tax where the 2026 HOH schedule (10% to $17,700, 12% to $67,450, 22% above) yields $10,349.45 on $79,311.15; those two items are the entire $95.15 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It filed the head as single with a $14,600 standard deduction rather than head of household with $24,150, reached $8,506 of tax, and then submitted $23,437 while stating that a 'detailed calculation' yielded it — no step it describes produces that number, which exceeds even the $12,363.05 of tax before any credits. It also treated the $9,600 of childcare as fully creditable at 20% instead of applying the $3,000 one-child expense cap that gives $600." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It guessed a $23,500 HOH standard deduction instead of $24,150 and omitted the 2026 non-itemizer charitable deduction for the $700 cash gift, leaving taxable income $1,359 too high, and applied bracket thresholds of $17,000/$64,850 rather than the 2026 HOH $17,700/$67,450. Its CTC ($2,200), CDCC ($600), $3,000 capital-loss limit, and 15% qualified-dividend rate were all right, so those three deduction and bracket errors account for the full $572.95 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed $10,460 and then wrote 'adjusting estimate' and submitted $11,150, adding $690 that no step supports. The underlying $10,460 was itself high because it used a $23,000 HOH standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, counted the $68 state tax refund as income, and applied a $2,000 CTC instead of the 2026 $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It omitted the 2026 non-itemizer charitable deduction that makes the $700 cash gift deductible alongside the standard deduction, and used bracket cutoffs of $17,000/$64,850 instead of the 2026 HOH $17,700/$67,450, producing $10,716 of ordinary tax against the correct $10,349.45. Its $24,500 standard-deduction guess (versus $24,150) partly offset the charitable omission, leaving the residual $366.95 overstatement driven by the stale brackets." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never added the $13,424 of qualified dividends to AGI — its $109,708 ordinary gross income excludes them — and then subtracted that same $13,424 again when carving ordinary income out of taxable income, so the dividends were removed from the base twice while being taxed only once at 15%. It compounded this by counting the $2,289 short-term gain as ordinary income while also netting it inside the capital-loss computation, by using a $22,500 standard deduction instead of $24,150, and by taking a $2,000 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its reasoning arrived at $10,739 and it submitted $13,920, a value none of its steps yields. The $10,739 was itself wrong because it applied 2025 bracket cutoffs ($16,550/$63,100) rather than the 2026 HOH $17,700/$67,450, used a $22,500 standard deduction instead of $24,150, took a $2,000 CTC instead of $2,200, and omitted the $700 non-itemizer charitable deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $33,000 long-term loss from income and then subtracted the $3,000 capital-loss allowance again, double-counting the loss and cutting AGI to $86,949 instead of $117,585.14, and it used a $46,000 head-of-household standard deduction where the 2026 figure is $24,150. It also applied a 35% CDCC rate on a $5,000 expense base for a $1,750 credit, when one qualifying child caps expenses at $3,000 and this AGI puts the rate at 20% for $600, and used a $300 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law for 2026 — 10/15/25% rate brackets, $9,680 of personal exemptions, and a $1,000 CTC — but the 2026 rules keep the 10/12/22% schedule, allow no personal exemptions, and set the CTC at $2,200. It also itemized $26,228 in place of the $24,150 standard deduction plus the separate $700 non-itemizer charitable deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"It zeroed the child tax credit as 'fully phased out,' applying the pre-TCJA $75,000 head-of-household phaseout to a $1,000 credit; the 2026 credit is $2,200 and phases out starting at $200,000 for this filer, so the full $2,200 is usable against the $11,763.05 of tax remaining after the CDCC. It also claimed $11,000 of repealed personal exemptions and itemized $15,528.77 instead of taking the larger $24,150 standard deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It taxed $88,390 of income at pre-TCJA rates to reach $14,456, where the 2026 schedule on $92,735.15 (with $13,424 at 15%) gives $12,363.05, and it claimed $10,000 of personal exemptions that no longer exist. It also built an $18,564 SALT deduction out of $9,217 of listed real estate taxes and used a $1,000 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated $100,571 AGI drops the $20,480 of dividend income that belongs in gross income, and it applied only a $2,000 CTC with no CDCC. Even that AGI yields about $7,200 of tax after its own credit, so the submitted $11,093 tracks none of its steps; the correct chain is $117,585.14 AGI, $92,735.15 taxable, $12,363.05 of tax, less the $600 CDCC and $2,200 CTC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $96,446 AGI is $117,654 less the $21,208 of employer-sponsored insurance premiums, double-counting an exclusion already reflected in the $100,195 wage figure — those premiums are pre-tax and are not subtracted a second time. It also claimed $10,400 of repealed personal exemptions, itemized $14,298 rather than taking the $24,150 standard deduction plus the $700 charitable deduction, and applied only the $600 CDCC while dropping the $2,200 CTC." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $96,454 AGI subtracts the $21,208 of employer-sponsored insurance premiums from wages that are already net of them, understating income by that amount; PolicyEngine's employment income is $100,013.62, the $100,195 wage less only the $181 traditional 401(k) deferral. It then layered on personal exemptions that 2026 law does not allow and used a $1,000 CTC instead of $2,200." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Its $12,790.25 is tax before any credits: it equals the pre-credit liability computed without the $700 non-itemizer charitable deduction and with $17,000/$64,850 bracket cutoffs ($10,777.45 ordinary plus $2,013.60 on qualified dividends), and it never subtracted the $600 CDCC or the $2,200 CTC. The requested output is tax after nonrefundable credits, which is $9,563.05." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Its explanation applies only the child and dependent care credit among nonrefundable credits, dropping the $2,200 child tax credit that fully offsets tax at this income (the 2026 HOH phaseout starts at $200,000), and its $7,380 sits $2,183 below the reference — the CTC amount. It also claimed personal exemptions, which 2026 law does not provide." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,384 corresponds to about $6,184 of pre-credit tax, which is what results from deducting the entire $30,710.82 net capital loss against ordinary income instead of applying the $3,000 §1211(b) annual limit and carrying the remaining $27,710.82 forward. The correct base is $92,735.15 of taxable income producing $12,363.05 of tax before the $2,800 of nonrefundable credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $2,000 child tax credit where 2026 sets it at $2,200, and omitted the non-itemizer charitable deduction that lets this standard-deduction filer subtract the $700 cash gift on top of the $24,150 standard deduction. Those two items plus bracket cutoffs below the 2026 HOH $17,700/$67,450 schedule account for the $691.95 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented a $23,063 HOH standard deduction instead of the actual $24,150 and omitted the $700 non-itemizer charitable deduction, putting taxable income $1,795.85 too high, then used bracket cutoffs of $16,913/$64,575 rather than $17,700/$67,450 and a $2,000 CTC instead of $2,200. Its correct handling of the $3,000 capital-loss limit, the excluded $68 refund, the 15% dividend rate, and the $600 CDCC leaves those deduction and bracket errors as the whole $898.33 gap." -us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It itemized $23,266 — including the $21,208 of employer-paid-through-payroll insurance premiums as §213 medical expenses, which are already excluded from wages and cannot be deducted again — and that total is below the $24,150 standard deduction it should have taken. It also included the $68 state tax refund in income and never applied the separate $700 non-itemizer charitable deduction that stacks on the standard deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $12,719 exceeds the $12,363.05 of tax before any credits, so its claim to have applied both the CTC and CDCC does not hold. The figure matches taxing the $13,424 of qualified dividends at the 22% ordinary rate ($13,302.73 total on $92,735.15) and subtracting only the $600 CDCC, where the correct treatment taxes those dividends at 15% for $2,013.60 and subtracts $2,800 of nonrefundable credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized real estate taxes, medical, over-the-counter, and charitable amounts — about $9,900 of allowable deductions, since the $1,965 of medical costs falls far below the 7.5%-of-AGI floor — instead of taking the $24,150 HOH standard deduction plus the $700 non-itemizer charitable deduction, and its $17,303 reflects none of the $2,800 in nonrefundable credits. The correct base is $92,735.15 taxable, $12,363.05 of tax, $9,563.05 after credits." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a $2,000 child tax credit rather than the 2026 $2,200, and built a $24,415 itemized figure instead of the $24,150 standard deduction plus the separate $700 non-itemizer charitable deduction, leaving taxable income $512 high. It also added the $68 state and local tax refund to AGI, which is excluded for a filer taking the standard deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $89,951 AGI is gross income net of the full $30,710.82 capital loss: it deducted the entire loss instead of the $3,000 §1211(b) limit, understating AGI by $27,710.82. Everything downstream is right — $24,150 standard deduction, $2,013.60 on the qualified dividends at 15%, $2,200 CTC and $600 CDCC — so that single loss-limitation error is the whole $4,418.21 shortfall." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It ran the correct structure — $24,150 standard deduction, $3,000 capital-loss limit, 15% on the qualified dividends, $2,200 CTC, $600 CDCC — but never took the 2026 non-itemizer charitable deduction for the $700 cash gift and included the $68 state tax refund in its $117,662 AGI. Those $768 of extra taxable income at 22% produce the $170.95 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the right credits ($2,200 CTC and $600 CDCC) and the preferential dividend rate, but omitted the non-itemizer charitable deduction that removes the $700 cash gift on top of the standard deduction and understated both the $24,150 HOH standard deduction and the 2026 bracket cutoffs of $17,700/$67,450. Those items produce its $672.95 excess over the $9,563.05 reference." -us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized 'medical, charitable and state/local tax deductions,' which requires counting the $21,208 of pre-tax employer-sponsored insurance premiums to clear the 7.5%-of-AGI medical floor; those premiums are excluded from wages and are not deductible under §213, so the actual medical deduction is zero and the $9,917 of real estate taxes and charity loses to the $24,150 standard deduction. That inflated deduction, about $3,600 above the $24,850 the reference allows, drives its $792.31 shortfall." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It submitted a round $14,000 with no derivation, and that figure exceeds the $12,363.05 of tax before any credits, so it subtracted neither the $600 CDCC nor the $2,200 CTC. The number is consistent with taxing the $13,424 of qualified dividends at ordinary rates on 2025 bracket cutoffs and applying no nonrefundable credits at all." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its $96,454 AGI subtracts the $21,208 of employer-sponsored insurance premiums from wages that already exclude them — PolicyEngine's employment income is $100,013.62, the $100,195 wage less only the $181 traditional 401(k). It then applied post-sunset law that does not govern 2026: two personal exemptions, which are not allowed, and a fully phased-out CTC where the credit is $2,200 and unphased until $200,000 of AGI for a head of household." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It taxed ordinary income at post-sunset 10/15/25% rates and claimed two $5,300 personal exemptions; 2026 uses the 10/12/22% HOH schedule with cutoffs of $17,700 and $67,450 and allows no personal exemptions, giving $10,349.45 rather than its $12,053 of ordinary tax. It also zeroed the CTC against a pre-TCJA $75,000 head-of-household phaseout when the 2026 credit is $2,200 and fully usable here, and itemized $15,465 below the $24,150 standard deduction." -us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used a $12,591 standard deduction against a head-of-household filer whose 2026 amount is $24,150, inflating taxable income to $105,063 instead of $92,735.15, and it dropped the $2,200 child tax credit entirely, applying only the $600 CDCC. Those two errors account for the entire $7,020.95 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the correct $24,150 standard deduction, $3,000 capital-loss limit, 15% dividend rate, $2,200 CTC, and $600 CDCC, but omitted the 2026 non-itemizer charitable deduction for the $700 cash gift and carried the $68 state tax refund into AGI, leaving taxable income $768.85 high. Its ordinary tax of $10,638 versus the correct $10,349.45 on $79,311.15 also reflects bracket cutoffs below the 2026 HOH $17,700/$67,450 schedule." -us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so the required key was absent from its submit_outputs response. No substantive tax computation was produced to evaluate." -us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its single error is including the $68 state and local tax refund in gross income: under the §111 tax-benefit rule a filer taking the $24,150 standard deduction derived no prior-year benefit from state taxes, so the refund is excluded. That $68 of extra taxable income at the 22% marginal rate is exactly the $16.91 overstatement; every other step — the $700 non-itemizer charitable deduction, the $3,000 loss limit, $2,013.60 on the qualified dividends, the $2,200 CTC and $600 CDCC — matches the reference." -us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It concluded that 'significant capital loss offsetting income' plus the standard deduction and CTC drive tax to zero, but §1211(b) caps the deductible net capital loss at $3,000 per year, leaving the $30,710.82 loss to offset only $3,000 of the $120,593.62 of gross income. Taxable income is $92,735.15, tax before credits is $12,363.05, and the $2,800 of nonrefundable credits reduces it to $9,563.05, not zero." -us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used the correct 2026 HOH brackets, standard deduction, $3,000 loss limit, 15% dividend rate, $2,200 CTC, and $600 CDCC, but never took the non-itemizer charitable deduction that removes the $700 cash gift alongside the standard deduction, and it counted the $68 state tax refund as income. Its taxable income of $93,512 versus $92,735.15 is those $777 taxed at 22%, the full $170.95 overstatement." -us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning settled on $6,509 and it submitted $12,866.90, a value none of its steps produces. The $6,509 line was itself wrong because it netted the entire $30,711 capital loss into AGI instead of the $3,000 limit, used a $21,900 standard deduction rather than $24,150, taxed part of the qualified dividends at 0%, and split the CTC into a $400 nonrefundable piece when the 2026 credit is $2,200 and fully absorbed by this liability." -us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $139,775 AGI exceeds the $123,064 sum of every positive income item, so the $33,000 long-term capital loss was never applied as an offset at all, let alone limited to the $3,000 deduction. It also used the single filer's $15,750 standard deduction instead of the $24,150 head-of-household amount, claimed a $6,237 QBI deduction where there is no qualified business income, and used a $1,875 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Submitted $14,748, which contradicts its own worked result of about $10,353. That worked chain also skipped the $700 2026 non-itemizer charitable deduction (taxable income $93,436 instead of $92,735.15). It also used HoH thresholds of $17,300/$65,900 instead of the 2026 $17,700/$67,450, and applied a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its ordinary tax of $10,444.60 on $79,380 is $80 too high: the 2026 HoH schedule (10% to $17,700, 12% to $67,450, 22% above) gives $10,364.60. It also added the $68 state tax refund to AGI. The head takes the standard deduction, so the refund produced no prior-year tax benefit and is excluded." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Filed as single with a $14,600 standard deduction instead of head of household with the $24,150 2026 HoH standard deduction. It netted the whole $30,711 capital loss against income instead of capping the deduction at $3,000, and figured the CDCC on the full $9,600 instead of the $3,000 one-child cap. It then submitted $23,437, a figure unconnected to its own tax estimate of about $8,506." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used a $23,500 HoH standard deduction instead of $24,150 and omitted the $700 non-itemizer charitable deduction. It also taxed ordinary income on stale thresholds (10% to $17,000, 12% to $64,850) instead of the 2026 $17,700/$67,450. That gave $10,922 of ordinary tax on $80,670, versus $10,349.45 on the correct $79,311.15." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its own chain reached $10,460, and it then raised the answer to $11,150 with no stated reason. That chain used a $23,000 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund. It also used stale bracket thresholds ($17,000/$64,850 instead of $17,700/$67,450) and a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Taxed ordinary income on stale HoH thresholds (10% to $17,000, 12% to $64,850) instead of the 2026 $17,700/$67,450. That produced $10,716 on $79,730, where the 2026 schedule gives $10,441.60. It also used a $24,500 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund to income." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its only error is adding the $68 state tax refund to income. The head claims the standard deduction, so the refund carried no prior-year tax benefit and is excluded, leaving taxable income of $92,735.15. The extra $68.85 of taxable income at 22% accounts for its $15.15 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Left the $13,424 of qualified dividends out of AGI ($106,519) and then subtracted them again from taxable income, so ordinary taxable income was $70,595 instead of $79,311.15. It also added the $2,289 short-term gain as separate ordinary income after already netting it into the capital loss. On top of that it used a $22,500 standard deduction, stale brackets, and a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Submitted $13,920 even though its own chain ended at $10,739. That chain used a $22,500 standard deduction instead of $24,150 and omitted the $700 non-itemizer charitable deduction. It also used 2024 HoH thresholds ($16,550/$63,100) instead of the 2026 $17,700/$67,450, and a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the extra $700 charitable deduction that non-itemizers get starting in 2026. It also added the $68 state refund to income and dropped the $8 IRA deduction as phased out. That $776.86 of excess taxable income at 22% accounts for its $171 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the full $33,000 long-term loss and then another $3,000, instead of netting to a $30,711 loss and capping the deduction at $3,000, which cut AGI to $86,949. It also used a nonexistent $46,000 HoH standard deduction instead of $24,150. It computed the CDCC as 35% of $5,000 instead of 20% of the $3,000 one-child cap, and limited the nonrefundable CTC to $300 instead of the full $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Applied pre-TCJA law ($9,680 of personal exemptions, 10/15/25% brackets, a $1,000 CTC), but the OBBBA made the TCJA structure permanent for 2026. It also itemized $26,228 under those pre-TCJA rules. The correct 2026 path has no exemptions, the $24,150 standard deduction plus the $700 non-itemizer charitable deduction, 10/12/22% brackets, and a $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expired: it applied $11,000 of personal exemptions, pre-TCJA brackets, and a $1,000 CTC that it phased out entirely. The 2026 regime, made permanent by the OBBBA, has no exemptions and a $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. It also gives the full $2,200 CTC, since income is below the $200,000 phase-out threshold." +us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Used a $23,100 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and added the $68 refund. Its thresholds were below the 2026 HoH schedule, giving $11,059.60 on $81,130 where 10% to $17,700 and 12% to $67,450 give $10,749.60. It also applied a $2,300 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Applied pre-TCJA law: $10,000 of personal exemptions, $18,564 of uncapped SALT as an itemized deduction, and a $1,000 CTC. Under 2026 law there are no exemptions, and the $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction beats itemizing. The 2026 CTC is $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Reported AGI of $100,571, about $17,000 below the correct $117,585.14. The correct AGI is wages net of the 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capped loss and the $8 IRA deduction. It subtracted only a $2,000 CTC and omitted the $600 CDCC and the 2026 CTC amount of $2,200, and its implied $13,093 pre-credit tax does not follow from the 2026 HoH brackets on its own AGI." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $21,208 of employer-sponsored insurance premiums from wages (AGI $96,446), but employment income stays at $100,013.62 after only the 401(k) deferral. It also applied pre-TCJA rules: $10,400 of personal exemptions, itemizing $14,298, and a fully phased-out CTC. As a result it subtracted only the $600 CDCC and dropped the $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $21,208 of employer-sponsored insurance premiums from wages (AGI $96,454), and applied pre-TCJA personal exemptions and a $1,000 CTC. The 2026 computation keeps employment income at $100,013.62 after only the 401(k) deferral and has no personal exemptions. It allows the $700 non-itemizer charitable deduction and a $2,200 CTC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no computation. Its $12,790.25 sits just above the correct pre-credit tax of $12,363.05 and $3,227 above the after-credit amount. That is the pattern of a regular-tax figure that never subtracted the $2,200 CTC and the $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted personal exemptions, which the OBBBA permanently eliminated for 2026, understating taxable income and landing $2,183 below the correct amount. The correct taxable income is $92,735.15: AGI of $117,585.14 less the $24,150 HoH standard deduction and the $700 non-itemizer charitable deduction." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. Its $3,384 tracks deducting the full $30,711 net capital loss instead of the $3,000 §1211(b) cap, which drops taxable income to about $65,000 and puts the qualified dividends in the 0% bracket. The correct chain taxes $92,735.15: $10,349.45 on ordinary income plus $2,013.60 on dividends, less $2,800 of credits." +us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted a $2,000 CTC instead of the 2026 $2,200. Its implied pre-credit tax of $12,855 is $492 above the correct $12,363.05. It skipped the $700 non-itemizer charitable deduction and used HoH bracket thresholds below the 2026 $17,700/$67,450." +us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Used an estimated $23,063 standard deduction instead of $24,150, omitted the $700 non-itemizer charitable deduction, and dropped the $8 IRA deduction. It used bracket thresholds of $16,913/$64,575 instead of the 2026 $17,700/$67,450, and applied a $2,000 CTC instead of $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Counted the $21,208 of employer-sponsored insurance premiums as deductible medical expenses and itemized $23,266. It used that total even though it is below the $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. It also taxed ordinary income on thresholds below the 2026 schedule, giving $11,046 on $80,972 instead of $10,714.84." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its $12,719 is $3,156 above the correct $9,563.05 and close to the correct pre-credit tax of $12,363.05. Although it says it applied them, it never subtracted the $2,200 CTC and $600 CDCC from regular tax." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Itemized real estate taxes, medical and OTC costs, and charity instead of taking the larger $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. Its $17,303 exceeds even the correct pre-credit tax of $12,363.05, so it also failed to subtract the $2,800 of CTC and CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized $24,415 instead of taking the $24,150 standard deduction plus the $700 non-itemizer charitable deduction ($24,850 total). It also added the $68 refund, dropped the $8 IRA deduction, and subtracted a $2,000 CTC instead of the 2026 $2,200." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the entire $30,711 net capital loss (AGI $89,951 = $100,014 + $20,480 + $100 + $68 − $30,711) instead of capping the deduction at $3,000 under §1211(b). That understated AGI by about $27,600 and dropped the tax far below $9,563.05." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the extra $700 charitable deduction that non-itemizers get starting in 2026. It also added the $68 refund to AGI ($117,662) and dropped the $8 IRA deduction. That $776.86 of excess taxable income at 22% produces its $171 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its implied pre-credit tax of $13,036 is $673 above the correct $12,363.05, an overstatement of ordinary tax. It skipped the $700 non-itemizer charitable deduction and applied bracket thresholds below the 2026 HoH schedule (10% to $17,700, 12% to $67,450)." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Itemized roughly $28,500 of medical, charitable and SALT deductions by treating the $21,208 of employer-sponsored insurance premiums as deductible medical expenses. Without those premiums, itemized deductions fall well below the $24,150 standard deduction. The correct path takes the standard deduction plus the $700 non-itemizer charitable deduction, for taxable income of $92,735.15." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the $700 non-itemizer charitable deduction available from 2026. It also included the $68 refund and dropped the $8 IRA deduction, giving taxable income of $93,512 instead of $92,735.15 and a $171 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Used the $24,150 standard deduction but skipped the $700 non-itemizer charitable deduction and added the $68 refund, giving taxable income of $93,504 instead of $92,735.15. The $768.85 excess at 22% explains its $169.15 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Itemized roughly $28,400 of medical, charitable and SALT deductions by counting the $21,208 of employer-sponsored insurance premiums as medical expenses. Without them, itemizing loses to the $24,150 standard deduction. The correct path takes the standard deduction plus the $700 non-itemizer charitable deduction, for taxable income of $92,735.15." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation, and its round $14,000 exceeds even the correct pre-credit tax of $12,363.05. It therefore neither derived the $92,735.15 of taxable income nor subtracted the $2,200 CTC and $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA sunset: a pre-TCJA standard deduction, two personal exemptions, and a CTC fully phased out. It also subtracted the $21,208 of employer-sponsored premiums from wages (AGI $96,454). Under 2026 law there are no exemptions, employment income stays at $100,013.62 after the 401(k) deferral, and the full $2,200 CTC applies." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset law: two $5,300 exemptions, 10/15/25% brackets, and a $1,000 CTC phased out above a $75,000 threshold. Under the permanent 2026 regime there are no exemptions and brackets are 10/12/22% (to $17,700/$67,450). The standard deduction plus the $700 non-itemizer charitable deduction beats its $15,465 itemized total, and the full $2,200 CTC applies." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Itemized $28,094 by counting the $21,208 of employer-sponsored premiums as medical expenses above the 7.5% floor, alongside property tax and NY income tax. Without those premiums, itemized deductions fall below the $24,150 standard deduction plus the $700 non-itemizer charitable deduction. It therefore taxed $89,500 instead of $92,735.15." +us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration, using a $12,591 standard deduction and pre-TCJA brackets for $15,170 of ordinary tax on $105,063 of taxable income, and subtracted no CTC. 2026 law gives a $24,150 HoH standard deduction plus the $700 charitable deduction and 10/12/22% brackets. It also allows a $2,200 CTC alongside the $600 CDCC." +us,scenario_082,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Omitted the $700 non-itemizer charitable deduction and added the $68 refund, giving taxable income of $93,504 instead of $92,735.15. Its ordinary tax of about $10,638 on $80,080 is about $120 above the $10,518.60 that the 2026 HoH schedule (10% to $17,700, 12% to $67,450) produces." +us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to grade against $9,563.05." +us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Correctly took the $24,150 standard deduction plus the $700 non-itemizer charitable deduction and used the right 2026 brackets. However, it added the $68 state refund to income, even though the non-itemizing head got no prior-year tax benefit from it, and dropped the $8 IRA deduction. That $76.86 of excess taxable income at 22% accounts for its $16.91 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Treated the $33,000 long-term loss as wiping out ordinary income and reported $0 tax, ignoring the §1211(b) rule that net capital losses offset ordinary income only up to $3,000. With the capped loss, taxable income is $92,735.15 and tax after the $2,800 of credits is $9,563.05." +us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $24,150 standard deduction and skipped the $700 non-itemizer charitable deduction that applies from 2026. It also added the $68 refund to AGI ($117,662) and dropped the $8 IRA deduction, reaching taxable income of $93,512 instead of $92,735.15 and a $171 overstatement." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Netted the full $30,711 capital loss against income (AGI $90,132) instead of capping the deduction at $3,000. It used a $21,900 standard deduction and treated only $400 of the CTC as nonrefundable. It then submitted $12,866.90, which contradicts its own final computation of $6,509." +us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Filed single with a $15,750 standard deduction instead of head of household with $24,150, and claimed a $6,237 QBI deduction with no qualified business income. Its AGI of $139,775 is far above the correct $117,585.14. It also applied a $1,875 CTC instead of $2,200 and dropped the $600 CDCC." us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied only the ACTC limitation formula — lesser of $1,700 and 15% of earned income over $2,500 — and never performed the ordering step in which the nonrefundable CTC is first applied against tax liability and only the unused remainder becomes refundable. Its own reasoning conceded the head 'has sufficient nonrefundable CTC capacity' and then discarded that conclusion; with roughly $117,650 of AGI the pre-credit federal tax absorbs the entire credit, leaving $0. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200 amount." us,scenario_082,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the $1,700 statutory refundable ceiling as the answer after checking only the $2,500 earned-income floor, skipping the requirement that the refundable ACTC equals the CTC remaining after the nonrefundable portion offsets tax liability. Even at its own stated AGI of about $86,949, head-of-household tax before credits far exceeds the $2,200 CTC, so the credit is fully used nonrefundably and refundable credits are $0." us,scenario_082,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It quoted $1,000, the pre-TCJA additional child tax credit cap, which has no basis in 2026 law where the credit is $2,200 per child with a $1,700 refundable ceiling. The controlling error is treating a refundable amount as automatic: this filer's federal tax before credits exceeds the full $2,200 credit, so the nonrefundable CTC absorbs all of it and $0 is refundable." @@ -6159,123 +6756,141 @@ us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_l us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model calculated NYC head-of-household tax brackets on $79,254 of taxable income without any fact establishing NYC residence. New York State residence alone triggers no local income tax in this benchmark, yielding zero." us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"The model explicitly assumed NYC residence because NYC is a New York locality with an income tax, contradicting the instruction that unlisted household facts are false. Its detailed NYC taxable-income, bracket, school-credit, and dependent-care-credit computation is inapplicable because no NYC location was provided." us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"The model treated the household as an NYC resident and applied NYC brackets and credit rules despite the prompt listing only New York State. Without a specified NYC residence, none of those local rules applies and the local-income-tax output is zero." -us,scenario_082,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model correctly computed FICA of $7,664.92 on the full $100,195, then explicitly identified NY PFL/SDI and discarded them as ""negligible... excluded here"" — but the output definition counts mandatory employee state payroll taxes, and NY PFL ($411.91) plus DBL ($31.20) add $443.11. It recognized the right components and then deliberately dropped them." -us,scenario_082,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model computed only the two federal components ($6,212.09 Social Security + $1,452.83 Medicare) and never considered a state layer at all, omitting New York's mandatory employee Paid Family Leave contribution of $411.91 and disability (DBL) contribution of $31.20 — the full $443.11 shortfall." -us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model applied a 0.62% NY Paid Family Leave rate ($621.21) instead of the 0.411% that yields $411.91, overstating PFL by $209.30, and then added a further unnamed ""other NY state payroll taxes"" increment to reach $8,304.92 — a figure its own listed components ($6,212.09 + $1,452.83 + $621.21 = $8,286.13) do not produce. The correct NY layer is $411.91 PFL + $31.20 DBL = $443.11." -us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model computed NY PFL at 0.388% × $100,195 ≈ $388.76 — close to the correct $411.91 — then abandoned it for an invented cap and substituted $49.31, keeping only $80.51 of NY tax instead of $443.11. Its DBL figure of $31.20 was exactly right; the self-imposed PFL cap, which does not bind at this wage level, cost it $362.60." -us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model asserted ""NY has no mandatory employee payroll tax,"" which is false: New York statutory disability (DBL, $0.60/week = $31.20) and Paid Family Leave (0.411% of $100,195 = $411.91) are both employee-funded mandatory contributions, together $443.11 above the $7,664.92 FICA-only answer it submitted." -us,scenario_082,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model stated ""NY does not impose a mandatory employee payroll tax counted here"" and submitted rounded FICA of $7,665, omitting the employee-funded NY Paid Family Leave contribution of $411.91 and the DBL contribution of $31.20 ($0.60/week × 52) that the output definition's ""mandatory employee state payroll taxes"" clause requires." -us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model got NY DBL exactly right at $31.20 but capped the PFL base at a $89,343 statewide-average-weekly-wage ceiling and applied 0.388%, yielding $346.65; the PFL contribution here is 0.411% applied to the full $100,195 = $411.91 with no wage cap binding, leaving it $65.26 short." -us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model first declared NY SDI/PFL ""modeled here as $0"" and then bolted on an unexplained $129.39 that corresponds to no New York parameter; the actual mandatory employee state layer is PFL $411.91 (0.411% of $100,195) plus DBL $31.20, i.e. $443.11, so it understated the state component by $313.72." -us,scenario_082,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model subtracted the $181 traditional 401(k) deferral from the FICA base — elective deferrals are included in Social Security and Medicare wages, so the base is the full $100,195 — and separately asserted NY has no mandatory employee state payroll tax, dropping PFL $411.91 and DBL $31.20. The state omission accounts for $443.11 of the $457.03 shortfall." -us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model included the NY disability (DBL) contribution of $31.20 but stopped there, omitting the second mandatory NY employee contribution — Paid Family Leave at 0.411% of the full $100,195 = $411.91 — which is exactly its $411.91 shortfall from the reference." -us,scenario_082,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model applied a NY Paid Family Leave ""maximum annual employee contribution"" cap of $369.68 rather than the uncapped 0.411% × $100,195 = $411.91, and omitted the NY disability (DBL) contribution of $31.20 entirely, producing a $73.43 shortfall in the state layer." -us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly identified both NY components but sized Paid Family Leave at roughly $333 (about 0.332% of wages) instead of 0.411% × $100,195 = $411.91, and rounded DBL to $31 instead of $31.20; the understated PFL rate accounts for essentially all of the $79.03 gap." -us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied a flat 7.65% FICA rate to $100,195 and stopped, omitting New York's mandatory employee Paid Family Leave contribution ($411.91 at 0.411% of wages) and statutory disability contribution ($31.20 at $0.60/week), a combined $443.11." -us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $21,208 employer-sponsored insurance premium from the FICA base, taxing $78,987 instead of the stated gross wage of $100,195 — the payroll tax base here is the full $100,195 — which alone cost $1,622.42 in Social Security and Medicare tax. Its NY layer was also short, using $294.62 for Paid Family Leave against the correct $411.91." -us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced the FICA wage base by the $21,208 ESI premium to $78,987, computing $4,897.19 Social Security and $1,145.31 Medicare instead of $6,212.09 and $1,452.83 on the full $100,195, a $1,622.42 understatement; its ~$394 NY Paid Family Leave figure also fell short of the $411.91 PFL plus $31.20 DBL that make up the $443.11 state layer." -us,scenario_082,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model computed Social Security and Medicare on gross wages and submitted $7,664.92, omitting the state tier entirely; New York's mandatory employee Paid Family Leave ($411.91) and disability ($31.20) contributions are part of this output and account for the full $443.11 miss." -us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model applied FICA to $78,987 after ""pre-tax ESI deductions,"" but the employee payroll tax base is the stated $100,195 gross wage with no reduction for the $21,208 insurance premium, understating Social Security and Medicare by $1,622.42; its combined NY disability and PFL estimate of about $270 also fell $173 short of the $443.11 state layer." -us,scenario_082,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model set ""FICA-taxable wages"" at $78,987 by netting the $21,208 ESI premium out of the $100,195 gross wage, which understates Social Security plus Medicare by $1,622.42; the payroll tax base is the full stated wage, and its roughly $333 NY SDI/PFL add-on is short of the actual $411.91 PFL plus $31.20 DBL." -us,scenario_082,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"The model listed only the two federal components ($6,212.09 and $1,452.83) and treated payroll tax as consisting of nothing else, dropping New York's mandatory employee Paid Family Leave contribution of $411.91 and disability contribution of $31.20." -us,scenario_082,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model got NY DBL exactly right at $31.20 but computed Paid Family Leave as 0.388% against an assumed $96,200 statewide-average-weekly-wage cap ($373.26); PFL applies at 0.411% to the full $100,195 with no cap binding, giving $411.91 and leaving a $38.65 shortfall." -us,scenario_082,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model claimed ""NY has no employee-side state payroll tax,"" dropping the $411.91 Paid Family Leave and $31.20 disability contributions, and then reported $7,765 against its own stated components of $6,212 + $1,453 = $7,665 — a $100 arithmetic slip layered on the $443.11 state omission." -us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model explicitly concluded there is ""no mandatory employee state payroll tax in NY"" and submitted a rounded FICA-only $7,660; New York's employee-funded Paid Family Leave ($411.91 at 0.411% of $100,195) and statutory disability ($31.20) contributions are mandatory and add $443.11." -us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"The model computed only 6.2% and 1.45% on $100,195 and rounded to $7,659, never reaching the state tier; the missing $449.03 is New York's mandatory employee Paid Family Leave contribution of $411.91 plus the $31.20 disability contribution, with the residual from its own rounding." -us,scenario_082,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model added a lumped ""estimated NY employee PFL/DBL"" figure of $385.73 rather than deriving the two components separately: PFL at 0.411% × $100,195 = $411.91 and DBL at $0.60/week × 52 = $31.20, totaling $443.11, so its estimate was $57.38 light." -us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model captured the NY disability contribution at $31.20 but treated it as New York's only employee payroll tax, omitting the mandatory Paid Family Leave contribution of $411.91 (0.411% of $100,195) — precisely its shortfall from the reference." -us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied the combined 7.65% federal employee rate to $100,195 and submitted $7,664.92 with no state component, omitting New York's mandatory employee Paid Family Leave ($411.91) and disability ($31.20) contributions that bring the total to $8,108.03." -us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model considered only the 6.2% and 1.45% federal rates — which yield $7,664.92, not the $7,654 it submitted — and included nothing for New York's mandatory employee Paid Family Leave contribution of $411.91 or the $31.20 disability contribution." -us,scenario_082,payroll_tax,gpt-6-astra,llm_error,state_local_rule,False,"The model correctly built federal FICA of $7,664.92 and the $31.20 NY disability contribution, but priced Paid Family Leave at $432.84 (about 0.432% of wages) instead of 0.411% × $100,195 = $411.91, overstating the state layer by $20.93." -us,scenario_082,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model applied a single 7.65% rate to wages and reported $7,665 with no further components, omitting New York's mandatory employee-side Paid Family Leave contribution of $411.91 and statutory disability contribution of $31.20." -us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model set FICA wages to gross wages minus the $21,208 ESI premium ($78,987), but the employee payroll tax base is the full stated $100,195, so its Social Security and Medicare figures are $1,622.42 too low; it additionally asserted ""no NY employee payroll tax,"" dropping the $411.91 PFL and $31.20 DBL contributions." -us,scenario_082,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"The model applied a capped ""2026 PFL wage base"" to arrive at roughly $367 of Paid Family Leave instead of 0.411% of the full $100,195 = $411.91, and concluded no other employee state payroll tax applies, thereby omitting the $31.20 New York disability contribution." -us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model stripped both the $21,208 ESI premium and the $181 traditional 401(k) deferral out of the wage base to reach $78,806; both amounts remain in Social Security and Medicare wages, so the base is $100,195 and its federal tax is $1,636.28 too low, and it also omitted the $411.91 NY Paid Family Leave and $31.20 disability contributions." -us,scenario_082,payroll_tax,inkling,llm_error,state_local_rule,False,"The model identified both New York components but sized Paid Family Leave at about $365 instead of 0.411% × $100,195 = $411.91 and rounded DBL to $31 rather than $31.20, producing a $396 state layer against the actual $443.11; its whole-dollar rounding of the federal components adds the remainder of the $47.03 gap." -us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive computation was submitted; the required derivation is $6,212.09 Social Security plus $1,452.83 Medicare on $100,195, plus $411.91 NY Paid Family Leave and $31.20 NY disability, totaling $8,108.03." -us,scenario_082,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model computed the two federal components correctly and treated $7,664.92 as the complete answer, omitting New York's mandatory employee Paid Family Leave contribution of $411.91 (0.411% of $100,195) and the $31.20 statutory disability contribution." -us,scenario_082,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model verified the wage base and the $200,000 Additional Medicare threshold but never considered a state tier, submitting rounded FICA of $7,665 and omitting the $411.91 New York Paid Family Leave and $31.20 disability contributions that the output definition's mandatory employee state payroll taxes clause requires." -us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model cycled through several NY estimates — $8,038.33 with PFL at 0.373%, $8,168.80 with a $431.88 PFL cap — and then submitted $7,464.33, which is $200.59 below even its own correct FICA-only figure of $7,664.92 and corresponds to subtracting an invented negative state adjustment. The correct state layer adds $411.91 PFL plus $31.20 DBL." -us,scenario_082,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model stated its components as $6,212.09 plus $1,452.83 with ""no mandatory employee state payroll tax"" — a total of $7,664.92 — yet submitted $7,752.42, $87.50 above its own arithmetic; the actual gap is New York's mandatory $411.91 Paid Family Leave and $31.20 disability contributions." +us,scenario_082,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Computed federal FICA correctly at $7,664.92 but left out New York's mandatory employee PFL ($411.91) and DBL ($31.20) contributions, calling them negligible or excluded. The output definition explicitly includes mandatory employee state payroll taxes, so $443.11 is missing." +us,scenario_082,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Summed only employee Social Security ($6,212.09) and Medicare ($1,452.83). It left out New York's mandatory employee PFL contribution (0.432% capped at $411.91) and DBL contribution ($31.20), both of which the payroll_tax output includes." +us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"Applied a made-up 0.62% NY PFL rate to the full $100,195 without the wage cap ($621.21), then added an unspecified plug for other NY withholdings. The 2026 PFL is 0.432% capped at $411.91 and DBL is $31.20, so NY payroll tax is overstated by about $197." +us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Correctly identified DBL at $31.20 but then dropped its own PFL estimate and replaced it with an unsupported $49.31. That gives only $80.51 of NY contributions instead of $443.11. The 2026 PFL rate is 0.432%, not the 2025 rate of 0.388%, and the $95,350.72 wage cap makes the PFL contribution $411.91." +us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Wrongly stated that New York has no mandatory employee payroll tax. It therefore left out the employee PFL contribution ($411.91 at the 2026 cap) and the DBL contribution ($31.20), reporting only the $7,664.92 federal FICA." +us,scenario_082,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Asserted that NY imposes no mandatory employee payroll tax for this output and reported only rounded FICA ($7,665). The NY PFL ($411.91) and DBL ($31.20) employee contributions are mandatory and belong in payroll_tax." +us,scenario_082,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"Stopped at federal FICA ($7,664.92) on the claim that no NY employee payroll tax is modeled. NY PFL (0.432% capped at $411.91) and DBL ($0.60/week = $31.20) are both included in the payroll tax total." +us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Included DBL correctly at $31.20 but computed PFL with the 2025 parameters (0.388% on an ~$89,343 cap = $346.65). The 2026 parameters are 0.432% on a $95,350.72 cap, giving $411.91, so the total is short by $65.26." +us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Treated NY SDI/PFL as minimal and added an unsupported $129.39 estimate. The actual 2026 contributions are $411.91 for PFL (0.432% capped at the $95,350.72 wage base) plus $31.20 for DBL, so NY payroll tax is understated by about $314." +us,scenario_082,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Declared that no NY payroll tax is modeled and reported only FICA of $7,664.92. It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Subtracted the $181 traditional 401(k) deferral from FICA wages. Elective deferrals remain Social Security and Medicare wages under IRC §3121(v)(1)(A), so the base stays $100,195. It also asserted NY has no employee payroll tax, dropping PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Included NY DBL ($31.20) but left out the mandatory NY PFL employee contribution. In 2026 that is 0.432% of wages capped at $411.91, so the total is short by exactly that amount." +us,scenario_082,payroll_tax,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used an incorrect NY PFL maximum of $369.68 instead of the 2026 maximum of $411.91 (0.432% × $95,350.72). It also left out the $31.20 NY DBL employee contribution." +us,scenario_082,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used the correct 2026 PFL rate of 0.432% but applied it to all $100,195 of wages ($432.84) without the statewide average weekly wage cap of $95,350.72. That cap limits the contribution to $411.91, so the total is overstated by $20.93." +us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Included DBL ($31) but used a stale NY PFL figure of about $333 (the 2024 maximum of $333.25). The 2026 PFL maximum is $411.91 (0.432% on a $95,350.72 cap), so the total falls about $79 short." +us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Applied only the 7.65% federal FICA rate to wages ($7,665). It left out New York's mandatory employee PFL ($411.91) and DBL ($31.20) contributions, which are part of payroll_tax." +us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Subtracted the $21,208 employer-sponsored insurance premium from wages, giving a FICA base of $78,987. The payroll tax base is the full $100,195, so Social Security and Medicare come to $7,664.92. It also applied PFL to the reduced base at a stale rate ($294.62) instead of the 2026 capped $411.91." +us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Reduced FICA wages to $78,987 by subtracting the $21,208 ESI premium, which understates Social Security and Medicare by $1,622.42. It also used about $394 for PFL instead of $411.91 and left out the $31.20 NY DBL contribution." +us,scenario_082,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only employee Social Security and Medicare on gross wages ($7,664.92). It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Computed FICA on $78,987 after subtracting the $21,208 ESI premium, instead of the full $100,195 wage base. That understates federal payroll tax by $1,622.42. Its NY PFL/DBL add-on of about $270 also falls short of the correct $443.11." +us,scenario_082,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Treated the $21,208 ESI premium as reducing FICA wages to $78,987, but Social Security and Medicare apply to the full $100,195. Its NY SDI/PFL amount of about $333 also understates the correct $443.11 ($411.91 PFL + $31.20 DBL)." +us,scenario_082,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Reported only employee Social Security ($6,212.09) and Medicare ($1,452.83). It left out New York's mandatory employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"Included DBL ($31.20) but computed PFL with the 2025 rate of 0.388% on a guessed $96,200 cap ($373.26). The 2026 rate is 0.432% on the $95,350.72 cap, giving $411.91, so the total is short by $38.65." +us,scenario_082,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Asserted NY has no employee-side payroll tax, leaving out PFL ($411.91) and DBL ($31.20). It also mis-added its own components: $6,212 + $1,453 is $7,665, not $7,765." +us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Asserted NY has no mandatory employee payroll tax and reported a rounded-down FICA of $7,660 (the exact figure is $7,664.92). It left out the NY PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Reported only federal Social Security and Medicare, under-rounded to $7,659 instead of $7,664.92. It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Its NY add-on of $385.73 equals the 2025 PFL maximum ($354.53) plus DBL ($31.20). It used the 2025 cap instead of the 2026 PFL maximum of $411.91 (0.432% × $95,350.72), so the total is short by $57.38." +us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,Added the NY DBL contribution ($31.20) but left out the mandatory NY PFL employee contribution. In 2026 that is 0.432% of wages capped at $411.91. +us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Applied only the 7.65% federal FICA rate ($7,664.92). It left out New York's mandatory employee PFL ($411.91) and DBL ($31.20) contributions, which the payroll_tax output includes." +us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Reported only Social Security and Medicare, and understated even that figure ($7,654 instead of $7,664.92). It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions entirely." +us,scenario_082,payroll_tax,gpt-6-astra,llm_error,thresholds_rates,False,"Applied the 2026 PFL rate of 0.432% to all $100,195 of wages ($432.84) without the $95,350.72 statewide average weekly wage cap. That cap limits the PFL contribution to $411.91, so the total is overstated by $20.93." +us,scenario_082,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Applied only the 7.65% federal FICA rate ($7,665). It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"Subtracted the $21,208 ESI premium from wages, giving a FICA base of $78,987, when Social Security and Medicare apply to the full $100,195. It also asserted NY has no employee payroll tax, dropping PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Used a stale NY PFL figure of about $367 instead of the 2026 maximum of $411.91 (0.432% on the $95,350.72 cap). It also stated that no other employee state payroll tax applies, which leaves out the $31.20 NY DBL contribution." +us,scenario_082,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"Reported only employee Social Security and Medicare, rounded to $7,665. It left out New York's mandatory employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted both the $21,208 ESI premium and the $181 traditional 401(k) deferral from FICA wages. Elective deferrals are FICA wages, and the payroll tax base is the full $100,195. It also left out NY PFL ($411.91) and DBL ($31.20)." +us,scenario_082,payroll_tax,inkling,llm_error,thresholds_rates,False,"Included DBL (~$31) but estimated PFL at about $365 using outdated parameters. The 2026 PFL is 0.432% of wages capped at $95,350.72, which gives $411.91, so the total is short by about $47." +us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for payroll_tax, so there is no answer to score. The correct derivation is $6,212.09 Social Security + $1,452.83 Medicare + $411.91 NY PFL + $31.20 NY DBL = $8,108.03." +us,scenario_082,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Computed federal FICA ($7,664.92, rounded to $7,665) and stopped there. It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions." +us,scenario_082,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Summed only employee Social Security and Medicare ($7,665). It left out the mandatory NY employee PFL ($411.91) and DBL ($31.20) contributions, which the payroll_tax output includes." +us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,other,False,"Correctly found FICA of $7,664.92 and even reached a NY PFL estimate near the cap. It then subtracted an invented '~$-200' of NY deductions and reported $7,464.33, which is below FICA alone. NY PFL ($411.91) and DBL ($31.20) are added to FICA, never subtracted." +us,scenario_082,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Explicitly excluded NY PFL ($411.91) and DBL ($31.20), even though the output definition requires mandatory employee state payroll taxes. It also mis-added its own components: $6,212.09 + $1,452.83 is $7,664.92, not $7,752.42." us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It reached NYAGI of $117,654 by skipping New York's $68 state-tax-refund subtraction and then priced taxable income with the 2025 4/4.5/5.25/5.5 schedule, inflating the bracket tax to $5,891. It subtracted a $396 'nonrefundable' New York child and dependent care credit, but NY's CDCC is refundable and belongs in state_refundable_credits, and it never added the $156.38 supplemental tax that NYAGI above $107,650 triggers." -us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It recovered the 2026 HOH rate schedule (3.9/4.4/5.15/5.4) correctly but built the supplemental tax from the wrong recapture base: it used the $248.63 bracket benefit measured against the 5.4% rate instead of the $786.88 benefit measured against the 5.9% rate that applies once NYAGI passes $107,650, producing $49.75 where NY's worksheet gives $156.38. It also omitted the $68 state-refund subtraction, leaving its taxable income $69 above the correct $105,385.15." -us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It set New York taxable income equal to federal taxable income after a $14,600 federal standard deduction, discarding NY's own $11,200 HOH standard deduction and $1,000 dependent exemption; New York starts from federal AGI ($117,585 here), not federal taxable income. Its own $62,149 base yields about $3,107 under the 2026 HOH schedule, so the $5,649 it reported is not consistent with the arithmetic it described." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed the household as single despite the qualifying dependent child, taking the $8,000 single standard deduction instead of the $11,200 HOH amount, skipping the $1,000 dependent exemption, and using the single schedule that reaches 6% above $80,650 rather than HOH's 5.4% through $107,650. After computing $6,221 it wrote the answer down to $5,076 with no credit to justify the $1,145 reduction, and it never added the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It applied the $8,000 single standard deduction in place of the $11,200 head-of-household deduction and omitted the $1,000 dependent exemption, then used a fabricated '$4,579 + 6.00% over $80,650' schedule instead of NY's 2026 HOH rates. Having correctly found the household credit to be $0 at this income, it still cut its own $6,326 result to $5,276 as an unexplained 'modest adjustment,' and it omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It paired the HOH bracket schedule with the $8,000 single standard deduction and omitted the $1,000 dependent exemption, overstating taxable income by about $4,270. It then rounded to '$6,000' at a guessed 5.5%-6% marginal rate instead of running the 2026 HOH schedule (3.9/4.4/5.15/5.4 through $107,650), which gives $5,442.17 plus $156.38 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied the married-filing-jointly bracket thresholds ($17,150/$23,600/$27,900/$161,550) with a nonexistent 5.85% rate rather than the 2026 HOH schedule that taxes this income at 5.4% through $107,650, and it used a single-filer frame with $9,917 of itemized deductions instead of the $11,200 HOH standard deduction plus the $1,000 dependent exemption. It then subtracted a $120 NY child and dependent care credit, which New York makes refundable, and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After computing $6,193 from a fabricated single-filer '$4,504 + 6.25% over $80,650' formula, it cut roughly $1,400 for 'nonrefundable credits' citing the NY household credit, which is fully phased out above $32,000 of NYAGI, while NY's CDCC and Empire State Child Credit are refundable and excluded from this output. It also used the single $8,000/itemized frame instead of the $11,200 HOH standard deduction and $1,000 dependent exemption, and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It set federal AGI at $86,949 by absorbing the full $30,711 net capital loss rather than the $3,000 annual deduction limit, understating NYAGI by roughly $30,600, and then used a $10,000 standard deduction and $2,000 of exemptions in place of the $11,200 HOH deduction and $1,000 dependent exemption. It closed by subtracting a $217 Empire State Child Credit, which is refundable and excluded from this pre-refundable-credit output." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of NY itemized deductions against the $11,169 the facts support ($11,057 of SALT plus $112.07 of charity after the 0.5%-of-AGI floor), so it never reached the $11,200 HOH standard-deduction floor and understated taxable income by $9,414. It also used the 2025 5.5% top-applicable rate rather than 2026's 5.4% and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It got NYAGI ($117,594 after the $68 refund subtraction) and the $11,200 HOH standard deduction right but dropped the $1,000 dependent exemption, leaving $106,394 of taxable income instead of $105,385. It then subtracted a $120 New York child and dependent care credit; NY's CDCC is refundable and belongs in state_refundable_credits, not in this pre-refundable-credit figure." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 HOH amount and omitted the $1,000 dependent exemption, arriving at $107,086 of taxable income, then subtracted a $30 household credit that is fully phased out above $32,000 of NYAGI. It also omitted the $156.38 supplemental tax that NYAGI of $117,585 triggers." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It taxed 'annual wage income after standard deduction,' discarding the $20,480 of dividends, the $2,289 of short-term gains net of the $3,000 capital-loss allowance, and the $100 of interest that carry NYAGI to $117,585. Its $4,380 corresponds to roughly $85,700 of taxable income under the 2026 HOH schedule, about $19,700 below the correct $105,385, and it includes none of the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $84,178 of taxable income is exactly $21,207 below the correct $105,385: it deducted the $21,208 employer-sponsored insurance premium a second time, though that amount is already excluded from the $100,195 of stated wages and is not a New York deduction. It also priced the result with the 2025 4/4.5/5.25/5.5 schedule (its $4,381.17 is that schedule applied to $84,178) rather than 2026's 3.9/4.4/5.15/5.4 rates, and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its NYAGI of $96,454 is federal AGI of $117,662 less the $21,208 employer-sponsored insurance premium — a second deduction of an amount already excluded from the stated wages — and it then removed $17,200 of deductions and exemptions instead of the $11,200 HOH standard deduction plus the $1,000 dependent exemption. The two errors put taxable income $26,131 below the correct $105,385, and it omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted $6,040.50 with no derivation; that figure is an effective 5.73% of the correct $105,385 taxable income, i.e., a near-flat rate that ignores the 3.9%, 4.4% and 5.15% brackets below $20,900 in New York's 2026 HOH schedule. Running that schedule gives $5,442.17 of main tax, and the $107,650 NYAGI threshold adds $156.38 of supplemental tax for $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $4,220 corresponds to about $82,750 of NY taxable income under the 2026 HOH schedule, roughly $22,600 below the correct $105,385 — the size of the $20,480 of dividend income plus $2,289 of short-term gains it left out of NYAGI. It also omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,591 corresponds to about $71,100 of NY taxable income, roughly $34,300 below the correct $105,385, consistent with deducting the whole $30,711 net capital loss instead of the $3,000 annual limit. It then shaved 'allowable nonrefundable credits' that do not exist at this income: the household credit is zero above $32,000 of NYAGI, and the CDCC and Empire State Child Credit are refundable." -us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It set the base up correctly with the $11,200 HOH standard deduction and $1,000 dependent exemption but priced it with the pre-2026 4/4.5/5.25/5.5 schedule instead of 2026's reduced 3.9/4.4/5.15/5.4 rates. It also omitted the $156.38 supplemental tax that NYAGI of $117,585 — above the $107,650 threshold — triggers." -us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no substantive computation to assess. The correct chain is $105,385.15 of taxable income yielding $5,442.17 of main tax plus $156.38 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It built $27,966 of NY itemized deductions on a fabricated $13,349 medical deduction — the household's medical, OTC and child medical costs total $1,965, entirely below the 7.5%-of-AGI floor of $8,819 — and added roughly $4,700 of NY income tax to SALT, which New York excludes from its itemized deduction. The real itemized total is $11,169.24, below the $11,200 HOH standard deduction, and the $120 NY child and dependent care credit it subtracted is refundable, not nonrefundable." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It submitted an unreasoned estimate rather than running the schedule; $5,150 corresponds to about $99,975 of NY taxable income under the 2026 HOH rates, roughly $5,400 short of the correct $105,385.15. It also includes none of the $156.38 supplemental tax that NYAGI of $117,585 triggers." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted $7,777 as a self-described 'rounded estimate' instead of computing the schedule, overshooting the correct $5,598.55 by $2,178. Its stated basis is also wrong: medical and OTC costs of $1,965 are entirely below the 7.5%-of-AGI floor and charity contributes only $112.07 after the 0.5%-of-AGI floor, leaving itemized deductions of $11,169 below the $11,200 standard deduction." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Every input was right — NYAGI $117,594 after the $68 refund subtraction, the $11,200 HOH standard deduction, the $1,000 dependent exemption, taxable income $105,394 — but it priced them with the 2025 schedule (base $900.88 plus 5.5% over $20,900), which reproduces its $5,548 exactly, instead of 2026's reduced 3.9/4.4/5.15/5.4 rates. It also omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,954.80 corresponds to roughly $59,300 of NY taxable income, about $46,000 below the correct $105,385.15 — consistent with taxing wages alone after absorbing the entire $30,711 net capital loss instead of the $3,000 annual limit and dropping the $20,480 of dividends and $100 of interest. It also omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Its stated $106,394 of taxable income is NYAGI less the $11,200 standard deduction with the $1,000 dependent exemption omitted. Its $5,815 is the single-filer schedule (5.5% to $80,650, then 6%) applied to that base, not the head-of-household schedule that stays at 5.4% through $107,650, and it omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It omitted the $1,000 dependent exemption, leaving $106,394 of taxable income instead of $105,385.15, and priced it with the single-filer schedule that jumps to 6% above $80,650 rather than the head-of-household schedule's 5.4% through $107,650. That single-schedule tax of about $5,815 plus roughly $157 of supplemental tax is its $5,977.08." -us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It claimed about $23,624 of NY itemized deductions against the $11,169.24 the facts support ($11,057.16 of SALT plus $112.07 of charity after the 0.5%-of-AGI floor), an amount that is itself below the $11,200 HOH standard deduction. Its $92,961.65 taxable income is therefore $12,423.50 too low, even though it correctly included the income-based supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It submitted a bare $5,500 with no derivation; that is the $5,442.17 main NY tax rounded, with the $156.38 supplemental tax left out entirely. NYAGI of $117,585 exceeds the $107,650 threshold, so the recapture applies: $786.88 of base benefit times the 19.87% phase-in over $50,000." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its NYAGI of $96,454 is federal AGI of $117,662 less the $21,208 employer-sponsored insurance premium, deducted a second time despite already being excluded from the $100,195 of stated wages. It also used a $12,100 standard deduction rather than the $11,200 HOH amount and subtracted a $120 NY child and dependent care credit, which is refundable in New York and excluded from this output." -us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got NYAGI ($117,594), the $11,200 HOH standard deduction, the $1,000 dependent exemption and taxable income ($105,394) exactly right, then applied the 2025 rate schedule — '901 plus 5.5 percent of the amount over 20,900' — instead of 2026's reduced 3.9/4.4/5.15/5.4 rates, which give $5,442.17. It also omitted the $156.38 supplemental tax triggered by NYAGI above $107,650." -us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It claimed $2,000 of exemptions for a single dependent child where New York allows $1,000 each, and skipped the $68 state-refund subtraction from federal AGI. It then subtracted a $120 New York child and dependent care credit, which is refundable and belongs in state_refundable_credits, and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no substantive computation to assess. The correct chain is $105,385.15 of taxable income yielding $5,442.17 of main tax plus $156.38 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It carried the base exactly right — $117,594 of NYAGI after the $68 refund subtraction, the $11,200 HOH standard deduction, the $1,000 dependent exemption, $105,394 of taxable income — then taxed it 'through the 5.75% bracket,' a rate that does not exist in New York's schedule. The 2026 head-of-household schedule stays at 5.4% through $107,650, giving $5,442.17, to which the $107,650 NYAGI threshold adds $156.38 of supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the entire $33,000 long-term capital loss against ordinary income instead of the $3,000 annual deduction limit, and then treated a child tax credit as zeroing out New York tax. NY taxable income is $105,385.15, and neither the federal CTC nor the Empire State Child Credit reduces state tax before refundable credits; the correct liability is $5,598.55." -us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used the $8,000 single standard deduction as if it were the head-of-household amount ($11,200), inflating taxable income by $3,200, and applied a 5.85% middle rate where the 2026 HOH schedule uses 5.4% through $107,650. It then subtracted a $120 NY child and dependent care credit, which New York makes refundable, and omitted the $156.38 supplemental tax." -us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrote net capital gains as −$30,711 and summed that into AGI in full, ignoring the $3,000 annual capital-loss limit and cutting NYAGI to $89,951 instead of $117,585, then took the $8,000 single standard deduction rather than the $11,200 HOH amount. Its own bracket arithmetic produced $4,292.89, and it then added $462.55 to that figure while describing the addition as a household credit." -us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $139,586, about $22,000 above the correct $117,586, then used the $8,000 single standard deduction instead of the $11,200 HOH amount and invented a $5,516 'federal deduction adjustment.' It closed by subtracting a $75 household credit that is zero above $32,000 of NYAGI, a $330 Empire State Child Credit and a $116 'NY CTC' — all refundable or nonexistent, and excluded from this pre-refundable-credit output." -us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Computed the Empire State Child Credit under the superseded rule of 33% of the pre-TCJA $1,000 federal CTC ($330) instead of the enhanced under-4 credit of $1,000 reduced by $16.50 per $1,000 of AGI over the $75,000 head-of-household threshold, which yields $290.50. It then guessed roughly $150 for the NY dependent care credit rather than computing the $3,000 expense cap times the 20% federal rate times New York's 0.60 applicable percentage at this AGI, which is $360." -us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It reconstructed the ESCC correctly in structure ($1,000 less 1.65% of AGI over $75,000, off by $5.71 only because the statutory reduction runs in $16.50-per-$1,000 steps and its AGI differed slightly). The $240 shortfall is entirely the dependent care credit: it applied the pre-expansion rule that NY AGI over $65,000 gets 20% of the federal credit, missing New York's expanded applicable-percentage schedule that pays 0.60 of the $600 federal credit at NY AGI $117,585, or $360." -us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that New York has no refundable child credit or dependent care credit; both the Empire State Child Credit and the NY CDCC are refundable state credits. At AGI $117,585.15 the $1,000 under-4 ESCC retains $290.50 after the $75,000 HOH phase-out, and the CDCC pays 60% of the $600 federal credit for $360." -us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Invented a $110 per-child under-4 ESCC amount in place of the statutory $1,000 base reduced by $16.50 per $1,000 of AGI over $75,000, which gives $290.50. It also declared the NY child and dependent care credit nonrefundable above roughly $50,000 of AGI; that credit is fully refundable for full-year residents and equals $360 here (0.60 of the $600 federal credit)." -us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Treated the Empire State Child Credit as fully phased out and the dependent care credit as nonrefundable. The ESCC reduction of $16.50 per $1,000 of excess AGI removes only $709.50 of the $1,000 under-4 credit at $117,585.15 of AGI, leaving $290.50, and the refundable NY CDCC has no income cutoff, adding $360." -us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Declared the income too high for every New York credit without computing either phase-out. The enhanced under-4 ESCC loses $709.50 to the $75,000 HOH phase-out and still pays $290.50, and the NY dependent care credit is refundable at every income level, paying 0.60 of the $600 federal credit for $360." -us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the repealed ESCC formula — greater of $100 or 33% of the current $2,000 federal CTC, with no income phase-out — producing $660 instead of the enhanced $1,000 under-4 credit reduced by $16.50 per $1,000 of AGI above $75,000, which is $290.50. It also used the pre-expansion 20%-of-federal CDCC rate for NY AGI over $75,000 ($120) rather than the 0.60 applicable percentage that yields $360." -us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Concluded the household's NY AGI put it above the Empire State Child Credit range and considered no dependent care credit at all. The under-4 ESCC phases out at $16.50 per $1,000 above $75,000 and still pays $290.50 at AGI $117,585.15, and the refundable NY CDCC adds $360." -us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Evaluated only the NY earned income credit (30% of a zero federal EITC) and stated no other refundable NY credits exist. The two credits that produce the reference are the Empire State Child Credit of $290.50 for the child under age 4 and the refundable NY child and dependent care credit of $360. -us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Found the dependent care pathway but applied the pre-expansion 20%-of-federal rate, giving $120 instead of New York's 0.60 applicable percentage on the $600 federal credit, which is $360. It also omitted the $290.50 Empire State Child Credit for the one-year-old entirely." -us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied the pre-2023 age gate that the Empire State Child Credit covers only children age 4 and older; New York pays $1,000 for a child under 4 in 2026, reduced to $290.50 by the $16.50-per-$1,000 phase-out above $75,000. It also treated the NY EITC as the only other candidate, missing the $360 refundable dependent care credit." -us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Assembled $693 from three wrong parts: the superseded 33%-of-federal-CTC ESCC of $330 instead of the $290.50 enhanced under-4 credit after phase-out, an unspecified $300 estimate for the dependent care credit instead of the 0.60 × $600 = $360 computation, and a $63 NYC school tax credit for a household with no New York City residence listed." -us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Used the superseded ESCC amount of 33% of the $1,000 pre-TCJA federal CTC ($330) instead of the enhanced $1,000 under-4 credit reduced by $709.50 through the $75,000 HOH phase-out, which is $290.50. It also omitted the refundable NY child and dependent care credit of $360, the larger of the two components." -us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Fell back to the repealed $100 ESCC floor rather than the enhanced $1,000 under-4 credit less the $16.50-per-$1,000 phase-out, which pays $290.50. Its dependent care figure used the pre-expansion 20%-of-federal rate ($120) instead of New York's 0.60 applicable percentage at NY AGI $117,585, which gives $360." -us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied the 110% dependent care applicable percentage, which is reserved for NY AGI at or below $25,000, to a household at $117,585 of AGI, producing $660 instead of the 0.60 rate that yields $360. It paired that with the superseded $330 ESCC rather than the $290.50 enhanced under-4 credit after phase-out." -us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Submitted zero with no derivation. The correct build is the Empire State Child Credit of $1,000 for a child under 4 less a $709.50 phase-out at AGI $117,585.15, or $290.50, plus the refundable NY dependent care credit of 0.60 × $600 = $360; the zero is consistent with treating all NY credits as barred by income, which no NY rule does here." -us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the superseded 33%-of-federal-CTC ESCC figure of $330 rather than the enhanced $1,000 under-4 credit reduced by $16.50 per $1,000 of AGI over $75,000, which is $290.50. It also omitted the refundable NY child and dependent care credit of $360." -us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Reported the legacy $330 Empire State Child Credit instead of the 2026 enhanced under-4 credit of $1,000 less the $709.50 phase-out above the $75,000 HOH threshold, which is $290.50. It never evaluated the NY child and dependent care credit, which contributes $360 of the $650.50 total." -us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Applied the pre-expansion 20%-of-federal NY dependent care rate to get $120 instead of the 0.60 applicable percentage on the $600 federal credit, which is $360. It also asserted the Empire State Child Credit phases out at this income, when the $16.50-per-$1,000 reduction above $75,000 leaves $290.50." -us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_refundable_credits, so the submission carried no substantive computation to evaluate. The required answer is the $290.50 Empire State Child Credit plus the $360 refundable NY dependent care credit, totaling $650.50." -us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Claimed New York's enhanced credit equals 100% of the federal CTC and submitted the $2,200 federal per-child amount. The enhanced NY credit is a flat $1,000 for a child under age 4, reduced by $16.50 per $1,000 of AGI above the $75,000 HOH threshold to $290.50, and the model also omitted the $360 refundable dependent care credit." -us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Asserted that the facts indicate no refundable New York credits, never evaluating the two that apply: the Empire State Child Credit for the one-year-old, worth $290.50 after the $75,000 HOH phase-out, and the refundable NY child and dependent care credit on the $9,600 of childcare expenses, worth $360." -us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Used the head's high wages as a blanket disqualifier and demanded an explicit refundable-credit input. The $9,600 of childcare expenses and the child under age 4 are the only inputs required: they generate a $360 NY dependent care credit and a $290.50 Empire State Child Credit after the $16.50-per-$1,000 phase-out above $75,000." -us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Correctly identified the $600 federal dependent care credit but applied New York's pre-expansion 20% rate rather than the 0.60 applicable percentage at NY AGI $117,585, understating the CDCC by $240. It further concluded no NY child credit remains, when the enhanced $1,000 under-4 credit retains $290.50 after its $709.50 phase-out." -us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Took the full $1,000 enhanced under-4 Empire State Child Credit and skipped the phase-out entirely; the $16.50-per-$1,000 reduction on $42,585 of AGI above the $75,000 HOH threshold cuts $709.50, leaving $290.50. It also omitted the $360 refundable NY dependent care credit." -us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Reproduced the Empire State Child Credit exactly at $290.50 but valued the NY dependent care credit at $120 using the pre-expansion 20%-of-federal rate. New York's applicable percentage at NY AGI $117,585 is 0.60, so the credit on the $3,000 capped expenses at the 20% federal rate is $360, a $240 shortfall that accounts for the entire error." -us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the pre-2023 rule that the Empire State Child Credit requires a child age 4 or older, and stacked an income bar on top of it. New York's enhanced credit pays $1,000 for a child under 4, reduced to $290.50 at this AGI, and the refundable NY dependent care credit of $360 has no income cutoff." -us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Computed the Empire State Child Credit correctly at $290.50 after its income phase-out, then applied the pre-expansion 20%-of-federal rate to the dependent care credit for $120. The NY applicable percentage at NY AGI $117,585 is 0.60 against the $600 federal credit, giving $360 and the missing $240." -us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Stated flatly that no qualifying state refundable credits apply, with no evaluation of New York's schedule. The household qualifies for the Empire State Child Credit of $290.50 for the child under 4 after the $75,000 HOH phase-out and the refundable NY dependent care credit of $360 on the $9,600 of childcare expenses." -us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Zeroed the Empire State Child Credit on the claim that the federal CTC is fully phased out; the federal CTC phase-out for head of household begins at $200,000, so it is intact at $117,585, and New York's enhanced under-4 credit is a flat $1,000 with its own $75,000 threshold, leaving $290.50. It also missed the $360 refundable NY dependent care credit, which is independent of the EITC it did check." -us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Fell back to the repealed $100 ESCC floor after wrongly concluding the federal CTC is phased to zero; the 2026 NY credit for a child under 4 is $1,000 reduced by $16.50 per $1,000 of AGI over $75,000, or $290.50. Its dependent care figure used the pre-expansion 20% rate rather than the 0.60 applicable percentage, giving $120 instead of $360." -us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Reasoned only from the zero federal EITC to a zero state EITC and asserted no other qualifying credits. The reference is built from the Empire State Child Credit of $290.50 for the one-year-old and the refundable NY child and dependent care credit of $360, neither of which depends on the EITC." -us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as absorbed against tax liability; the NY child and dependent care credit is refundable for full-year residents and pays $360 here (0.60 of the $600 federal credit) regardless of liability. It also considered only the EITC as a child-related credit, omitting the $290.50 Empire State Child Credit." -us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were submitted for state_refundable_credits, leaving no computation to assess. The required figure is $650.50: a $290.50 Empire State Child Credit after phase-out plus a $360 refundable NY dependent care credit." -us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Recomputed a 2017-law federal CTC to zero and fell back to the repealed $100 ESCC floor; New York's 2026 credit for a child under 4 is a flat $1,000 independent of the federal CTC, reduced by $16.50 per $1,000 of AGI over $75,000 to $290.50. It also applied the pre-expansion 20%-of-federal dependent care rate ($120) instead of the 0.60 applicable percentage ($360)." -us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Reasoned from near-zero NY liability and an assumed income range cutoff to zero credits. Both NY credits here are refundable and paid without regard to liability, and the enhanced under-4 Empire State Child Credit retains $290.50 after its $709.50 phase-out while the dependent care credit adds $360." -us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Used a 5%-of-excess phase-out rate on the $1,000 under-4 Empire State Child Credit, which zeroes it at $95,000 of AGI; the statutory reduction is $16.50 per $1,000 of excess over the $75,000 HOH threshold, so $42,585 of excess cuts $709.50 and leaves $290.50. It also omitted the $360 refundable NY dependent care credit." -us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Tested only the NY earned income credit against the household's income and concluded no refundable credits apply. The listed facts support two others: the Empire State Child Credit of $290.50 for the child under 4 after the $75,000 phase-out and the refundable NY dependent care credit of $360 on the $9,600 of childcare expenses." -us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Named the child credit as the source but submitted an unsupported $313 with no computation, and treated it as only partially refundable. The Empire State Child Credit is fully refundable at $290.50 after the $16.50-per-$1,000 phase-out above $75,000, and the separate refundable NY dependent care credit adds $360." +us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It subtracted a ~$396 NY child and dependent care credit as if it were nonrefundable, but New York's CDCC is refundable and excluded from this output. It also never added the $156.38 supplemental recapture tax that applies because NY AGI is above $107,650. Its ~$5,891 bracket tax on $105,454 is too high: the 2026 HOH schedule (3.9%/4.4%/5.15%/5.4%) gives about $5,446." +us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its bracket tax was right (~$5,446), but it put the supplemental recapture tax at only $49.75 by measuring the benefit against the 5.4% rate of its own bracket. New York's recapture for a HOH filer with NY AGI above $107,650 comes to $156.38 here. It also skipped the $68 state-refund subtraction, which slightly overstates taxable income." +us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used federal taxable income ($62,149, after the $14,600 federal standard deduction) as NY taxable income. The correct path starts from NY AGI of $117,585 and subtracts the $11,200 NY HOH standard deduction and the $1,000 dependent exemption, giving $105,385.15. Its $5,649 is not even the NY tax on $62,149 (about $3,100 on the HOH schedule), and it leaves out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed the head as single despite a qualifying 1-year-old child. That meant the $8,000 single standard deduction instead of $11,200 HOH, no $1,000 dependent exemption, and single brackets. It also overstated AGI at ~$119,943 (federal AGI is $117,654). It then cut its $6,221 to $5,076 with an unexplained ~$1,145 reduction, although no nonrefundable NY credit applies." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It filed as single rather than head of household: the $8,000 single deduction, no $1,000 dependent exemption, and 6% on income above $80,650. It also skipped the $181 traditional 401(k) and $8 IRA reductions. It then took ~$1,050 off its $6,326 for a 'child/dependent care nonrefundable portion', but New York's CDCC is refundable and the household credit is zero at this income." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used head of household but took an ~$8,000 standard deduction (the single amount) instead of the $11,200 HOH deduction, and it left out the $1,000 dependent exemption. That gave taxable income of ~$109,654 instead of $105,385.15. It then rounded to $6,000 using 5.5%–6% marginal rates, instead of the 2026 HOH schedule (5.4% in this bracket) plus the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It got taxable income right ($105,386) but put the 2026 HOH bracket tax at $5,808.15. The schedule gives $5,442.17 (12,800×3.9% + 4,850×4.4% + 3,250×5.15% + 84,485×5.4%). It also understated the supplemental recapture tax at $81.40 versus $156.38." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated the head as single, so it compared $9,917 of itemized deductions with the $8,000 single deduction instead of $11,200 HOH. It dropped the $1,000 dependent exemption and used made-up single brackets (5.85% above $27,900). It also subtracted a $120 NY child and dependent care credit, which is refundable and excluded here, and never added the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It filed as single and itemized $9,917 against the $8,000 single deduction, missing the $11,200 HOH standard deduction and the $1,000 dependent exemption. It then used single brackets (6.25% above $80,650) to get $6,193. It cut that to $4,800 for the household credit, which is fully phased out at this income; no nonrefundable NY credit applies." +us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Its taxable income was right (~$105,394), but it rounded the graduated tax to $5,400. It never added the $156.38 supplemental recapture tax, which applies because NY AGI of $117,585 exceeds $107,650. The 2026 HOH schedule gives $5,442.17, so the total is $5,598.55." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It put federal AGI at ~$86,949, which matches deducting the net capital loss far beyond the $3,000 annual cap; NY AGI is $117,585. It also used a ~$10,000 standard deduction instead of $11,200 and claimed $2,000 of exemptions when only the child gets the $1,000 exemption. It subtracted the Empire State Child Credit, which is refundable and excluded from this output." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of NY itemized deductions. The NY itemized total here is only $11,169 (SALT $11,057 plus a reduced $112 charitable deduction), below the $11,200 HOH standard deduction, so taxable income is $105,385.15, not $95,971. It also used pre-2026 rates (5.5% instead of 5.4%) with wrong bracket widths and left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It left out the $1,000 dependent exemption for the child, so taxable income was $106,394 instead of $105,385.15. Its $5,731.77 pre-credit tax does not match the 2026 HOH schedule plus supplemental tax. It then subtracted a $120 NY child and dependent care credit, which is refundable in New York and excluded from tax before refundable credits." +us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted $2,000 of exemptions, but New York gives the filer no exemption, only $1,000 per dependent, so taxable income is $105,385.15, not $104,386. It also left out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 HOH amount and left out the $1,000 dependent exemption, giving taxable income of $107,086 instead of $105,385.15. It then subtracted a $30 household credit that is not available at this income. It never added the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its only explanation was 'wage income after standard deduction'. The $4,380 matches taxing about $84,150 at pre-2026 HOH rates, which is what you get by subtracting the $21,208 employer health premiums from the listed wages; those wages count in full in AGI. The correct path taxes $105,385.15 at 2026 HOH rates ($5,442.17) and adds the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $84,178 taxable income equals NY AGI minus the $21,208 employer-sponsored insurance premiums minus $12,200 of deduction and exemption. It wrongly took the premiums out of the listed gross wages; taxable income is $105,385.15. It also used pre-2026 HOH rates (4%/4.5%/5.25%/5.5%). Pushing AGI below $107,650 made it miss the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut NY AGI to $96,454 by subtracting the $21,208 employer-sponsored insurance premiums, but the listed wages are fully taxable and NY AGI is $117,585. It then subtracted $17,200 instead of the $11,200 HOH standard deduction plus the $1,000 dependent exemption. With AGI below $107,650 in its calculation, it also missed the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $6,040.50 is $442 more than the correct $5,442.17 HOH bracket tax plus $156.38 supplemental tax. That fits single-filer treatment ($8,000 deduction, no $1,000 dependent exemption, 6% above $80,650), which gives about $6,000, instead of head of household with $11,200 and the 2026 HOH schedule." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $4,220 corresponds to about $82,000–$83,000 of taxable income on the HOH schedule, roughly $23,000 below the correct $105,385.15, which matches taking the $21,208 employer health premiums out of income. It also left out the $156.38 supplemental tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $3,591 corresponds to about $71,000 of taxable income on the HOH schedule, roughly $34,000 short of the correct $105,385.15. So it left out income that belongs in NY AGI: wages reduced only by the $181 traditional 401(k), $20,480 of dividends, and a capital loss capped at $3,000. It also left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the correct $11,200 HOH standard deduction and $1,000 dependent exemption. But its $5,533 follows the pre-2026 rates (4%/4.5%/5.25%/5.5%) instead of the 2026 cut rates (3.9%/4.4%/5.15%/5.4%). It never added the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so nothing could be scored against the $5,598.55 that comes from $105,385.15 of taxable income on the 2026 HOH schedule plus the supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It itemized $27,966 by adding ~$4,700 of NY income tax to SALT (New York does not allow deducting its own income tax) and a $13,349 medical deduction built on the employer-sponsored insurance premiums. NY itemized deductions here are only $11,169, below the $11,200 HOH standard deduction. It also subtracted a refundable $120 NY CDCC and left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It gave a vague estimate citing 'limited credits', but no nonrefundable NY credit applies: the CDCC and Empire State Child Credit are refundable and the household credit is phased out. The correct result also adds a $156.38 supplemental tax on top of the $5,442.17 HOH bracket tax. Its $5,150 is below even the bracket tax alone, which matches subtracting out-of-scope credits and skipping the supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It gave no computation and claimed NY itemized deductions from real estate taxes, medical, and charity. But NY itemized deductions ($11,169) are below the $11,200 HOH standard deduction, and the medical expenses fall under the floor. The correct $105,385.15 at 2026 HOH rates gives $5,442.17 plus $156.38. Its $7,777 overstates that by $2,178, an effective rate far above the 5.4% top rate that applies here." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got taxable income right ($105,394) but applied pre-2026 HOH rates (5.5% above $20,900), producing $5,548. The 2026 cut rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17. It also left out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $2,954.80 corresponds to only about $59,000 of taxable income on the HOH schedule, roughly $46,000 short of the correct $105,385.15. That gap means it dropped income that stays in NY AGI: the $21,208 employer premiums are not subtracted from the listed wages, and capital losses are capped at $3,000. It also left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $106,394 taxable income leaves out the $1,000 dependent exemption for the child. Its $5,815 is also more than the 2026 HOH schedule produces. Correct taxable income of $105,385.15 gives $5,442.17 at 3.9%/4.4%/5.15%/5.4%, plus the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted only the $11,200 standard deduction and never took the $1,000 dependent exemption for the child. Its $5,977.08 overstates the 2026 HOH tax. The correct $105,385.15 of taxable income gives $5,442.17 at 3.9%/4.4%/5.15%/5.4%, plus a $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized about $23,624, which requires a medical deduction built on the $21,208 employer-sponsored insurance premiums. NY itemized deductions here are only $11,169 (SALT $11,057 plus $112 charitable), so the $11,200 HOH standard deduction applies and taxable income is $105,385.15, not $92,961.65. The supplemental tax it added was computed on that understated base." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied the 2026 HOH brackets after the standard deduction and dependent exemption and landed near the $5,442.17 bracket tax. It left out the $156.38 supplemental recapture tax that New York imposes because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It subtracted a $120 NY child and dependent care credit as nonrefundable, but New York's CDCC is refundable and excluded from tax before refundable credits. Its $5,665.24 pre-credit tax is also above the correct $5,598.55 ($5,442.17 on the 2026 HOH schedule plus the $156.38 supplemental tax)." +us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized about $23,624, which requires a medical deduction built on the $21,208 employer-sponsored insurance premiums. NY itemized deductions here total only $11,169 (SALT $11,057 plus $112 charitable), so the $11,200 HOH standard deduction applies and taxable income is $105,385.15, not $92,961.65. Its $4,754.16 is the tax on that understated base." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It gave no derivation. Its round $5,500 sits just above the $5,442.17 2026 HOH bracket tax on $105,385.15, which matches leaving out the $156.38 supplemental recapture tax that applies because NY AGI exceeds $107,650." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It set NY AGI to $96,454 by subtracting the $21,208 employer-sponsored insurance premiums from the listed wages, and it used a ~$12,100 HOH standard deduction instead of $11,200. It subtracted a $120 NY CDCC, which is refundable in New York. The understated AGI also meant it skipped the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed $901 + 5.5% over $20,900 on the correct $105,394, which is the pre-2026 HOH schedule. The 2026 cut rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17. It never added the $156.38 supplemental recapture tax." +us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It took about $23,000 of NY itemized deductions, but they total only $11,169 here, below the $11,200 HOH standard deduction, so taxable income is $105,385.15 rather than $93,545. It also used the single schedule ($4,271 + 6% over $80,650) instead of the HOH schedule. It left out the supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It claimed $2,000 of exemptions, but New York gives the $1,000 exemption only for dependents, not the filer. It overstated the bracket tax at ~$5,699 and subtracted a $120 NY CDCC that is refundable in New York. It also left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so nothing could be scored against the $5,598.55 that comes from $105,385.15 of taxable income on the 2026 HOH schedule plus the supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its taxable income of $105,394 is right, but it applied a HOH schedule running 'through the 5.75% bracket'. For 2026 the rate on HOH income from $20,900 to $107,650 is 5.4%, giving $5,442.17. It also left out the $156.38 supplemental recapture tax." +us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It let the $33,000 long-term capital loss offset most income and treated the CTC as cutting NY tax to $0. But a net capital loss offsets only $3,000 of ordinary income, so NY AGI is $117,585 and taxable income is $105,385.15, taxed at 2026 HOH rates for $5,442.17 plus $156.38. It also used the $8,000 single deduction instead of $11,200 HOH." +us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used an $8,000 standard deduction (the single amount) instead of the $11,200 HOH deduction. It applied outdated rates (5.85% up to $107,650, 6.25% above) instead of the 2026 HOH rates (3.9%/4.4%/5.15%/5.4%). It subtracted a $120 NY CDCC, which is refundable in New York, and left out the $156.38 supplemental tax." +us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $30,711 net capital loss instead of capping it at $3,000, cutting federal AGI to $89,951 from $117,654. It added the $68 refund instead of subtracting it, used the $8,000 single deduction instead of $11,200 HOH, and applied single brackets. It then raised its $4,292.89 tax to $4,755.44 'after household credit', a credit that is zero at this income." +us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It overstated federal AGI at $139,586; with the capital loss capped at $3,000 it is $117,654. It used the $8,000 single deduction instead of the $11,200 HOH deduction and subtracted an invented 'federal deduction adjustment'. It then subtracted the refundable Empire State Child Credit and a household credit that is phased out at this income." +us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the old ESCC formula of 33% of a $1,000 pre-TCJA federal CTC, giving $330, instead of the enhanced $1,000 under-4 credit less the $693 phase-out, which gives $307. It also guessed about $150 for the NY CDCC instead of 60% of the $600 federal credit, which is $360." +us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Applied a 20% NY CDCC percentage and got $120 instead of the 60% percentage, which gives $360; this $240 gap accounts for most of the miss. It also applied the ESCC phase-out as a continuous 1.65% ($703.79) instead of $16.50 per whole $1,000 over $75,000 (42 steps, $693), so it got $296.21 instead of $307." +us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that New York has no refundable child credit, missing the refundable Empire State Child Credit ($307 after phase-out) and the refundable NY child and dependent care credit ($360)." +us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Invented a $110 under-4 ESCC instead of applying the $1,000 under-4 amount reduced by $693 to $307. It also wrongly treated the NY child and dependent care credit as nonrefundable at this AGI, dropping the refundable $360 (60% of the $600 federal CDCC)." +us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Assumed the ESCC phases out completely at this income. The $16.50-per-$1,000 reduction above $75,000 removes only $693 of the $1,000 under-4 credit, leaving $307. It also wrongly called the NY CDCC nonrefundable at higher AGI, dropping the refundable $360." +us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Declared income too high for the ESCC and the NY child care credit. At $117,653 federal AGI, the $1,000 under-4 ESCC is reduced by only $693 to $307, and the NY CDCC is a refundable $360 (60% of $600) with no income cutoff." +us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Applied the 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also counted 43 phase-out steps instead of 42 whole $1,000 steps over $75,000, giving an ESCC of $290.50 instead of $307." +us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used the superseded ESCC formula, the greater of $100 or 33% of a $2,000 federal CTC ($660), and asserted there is no phase-out. The enhanced credit is $1,000 reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360), and it understated AGI by deducting the full $33,000 capital loss instead of the $3,000 cap." +us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Assumed the ESCC is fully phased out at this income, when the $1,000 under-4 credit is only reduced to $307. It also omitted the refundable NY child and dependent care credit of $360 on the $9,600 of childcare expenses." +us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Assumed the ESCC is phased out when $307 remains ($1,000 less 42 × $16.50). It also used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360)." +us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Checked only the NY earned income credit and ignored the refundable Empire State Child Credit ($307 for the child under 4) and the refundable NY child and dependent care credit ($360). +us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also omitted the $307 Empire State Child Credit for the child under 4. +us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"Applied the old rule that excludes children under 4 from the ESCC, but the 2026 enhanced credit pays $1,000 for a child under 4, which phases down to $307 here. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Took the full $1,000 under-4 ESCC and never applied the $16.50-per-$1,000 reduction for federal AGI above $75,000 (42 steps, $693), which leaves $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)." +us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount (33% of a $1,000 CTC) instead of the $1,000 under-4 credit phased down to $307. It estimated the NY CDCC at $300 instead of $360 and added a $63 NYC school tax credit that is not a refundable state credit for this household." +us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the enhanced $1,000 under-4 credit phased down to $307. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied the old $100 ESCC minimum instead of the $1,000 under-4 credit reduced by $693 to $307. It also used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360)." +us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied the 110% NY CDCC percentage that applies only at the lowest incomes, giving $660, instead of the 60% percentage at this income, which gives $360. It also used the superseded $330 ESCC amount instead of the phased-down $307." +us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported zero without reasoning. The correct derivation adds the $307 Empire State Child Credit ($1,000 under-4 amount less the $693 phase-out) and the $360 refundable NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the $1,000 under-4 credit phased down to $307. It also left out the refundable $360 NY child and dependent care credit on the $9,600 of childcare costs." +us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Used the superseded $330 ESCC amount instead of the enhanced $1,000 under-4 credit phased down to $307. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also assumed the ESCC is phased out when $307 remains after the $693 reduction. +us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_082,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Set the ESCC equal to the full $2,200 federal CTC instead of the $1,000 under-4 amount reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable NY credits. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit generated by the $9,600 of childcare expenses." +us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,Looked only for NY EITC eligibility and returned zero. It missed the $307 Empire State Child Credit and the $360 refundable NY child and dependent care credit. +us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Applied a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also wrongly concluded that no child credit remains, when the $1,000 under-4 ESCC is only reduced to $307." +us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Took the unreduced $1,000 under-4 ESCC and skipped the $693 phase-out for federal AGI above $75,000, which leaves $307. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also counted 43 phase-out steps instead of 42 whole $1,000 steps, giving an ESCC of $290.50 instead of $307." +us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"Applied the old ESCC age floor that excludes children under 4, but the 2026 enhanced credit pays $1,000 for a child under 4, which phases down to $307 here. It also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also over-reduced the ESCC to $290.50 by counting 43 steps instead of 42 whole $1,000 steps over $75,000, which gives $307." +us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). It also treated the ESCC as fully phased out when $307 of the $1,000 under-4 credit remains." +us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Wrongly treated the NY child and dependent care credit as nonrefundable; it is refundable at $360 (60% of $600). It also assumed there is no ESCC at this income, when $307 remains after the $693 phase-out." +us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Used a 20% NY CDCC percentage ($120) instead of 60% of the $600 federal credit ($360). This $240 gap accounts for almost all of the miss; its $312 ESCC is close to the correct $307 ($1,000 less 42 × $16.50)." +us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,Found no qualifying refundable credits. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit. +us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Tied the ESCC to a federal CTC it wrongly called fully phased out; the federal phase-out starts at $200,000. The enhanced ESCC is $1,000 for a child under 4 less $16.50 per $1,000 over $75,000, which gives $307. It also ignored the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Applied the old $100 ESCC minimum on the false premise that the federal CTC is phased to zero, instead of the $1,000 under-4 credit reduced to $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)." +us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Computed the ESCC as 33% of a $2,000 federal CTC ($660) under the superseded formula with no phase-out, instead of the $1,000 under-4 amount less $693, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)." +us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Checked only the NY EITC and ignored the refundable Empire State Child Credit ($307) and the refundable NY child and dependent care credit ($360). +us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as nonrefundable and absorbed by tax liability, but it is refundable in full at $360 (60% of $600). It also omitted the $307 Empire State Child Credit for the child under 4." +us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Computed the ESCC from a recomputed 2017-law federal CTC and fell back to the $100 minimum. The 2026 credit is a flat $1,000 for a child under 4, reduced by $16.50 per $1,000 of AGI over $75,000, which gives $307. It also used a 20% NY CDCC percentage ($120) instead of 60% ($360)." +us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Asserted that income exceeds the ESCC range. The $1,000 under-4 credit is only reduced by $693 to $307 at $117,653 federal AGI, and the model also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Correctly started from the $1,000 under-4 ESCC and the $75,000 head-of-household threshold. It then used a 5% phase-out rate instead of $16.50 per whole $1,000 (1.65%), which wiped out the credit instead of leaving $307, and it also omitted the refundable $360 NY child and dependent care credit." +us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Checked only the NY earned income credit. It missed the $307 Empire State Child Credit for the child under 4 and the $360 refundable NY child and dependent care credit. +us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Its $313 covers only an ESCC-sized amount, close to the correct $307. It omitted the refundable NY child and dependent care credit of $360 (60% of the $600 federal CDCC) that brings the total to $667." us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero." us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output." us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation." @@ -6296,165 +6911,187 @@ us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's subm us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,The model correctly derived $539.33 and then improperly changed it to $538 under the label of rounding. Ordinary currency rounding preserves $539.33; it does not reduce the result by $1.33. us,scenario_083,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $649.62 is inconsistent with applying employee Social Security and Medicare taxes to the stated $7,050 wages. The correct 7.65% combined tax on that wage base is $539.33, so the answer used an inflated base or rate unsupported by the household facts." us,scenario_083,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $3,960 of rental income as self-employment income and applied the 92.35% adjustment and 15.3% tax rate to it. Ordinary rental income is excluded from net earnings from self-employment, leaving no self-employment earnings subject to the $400 threshold and yielding $0 of self-employment tax." -us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model ran the full allotment calculation without ever applying a SNAP resource test, so it never compared the head's $6,620 bank balance to Texas's $5,000 BBCE liquid-asset limit that zeroes the benefit. It compounded this by submitting $2,718 after its own reasoning concluded $1,428–$1,469, an unexplained jump to an implied $226.50/month allotment that contradicts its stated $122.40/month result." -us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model identified the decisive fact — that $6,620 exceeds the standard resource limit — and then discarded it on the false premise that 'under broad-based categorical eligibility in TX asset test is generally waived'; Texas is a BBCE state that expressly keeps a $5,000 liquid-resource limit, which $6,620 breaches, making the household ineligible. It also submitted $2,934 against its own computed $1,430, doubling the monthly figure from $119.20 to $244.50 with no stated basis." -us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"The model verified only the 130% FPG gross-income screen and then applied the 30%-of-net-income formula, omitting the resource test entirely; the head's $6,620 in countable bank assets exceeds Texas's $5,000 BBCE asset limit, so no benefit is payable. Its $134/month result is exactly what the allotment formula returns when the asset screen is skipped." -us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model never tested countable resources against Texas's $5,000 BBCE liquid-asset limit, which the $6,620 bank balance exceeds and which drives the benefit to zero. It further inflated its answer by manufacturing a shelter deduction from the $200,000 mortgage balance even though no mortgage payment, tax, insurance, or utility cost is listed, pushing $119/month up to an asserted $183/month." -us,scenario_083,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"The model applied only the 130% FPL gross-income test and the standard net-income allotment formula, skipping the resource screen under which the head's $6,620 in bank assets exceeds Texas's $5,000 BBCE liquid-asset limit and zeroes the benefit. Its $1,381 is the pure formula output with the asset test omitted." -us,scenario_083,snap,deepseek-v4-pro-0813,llm_error,asset_resource,False,"The model computed net monthly income of $602 and applied the $292 maximum allotment minus 30% of net income, never testing countable resources; the $6,620 bank balance exceeds Texas's $5,000 BBCE asset limit, so the household is ineligible and the benefit is $0. It also used the FY2025 $292/$198 parameters rather than FY2026 figures, but the asset test is what makes the answer wrong." -us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"The model applied only income deductions and the allotment formula, omitting any resource test; the head's $6,620 in countable bank assets exceeds Texas's $5,000 BBCE liquid-resource limit, which terminates eligibility before any allotment is computed. Its $1,336.80 is the FY2025-parameter formula result with the asset screen skipped." -us,scenario_083,snap,gemini-3.8-flash,llm_error,asset_resource,False,"The model went straight from gross income to deductions to $292 minus 30% of net income without applying the SNAP resource test, under which the $6,620 bank balance exceeds Texas's $5,000 BBCE asset limit and disqualifies the household. Its $1,344 reflects that omitted screen, not a parameter error." -us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"The model never evaluated countable resources against Texas's $5,000 BBCE liquid-asset limit, which the head's $6,620 bank balance exceeds and which zeroes the benefit. It also invented non-existent parameters — a $2,688 annual standard deduction and a $316 maximum allotment — and applied deductions on an annual basis, producing $1,718.40 from a formula that should never have been reached." -us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model applied the earned-income and standard deductions and the $298 maximum allotment minus 30% of net income, omitting the resource test that disqualifies this household: $6,620 in bank assets exceeds Texas's $5,000 BBCE liquid-resource limit. Its $1,448.40 is the allotment formula output with eligibility assumed rather than tested." -us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"The model explicitly asserted 'TX BBCE 165% FPL limit with assets waived,' which is the exact error: Texas's broad-based categorical eligibility raises the gross-income screen but retains a $5,000 liquid-resource limit, and the head's $6,620 bank balance exceeds it. Waiving the asset test let it proceed to the $292-minus-30%-of-net-income calculation and report $1,358 instead of $0." -us,scenario_083,snap,grok-4.6,llm_error,asset_resource,False,"The model checked the BBCE gross-income test and the ABAWD work requirement — both of which this 48-hour-per-week worker passes — but never checked countable resources, where the $6,620 bank balance exceeds Texas's $5,000 BBCE liquid-asset limit and ends eligibility. Its $1,356 is the FY2025-scale allotment formula applied to a household that fails the resource screen." -us,scenario_083,snap,ox-alpha,llm_error,asset_resource,False,"The model tested only the 130% FPL gross-income limit and then computed $298 minus 30% of $591 net income, never applying the resource test under which $6,620 in bank assets exceeds Texas's $5,000 BBCE liquid-asset limit. Skipping that single screen is the entire gap between its $1,448 and $0." -us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model omitted the SNAP resource test, under which the head's $6,620 bank balance exceeds Texas's $5,000 BBCE liquid-asset limit and zeroes the benefit. Its arithmetic is also internally broken: it subtracted a $198 monthly standard deduction from annual income to get $9,402 net, and its own stated figures ($3,516 maximum minus $2,821) yield $695, not the $2,568 it submitted." +us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"It checked only the gross and net income tests and never tested the $6,620 bank balance against Texas's $5,000 SNAP resource limit, which makes the household ineligible. Its own work gave about $122 a month, or $1,469 a year, but it submitted $2,718, which none of its arithmetic supports." +us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"It noticed the $6,620 exceeds the standard resource limit but wrongly said Texas BBCE waives the asset test. Texas BBCE keeps a $5,000 asset limit, so the household is ineligible. It then abandoned its own $1,430 calculation for an unexplained $244.50 a month." +us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"It applied only the 130% FPG gross income test and the net-income benefit formula, arriving at about $134 a month. It never applied the resource test, and the $6,620 bank balance exceeds Texas's $5,000 BBCE asset limit, which zeroes out SNAP." +us,scenario_083,snap,claude-opus-5.5,llm_error,asset_resource,False,"It said Texas BBCE means there is no asset test. In fact Texas's BBCE keeps a $5,000 resource limit, and the $6,620 bank balance fails it. The $120.70 a month it computed from the deduction chain therefore never applies." +us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"It skipped the resource test, and the $6,620 in bank assets exceeds Texas's $5,000 SNAP asset limit, so the household is ineligible. It also invented a shelter deduction from a mortgage balance, with no housing payment listed, to push the benefit to about $183 a month." +us,scenario_083,snap,claude-sonnet-5.5,llm_error,asset_resource,False,"It checked only the gross income limit and ran the $298 minus 30%-of-net formula, getting $120.70 a month. It never tested the $6,620 bank balance against Texas's $5,000 BBCE asset limit, which disqualifies the household." +us,scenario_083,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It confirmed gross income is under 130% FPL and went straight to the benefit formula ($3,516 minus 30% of $7,116). It never applied the resource test, which the $6,620 bank balance fails against Texas's $5,000 BBCE asset limit." +us,scenario_083,snap,deepseek-v4-pro-0813,llm_error,asset_resource,False,"It computed net income of $602 and a benefit of $292 minus 30% of that, using FY2025 parameters. It never checked resources, and the $6,620 bank balance exceeds Texas's $5,000 SNAP asset limit, making the household ineligible." +us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"It went straight from gross income through the $198 standard deduction and 20% earned-income deduction to a $111.40 monthly benefit. It skipped the resource test, and the $6,620 bank balance fails it against Texas's $5,000 BBCE limit." +us,scenario_083,snap,gemini-3.8-flash,llm_error,asset_resource,False,"It computed $600 a month of net income and a $112 monthly benefit from the $292 maximum allotment without any eligibility screen on assets. The $6,620 bank balance exceeds Texas's $5,000 SNAP resource limit, so the benefit is $0." +us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"It never applied the SNAP resource test, which the $6,620 bank balance fails against Texas's $5,000 BBCE asset limit. It also inflated the parameters, using a $2,688 annual standard deduction and a $316 monthly maximum allotment, to reach $1,718.40." +us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"It applied only the deduction chain ($591 countable income) and the $298 minus 30% formula. It never tested the $6,620 bank balance against Texas's $5,000 SNAP asset limit, which makes the household ineligible." +us,scenario_083,snap,gpt-6-luna,llm_error,asset_resource,False,"Its $119 a month equals the one-person maximum allotment minus 30% of about $596 net income. That means it treated the household as eligible without applying the resource test, which the $6,620 bank balance fails against Texas's $5,000 BBCE asset limit." +us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"It said Texas BBCE waives assets. Texas's BBCE applies a 165% FPL gross income limit together with a $5,000 asset limit, and the $6,620 bank balance fails it, so SNAP is $0 rather than about $113 a month." +us,scenario_083,snap,grok-4.6,llm_error,asset_resource,False,"It checked the BBCE gross income test and the ABAWD work rule but never the resource test. The $6,620 bank balance exceeds Texas's $5,000 BBCE asset limit, which disqualifies the household." +us,scenario_083,snap,grok-4.7,llm_error,asset_resource,False,"It said Texas BBCE waives the asset test, so $6,620 in bank assets would not disqualify the household. Texas's BBCE keeps a $5,000 asset limit, so that balance makes the household ineligible, and its $1,448.40 is the result of a formula that never applies." +us,scenario_083,snap,ox-alpha,llm_error,asset_resource,False,"It applied only the 130% FPL gross income test and the $298 minus 30%-of-$591 formula. It never tested the $6,620 bank balance against Texas's $5,000 SNAP resource limit, which makes the household ineligible." +us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"It never applied the resource test, which the $6,620 bank balance fails against Texas's $5,000 BBCE asset limit. Its arithmetic was also broken: it subtracted a monthly $198 standard deduction from annual income, then reported $2,568 without deriving it." us,scenario_084,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness, disability, and SSDI receipt as categorical Medicaid eligibility despite acknowledging that the head exceeds the SSI-related income and resource limits. None of those facts creates unconditional North Carolina Medicaid eligibility, and the head satisfies no qualifying pathway." us,scenario_084,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly reasoned that income exceeds the North Carolina aged/blind/disabled limit and that no MAGI parent or child category applies, but then submitted eligible. Its numeric answer directly contradicts its own determination that the head satisfies no disability or MAGI pathway." us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented medically needy spend-down eligibility without applying North Carolina's full financial test. Even subtracting all $6,754 of listed premiums and medical expenses from $33,233 leaves $26,479, and the model also ignored the $56,500 in assets; the head therefore does not enter a medically needy pathway." us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and blindness as sufficient for Medicaid while leaving the required income, resource, and coverage-group tests unapplied. PolicyEngine assigns no Medicaid category because those statuses alone do not establish eligibility." us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded SSDI from Medicaid MAGI merely because it considered the benefit nontaxable. Medicaid MAGI includes Social Security benefits through its program-specific adjustment, producing 1.56 times FPL rather than approximately $8,405 and placing the head above the expansion limit." us,scenario_084,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model calculated expansion MAGI as approximately $8,405 by omitting the head's Social Security disability income. The Medicaid MAGI computation yields 1.56 times FPL, so the head exceeds North Carolina's expansion threshold and has no other qualifying category." -us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"Applied the real-world 24-month SSDI-to-Medicare entitlement pathway, which PolicyEngine's is_medicare_eligible does not open below 65 without an explicit disability/ESRD entitlement input. It manufactured the missing 24 months from the prompt's clause that disability status is constant through the tax-benefit year, but that clause fixes facts inside 2026 and supplies no prior-year entitlement, leaving the age-65 test, which a 62-year-old fails." -us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Asserted that SSDI receipt by itself confers pre-65 Medicare, dropping even the 24-month qualifying period the real-world rule requires and that no listed fact establishes. PolicyEngine's is_medicare_eligible keys on the age-65 threshold plus explicit entitlement inputs, and the 62-year-old head has none of them set." -us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Stated the rule two incompatible ways and settled on the loosest version — that anyone disabled and receiving SSDI is eligible — applying no qualifying-period test at all. It converted the is_disabled flag and the $24,828 SSDI amount into Medicare entitlement, while PolicyEngine requires age 65 or an explicit disability/ESRD entitlement input, neither of which this 62-year-old household carries." -us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Invoked the 24-month SSDI-to-Medicare rule and treated the prompt's within-year constancy instruction as proof those 24 months had already elapsed. Constancy fixes 2026 status only and creates no pre-2026 entitlement, so PolicyEngine's operative test is age 65, which the 62-year-old head fails." -us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Wrote that it was 'assuming established disability benefit receipt' — precisely the unlisted fact the prompt sets to false, since no SSDI entitlement start date or duration appears in the household. Stripped of that assumption the facts give only age 62 and an is_disabled flag, and PolicyEngine's is_medicare_eligible begins at 65." -us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"Named an 'SSDI disability pathway' with no duration or entitlement test, converting the is_disabled input and the disability income amount straight into Medicare eligibility. PolicyEngine models Medicare entitlement from the age-65 threshold plus explicit disability/ESRD entitlement inputs, all false for this 62-year-old who carries employer-sponsored insurance." -us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Recited the 24-month waiting period correctly, then declared it met from the bare facts of age 62 and SSDI receipt, which state nothing about months of entitlement. Under the treat-unlisted-as-false instruction the waiting period is unmet, leaving PolicyEngine's age-65 test, which the head fails." -us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Concluded eligibility 'after the SSDI waiting period' while giving no basis that the period had run — the household lists an annual SSDI amount, not an entitlement start date. PolicyEngine's is_medicare_eligible turns on the age-65 threshold for this household, and the head is 62." -us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Inferred 24 months of SSDI entitlement as 'implied by SSDI receipt at age 62', using the head's age as a proxy for benefit history. Age carries no entitlement duration, the prompt sets unlisted facts to false, and PolicyEngine's age-65 Medicare test fails at 62." -us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied the under-65 'Medicare disability provision' as though the is_disabled flag alone triggers it, skipping the entitlement requirement entirely. PolicyEngine opens Medicare at age 65 or on an explicit disability/ESRD entitlement input, and neither holds for this 62-year-old." -us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Equated 'disabled and receiving SSDI' with Medicare eligibility in a single step, applying no age threshold and no qualifying-period test. PolicyEngine's is_medicare_eligible resolves on age 65 here, and the head is 62 with employer-sponsored insurance and no Medicare Part B premium listed." -us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Made SSDI receipt sufficient on its own for Medicare, collapsing the separate 24-month entitlement requirement that no listed fact satisfies. PolicyEngine's Medicare eligibility begins at 65 and the head is 62, so the engine returns false." -us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Treated SSDI benefit receipt as automatic Medicare qualification and never tested the head's age against the 65 threshold that governs PolicyEngine's is_medicare_eligible. At 62 with no Medicare entitlement input — the health-premium line is stated as excluding Medicare Part B, with no Part B premium given — the head fails that test." -us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Combined the is_disabled flag with SSDI income to assert Medicare eligibility, omitting both the qualifying-period step and the age test. The operative rule in PolicyEngine is the age-65 threshold, which a 62-year-old fails with no disability-entitlement input set." -us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Read SSDI receipt plus disability status as a completed Medicare entitlement while supplying no start date for the 24-month qualifying period. Unlisted facts are false under the prompt, so the only remaining route is the age-65 threshold, and the head is 62." -us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"Attributed the under-65 disability pathway to 'PolicyEngine's Medicare eligibility rules'; the engine's is_medicare_eligible keys on the age-65 threshold plus explicit disability/ESRD entitlement inputs, none of which this household sets. It misread the is_disabled flag and SSDI income — which drive SSI, the blind/disabled deductions, and taxable Social Security — as Medicare-entitlement inputs." -us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Cited 'disabled and a survivor' as the eligibility basis, importing surviving-spouse status — a federal income-tax filing status with no role in Medicare — into the Medicare test. PolicyEngine's is_medicare_eligible turns on the age-65 threshold, which the 62-year-old head fails, and the disability route requires an entitlement input this household does not have." -us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Correctly noted that age 62 does not qualify, then reversed on the claim that PolicyEngine marks a person Medicare-eligible whenever the disabled flag and SSDI income appear. The is_disabled input is a disability status for SSI and the elderly/disabled deductions, not a Medicare entitlement flag; is_medicare_eligible requires age 65 or an explicit entitlement input here." -us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"Claimed 'PolicyEngine disability rules' grant pre-65 Medicare from SSDI receipt; the engine's is_medicare_eligible is governed by the age-65 threshold plus explicit disability/ESRD entitlement inputs that are false for this household. At 62, with employer-sponsored insurance and premiums stated as excluding Medicare Part B, the head fails." -us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Derived Medicare eligibility from SSDI receipt alone, with no entitlement duration and no age test. PolicyEngine's Medicare eligibility starts at 65; the head is 62 and no disability-entitlement input is set for this household." -us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Invoked 'the disability pathway' as a standing entitlement without the 24 months of SSDI entitlement that pathway requires and that the household facts do not supply. PolicyEngine resolves Medicare eligibility on the age-65 threshold here, which the 62-year-old head fails." -us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"Asserted a 'disability-based Medicare pathway modeled for a disabled Social Security disability beneficiary'; PolicyEngine does not open that pathway from the is_disabled flag or SSDI income, since is_medicare_eligible turns on age 65 plus explicit disability/ESRD entitlement inputs that are all false here. The head is 62." -us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"Stated the head was 'on SSDI for sufficient duration' — a duration found nowhere in the household facts, which give only an annual benefit amount and set every unlisted fact to false. PolicyEngine's age-65 Medicare test governs, and the head is 62." -us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"Declared the 24-month SSDI waiting-period rule satisfied from disability status and benefit receipt, neither of which dates the entitlement. With no entitlement history in the facts and the head at 62, PolicyEngine's is_medicare_eligible resolves on the age-65 threshold and returns false." -us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"Concluded eligibility 'after the SSDI waiting period' without establishing that the period had run; the household lists an annual benefit amount, not months of entitlement. PolicyEngine's Medicare eligibility for this household is the age-65 test, failed at 62." -us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"Turned the is_disabled flag and SSDI income directly into Medicare qualification, skipping both the age-65 threshold and the 24-month entitlement requirement. Those inputs feed SSI and the disability-related deductions in PolicyEngine, not is_medicare_eligible, which the 62-year-old head fails." -us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"Treated the dollar amounts — $8,400 in disability benefits and $24,828 in Social Security disability income — as evidence that the 24-month waiting period was satisfied, but annual amounts carry no information about months of entitlement. The prompt sets unlisted facts to false, leaving the age-65 threshold as the operative test, which the 62-year-old head fails." -us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Read the prompt's within-year constancy instruction as a multi-year SSDI history satisfying the 24-month Medicare qualifying period; that clause fixes status inside the 2026 tax-benefit year and creates no prior entitlement. Without prior entitlement, PolicyEngine's is_medicare_eligible resolves on the age-65 threshold, and the head is 62." +us,scenario_084,head_medicare_eligible,claude-fable-5,prompt_ambiguity,age_disability,False,"The model treated the instruction to hold disability constant through the year as meeting the 24-month SSDI waiting period. That instruction covers only 12 months of the tax year. The number of prior SSDI months is not listed, so it is 0, and the 24-month disability pathway fails for a 62-year-old." +us,scenario_084,head_medicare_eligible,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated current SSDI receipt as giving Medicare eligibility by itself and skipped the rule requiring 24 months of disability-benefit entitlement. The Head's months of SSDI receipt are not listed, so they are 0, which leaves a 62-year-old ineligible." +us,scenario_084,head_medicare_eligible,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI requirement, then dropped it and concluded that being disabled and receiving SSDI is enough. No fact establishes 24 months of receipt. That count is not listed and is therefore 0, so the Head, at 62, is not Medicare eligible." +us,scenario_084,head_medicare_eligible,claude-opus-4.7,prompt_ambiguity,age_disability,False,"The model cited the 24-month SSDI waiting period but treated disability held constant for the year as satisfying it. A full year of receipt is only 12 months, and prior SSDI months are not listed, so they are 0. The disability pathway fails." +us,scenario_084,head_medicare_eligible,claude-opus-4.8,prompt_ambiguity,age_disability,False,"The model said outright that it was 'assuming established disability benefit receipt'. That invents SSDI history the prompt tells it not to infer. With months of SSDI receipt at 0, the 24-month requirement is not met, and at age 62 the Head is not Medicare eligible." +us,scenario_084,head_medicare_eligible,claude-opus-5,prompt_ambiguity,age_disability,False,"The model treated the SSDI disability pathway as automatic once disability benefits are paid. It left out the rule requiring 24 months of benefit entitlement. That count is not listed, so it is 0, and the Head is not eligible." +us,scenario_084,head_medicare_eligible,claude-sonnet-5,prompt_ambiguity,age_disability,False,"The model wrote that the Head 'has been receiving' SSDI, which invents a history of receipt long enough to meet the 24-month waiting period. The prompt gives no months of SSDI receipt, so that count is 0. A 62-year-old without 24 months of entitlement does not qualify." +us,scenario_084,head_medicare_eligible,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model mentioned the SSDI waiting period but never checked whether it had passed. The months of SSDI receipt are not listed, so they are 0, well short of the 24 months required. The Head, at 62, is therefore not eligible." +us,scenario_084,head_medicare_eligible,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"The model inferred 24 months of SSDI receipt from the Head's age of 62. Age says nothing about how long benefits have been paid. The months of receipt are not listed, so they are 0, and the 24-month disability pathway is not met." +us,scenario_084,head_medicare_eligible,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model applied the Medicare disability provision without its main condition: 24 months of entitlement to SSDI. With months of SSDI receipt at 0 (not listed), a 62-year-old does not qualify under that provision." +us,scenario_084,head_medicare_eligible,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"The model claimed PolicyEngine makes every disabled SSDI recipient eligible for Medicare. PolicyEngine actually requires age 65, ESRD, or at least 24 months of SSDI receipt. The months count is not listed and is therefore 0, so none of these apply." +us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model concluded that disabled status plus SSDI income makes the Head Medicare eligible. It never applied the 24-month SSDI entitlement requirement, which fails because months of receipt are not listed and default to 0." +us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"The model treated SSDI income by itself as qualifying for Medicare and ignored the 24-month entitlement waiting period. Months of SSDI receipt are 0 because they are not listed, so the Head, at 62, is not eligible." +us,scenario_084,head_medicare_eligible,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"The model equated receiving SSDI with Medicare eligibility and skipped the rule requiring 24 months of disability-benefit entitlement. The months count is not listed, so it is 0, and the Head is not eligible." +us,scenario_084,head_medicare_eligible,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"The model concluded that disabled status plus SSDI income gives Medicare eligibility without applying the 24-month SSDI waiting period. Prior months of receipt are 0 because they are not listed, so a 62-year-old does not qualify." +us,scenario_084,head_medicare_eligible,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt and disability as enough for Medicare and left out the 24-month entitlement requirement. With months of SSDI receipt at 0 (not listed), the disability pathway fails." +us,scenario_084,head_medicare_eligible,glm-5.2,prompt_ambiguity,age_disability,False,"The model said PolicyEngine's rules cover anyone under 65 who is disabled and receiving SSDI. PolicyEngine's disability pathway actually requires at least 24 months of SSDI receipt. That count is not listed and is therefore 0, so the Head is not eligible." +us,scenario_084,head_medicare_eligible,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"The model cited disabled and surviving-spouse status as grounds for Medicare. Neither qualifies anyone without 24 months of entitlement to disability benefits, and that also applies to disabled widow(er) benefits. The months of receipt are not listed and are therefore 0, so the Head, at 62, is not eligible." +us,scenario_084,head_medicare_eligible,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"The model correctly said someone aged 62 needs disability-based entitlement, then treated having SSDI income as proof of that entitlement. Medicare eligibility requires 24 months of SSDI entitlement. That count is not listed and is therefore 0, so the Head is not eligible." +us,scenario_084,head_medicare_eligible,gpt-5.5,prompt_ambiguity,age_disability,False,"The model treated disability status plus SSDI income as meeting PolicyEngine's disability rule. That rule requires at least 24 months of SSDI receipt, and the months count is 0 because it is not listed. The Head, at 62, is therefore not eligible." +us,scenario_084,head_medicare_eligible,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"The model gave Medicare eligibility to someone under 65 based only on receiving SSDI. It never applied the 24-month entitlement requirement, which fails because months of SSDI receipt are not listed and default to 0." +us,scenario_084,head_medicare_eligible,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model used the disability pathway without checking its requirement of 24 months of SSDI entitlement. The months of SSDI receipt are 0 because they are not listed, so the Head is not eligible at 62." +us,scenario_084,head_medicare_eligible,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model assumed PolicyEngine's disability-based Medicare pathway applies to any disabled SSDI recipient. That pathway requires at least 24 months of SSDI receipt. The months count is not listed and is therefore 0, so the pathway does not apply." +us,scenario_084,head_medicare_eligible,gpt-6-luna,prompt_ambiguity,age_disability,False,"The model counted disabled status plus SSDI income as meeting PolicyEngine's disability-based Medicare rule. That rule requires 24 months of SSDI receipt, and the months count is 0 because it is not listed. The Head is therefore not eligible." +us,scenario_084,head_medicare_eligible,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model gave disability-based Medicare to a disabled SSDI recipient without applying the rule requiring 24 months of benefit entitlement. Months of receipt are not listed, so they are 0, which leaves a 62-year-old ineligible." +us,scenario_084,head_medicare_eligible,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model treated reported disability status and SSDI income as meeting the disability-based Medicare rule. That rule turns on 24 months of SSDI receipt, and the prompt gives none, so the count is 0. The Head is not eligible." +us,scenario_084,head_medicare_eligible,grok-4.3,prompt_ambiguity,age_disability,False,"The model asserted the Head had been on SSDI 'for sufficient duration', but no listed fact gives any duration of receipt. Months of SSDI receipt are 0 because they are not listed, so the 24-month requirement fails." +us,scenario_084,head_medicare_eligible,grok-4.5,prompt_ambiguity,age_disability,False,"The model declared the 24-month SSDI waiting period satisfied without any fact to support it. Months of SSDI receipt are not listed, so they are 0, and a 62-year-old without 24 months of entitlement is not Medicare eligible." +us,scenario_084,head_medicare_eligible,grok-4.6,prompt_ambiguity,age_disability,False,"The model mentioned the SSDI waiting period but assumed it had passed. Months of SSDI receipt are not listed, so they are 0, short of the 24 months required. The Head is therefore not eligible." +us,scenario_084,head_medicare_eligible,grok-4.7,prompt_ambiguity,age_disability,False,"The model misstated PolicyEngine's Medicare rule as 'age 65 or older or if disabled'. Disability counts only after at least 24 months of SSDI receipt. That count is not listed and is therefore 0, so a 62-year-old does not qualify." +us,scenario_084,head_medicare_eligible,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated SSDI receipt by someone under 65 as immediately qualifying for Medicare and left out the 24-month entitlement requirement. Months of receipt are 0 because they are not listed, so the Head is not eligible." +us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disability,False,"The model inferred that the 24-month waiting period was met from the sizes of the Head's disability and SSDI payments. Dollar amounts say nothing about how long benefits have been paid. The months of SSDI receipt are not listed, so they are 0, and the Head is not eligible." +us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model treated the instruction that disability and SSDI status are constant through the year as establishing 24 months of SSDI receipt. That instruction covers only the 12-month tax year. Prior months of receipt are not listed, so they are 0, and the 24-month requirement fails." us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Sonnet derived monthly net income of $1,737.59 but never compared it to the SNAP net income limit of 100% FPL (about $1,340/month for one person), which that figure fails by roughly 30%, and it ignored both NC's 200%-FPL BBCE gross cap (about $2,680/month against its own $2,769) and the $56,500 in bank assets against the elderly/disabled resource limit of roughly $4,650. Its own benefit formula returned $0 because 30% of net income ($521.28) exceeds the one-person maximum allotment ($298), after which it discarded that result and substituted an unsupported $183/month figure attributed to a conditional minimum benefit, a rule that provides $23-$24, not $183. It further inflated the excess medical deduction by double-counting the same $3,600 premium (listed once as health insurance premiums excluding Medicare Part B and again as other health insurance premiums) and by including $1,054 of non-prescription over-the-counter costs, which are not deductible medical expenses." us,scenario_084,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"DeepSeek asserted the household passes the net income test after the standard and medical deductions, but net income of about $1,738/month exceeds the one-person net income limit near $1,340/month, gross income of $2,769/month exceeds North Carolina's 200%-FPL BBCE gross cap of about $2,680/month, and $56,500 in countable resources exceeds the elderly/disabled asset limit of roughly $4,650. Having wrongly cleared eligibility, it applied the $23/month minimum allotment for one- and two-person households, a floor that attaches only after a household qualifies; an income-ineligible household receives $0, not $276." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its worksheet reproduced the reference chain exactly — taxable Social Security $14,051, AGI $45,574, $18,150 standard deduction plus the $6,000 senior deduction, taxable income $21,424, and $1,240 + 12% × $9,024 = $2,323 — and it then submitted 940, a figure that appears nowhere in that derivation. The substantive computation was correct; the reported value discarded it." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the entire $27,426 of survivor benefits into AGI instead of running the provisional-income worksheet that includes only $14,050.60, then wiped out the resulting liability with a §22 elderly credit computed from a $7,500 base. The §22 initial amount for a single filer is $5,000 and is reduced dollar-for-dollar by nontaxable Social Security, and this filer's $13,375 of excluded benefits drives that credit to zero, leaving the full $2,322.76 standing." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached the correct $45,574 AGI, then guessed a combined standard deduction of about $17,000 instead of the 2026 $16,100 basic amount plus the $2,050 age-65 addition ($18,150), and applied the 2025 $11,925 bracket break rather than 2026's $12,400. The $1,150 of understated deduction and the stale bracket ceiling produce $2,470 instead of $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It abandoned the Tier-2 Social Security formula mid-calculation and taxed roughly $20,705 of benefits instead of the $14,050.60 that 0.85 × ($45,236 − $34,000) + $4,500 produces, inflating AGI to $52,228. It also used a $15,750 base plus $2,000 aged amount rather than $16,100 plus $2,050, so taxable income came out $28,478 instead of $21,423." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its derivation is the reference chain line for line — AGI $45,574, $18,150 + $6,000 in deductions, taxable income $21,424, tax about $2,323 — and it then submitted 1170, a value its own arithmetic never generates. The computation was right and the reported number abandoned it." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Correct through AGI, it then deducted only $17,000 ($15,000 base + $2,000 aged, both stale figures) and never applied the OBBBA §70103 $6,000 senior deduction, which this filer takes in full because MAGI $45,574 is below the $75,000 phase-out start. That $7,150 of missing deduction, plus the 2025 $11,925 bracket break instead of $12,400, yields $3,189." -us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It omitted the $6,000 OBBBA senior deduction and used a $17,000 standard deduction to arrive at $3,189, then submitted 626 after an unexplained 'adjustment' for itemized mortgage interest it had already ruled out in favor of the standard deduction. Both the dropped senior deduction and the final number that contradicts its own arithmetic are wrong." -us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Taxable Social Security and the $45,574 AGI are right, but it deducted only $17,460 from an invented $15,400 base plus $2,060 aged amount and applied no $6,000 senior deduction. The correct 2026 stack of $16,100 + $2,050 + $6,000 = $24,150 leaves taxable income of $21,423, not the $28,114 it taxed." -us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a pre-TCJA rate schedule with a 15% second bracket and a $9,800 standard deduction, and set taxable Social Security at $19,668.60 rather than the worksheet's $14,050.60. For 2026 the rates remain 10%/12% with a $12,400 10% ceiling, and the deduction stack is $16,100 base + $2,050 aged + $6,000 senior." -us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"AGI $45,574 is correct; it then took a $17,050 deduction that leaves out the $6,000 OBBBA senior deduction and applied the 2025 $1,222.50/$11,925 bracket boundary instead of 2026's $1,240/$12,400. The full $24,150 deduction gives taxable income $21,423 and tax $2,322.76, not $3,178." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It modeled 2026 as a TCJA-sunset year, using an $8,100 standard deduction, a $5,050 personal exemption, a $1,950 aged addition, and a 15% second bracket. The TCJA rate schedule and the repeal of personal exemptions are permanent, and 2026 adds the $6,000 senior deduction, so the correct stack is $24,150 against 10%/12% rates on $21,423 of taxable income." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It started from $58,949 of 'gross taxable income,' which is every income line plus the full $27,426 of survivor benefits, skipping the provisional-income worksheet that includes only $14,050.60 of those benefits in AGI. It also applied no $6,000 senior deduction, so its base was overstated by about $13,375 and its deduction understated by $6,000." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"AGI $45,573.60 is exact, but it then applied 'pre-TCJA baseline' rules with a personal exemption and a 15% second bracket, and skipped the $6,000 senior deduction. Personal exemptions remain repealed for 2026 and the applicable rates here are 10% to $12,400 and 12% above, applied to $21,423 after the $24,150 deduction stack." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied 'post-TCJA expiration' rules to 2026 — a personal exemption on top of the standard deduction and a 15% second bracket — reaching taxable income of $30,716. The permanent 10%/12% schedule applies, personal exemptions stay repealed, and the $16,100 + $2,050 + $6,000 deduction stack leaves $21,423 of taxable income." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It submitted 0 with no derivation; that value is consistent with treating the $20,720 pension, $7,568 IRA distribution, $1,560 401(k) distribution, $1,675 of interest, and $14,050.60 of taxable survivor benefits as fully sheltered by deductions. The actual $24,150 deduction stack leaves $21,423 of taxable income, which carries $1,240 at 10% plus 12% of $9,023 for $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The Social Security worksheet and the $45,573.60 AGI are exact, but it deducted $17,450 with no OBBBA $6,000 senior deduction and used the 2025 $11,925 bracket break instead of $12,400. Adding the senior deduction moves taxable income from $28,123.60 to $21,423 and the tax from $3,136.33 to $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It built a $15,650 deduction that includes a personal exemption and taxed the excess over $11,925 at 15%, treating 2026 as a TCJA-sunset year. Personal exemptions remain repealed, the second bracket is 12% starting at $12,400, and the $6,000 senior deduction brings the deduction stack to $24,150 and taxable income to $21,423." -us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"AGI is exact at $45,573.60, but it deducted $17,000 built from a $15,000 base and $2,000 aged amount and omitted the $6,000 OBBBA senior deduction entirely, and used the 2025 $11,925 bracket break. The 2026 stack of $16,100 + $2,050 + $6,000 = $24,150 gives taxable income $21,423 and tax $2,322.76, not $3,190.33." -us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It computed the Social Security inclusion and $45,573.60 AGI correctly, then applied a roughly $17,000 standard deduction with no OBBBA senior deduction and explicitly used 2025 brackets. For 2026 the deduction is $16,100 + $2,050 + $6,000 = $24,150 and the 10% bracket runs to $12,400, giving $2,322.76 on $21,423 of taxable income." -us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"AGI $45,574 is right; it then guessed a $17,100 deduction ($15,450 base + $1,650 aged) and never applied the $6,000 senior deduction, which is available in full below the $75,000 MAGI phase-out. That leaves taxable income at $28,474 instead of $21,423 and tax at $3,170 instead of $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It concluded that the absence of wage income and dependents means no liability. The $20,720 private pension, $7,568 IRA distribution, $1,560 401(k) distribution, $1,675 of taxable interest, and $14,050.60 of taxable survivor benefits are all ordinary income, and after the $24,150 deduction stack $21,423 remains taxable at 10%/12% for $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction and retirement contributions offset the income; the listed contributions total $10 ($8 traditional 401(k) plus $1 each of Roth 401(k) and Roth IRA) and change nothing, and the Roth amounts are not deductible at all. AGI $45,573.60 less the $24,150 deduction stack leaves $21,423 taxable, producing $2,322.76 rather than zero." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $4,200 requires taxable income near $37,100 under the 2026 schedule — the figure that results from taxing 85% of the survivor benefits ($23,312) instead of the worksheet's $14,050.60 and deducting roughly $17,700 with no $6,000 senior deduction. The correct chain is AGI $45,573.60 less $24,150, giving $21,423 and $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It explicitly applied 'current-law rules after TCJA sunset,' combining a $15,889 standard-deduction-plus-personal-exemption total with a 15% bracket above $12,409. Personal exemptions remain repealed for 2026, the second bracket is 12%, and the deduction stack is $16,100 + $2,050 + $6,000 = $24,150, leaving $21,423 taxable." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied a TCJA-sunset schedule for 2026 — an $8,550 basic deduction, a $2,100 aged addition, a $5,450 personal exemption, and 15% above $12,550. The 2026 amounts are a $16,100 basic deduction plus a $2,050 aged addition plus the $6,000 senior deduction, with no personal exemption and a 12% second bracket starting at $12,400." -us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It taxed $23,264 of survivor benefits, near the 85% ceiling, instead of the $14,050.60 the Tier-2 formula produces, then took a $10,631 'post-TCJA reversion' standard deduction and a 15% bracket. The 2026 deduction is $18,150 plus the $6,000 senior amount and the top rate reached here is 12%, so taxable income is $21,423, not $44,148." -us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Everything is right through the $18,150 standard deduction for a single filer aged 67, including the $14,051 of taxable Social Security and the 10%/12% brackets with the $12,400 break. The single miss is the OBBBA §70103 $6,000 senior deduction, taken in full because MAGI $45,574 is under the $75,000 phase-out start; adding it moves taxable income from $27,424 to $21,423 and the tax from $3,043 to $2,322.76." -us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so the model submitted no substantive computation to evaluate. The contract required federal_income_tax_before_refundable_credits as a numeric key in the outputs object and it is absent." -us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It included 85% of the survivor benefits ($23,312) rather than the $14,050.60 the provisional-income worksheet yields, reached about $61,449 of gross income and a $16,850 deduction, and then declared the remainder 'below the 10% bracket threshold' and submitted 0. That remainder is roughly $44,600 by its own numbers, and the correct taxable income of $21,423 carries $2,322.76 of tax." -us,scenario_085,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It dropped the '+ lesser of $4,500 or half the benefits' term from the Tier-2 Social Security formula, taxing only $9,550.60 of benefits instead of $14,050.60 and understating AGI by exactly $4,500; with its otherwise correct $24,150 deduction and 10%/12% brackets that costs $540 of tax. Its submitted $3,680 matches neither the reference nor the $1,782.83 its own explanation computes." -us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It taxed Social Security at the 85% ceiling ($23,312.10) instead of applying the Tier-2 alternative that yields $14,050.60, used a $15,750 deduction with no $2,050 aged addition and no $6,000 senior deduction, computed about $4,500, and then submitted $8,587.50 — a number none of its own arithmetic produces. The correct chain gives $21,423 of taxable income and $2,322.76 of tax." -us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It added the $23,312 of taxable benefits to the $45,236 provisional-income figure, which already contains all the other income, producing a double-counted AGI of $68,548, and it used the 85% ceiling rather than the $14,050.60 the worksheet gives. It then deducted only the $16,100 base with no $2,050 aged addition and no $6,000 senior deduction, and subtracted two credits that do not exist — a '$1,600 Senior Bonus credit' and a '$2,100 credit for survivors with dependent children.'" +us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning matches the reference exactly: taxable SS $14,051, AGI $45,574, deductions of $18,150 plus the $6,000 senior deduction, taxable income $21,424, and tax of $2,323. It then submitted 940, a number its own explanation never derives." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the full $27,426 of Social Security into AGI instead of the $14,050.60 taxable portion, and it used a made-up $27,750 deduction. It then zeroed the tax with nonrefundable credits the filer does not get. The Credit for the Elderly is reduced to zero because the $13,375 of nontaxable Social Security alone exceeds the $5,000 initial amount. AGI of $45,574 is above the Saver's Credit income limit." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its AGI of $45,574 is correct. However, it used about $17,000 for the aged standard deduction instead of $16,100 + $2,050 = $18,150, so its total deduction was $23,000 rather than $24,150. It also ended the 10% bracket at the 2025 figure of $11,925 instead of the 2026 figure of $12,400. That produced $2,470 instead of $2,322.76." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It put taxable Social Security at about $20,705 without applying the formula, which caps it at 0.85 × ($45,236 − $34,000) + $4,500 = $14,051. That inflated AGI to $52,228. It also used $15,750 + $2,000 instead of the 2026 amounts of $16,100 + $2,050. Together these pushed taxable income to $28,478 and the tax to $3,169." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its reasoning matches the reference: AGI $45,574, deductions of $24,150 including the $6,000 senior deduction, taxable income $21,424, and tax of about $2,323. It then submitted 1,170, which contradicts its own calculation." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left out the OBBBA $6,000 senior deduction entirely. It also used 2025-level deductions ($15,000 + $2,000 = $17,000) instead of $16,100 + $2,050 = $18,150, and 2025 brackets. It subtracted the $8 traditional 401(k) contribution even though there are no wages for it to reduce. The result was taxable income of $28,566 instead of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its derivation used a $17,000 aged standard deduction with no $6,000 senior deduction, giving $3,189. It then submitted 626, backed only by a vague reference to 'additional itemized deduction consideration' that it never computed. The correct $24,150 of deductions gives $2,322.76." +us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It took only a $17,460 standard deduction ($15,400 + $2,060) and left out the $6,000 senior deduction. It also understated the 2026 base deduction, which is $16,100. That overstated taxable income at $28,114 instead of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law: a $9,800 aged standard deduction and a 15% second bracket. OBBBA made the TCJA brackets and larger standard deduction permanent, so those do not apply in 2026. It also overstated taxable Social Security at $19,668.60 instead of $14,050.60." +us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $45,574 is correct. It subtracted only a $17,050 aged standard deduction, however, leaving out the $6,000 senior deduction and understating the base-plus-aged amount of $18,150. Taxable income came out at $28,524 instead of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, $18,150 standard deduction, and 2026 brackets are all correct, but it left out the $6,000 OBBBA senior deduction for filers 65 and older. That added $6,000 × 12% = $720, turning $2,322.76 into $3,042.83." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying an $8,100 standard deduction, a $5,050 personal exemption, and a 15% bracket. For 2026, OBBBA's permanent rules apply instead: a $16,100 + $2,050 standard deduction, the $6,000 senior deduction, and 10%/12% brackets." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It started from $58,949 of gross taxable income, which counts the full $27,426 of Social Security as taxable instead of the $14,050.60 that the provisional-income formula allows. It also never applied the full $24,150 of deductions, which includes the $6,000 senior deduction." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its AGI of $45,573.60 is correct, but it applied pre-TCJA baseline rules with a personal exemption and the older, smaller standard deduction. For 2026, OBBBA's $16,100 + $2,050 standard deduction, $6,000 senior deduction, and 10%/12% brackets apply." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied rules that assume the TCJA had expired, with a personal exemption and a 15% bracket, instead of OBBBA's permanent 10%/12% brackets and the $24,150 of deductions that includes the senior deduction. It also subtracted the $8 traditional 401(k) contribution even though there are no wages for it to reduce." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no working. The correct calculation leaves $21,423 of taxable income after $24,150 of deductions and a tax of $2,322.76. A $0 answer implies the model treated the $31,523 of pension, IRA, 401(k), and interest income, plus the taxable Social Security, as fully offset, which the deductions do not do." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its AGI of $45,573.60 is correct, but it subtracted only a $17,450 aged standard deduction and left out the $6,000 senior deduction. It also used the 2025 top of the 10% bracket ($11,925) instead of $12,400." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, with a standard deduction, aged addition, and personal exemption totaling $15,650 and higher post-TCJA rates. For 2026, OBBBA's $24,150 of deductions and 10%/12% brackets apply." +us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 deduction amounts ($15,000 + $2,000 = $17,000) and left out the $6,000 senior deduction. The correct 2026 total is $16,100 + $2,050 + $6,000 = $24,150. It also used the 2025 10% bracket limit of $11,925." +us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $17,000 aged standard deduction and 2025 brackets. It left out the $6,000 OBBBA senior deduction and understated the 2026 standard deduction of $16,100 + $2,050, giving taxable income of $28,573.60 instead of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted only $17,100 ($15,450 + $1,650) and left out the $6,000 senior deduction. It also understated the 2026 base-plus-aged standard deduction, which is $18,150. Taxable income came out at $28,474 instead of $21,423." +us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It concluded there was no liability because the household has no wages and no dependents. It ignored that the $20,720 pension, $7,568 IRA distribution, $1,560 401(k) distribution, $1,675 of interest, and $14,050.60 of taxable Social Security are ordinary taxable income. After $24,150 of deductions, that leaves $21,423 of taxable income." +us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the income was offset by the standard deduction and by retirement contributions or credits. The contributions total only $8 of traditional 401(k), with no wages for them to reduce, and no nonrefundable credits apply. The $24,150 of deductions still leaves $21,423 taxable, for $2,322.76 of tax." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its reference to 'deductions and exemptions' shows it applied rules that assume the TCJA had expired, with personal exemptions and higher rates. For 2026, OBBBA's $24,150 of deductions and 10%/12% brackets apply, giving $2,322.76, not about $4,200." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its AGI of $45,574 is correct, but it assumed the TCJA had expired, with an aged standard deduction plus personal exemption of $15,889 and a 15% bracket. For 2026, OBBBA's permanent rules apply: $16,100 + $2,050 + a $6,000 senior deduction, and 10%/12% brackets." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the TCJA sunset: an $8,550 basic deduction, a $2,100 aged amount, a $5,450 personal exemption, and a 15% bracket. OBBBA made the TCJA structure permanent and added the $6,000 senior deduction, so the correct deductions total $24,150 with 10%/12% brackets." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration rules: a standard deduction, aged addition, and personal exemption totaling $15,350, with higher brackets. For 2026, OBBBA's $24,150 of deductions, including the $6,000 senior deduction, and 10%/12% brackets apply." +us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the TCJA reversion, with a $10,631 aged standard deduction and a 15% bracket, instead of OBBBA's permanent 10%/12% brackets and $24,150 of deductions. It also taxed Social Security at about the 85% maximum ($23,264) instead of the $14,050.60 the provisional-income formula gives." +us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI, $18,150 standard deduction, and 2026 brackets are correct, but it left out the $6,000 OBBBA senior deduction. That overstated taxable income by $6,000 and the tax by $720 ($3,043 instead of $2,322.76)." +us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It taxed 85% of Social Security ($23,312) instead of applying the provisional-income formula, which gives $14,050.60. It also left out the $6,000 senior deduction. It then concluded the tax was near zero, even though its own figures leave more than $40,000 of taxable income after a $16,850 deduction." +us,scenario_085,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as only 0.85 × ($45,236 − $34,000) = $9,550.60. It dropped the $4,500 base amount (the lesser of $4,500 or half of benefits) that the worksheet adds, so it understated AGI at $41,073.60 and its explanation reached $1,782.83. The value it submitted, $3,680, matches neither that explanation nor the correct $2,322.76." +us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It taxed the full 85% maximum of Social Security ($23,312.10) instead of the $14,050.60 the formula gives. It subtracted only about $15,750 in deductions and left out the $6,000 senior deduction. It then submitted 8,587.50, which its own figure of about $4,500 does not support." +us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It taxed 85% of Social Security instead of $14,050.60 and miscounted AGI as $68,548. It took only a $16,100 deduction, with no aged addition or senior deduction. It then invented a '$1,600 Senior Bonus credit' and a '$2,100 survivor credit', although the OBBBA senior provision is a $6,000 deduction and no survivor credit exists." us,scenario_085,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no wages and computed zero employee Social Security, Medicare, and state payroll taxes, but then submitted 1459 instead of carrying that zero into the value field. The submitted number is unrelated to its stated derivation." us,scenario_085,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_085,snap,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_085,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model built the correct $1,675 interest base and $51.42 gross tax, then denied Tax Forgiveness on the ground that ""eligibility income including Social Security ($27,426) and pensions exceeds limits."" PA-40 Schedule SP eligibility income specifically excludes Social Security and Railroad Retirement benefits and old-age/retirement-plan distributions received after retirement age, so eligibility income is the $1,675 of interest — under the $6,500 single-claimant limit, giving 100% forgiveness and a nonrefundable credit that zeroes the $51.42." -us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model computed the correct $51.42 gross PA tax on $1,675 of interest but asserted ""total eligibility income far exceeds PA tax forgiveness limits,"" counting the Social Security survivor benefits and retirement distributions it had just excluded from taxable income. Schedule SP excludes those same items from eligibility income, leaving $1,675 against the $6,500 unmarried limit, so 100% forgiveness applies and the nonrefundable credit reduces the tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed the full $58,949 including Social Security survivor benefits and all pension, IRA, and 401(k) distributions, none of which fall in PA's eight taxable income classes for a 67-year-old, and then subtracted a fabricated $8,700 PA standard deduction — Pennsylvania grants no standard deduction or personal exemption. It also invented a $206.22 partial forgiveness figure instead of applying Schedule SP's 100% forgiveness tier, which at $1,675 of eligibility income cancels the entire $51.42 of tax actually owed on the interest." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly limited the PA base to $1,675 of interest and computed $51.4225 of gross tax (reporting it as $51.45), but stopped at gross tax and never applied PA-40 Schedule SP. With eligibility income of $1,675 — Social Security and post-retirement-age plan distributions are excluded from that measure — the claimant qualifies for 100% forgiveness, a nonrefundable credit that brings state income tax before refundable credits to $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model got the taxable base and 3.07% flat rate right, yielding $51.42, and then submitted gross tax as the answer without evaluating any nonrefundable credit. Pennsylvania's Tax Forgiveness credit gives this single 67-year-old with $1,675 of Schedule SP eligibility income 100% forgiveness, so the after-nonrefundable-credit liability is $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model reached the correct $51.42 gross tax on interest, then concluded ""PA has no nonrefundable credits applicable here (no children, no special credits noted)."" Tax Forgiveness under Schedule SP is PA's nonrefundable credit and requires no dependents: at $1,675 of eligibility income, below the $6,500 unmarried poverty limit, forgiveness is 100% and the liability is $0." -us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model derived the right $1,675 base and $51.42 gross tax, then submitted 500 as a padded guess ""to reflect potential additional taxable classes"" that contradicts its own stated $51 conclusion. Both the stated $51 and the submitted $500 skip PA-40 Schedule SP, whose 100% forgiveness tier at $1,675 of eligibility income (Social Security and post-retirement-age distributions excluded) wipes out the entire tax, giving $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model correctly excluded Social Security and retirement distributions and computed $51.42 of gross tax on $1,675 of interest, rounding to $51, but never applied a nonrefundable credit. Schedule SP eligibility income here is $1,675, under the $6,500 single limit, so the 100% Tax Forgiveness credit reduces state income tax before refundable credits to $0." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a ""$1,250 exclusion (age ≥59.5)"" against interest income; Pennsylvania grants no age-based interest exclusion — the retirement exclusion covers old-age distributions from eligible plans, not interest — so the full $1,675 is taxable and gross tax is $51.42, not $13.05. It also omitted PA-40 Schedule SP, whose 100% forgiveness tier at $1,675 of eligibility income eliminates that $51.42 entirely." -us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The model computed the correct $51.42 gross tax and then stated ""no PA nonrefundable credits apply."" PA's Tax Forgiveness credit applies here: Schedule SP eligibility income excludes the Social Security survivor benefits and the post-retirement-age pension, IRA, and 401(k) distributions, leaving $1,675 and a 100% forgiveness rate that produces $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model applied the exclusions and 3.07% rate correctly to reach $51.42 but reported gross tax rather than tax after nonrefundable credits. Schedule SP's 100% forgiveness tier, reached because eligibility income is only the $1,675 of taxable interest, cancels the full $51.42 and yields $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's one-line rationale excludes only Social Security, which at 3.07% on the remaining $31,523 gives $967.76; its submitted $1,782 instead corresponds to applying the flat rate to essentially the entire $58,949 of income, so it taxed the Social Security survivor benefits and all retirement distributions that PA's eight income classes exclude for a 67-year-old. The correct derivation taxes only $1,675 of interest for $51.42 of gross tax, which the 100% Schedule SP Tax Forgiveness credit at $1,675 of eligibility income reduces to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model correctly taxed only the $1,675 of interest at 3.07% for $51 but treated gross tax as the final answer. PA-40 Schedule SP grants this single claimant 100% forgiveness at $1,675 of eligibility income — Social Security and post-retirement-age plan distributions are excluded from that measure — so the nonrefundable credit brings the liability to $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model applied the retirement and Social Security exclusions correctly and stopped at $51.42 of gross tax without checking PA's nonrefundable Tax Forgiveness credit. Eligibility income of $1,675 falls below the $6,500 unmarried poverty limit, so Schedule SP forgives 100% of the tax and the answer is $0." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model reached the right $51.42 of gross tax on $1,675 of interest but never evaluated Schedule SP. With Social Security and post-retirement-age distributions excluded from eligibility income, the claimant's $1,675 qualifies for 100% Tax Forgiveness, a nonrefundable credit that zeroes the liability." -us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"The model correctly identified $1,675 of interest as the only PA-taxable income and computed $51.42, then submitted gross tax instead of tax after nonrefundable credits. PA's Tax Forgiveness credit forgives 100% at $1,675 of Schedule SP eligibility income, producing $0." -us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model handled the exclusions and the flat rate correctly for $51.42 of gross tax but omitted PA-40 Schedule SP from its credit analysis entirely. Because Schedule SP eligibility income excludes the $27,426 of Social Security survivor benefits and the $29,848 of post-retirement-age distributions, eligibility income is $1,675 and forgiveness is 100%, leaving $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model computed $51.42 on the correct $1,675 base and stated ""no nonrefundable credits applied."" Pennsylvania's Tax Forgiveness credit is nonrefundable and applies at 100% here, since Schedule SP eligibility income of $1,675 is below the $6,500 unmarried limit, so state income tax before refundable credits is $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly excluded Social Security and qualified retirement distributions and computed $51.4225 of gross tax, but reported that figure without applying any nonrefundable credit. Schedule SP forgives 100% of PA tax at $1,675 of eligibility income, so the required after-credit amount is $0." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model derived the correct $1,675 taxable base and $51.42 of gross tax and stopped there. PA-40 Schedule SP grants this 67-year-old single claimant 100% Tax Forgiveness because Social Security and post-retirement-age distributions drop out of eligibility income, leaving $1,675 and a $0 after-credit liability." -us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model reached the correct $51.42 gross tax but ruled out forgiveness because ""total household income precludes tax forgiveness,"" applying gross household income instead of Schedule SP eligibility income. Schedule SP excludes Social Security and Railroad Retirement benefits and old-age distributions from eligible retirement plans, so eligibility income is $1,675, forgiveness is 100%, and the liability is $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model correctly restricted the PA base to $1,675 of interest and reported the resulting $51 of gross tax with no credit step. The Schedule SP Tax Forgiveness credit is nonrefundable and forgives 100% at $1,675 of eligibility income, so the value requested — tax after nonrefundable credits — is $0." -us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed the $20,720 pension, $7,568 IRA, and $1,560 401(k) distributions along with interest for a $31,523 base; Pennsylvania excludes old-age distributions from eligible retirement plans received after retirement age, and the head is 67, so only the $1,675 of interest is taxable. Correcting the base gives $51.42 of gross tax, which Schedule SP's 100% forgiveness tier at $1,675 of eligibility income then eliminates, leaving $0 rather than $968." -us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model correctly taxed only the $1,675 of interest at 3.07% for $51 but omitted PA's nonrefundable Tax Forgiveness credit. At $1,675 of Schedule SP eligibility income the forgiveness rate is 100%, so the after-nonrefundable-credit liability is $0." -us,scenario_085,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model built a $31,523 base by taxing the pension, IRA, and 401(k) distributions, which Pennsylvania excludes as old-age benefits from eligible retirement plans for this 67-year-old, and then compounded the error by declaring ""no PA tax forgiveness... at this income level."" The correct base is the $1,675 of interest, and Schedule SP eligibility income is that same $1,675 — below the $6,500 unmarried limit — so 100% forgiveness reduces the $51.42 of gross tax to $0." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"The model got the exclusions and the $51.42 gross tax exactly right and then stated ""no nonrefundable credits apply."" PA's Tax Forgiveness credit is the applicable nonrefundable credit: with Schedule SP eligibility income of $1,675 for an unmarried claimant with no dependents, forgiveness is 100% and the liability is $0." -us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model computed the correct $51.42 gross tax but dismissed forgiveness as unavailable ""at this much higher eligibility income,"" treating the Social Security survivor benefits and retirement distributions as part of Schedule SP eligibility income. Schedule SP excludes Social Security and Railroad Retirement benefits and post-retirement-age distributions from eligible plans, leaving $1,675 of eligibility income, a 100% forgiveness rate, and $0 of tax." -us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model placed the $20,720 pension, $7,568 IRA, and $1,560 401(k) distributions inside PA's eight taxable classes for a $31,523 base, but distributions received as old-age benefits from eligible retirement plans after retirement age are not compensation or taxable gain in Pennsylvania for this 67-year-old, leaving only $1,675 of interest. It further concluded ""no nonrefundable credits apply,"" whereas Schedule SP forgives 100% of the $51.42 of correctly computed tax at $1,675 of eligibility income, giving $0 instead of $967.76." +us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It correctly got PA taxable income down to the $1,675 of interest and $51.42 of tax. It then ruled out Tax Forgiveness by adding the $27,426 of Social Security and the pensions to eligibility income. Schedule SP eligibility income leaves out both, so it is $1,675, under the $6,500 single threshold, which gives 100% forgiveness and $0 tax." +us,scenario_085,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It computed $51.42 on the $1,675 of interest. It then denied Tax Forgiveness by treating Social Security and exempt retirement distributions as eligibility income. Schedule SP leaves both out, so eligibility income is $1,675, below the $6,500 single limit, and full forgiveness brings the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built PA taxable income from federal AGI including Social Security and all retirement distributions, and applied a made-up $8,700 PA standard deduction. It also used a 3.077% rate. PA exempts Social Security and a 67-year-old's pension, IRA and 401(k) distributions and has no standard deduction, so taxable income is only $1,675 of interest. Schedule SP Tax Forgiveness then cancels that tax in full, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It stopped at 3.07% tax on the $1,675 of interest and never applied PA Special Tax Forgiveness. Eligibility income is $1,675, below the $6,500 single threshold, so forgiveness is 100% and tax is $0. It also misrounded $51.4225 to $51.45." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly limited PA taxable income to the $1,675 of interest but never applied Schedule SP Tax Forgiveness. Eligibility income excludes Social Security and retirement benefits, so it is $1,675, under the $6,500 single limit. Full forgiveness wipes out the $51.42, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It assumed forgiveness eligibility income includes the exempt retirement income and so exceeds the forgiveness limit. Schedule SP eligibility income leaves out commonly recognized retirement benefits and Social Security, so it equals the $1,675 of interest. That is below $6,500, so the $51.42 is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded that no PA nonrefundable credits apply and overlooked Special Tax Forgiveness. Eligibility income of $1,675 (interest only) is below the $6,500 single threshold, so 100% forgiveness reduces the tax to $0. It submitted $51, the tax before forgiveness, rounded." +us,scenario_085,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own reasoning reached $51 on the $1,675 of interest, but it submitted an arbitrary $500 'to reflect potential additional taxable classes', which contradicts that derivation. It also never applied Schedule SP Tax Forgiveness, which fully forgives tax when eligibility income ($1,675) is below $6,500. The correct result is $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It taxed the $1,675 of interest at 3.07% and stopped, never applying PA Special Tax Forgiveness. Eligibility income excludes Social Security and exempt retirement distributions, so it is $1,675, under the $6,500 single limit. Full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It made up a $1,250 senior interest exclusion that does not exist in PA law, cutting taxable interest to $425. It also said no state credits apply, missing Tax Forgiveness. With eligibility income of $1,675, well under the $6,500 single threshold, Schedule SP forgives 100% of the tax, so the answer is $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It declared that no PA nonrefundable credits apply and overlooked Special Tax Forgiveness. Eligibility income is only the $1,675 of interest, since Social Security and retirement benefits are excluded. That is below $6,500, so the $51 is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single-filer threshold, so 100% forgiveness gives $0 tax." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It correctly exempted Social Security and retirement distributions but stopped at $51.42 without applying Schedule SP Tax Forgiveness. Eligibility income is $1,675, below $6,500, so the tax is 100% forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"$1,782 is about 3.07% of roughly $58,000, which is the household's full $58,949 of gross income. So despite claiming to exclude Social Security, it taxed essentially all income, including pension, IRA and 401(k) distributions that PA exempts after age 59½. Correctly, taxable income is $1,675 of interest, and Tax Forgiveness then eliminates that tax, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It taxed only the $1,675 of interest but left out PA Special Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single threshold, so full forgiveness reduces the $51 to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It reported $51.42 as final tax without applying Schedule SP Tax Forgiveness. Social Security and retirement benefits are excluded from eligibility income, so eligibility income is $1,675, under the $6,500 single limit, and the tax is fully forgiven to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It stopped at 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Eligibility income of $1,675 is below $6,500, which gives 100% forgiveness and $0 tax." +us,scenario_085,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It correctly identified $1,675 as the only PA-taxable income but skipped Special Tax Forgiveness. That same $1,675 is the Schedule SP eligibility income, below the $6,500 single threshold, so the $51.42 is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It computed $51.42 on the interest and treated that as the final tax, never applying PA Tax Forgiveness. Eligibility income excludes Social Security and exempt retirement distributions, so it is $1,675, below $6,500, and full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It explicitly applied no nonrefundable credits and so missed Schedule SP Special Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single limit, so 100% forgiveness reduces tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed the $1,675 of interest at 3.07% and stopped without applying PA Tax Forgiveness. Eligibility income is $1,675, below the $6,500 single threshold, so the tax is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It correctly excluded retirement distributions and Social Security but never applied Special Tax Forgiveness. Eligibility income of $1,675 is under $6,500, so 100% forgiveness gives $0 instead of $51.42." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It ruled out Tax Forgiveness on the basis of total household income, counting Social Security and retirement distributions. Schedule SP eligibility income leaves those out, so it is only the $1,675 of interest, below the $6,500 single limit, and full forgiveness brings the tax to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It stopped at about $51 of tax on the interest without applying PA Special Tax Forgiveness. Eligibility income of $1,675 is below the $6,500 single threshold, so the tax is fully forgiven to $0." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It computed 3.07% of the $1,675 of interest and never applied Schedule SP Tax Forgiveness. Eligibility income of $1,675 is under $6,500, which gives 100% forgiveness and $0 tax." +us,scenario_085,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It concluded that no nonrefundable credit applies and overlooked PA Special Tax Forgiveness. Eligibility income excludes Social Security and retirement benefits, so it is $1,675, below the $6,500 single limit. Full forgiveness wipes out the $51.42, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It applied the flat 3.07% rate to the $1,675 of interest and stopped, never applying PA Tax Forgiveness. Eligibility income of $1,675 is under the $6,500 single threshold, so the tax is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It included the $20,720 pension and the $7,568 IRA and $1,560 401(k) distributions in PA taxable income. PA exempts these retirement distributions for a retiree aged 59½ or older. Correct taxable income is $1,675 of interest, and Schedule SP forgiveness then eliminates that tax because eligibility income is below $6,500, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It taxed only the $1,675 of interest but never applied Special Tax Forgiveness. Eligibility income of $1,675 is below $6,500, so 100% forgiveness reduces the $51 to $0." +us,scenario_085,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It stopped at 3.07% of the $1,675 of interest and never applied PA Tax Forgiveness. Social Security and retirement benefits are excluded from eligibility income, so it is $1,675, under the $6,500 single threshold, and full forgiveness gives $0." +us,scenario_085,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It taxed the $29,848 of pension, IRA and 401(k) distributions, which PA excludes for a 67-year-old retiree, reaching $31,523 of taxable income. On that inflated base it then rejected Tax Forgiveness. The correct base is $1,675 of interest, which also serves as eligibility income; that is below $6,500, so forgiveness is 100% and tax is $0." +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It correctly reduced PA taxable income to the $1,675 of interest but asserted that no nonrefundable credits apply, missing Special Tax Forgiveness. Eligibility income of $1,675 is below the $6,500 single limit, so the $51.42 is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It rejected Tax Forgiveness by treating Social Security and exempt retirement distributions as eligibility income. Schedule SP leaves out both, so eligibility income is only $1,675, under the $6,500 single threshold. The $51.42 is fully forgiven, giving $0." +us,scenario_085,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the pension, IRA and 401(k) distributions as PA-taxable, giving $31,523 of taxable income. PA exempts retirement distributions received after age 59½/retirement, so taxable income is only the $1,675 of interest. It also missed that Schedule SP Tax Forgiveness zeroes that tax because eligibility income is below $6,500, so the answer is $0." us,scenario_086,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated Medicaid ineligibility as sufficient for CHIP eligibility and never applied Georgia’s CHIP income limit. Although the 12-year-old satisfies the age criterion, the household fails the CHIP income criterion, so the child is not eligible." us,scenario_086,child1_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly found that income exceeds Georgia’s child Medicaid limit, then reversed its result by treating PeachCare/CHIP as part of PolicyEngine’s Medicaid output. Medicaid and CHIP are separate benchmark variables, and child1 qualifies for no Medicaid category at 3.44 times FPL." us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the head’s employer-sponsored insurance premiums from Medicaid MAGI and then asserted that $74,824 was within a roughly $50,487 limit. Those premiums do not reduce MAGI on the stated facts, and 3.44 times FPL exceeds Georgia’s child Medicaid thresholds." us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category." us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE." us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It reassigned the $12,824 of Social Security dependent benefits to the child and excluded them from the filer's income, dropping the $10,900.40 of taxable benefits that provisional income of $68,074.50 (well above the $34,000 second threshold for a non-joint filer) makes includible at the 85% cap. It also capped the educator deduction at $300 instead of the $337.50 PolicyEngine applies for 2026 and subtracted $275 of auto-loan interest. Its own arithmetic terminated at $1,925 and it submitted $3,305, a figure no step of its derivation produces." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It reproduced the reference chain — 85% inclusion of $10,900.40, AGI $72,562.40, $24,150 head-of-household standard deduction — then subtracted an additional $275 auto-loan interest deduction, which does not apply because the OBBBA vehicle-loan conditions (new personal-use vehicle, US final assembly, first-lien security) are unlisted and therefore false. It compounded this by placing the 10% bracket ceiling at $17,000 rather than the 2026 head-of-household $17,700, so the $275 over-deduction (-$33) and the bracket error (+$14) net to the $19.06 shortfall." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted $22,768 of insurance premiums ($19,648 employer-sponsored plus two $1,560 personal-premium entries) as above-the-line adjustments; employer-sponsored premiums are already excluded from the $62,000 of Box 1 wages, and personal premiums are itemized medical expenses that fail the 7.5%-of-AGI floor. It then used the single filer's $14,600 standard deduction instead of the $24,150 head-of-household amount, and submitted $3,779 as a pre-credit figure while the requested output is after the $2,200 nonrefundable CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It declared the $12,824 of Social Security dependent benefits to be the child's income and excluded them, omitting the $10,900.40 of taxable benefits that the 85% inclusion tier produces at $68,074.50 of provisional income. It further used a $23,000 head-of-household standard deduction instead of $24,150 and a $2,000 CTC instead of the 2026 amount of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It excluded the $12,824 of Social Security dependent benefits as the child's income, dropping the $10,900.40 taxable at the 85% tier, and used a $23,000 standard deduction rather than $24,150. It then abandoned its own $4,304 bracket computation for an unexplained $4,673 and submitted that pre-credit tentative tax instead of the $2,673 after-CTC figure its reasoning reached, so the answer is the wrong quantity as well as the wrong amount." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It asserted the $12,824 of Social Security benefits are not taxable at this income level; provisional income of $68,074.50 exceeds the $34,000 upper threshold for a head-of-household filer by more than $34,000, so the 85% cap binds and $10,900.40 enters AGI. It also used a $24,500 standard deduction instead of $24,150 and a $2,000 CTC instead of $2,200, then abandoned its $4,119 bracket result for an unexplained $4,519." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied a pre-TCJA-sunset 2026 — an $8,300-to-$11,900 single standard deduction, two ~$5,300 personal exemptions, a 15% second bracket, and a $1,000 CTC — where 2026 runs on the permanent OBBBA schedule: $24,150 head-of-household standard deduction, zero personal exemptions, 10%/12% brackets breaking at $17,700, and a $2,200 CTC. It compounded this by filing the head as single rather than head of household and by excluding the $10,900.40 of taxable Social Security benefits as the child's income." -us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security benefits as non-taxable, omitting the $10,900.40 includible at the 85% tier given $68,074.50 of provisional income, and used a $23,350 standard deduction with a $16,550 bracket break rather than $24,150 and $17,700. Its stated arithmetic ends at $2,271 and it submitted $3,892, a number its derivation never produces." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly applied the 85% Social Security inclusion but used a $23,100 head-of-household standard deduction instead of the 2026 amount of $24,150 and a $2,000 CTC instead of $2,200, then rounded the bracket tax to $5,600. Those two parameter substitutions account for essentially the entire $344 shortfall." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It built 2026 on repealed pre-TCJA parameters — an $11,000 standard deduction, $9,600 of personal exemptions, a 15% second bracket, and a $1,000 CTC — instead of the $24,150 head-of-household standard deduction, no exemptions, the 12% bracket above $17,700, and the $2,200 CTC. It also excluded the $10,900.40 of taxable Social Security benefits entirely and subtracted $1,560 of personal health premiums, which are itemized medical expenses below the 7.5%-of-AGI floor, not an adjustment to income." -us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It got the income side right — $10,900.40 of taxable Social Security and an AGI near $72,600 — then applied a $23,100 standard deduction instead of $24,150 and a $2,000 CTC instead of the 2026 $2,200. Those two substitutions, plus capping the educator deduction at $300 rather than $337.50, produce its $664 shortfall." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored insurance premiums from the $62,000 of wages, double-counting an exclusion already reflected in the stated gross wage figure, cutting taxable wages to $42,352. It then applied pre-TCJA-sunset parameters — a $16,500 standard deduction, $10,100 of personal exemptions, a 15% bracket, and a $1,000 CTC — instead of the $24,150 head-of-household standard deduction, no exemptions, the 12% bracket, and the $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It included the full $12,824 of Social Security benefits rather than the 85% inclusion of $10,900.40, used a $15,300 standard deduction instead of the $24,150 head-of-household amount, and a $2,000 CTC instead of $2,200. Its stated inputs yield about $59,524 of taxable income and roughly $6,789 of pre-credit tax, so the submitted $1,782 does not follow from its own derivation either." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It reached the correct AGI of $72,560 with the 85% Social Security inclusion, then assumed the TCJA individual provisions expire for 2026 and restored personal exemptions and the $1,000 nonrefundable CTC. For 2026 there are no personal exemptions, the head-of-household standard deduction is $24,150, and the CTC is $2,200, which turns its $6,606-less-$1,000 into $5,455.55 less $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It removed the $19,648 of employer-sponsored insurance premiums from wages, double-counting an exclusion already built into the $62,000 gross wage figure, and then applied sunset-law parameters: a $15,000 standard deduction, $10,100 of personal exemptions, and a $1,000 CTC. The 2026 computation keeps wages at $62,000, allows no personal exemptions, deducts $24,150, and applies a $2,200 CTC." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It supplied no derivation, only the number. The correct chain — $10,900.40 of taxable Social Security into an AGI of $72,562.90, less the $24,150 head-of-household standard deduction, taxed at 10% to $17,700 then 12%, less the $2,200 CTC — gives $3,255.55; its $3,043 corresponds to about $46,642 of taxable income, roughly $1,771 short, consistent with an inflated standard deduction." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It named a $2,000 nonrefundable CTC where 2026 provides $2,200, and its $3,678 implies $50,267 of taxable income — about $1,854 above the correct $48,412.90, consistent with a standard deduction near $22,300 rather than the head-of-household $24,150. Those two parameter errors fully account for the gap." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions on top of the head-of-household standard deduction; personal exemptions are zero for 2026, so the only allowance against the $72,562.90 AGI is the $24,150 standard deduction. Subtracting phantom exemptions from the correct $48,412.90 of taxable income is what pushes its answer $1,366.81 below the $3,255.55 that the 10%/12% schedule less the $2,200 CTC produces." -us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It handled the 85% Social Security inclusion correctly but stacked four parameter errors: a $22,500 standard deduction instead of $24,150, a $17,000 bracket break instead of $17,700, a $300 educator cap instead of $337.50, and a $2,000 CTC instead of $2,200. Correcting the standard deduction and CTC alone moves its $3,672.05 to within a few dollars of the reference." -us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It assigned the $12,824 of Social Security benefits to the child as representative-payee income and tested the child's own $25,000 provisional-income base, excluding the benefits entirely; the benefits belong to the head, whose provisional income of $68,074.50 makes $10,900.40 taxable at the 85% cap. It also used a $22,500 standard deduction instead of $24,150 and a $2,000 CTC instead of $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It got AGI ($72,562), the standard deduction (~$24,350), and the $2,200 CTC essentially right, then computed $6,347 of tax on roughly $48,212 of taxable income. The 2026 head-of-household schedule — 10% to $17,700 then 12% — yields $5,431 on that base, so it overstated the bracket tax by about $916 by applying rates above 12% to income that never leaves the 12% bracket (which runs to $67,450)." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It asserted that the standard deduction and nonrefundable credits fully offset the liability without computing either. The $24,150 head-of-household standard deduction against $72,562.90 of AGI leaves $48,412.90 of taxable income and $5,455.55 of tentative tax, and the only nonrefundable credit available is the $2,200 CTC for one child, which leaves $3,255.55 standing." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed a deduction for the household's listed medical expenses; those total $3,060 including premiums, far below the 7.5%-of-AGI floor of $5,442, and the $24,150 standard deduction exceeds any itemized total in any case. It also omitted the $10,900.40 of taxable Social Security benefits, and its $2,682 implies roughly $43,600 of taxable income against the correct $48,412.90." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It excluded the $10,900.40 of taxable Social Security benefits from AGI and used a $22,500 standard deduction instead of $24,150, then allowed only $384.40 of the $2,200 CTC against liability on the theory that a refundable portion is claimed first. The CTC offsets tax nonrefundably up to the full $2,200 before any refundable ACTC arises, so the entire $2,200 applies against the $5,455.55 tentative tax." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included $6,412 — half the benefits — as taxable Social Security, applying the 50% tier. Provisional income of $68,074.50 exceeds the $34,000 upper threshold for a head-of-household filer, so the inclusion is the lesser of 85% of benefits ($10,900.40) or $4,500 plus 85% of the excess; the 85% cap binds and $10,900.40 is includible, and the missing $4,488.40 of income costs $538.61 of tax at 12%." -us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It identified every component correctly — the $338 educator deduction, taxable Social Security, the head-of-household standard deduction, and the $2,200 nonrefundable CTC — but its bracket arithmetic overstated the result by $13.45. Twelve percent of the $30,712.40 above the $17,700 break is $3,685.49, giving $5,455.49 of tentative tax and $3,255.49 after the credit, not the ~$5,469 its answer implies." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It deducted mortgage interest; only a $55,000 mortgage balance is listed, no interest amount, so that input is zero, and the $24,150 standard deduction dominates any itemized total regardless. It also gave no treatment of the $10,900.40 of taxable Social Security, and its $4,523 requires roughly $58,900 of taxable income against the correct $48,412.90." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It reached the correct AGI of $72,562 with the 85% Social Security inclusion, then applied post-TCJA-expiration parameters: a $12,389 standard deduction, $10,733 of personal exemptions, 10%/15% brackets, and a $1,000 CTC. For 2026 the head-of-household standard deduction is $24,150, personal exemptions are zero, the second bracket is 12% above $17,700, and the CTC is $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed AGI correctly at about $72,600 but then used a sunset-law $12,400 standard deduction, two $5,350 personal exemptions, a 15% bracket, and a $1,000 CTC. The 2026 schedule allows no personal exemptions, a $24,150 head-of-household standard deduction, a 12% rate above $17,700, and a $2,200 credit, which converts its $6,542-less-$1,000 into $5,455.55 less $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It stated the case as 'post-TCJA' and used a $12,538 standard deduction, $10,862 of personal exemptions, pre-TCJA 10%/15% brackets, and a $1,000 CTC, despite reaching the correct $72,562 AGI. For 2026 personal exemptions are zero, the head-of-household standard deduction is $24,150, the rate above $17,700 is 12%, and the CTC is $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reproduced the entire correct structure and missed by $4.45 on rounding: it took taxable Social Security as $10,890 rather than the exact 85% figure of $10,900.40, capped the educator deduction at $300 instead of $337.50, and rounded the bracket tax to about $5,460 instead of $5,455.55." -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the required key never reached the outputs object. This is a contract failure in emitting the answer, not a substantive tax computation error." -us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It got the 85% Social Security inclusion, the $24,150 standard deduction, the $17,700 bracket break, and the $2,200 CTC all correct, then subtracted $275 of auto-loan interest from AGI. That deduction requires a new personal-use vehicle with US final assembly securing the loan — conditions not listed and therefore false — and its $33.06 tax effect at 12% is the entire gap to $3,255.55." -us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It excluded the $12,824 of Social Security benefits, dropping the $10,900.40 includible at the 85% tier, and filed the head as single — a $15,000 standard deduction and single brackets breaking at $11,925 — despite the qualifying 12-year-old that establishes head-of-household status with a $24,150 standard deduction and a $17,700 break. It also used a $2,000 CTC rather than the 2026 $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed the head as single with a $16,100 standard deduction rather than head of household with $24,150, which pushed income into a 22% bracket that head-of-household taxable income of $48,412.90 never reaches (the 12% bracket runs to $67,450). It then allowed only $500 of the $2,200 CTC against tax by subtracting the $1,700 refundable cap first; the credit applies nonrefundably up to its full $2,200 before any refundable portion arises, and it additionally deducted $275 of non-qualifying auto-loan interest." -us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It filed the head and the 12-year-old as married filing jointly; a dependent child is not a spouse, and one adult with a qualifying child files as head of household. That error cascaded through the entire computation — it used the $32,000 MFJ provisional-income base, a $31,200 MFJ standard deduction instead of $24,150, and the MFJ bracket breaking at $23,850 instead of the head-of-household $17,700 — and it also used a $2,000 CTC rather than $2,200." -us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed a $32,200 standard deduction — the married-filing-jointly figure, where head of household is $24,150 — together with a full exclusion of the Social Security benefits, when provisional income of $68,074.50 makes $10,900.40 taxable at the 85% cap. Its own numbers contradict its answer: $74,824 less $32,200 is $42,624 of taxable income, which cannot yield zero tax after a $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security dependent benefits as the child's income, so the $10,900.40 taxable portion (85% cap, provisional income about $68,074) never reached the Head's AGI. It also capped educator expense at $300 and deducted $275 of auto loan interest. Its submitted $3,305 does not even match its own computed $1,925." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It deducted $275 of auto loan interest, but the facts establish no qualifying new vehicle, so no OBBBA vehicle-loan-interest deduction applies; that lowered taxable income to $48,137.40 instead of $48,412.90. It also topped the 2026 HoH 10% bracket at $17,000 instead of $17,700, which adds $14 of tax. Net result: $3,236.49 instead of $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included 100% of the $12,824 in SS benefits instead of the $10,900.40 taxable portion. It also subtracted $19,648 of employer-sponsored premiums and $3,120 of health premiums as above-the-line deductions, which are not allowed. It used single status with a $14,600 standard deduction instead of the $24,150 HoH amount. It then submitted $3,779, its tax before credits, without subtracting any CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It excluded the SS dependent benefits as the child's income, omitting $10,900.40 of taxable SS from the Head's AGI. It also used stale parameters: a $23,000 HoH standard deduction instead of $24,150, a $17,000 bracket top instead of $17,700, and a $2,000 CTC instead of $2,200. With the $300 educator cap, this gave $2,304." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It excluded the $10,900.40 of taxable SS benefits as the child's income and used a $23,000 HoH standard deduction and a $2,000 CTC. It then submitted $4,673, an inflated tax figure before credits, instead of its own after-CTC amount of $2,673." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It declared the SS benefits non-taxable 'at this income level' even though provisional income of $68,074 exceeds the $34,000 base, so $10,900.40 is taxable. It used a $24,500 standard deduction and a $2,000 CTC, then inflated its own $4,119 tax figure to $4,519 before subtracting the credit." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its computation is right except for the educator expense: it capped it at $300, but PolicyEngine's 2026 limit allows the full $337.50. That overstates taxable income by $37.50 ($48,450.40 vs $48,412.90) and tax by $4.50, giving $3,260.05." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA sunset for 2026, but OBBBA made the TCJA structure permanent. So it used pre-TCJA rules that no longer apply: an $8,300 single standard deduction, $10,600 of personal exemptions, a 15% bracket, and a $1,000 CTC. It also dropped the $10,900.40 of taxable SS as the child's income and filed as single instead of HoH." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the SS benefits as non-taxable even though provisional income exceeds $34,000, so the 85% cap makes $10,900.40 taxable. It used the stale $23,350 HoH deduction, $16,550 bracket and $2,000 CTC. It then replaced its own $2,271 result with an unexplained $3,892." +us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest, but no qualifying new vehicle is established, so the OBBBA vehicle-loan-interest deduction does not apply. That cut taxable income to about $48,137 instead of $48,412.90 and understated tax by about $33, giving $3,222." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It correctly included $10,900 of taxable SS but used a $23,100 HoH standard deduction instead of the 2026 $24,150. It subtracted a $2,000 CTC instead of $2,200, capped educator expense at $300, and rounded tax to $5,600, which gave $3,600." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (an $11,000 standard deduction, $9,600 of personal exemptions, a 15% bracket and a $1,000 CTC), but OBBBA made the TCJA structure permanent. It also subtracted the $1,560 health premium from wages and omitted the $10,900.40 of taxable SS." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,600.40 is right apart from the $300 educator cap. But it used a stale $23,100 HoH standard deduction instead of $24,150, which overstates taxable income by about $1,088. It also subtracted a $2,000 CTC instead of the 2026 $2,200, giving $3,591.05." +us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (a $12,200 standard deduction, $10,600 of personal exemptions, a 15% bracket and a $1,000 CTC), but OBBBA permanently kept the $24,150 HoH deduction, zero exemptions, the 12% bracket and the $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used a $16,500 standard deduction, $10,100 of personal exemptions, a 15% bracket and a $1,000 CTC, which OBBBA superseded. It also wrongly subtracted $19,648 of ESI premiums from the $62,000 gross wages." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted 100% of the $12,824 SS benefits as income instead of the $10,900.40 taxable portion and dropped the $337.50 educator deduction. It used a $15,300 standard deduction instead of the $24,150 HoH amount and a $2,000 CTC instead of $2,200, arriving at $1,782." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,560 is essentially correct, but it assumed the TCJA sunset: it used a pre-TCJA HoH standard deduction, two personal exemptions, the 15% bracket and a $1,000 CTC. OBBBA permanently set the 2026 values at $24,150, zero exemptions, 10%/12% brackets and a $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules (a $15,000 standard deduction, $10,100 of exemptions and a $1,000 CTC) that OBBBA superseded. It also subtracted the $19,648 of ESI premiums from gross wages, although the $62,000 wage input is already the taxable wage base." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. The correct path is AGI of $72,562.90 (including $10,900.40 of taxable SS), less the $24,150 HoH deduction, for taxable income of $48,412.90, tax of $5,455.55, and $3,255.55 after the $2,200 CTC. Its $3,043 is $212.55 too low, equivalent to understating taxable income by about $1,770 at the 12% rate." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $2,000 CTC instead of the 2026 $2,200. Its implied tax before credits of $5,678 is about $222 above the correct $5,455.55. That matches the stale $22,500 HoH standard deduction plus the $17,000 bracket top, instead of $24,150 and $17,700." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted personal exemptions, which OBBBA permanently set to zero, on top of the standard deduction. This understated taxable income and gave $1,888.74 instead of $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $72,600.40 AGI is right apart from the $300 educator cap, but it used the stale $22,500 HoH standard deduction instead of $24,150 and the $17,000 bracket top instead of $17,700. It also subtracted a $2,000 CTC instead of $2,200, giving $3,672.05." +us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It treated the SS dependent benefits as the child's non-taxable income, but they are the Head's, and 85% ($10,900.40) is taxable. It also used a $22,500 HoH deduction instead of $24,150 and a $2,000 CTC instead of $2,200, arriving at $2,360." +us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 and taxable income of about $48,212 are close to correct, but it computed tax of $6,347 on that income. The 2026 HoH schedule gives $1,770 + 12% of the excess over $17,700, about $5,431, so its bracket arithmetic overstated tax by roughly $900." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed nonrefundable credits fully offset the tax. The only nonrefundable credit is the $2,200 CTC, and taxable income of $48,412.90 generates $5,455.55 of tax, leaving $3,255.55, not $0." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted itemized medical expenses on top of the standard deduction, although the $3,060 of medical costs falls below the 7.5%-of-AGI floor and cannot be combined with the standard deduction. It also omitted the $10,900.40 of taxable SS, arriving at $2,682." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $10,900.40 of taxable SS and used a stale $22,500 HoH deduction. It then subtracted only $384.40 of CTC by reserving a 'refundable portion' first, but the nonrefundable CTC applies in full up to liability, so the full $2,200 is used." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included only $6,412 (50%) of the SS benefits as taxable. Provisional income of about $68,074 exceeds the $34,000 second base, so the 85% cap applies and $10,900.40 is taxable. This understated AGI and tax." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its components (taxable SS, $338 educator deduction, $24,150 HoH deduction, $2,200 CTC) match the correct path. Its $3,269 is $13.45 high, which matches taxing taxable income of about $48,413 with a $17,000 bracket top instead of the 2026 HoH $17,700 threshold (+$14)." +us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its brackets, $24,150 deduction and $2,200 CTC are right, but taxable income of $37,550 omits the $10,900.40 of taxable SS dependent benefits (85% cap at provisional income of about $68,074). It also capped educator expense at $300, understating tax by about $1,304." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It built taxable income from wages only, leaving out the $10,900.40 of taxable SS. It deducted 'limited mortgage interest' although only a mortgage balance is given, so mortgage interest is $0 and the $24,150 standard deduction applies alone. Its $4,523 does not follow from that base." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 is right, but it applied TCJA-expiration rules (a $12,389 deduction, $10,733 of personal exemptions, a 15% bracket and a $1,000 CTC). OBBBA made permanent the $24,150 HoH deduction, zero exemptions, the 12% bracket and the $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It correctly included $10,900 of taxable SS, but it used 'sunset' rules (a $12,400 HoH deduction, two $5,350 exemptions, a 15% bracket and a $1,000 CTC). OBBBA replaced these with the $24,150 deduction, zero exemptions, 10%/12% brackets and a $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It capped educator expense at $300 instead of deducting the full $337.50 and understated 85% of $12,824 as $10,890 instead of $10,900.40. It also computed $5,464 of tax on $48,440, where the correct brackets give $5,458.80, which left it at $3,264." +us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $72,562 AGI is right, but it applied post-TCJA-sunset rules (a $12,538 deduction, two $5,431 exemptions, a 15% bracket and a $1,000 CTC) that OBBBA superseded. The 2026 law uses the $24,150 HoH deduction, no exemptions, a 12% bracket and a $2,200 CTC." +us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It capped educator expense at $300 instead of deducting the full $337.50 and understated taxable SS as $10,890 instead of $10,900.40. That produced taxable income of about $48,440 instead of $48,412.90 and a result about $4.45 high." +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no computation to evaluate against the correct $3,255.55." +us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest from AGI, but no qualifying new vehicle is established, so the OBBBA vehicle-loan-interest deduction does not apply. That reduced taxable income to $48,137.40 instead of $48,412.90 and understated tax by $33, giving $3,222.49." +us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It filed as single with a $15,000 deduction and 2025 single brackets, but an unmarried parent with a 12-year-old qualifying child files head of household ($24,150 deduction, $17,700 10% bracket). It also omitted the $10,900.40 of taxable SS and used a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as single with a $16,100 deduction, which pushed income into the 22% bracket, instead of head of household with $24,150. It deducted $275 of auto loan interest without a qualifying vehicle. It also applied only $500 of the CTC as nonrefundable, although the full $2,200 is used against liability before any refundable portion." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed married filing jointly although the Head is an unmarried single parent who files head of household. So it used a $31,200 MFJ deduction and the MFJ $23,850 bracket instead of $24,150 and $17,700, and it subtracted a $2,000 CTC instead of $2,200." +us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the $32,200 MFJ standard deduction to a single parent who files head of household ($24,150). It also invented an 'SS dependent benefits exclusion' that zeroed out the $10,900.40 of taxable SS, driving tax to $0 instead of $3,255.55." us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model computed pre-credit tax of about $4,300 against a $2,200 CTC and then contradicted its own arithmetic by awarding $1,700, writing that the nonrefundable portion ""absorbs tax"" while still paying out the refundable cap. It applied only two of the three ACTC limitations — the $1,700 per-child cap and the 15%-of-earnings-over-$2,500 phase-in — and omitted the controlling leg of IRC 24(d)(1)(B), the amount of CTC blocked by the tax-liability limitation, which is $0 when liability ($5,455.55 here) exceeds the entire $2,200 credit." us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model stated that ""tax liability is largely offset"" by the CTC and then treated the $1,700 refundable ceiling as the amount payable, inverting Schedule 8812: line 16a (total CTC minus the nonrefundable amount actually used) is $0 here, and the $1,700 cap can only limit that residual, never create one. It also used a stale $2,000 CTC rather than the 2026 OBBBA amount of $2,200, but the credit is fully absorbed at either figure since pre-credit tax is $5,455.55." us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model derived the correct answer explicitly — ""the full $2,000 CTC would be absorbed by tax liability, leaving no refundable Additional Child Tax Credit"" — then overrode it with $1,700 by invoking unnamed ""itemized/above-the-line deductions reducing taxable income further."" Reaching any ACTC requires taxable income below roughly $21,280 under the 10%/12% HoH schedule, i.e. about $51,300 of deductions against $72,562.90 of AGI; the actual itemizable items here (mortgage interest on a $55,000 balance, Georgia income tax, and medical costs entirely below the 7.5%-of-AGI floor) fall far short of even the $24,150 standard deduction." @@ -6476,44 +7113,51 @@ us,scenario_086,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"T us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component amounts—$3,844 of Social Security tax and $899 of Medicare tax—but failed to add them correctly. Those components total $4,743, not $3,979, and no Georgia-specific adjustment changes that sum." us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct $3,844 Social Security tax and $899 Medicare tax but submitted $3,813 instead of adding them. The stated components total $4,743." -us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Used a $24,000 head-of-household standard deduction and a $4,000 dependent exemption at 5.19%, where Georgia 2026 allows a $15,000 head-of-household standard deduction, a $5,000 dependent exemption, and a 4.99% flat rate. It then computed $1,749.03 from its own figures and submitted $2,732, a number its reasoning explicitly disowns as 'single-filer treatment' and that no stated calculation produces." -us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Subtracted $275 of auto loan interest from Georgia AGI, which Georgia does not allow as a subtraction, and then applied a $24,000 standard deduction with a $4,000 dependent exemption at 5.09%. Georgia's 2026 figures are a $15,000 head-of-household standard deduction, a $5,000 dependent exemption, and a 4.99% flat rate on Georgia AGI of $61,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied Georgia's pre-2024 graduated 1%–6% bracket schedule, which HB 1437 replaced with the flat tax, directly to the full $62,000 of gross wages with no standard deduction, no dependent exemption, and no Social Security subtraction. The correct computation is Georgia AGI $61,662.50 less $15,000 and $5,000, taxed at the 4.99% flat rate." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Used an $18,500 standard deduction and a $4,000 dependent exemption at 5.19% instead of $15,000, $5,000, and 4.99%, arriving at $2,032.51 from its own arithmetic. It then submitted $2,191, a figure no step of its derivation produces." -us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Took a $12,000 standard deduction and a $4,000 dependent exemption at 5.09%, computed $2,324, then wrote the answer down to $1,869 for unspecified 'personal exemption considerations' — Georgia's flat tax eliminated filer personal exemptions and leaves only the $5,000 dependent exemption alongside the $15,000 head-of-household standard deduction, at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Removed Social Security twice: it built federal AGI as $61,662 with the benefits already excluded, then subtracted the full $12,824 again to reach a $48,838 Georgia base instead of Georgia AGI of $61,662.50. It compounded that with the repealed pre-2022 stack of a $5,400 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.39%, rather than $15,000, $5,000, and 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Applied a $12,000 standard deduction and $4,000 dependent exemption at 5.19% to reach $2,372, then submitted $2,765 as an adjustment 'to nearest state tax tables' — Georgia has a single 4.99% flat rate for 2026 with no tax tables, and the correct subtractions are $15,000 and $5,000." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Never applied Georgia's subtraction for the $10,900.40 of taxable Social Security, taxing a $72,600 base rather than Georgia AGI of $61,662.50, and then used a $9,000 standard deduction with two $2,700 personal exemptions at 5.29%. Georgia's 2026 subtractions are a $15,000 head-of-household standard deduction and a $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Used the repealed pre-flat-tax structure — a $5,400 standard deduction plus two $2,700 personal exemptions — at 5.29%, and started from an unexplained $60,140 AGI instead of Georgia AGI of $61,662.50. HB 1437 replaced personal exemptions with a larger standard deduction: $15,000 for head of household plus a $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Handled the Social Security subtraction correctly but claimed a $5,700 standard deduction with $2,700 filer and $3,000 dependent exemptions ($11,400 total) at 5.09%, versus Georgia's $15,000 standard deduction plus $5,000 dependent exemption ($20,000) at 4.99%. That leaves $50,300 of taxable income where $41,662.50 is correct." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Deducted the $19,648 of employer-sponsored insurance premiums from wages a second time — they are already excluded from the $62,000 W-2 wage figure — and left the $10,900.40 of taxable Social Security in the Georgia base rather than subtracting it, producing a $52,952.40 starting figure instead of $61,662.50. It then used a $15,800 head-of-household 'personal exemption' with a $3,000 dependent exemption at the 2024 rate of 5.39%, rather than $15,000, $5,000, and 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Named no Georgia deduction, exemption, or rate; $2,690 corresponds to taxing roughly $50,000 at 5.39%, i.e. the $62,000 of wages reduced only by a $12,000 standard deduction. The correct base is Georgia AGI of $61,662.50 less a $15,000 standard deduction and a $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Reached the correct Georgia AGI of $61,662 after excluding Social Security, then applied a $12,000 standard deduction and a $3,000 dependent exemption at 5.39%. Georgia's 2026 amounts are a $15,000 head-of-household standard deduction and a $5,000 dependent exemption at 4.99%, so it overstated taxable income by $5,000 and the rate by 40 basis points." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Started from a $52,952 base built by deducting the $19,648 of employer-sponsored insurance premiums from wages a second time while retaining the taxable Social Security amount, instead of Georgia AGI of $61,662.50 after the $10,900.40 subtraction. It then applied 5.29% rather than Georgia's 2026 flat 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Submitted a bare $3,212 with no derivation; that amount corresponds to taxing essentially the whole $61,662.50 of Georgia AGI at roughly 5.2%, with neither the $15,000 head-of-household standard deduction nor the $5,000 dependent exemption subtracted. The correct computation is $41,662.50 × 4.99% = $2,078.96." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Named no deduction, exemption, or rate; $3,047 corresponds to applying a rate near Georgia's 4.99% to the full $61,662.50 of Georgia AGI, omitting the $15,000 standard deduction and $5,000 dependent exemption that bring taxable income to $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $1,432.32 corresponds to a taxable base near $28,800, which is Georgia AGI of $61,662.50 with the full $12,824 of Social Security benefits stripped a second time on top of the $10,900.40 taxable portion already subtracted, then reduced by $20,000 of deductions. Georgia makes one Social Security subtraction of $10,900.40, leaving $41,662.50 taxable at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Correctly subtracted the $10,900.40 of taxable Social Security to reach $61,700, then applied a $12,000 standard deduction and a $3,000 dependent exemption at 5.39%. Georgia's 2026 head-of-household standard deduction is $15,000, the dependent exemption is $5,000, and the flat rate is 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Used a $10,600 head-of-household standard deduction and took no dependent exemption at all, then applied 5.39%. Georgia grants a $15,000 head-of-household standard deduction plus a $5,000 exemption for the 12-year-old dependent at a 4.99% rate, so its $51,062 of taxable income overstates the correct $41,662.50 by $9,400." -us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Reached the correct Georgia AGI of $61,662 but then stacked a $24,000 standard deduction — the married-filing-jointly amount — with roughly $8,400 of personal exemptions that Georgia's flat tax repealed, and applied 5.19%. The correct subtractions are a $15,000 head-of-household standard deduction and one $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Cited no Georgia figures; $1,639 corresponds to taxing about $32,800 at 4.99%, which is Georgia AGI of $61,662.50 cut by a $24,000 married-level standard deduction plus the $5,000 dependent exemption. Georgia's head-of-household standard deduction is $15,000, leaving $41,662.50 taxable and $2,078.96 of tax." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Named no Georgia standard deduction, dependent exemption, or rate and described a 'federal-style' taxable income; $498 corresponds to a base near $10,000 at the 4.99% flat rate, about $31,700 below the correct $41,662.50 base of Georgia AGI $61,662.50 less $15,000 and $5,000." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Applied a $12,000 head-of-household standard deduction with $4,500 of personal/dependent exemptions at 5.19%, giving $45,162 of taxable income. Georgia's 2026 amounts are a $15,000 standard deduction and a $5,000 dependent exemption, giving $41,662.50 at a 4.99% flat rate." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The $1,357.80 corresponds to a taxable base near $27,200 — Georgia AGI of $61,662.50 reduced by roughly $34,400, i.e. a $24,000 married-level standard deduction stacked with repealed personal exemptions on top of the dependent exemption. Georgia allows $15,000 plus a single $5,000 dependent exemption, leaving $41,662.50 taxable at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Applied a $12,000 personal deduction and a $4,000 dependent exemption at an estimated 5.09% rate. Georgia's 2026 figures are a $15,000 head-of-household standard deduction, a $5,000 dependent exemption, and a 4.99% flat rate, so it overstated the base by $4,000 and the rate by 10 basis points." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Subtracted about $25,000 of combined Georgia deductions and exemptions from Georgia AGI of $61,662.50 and applied 5.19%, producing a $36,662 base. The statutory subtractions are a $15,000 head-of-household standard deduction and a $5,000 dependent exemption, leaving $41,662.50 taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Excluded Social Security correctly, then used a $12,000 standard deduction and a $4,000 dependent exemption at a 5.09% 'scheduled' rate, leaving $45,662 taxable. Georgia's 2026 head-of-household standard deduction is $15,000, the dependent exemption is $5,000, and the flat rate is 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Stated no Georgia parameters; $2,184 corresponds to Georgia AGI of $61,662.50 less the $15,000 standard deduction but only a $3,000 dependent exemption, taxed at 4.99%. Georgia's per-dependent exemption is $5,000, and that missing $2,000 of exemption accounts for the entire gap to $2,078.96." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Reached the correct Georgia AGI of $61,662 but used an $18,000 head-of-household standard deduction, claimed no dependent exemption for the 12-year-old, and applied 5.29%. The correct figures are a $15,000 standard deduction, a $5,000 dependent exemption, and a 4.99% flat rate." -us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Subtracted taxable Social Security correctly, then applied a $12,000 standard deduction plus $5,700 of personal and dependent exemptions at 5.39%. Georgia's flat tax replaced filer personal exemptions: the correct subtractions are the $15,000 head-of-household standard deduction and a $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Subtracted the $338 educator expense a second time from a federal AGI that already reflected it, then applied the repealed $5,400 standard deduction with two $2,700 personal exemptions at the 2024 rate of 5.49%. The 2026 computation is Georgia AGI $61,662.50 less a $15,000 standard deduction and a $5,000 dependent exemption, taxed at 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Built roughly the right structure — Georgia AGI less the head-of-household standard deduction and exemptions — but applied 5.29% instead of Georgia's 2026 flat 4.99%. On its own $40,700 base the 30-basis-point rate error adds about $122, more than the whole $71 overstatement, offset only by its base running $962 below the correct $41,662.50." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Applied Georgia's correct 4.99% flat rate but subtracted a $12,000 standard deduction plus two repealed $2,700 personal exemptions ($17,400) instead of the $15,000 head-of-household standard deduction plus the $5,000 dependent exemption ($20,000). The resulting $2,600 of excess taxable income is exactly the $130 overstatement in its answer." -us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Subtracted $275 of nondeductible auto loan interest to reach a $61,387 Georgia AGI, then used an $18,500 standard deduction with a $3,000 dependent exemption at 5.19%. Georgia AGI is $61,662.50, the head-of-household standard deduction is $15,000, the dependent exemption is $5,000, and the rate is 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Ignored Georgia's post-reform deduction schedule, subtracting only a $5,000 standard deduction and a $2,700 personal exemption from wages with no dependent exemption, and applied 5.19%. Georgia's 2026 head-of-household standard deduction is $15,000 with a $5,000 dependent exemption at 4.99%, so its $54,000 base overstates the correct $41,662.50 by more than $12,000." -us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its deduction stack — a $12,000 standard deduction, a $3,700 personal exemption, a $4,000 dependent exemption, and a $275 auto loan interest subtraction — lands taxable income at $41,687, within $25 of the correct $41,662.50, so the entire $43 error is the rate. It used 5.09% where Georgia's 2026 flat rate is 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Kept the full $12,824 of Social Security benefits in the Georgia base for a $74,824 starting figure, when Georgia subtracts the entire $10,900.40 taxable amount to give Georgia AGI of $61,662.50. It compounded that with a $12,000 standard deduction, a $2,700 personal exemption, and a $3,000 dependent exemption at 5.12%, versus $15,000, $5,000, and 4.99%." -us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Taxed $74,824 — gross wages plus the full Social Security benefits — while asserting the benefits were excluded, and subtracted a single $23,100 federal-style standard deduction instead of Georgia's $15,000 head-of-household standard deduction and $5,000 dependent exemption. Georgia AGI is $61,662.50, taxable income is $41,662.50, and the flat rate is 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used a $24,000 head-of-household standard deduction (the joint-filer amount) instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $1,749. It then dropped that result and submitted $2,732 based on an unexplained 'single-filer' treatment that matches none of its own arithmetic." +us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the $10,900.40 of taxable Social Security, but then also took $275 of auto-loan interest out of Georgia AGI; that is a below-the-line federal deduction and never enters AGI. It also used a $24,000 standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%, so it taxed $33,387 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied Georgia's repealed six-bracket schedule (1%–6%) to the full $62,000 of wages with no standard deduction and no dependent exemption. Georgia has had a flat tax since 2024; for 2026 the correct base is $61,662.50 − $15,000 − $5,000 = $41,662.50, taxed at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an $18,500 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, which gave $2,032.51. It then submitted $2,191 without any derivation." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the head as a single filer with a $12,000 standard deduction instead of the $15,000 head-of-household amount, used a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of 4.99%, which gave $2,324. It then reported an unsupported $1,869." +us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"From about $61,700 of Georgia AGI it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $4,000 dependent exemption instead of $5,000, then applied 5.09% instead of the 2026 rate of 4.99%. That taxed $45,700 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It subtracted the full $12,824 of Social Security from an AGI of $61,662 that already excluded those benefits, counting the exclusion twice. It then used the repealed pre-2024 structure ($5,400 standard deduction, $2,700 personal exemption, $3,000 dependent exemption) and a 5.39% rate instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and 5.19% instead of 4.99%, which gave $2,372. It then inflated that to $2,765 with an unexplained 'standard deduction phase' adjustment that Georgia law does not have." +us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.09%. That uses the $12,000 single standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and 5.09% instead of the 2026 rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $72,600 and never subtracted the $10,900.40 of taxable Social Security, which Georgia fully exempts. It also used a $9,000 standard deduction plus two repealed $2,700 personal exemptions instead of the $15,000 head-of-household deduction and $5,000 dependent exemption, and a 5.29% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the repealed pre-2024 structure (a $5,400 standard deduction plus $2,700 personal exemptions for both head and child) and a 5.29% rate. The 2026 law gives a $15,000 head-of-household deduction and a $5,000 dependent exemption at 4.99%. It also reduced AGI to $60,140 by subtracting $1,560 of health insurance premiums, which are not an above-the-line deduction for a wage earner." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the repealed pre-2024 structure: a $5,700 standard deduction, a $2,700 filer exemption, and a $3,000 child exemption, which left $50,300 taxable. The correct 2026 amounts are a $15,000 head-of-household deduction and a single $5,000 dependent exemption. It also used 5.09% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the correct 4.99% rate but started from $62,000 of wages without the educator-expense adjustment. It then took a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus $6,700 of exemptions that include a taxpayer exemption Georgia no longer allows, instead of the single $5,000 dependent exemption. The result was $43,300 taxable instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $52,952.40 AGI subtracts the $19,648 of employer-sponsored insurance premiums from wages, which are not excluded from the listed gross wages here. It also failed to subtract the $10,900.40 of taxable Social Security for Georgia. On top of that it used a $15,800 head-of-household exemption, a $3,000 dependent exemption, and the 5.39% rate instead of $15,000, $5,000, and 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $2,690 matches taxing about $50,000 (the $62,000 of wages less only a roughly $12,000 standard deduction) at an old rate of about 5.39%. It omitted the $5,000 dependent exemption and used neither the $15,000 head-of-household deduction nor the 2026 rate of 4.99%. The correct result is ($61,662.50 − $20,000) × 4.99% = $2,078.96." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly reached Georgia AGI of $61,662, but then subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It also applied the old 5.39% rate instead of 4.99%, taxing $46,662 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a $52,952 AGI that wrongly subtracts the $19,648 of employer-sponsored insurance premiums from wages and still includes the $10,900 of taxable Social Security that Georgia exempts. It then took only $15,000 of combined deductions and exemptions instead of $20,000, and applied 5.29% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 is about $1,133 too high. It matches taxing federal AGI of about $72,562 less only a $12,000 deduction at roughly 5.29%, so the model kept the $10,900.40 of taxable Social Security that Georgia exempts. It also did not apply the $15,000 head-of-household deduction, the $5,000 dependent exemption, or the 4.99% rate." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $3,047 matches taxing about $56,500 at roughly 5.39%, which is federal AGI less about $16,000 of deductions. That means it left the $10,900.40 of taxable Social Security in Georgia income instead of subtracting it. The correct base is $41,662.50, taxed at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,432.32 equals tax on about $28,700 at 4.99%, roughly $13,000 less than the correct $41,662.50. So it subtracted about $33,000 of deductions and exemptions (the size of a $24,000 joint-level standard deduction plus exemptions) instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption." +us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"From Georgia AGI of about $61,700 it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It then applied the old 5.39% rate instead of 4.99%, taxing $46,700 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly reached Georgia AGI of $61,662, but then used a $10,600 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely. It also applied 5.39% instead of 4.99%, taxing $51,062." +us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"From the correct Georgia AGI of $61,662 it subtracted a $24,000 standard deduction (the joint-filer amount) instead of $15,000, plus about $8,400 of personal exemptions instead of the single $5,000 dependent exemption. That cut taxable income to $29,262, and it applied 5.19% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $1,639 equals tax on only about $32,800 at 4.99%, so it subtracted roughly $29,000 of deductions and personal exemptions from Georgia AGI. Georgia allows only $20,000 here: the $15,000 head-of-household deduction and the $5,000 dependent exemption, which leaves $41,662.50 taxable." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It computed Georgia tax from federal-style taxable income with federal standard-deduction and medical adjustments, so its $498 implies only about $10,000 of taxable income. Georgia instead subtracts just the $15,000 head-of-household deduction and the $5,000 dependent exemption from $61,662.50 of Georgia AGI, and taxes the $41,662.50 at 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction instead of the $15,000 head-of-household amount, $4,500 of personal and dependent exemptions instead of the $5,000 dependent exemption, and 5.19% instead of the 2026 rate of 4.99%. It taxed $45,162 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It correctly removed taxable Social Security, but its $1,357.80 equals tax on only about $27,200 at 4.99%. That means it subtracted roughly $34,500 of deductions and exemptions instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption, which leave $41,662.50 taxable." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a $12,000 deduction instead of the $15,000 head-of-household standard deduction, a $4,000 dependent exemption instead of $5,000, and 5.09% instead of 4.99%. It taxed $45,662.40 instead of $41,662.50, and it also misstated the product ($45,662.40 × 5.09% = $2,324.22, not $2,323.22)." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied 5.19% instead of the 2026 rate of 4.99%, and its $1,903 implies about $36,670 of taxable income. That means it subtracted about $25,000 of deductions and exemptions instead of $20,000 (the $15,000 head-of-household deduction plus the $5,000 dependent exemption)." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.09%. That uses a $12,000 standard deduction instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It had the correct 4.99% rate but claimed a $4,000 taxpayer exemption that Georgia repealed. It also used a $12,000 standard deduction instead of the $15,000 head-of-household amount, a $3,000 dependent exemption instead of $5,000, and capped the educator expense at $300, so it taxed $42,700 instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It computed ($61,662 − $12,000 − $4,000) × 5.19%. That uses a $12,000 deduction instead of the $15,000 head-of-household standard deduction, a $4,000 dependent exemption instead of $5,000, and 5.19% instead of the 2026 rate of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"From the correct $61,662 of Georgia AGI it subtracted a $12,000 deduction instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000. It then applied the scheduled 5.09% rate instead of the 4.99% rate in effect for 2026." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The correct computation is ($61,662.50 − $15,000 − $5,000) × 4.99% = $2,078.96. The model's $2,184 overstates tax by $105, which matches subtracting only about $17,900 of deductions and exemptions or applying a rate above 4.99%. It did not use the $15,000 head-of-household deduction, the $5,000 dependent exemption, and the 4.99% rate together." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It subtracted an $18,000 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely. It then applied 5.29% instead of 4.99%, taxing $43,662." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction instead of the $15,000 head-of-household amount, plus $5,700 of combined personal ($2,700) and dependent ($3,000) exemptions. Georgia allows only a $5,000 dependent exemption and no taxpayer exemption. It also applied 5.39% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"From Georgia AGI of $61,700 it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and a $3,000 dependent exemption instead of $5,000. It then applied 5.19% instead of 4.99%, taxing $46,700 rather than $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted the educator expense a second time even though federal AGI already includes that adjustment. It then applied the repealed pre-2024 structure ($5,400 standard deduction plus two $2,700 personal exemptions) and the 2024 rate of 5.49%, instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its taxable base of about $40,700 is close to the correct $41,662.50, but it applied 5.29% instead of Georgia's 2026 flat rate of 4.99%. That rate error inflates the tax by more than $120." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It had the correct 4.99% rate and $61,662 of Georgia AGI, but subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount. It also claimed two $2,700 personal exemptions ($5,400) instead of the single $5,000 dependent exemption, since Georgia no longer allows a taxpayer exemption, so it taxed $44,262 instead of $41,662.50." +us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reduced Georgia AGI by $275 of auto-loan interest, which is a below-the-line federal deduction and not part of AGI. It also used an $18,500 head-of-household standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and 5.19% instead of 4.99%, and those errors happened to offset to $2,070.14." +us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It subtracted only a $5,000 standard deduction and a $2,700 personal exemption from wages, instead of the $15,000 head-of-household deduction and the $5,000 dependent exemption, which left $54,000 taxable instead of $41,662.50. It also applied 5.19% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It cut Georgia AGI to $61,387 by subtracting $275 of auto-loan interest, which is not an AGI adjustment. It then used a $12,000 single standard deduction instead of the $15,000 head-of-household amount, a repealed $3,700 personal exemption, and a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It included all $12,824 of Social Security benefits in Georgia income, even though Georgia fully exempts Social Security (the $10,900.40 taxable portion is subtracted from federal AGI). It then used a $12,000 standard deduction, a repealed $2,700 personal exemption, a $3,000 dependent exemption, and a 5.12% rate instead of the $15,000 head-of-household deduction, the $5,000 dependent exemption, and 4.99%." +us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said Social Security was excluded but started from gross income of $74,824, which includes the full $12,824 of benefits. It then subtracted a $23,100 standard deduction instead of the $15,000 head-of-household deduction plus the $5,000 dependent exemption. Its implied rate of about 4.57% on $51,700 also does not match Georgia's 4.99% flat rate." us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits." us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit." us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract." @@ -6542,68 +7186,74 @@ us,scenario_088,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It removed us,scenario_088,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It asserted the head had ""no earnings subject to employee-side payroll tax"" and returned $0, discarding the $18,577 of gross wages and salaries that constitute the FICA base. Employee Social Security at 6.2% ($1,151.78) and Medicare at 1.45% ($269.37) apply to those wages regardless of Texas having no state employee payroll tax, which is the only genuinely zero component in this household." us,scenario_088,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It set up the correct computation — 0.062 × 18,577 + 0.0145 × 18,577, no wage-base cap, no Additional Medicare Tax — but never executed it, reporting ""≈1270"" when 7.65% of $18,577 is $1,421.14. Its $1,270 is consistent with applying the combined 7.65% rate to roughly $16,600 of wages, an arithmetic slip of $151 rather than a rule error." us,scenario_088,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value and no explanation were returned, so nothing was submitted against the required key. The correct derivation is 0.062 × $18,577 = $1,151.78 plus 0.0145 × $18,577 = $269.37 = $1,421.14; the model produced no answer to evaluate." -us,scenario_088,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model performed the disabled-household net-income test and an uncapped excess-shelter deduction but never applied the SNAP countable-resource test to the head’s $31,776 bank balance. Those liquid assets exceed the applicable Texas SNAP resource limit, so the household is ineligible and the computed $159 monthly allotment never arises." -us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model calculated a benefit from income and shelter deductions while omitting the $31,776 bank account from SNAP eligibility. That countable resource balance exceeds the applicable limit for a household with a disabled member, ending eligibility before its estimated $100 monthly benefit calculation; it also incorrectly applied a shelter-deduction cap despite the disabled member." -us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"The model treated high mortgage interest and property taxes as reducing net income to zero and then awarded the maximum allotment, without testing the $31,776 bank balance against the SNAP resource limit. The excess countable resources make the household ineligible regardless of the shelter deduction." -us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model moved directly from the disabled-member net-income pathway and large shelter deduction to the maximum one-person allotment, omitting the SNAP asset test. The head’s $31,776 in bank-account assets exceeds the applicable countable-resource limit, so no allotment is payable." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its chain was correct through taxable income of $64,432 — exactly the reference's $60,964.27 plus the $3,467.43 qualified overtime deduction it never applied, a 2026 below-the-line deduction stacked on top of the $32,200 standard deduction. It then discarded its own $7,236 bracket result and submitted $8,225 under an unexplained 'wage-based 401k treatment nuances' adjustment." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It omitted the $3,467.43 qualified overtime deduction entirely and instead invented a ~$1,200 self-employed health insurance deduction, which §162(l) denies because the spouse is covered by an employer-sponsored plan. It also sized §199A at ~$9,330 rather than $9,568.44 (20% of $47,842.18), leaving taxable income of $63,722 against the correct $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It fabricated ~$57,893 of head wages by inferring a wage rate from the head's 45 usual weekly hours; no wage or gross wage is listed for the head, whose employment income is $0. It then used a $29,200 standard deduction instead of $32,200 and applied neither the $9,568.44 §199A deduction nor the $3,467.43 overtime deduction, taxing $127,857 instead of $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of $106,200 but used a $31,500 standard deduction instead of $32,200, sized §199A at ~$9,200 instead of $9,568.44, and never applied the $3,467.43 overtime deduction. It then abandoned its own $7,377 bracket computation and submitted $9,089 'after adjustments' with no stated basis." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It added the $4,286 self-employment tax to the answer, but this output is federal income tax after nonrefundable credits and excludes SE tax. It also took no §199A deduction on the $47,842.18 of qualified business income, no $3,467.43 overtime deduction, and a $31,500 standard deduction, so its pre-SE-tax figure of $8,468 was already $1,648 high." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its own chain produced roughly $7,150 on a near-correct AGI, standard deduction, and §199A estimate, and it then discarded that to submit $12,850 by taxing self-employment and partnership income 'at higher effective rates' — folding self-employment tax into an output that excludes it. It also never applied the $3,467.43 qualified overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) although the head has no wages to defer from, understating AGI at $100,797 versus the correct $106,200.13, and it used a $30,800 standard deduction rather than $32,200. It also omitted the $3,467.43 qualified overtime deduction; the offsetting errors left it $59 low." -us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It ran the $19,650 partnership/S-corp income through Schedule SE as self-employment earnings, inflating the half-SE-tax deduction to $3,532 and mis-stating income, then took no §199A deduction at all, used a $30,000 standard deduction, and omitted the $3,467.43 overtime deduction. It finally inflated its own $8,054 result to $8,300 for unspecified 'minor deduction adjustments'." -us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It stripped the $7,789 employer-sponsored insurance premium out of wages and added a $2,400 self-employed health insurance deduction, neither of which reduces income here — AGI is $106,200.13. It then applied no §199A deduction on the $47,842.18 of qualified business income, no $3,467.43 overtime deduction, and a $30,000 standard deduction instead of $32,200." -us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $29,200 standard deduction — a 2024 MFJ amount, versus 2026's $32,200 — and computed §199A as 20% of the full $49,985 rather than of $47,842.18, which is business income net of the $2,143.12 half-SE-tax deduction. It also omitted the $3,467.43 qualified overtime deduction and applied 2025 brackets instead of the 2026 schedule." -us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied 'post-TCJA-expiration' 2026 law with personal exemptions; OBBBA made the $32,200 standard deduction and the 10%/12% schedule the operative 2026 law and personal exemptions remain repealed. It also excluded the $7,789 employer health premium from wages and took neither the $9,568.44 §199A deduction nor the $3,467.43 overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA sunset for 2026 — a $16,000 standard deduction, $10,100 of personal exemptions, and a 15% second bracket — when 2026 carries a $32,200 MFJ standard deduction, no personal exemptions, and a 12% bracket to $100,800. It compounded this by subtracting the $7,789 employer premium from wages and taking no §199A or overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It substituted ~$29,000 of net self-employment earnings for the $30,335.29 of self-employment income and applied neither the $9,568.44 §199A deduction nor the $3,467.43 overtime deduction. Its $11,090 corresponds to taxing roughly $96,500 under the 10%/12% schedule against the correct taxable income of $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a pre-TCJA sunset schedule for 2026 — a $16,600 standard deduction plus $10,600 of personal exemptions and pre-TCJA rates — when 2026 law gives a $32,200 standard deduction, no exemptions, and 10%/12% brackets. It also cut wages by the $7,789 employer premium and took neither the $9,568.44 §199A deduction nor the $3,467.43 overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction with $10,100 of personal exemptions and a 15% bracket beginning at $18,650 — a repealed pre-TCJA schedule, not 2026's $32,200 standard deduction and 10%/12% brackets with the 12% rate starting at $24,800. It also removed the $7,789 employer premium from wages and applied no §199A or overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, and its $15,622 is 2.3 times the correct $6,819.71 — a figure reachable only by taxing well over $100,000 at rates above the 12% bracket that governs this return. The correct chain applies the $32,200 standard deduction, the $9,568.44 §199A deduction, and the $3,467.43 overtime deduction to AGI of $106,200.13, none of which its number reflects." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded the $7,789 employer-sponsored insurance premium from wages, putting AGI at $98,410.90 instead of $106,200.13, and used a $30,800 standard deduction rather than $32,200. It also omitted the $3,467.43 qualified overtime deduction; only its $9,568.38 §199A figure matched the reference." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its stated AGI of $98,663 is the correct $106,200.13 less the $7,789 employer health premium it wrongly excluded from wages and less the head's $252 spousal traditional IRA it dropped. It also applied no $3,467.43 qualified overtime deduction, so its taxable income never reached the reference's $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $7,612 implies taxable income near $67,570 under the 2026 MFJ brackets — about $6,600 above the correct $60,964.27. That gap is the $3,467.43 qualified overtime deduction it omitted plus an understated standard deduction and §199A deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It nailed AGI at $106,199.91 and §199A at $9,568.38 but omitted the $3,467.43 qualified overtime deduction, which alone costs $416 of tax. It also used a $30,600 standard deduction instead of $32,200 and ended the 10% bracket at $24,300 rather than 2026's $24,800." -us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It zeroed the head's $252 traditional IRA deduction on the theory that the head has no earned income; a spousal IRA under §219(c) is deductible against the couple's joint compensation on a joint return, and PolicyEngine allows both IRAs in the $577 of above-the-line deductions. It further used a $31,500 standard deduction instead of $32,200, a $9,533 §199A figure instead of $9,568.44, and no $3,467.43 overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It declined to compute anything and submitted 0, asserting that filing status, deductions, and credits were not established. The facts fully determine the return: joint filing for the married couple, $108,920.27 of gross income, $2,720.14 of above-the-line deductions, and $45,235.87 of deductions from AGI yield $60,964.27 taxable and $6,819.71 of tax." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no derivation, and its $6,828 sits $8.29 above the correct $6,819.71 — precisely the effect of ending the 10% bracket near $24,400 instead of the 2026 MFJ $24,800 while applying the correct $60,964.27 of taxable income. Its deduction chain matched; the rate schedule's first bracket boundary did not." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It caught both the $32,200 standard deduction and the $3,467.43 qualified overtime deduction, then substituted a $7,789 pre-tax employer-premium exclusion for the $9,568.44 §199A deduction and dropped the head's $252 spousal IRA deduction. Those two substitutions put taxable income at $62,996 rather than $60,964.27, overstating the tax by $244." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the traditional 401(k), half-SE-tax, standard deduction and §199A steps but never the $3,467.43 qualified overtime deduction, and it understated the 2026 MFJ standard deduction below $32,200. Its $7,530 implies taxable income near $66,880 against the correct $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It had the full structure right — $32,200 standard deduction, $3,467.43 overtime deduction, §199A — but additionally excluded the $7,789 employee share of employer-sponsored insurance premiums as 'eligible pretax insurance,' which does not reduce the $58,934.98 of employment income already net of the 401(k) deferral. That single extra $7,789 deduction at 12% is $934.65, exactly its shortfall against $6,819.71." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $12,450 corresponds to taxing roughly $105,000 — essentially the entire $106,200.13 AGI — under the 2026 MFJ brackets. It applied none of the $32,200 standard deduction, $9,568.44 §199A deduction, or $3,467.43 overtime deduction that reduce taxable income to $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It reached the correct AGI of $106,200 and then applied a TCJA-sunset schedule — a $16,637 standard deduction, $10,611 of personal exemptions, and a 15% second bracket — when 2026 carries a $32,200 standard deduction, no personal exemptions, and a 12% bracket to $100,800. It also took neither the $9,568.44 §199A deduction nor the $3,467.43 overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It reached the correct AGI of $106,200 and then applied a repealed pre-TCJA schedule — a $16,600 standard deduction, $10,600 of personal exemptions, and 15% above roughly $25,000 — instead of 2026's $32,200 standard deduction and 10%/12% brackets. It also omitted both the $9,568.44 §199A deduction and the $3,467.43 qualified overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $3,467 FLSA overtime premium to wages as extra income, reaching $119,333 of gross income; the $65,881 of gross wages already includes overtime pay, and for 2026 that premium generates a $3,467.43 deduction rather than additional income. It also deducted the head's $5,403 401(k) against zero wages, used a $30,660 standard deduction, and took no §199A deduction on the $47,842.18 of qualified business income." -us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It had AGI at $106,200 and the $32,200 standard deduction right but computed §199A as 20% of the full $49,985, producing $9,997 instead of the $9,568.44 that comes from the $47,842.18 of business income net of the $2,143.12 half-SE-tax deduction. Its dominant error is omitting the $3,467.43 qualified overtime deduction, leaving taxable income of $64,003 against the correct $60,964.27." -us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It added the $3,467 FLSA overtime premium on top of the $65,881 of gross wages, which already include overtime pay and which entitle the couple to a $3,467.43 deduction rather than extra income, and it deducted the head's $5,403 traditional 401(k) against zero wages. It further used a $29,900 standard deduction instead of $32,200 and took no §199A deduction on the $47,842.18 of qualified business income." -us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Every income and deduction step matched — its taxable income of $60,964.52 sits within a quarter of the reference's $60,964.27, including the $3,467.43 overtime deduction, $32,200 standard deduction, and $9,568.38 §199A deduction. It ended the 10% bracket at $24,400 instead of the 2026 MFJ $24,800, taxing $400 at 12% rather than 10% and overstating the tax by exactly $8.03." -us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It ran the $19,650 partnership/S-corp income through Schedule SE to get a $3,531 half-SE-tax deduction rather than the $2,143.12 owed on self-employment income alone, and it deducted the head's $5,403 traditional 401(k) with no wages behind it. It also used a $30,000 standard deduction, took §199A on the full $49,985, and omitted the $3,467.43 overtime deduction." -us,scenario_089,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It allowed the head's $5,403 traditional 401(k) deferral against zero wages, putting AGI at $100,797 instead of $106,200.13, and computed §199A as 20% of $49,985 rather than of the half-SE-tax-reduced $47,842.18. It also omitted the $3,467.43 qualified overtime deduction, and the offsetting errors left taxable income $2,364 low." -us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) although the head has no wages, took no §199A deduction on the $47,842.18 of qualified business income, and omitted the $3,467.43 qualified overtime deduction. It then applied a 12% bracket beginning at $23,200 with a $2,783.50 base — a pre-2026 schedule, when 2026 MFJ runs 10% to $24,800." -us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It fabricated $132,619 of head wages from the spouse's $27 hourly rate and the head's 45 weekly hours; no wage is listed for the head, whose employment income is $0, so AGI is $106,200.13, not $248,485. It then offset that inflated tax with an $8,000 nonrefundable child tax credit and a $400 saver's credit although the household has no children and no such credits apply." +us,scenario_088,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It waived the gross income test because of the disability, then built an uncapped excess shelter deduction from mortgage interest and property taxes, which gave a net income of about $445 per month and a benefit of $159 per month using a $292 maximum. It never applied the SNAP resource test. The $31,776 bank balance exceeds the $4,500 limit for elderly or disabled households and Texas's $5,000 categorical-eligibility limit, so the household is ineligible and SNAP is $0." +us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"It skipped the gross income test because of the disability and then estimated a small allotment of about $100 per month from a capped shelter deduction. It left out the SNAP asset test entirely. The head's $31,776 in bank assets exceeds the $4,500 resource limit for elderly or disabled households (and Texas's $5,000 limit), so the household is ineligible and SNAP is $0." +us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"It reasoned that the disability plus $20,814 in mortgage interest and property taxes drove net income to zero, and it awarded the full one-person maximum of $3,492 ($291 x 12). It never checked countable resources. The $31,776 bank balance breaks the $4,500 elderly or disabled asset limit and Texas's $5,000 limit, so the household is ineligible and SNAP is $0." +us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"It used the disability to move straight to the net-income test and let the homeowner shelter costs zero out net income, which produced the maximum one-person allotment of $298 x 12 = $3,576. It never applied the SNAP resource test. The $31,776 bank balance exceeds the $4,500 limit for elderly or disabled households and Texas's $5,000 limit, so the household is ineligible and SNAP is $0." +us,scenario_088,snap,gpt-6-sol,llm_error,asset_resource,False,"It applied the uncapped shelter deduction for disabled households, zeroed out net income, and paid the $298 monthly maximum, for $3,576 a year. It ignored the SNAP resource test. The head's $31,776 in bank assets exceeds the $4,500 elderly or disabled asset limit and Texas's $5,000 limit, so the household is ineligible and SNAP is $0." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly derived AGI $106,200, the $32,200 standard deduction, the $9,568 QBI deduction and 2026 brackets for $7,236. It never applied the $3,467.43 qualified overtime deduction for the spouse's FLSA overtime premium, which lowers taxable income to $60,964 and tax to $6,820. It then discarded its own $7,236 for an unexplained $8,225." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It omitted the $3,467.43 qualified overtime deduction. It also took a ~$1,200 self-employed health insurance deduction, which is barred because both spouses have employer-sponsored coverage, and dropped the head's $252 spousal IRA deduction. That left taxable income at $63,722 instead of $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented ~$57,893 of head wages from weekly hours, although the head has no listed wages (unlisted = 0). It used the stale $29,200 standard deduction and never applied the $9,568 QBI or $3,467 overtime deductions, so its taxable income was ~$127,857 versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It got AGI $106,200 right but used the 2025 $31,500 standard deduction instead of 2026's $32,200 and a $24,150 bracket edge instead of $24,800. It omitted the $3,467 qualified overtime deduction, then inflated its own $7,377 result to $9,089 with unexplained 'adjustments'." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted both the $9,568 QBI deduction and the $3,467 overtime deduction, and used a $31,500 standard deduction, giving taxable income of $74,700. It then added the $4,286 SE tax and other amounts to income tax, although SE tax is a separate liability excluded from this output." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its bracket computation (~$7,150) left out the $3,467 qualified overtime deduction. It then abandoned that figure for $12,850 by asserting 'higher effective rates' on self-employment and partnership income, although that income is taxed at the same 10%/12% ordinary brackets." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) even though the head has no wages for an elective deferral to reduce, which understated AGI at $100,797 versus $106,200. It also used a $30,800 standard deduction and inflated 2025 brackets instead of the 2026 $32,200 and $24,800, and omitted the $3,467 overtime deduction. These errors roughly offset to $6,761." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $19,650 partnership/S-corp income as SE earnings (half SE tax $3,532 instead of $2,143) and disallowed the head's $252 spousal IRA. It used a $30,000 standard deduction and never applied the $9,568 QBI or $3,467 overtime deductions, then rounded its $8,054 up to $8,300 arbitrarily." +us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Every step matched the reference (AGI $106,200, $32,200 standard deduction, $9,568 QBI, 2026 brackets) except that it omitted the $3,467.43 qualified overtime deduction for the spouse's FLSA overtime premium. That left taxable income $3,467 too high and tax $416 too high ($7,236 vs $6,820)." +us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $3,467 FLSA overtime premium on top of gross wages that already include it, excluded the $7,789 ESI premiums from wages, and claimed a $2,400 SE health insurance deduction that employer coverage bars. It then used the 2025 $30,000 standard deduction and 2025 brackets, and omitted both the $9,568 QBI and $3,467 overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It misstated spouse wages net of 401(k) as $57,735 instead of $58,935 and used the 2024 $29,200 standard deduction instead of 2026's $32,200. It computed QBI on the full $49,985 without subtracting the $2,143 half-SE-tax deduction, and omitted the $3,467 qualified overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a post-TCJA-sunset regime (personal exemptions, pre-TCJA standard deduction and brackets), although the OBBBA made the TCJA structure permanent with a $32,200 MFJ standard deduction and 10%/12% brackets for 2026. It also excluded the $7,789 ESI premiums from wages (AGI $98,411 vs $106,200) and omitted the QBI and overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed AGI $106,199.90 correctly but omitted both the $9,568 Section 199A QBI deduction and the $3,467 qualified overtime deduction. It used the $31,500 standard deduction instead of $32,200, leaving taxable income at $74,700 versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 ($16,000 standard deduction, $10,100 personal exemptions, 15% bracket), but the OBBBA kept the $32,200 standard deduction and 10%/12% brackets. It also subtracted the $7,789 ESI premiums from wages and omitted the $9,568 QBI and $3,467 overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its reasoning subtracts only the standard deduction and retirement contributions and never applies the $9,568 QBI deduction or the $3,467 overtime deduction. Its $11,090 corresponds to taxable income near $96,500 under 2026 brackets, about $35,500 above the correct $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a TCJA sunset ($16,600 standard deduction plus $10,600 personal exemptions, pre-TCJA brackets), whereas 2026 law under the OBBBA uses a $32,200 standard deduction, no exemptions and 10%/12% brackets. It also excluded the $7,789 ESI premiums from wages and omitted the QBI and overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a TCJA-sunset structure ($15,300 standard deduction, $10,100 exemptions, 15% bracket above $18,650) instead of the permanent OBBBA $32,200 standard deduction and 10%/12% brackets. It also subtracted the $7,789 ESI premiums from wages and applied neither the $9,568 QBI nor the $3,467 overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $15,622 exceeds the ~$14,900 that 2026 brackets impose on the household's entire $115,866 of gross receipts with zero deductions, so it both skipped the $32,200 standard deduction, $9,568 QBI deduction and $3,467 overtime deduction and imputed extra income or bundled SE tax. The correct derivation yields $60,964 taxable and $6,819.71 tax." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded the spouse's $7,789 ESI premiums from wages, understating AGI at $98,410.90 versus $106,200.13. It then used a $30,800 standard deduction and the 2024 $23,200 bracket edge instead of $32,200 and $24,800, and omitted the $3,467 qualified overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its ~$98,663 AGI reflects subtracting the $7,789 ESI premiums from the spouse's wages and dropping the head's $252 spousal IRA; the correct AGI is $106,200. It also never applied the $3,467 qualified overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its reasoning cites only a standard deduction, omitting the $9,568 QBI deduction and the $3,467 qualified overtime deduction. Its $7,612 corresponds to about $67,600 of taxable income under 2026 brackets versus the correct $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"AGI ($106,199.91) and QBI ($9,568.38) were right. It used a $30,600 standard deduction and a $24,300 10% bracket edge instead of 2026's $32,200 and $24,800, and omitted the $3,467 qualified overtime deduction, giving taxable income of $66,032 versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $3,467 qualified overtime deduction and wrongly denied the head's $252 spousal IRA, which is deductible on joint compensation under IRC 219(c). It also misstated AGI as ~$104,131 and used the $31,500 standard deduction and a $24,600 bracket edge instead of $32,200 and $24,800." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It declared the liability indeterminable and submitted $0, even though the facts fully specify $65,881 wages, $30,335 SE income and $19,650 partnership income for a joint return. Those produce $60,964 of taxable income and $6,819.71 of tax." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its explanation subtracts only standard deductions and retirement contributions and names neither the $9,568.44 QBI deduction nor the $3,467.43 overtime deduction. Its $6,828 is $8.29 above the exact $6,819.71 (about $69 of excess taxable income), the result of approximation rather than the exact deduction stack." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the overtime deduction but subtracted the $7,789 ESI premiums as 'pretax health' deductions, dropped the head's $252 spousal IRA, and omitted the $9,568 QBI deduction entirely. That yields taxable income of $62,996 versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its deduction list (retirement, half SE tax, standard deduction, QBI) omits the $3,467 qualified overtime deduction for the spouse's FLSA overtime premium. Its $7,530 corresponds to about $66,900 of taxable income versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $7,789 employer-sponsored insurance premiums as a pre-tax Section 125 wage exclusion, but nothing marks them as salary reductions; the head carries identical premiums with no wages. That understated AGI at $98,410.90 instead of $106,200.13. With the correct $32,200, $3,467 and $9,568 deductions, this single error cut tax by $934.65." +us,scenario_089,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the spouse's $7,789 employer-sponsored insurance premiums as 'eligible pretax wage deductions', lowering AGI to $98,410.90. Employment income is only net of the $6,946 traditional 401(k), so AGI is $106,200.13. Its standard, overtime and QBI deductions were all correct, so this one exclusion accounts for the entire $934.65 shortfall." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated combined income as ~$120,000, above the $108,920 gross income after the spouse's 401(k). It applied only a standard deduction, omitting the $9,568 QBI and $3,467 overtime deductions, and overstated tax by ~$5,600." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"AGI $106,200 was right, but it applied a TCJA-sunset regime ($16,637 standard deduction, $10,611 exemptions, 15% bracket) instead of the OBBBA-permanent $32,200 standard deduction and 10%/12% brackets. It also omitted the $9,568 QBI and $3,467 overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It computed AGI $106,200 correctly but assumed the TCJA expired ($16,600 standard deduction plus $10,600 exemptions, 15% bracket), whereas 2026 law uses a $32,200 standard deduction with 10%/12% brackets. It also skipped the $9,568 QBI and $3,467 overtime deductions." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly reached AGI $106,200 but applied pre-TCJA 2026 rules ($16,600 standard deduction, $10,600 personal exemptions, 15% bracket) that the OBBBA repealed. It omitted the $9,568 QBI and $3,467 overtime deductions, yielding $79,000 taxable versus $60,964." +us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $3,467 overtime premium on top of wages that already include it and subtracted the head's $5,403 401(k) with no head wages. It claimed a $600 SE health insurance deduction that employer coverage bars, used a $30,660 standard deduction, and applied neither the QBI nor the overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"AGI and the $32,200 standard deduction were right. It computed QBI as 20% of $49,985 without reducing it by the $2,143 deductible half of SE tax (correct QBI is $47,842.18), omitted the $3,467 qualified overtime deduction, and used a $24,500 bracket edge instead of $24,800." +us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It double-counted the $3,467 overtime premium on top of gross wages and subtracted the head's $5,403 401(k) though the head has no wages. It used a $29,900 standard deduction and 2025 brackets, and applied neither the $9,568 QBI nor the $3,467 overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reached essentially the correct taxable income ($60,964.52) but used $24,400 as the top of the 2026 MFJ 10% bracket instead of $24,800. That moved $400 from 10% to 12% and overstated tax by exactly $8.00." +us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $5,403 401(k) despite no head wages and treated the partnership income as SE earnings (half SE tax $3,531 vs $2,143). It used QBI of 20% of the unreduced $49,985, a $30,000 standard deduction and 2025 brackets, and omitted the $3,467 overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted the head's $5,403 traditional 401(k) though the head has no wages, understating AGI at $100,797 versus $106,200. It also computed QBI on $49,985 without subtracting the half-SE-tax deduction ($9,997 vs $9,568) and omitted the $3,467 qualified overtime deduction." +us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $5,403 401(k) with no head wages, used a $32,300 standard deduction, and omitted both the $9,568 QBI and $3,467 overtime deductions. It then applied an internally inconsistent bracket base ($2,783.50 over a $23,200 threshold)." +us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It invented $132,619 of head wages from the spouse's $27 hourly rate, although the head has no listed wages, and inflated the half-SE-tax deduction to $14,180. It subtracted an $8,000 child tax credit for nonexistent children and a saver's credit, and omitted the QBI and overtime deductions." us,scenario_089,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded a $2,242.80 Additional Child Tax Credit despite the household having no qualifying children. It also fabricated $120,000 of head wages from hours worked and mislabeled $2,400 of supposed excess Additional Medicare withholding as a refundable credit, even though no head wage or hourly rate was listed and tax withholding refunds are not included in federal_refundable_credits." -us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It fabricated roughly $6,200 of head payroll tax from ""45 usual weekly hours"" even though the head has no gross wages listed and unlisted numeric inputs are 0, then added the $3,467 FLSA overtime premium on top of the spouse's $65,881 gross wages, which the prompt already defines as including overtime pay. It also invented a nonexistent North Carolina employee payroll tax (a 0.56% rate on the spouse plus $1,000 on the head); the only FICA base in this household is the spouse's $65,881, giving $4,084.63 + $955.28 = $5,039.91." -us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its derivation was exactly right — no head wages, $65,881 spouse base with no 401(k) or ESI reduction, 6.2% + 1.45%, no Additional Medicare Tax — and it reached $5,039.89 in its own text. It then re-added the two components as $5,040.20 and applied an unexplained ""variance"" adjustment to submit $5,085.44, discarding the correct total it had already computed." -us,scenario_089,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It built the FICA base as $65,881 + $3,467 − $6,946 − $7,789 = $54,613, making three separate base errors: the FLSA overtime premium is already inside the stated gross wage total, elective traditional 401(k) deferrals remain fully subject to Social Security and Medicare tax, and the employer-sponsored insurance premium is a listed household expense, not a reduction of the stated gross wage. The base is $65,881, so employee FICA is $5,039.91, not $4,178." -us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the two $600 health insurance premium entries ($1,200) from the spouse's gross wages to get a $64,681 FICA base. Those premium inputs are separate household expense facts and do not reduce the stated $65,881 gross wage total, so Social Security is $4,084.63 and Medicare $955.28 for a total of $5,039.91." -us,scenario_089,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a section 125 pre-tax salary reduction and taxed only $58,092. That input is the total ESI premium listed as a household fact, not an amount withheld from the stated gross wage, so the full $65,881 is subject to the 6.2% and 1.45% employee rates, giving $5,039.91 rather than $4,444." -us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It reduced the spouse's FICA base to $58,092 by subtracting the $7,789 employer-sponsored insurance premium as a cafeteria-plan exclusion. The prompt fixes gross wages and salaries at $65,881 as the annual total and lists the ESI premium separately, so 7.65% applies to $65,881 and the employee payroll tax is $5,039.91." -us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It added the $3,467 FLSA overtime premium to the $65,881 gross wage figure and applied 7.65% to $69,348. The prompt states gross wage and salary amounts already include overtime pay, so the overtime premium is a component of the $65,881, not an addition; the correct employee FICA is $5,039.91." -us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It netted the $7,789 employer-sponsored insurance premium out of wages as a pre-tax deduction, taxing $58,092 at 7.65% for $4,444.04. The ESI premium is a separate listed household fact and does not reduce the stated $65,881 gross wage, whose employee Social Security and Medicare tax is $5,039.91." -us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It characterized the $7,789 ESI premium as a pre-tax health insurance salary reduction and computed 7.65% of $58,092. Gross wages of $65,881 are the annual total subject to the 6.2% Social Security and 1.45% Medicare employee rates, producing $5,039.91." -us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $7,789 employer-sponsored insurance premium from the spouse's wages, leaving a $58,092 FICA base. That premium is a separately listed household fact rather than a withholding from the stated gross wage, so the full $65,881 is taxed, giving $4,084.63 Social Security plus $955.28 Medicare = $5,039.91." -us,scenario_089,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied the employee 6.2% and 1.45% rates to $58,092 after removing the $7,789 ESI premium as a pre-tax deduction from W-2 wages. The stated gross wage of $65,881 is the FICA base, yielding $5,039.91." -us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its explanation states the correct rule — employee Social Security and Medicare on the spouse's $65,881 wages, head with no wages, no Additional Medicare Tax — which computes to $5,039.91, but it submitted $8,107. That figure abandons the 7.65% employee share it described and instead tracks a combined employer-plus-employee Social Security loading on those wages, which the question explicitly excludes." -us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It applied 6.2% and 1.45% to ""W-2 wages for both spouses,"" imputing wage income to the head, who has no gross wages listed and whose unlisted income the prompt fixes at 0. Its $9,711 corresponds to a FICA base near $127,000, roughly double the household's only wage base of $65,881, which produces $5,039.91." -us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It applied an ""employee health-premium exclusion"" of $7,789 to reach a $58,092 base. No such exclusion applies here: the ESI premium is a listed household expense, not a reduction of the $65,881 stated gross wage, so employee Social Security and Medicare total $5,039.91." -us,scenario_089,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It computed 7.65% of $58,092 after removing the $7,789 ESI premium as a pretax salary reduction. It was right that there is no Additional Medicare Tax and no mandatory NC employee payroll tax, but the base is the full $65,881 of gross wages, giving $5,039.91." -us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"Its own one-line basis — ""spouse wages and overtime"" — double-counts the $3,467 FLSA overtime premium that is already inside the $65,881 gross wage total, and its $8,120 exceeds 7.65% of even the wage-plus-self-employment base, so it also loaded earnings that the question routes to the separate self-employment tax output. Employee FICA on $65,881 is $4,084.63 + $955.28 = $5,039.91." -us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It taxed $69,348 by adding the $3,467 FLSA overtime premium to the $65,881 gross wages, producing $4,299.58 Social Security and $1,005.55 Medicare. The prompt states gross wage totals already include overtime pay, so the base is $65,881 and the employee payroll tax is $5,039.91." -us,scenario_089,payroll_tax,inkling,llm_error,other,False,"It identified the base and rules correctly — $65,881 of spouse wages only, 6.2% and 1.45%, no NC employee payroll tax, self-employment tax excluded — but miscomputed the Social Security component as $4,083 instead of $4,084.62 and rounded Medicare to $955 instead of $955.27. The correct sum is $5,039.91, not the $5,038 it submitted." -us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It computed employee FICA correctly at $5,039.89 on the spouse's $65,881 wages, then added $2,145.68 — the employee half of the self-employment tax on $30,335 — to the payroll_tax output. The question defines payroll_tax as employee Social Security, Medicare, Additional Medicare, and mandatory state payroll taxes only, and explicitly excludes self-employment tax, which is reported in its own output, so the answer is $5,039.91." -us,scenario_089,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Its stated components imply Social Security on about $162,400 of wages and Medicare on about $97,700 — two different fabricated bases, neither of which exists in this household — while it simultaneously asserted the head has no wages. The household's only FICA wages are the spouse's $65,881, giving $4,084.63 Social Security and $955.28 Medicare for $5,039.91." +us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It made up a $6,200 payroll tax for the head, who has no listed wages, so their FICA wages are $0. It also added the $3,467 overtime premium on top of $65,881 gross wages that already include overtime, and it added a North Carolina state payroll tax that does not exist. The correct total is 7.65% of $65,881, or $5,039.91." +us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"It used the right base of $65,881 and reached $5,039.89. It then miscalculated Medicare as $955.57 instead of $955.27 and made an unexplained upward 'variance' adjustment to $5,085.44. Nothing in the payroll tax rules supports that adjustment." +us,scenario_089,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It added the $3,467 overtime premium to gross wages that already include it. It then subtracted the $6,946 traditional 401(k) deferral, even though elective deferrals remain FICA wages under IRC §3121(v)(1)(A). It also subtracted the $7,789 ESI premium, although the facts list no pre-tax salary reduction. The result was a $54,613 base instead of $65,881." +us,scenario_089,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted $1,200 of 'health insurance premiums excluding Medicare Part B' and 'other health insurance premiums' from wages. These are individually paid premiums, not salary reductions, so they do not reduce the FICA base. That left $64,681 instead of the full $65,881." +us,scenario_089,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax cafeteria-plan exclusion and cut FICA wages to $58,092. The facts list no pre-tax salary reduction, so the full $65,881 is taxable, which gives $5,039.91 instead of $4,444." +us,scenario_089,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It subtracted the $7,789 employer-sponsored insurance premium from the spouse's $65,881 wages as if it were a Section 125 salary reduction. The listed facts do not reduce the payroll wage base, so FICA applies to all $65,881, giving $5,039.91." +us,scenario_089,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It added the $3,467 FLSA overtime premium to $65,881 gross wages that already include overtime, making the base $69,348. It also miscalculated 7.65% of that base, which is $5,305 rather than $5,373. The correct base of $65,881 yields $5,039.91." +us,scenario_089,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as a pre-tax deduction from FICA wages and got $58,092 × 7.65% = $4,444.04. The spouse's full $65,881 gross wage is the payroll tax base because no salary-reduction election is listed." +us,scenario_089,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It labeled the $7,789 ESI premium 'pre-tax' and removed it from FICA wages, leaving $58,092. With no Section 125 election in the facts, Social Security and Medicare apply to the full $65,881, which yields $5,039.91." +us,scenario_089,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted $7,789 of 'pre-tax ESI premiums' from gross wages to get a $58,092 FICA base. The household facts give no pre-tax salary reduction, so the base is the full $65,881 and the tax is $5,039.91." +us,scenario_089,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It applied a 'pre-tax ESI deduction' of $7,789 to reach $58,092 in taxable wages. Employee FICA applies to the spouse's full $65,881 gross wage because no cafeteria-plan salary reduction is listed, giving $5,039.91." +us,scenario_089,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"It correctly named the spouse's $65,881 wages as the only payroll tax base, which gives 7.65% × $65,881 = $5,039.91. It then submitted $8,107, about $3,067 higher than its own stated base supports. That figure matches a base of about $106,000, which means non-wage income was folded into it." +us,scenario_089,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It applied FICA to 'W-2 wages for both spouses', but the head has no listed wages and therefore owes no employee payroll tax. The $9,711 answer implies a base of about $127,000, far above the spouse's $65,881 of actual wages, which yield $5,039.91." +us,scenario_089,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It applied an 'employee health-premium exclusion' of $7,789 to reach $58,092 in FICA wages. The facts contain no pre-tax salary-reduction election, so payroll tax applies to all $65,881, giving $5,039.91 rather than $4,444." +us,scenario_089,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated the $7,789 employer-sponsored insurance premium as pretax and reduced FICA wages to $58,092. The payroll tax base is the spouse's full $65,881 gross wage, so the correct figure is $5,039.91." +us,scenario_089,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It correctly kept the traditional 401(k) deferrals in payroll wages but subtracted the $7,789 employer-sponsored insurance premium, leaving $58,092. No pre-tax salary-reduction election is listed, so FICA applies to the full $65,881, which yields $5,039.91." +us,scenario_089,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"It taxed 'spouse wages and overtime' as separate amounts, but the $65,881 gross wage already includes overtime. The $8,120 answer implies a base of about $106,000, so it also pulled in non-wage self-employment or partnership income. The correct base of $65,881 yields $5,039.91." +us,scenario_089,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It added the $3,467 FLSA overtime premium to the $65,881 gross wage, making the base $69,348. The prompt states that gross wages already include overtime pay, so the base is $65,881 and the tax is $5,039.91, not $5,305." +us,scenario_089,payroll_tax,inkling,llm_error,other,False,"It used the correct $65,881 base but miscalculated 6.2% × $65,881 as $4,083 instead of $4,084.63, and truncated Medicare to $955 instead of $955.28. That arithmetic slip left it at $5,038 rather than $5,039.91." +us,scenario_089,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It correctly computed $5,039.89 of employee FICA on $65,881, then added $2,145.68 as the 'employee portion' of self-employment tax. Self-employment tax is not employee payroll tax and the question explicitly excludes it." +us,scenario_089,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Its $10,068.71 of Social Security and $1,416.24 of Medicare imply bases of about $162,000 and $97,700. It folded self-employment and partnership income into the employee FICA base, when only the spouse's $65,881 of wages count, yielding $4,084.63 plus $955.28." us,scenario_089,self_employment_tax,claude-haiku-4.5,llm_error,other,False,"It identified the correct base ($30,335 of self-employment income, partnership/S-corp income excluded) and the correct 92.35% × 15.3% formula, but mis-multiplied: 30,335 × 0.9235 = 28,014.37, not the 27,998.80 it wrote. It then discarded even its own $4,285.34 result and submitted $4,275.23 under the label ""standard rounding,"" whereas correct execution of its stated method gives the reference $4,286.24." us,scenario_089,self_employment_tax,claude-sonnet-4.6,llm_error,other,False,"Its rule application was exactly right — partnership/S-corp income excluded from net earnings from self-employment, 92.35% × 15.3% on the $30,335 — but the arithmetic slipped: 30,335 × 0.9235 = 28,014.37, not the 28,024.32 it used. That $9.95 overstatement of net earnings carried through the 15.3% rate to $4,287.72 instead of $4,286.24." us,scenario_089,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"It folded the $19,650 of partnership/S-corp income into the SECA base for a $49,985 total; PolicyEngine's self-employment tax draws only on the self-employment income field of $30,335, and partnership/S-corp income flows to taxable income without entering net earnings from self-employment. It compounded that by computing $7,062.66 from the inflated base and then submitting an unexplained $6,355 that its own arithmetic never produced." @@ -6616,44 +7266,45 @@ us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"It combined the $19,650 partnership/S-corp income with the $30,335 self-employment income into a single $49,985 SECA base. Partnership/S-corp income is outside net earnings from self-employment in PolicyEngine, so its $7,063 overstates the $4,286.24 reference by precisely the 14.13% effective SECA tax on that $19,650." us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It explicitly weighed whether partnership/S-corp income belongs in net earnings from self-employment and resolved it the wrong way, taxing the full $49,985. The SECA base here is the $30,335 self-employment income alone, which at 92.35% × 15.3% gives $4,286.24 rather than its $7,062.66." us,scenario_089,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"It made two errors: it inflated the base to ~$50,000 by adding the $19,650 of partnership/S-corp income, which does not enter net earnings from self-employment, and it then subtracted a fabricated $668.78 ""offset"" for Social Security tax already paid on wages. Wage-base coordination reduces the SE Social Security base only when wages exceed the annual Social Security cap, and this household's $69,348 of wages are far below it; on the correct $30,335 base with no offset the tax is $4,286.24." -us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It reached NC taxable income of $80,952 and stated its own final result as $3,230, then submitted $4,022 — a figure produced by none of its arithmetic. Its base was also $252 too high because it denied the head's traditional IRA deduction, which is allowed as a spousal IRA against the couple's joint compensation." -us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the correct 3.99% rate and $25,500 joint standard deduction but built AGI of $105,252, $948 below the correct $106,200.13. It over-deducted above-the-line items beyond the only three that apply: the spouse's $6,946 traditional 401(k), the $577 of traditional IRA contributions, and $2,143 of half-SE-tax." -us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never subtracted North Carolina's $25,500 joint standard deduction — its explanation moves straight from adjusted income to 'NC rates' and nonrefundable credits that do not exist here (no children, no education expenses). Its $4,532 is what a rate near 4.25% yields on the full $106,200 AGI with no standard deduction, versus the correct 3.99% on $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It deducted the head's $5,403 traditional 401(k) even though the head has no wages, cutting AGI to $100,797, and then submitted $4,710 after computing $3,004 as its own final figure. A 401(k) elective deferral requires wage compensation, so only the spouse's $6,946 reduces AGI." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived the reference exactly — AGI $106,199, less the $25,500 joint standard deduction, times 3.99% = $3,220 — and then discarded it for $5,290 on the basis of an invented '$132,500 NC taxable' base that appears nowhere in the household facts and exceeds the $115,866 of gross income." -us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stated the right structure (AGI ~$106,500 less the $25,500 joint standard deduction, giving ~$81,000) and then multiplied 3.99% by $87,100, a taxable income it never derived and $6,400 above its own figure. Its own stated base would have produced $3,232 against the correct $3,219.94." -us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its AGI of $106,126 was within $74 of correct, but it asserted a 2026 NC joint standard deduction of $30,000 and a 4.25% flat rate. The 2026 parameters are a $25,500 joint standard deduction and a 3.99% rate; the two errors partially offset, leaving it $15 high." -us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It folded the $19,650 of partnership/S-corp income into the self-employment tax base, taking a $3,529 half-SE-tax deduction instead of $2,143, applied the 92.35% net-earnings factor to income entering AGI, and used a 4.25% rate — reaching $2,979, then submitting $3,800 as a 'rounding to the nearest hundred' of that number." -us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $27,500 NC joint standard deduction rather than the $25,500 that applies for 2026, and built AGI of $99,477 — $6,723 short, consistent with deducting the head's non-deductible $5,403 traditional 401(k) plus extra self-employment adjustments. Only the rate, 3.99%, was right." -us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $104,999.90 is exactly $1,200 below the correct $106,200.13 — it took the head's $600 health insurance premiums excluding Medicare Part B plus $600 of other health insurance premiums as an above-the-line self-employed health insurance deduction. Those premiums belong to a person with employer-sponsored coverage and no self-employment income, so they do not reduce AGI." -us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Three parameter errors compound: it excluded the $7,789 employer-sponsored insurance premium from wages to reach AGI of $98,411, used a $30,600 joint standard deduction instead of $25,500, and applied a 4.0% rate instead of 3.99%. The correct base is $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used the right $25,500 standard deduction and 3.99% rate but started from AGI of $98,411, which is $106,200.13 minus the $7,789 employer-sponsored insurance premium. The listed ESI premium does not reduce the $65,881 of stated gross wages in this computation, so AGI stays at $106,200.13." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation at all, asserting only 'the 2026 North Carolina flat tax rate applied to the taxable income base.' Its $4,410 is what a stale 4.25–4.5% rate produces on roughly $100,000 of AGI with the $25,500 joint standard deduction never subtracted; the correct computation is 3.99% × ($106,200.13 − $25,500) = $3,219.94." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied the correct $25,500 joint standard deduction and 3.99% rate to an AGI of $98,411, which is the correct $106,200.13 reduced by the $7,789 employer-sponsored insurance premium treated as a pre-tax cafeteria-plan exclusion. That premium does not reduce the stated gross wages here, leaving taxable income at $80,700.13 rather than $72,911." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It excluded the $7,789 employer-sponsored insurance premium from wages to reach AGI of $98,411 and then used a $29,250 joint standard deduction instead of $25,500. Both moves shrink the base: correct taxable income is $80,700.13, not $69,161." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted $6,787 with no income, deduction, or rate computation. At NC's 3.99% flat rate that implies taxable income of $170,100 — nearly $55,000 more than the household's entire $115,866 of gross income; the correct base is $106,200.13 AGI less the $25,500 joint standard deduction." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its stated NC AGI of $98,410.90 is precisely the correct $106,200.13 less the $7,789 employer-sponsored insurance premium, which it removed from wages as a pre-tax exclusion. Standard deduction and rate were right, so the entire $310.80 shortfall traces to that single subtraction." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $2,919.20 implies NC taxable income of $73,163 against the correct $80,700.13, an AGI near $98,660 — it stripped the $7,789 employer-sponsored insurance premium out of the spouse's wages. Gross wages of $65,881 enter AGI in full, reduced only by the spouse's $6,946 401(k), the $577 of traditional IRAs, and $2,143 of half-SE-tax." -us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $3,514 corresponds to NC taxable income of about $88,070, i.e. an AGI near $113,570 — essentially the $115,866 of gross income less only the $2,143 half-SE-tax deduction. It omitted the spouse's $6,946 traditional 401(k) deferral and the $577 of deductible traditional IRA contributions, which bring AGI to $106,200.13." -us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It included self-employment income in AGI at 92.35% of its value ($28,014 instead of $30,335), removing $2,321 that belongs in AGI; the 92.35% factor computes net earnings for the SE tax only and never reduces gross income. That single error explains the entire gap between its $103,879.36 AGI and the correct $106,200.13, since its rate and standard deduction were right." -us,scenario_089,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income of $63,098 — already net of the federal standard deduction — and then subtracted a state standard deduction again, double-counting roughly $32,000 of deduction. North Carolina's starting point is federal AGI of $106,200.13, and its 2026 joint standard deduction is $25,500, not the $21,500 used." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It reported zero NC tax for a joint filer with $106,200.13 of AGI, asserting that the 'limited taxable-income facts and standard deductions' wipe out liability. North Carolina's $25,500 joint standard deduction leaves $80,700.13 of taxable income, and the 3.99% flat rate has no zero bracket, exemption, or low-income threshold that could produce $0." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It described the method without executing it; $3,055 implies NC taxable income of $76,566, $4,134 below the correct $80,700.13. That gap tracks deducting the head's $5,403 traditional 401(k), which is disallowed because the head has no wages from which to defer." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Its explanation names the error outright: it computed 'taxable wages after the spouse's traditional 401(k) and pretax ESI premiums,' subtracting the $7,789 employer-sponsored insurance premium to reach $72,911 of taxable income. Only the $6,946 401(k) deferral reduces those wages; AGI is $106,200.13 and taxable income $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $69,696 is $11,004 below the correct $80,700.13, stripping out the head's $5,403 traditional 401(k) — disallowed with no wages — along with pre-tax health-premium exclusions for the $7,789 ESI and $1,200 of other premiums. Those amounts stay in AGI, which is $106,200.13." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its method statement is exact — deductible traditional retirement contributions, half of SE tax, the joint standard deduction, 3.99% — but its $3,230 corresponds to AGI of $106,452, exactly $252 above the correct figure. It denied the head's $252 traditional IRA deduction, which is allowed as a spousal IRA against the couple's joint compensation." -us,scenario_089,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied the correct $25,500 joint standard deduction and 3.99% rate to an AGI of $98,410.90, which is the correct $106,200.13 less the $7,789 employer-sponsored insurance premium. That premium is not netted out of the stated gross wages, so taxable income is $80,700.13." -us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation; $3,120 implies NC taxable income of $78,196, roughly $2,500 under the correct $80,700.13. That shortfall matches carrying the 92.35% net-earnings factor into the AGI inclusion of the $30,335 of self-employment income, which enters AGI in full with only the $2,143 half-SE-tax deduction removed." -us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It built the base perfectly — $106,200 AGI less the $25,500 joint standard deduction, giving $80,700 — and then applied a 4.25% rate. North Carolina's 2026 flat rate is 3.99%; 4.25% overstates the tax by exactly $210." -us,scenario_089,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Every input is correct — $106,200 AGI, the $25,500 joint standard deduction, and $80,700 of taxable income — but it used a 4.25% flat rate instead of the 2026 rate of 3.99%. The rate is the sole source of the $210 overstatement." -us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used AGI of $103,664, $2,536 below the correct $106,200.13, and rounded the rate to 4.0% from 3.99%. The AGI shortfall drives the error: gross income of $115,866 is reduced only by the spouse's $6,946 401(k), $577 of traditional IRAs, and $2,143 of half-SE-tax." -us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It started from the correct AGI near $106,200 and the correct 3.99% rate but applied a joint standard deduction of 'about $26,700' instead of $25,500. North Carolina's 2026 joint standard deduction is $25,500, making taxable income $80,700.13 rather than the $79,500 it used." -us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so no NC computation was submitted for grading. The required derivation is $106,200.13 of AGI less the $25,500 joint standard deduction, taxed at the 2026 flat 3.99% rate, giving $3,219.94." -us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $102,732.90 is exactly the correct $106,200.13 less the spouse's $3,467 FLSA overtime premium, which it removed from income. The federal deduction for qualified overtime is taken below the line and leaves AGI untouched, and North Carolina starts from AGI, so the $3,467 stays in the $106,200.13 base." -us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied a 4.5% rate — North Carolina's 2026 flat rate is 3.99% — to an AGI of $99,409 that is $6,791 short, exactly the head's disallowed $5,403 traditional 401(k) plus $1,388 of extra half-SE-tax from treating the $19,650 of partnership/S-corp income as self-employment earnings. Partnership and S-corp income carries no SE tax, and the head has no wages to defer." -us,scenario_089,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted the full $12,349 of traditional 401(k) contributions, including the head's $5,403, producing AGI of $100,797 instead of $106,200.13. A 401(k) elective deferral requires wage compensation and the head has none, so only the spouse's $6,946 reduces AGI; its $25,500 standard deduction and 3.99% rate were otherwise correct." -us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 4.5% rate rather than the 2026 flat 3.99%, and compounded that by adding the $3,467 FLSA overtime premium on top of gross wages that already include overtime, folding the $19,650 of partnership income into the SE tax base for a $3,531 half-SE deduction, and deducting the head's disallowed $5,403 401(k). The correct computation is 3.99% × ($106,200.13 − $25,500)." -us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It fabricated an AGI of $248,485 — more than double the household's $115,866 of gross income — subtracted a $32,200 standard deduction, applied a 4.5% rate, and then invented a $2,625.56 credit for taxes paid to other states on partnership income that carries no other-state sourcing in these facts. The base is $106,200.13 of AGI less the $25,500 joint standard deduction at 3.99%, with no nonrefundable credits." +us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work reached NC taxable income of $80,952 and tax of $3,230, which already contained an error: it disallowed Head's $252 traditional IRA deduction, but on a joint return that deduction is allowed as a spousal IRA under IRC §219(c). It then submitted $4,022, a number no step of its work produces. The correct base is AGI of $106,200.13 minus the $25,500 joint standard deduction, taxed at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the right 3.99% rate and $25,500 standard deduction but started from a federal AGI of $105,252, which is $948 below the correct $106,200.13. Correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus $30,335 self-employment and $19,650 pass-through income, minus about $2,143 half-SE-tax and $577 of IRA deductions. As a result NC taxable income was understated at $79,752 instead of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It showed no standard-deduction or rate step, and its $4,532 is roughly 4.25% of the full ~$106,200 AGI. That matches taxing AGI without subtracting North Carolina's $25,500 joint standard deduction and using an outdated rate instead of the 2026 rate of 3.99%. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It deducted Head's $5,403 traditional 401(k) deferral even though Head has no wages; 401(k) deferrals can only come out of pay. That gave AGI of $100,797 and tax of $3,004, but it then submitted $4,710, which none of its calculations support. Excluding only the spouse's $6,946 deferral gives AGI of $106,200.13, NC taxable income of $80,700.13, and tax of $3,219.94 at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning followed the correct path: it excluded only the spouse's 401(k), took the $577 IRA and ~$2,144 half-SE-tax adjustments, reached AGI of ~$106,199, subtracted $25,500, and applied 3.99% to get $3,220. It then dropped that result for an invented ~$132,500 income base, larger than the household's entire $115,866 gross income, and submitted $5,290." +us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stated NC taxable income of about $81,000 (AGI ~$106,500 minus $25,500) but then multiplied 3.99% by $87,100, inflating its own base by about $6,100 with no explanation. Applying 3.99% to the correct $80,700.13 gives $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a made-up $30,000 joint standard deduction (North Carolina's is $25,500 under G.S. 105-153.5) and a 4.25% rate instead of the 3.99% rate set for 2026 under G.S. 105-153.7. Its AGI of $106,126 was close, so the two errors partly cancelled out to $3,235." +us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It counted only 92.35% of self-employment income in AGI, but the 0.9235 factor applies only to the self-employment tax base. It also treated the $19,650 partnership/S-corp income as subject to self-employment tax and used 4.25% instead of 3.99%. Finally it submitted $3,800, which is not a rounding of its own $2,979 result." +us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $99,477, which is $6,723 below the correct $106,200.13; correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus self-employment and pass-through income, minus half-SE-tax and $577 of IRA deductions. It also used a $27,500 joint standard deduction instead of North Carolina's $25,500, so NC taxable income came out at $71,977 instead of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $104,999.90 is $1,200 below the correct $106,200.13, which matches deducting the two spouses' $600 'other health insurance premiums' as a self-employed health insurance deduction. IRC §162(l)(2)(B) bars that deduction because both spouses are eligible for employer-sponsored coverage. Without it, ($106,200.13 − $25,500) × 3.99% = $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It removed the spouse's $7,789 employer-sponsored insurance premium from wages, giving AGI of $98,411 instead of $106,200.13. It also used a $30,600 joint standard deduction instead of North Carolina's $25,500 and a 4.0% rate instead of the 2026 rate of 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from AGI of $98,411, which is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium. The facts give gross wages with no pre-tax payroll deduction for that premium, so AGI stays $106,200.13 and NC taxable income is $80,700.13, not $72,911." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $4,410 equals 3.99% of about $110,500, which is more than AGI, so it never subtracted North Carolina's $25,500 joint standard deduction from an income base close to gross income. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $98,411, which removes the spouse's $7,789 employer-sponsored insurance premium from gross wages; the facts give no pre-tax payroll deduction for that premium. The correct AGI of $106,200.13 minus the $25,500 standard deduction gives $80,700.13 of taxable income, not $72,911." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from AGI of $98,411 by wrongly removing the spouse's $7,789 employer-sponsored insurance premium from wages. It also subtracted a $29,250 standard deduction instead of North Carolina's $25,500 joint amount, producing taxable income of $69,161 instead of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It showed no calculation. $6,787 is more than 3.99% of the household's entire $115,866 gross income ($4,623), so it did not apply North Carolina's 2026 flat 3.99% rate to AGI minus the $25,500 joint standard deduction. That calculation gives $3,219.94." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its NC AGI of $98,410.90 is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, which it treated as a pre-tax wage deduction the facts do not support. NC taxable income is $80,700.13, not $72,910.90." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $2,919.20 is 3.99% of $73,163, which implies AGI of $98,663. That equals the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, with Head's $252 IRA deduction added back. So it wrongly treated the insurance premium as a pre-tax wage deduction and disallowed Head's spousal IRA deduction, which IRC §219(c) permits on a joint return." +us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $3,514 is 3.99% of about $88,070 of taxable income, roughly $7,370 more than the correct $80,700.13. The overstatement is about the size of the spouse's $6,946 traditional 401(k) deferral plus IRA deductions, which it left in income." +us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It counted self-employment income in AGI as $28,014.47, the $30,335 multiplied by 0.9235, but that factor applies only to the self-employment tax base. AGI includes the full $30,335 minus the $2,143 half-SE-tax deduction, which gives $106,200.13 instead of $103,879.36." +us,scenario_089,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($63,098) instead of federal AGI, which is North Carolina's actual starting point under G.S. 105-153.5. It then subtracted a $21,500 standard deduction instead of the $25,500 joint amount. The correct base is $106,200.13 − $25,500 = $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It returned $0 even though federal AGI of $106,200.13 leaves $80,700.13 of NC taxable income after the $25,500 joint standard deduction. North Carolina has no nonrefundable credit for this childless couple that offsets the resulting $3,219.94 of tax at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $3,055 is 3.99% of about $76,570 of taxable income, roughly $4,130 below the correct $80,700.13. It subtracted too much for retirement contributions: only the spouse's $6,946 traditional 401(k) deferral (Head has no wages) and the couple's $577 of IRA contributions reduce AGI." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It removed $7,789 of 'pretax ESI premiums' from the spouse's wages, lowering AGI to $98,411. The facts give gross wages with no pre-tax premium deduction, so AGI is $106,200.13 and NC taxable income is $80,700.13, not $72,911." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its NC taxable income of $69,696 is $11,004 below the correct $80,700.13 (AGI of $106,200.13 minus $25,500). It subtracted items that do not reduce AGI in this household, such as the spouse's $7,789 employer-sponsored insurance premium and deferrals beyond the spouse's own $6,946 401(k) contribution." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,230 is 3.99% of $80,952, which implies AGI of $106,452: the correct $106,200.13 with Head's $252 traditional IRA deduction taken out. Head has no earnings, but on a joint return the spousal-IRA rule in IRC §219(c) lets Head deduct against the spouse's earnings, so the full $577 IRA deduction applies." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $98,410.90 is the correct $106,200.13 minus the spouse's $7,789 employer-sponsored insurance premium, which it treated as a pre-tax reduction of the stated gross wages without any basis in the facts. NC taxable income is $80,700.13, not $72,910.90." +us,scenario_089,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $98,410.90 removes the spouse's $7,789 employer-sponsored insurance premium from wages as if it were a pre-tax payroll deduction, which the facts do not state. The correct AGI of $106,200.13 minus the $25,500 standard deduction gives $80,700.13 taxed at 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It showed no calculation. Its $3,120 is 3.99% of about $78,200 of taxable income, roughly $2,500 below the correct $80,700.13. So it understated the correct $106,200.13 AGI by over-reducing self-employment or retirement items before subtracting the $25,500 standard deduction." +us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got AGI ($106,200) and taxable income ($80,700) right but applied 4.25%, North Carolina's 2025 rate, instead of the 3.99% rate for 2026 under G.S. 105-153.7." +us,scenario_089,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got the $106,200 AGI and $80,700 taxable income right but taxed them at the 2025 rate of 4.25% instead of North Carolina's 2026 rate of 3.99%." +us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used AGI of $103,664, which is $2,536 below the correct $106,200.13, and a 4% rate instead of the 2026 rate of 3.99%. Correct AGI is the spouse's wages minus only the $6,946 401(k) deferral, plus the full $30,335 self-employment and $19,650 pass-through income, minus $2,143 half-SE-tax and $577 of IRA deductions, which leaves taxable income of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It started from the correct ~$106,200 AGI but subtracted a ~$26,700 joint standard deduction instead of North Carolina's statutory $25,500 (G.S. 105-153.5(a)(1)). That understated taxable income at ~$79,500 instead of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its AGI of $102,732.90 is the correct $106,200.13 minus the spouse's $3,467 FLSA overtime premium. The federal qualified-overtime deduction under IRC §225 is taken after AGI and does not reduce it, so it never reaches North Carolina's AGI starting point." +us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied 4.5%, North Carolina's 2024 rate, instead of the 3.99% rate for 2026. Its AGI of $99,409 is also $6,791 below the correct $106,200.13." +us,scenario_089,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI of $100,797 subtracts Head's $5,403 traditional 401(k) deferral even though Head has no wages, and 401(k) deferrals can only come out of pay. Excluding only the spouse's $6,946 deferral gives AGI of $106,200.13 and taxable income of $80,700.13." +us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It applied 4.5% instead of the 2026 rate of 3.99%. It also added the $3,467 FLSA overtime premium on top of gross wages that already include it, computed self-employment tax on the $19,650 partnership/S-corp income as if it were self-employment income, and deducted Head's $5,403 401(k) deferral even though Head has no wages." +us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used an invented AGI of $248,485, more than double the household's $115,866 gross income, along with a $32,200 standard deduction and a 4.5% rate. It also made up a credit for taxes paid to other states, although no fact places the partnership income outside North Carolina. The correct result is ($106,200.13 − $25,500) × 3.99% = $3,219.94." us,scenario_089,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_090,federal_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model treated the 22-year-old as eligible for the childless EITC, missing its minimum age of 25. Its subsequent phase-in and phaseout estimate is irrelevant because age disqualifies the filer before the credit schedule is applied." us,scenario_090,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model applied the childless EITC schedule to a 22-year-old even though the minimum qualifying age is 25. It also submitted $721 after calculating a $535.50 phase-in amount, but the controlling error is that no EITC schedule applies to this filer." @@ -6674,73 +7325,83 @@ us,scenario_090,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibil us,scenario_090,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly assumed categorical eligibility even though the prompt sets every unlisted status to false. At age 22, the head is neither an infant nor a child under five, and no pregnant, postpartum, or breastfeeding status is listed, so the mandatory WIC categorical test fails." us,scenario_090,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model calculated both FICA components correctly but added $434.00 and $101.50 incorrectly: their sum is $535.50, not $539.50." us,scenario_090,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly derived $434.00 of Social Security tax and $101.50 of Medicare tax, totaling $535.50, then replaced that result with an unsupported $511.00 final value. No rounding rule changes $535.50 to $511.00." -us,scenario_090,snap,claude-fable-5,llm_error,asset_resource,False,"Ran the gross-income screen, 20% earned income deduction, standard deduction and 30% net-income contribution to a positive allotment while never applying the SNAP resource test to the stated $20,300 in bank assets, which exceeds the roughly $3,000 countable-resource limit Kansas enforces because it has no broad-based categorical eligibility. Its submitted $2,137 also does not follow from its own stated $101/month benefit ($1,212/year), so the number is unreachable from its own derivation as well as from the correct one." -us,scenario_090,snap,claude-opus-4.8,llm_error,asset_resource,False,"Identified the decisive fact — 'Bank assets of $20,300 exceed standard $2,750 asset limit' — and then discarded it by asserting that Kansas uses broad-based categorical eligibility to raise or eliminate the asset test. Kansas repealed BBCE under the 2015 HOPE Act and applies the federal ~$3,000 resource limit to households with no elderly or disabled member, so the resource failure alone zeroes the benefit and the $2,496 income-formula result never applies." -us,scenario_090,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"Reached the correct interim conclusion that $20,300 in bank assets exceeds the SNAP asset limit and makes the household ineligible, then reversed it on the false premise 'Kansas does have BBCE, so the asset limit does not apply'; Kansas has no broad-based categorical eligibility and applies the federal ~$3,000 resource limit. It then spent the rest of its reasoning adjudicating whether the $4,800 in financial assistance is countable unearned income or excluded educational aid, a question that never binds because the resource test disqualifies the household before any income computation." -us,scenario_090,snap,claude-sonnet-5,llm_error,asset_resource,False,"Correctly found the household resource-ineligible on $20,300 in liquid assets against the roughly $3,000 limit, then restored eligibility by claiming Kansas waives resource limits through TANF-linked broad-based categorical eligibility — Kansas repealed BBCE in 2015 and the federal asset test governs. It compounded the error by submitting $2,650 after computing $99/month ($1,188/year), inflating its own result with an unexplained 'adjusting for rounding and standard deduction updates' step." -us,scenario_090,snap,glm-5.3,llm_error,asset_resource,False,"Screened the household on the 130% FPG gross income test only and went straight to the allotment formula, never running the countable-resource test that Kansas applies as a non-BBCE state; the $20,300 in bank assets exceeds the roughly $3,000 limit for a household with no elderly or disabled member and disqualifies the household outright. Its $1,286 is the pure benefit-formula output for an income-eligible household and reflects no resource screen at all." -us,scenario_090,snap,gpt-5.4-mini,llm_error,asset_resource,False,"Asserted the household has 'low net resources' when the prompt states $20,300 in bank account assets, contradicting the one fact that decides the case: Kansas has no broad-based categorical eligibility, so the federal countable-resource limit of roughly $3,000 for a household with no elderly or disabled member applies and this household fails it. The $438 figure corresponds to no stated derivation and is simply a positive allotment asserted for a household that is resource-ineligible." -us,scenario_090,snap,grok-build-0.1,llm_error,asset_resource,False,"Stated flatly that 'Kansas uses BBCE so no asset test' and therefore skipped the resource screen entirely; Kansas repealed broad-based categorical eligibility under the 2015 HOPE Act and applies the federal countable-resource limit of roughly $3,000, which the household's $20,300 in bank assets exceeds. Everything downstream — the $2,448 standard deduction, $1,400 earned income deduction, and $3,504 maximum annual allotment — is moot because the household never clears the resource test." -us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used the repealed pre-2024 Kansas personal exemption of $2,250 instead of the $9,160 single-filer exemption enacted by SB 1 (2024 special session), and a stale $3,500 standard deduction instead of $3,605. The correct $3,605 + $9,160 = $12,765 of subtractions exceeds the $7,020 Kansas AGI, so Kansas taxable income is zero; its 5.2% rate selection was right but was applied to $1,270 of taxable income that does not exist." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It invented a Kansas standard deduction as ""about one-third of the federal single standard deduction"" (~$5,100) and applied no personal exemption whatsoever. Kansas grants a $3,605 standard deduction plus a $9,160 single personal exemption under SB 1 (2024), which together shelter all $7,020 of Kansas AGI and produce $0 tax." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It subtracted only a $3,500 standard deduction, omitting the $9,160 single personal exemption entirely, and then applied the 3.10% bottom bracket that SB 1 (2024) repealed when it consolidated Kansas to a 5.2%/5.58% two-bracket schedule. With the $9,160 exemption and $3,605 standard deduction applied, $7,020 of Kansas AGI leaves zero taxable income and $0 liability." -us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It carried forward the pre-2024 Kansas personal exemption of $2,250 rather than the $9,160 single exemption set by SB 1 (2024), and used a stale $3,500 standard deduction instead of $3,605. Its 5.2% rate was correct, but $3,605 + $9,160 of subtractions wipes out the entire $7,020 Kansas AGI, leaving no base to tax." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used $2,320 — the SB 1 per-dependent exemption amount — as the filer's own personal exemption instead of the $9,160 single exemption, and paired it with a stale $3,500 standard deduction. It also asserted a $7,000 single-filer low-income exclusion threshold; Kansas's low-income exclusion threshold for single filers is $5,000, and it is irrelevant here because the $3,605 standard deduction plus $9,160 exemption already zero out $7,020 of Kansas AGI." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It substituted the $2,320 per-dependent exemption amount for the $9,160 single personal exemption enacted by SB 1 (2024) and used $3,600 rather than $3,605 for the standard deduction. The full $3,605 + $9,160 = $12,765 of Kansas subtractions exceeds $7,020 of Kansas AGI, so the 5.2% rate applies to zero taxable income." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It got the $3,605 standard deduction right but applied the repealed $2,250 personal exemption instead of the $9,160 SB 1 (2024) single exemption, and then taxed the residual at the 3.1% bottom bracket that the same bill eliminated in favor of a 5.2%/5.58% schedule. Subtracting the actual $9,160 exemption from $7,020 of Kansas AGI already drives taxable income to zero, making the rate choice moot." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It combined a stale $3,500 standard deduction with the repealed $2,250 personal exemption and the repealed 3.1% bottom bracket, all superseded by SB 1 (2024). Kansas's current $3,605 standard deduction plus $9,160 single personal exemption exceeds the household's $7,020 Kansas AGI, leaving $0 of taxable income." -us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It applied the pre-SB 1 parameter set — $3,500 standard deduction, $2,250 personal exemption, and the 3.1% initial bracket — none of which are in force for 2026. Under the SB 1 (2024) regime, $3,605 + $9,160 of subtractions absorbs the entire $7,020 Kansas AGI and yields $0 before refundable credits." -us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It subtracted only a $3,500 standard deduction and no personal exemption at all, then applied the 3.1% bracket that SB 1 (2024) repealed. The omitted $9,160 single personal exemption by itself exceeds the $7,020 Kansas AGI, so Kansas taxable income and tax are zero." -us,scenario_090,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It fabricated a 5.15% Kansas flat rate — Kansas has a two-bracket 5.2%/5.58% schedule, not a flat tax — used an approximate $3,700 standard deduction in place of $3,605, and omitted the personal exemption entirely. The $9,160 single personal exemption alone exceeds the $7,020 Kansas AGI, producing $0 liability regardless of rate." -us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $1,120 of Kansas taxable income implies combined standard deduction and personal exemption of only about $5,900, meaning it used the pre-2024 exemption tier rather than the $9,160 single exemption from SB 1 (2024). Kansas's actual $3,605 + $9,160 in subtractions exceeds $7,020 of Kansas AGI, leaving nothing for the 5.2% rate to tax." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It correctly identified the $3,605 standard deduction and the 5.2% bottom rate but paired them with the repealed $2,250 personal exemption instead of the $9,160 single exemption enacted by SB 1 (2024). That single substitution creates the entire $1,165 of phantom taxable income; with $9,160 applied, $7,020 of Kansas AGI produces $0 tax." -us,scenario_090,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Its $1,165 residual shows it subtracted $3,605 plus a $2,250 personal exemption, using the pre-SB 1 exemption amount rather than the $9,160 single exemption in force for 2026. Kansas's actual subtractions total $12,765 for a single filer with no dependents, exceeding the $7,020 Kansas AGI and zeroing the tax." -us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It invented a 5.7% Kansas flat rate — the statutory bottom bracket is 5.2% — used a stale $3,500 standard deduction, and applied no personal exemption. The $9,160 single personal exemption from SB 1 (2024) alone exceeds the $7,020 Kansas AGI, so Kansas taxable income is zero and no rate produces a positive liability." -us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It approximated the Kansas personal exemption at about $2,300 — the per-dependent amount — instead of the $9,160 single-filer exemption, and guessed a ~5.5% flat rate rather than the 5.2% bottom bracket. With the correct $3,605 standard deduction and $9,160 exemption, $7,020 of Kansas AGI leaves zero taxable income and $0 tax." -us,scenario_090,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It converted the Kansas personal exemption into a fictitious $110 nonrefundable credit; Kansas grants the exemption as a $9,160 subtraction from Kansas AGI for a single filer under SB 1 (2024), not as a credit against tax. It also used a stale $3,500 standard deduction and misstated 3,520 × 5.2% as $182.96 rather than $183.04, but the decisive error is the missing $9,160 deduction, which alone reduces $7,020 of Kansas AGI to zero taxable income." +us,scenario_090,snap,claude-fable-5,llm_error,asset_resource,False,"The model ran only the gross-income, net-income and allotment steps and never applied the SNAP resource test to the head's $20,300 bank balance. Kansas has no BBCE, so the federal $3,000 resource limit applies and makes the household ineligible, giving $0 rather than its (internally inconsistent) $2,137." +us,scenario_090,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model correctly noted that the $20,300 in bank assets exceeds the federal resource limit. It then claimed that Kansas uses broad-based categorical eligibility to waive the asset test, which is wrong because Kansas does not use BBCE. The household therefore fails the resource test and gets $0 SNAP, not $2,496." +us,scenario_090,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model first concluded that the $20,300 in assets makes the household ineligible, then reversed itself by claiming that Kansas has BBCE, which removes the asset limit. Kansas does not use BBCE, so the federal $3,000 resource limit disqualifies the household and SNAP is $0, not $1,116." +us,scenario_090,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model found the household resource-ineligible because its $20,300 in liquid assets exceeds the roughly $3,000 limit. It then restored eligibility by claiming that Kansas uses TANF-funded BBCE to waive the limit. Kansas does not use BBCE, so resource ineligibility stands and SNAP is $0. The model's $2,650 was also an unexplained upward adjustment from its own $1,188 calculation." +us,scenario_090,snap,claude-sonnet-5.5,llm_error,asset_resource,False,"The model stated that Kansas broad-based categorical eligibility means the $20,300 in assets does not block eligibility. Kansas does not use BBCE, so the federal $3,000 resource limit applies and the household is ineligible, giving $0 rather than $2,642." +us,scenario_090,snap,glm-5.3,llm_error,asset_resource,False,"The model applied only the 130% FPG gross-income test and the benefit formula and never tested the head's $20,300 bank balance against the SNAP resource limit. Kansas has no BBCE, so the $3,000 federal resource limit makes the household ineligible, giving $0 rather than $1,286." +us,scenario_090,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model asserted that the household has 'low net resources' even though the head holds $20,300 in bank assets. Kansas does not use BBCE, so that balance exceeds the federal $3,000 SNAP resource limit and makes the household ineligible. The correct value is $0, not the model's unsupported $438." +us,scenario_090,snap,grok-build-0.1,llm_error,asset_resource,False,"The model stated 'Kansas uses BBCE so no asset test' and skipped the resource test. Kansas does not use broad-based categorical eligibility, so the $20,300 bank balance exceeds the federal $3,000 resource limit, the household is ineligible, and SNAP is $0, not $1,112." +us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,500 standard deduction and a $2,250 personal exemption, then taxed $1,270 at 5.2% to get $66. The 2024 Kansas law's $9,160 single personal exemption plus the $3,605 standard deduction totals $12,765, which is more than the $7,020 KAGI, so Kansas taxable income and tax are both $0." +us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $3,605 standard deduction but the pre-2024 $2,250 personal exemption, which left $1,165 taxed at 5.2%. Kansas's current single personal exemption is $9,160, so the $12,765 in combined deductions wipes out the $7,020 KAGI and the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a standard deduction of about $5,100 ('one-third of federal'), left out the personal exemption entirely, and taxed $1,920 at 5.2%. The actual Kansas single standard deduction is $3,605 and the personal exemption is $9,160, which together exceed the $7,020 KAGI and give $0 tax." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, skipped the personal exemption, and taxed $3,520 at the repealed 3.1% bottom rate. Kansas's $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so taxable income is $0." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $3,500 standard deduction and the old $2,250 personal exemption, then taxed the remaining $1,270 at 5.2%. Under the 2024 Kansas law, the single personal exemption is $9,160 and the standard deduction is $3,605, so the $7,020 KAGI is fully sheltered and the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,800 standard deduction and a $2,320 exemption, which is the Kansas dependent amount rather than the $9,160 single-filer exemption. It then taxed $900 at the repealed 3.1% rate. The correct $3,605 + $9,160 = $12,765 exceeds the $7,020 KAGI, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It paired a $3,500 standard deduction with a $2,320 exemption, which is the dependent amount, and taxed $1,200 at 5.2%. It never applied the $9,160 single personal exemption, which together with the $3,605 standard deduction brings Kansas taxable income to $0 with no need for any low-income exclusion." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $3,600 standard deduction and a $2,320 personal exemption, leaving $1,100 taxed at 5.2%. The single-filer exemption under current Kansas law is $9,160, not the $2,320 dependent figure, so the combined $12,765 exceeds the $7,020 KAGI and the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the $3,605 standard deduction but the obsolete $2,250 exemption, then taxed $1,165 at the repealed 3.1% rate. Kansas's $9,160 single personal exemption alone exceeds the income left after the standard deduction, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied the pre-2024 Kansas amounts ($3,500 standard deduction, $2,250 exemption, 3.1% bottom rate) to get $1,270 × 3.1%. Under the 2024 law, the $3,605 standard deduction plus the $9,160 single exemption exceed the $7,020 KAGI, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the old $3,500 standard deduction and $2,250 exemption and taxed $1,270 at the repealed 3.1% rate. It missed the 2024 increase of the single personal exemption to $9,160, which together with the $3,605 standard deduction leaves $0 Kansas taxable income." +us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, left out the personal exemption entirely, and taxed $3,520 at the repealed 3.1% rate. Kansas's $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted only a standard deduction of about $3,700, never applied a personal exemption, and used a nonexistent 5.15% flat rate on $3,320. Kansas allows a $9,160 single personal exemption on top of the $3,605 standard deduction, which takes taxable income to $0." +us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,120 of taxable income means it subtracted only about $5,900 in combined standard deduction and exemption, which fits the old ~$2,300 per-person exemption. The current Kansas single exemption is $9,160, so the $12,765 in combined deductions leaves $0 to tax at 5.2%." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,605 standard deduction but only a $2,250 personal exemption, then taxed $1,165 at 5.2% and rounded to $61. The 2024 Kansas law sets the single personal exemption at $9,160, which zeroes out Kansas taxable income on $7,020 of KAGI." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Its $1,165 of taxable income means it subtracted about $5,855, which matches the $3,605 standard deduction plus the obsolete $2,250 exemption. It then taxed that at 5.2%. The correct $9,160 single personal exemption plus the $3,605 standard deduction exceed the $7,020 KAGI, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It paired the correct $3,605 standard deduction with a $2,320 exemption, which is the Kansas dependent amount, and taxed $1,095 at 5.2%. A single filer's own exemption is $9,160, so combined deductions of $12,765 exceed the $7,020 KAGI and the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted only a $3,500 standard deduction, skipped the personal exemption, and applied a made-up 5.7% flat rate to $3,520. Kansas's $3,605 standard deduction plus its $9,160 single personal exemption exceed the $7,020 KAGI, so the tax is $0." +us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted a standard deduction of about $3,600 and an exemption of about $2,300, which is the old per-person figure, then taxed about $1,100 at an assumed 5.5% flat rate. The current Kansas single exemption of $9,160 plus the $3,605 standard deduction leave $0 of taxable income." +us,scenario_090,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It treated the personal exemption as a made-up $110 credit rather than a deduction from income, so it taxed $3,520 at 5.2% and subtracted $110. Kansas deducts a $9,160 single personal exemption along with the $3,605 standard deduction, which exceeds the $7,020 KAGI and gives $0 tax." us,scenario_090,state_refundable_credits,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It priced the Kansas food sales tax credit off an income test alone, inventing an ""under-60, AGI below the poverty guideline"" rule; the actual credit (K.S.A. 79-32,271) requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 residing in the home, and a 22-year-old single filer with no dependents meets none of those categories. It also missed that the credit has been nonrefundable since tax year 2013, so it cannot enter state_refundable_credits at all. Its $60 corresponds to no Kansas parameter — the statutory amount is $125 per exemption." us,scenario_090,state_refundable_credits,glm-5.2,llm_error,age_disability,False,"It applied the 7.65% childless phase-in rate to $7,000 of wages to get a $535.50 federal EITC while ignoring the childless-EITC age requirement of 25 through 64; the head is 22, so the federal EITC is $0 and the Kansas EITC — which is a flat percentage of the federal credit — is $0. It compounded this by using an 18% Kansas match; the Kansas rate is 17%, though at a $0 federal base either rate yields $0." us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It used the correct 17% Kansas match but multiplied it against a federal EITC of $535.50 that does not exist: the childless EITC is available only to filers aged 25 through 64, and the head is 22, so the federal credit phases in to $0 and 17% of $0 is $0. Kansas has no independent earned-income credit computation, so a zero federal EITC forces ks_eitc to zero." us,scenario_090,state_refundable_credits,grok-4.5,llm_error,age_disability,False,"It computed the Kansas refundable EITC as 17% of a $536 federal EITC without applying the age-25 minimum for the childless federal EITC; at age 22 the head's federal EITC is $0, making the Kansas EITC $0. Its Kansas rate and arithmetic were right, but the federal base it multiplied was disallowed by the age test." us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied only the Kansas food sales tax credit's income ceiling and skipped the categorical qualification, which limits the credit to filers who are 55 or older, blind or disabled, or have a dependent child under 18 living with them; a 22-year-old single filer with no dependents satisfies none of those, so the credit is $0. It further treated the credit as refundable, but Kansas made it nonrefundable beginning in tax year 2013, so even a qualifying $125 credit would not appear in state_refundable_credits." us,scenario_090,tanf,glm-5.3,llm_error,categorical_eligibility,False,"Kansas TANF requires a dependent child in the assistance unit, and this 22-year-old head is a childless adult, so the benefit is $0 regardless of income; the household also holds $20,300 in bank assets, far above Kansas's TANF resource limit. glm-5.3 ran no eligibility test at all - it copied the $4,800 'financial assistance' input, a separate unearned-income variable in PolicyEngine rather than a TANF payment, straight into the tanf output. Its own stated reasoning ('financial assistance during the year, which is its annual TANF cash assistance amount') shows the substitution, and the prompt's instruction not to infer unlisted benefit receipt forbids reclassifying that input as TANF." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It omitted the $627.91 qualified-overtime deduction created by the $628 FLSA overtime premium and pulled the $1,170 of non-Sch D gains into the income base, so its ordinary base of $14,945 exceeds the correct $14,316.66. Its own chain ends at $1,545 of tax less a $120 saver's credit, i.e. $1,425, yet it submitted $1,276, a figure no step of its derivation produces." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It left the $628 overtime premium undeducted and kept the $1,170 of non-Sch D capital gains in the ordinary bracket base, inflating ordinary taxable income to $16,115 against the correct $14,316.66. That extra $1,798 taxed at 12% is precisely the $215.90 by which its $1,565.90 exceeds the reference; its $119.90 saver's credit was otherwise correct." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied a $2,500 American Opportunity Credit although no qualified tuition or education expense amount is listed, so AOTC is $0 and the only nonrefundable credit is the $120 saver's credit. It compounded this with a $14,600 standard deduction instead of $16,100, no exclusion of the $926.16 of pre-tax 401(k) deferrals, and a 15% rate on the $3,968 of qualified dividends when taxable income of $18,284.66 sits far below the 0%-rate ceiling; its submitted $650 matches neither its own $490 result nor any correct chain." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It skipped the $627.91 qualified-overtime deduction and taxed the $1,170 of non-Sch D gains, raising the ordinary base to $16,115 against the correct $14,316.66. It then declared that no nonrefundable credits apply, missing the $120 saver's credit that $1,199 of elective deferrals and IRA contributions earn at the 10% rate on a $35,012.56 AGI." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It treated the $27,000 wage figure as already net of the $926.16 traditional 401(k) deferral and never removed it, and it omitted the $627.91 overtime deduction, reaching $1,797 of tax. It then back-solved an arbitrary $331 nonrefundable AOTC to land on $1,466, when no qualified tuition expense is listed so AOTC is $0 and the sole nonrefundable credit is the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its own derivation reaches about $1,684 of ordinary tax with no nonrefundable credits, and it then submitted $2,109 — a number no bracket application produces on its stated $20,083 taxable base. It also never applied the $627.91 qualified-overtime deduction or the $120 saver's credit, both of which are required to reach $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It guessed a $15,000 standard deduction and used the 2025 $11,925 ceiling for the 10% bracket instead of the 2026 $16,100 deduction and $12,400 ceiling. It also never took the $627.91 qualified-overtime deduction from the $628 FLSA overtime premium and concluded no nonrefundable credits apply, missing the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Its $1,470 equals the tax before credits, but its derivation used a $15,750 standard deduction and subtracted a fictitious $1,500 nonrefundable AOTC to reach $236 before overriding that with $1,470. The credit that actually applies is the $120 saver's credit — 10% of $1,199 in traditional 401(k), Roth 401(k), traditional IRA and Roth IRA contributions — which brings tax to $1,350." -us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a guessed $15,400 standard deduction instead of the 2026 $16,100 and skipped the $627.91 qualified-overtime deduction. Those two omissions add $1,328 to the ordinary base ($15,645 versus the correct $14,316.66), which at 12% is the $153 by which its $1,515 exceeds the reference; its $119.90 saver's credit was correct." -us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $1,800 of health insurance premiums from wages, which are not an above-the-line deduction, and used a $15,000 standard deduction with the 2025 $11,925 bracket ceiling. Those understatements nearly cancel its omission of the $627.91 overtime deduction, so it lands $1 from the reference by offsetting errors rather than by the correct chain of $35,012.56 AGI, $18,284.66 taxable, $1,470 of tax, and a $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It guessed a $15,400 standard deduction and a $12,225 ceiling for the 10% bracket instead of the 2026 $16,100 and $12,400, and never applied the $627.91 qualified-overtime deduction. Its ordinary base of $15,645 therefore exceeds the correct $14,316.66; its $119.90 saver's credit was right." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It zeroed the entire liability with the American Opportunity Credit even though no qualified tuition expense is listed, making AOTC $0. It also deducted the $6,589 of employer-sponsored insurance premiums, which sit outside the $27,000 of employment income and are not an above-the-line deduction, and applied a pre-TCJA-style $13,000 deduction-plus-exemption instead of the 2026 $16,100 standard deduction." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted a taxable base of about $25,500 against the correct $18,284.66, using a standard deduction far below $16,100 and never taking the $627.91 qualified-overtime deduction. Its submitted $1,399 does not follow even from its own base, which yields $2,812 at 2026 rates, and it never applied the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA parameters — a reduced standard deduction plus a personal exemption — when 2026 law gives a single filer a $16,100 standard deduction and no personal exemption. It also cut wages by the $6,589 of ESI premiums, producing $29,594 of AGI against the correct $35,012.56; its $120 saver's credit was the one correct credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used an $8,300 standard deduction plus a $5,300 personal exemption on a TCJA-sunset reading, when 2026 sets the single standard deduction at $16,100 with no personal exemption. It also removed the $6,589 of ESI premiums from income and omitted both the $627.91 overtime deduction and the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It stopped at tax before credits and named only the AOTC as a candidate nonrefundable credit, so it never applied the $120 saver's credit on $1,199 of retirement contributions. Its $1,475 is the pre-credit figure of $1,470 plus rounding; the reference is that amount net of the saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 of employer-sponsored insurance premiums from income and applied a personal exemption on top of the standard deduction; 2026 law allows neither, leaving AGI at $35,012.56 with only the $16,100 standard deduction and the $627.91 qualified-overtime deduction to subtract. It also omitted the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Its explanation states the answer is $1,756 while the submitted value is $1,086, and neither follows from a correct chain. It built AGI of $36,183 by including the $1,170 of non-Sch D gains rather than the correct $35,012.56, and it applied neither the $627.91 qualified-overtime deduction nor the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,000 standard deduction and the 2025 $11,925 bracket ceiling and skipped the $627.91 qualified-overtime deduction, reaching $1,567 in its own explanation, then submitted $1,617 — $50 above its stated result. The correct chain gives $18,284.66 of taxable income, $1,470 of tax, and $1,350 after the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It guessed a $15,400 standard deduction and a $12,200 ceiling for the 10% bracket instead of the 2026 $16,100 and $12,400, and never took the $627.91 qualified-overtime deduction. Its ordinary base of $15,645 therefore exceeds the correct $14,316.66 by $1,328, the source of the entire $163.50 gap; its $119.90 saver's credit was right." -us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It declared that no nonrefundable credits apply, missing the $120 saver's credit earned by $1,199 of elective deferrals plus Roth 401(k) and Roth IRA contributions at a $35,012.56 AGI in the 10% tier. It also used a guessed $15,200 standard deduction rather than $16,100 and omitted the $627.91 qualified-overtime deduction." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction fully offsets taxable income, but $35,012.56 of AGI less the $16,100 standard deduction and the $627.91 overtime deduction leaves $18,284.66 taxable and $1,470 of tax. It then invoked the American Opportunity Credit although no qualified tuition expense is listed, so AOTC is $0." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and nonrefundable credits leave no liability, but the correct chain produces $18,284.66 of taxable income and $1,470 of tax, of which only the $120 saver's credit is offsettable. Its AOTC assumption fails because no qualified tuition expense is listed, making the credit $0." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Its derivation is correct through $1,470 of tax before credits, including the $16,100 standard deduction and the $628 overtime deduction. It then concluded no usable nonrefundable credits exist and so omitted the $120 saver's credit — 10% of $1,199 in traditional 401(k), Roth 401(k), traditional IRA and Roth IRA contributions at a $35,012.56 AGI, squarely in the 2026 10% tier." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It zeroed the liability with the nonrefundable portion of an American Opportunity Credit, but no qualified tuition or education expense amount is listed, so AOTC is $0. The only nonrefundable credit is the $120 saver's credit against $1,470 of tax on $18,284.66 of taxable income." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the $16,100 standard deduction and the qualified-overtime deduction correctly but taxed the $1,170 of non-Sch D capital gains at ordinary rates, adding about $140, and it never applied the $120 saver's credit. Those two items are exactly the $260 by which its $1,610 exceeds the reference." -us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It cut AGI to $29,594 by deducting the $6,589 of employer-sponsored insurance premiums, which lie outside the $27,000 of employment income and are not an above-the-line deduction; correct AGI is $35,012.56. That single error shrinks its ordinary base to $7,728 against the true $14,316.66, even though its $16,100 standard deduction, $627.91 overtime deduction, and $119.90 saver's credit were all right." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It asserted the standard deduction plus an AOTC wipes out the liability, but no qualified tuition expense is listed so AOTC is $0. AGI of $35,012.56 less the $16,100 standard deduction and $627.91 overtime deduction leaves $18,284.66 taxable, $1,470 of tax, and $1,350 after the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law — a personal exemption and a restored 15% second bracket — when 2026 keeps the $16,100 single standard deduction, no personal exemption, and 10%/12% brackets, giving an ordinary base of $18,487 against the correct $14,316.66. It also omitted the $627.91 qualified-overtime deduction, though its $120 saver's credit was correct." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used an $8,300 standard deduction plus a $5,300 personal exemption and a 15% second bracket on the assumption TCJA expired; 2026 law provides a $16,100 standard deduction, no exemption, and a 12% second bracket. It also never took the $627.91 deduction generated by the $628 FLSA overtime premium." -us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It combined an $8,350 standard deduction with a $5,330 personal exemption from a TCJA-sunset reading and additionally removed the $6,589 of ESI premiums from wages; 2026 law gives a $16,100 standard deduction, no exemption, and no ESI deduction. It also taxed the $1,170 of non-Sch D gains at ordinary rates, skipped the $627.91 overtime deduction, and applied no saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied an ~$8,300 standard deduction plus a ~$5,200 personal exemption and 10%/15% brackets on a TCJA-sunset assumption, when 2026 provides a $16,100 standard deduction, no exemption, and 10%/12% brackets. It also omitted the $627.91 overtime deduction and computed the saver's credit as $97 instead of 10% of the $1,199 eligible contribution base, or $120." -us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so no computation reached the grader. The required chain is $35,012.56 of AGI, $18,284.66 of taxable income after the $16,100 standard deduction and $627.91 overtime deduction, $1,470 of tax with qualified dividends at 0%, and $1,350 after the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It offset the entire liability with an American Opportunity Credit, but no qualified tuition expense is listed, so AOTC is $0 and nothing offsets tax beyond the $120 saver's credit. The correct result is $1,470 of tax on $18,284.66 of taxable income, less that credit." -us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% saver's credit rate to a $969 contribution base for a $484.50 credit, when a $35,012.56 AGI falls in the 2026 10% tier and the eligible base is $1,199 including the Roth 401(k) and Roth IRA contributions, giving $120. It also omitted the $627.91 qualified-overtime deduction, leaving its ordinary base at $16,115 against the correct $14,316.66." -us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It added the $628 FLSA overtime premium on top of the $27,000 of wages, when that premium is already inside the stated gross wage and instead generates a $627.91 deduction. It then zeroed the liability with an AOTC despite $0 of listed qualified tuition; the correct answer is $1,470 of tax less the $120 saver's credit." -us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It stated taxable income of $40,571 and in the same explanation claimed the single standard deduction reduces taxable income to $0, a self-contradiction. AGI of $35,012.56 less the $16,100 standard deduction and the $627.91 overtime deduction leaves $18,284.66 taxable and $1,470 of tax before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified overtime deduction, so ordinary taxable income came out at $14,945 instead of $14,317. That gives $1,545 − $120 = $1,425 rather than $1,470 − $120 = $1,350. It then submitted $1,276, which does not follow from its own $1,425 working." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified overtime deduction and taxed the $1,170 of capital gains at ordinary rates instead of in the 0% band. That put ordinary income at $16,115 instead of $14,317, adding about $216 of 12%-bracket tax ($1,685.80 − $119.90 instead of $1,470 − $120)." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It claimed a $2,500 American Opportunity Credit even though qualified tuition expenses are $0, which makes the AOTC $0. It also used the old $14,600 standard deduction, did not exclude the $926 traditional 401(k), taxed qualified dividends at 15% instead of 0%, and left out the overtime deduction and the $120 saver's credit. Its $650 does not match its own $490 working." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It taxed the $1,170 of capital gains as ordinary income, left out the $628 overtime deduction, and said no nonrefundable credits apply, so it missed the $120 saver's credit on $1,199 of retirement contributions. Its $1,656 does not even match its own $1,686 tentative tax." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed the $27,000 wages were already net of the $926 traditional 401(k), which overstated AGI at $37,109. It also taxed the $1,170 of gains as ordinary income and left out the $628 overtime deduction. It then subtracted an arbitrary $331 AOTC even though qualified expenses are $0, and it never applied the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $628 overtime deduction (ordinary income $14,945 instead of $14,317) and the $120 saver's credit. It then abandoned its own ~$1,684 bracket result and submitted an unsupported $2,109." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used pre-OBBBA 2026 parameters: a $15,000 standard deduction instead of $16,100 and a $11,925 top for the 10% bracket instead of $12,400. It also left out the $628 qualified overtime deduction, and it never considered the $120 saver's credit on $1,199 of 401(k)/IRA contributions." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It explicitly said the saver's credit does not reduce the tax, and it used a $1,500 nonrefundable AOTC even though qualified education expenses are $0. It then drifted to $1,470, which is the pre-credit regular tax, without subtracting the $120 saver's credit (10% of $1,199) that brings it to $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It left out the 2026 qualified overtime deduction for the $628 FLSA overtime premium. Ordinary taxable income stayed at $14,945 instead of $14,317, which overstates tax by $75.50 ($1,545.40 instead of $1,470 before the $120 saver's credit)." +us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 OBBBA $16,100 and left out the $628 qualified overtime deduction. That inflated ordinary taxable income to $15,645 instead of $14,317, so regular tax came out at $1,634.90 instead of $1,470 before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It subtracted the $1,800 of other health insurance premiums from wages, used a $15,000 standard deduction instead of $16,100, left out the $628 overtime deduction, and used $11,925 as the top of the 10% bracket instead of $12,400. These errors roughly cancel ($14,245 of ordinary income vs. $14,316.66). Its unrounded $1,470.90 − $119.90 gives $1,351 instead of $1,470 − $120 = $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 $16,100, a wrong 10% bracket width, and left out the $628 qualified overtime deduction. That produced $15,645 of ordinary taxable income and $1,632.90 of tax instead of $14,317 and $1,470, before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 $16,100 and left out the $628 qualified overtime deduction. That raised taxable income to $20,883 and regular tax to $1,645.40 instead of $1,470 before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It wiped out the tax with a nonrefundable AOTC even though qualified education expenses are $0, which makes the AOTC $0. It also treated the $6,589 ESI premiums as a pre-tax wage reduction and applied a TCJA-sunset standard deduction plus personal exemption instead of the $16,100 OBBBA standard deduction. As a result, it never reached the $1,470 tax less the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $25,500 is about $7,200 too high: the correct figure is $18,285 after the $16,100 standard deduction and $628 overtime deduction, with only $14,317 taxed at ordinary rates. It also never subtracted the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It cut wages to $20,411 by treating the $6,589 employer-sponsored insurance premiums as pre-tax, but only the $926 traditional 401(k) reduces the $27,000 wages. It also assumed the TCJA sunset (pre-TCJA standard deduction plus personal exemption) instead of the permanent $16,100 OBBBA standard deduction, and it left out the $628 overtime deduction. The $120 saver's credit was the only step it got right." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It treated the $6,589 ESI premiums as a pre-tax exclusion (AGI $29,594 instead of $35,013) and applied an $8,300 standard deduction plus $5,300 exemption for a TCJA sunset that OBBBA prevented. It left out the $628 overtime deduction and never subtracted the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave no computation. Its $1,475 roughly matches the $1,470 pre-credit tax on $14,317 of ordinary income, with no subtraction for the $120 saver's credit (10% of $1,199 in 401(k)/IRA contributions) that brings the answer to $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted the $6,589 of employer-sponsored insurance premiums from wages and applied a personal exemption under an assumed TCJA sunset instead of the $16,100 OBBBA standard deduction plus the $628 overtime deduction. It also never applied the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation computes $1,756 on AGI of $36,183 using only the standard deduction, leaving out the $628 qualified overtime deduction and the $120 saver's credit. It then submitted $1,086, which matches neither its own explanation nor the correct $1,470 − $120." +us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and a $11,925 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 overtime deduction. That gave $1,686.90 before the saver's credit. It then submitted $1,617, which contradicts its own $1,567 result." +us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,200 top for the 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 qualified overtime deduction. That produced $1,633.40 of regular tax instead of $1,470 before the correctly computed $119.90 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It said no nonrefundable credits apply after checking only the AOTC and EITC, which misses the $120 saver's credit on $1,199 of 401(k)/IRA contributions. It also used a $15,200 standard deduction instead of $16,100 and left out the $628 overtime deduction." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the standard deduction and AOTC eliminate the tax. But the $16,100 standard deduction leaves $18,285 of taxable income ($1,470 of tax), and the AOTC is $0 because no qualified education expenses are listed. Only the $120 saver's credit applies, giving $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It used an American Opportunity Credit to zero out the tax, but qualified education expenses are $0, so the AOTC is $0. The $14,317 of ordinary taxable income produces $1,470 of tax, reduced only by the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It got the $1,470 regular tax exactly right but said there were no usable nonrefundable credits. That misses the saver's credit: 10% of the $1,199 in traditional and Roth 401(k)/IRA contributions at AGI of about $35,013 is $120, which lowers the tax to $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It offset the whole tax with a nonrefundable AOTC even though qualified tuition expenses are $0, which makes the AOTC $0. The correct result is $1,470 of regular tax less only the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied the standard and overtime deductions but put only the qualified dividends in the 0% band. Taxing the $1,170 of capital gains at 12% added about $140. It also left out the $120 saver's credit, so it got $1,610 instead of $1,470 − $120 = $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It treated the $6,589 employer-sponsored insurance premiums as a pre-tax wage reduction, cutting AGI to $29,594 and ordinary taxable income to $7,728. Wages are reduced only by the $926 traditional 401(k), which gives $14,317 of ordinary income and $1,470 of tax before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It got the $1,470 regular tax right but computed the saver's credit on only the $969 of traditional contributions ($96.90). It left out the $163 Roth 401(k) and $67 Roth IRA contributions, which also count; the full $1,199 base gives a $120 credit." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It computed the $1,470 regular tax correctly but applied no nonrefundable credits. It missed the saver's credit of 10% × $1,199 in 401(k)/IRA contributions = $120, which lowers the tax to $1,350." +us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages as pre-tax, which cut ordinary taxable income to $7,728 instead of $14,317. It also said no credits were usable, which misses the $120 saver's credit on $1,199 of retirement contributions." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It claimed the AOTC and other nonrefundable credits wipe out the liability, but the AOTC is $0 because qualified education expenses are $0. The $18,285 of taxable income carries $1,470 of tax, and only the $120 saver's credit reduces it." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a personal exemption, a smaller standard deduction and a 15% bracket. OBBBA made the TCJA structure permanent, so the correct rules are a $16,100 standard deduction, no exemption and 10%/12% brackets, plus the $628 overtime deduction. Only its $120 saver's credit is correct." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset law (an $8,300 standard deduction, a $5,300 exemption and a 15% bracket) instead of the OBBBA-permanent $16,100 standard deduction with 10%/12% brackets, and it left out the $628 overtime deduction. That produced $2,017 of regular tax instead of $1,470 before the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct $14,317 of ordinary taxable income and the saver's credit. However, it used $12,200 as the top of the 10% bracket instead of the 2026 $12,400, which overstates regular tax by $4 ($1,474.04 instead of $1,470)." +us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted the $6,589 ESI premiums from wages and applied a TCJA-sunset $8,350 standard deduction plus $5,330 exemption instead of the $16,100 OBBBA standard deduction plus the $628 overtime deduction. It also said no nonrefundable credits apply, which misses the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the pre-TCJA $8,300 standard deduction, a $5,200 exemption and a 15% bracket instead of the OBBBA-permanent $16,100 standard deduction and 10%/12% brackets, and it left out the $628 overtime deduction. It also cut the saver's credit to $97 by leaving out the Roth 401(k) and Roth IRA contributions from the $1,199 base." +us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score against the $1,350 derived from $1,470 of regular tax less the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used the American Opportunity Credit to fully offset the liability, but qualified education expenses are $0, so the AOTC is $0. The $1,470 regular tax is reduced only by the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% saver's credit rate. At AGI of about $35,013, a single filer is in the 10% tier, which gives $120 on the full $1,199 of traditional plus Roth contributions. Instead it took 50% of only $969 of traditional contributions ($484.50). It also left out the $628 overtime deduction and taxed the $1,170 of gains as ordinary income." +us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It invented a $1,960 AOTC to zero out the tax even though qualified education expenses are $0, which makes the AOTC $0. It also added the $628 overtime premium on top of the $27,000 wages, which already include it. The correct result is $1,470 of tax less the $120 saver's credit." +us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed the single standard deduction reduces taxable income to $0, which contradicts its own $40,571 figure. The $16,100 standard deduction and $628 overtime deduction leave $18,285 of taxable income, producing $1,470 of tax less the $120 saver's credit." us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached the correct result — no qualified tuition expenses are listed, so under the prompt's unlisted-inputs-are-zero rule the American Opportunity Credit and its 40% refundable portion are $0 — and it then overrode that derivation to submit $400 as a 'minimal refundable AOTC assumption'. It also mis-stated the EITC bar as the AGI phase-out ceiling while never applying the age-25 floor that independently zeroes the childless EITC for this 22-year-old." us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated the AOTC administrative flags (1098-T, institution EIN, half-time enrollment, credential pursuit) as if they established up to $4,000 of qualified expenses, when tuition is an unlisted input equal to $0, so the credit computed as 100% of the first $2,000 plus 25% of the next $2,000 is $0 and the refundable portion is $0. It then submitted $1,200, exceeding the $1,000 statutory refundable cap it had itself just cited." us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It jumped from AOTC eligibility flags straight to the $2,500 statutory maximum and took 40% of it, never computing the credit base: qualified tuition expenses are $0 here, so the credit and its refundable portion are $0. Its EITC rationale is also wrong on its own numbers — $10,152 of investment income is below the disqualifying limit — and it missed the actual disqualifier, the age-25 floor for childless EITC claimants." @@ -6752,62 +7413,70 @@ us,scenario_091,federal_refundable_credits,kimi-k2.6,parse_contract_failure,miss us,scenario_091,federal_refundable_credits,minimax-m3,llm_error,other,False,"It quoted only the statutory ceiling — 40% of a $2,500 maximum, capped at $1,000 — and submitted that ceiling as the answer, never computing the credit from qualified tuition expenses, which are unlisted and therefore $0, making the American Opportunity Credit and its refundable portion $0." us,scenario_091,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"It verified every AOTC administrative condition (half-time, credential, 1098-T) and then equated eligibility with the maximum credit, taking $1,000 as the 40% refundable portion; qualified tuition expenses are $0, so the credit base is $0 and nothing is refundable. Its EITC analysis correctly found investment income under the cap but rested the zero on the AGI phase-out while missing the age-25 floor that disqualifies a 22-year-old childless filer." us,scenario_091,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model incorrectly placed MAGI of 2.19 times FPL below Wisconsin's Medicaid income threshold for a single adult. It also treated that asserted income threshold as sufficient despite the head qualifying through no Medicaid eligibility category. -us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model computed the two federal components exactly right ($1,674.00 Social Security + $391.50 Medicare on the full $27,000) and then invented a $528.18 ""Wisconsin state payroll tax (employee withholding)"" by applying a 5.85% state income-tax bracket rate to wages. Wisconsin imposes no mandatory employee-side payroll tax — its unemployment insurance is employer-financed and the state has no disability or paid-family-leave employee contribution — so the requested payroll_tax output is federal FICA only; state income-tax withholding belongs in state_income_tax_before_refundable_credits, not here." -us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model shrank the FICA base to $25,200 by subtracting the $1,800 ""health insurance premiums excluding Medicare Part B"" input as a pre-tax cafeteria-plan deduction. That input records premiums paid, not a section 125 salary reduction carved out of the stated wage figure; the payroll tax base is the full $27,000 of gross wages, so the correct components are $1,674.00 and $391.50 rather than $1,562.40 and $365.40." -us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model reduced the FICA base to $20,411 by treating the $6,589 employer-sponsored insurance premium input as a pre-tax employee salary reduction excluded from Social Security and Medicare wages. The $27,000 gross wage figure is the FICA base as given, so applying 7.65% to $27,000 yields $2,065.50, not the $1,561.44 produced by the reduced base." -us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 employer-sponsored insurance premium from wages before applying the 7.65% combined employee rate, converting the ESI premium input into a section 125 exclusion from Social Security and Medicare wages. The stated $27,000 of gross wages is the payroll tax base, giving $1,674.00 + $391.50 = $2,065.50." -us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) contributions in the FICA base but then removed $6,589 of employer-sponsored insurance premiums from it, computing 7.65% on $20,411. The ESI premium input does not reduce the $27,000 of Social Security and Medicare wages, so the tax is 6.2% and 1.45% of $27,000 = $2,065.50." -us,scenario_091,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used the right base ($27,000 of gross wages) but summed the employee rates as 7.5% instead of 6.2% + 1.45% = 7.65%, producing $2,025.00. At the statutory rates the components are $1,674.00 and $391.50 for a total of $2,065.50." -us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model computed Social Security of $1,265.48 and Medicare of $295.96 on a base of $20,411, having subtracted the $6,589 employer-sponsored insurance premium from wages as a pre-tax exclusion. Social Security and Medicare wages here are the full $27,000, so the correct components are $1,674.00 and $391.50." -us,scenario_091,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The submitted value of $1,561.44 is 7.65% of $20,411 — the base left after subtracting the $6,589 employer-sponsored insurance premium — even though the model's own explanation lays out the correct $1,674.00 + $391.50 = $2,065.50 on the full $27,000. The model applied a pre-tax ESI exclusion to Social Security and Medicare wages in the field that was graded, contradicting the derivation it wrote." -us,scenario_091,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The submitted value of $1,561.44 equals 7.65% of $27,000 minus the $6,589 employer-sponsored insurance premium, while the explanation states the correct $1,674.00 + $391.50 = $2,065.50 on the full wage base. The graded number carries a pre-tax ESI exclusion from FICA wages that the reference does not apply and that the model's own text disclaims." -us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identified the base and rules exactly right — 6.2% and 1.45% on the full $27,000, no Additional Medicare Tax, no Wisconsin employee payroll tax — but botched the final addition, reporting $2,063.10 instead of $1,674.00 + $391.50 = $2,065.50. The $2.40 shortfall is a pure arithmetic slip in summing the two correctly identified components." -us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model derived $1,674 of Social Security plus roughly $392 of Medicare on $27,000 and then discarded its own total by ""rounding"" $2,066 up to $2,118 under a fabricated ""annual payroll-tax estimation"" convention. No such adjustment exists; the exact sum of the components it named is $2,065.50." -us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The submitted $2,457 is $1,674 (6.2% of $27,000) plus $783, which is the combined employer-plus-employee Medicare rate of 2.9% on $27,000 rather than the 1.45% employee share the model itself quoted. Doubling the Medicare component pulls in employer-side tax that the question explicitly excludes; the employee-only total is $1,674.00 + $391.50 = $2,065.50." -us,scenario_091,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model explicitly recharacterized the $6,589 employer-sponsored insurance figure as pre-tax employee premiums and applied 7.65% to the resulting $20,411. That input does not reduce Social Security and Medicare wages, which remain the stated $27,000, so the tax is $1,674.00 + $391.50 = $2,065.50; the model's other two calls (401(k) contributions stay FICA-taxable, Wisconsin adds no employee payroll tax) were correct." -us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model set the FICA base to $27,000 − $6,589 = $20,411 by treating the employer-sponsored insurance premium as a pre-tax health exclusion, then rounded the components to whole dollars for $1,561. Social Security and Medicare apply to the full $27,000 of gross wages, giving $1,674.00 + $391.50 = $2,065.50." -us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive computation was submitted to grade. The required derivation is 6.2% of $27,000 = $1,674.00 plus 1.45% of $27,000 = $391.50, totaling $2,065.50." -us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of gross wages to build a $27,628 FICA base, then applied 7.65% for $2,113.54. The prompt states that gross wage and salary amounts are annual totals including any overtime pay, so the $628 premium is already inside the $27,000; the base is $27,000 and the tax is $2,065.50." +us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got FICA right ($1,674.00 + $391.50) but then added a made-up Wisconsin 'state payroll tax' of $528.18. That figure is really state income tax withholding. Wisconsin has no mandatory employee state payroll tax (no SDI or paid-leave contribution), so the total should stay at $2,065.50." +us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $1,800 of other health insurance premiums from wages and taxed only $25,200. Those premiums are paid outside payroll, not withheld pre-tax through a cafeteria plan, so they do not reduce FICA wages. The 7.65% rate applies to the full $27,000." +us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $6,589 of employer-sponsored insurance premiums as a pre-tax employee payroll deduction and taxed only $20,411 at 7.65%. Those premiums do not reduce FICA wages, so the base is the full $27,000 and the tax is $2,065.50." +us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 of employer-sponsored insurance premiums from gross wages and taxed $20,411. The ESI premium input does not reduce the Social Security or Medicare wage base, which stays at $27,000 and gives $2,065.50." +us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept the traditional 401(k) contributions in FICA wages, but it wrongly subtracted the $6,589 of employer-sponsored insurance premiums as a pre-tax deduction. That shrank the base to $20,411 instead of the full $27,000." +us,scenario_091,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used the right base ($27,000) but added the rates wrong: it combined 6.2% and 1.45% as 7.5% instead of 7.65%. That gives $2,025 instead of $2,065.50." +us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model defined Medicare/Social Security wages as $27,000 minus $6,589 of 'pre-tax health insurance premiums' and computed $1,265.48 + $295.96. The employer-sponsored insurance premiums do not reduce FICA wages, so each rate applies to the full $27,000." +us,scenario_091,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model's explanation derives $2,065.50 correctly, but the value it submitted is 1,561.44. That is exactly 7.65% of $20,411, the base left after wrongly subtracting the $6,589 ESI premiums. The submitted value contradicts the explanation and carries the ESI-subtraction error." +us,scenario_091,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model's explanation totals $2,065.50 on $27,000 of wages, but the value it submitted is 1,561.44. That equals 7.65% of $20,411, meaning it subtracted the $6,589 employer-sponsored insurance premiums from the FICA base. The submitted value conflicts with the explanation and contains the base error." +us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct method (6.2% + 1.45% on $27,000, no Additional Medicare Tax, no Wisconsin payroll tax) but got the arithmetic wrong. It reported $2,063.10 instead of $1,674.00 + $391.50 = $2,065.50." +us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model reached about $2,066 using 7.65% on $27,000, then raised it to $2,118 with an invented 'rounded per annual payroll-tax estimation' step. No rule supports that adjustment; the correct total is $2,065.50." +us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model listed the correct components ($1,674 + $391.50) but submitted $2,457. That equals $27,000 × 9.1%, meaning it applied the combined 2.9% Medicare rate (employee plus employer) instead of the 1.45% employee rate. The employee-only Medicare tax is $391.50, for a total of $2,065.50." +us,scenario_091,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model explicitly assumed the employer-sponsored insurance premiums were pre-tax employee premiums and dropped the FICA base to $20,411. That exclusion does not apply: Social Security and Medicare tax apply to the full $27,000 of gross wages, giving $2,065.50." +us,scenario_091,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model treated the $6,589 of employer-sponsored insurance premiums as a pre-tax deduction and taxed $20,411 of 'estimated' Social Security and Medicare wages. The ESI premiums do not reduce FICA wages, so the 7.65% applies to $27,000." +us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model set the FICA base at $27,000 minus $6,589 of 'pre-tax health premiums', or $20,411, and taxed that. The employer-sponsored insurance premiums are not excluded from FICA wages, so the base is $27,000 and the tax is $2,065.50." +us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was nothing to score. The correct answer is 7.65% of $27,000 in wages, or $2,065.50." +us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of the $27,000 gross wages, taxing $27,628. The prompt states that gross wages already include overtime pay, so this counts the overtime twice. The correct base is $27,000, giving $2,065.50." us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Built AGI of $36,183 by adding the $1,170 non-Schedule-D distribution to the $35,012.56 wage/interest/dividend base, then derived the sliding-scale standard deduction from a $12,760 base phasing out from $18,400, producing $10,668 instead of the correct $12,067.66 at this AGI. Its own arithmetic yielded $975, and it submitted $1,257 with no computation behind the $282 increase." -us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Omitted the $700 Wisconsin personal exemption deduction entirely and taxed AGI of $36,183 that includes the $1,170 non-Schedule-D distribution, so its taxable income of $24,184 exceeded the correct $22,244.90 by $1,939. It also stopped the 3.5% band at $14,680, the 2025 threshold, rather than the 2026 figure near $15,013." -us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Dropped $10,152 of interest, dividends, and capital gains from income and substituted the federal $14,600 standard deduction for Wisconsin's sliding-scale deduction, then zeroed the remainder with the federal American Opportunity Credit, which does not offset Wisconsin income tax and which Wisconsin does not mirror. The correct base is $35,012.56 of AGI less $12,067.66 and $700, leaving $22,244.90 taxed at 3.5%/4.4%." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Guessed the phased Wisconsin standard deduction at ""approx $9,500"" when the 12% sliding scale gives $12,067.66 at this AGI, and kept the $1,170 non-Schedule-D distribution in the base, inflating taxable income to $25,983 against the correct $22,244.90. It then submitted $1,140 after computing $1,014, adding $126 with no stated basis." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Put the Wisconsin standard deduction at ""approx 7,700"" — $4,368 below the $12,067.66 the sliding scale produces at $35,012.56 of AGI — giving taxable income of $27,783 versus $22,244.90. It then added $70 to its own $1,093 result for unspecified ""itemized/school credit adjustments,"" none of which apply without property tax, rent, or excess federal itemized deductions." -us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Estimated the phased-down Wisconsin standard deduction at ~$9,000 rather than $12,067.66 and retained the $1,170 non-Schedule-D distribution in AGI, so its ~$26,500 of taxable income exceeded the correct $22,244.90 by more than $4,000. Nothing else in its schedule was wrong: the 3.5%/4.4% rates and $700 exemption it used are correct." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Used a $15,000 phase-out start for the single standard deduction when Wisconsin's sliding scale begins near $20,076 in 2026, and settled on unindexed 2024 parameters ($12,230 base, $13,810 bracket top), yielding a $9,688 deduction instead of $12,067.66 and taxable income of $25,795 against $22,244.90. It also carried the $1,170 non-Schedule-D distribution into the $36,183 base." -us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Set the phased standard deduction at ""roughly $11,340"" rather than $12,067.66 and taxed the $36,183 base that includes the $1,170 non-Schedule-D distribution. It then shaved its computed $933 to $900 with a ""Married/Single credit and standard personal exemption credit"" that does not exist for a single Wisconsin filer — the $700 exemption is a deduction, which it had already taken." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Landed within $3 on offsetting errors: it used a flat $13,450 standard deduction where the sliding scale gives $12,067.66, and taxed $36,183 including the $1,170 non-Schedule-D distribution, then capped the 3.5% band at the 2024 threshold of $14,320 instead of the 2026 figure near $15,013. Its stated taxable income of $22,033 is $212 below the correct $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied Wisconsin's repealed 3.54%/4.65% rate schedule instead of the current 3.50%/4.40% brackets, and took a flat $13,230 standard deduction with no sliding-scale phase-down and no $700 exemption. It also subtracted the $1,800 health-insurance premiums from AGI, which is not an above-the-line adjustment for an employee with employer coverage, reaching $34,383 instead of $35,012.56." -us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Used a single flat $14,050 deduction that both overstates the $12,067.66 sliding-scale standard deduction and stands in for the separate $700 exemption, applied to a $36,183 base that includes the $1,170 non-Schedule-D distribution. It also set the 3.5% band top at $15,200 rather than about $15,013, so its $22,133 taxable income missed $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $6,589 employer-sponsored insurance premium from income to reach AGI of $29,594, though the $27,000 gross wage figure is already the taxable wage base, understating AGI by $5,419. It compounded this with an $11,410 deduction instead of $12,067.66 and the 2024 $14,320 bracket top, leaving taxable income of $17,484 against $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its explanation computes tax on ""$27,000 adjusted gross income,"" omitting the $4,806 of interest and $4,176 of dividends that bring the reference AGI to $35,012.56. The submitted $905 matches no derivation from $27,000 either: that base carries a $13,029 sliding-scale deduction and the $700 exemption, leaving about $13,271 taxable and roughly $465 of tax." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Submitted a bare number with no figures; $588 corresponds to Wisconsin taxable income near $16,434, some $5,811 below the correct $22,244.90. That level is what the sliding-scale deduction and $700 exemption produce after roughly $6,500 of the income base is removed — the size of the employer-sponsored insurance premium that stays in wages." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Removed the $6,589 employer-sponsored insurance premium to get AGI of $29,594, omitted the $700 personal exemption, and stopped the 3.5% band at the 2024 threshold of $14,320. Its $16,887 of taxable income fell $5,358 short of the correct $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $1,177 corresponds to taxable income near $29,820, about $7,575 above the correct $22,244.90. That is what taxing the income base with essentially no Wisconsin standard deduction produces, when the sliding scale allows $12,067.66 at $35,012.56 of AGI plus the $700 exemption." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Claimed a 60% long-term capital-gain exclusion when Wisconsin's exclusion is 30% and reaches 60% only for qualifying farm assets, and claimed a ""medical expense credit"" Wisconsin does not offer — its itemized-deduction credit is 5% of federal itemized deductions exceeding the state standard deduction and is $0 here. The resulting $543.09 implies taxable income near $15,413 against the correct $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Submitted $944 while its explanation states ""value = 1018,"" and neither figure matches the schedule: $944 implies taxable income near $24,525, about $2,280 above the correct $22,244.90. That gap is exactly what carrying the $1,170 non-Schedule-D distribution into AGI and understating the $12,067.66 sliding-scale deduction produces." -us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Gave no figures; $1,113 implies taxable income near $28,366, roughly $6,121 above the correct $22,244.90, meaning it allowed only about $6,650 of deductions against the $35,012.56 AGI. The correct offsets are the $12,067.66 sliding-scale standard deduction plus the $700 personal exemption." -us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Asserted the Wisconsin single standard deduction is ""$0 due to phaseout"" at $36,183 of AGI; the 12% sliding scale from about $20,076 does not reach zero until roughly $132,000 and leaves $12,067.66 here. It also omitted the $700 exemption and invented a 5.3% third bracket beginning at $30,106, a rate that applies only above roughly $300,000." -us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Taxed the entire $36,183 on the claim that the standard deduction is ""fully phased out at this income""; Wisconsin reduces the deduction by 12% of AGI above about $20,076, leaving $12,067.66 at this AGI. The separate $700 personal exemption also went untaken, so its base exceeded the correct $22,244.90 by nearly $14,000." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted the income is too low to produce Wisconsin tax; after the $12,067.66 sliding-scale standard deduction and the $700 exemption, $22,244.90 of the $35,012.56 AGI remains taxable at 3.5%/4.4%. Wisconsin tax reaches zero only when AGI falls below roughly $14,600, and no nonrefundable state credit applies without property tax, rent, or excess itemized deductions." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Applied the federal American Opportunity Credit against Wisconsin tax; Wisconsin has no conforming education credit and offers no nonrefundable credit on these facts. The $35,012.56 AGI less the $12,067.66 standard deduction and $700 exemption leaves $22,244.90 taxable, producing $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Cut the base to $33,404 with three subtractions that do not belong: the $628 FLSA overtime premium (a federal below-the-line deduction that never reduces AGI and that Wisconsin does not follow), the $1,800 health-insurance premiums, and a 30% capital-gain exclusion, against the reference base of $35,012.56. Its $12,513.92 standard deduction also exceeded the $12,067.66 the sliding scale produces." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Gave no figures; $943 implies taxable income near $24,502, about $2,257 above the correct $22,244.90. That gap is the $1,170 non-Schedule-D distribution carried into AGI plus roughly $1,085 of understated standard deduction and exemption against the $12,067.66 plus $700 actually allowed." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Applied a 30% Wisconsin capital-gain subtraction to the $1,170 non-Schedule-D distribution, which sits outside the $35,012.56 AGI base entirely, so only $351 came out instead of the full amount. Its taxable income of about $22,320 exceeded the correct $22,244.90, overshooting the tax by $3.34." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Gave no figures; $860 implies taxable income near $22,616, $371 above the correct $22,244.90. That residual comes from keeping part of the $1,170 non-Schedule-D distribution in the base and approximating rather than computing the $12,067.66 sliding-scale standard deduction." -us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Stacked three subtractions the schedule does not allow — the $6,589 employer-sponsored insurance premium, the $1,800 health-insurance premiums, and a 30% capital-gain exclusion — reaching a $27,443 base against the reference $35,012.56. Paired with a $13,028.04 deduction rather than $12,067.66, that left $13,714.96 taxable entirely inside the 3.5% bracket, when $7,232 of the correct base belongs at 4.4%." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Claimed the standard deduction and credits fully offset the tax; the sliding scale gives $12,067.66 at $35,012.56 of AGI, which with the $700 exemption still leaves $22,244.90 taxable. No Wisconsin nonrefundable credit applies here — there is no property tax, no rent, and no federal itemized deduction exceeding the state standard deduction." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Started from $36,183 including the $1,170 non-Schedule-D distribution and applied an ""approximate 2026-inflated"" standard deduction of about $11,046 instead of the $12,067.66 the sliding scale produces, so its ~$24,437 taxable income exceeded the correct $22,244.90 by $2,192. Its rates and $700 exemption were right; only the deduction and base were not." -us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Excluded only 30% of the $1,170 capital gain rather than working from the $35,012.56 base that leaves it out entirely, set the sliding-scale deduction at $10,618 instead of $12,067.66, and stopped the 3.5% band at $14,320 (the 2024 threshold, not the 2026 figure near $15,013). Taxable income came to $24,514 against $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the $6,589 employer-sponsored premium to reach AGI of $29,594 and omitted the $700 personal exemption, then used a $14,035 deduction base with a $19,160 phase-out start and a $14,035 top for the 3.5% band. Its $16,811 of taxable income fell $5,434 below the correct $22,244.90." -us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Estimated the phased standard deduction at $10,900–$11,000 rather than $12,067.66 and taxed the $36,183 base including the $1,170 non-Schedule-D distribution, giving about $24,500 of taxable income against $22,244.90. It also applied a 4.65% second-bracket rate that Wisconsin replaced with 4.40% in 2023." -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for this variable, so no Wisconsin computation was submitted. The required derivation is $35,012.56 of AGI less the $12,067.66 sliding-scale standard deduction and the $700 personal exemption, with $22,244.90 taxed at 3.5% to about $15,013 and 4.4% above, for $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Applied Wisconsin's 30% exclusion to the $1,170 non-Schedule-D distribution instead of working from the $35,012.56 base that omits it, and set the phased deduction at $11,868.47 versus $12,067.66, so taxable income of $23,263.53 exceeded $22,244.90 by $1,018.63. It also placed the 3.5% band top at $15,163.62 rather than about $15,013." -us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Zeroed the tax with the Wisconsin School Property Tax Credit, which is 12% of property taxes paid or of rent constituting property taxes and caps at $300. No rent or property tax is listed and all unlisted numeric inputs are 0, so the credit is $0 and the $22,244.90 of taxable income stands at $843.66." -us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Set the standard-deduction phase-out to start near $27,800 instead of roughly $20,076 and used a $14,480 base, producing $13,480 rather than $12,067.66, and omitted the $700 exemption altogether. Those errors partly cancelled, leaving taxable income of about $22,700 against the correct $22,244.90 on a base that still included the $1,170 non-Schedule-D distribution." -us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Subtracted half the employer-sponsored premium ($3,294.50) from wages as an AGI adjustment and applied a 30% capital-gain exclusion, then substituted federal-style flat standard deductions ($12,760–$14,600) for Wisconsin's sliding scale, which gives $12,067.66 here. It also used the repealed 3.54%/4.65% rates and an 11% ""personal exemption credit"" that does not exist — the $700 exemption is a deduction in Wisconsin." -us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Started from a $40,571 income figure that exceeds the household's $37,152 of listed gross income, took a flat $12,200 deduction with no sliding-scale computation and no $700 exemption, then erased its own $1,374.94 with unnamed nonrefundable credits. Wisconsin provides none on these facts: no property tax, no rent, and no federal itemized deductions exceeding the state standard deduction." +us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It put the $1,170 capital gain distributions into WI income (net of a 30% exclusion). It estimated the sliding standard deduction at ~$10,668 from an old $12,760 base, versus the reference's $12,067.66, and never subtracted the $700 personal exemption. Its own arithmetic came to ~$975, but it submitted $1,257, which nothing in its derivation supports." +us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $1,170 capital gain distributions in WI AGI ($36,183 vs $35,012.56) and left out the $700 personal exemption, which gave taxable income of ~$24,184 instead of $22,244.90. It also used the 2025 first-bracket width of $14,680 rather than the 2026 figure of ≈$15,010." +us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It left all $8,982 of interest and dividends out of income and used the federal $14,600 standard deduction instead of WI's sliding-scale deduction and $700 exemption. It then zeroed the tax by applying the federal American Opportunity Credit to Wisconsin liability. Wisconsin has no AOTC counterpart and no qualified tuition is listed, so the $843.66 on $22,244.90 of taxable income has no nonrefundable offset." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It guessed the phased standard deduction at ~$9,500, far below the $12,067.66 the sliding scale gives at $35,012.56 of WI income. It also included the $1,170 capital gain distributions, so taxable income came to ~$25,983 instead of $22,244.90. It then raised its own computed $1,014 to $1,140 without giving a reason." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It estimated the phased standard deduction at only ~$7,700 (reference $12,067.66) and included the $1,170 capital gain distributions, giving taxable income of ~$27,783 versus $22,244.90. It then added an unexplained ~$70 to its computed $1,093 to reach $1,163." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put the phased-down standard deduction at ~$9,000 instead of $12,067.66 and started from $36,183, which includes the $1,170 capital gain distributions. That made taxable income ~$26,500 rather than $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of the $1,170 non-Schedule-D capital gain distributions in WI income after a 30% exclusion, but the reference leaves those distributions out of gross income entirely. That raised WI income to $35,832 and taxable income to ~$23,129 versus $22,244.90, putting it ~$38 too high." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It carried 2024-style parameters into 2026: a $12,230 standard-deduction base phased down from $15,000, which left only $9,688 versus the reference's $12,067.66, and a $13,810 first bracket instead of ≈$15,010. With the $1,170 capital gain distributions also included, taxable income came to $25,795 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions, underestimated the sliding standard deduction at ~$11,340 (reference $12,067.66), and used a $14,320 first bracket, reaching $933. It then subtracted a made-up 'Married/Single credit' and 'personal exemption credit' to get $900. Wisconsin's $700 exemption is a deduction it had already taken, and no nonrefundable credit applies." +us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its ~$23.9k of taxable income ($36.2k minus $11.6k minus $0.7k) counts the $1,170 capital gain distributions as income and uses a standard deduction ~$470 below the reference's $12,067.66. Together these overstated taxable income by ~$1,650 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $13,450 standard deduction without the 12% sliding-scale phase-down that cuts it to $12,067.66 at this income, and it included the $1,170 capital gain distributions in AGI. It also used a $14,320 first bracket instead of ≈$15,010. These errors partly cancel, leaving $840.57 instead of $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-2023 rates of 3.54% and 4.65% instead of Wisconsin's 3.5% and 4.4%. It also took an unphased $13,230 standard deduction, skipped the $700 exemption, and started from an unexplained $34,383 AGI instead of $35,012.56." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used an unphased $14,050 standard deduction and left out the $700 personal exemption; the reference applies the sliding-scale $12,067.66 plus $700. It also included the $1,170 capital gain distributions and set the first bracket at $15,200. These errors partly cancel, leaving $837.05." +us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions, phased the standard deduction down to only $9,963.40 (reference $12,067.66), and never subtracted the $700 personal exemption. That gave taxable income of $26,219.60 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages as if they were pre-tax, cutting AGI to $29,594. The reference reduces wages only by the $926.16 401(k) deferral, for WI income of $35,012.56. The understated income left taxable income at $17,484 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It based WI tax on $27,000 of wages alone. That skips the 401(k) exclusion, the $4,806 of interest, and the $4,176 of dividends that make WI income $35,012.56. It showed no standard deduction, exemption, or bracket calculation behind $905." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It showed no figures. $588 corresponds to about $16,300–16,450 of taxable income, which matches the pattern of subtracting the $6,589 employer insurance premiums from wages (AGI ≈ $29,594). The reference instead uses $35,012.56 of WI income and $22,244.90 of taxable income." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It removed the $6,589 employer-sponsored insurance premiums from income, giving AGI of $29,594 instead of $35,012.56, and left out the $700 personal exemption. Taxable income came to $16,887 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $1,177 implies roughly $29,700 of taxable income, which means it taxed ~$36k of income with little of the $12,067.66 sliding standard deduction and none of the $700 exemption." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied a 60% capital gain exclusion; Wisconsin's is 30%, and the reference keeps the $1,170 non-Schedule-D distributions out of gross income altogether. It also subtracted a medical expense credit that Wisconsin does not give on these facts, pulling the tax down to $543.09 instead of $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation, and its explanation says 1,018 while it submitted 944. Both figures are above $843.66, which fits counting the $1,170 capital gain distributions and understating the $12,067.66 sliding standard deduction." +us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $1,113 implies about $28,300 of taxable income, so it applied far less than the $12,067.66 sliding standard deduction and $700 exemption against WI income of $35,012.56." +us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It treated the WI standard deduction as fully phased out at $36,183, but the sliding scale still allows $12,067.66 at $35,012.56 of income, and it also skipped the $700 exemption. It then taxed the whole AGI, including the $1,170 capital gain distributions, reaching a 5.3% bracket that $22,244.90 of taxable income never touches." +us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the full $36,183 as though the standard deduction were fully phased out, and left out the $700 exemption. At $35,012.56 of WI income the sliding scale still gives $12,067.66, leaving $22,244.90 of taxable income." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the tax was eliminated without computing it. WI income of $35,012.56 minus the $12,067.66 standard deduction and $700 exemption leaves $22,244.90 taxable, which produces $843.66, and no nonrefundable credit offsets it." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no state AOTC and no qualified tuition is listed, so nothing offsets the $843.66 computed on $22,244.90 of taxable income." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 FLSA overtime premium and the $1,800 of other health insurance premiums from WI income, and applied a 30% exclusion to the capital gain distributions, arriving at $33,404. The reference makes neither the overtime nor the premium subtraction and leaves the non-Schedule-D distributions out entirely, giving $35,012.56. The understated income pulled taxable income down to ~$20,190 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $943 implies ~$24,500 of taxable income, about $2,250 above the reference's $22,244.90, which fits counting the $1,170 capital gain distributions and underestimating the $12,067.66 sliding standard deduction." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It kept the $1,170 non-Schedule-D capital gain distributions in WI income net of a 30% exclusion, while the reference leaves them out of gross income. Combined with its approximate 2026 deduction and bracket figures, this landed $3.34 above $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $860 implies ~$22,600 of taxable income, about $370 above the reference's $22,244.90, which fits including part of the $1,170 capital gain distributions that the reference leaves out of WI income." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages plus another $1,800 health-insurance deduction, cutting WI income to $27,443 versus $35,012.56. Taxable income fell to $13,714.96, all of it taxed at 3.5%." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It started from $36,183, which includes the $1,170 capital gain distributions the reference leaves out. Its deduction estimate left roughly $24,300 of taxable income against $22,244.90, overstating the tax by ~$92." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $36,183 of income, including the $1,170 capital gain distributions the reference leaves out, and its sliding standard deduction left roughly $23,900 taxable versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,589 employer-sponsored insurance premiums from wages plus another $1,800 insurance deduction, cutting WI income to $27,443 versus $35,012.56. That left only $13,769.76 of taxable income, all taxed at 3.5%, instead of $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the standard deduction and credits fully offset the tax. The $12,067.66 sliding standard deduction and $700 exemption still leave $22,244.90 taxable, and no WI nonrefundable credit applies because no rent, property tax, or other credit inputs are listed." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,170 capital gain distributions (AGI $36,183) and estimated the standard deduction at ~$11,046 instead of $12,067.66, leaving ~$24,437 taxable versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of capital gain distributions after a 30% exclusion (WI income $35,832 vs $35,012.56) and underestimated the sliding standard deduction at ~$10,618 (reference $12,067.66). It also used the $14,320 first bracket instead of ≈$15,010, overstating taxable income by ~$2,270." +us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium to get federal AGI of $35,555, but the federal overtime deduction is taken below AGI and does not reduce AGI or WI income. It also kept $819 of the capital gain distributions after a 30% exclusion, giving WI income of $35,204 versus $35,012.56, and its parameter estimates account for the rest of the $15 overshoot." +us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It started from $29,594, which removes the $6,589 employer-sponsored insurance premiums from wages, and left out the $700 personal exemption. Taxable income came to $16,811 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used an old 4.65% rate for the second bracket; Wisconsin's second rate is 4.4%. It also included the $1,170 capital gain distributions and estimated the standard deduction at ~$10,900–11,000 versus $12,067.66, leaving taxable income at ~$24,500 versus $22,244.90." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $843.66." +us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It kept $819 of the $1,170 capital gain distributions after a 30% exclusion (WI income $35,832 vs $35,012.56), which also shrank its phased standard deduction to $11,868.47 versus $12,067.66. That produced taxable income of $23,263.53 versus $22,244.90, and it also put the first bracket at $15,163.62 rather than ≈$15,010." +us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It offset the tax with the Wisconsin School Property Tax Credit, which requires property taxes or rent paid. None are listed, so the credit is zero and the $843.66 on $22,244.90 of taxable income stands." +us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It started the standard-deduction phase-down at ~$27,800, which is too high, giving ~$13,480 versus the reference's $12,067.66. It then left out the $700 exemption and included the $1,170 capital gain distributions. These errors partly cancel, leaving taxable income of ~$22,700 and tax of $863." +us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted half of the $6,589 ESI premiums from wages as an 'employer portion' and used the pre-2023 rates of 3.54% and 4.65% instead of 3.5% and 4.4%. It also took an unphased standard deduction and turned the $700 personal exemption into a $77 credit. It then averaged arbitrary deduction guesses to reach $720.73." +us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It overstated WI income at $40,571 (reference $35,012.56) and used an unphased $12,200 standard deduction. It then claimed unspecified nonrefundable credits wiped out its $1,374.94 of tax. No WI nonrefundable credit applies, so the tax on $22,244.90 of taxable income is $843.66." us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income." us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction." @@ -6820,26 +7489,29 @@ us,scenario_092,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit us,scenario_092,head_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an approximately $2,829 monthly SSI-related income limit that does not establish eligibility for this Alabama case, then compared income after a $20 disregard to that incorrect standard. The person receives no SSI and qualifies through no aged/disabled category, so that threshold calculation cannot produce Medicaid eligibility." us,scenario_092,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly recognized that the household has no wages, then improperly invented an “implied Social Security-taxable earnings equivalent” equal to the $17,400 Social Security retirement benefit. Retirement benefits are not employee payroll-taxable earnings, so applying the combined 7.65% FICA rate to that amount produced the erroneous $1,253.76 instead of $0." us,scenario_092,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_092,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Applied Alabama's Social Security exclusion but treated the $14,980 taxable private pension as fully includable in Alabama AGI, missing Alabama's exemption of defined-benefit pension payments under Ala. Code § 40-18-19, which removes that pension from Alabama gross income entirely. Its $484 is the bracket tax on $10,480 ($14,980 less the $3,000 standard deduction and $1,500 personal exemption), a deduction stack that is never reached because Alabama AGI is $0." -us,scenario_092,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Carried the entire $14,980 private pension into Alabama AGI, missing Alabama's full exemption for defined-benefit pension payments (Ala. Code § 40-18-19) that zeroes Alabama taxable income. It then compounded the error with a $1,500 'aged 65+ exemption' that does not exist in Alabama law, which allows only the $1,500 single personal exemption plus the standard deduction, arriving at $409 on a phantom $8,980 base." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Gave no derivation, but $639 inverts to an Alabama taxable income of $13,580 (2% of $500 + 4% of $2,500 + 5% of $10,580), meaning it taxed the full $14,980 pension after roughly $1,400 of subtractions. The correct derivation exempts that pension as a defined-benefit plan payment under Ala. Code § 40-18-19 and the Social Security alongside it, leaving $0 of Alabama taxable income and $0 tax." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Set Alabama AGI equal to the $14,980 federal pension amount, missing Alabama's exclusion of defined-benefit private pension income under Ala. Code § 40-18-19 which drops that income from the Alabama return before any deduction. Its itemized-deduction arithmetic ($9,028 of mortgage interest, real estate taxes, and charitable gifts, less the $1,500 personal exemption, taxed at 2%/4%/5%) is applied to a base that is $0, so the $183 has nothing to tax." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Treated the $14,980 pension as Alabama AGI, missing the Ala. Code § 40-18-19 exemption for defined-benefit pension payments that leaves Alabama taxable income at $0. It also claimed $2,500 of 'exemptions' against a single filer whose only Alabama personal exemption is $1,500, yielding $434 on a $9,480 base that does not exist." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Stopped at the Social Security exclusion and declared the $14,980 pension 'Alabama gross income,' missing Alabama's separate and complete exemption of defined-benefit private pension payments under Ala. Code § 40-18-19. Because that income never enters the Alabama return, the $9,028 of itemized deductions and $1,500 personal exemption it subtracted apply to a $0 base and the 2%/4%/5% brackets it ran produce $0, not $182.60." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Identified only the Social Security exclusion and taxed the $14,980 pension in full, missing Alabama's exemption of defined-benefit pension income (Ala. Code § 40-18-19) that removes it from Alabama AGI. Its $4,452 taxable base — pension less $9,028 itemized deductions and the $1,500 personal exemption — is computed from income Alabama does not tax, so the correct bracket result is $0 rather than $182.60." -us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Explicitly set 'Alabama AGI of $14,980 from private pension income,' which is the exact step Alabama forbids: defined-benefit pension payments are exempt under Ala. Code § 40-18-19, so Alabama AGI here is $0. The $3,000 standard deduction and $1,500 personal exemption it then applied to reach $10,480 and $484 operate on income that never enters the Alabama return." -us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Applied only the Social Security exclusion and taxed the $14,980 pension, missing Alabama's exemption of defined-benefit pension payments under Ala. Code § 40-18-19 that zeroes the Alabama base. It further invented a '$1,000 age 65+ exemption' on top of the $1,500 personal exemption; Alabama grants no additional age exemption, so its $9,480 taxable income and $434 tax rest on two separate fabrications." -us,scenario_092,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Taxed the full $14,980 pension in Alabama, missing the Ala. Code § 40-18-19 exemption for defined-benefit pension income that leaves $0 of Alabama taxable income. It also used a 1.5% bottom bracket where Alabama's first $500 is taxed at 2%, and its own components ($7.50 + $100 + $374) sum to $481.50 while it submitted $731.50." -us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Applied the Social Security exclusion but 'estimated' a bracket liability on the $14,980 pension, missing Alabama's full exemption of defined-benefit pension payments (Ala. Code § 40-18-19) that removes the only remaining income from the Alabama return. Its $414 inverts to an Alabama taxable income of $9,080 and rests on an 'age-65 exemption' Alabama does not provide; with the pension exempt, the tax is $0." -us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Started Alabama from federal AGI of $14,980 and stopped adjusting after Social Security, missing Alabama's exemption of private defined-benefit pension payments under Ala. Code § 40-18-19, which is a subtraction from Alabama gross income, not an itemized deduction. Its $9,028 of itemized deductions and $1,500 exemption produce $182.60 only because $14,980 of exempt pension was left in the base." -us,scenario_092,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Kept the $14,980 pension in Alabama AGI, missing Alabama's defined-benefit pension exemption (Ala. Code § 40-18-19) that reduces the Alabama base to $0. It then subtracted a '$1,500 additional age-65 exemption' that Alabama does not allow — only the single $1,500 personal exemption exists — driving its $2,952 base and $108 result." -us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Its stated method — 'federal AGI with state adjustments on the head's retirement income after deductions' — kept the $14,980 pension in the Alabama base, and its $182 matches the itemized path ($14,980 less $9,028 of deductions and the $1,500 personal exemption, taxed at 2%/4%/5%). Alabama exempts defined-benefit private pension payments under Ala. Code § 40-18-19 alongside Social Security, so the Alabama base is $0 and no bracket applies." -us,scenario_092,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Excluded Social Security but treated the $14,980 pension as Alabama-taxable, missing the Ala. Code § 40-18-19 exemption for defined-benefit pension payments that removes it from Alabama gross income. Its $4,452 taxable income built from $9,028 of itemized deductions and the $1,500 personal exemption is computed on income Alabama does not tax, so $183 replaces a $0 liability." -us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_income_tax_before_refundable_credits, so the key is absent from its outputs object rather than wrong. No Alabama computation was returned to evaluate against the $0 liability that follows from Alabama exempting both Social Security and defined-benefit private pension income under Ala. Code § 40-18-19." -us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Taxed the $14,980 pension in Alabama after excluding only Social Security, missing Alabama's exemption of defined-benefit pension payments under Ala. Code § 40-18-19 that leaves $0 of Alabama taxable income. It also used a '$2,500 single standard deduction' where Alabama's maximum single standard deduction is $3,000, producing a $10,980 base and $509 of tax on income the state does not reach." -us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Used a federal AGI of $29,770, which embeds 85% of the $17,400 Social Security ($14,790) even though provisional income of $23,680 is below the $25,000 single base amount so none of it is federally taxable, and Alabama excludes Social Security outright. It then taxed the pension too — Alabama exempts defined-benefit pension payments under Ala. Code § 40-18-19 — and applied a '$4,000 personal exemption' plus a '$1,000 age 65+' amount against a state whose single personal exemption is $1,500 and which grants no age exemption, inflating the answer to $1,199 against a $0 liability." -us,scenario_092,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Taxed the $14,980 private pension in Alabama, missing the Ala. Code § 40-18-19 exemption for defined-benefit pension payments that zeroes the Alabama base. It also phased the single standard deduction down to $1,750 at $14,980 of AGI when Alabama allows the full $3,000 at that income level, and applied a 4.95% top rate where Alabama's top marginal rate is 5%." -us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Left the $14,980 pension in the Alabama base, missing Alabama's exemption of defined-benefit private pension income under Ala. Code § 40-18-19, and invented a '$4,200' Alabama standard deduction plus a '$1,500 additional standard deduction for age 65+' where Alabama's single standard deduction caps at $3,000 with no age add-on. It then abandoned its own $349 bracket result and added an 'adjustment' for the Alabama federal income tax deduction to reach $852.45, even though federal tax here is $0 (AGI $14,980 is below the single age-65 standard deduction) and that deduction reduces rather than increases Alabama tax." +us,scenario_092,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It set Alabama AGI at the full $14,980 pension and missed the $6,000 retirement income exclusion for a filer aged 65 or older. It also took the $3,000 standard deduction even though itemized deductions of $9,028 are larger. With the exclusion and itemizing, taxable income is zero, but it taxed $10,480 and got $484." +us,scenario_092,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It itemized the $9,028 correctly, but it assumed the private pension is fully taxable and never applied Alabama's $6,000 retirement income exclusion for filers aged 65 or older. That exclusion cuts AGI to $8,980, which is below the itemized total. It also subtracted a $1,000 age-65 exemption that Alabama does not have, which gave it $3,452 taxable and $132.60." +us,scenario_092,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from the full $14,980 pension and missed the $6,000 retirement income exclusion for filers aged 65 or older. It took the $3,000 standard deduction instead of the larger $9,028 in itemized deductions and invented a $1,500 aged exemption. That left $8,980 taxable and $409, instead of zero taxable income." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its only reasoning is that it applied the standard deduction. Its $639 matches roughly $13,580 of taxable income at Alabama's brackets. That means it taxed nearly the whole pension, ignoring both the $6,000 retirement exclusion for filers aged 65 or older and the $9,028 in itemized deductions, which together reduce taxable income to zero." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It correctly itemized $9,028, but it kept Alabama AGI at $14,980 and missed the $6,000 retirement income exclusion for filers aged 65 or older. That exclusion cuts AGI to $8,980, below the itemized total, so taxable income is zero rather than $4,452 ($183)." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It taxed the full $14,980 pension without the $6,000 retirement income exclusion for filers aged 65 or older. It used the $3,000 standard deduction instead of the $9,028 in itemized deductions and added an age exemption that does not exist ($2,500 total exemptions), which gave $9,480 taxable and $434 instead of zero." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It itemized the $9,028 correctly, but it treated all $14,980 of pension income as Alabama gross income and missed the $6,000 retirement income exclusion for filers aged 65 or older. With AGI at $8,980, the itemized deductions wipe out taxable income, so the tax is $0, not $182.60." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It deducted the $9,028 in itemized deductions from the full $14,980 pension and never applied Alabama's $6,000 retirement income exclusion for filers aged 65 or older. That exclusion brings AGI to $8,980, below the itemized total, so taxable income is zero rather than $4,452." +us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It set Alabama AGI at the full $14,980 pension, missing the $6,000 retirement income exclusion for filers aged 65 or older. It also took the $3,000 standard deduction instead of the larger $9,028 in itemized deductions, so it taxed $10,480 ($484) instead of zero." +us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It missed the $6,000 retirement income exclusion for filers aged 65 or older and kept Alabama AGI at $14,980. It took the $3,000 standard deduction even though itemized deductions total $9,028, and it invented a $1,000 age-65 exemption. That gave $9,480 taxable and $434 instead of zero." +us,scenario_092,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It taxed the full $14,980 pension with the $3,000 standard deduction, ignoring the $6,000 retirement income exclusion for filers aged 65 or older and the $9,028 in itemized deductions, which together make taxable income zero. It also used a 1.5% first-bracket rate instead of 2%. Its own components ($7.50 + $100 + $374) add to $481.50, not the $731.50 it submitted." +us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It reduced the full $14,980 pension by a standard deduction, the personal exemption and an age-65 exemption that Alabama does not have. Its $414 implies about $9,080 of taxable income. It never applied the $6,000 retirement income exclusion for filers aged 65 or older and never itemized the $9,028, which together bring taxable income to zero." +us,scenario_092,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used a $2,000 standard deduction and $2,500 of personal and age exemptions against the full $14,980 pension. It missed the $6,000 retirement income exclusion for filers aged 65 or older and the $9,028 in itemized deductions, which reduce taxable income to zero. Its own figures give $10,480 taxable and $484." +us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It itemized $9,028 correctly, but it started from federal AGI of $14,980 and did not subtract Alabama's $6,000 retirement income exclusion for filers aged 65 or older. With that exclusion, AGI of $8,980 is fully offset by the itemized deductions, so the tax is $0, not $183." +us,scenario_092,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized $9,028 correctly, but it left the pension fully taxable at $14,980 and missed Alabama's $6,000 retirement income exclusion for filers aged 65 or older. It also subtracted a $1,500 age-65 exemption that Alabama does not have. That gave $2,952 taxable and $108, when the exclusion plus itemizing leaves zero taxable income." +us,scenario_092,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It correctly itemized $9,028 and applied the medical and employee-expense floors correctly. It still set Alabama AGI at the full $14,980 pension without the $6,000 retirement income exclusion for filers aged 65 or older. The exclusion brings AGI to $8,980, below the itemized total, so taxable income is zero rather than $4,452." +us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It gave only a generic explanation. Its $182 matches the itemized route on the full $14,980 pension: $14,980 - $9,028 - $1,500 = $4,452 taxable. It left out Alabama's $6,000 retirement income exclusion for filers aged 65 or older, which lowers AGI below the itemized deductions and makes the tax $0." +us,scenario_092,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly itemized $9,028, but it started from federal AGI of $14,980 and skipped Alabama's $6,000 retirement income exclusion for filers aged 65 or older. That exclusion reduces AGI to $8,980, which the itemized deductions fully offset, so the tax is $0 rather than $183." +us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so no substantive calculation was scored against the $0 reference." +us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It taxed the full $14,980 pension with a $2,500 standard deduction and the $1,500 exemption. It missed the $6,000 retirement income exclusion for filers aged 65 or older and did not itemize the $9,028 in deductions, which exceed the post-exclusion AGI of $8,980. That gave $10,980 taxable and $509 instead of zero." +us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It started from a federal AGI of $29,770 that includes the taxable share of Social Security, which Alabama fully exempts. It also missed the $6,000 retirement income exclusion for filers aged 65 or older, used a $4,000 personal exemption instead of the single $1,500, and took no standard or itemized deduction, which inflated taxable income to $24,770." +us,scenario_092,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It taxed the full $14,980 pension, missing the $6,000 retirement income exclusion for filers aged 65 or older. It used a phased-down $1,750 standard deduction instead of the $9,028 in itemized deductions, and it applied a 4.95% top rate instead of Alabama's 5%, which gave $11,730 taxable and $542.14 instead of zero." +us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It invented a federal-style $4,200 standard deduction plus a $1,500 age-65 add-on. It missed the $6,000 retirement income exclusion for filers aged 65 or older and the $9,028 in itemized deductions. It then raised its own $349 bracket result to $852.45 with an unexplained adjustment, even though the federal tax deduction can only lower Alabama tax, and the federal tax here is zero anyway." us,scenario_092,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_093,dependent1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the disability flag as sufficient for Missouri's SSI-related or MO HealthNet for the Disabled pathway and explicitly disregarded the pathway's additional eligibility requirements. Dependent 1 receives no SSI and qualifies for no Medicaid category, while their 2.82-times-FPL MAGI also disqualifies them from the MAGI pathways." us,scenario_093,dependent1_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model asserted that ""PolicyEngine treats disabled individuals as Medicare eligible regardless of age,"" inventing a disability branch that is_medicare_eligible does not contain — the variable keys solely on age >= 65. Applying the actual age test to Dependent 1's age of 27 yields False." @@ -6847,101 +7519,116 @@ us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disab us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model recited the correct statutory rule — 24 months of SSDI receipt confers Medicare — then discarded it, concluding that ""for tax-benefit modeling purposes, a disabled individual is treated as Medicare-eligible."" PolicyEngine's is_medicare_eligible implements no such shortcut: it is the age >= 65 test alone, which Dependent 1 at age 27 fails." us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model stated a disjunctive engine rule — ""In PolicyEngine, Medicare eligibility is determined by age 65 or older OR disability status"" — and fired the second branch off the is_disabled flag. That OR branch does not exist in is_medicare_eligible, which evaluates age >= 65 only, so Dependent 1 at 27 is not eligible." us,scenario_093,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Reached the correct $112,408 AGI and $32,200 standard deduction, then subtracted a $1,000 OBBBA auto-loan-interest deduction this household does not qualify for, cutting taxable income to $79,208 and its own result to $8,509 after the $500 other-dependent credit. It then discarded that figure and submitted $8,067 as an unexplained 'parameter adjustment,' compounding the phantom deduction with an arbitrary $442 reduction." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Pulled the 27-year-old dependent's $45,000 of wages onto the joint return, reporting $158,942 of wages and $156,408 of AGI instead of the filers-only $112,408.25, and additionally deducted $1,000 of personal auto-loan interest. The inflated $124,208 taxable income reached the 22% bracket and nearly doubled the correct $9,128.99 pre-credit tax." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"Included the dependent's $45,000 of wages in joint AGI and used a $29,200 MFJ standard deduction (a pre-2026 figure) rather than $32,200. It then subtracted a fabricated $4,246 of 'refundable credits and reconciliation' from its own $16,405 liability, which is exactly the quantity this output is defined to exclude." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,age_disability,False,"Correctly excluded the dependent's wages and reached $112,408 of AGI, but denied dependency by applying a self-support test to a permanently and totally disabled adult child, who is a qualifying child at any age and generates the $500 credit for other dependents. It also used a $31,500 standard deduction and a $24,150 top of the 10% bracket instead of the 2026 values of $32,200 and $24,800." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"Applied the qualifying-relative $5,200 gross income test to the 27-year-old disabled dependent; permanent and total disability makes that person a qualifying child at any age, to whom no gross income test applies, so the $500 other-dependent credit is allowed. Dropping that credit while also using a $31,500 standard deduction and a $24,000 10%-bracket ceiling instead of $32,200 and $24,800 left it $600 above the reference." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Deducted the full $3,000 of student loan interest instead of the $2,500 statutory cap and used a $32,600 standard deduction rather than $32,200, arriving at $8,371 in its own arithmetic after the $500 credit. It then submitted $10,371, a $2,000 uplift that no step of its stated derivation produces." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"Denied dependency for the 27-year-old by applying the qualifying-relative gross income test, which does not reach a permanently disabled child (a qualifying child at any age), so it omitted the $500 other-dependent credit. It also used the $30,000 MFJ standard deduction and $23,850 10%-bracket ceiling instead of the 2026 values of $32,200 and $24,800." -us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Used a $30,300 standard deduction instead of $32,200 and omitted the $500 other-dependent credit for the disabled 27-year-old, producing $9,376 in its own arithmetic. It then submitted $10,800, an unexplained $1,424 uplift attributed to vague 'adjustments.'" -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"Aggregated all three members' $163,000 of wages into one joint AGI of $160,098 when the dependent's $45,000 is excluded from the return, and declared the traditional IRA and student-loan-interest deductions 'fully phased out' although even its own AGI sits below the MFJ phaseout ranges. With a $30,600 standard deduction and no $500 other-dependent credit, it roughly doubled the correct tax." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"Subtracted the $16,778 of employer-sponsored insurance premiums from wages that are already reported net of them, cutting the joint liability to $7,398.60, and then added a separate $2,354.82 single return for the 27-year-old whose wages this output excludes entirely. It also omitted the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"Double-deducted the $8,389-per-person ESI premiums from already-net box 1 wages, dropping joint AGI to $95,630 instead of $112,408.25, then added a separate single return on the dependent's $36,611. The reference contains neither the dependent's wages nor a second liability, and it further omitted the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"Assumed TCJA expiration for 2026 and reverted to $5,050 personal exemptions with a $15,850 MFJ standard deduction; 2026 law provides a $32,200 standard deduction and no personal exemptions. It also filed the 27-year-old as a second tax unit and added $2,965.40 of that person's tax to the household total, on top of double-deducting the ESI premiums." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"Built AGI from the full $163,156 of household income, including the dependent's $45,000 of wages that the joint return excludes, and deducted the full $3,000 of student loan interest instead of the $2,500 cap. It also dropped the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"Premised the calculation on TCJA reversion — a pre-TCJA standard deduction plus personal exemptions — when 2026 gives a $32,200 MFJ standard deduction and zero personal exemptions, and split the household into two returns. The dependent's $45,000 belongs on neither return in this output, so the second liability it added is pure overstatement." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"Deducted the ESI premiums a second time from already-net wages to reach $95,630 of joint AGI, applied pre-TCJA personal exemptions that 2026 law does not provide, and added a $3,006.65 separate single return for the dependent. The correct computation is one joint return: $112,408.25 AGI, $32,200 standard deduction, $9,128.99 of tax, less the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"Submitted a bare number with no derivation. $11,048 corresponds to a joint taxable income near $96,200 with no credit applied — about $16,000 above the correct $80,208.25 — a level reachable only by absorbing part of the dependent's $45,000 into the return while omitting the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"Subtracted the $16,778 of employer-sponsored insurance premiums from wages already reported net of them, then added a second liability for the 27-year-old 'filing separately.' The dependent's $45,000 is excluded from this output entirely, so there is no second return to add to the joint $9,128.99 less $500." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"Split the household into an MFJ unit and a separate single unit for the 27-year-old and summed both liabilities; the requested output is the joint unit alone, $9,128.99 less the $500 other-dependent credit. Its $12,774 overshoots by roughly the size of a full single return on $45,000 of wages plus the omitted credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"Rolled all $163,000 of household wages into one AGI of about $157,598, when the dependent's $45,000 never enters the joint return, and then subtracted personal exemptions that 2026 law does not provide. Its ~$126,000 taxable income sits about $46,000 above the correct $80,208.25." -us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"Built the joint return nearly right, including the $500 other-dependent credit, but used a $30,840 standard deduction instead of $32,200 and then added a separate $3,353.74 return for the 27-year-old. That dependent's $45,000 of wages is excluded from this output rather than taxed on a second return." -us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"Deducted the full $3,000 of student loan interest over the $2,500 cap, ruled the disabled 27-year-old self-supporting and therefore not claimable — dropping the $500 other-dependent credit that a permanently disabled child of any age supports — and added a separate $3,210 single return whose income the joint return excludes." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction and retirement deductions eliminate taxable income entirely. Joint AGI of $112,408.25 exceeds the $32,200 standard deduction by $80,208.25, producing $9,128.99 at the 2026 MFJ 10%/12% brackets before the single $500 nonrefundable credit." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Gave no derivation and asserted that no dependent credits apply, when the disabled 27-year-old supports a $500 other-dependent credit. Its $6,347 corresponds to a joint taxable income near $57,000, roughly $23,000 below the $80,208.25 that follows from $112,408.25 of AGI less the $32,200 standard deduction." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Reduced already-net box 1 wages by all three members' $8,389 ESI premiums while simultaneously including the dependent's $45,000, giving $133,775 of wages and $132,241 of AGI instead of the filers-only $112,408.25. Its $32,200 standard deduction, bracket schedule, and $500 other-dependent credit were otherwise correct, so the entire $2,380 error is the income base." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Included the dependent's $45,000 of wages in joint income and took the full $3,000 of student loan interest over the $2,500 cap, giving $124,708 of taxable income against the correct $80,208.25 and reaching the 22% bracket. It also applied no other-dependent credit, leaving the $500 nonrefundable reduction out." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Included the dependent's $45,000 of wages and omitted the $190 of deductible traditional IRA contributions, yielding $157,598 of AGI and $125,398 of taxable income instead of $112,408.25 and $80,208.25. Its $500 other-dependent credit and 2026 bracket parameters were correct." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Used a joint taxable income of $85,208, exactly $5,000 above the correct $80,208.25 ($112,408.25 AGI less the $32,200 standard deduction), so it shorted the combination of $4,058.12 in pre-tax 401(k) deferrals and $2,689.62 of student-loan-interest and IRA adjustments by $5,000. Its bracket application and $500 other-dependent credit were right, leaving it $600 high." -us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Deducted all three members' $8,389 employer-sponsored insurance premiums from wages already reported net of them while also folding the dependent's $45,000 into joint income, producing $132,241 of AGI instead of $112,408.25. The resulting $100,041 taxable income overshoots the correct $80,208.25 by nearly $20,000 despite a correct $32,200 standard deduction and $500 credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"Gave no derivation beyond a one-line summary. $13,980 implies a joint taxable income above $110,000 versus the correct $80,208.25, a gap reachable only by taxing the 27-year-old dependent's $45,000 on the household return, and it applied no $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"Applied post-TCJA-sunset parameters — three personal exemptions, a reverted standard deduction, and pre-TCJA rates — when 2026 sets a $32,200 MFJ standard deduction, no personal exemptions, and 10%/12% breaks at $24,800/$100,800. It then added a $4,846 separate return for the dependent and omitted the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"Used a $16,600 standard deduction, two $5,300 personal exemptions, and a 15% second bracket from pre-TCJA law; 2026 provides a $32,200 standard deduction, no exemptions, and a 12% bracket running to $100,800. It compounded that with a $4,130 separate return for the 27-year-old and no $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"Reached the correct $112,408 AGI but applied a $30,750 standard deduction and post-sunset brackets, giving $11,001 on the joint return instead of $9,128.99. It then added a $3,820 separate return for the dependent, whose wages this output excludes, and omitted the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"Derived the joint return exactly — $112,408.25 AGI, $32,200 standard deduction, $80,208 taxable, about $9,129 of tax — then added a $3,220 separate single return for the 27-year-old and applied no nonrefundable credits. The dependent's wages are excluded from this output, and the disabled dependent supports the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for this variable, so no substantive computation reached the submission. The requested figure is $9,128.99 of 2026 MFJ tax on $80,208.25 of taxable income less the $500 other-dependent credit." -us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"Treated all three members as one filing unit and included the dependent's $45,000 of wages, giving $161,656 of AGI instead of $112,408.25, and denied the $190 traditional IRA deduction as 'fully phased out' when joint AGI sits far below the $126,000 MFJ phaseout start. Its $32,200 standard deduction, bracket schedule, and $500 other-dependent credit were correct." -us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Submitted zero with no derivation at all. Joint AGI of $112,408.25 less the $32,200 standard deduction leaves $80,208.25 of taxable income, producing $9,128.99 at the 2026 MFJ 10%/12% brackets, which the single $500 nonrefundable credit reduces to $8,628.99 rather than to zero." -us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"Every parameter was right — $32,200 standard deduction, $24,800/$100,800 bracket breaks, the $2,500 SLID cap, the $190 IRA deduction, and the $500 other-dependent credit — but it added the 27-year-old dependent's $45,000 of wages to joint income. That produced $157,408 of AGI instead of $112,408.25 and pushed $24,408 into the 22% bracket." -us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Subtracted all three $8,389 ESI premiums from already-net wages, took the full $3,000 of student loan interest over the $2,500 cap, added a separate return for the dependent, and omitted the $500 other-dependent credit, reaching about $9,345 in its own scratch work. It then submitted $7,625.75, a number none of its own computations produce." -us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Included all $163,000 of wages in joint AGI and used a $31,000 standard deduction instead of $32,200, computing $13,491 of liability, and explicitly declined to subtract the $500 other-dependent credit that this output does include. It then submitted $3,778, a figure its stated reasoning never yields." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly reached AGI of $112,408 and the $32,200 standard deduction, but then subtracted a $1,000 auto-loan-interest deduction that is not part of the computation. It then replaced its own $8,509 result with an unexplained $8,067 'parameter adjustment'. Without the auto-loan deduction, taxable income is $80,208.25, tax is $9,128.99, and the $500 ODC leaves $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It added the dependent's $45,000 wages to the joint return, which produced $124,208 of taxable income. The tax unit's gross income includes only the head's and spouse's wages net of 401(k) ($113,941.88) plus $1,156 interest, so AGI is $112,408.25 and taxable income is $80,208.25. It also took a $1,000 auto-loan-interest deduction that the computation does not include." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It pooled the dependent's $45,000 wages into the joint return, used the outdated $29,200 standard deduction and pre-2026 brackets, and omitted the $500 ODC. It then subtracted an arbitrary $4,246 for 'refundable credits and reconciliation', which does not belong in a before-refundable-credits measure. Using only the head's and spouse's income gives $80,208.25 of taxable income and $8,628.99 after the ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It rejected the 27-year-old as a dependent on support and gross-income grounds, but a permanently and totally disabled child is a qualifying child at any age with no gross income test, so the $500 credit for other dependents applies. It also used the 2025 $31,500 standard deduction and a $24,150 10% bracket instead of 2026's $32,200 and $24,800. With the correct figures, tax before the credit is $9,128.99, and $8,628.99 after it." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It applied the qualifying-relative gross income limit and concluded there is no dependent. The disabled 27-year-old is a qualifying child regardless of age or income, so the $500 ODC reduces tax. It also used a $31,500 standard deduction and a $24,000 10% bracket instead of 2026's $32,200 and $24,800, which overstated tax before credits ($9,229 versus $9,128.99)." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $32,600 standard deduction. After reaching $8,371 net of the ODC, it added an unexplained $2,000 for 'the disabled adult dependent's own tax'. The dependent's wages are not part of this tax unit's liability, so nothing is added; with the $2,500 cap and the $32,200 deduction, the result is $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It counted all three earners' wages ($163,000) in the joint return's income, which gave AGI of $157,408 and taxable income of $124,208. The joint return includes only the head's and spouse's income, which gives AGI of $112,408.25. It also took a $1,000 auto-loan-interest deduction that the computation does not allow; the correct taxable income is $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the qualifying-relative gross income test and declared no dependent, missing that a permanently disabled child is a qualifying child at any age, so the $500 ODC applies. It also used the 2025 pre-OBBBA $30,000 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800. With the correct figures, tax is $9,128.99 before the credit and $8,628.99 after it." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct $112,408 AGI but used an outdated ~$30,300 standard deduction instead of $32,200 and denied the $500 ODC for the disabled adult child. It then raised its own $9,376 computation to $10,800 with no stated basis. The correct taxable income is $80,208.25, which gives $9,128.99 of tax and $8,628.99 after the ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It ruled out the ODC because the 27-year-old earns $45,000, but a permanently disabled child is a qualifying child with no age or income limit, so the $500 credit applies. It also subtracted a $1,000 auto-loan-interest deduction that the computation does not include, which lowered taxable income to $79,208 instead of $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 wages in the joint return and then wrongly treated the IRA and student-loan deductions as phased out; the joint MAGI of about $115,000 is below both phase-out ranges. It also used a $30,600 standard deduction and omitted the $500 ODC for the disabled dependent." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It subtracted the listed ESI premiums from wages, which pushed joint AGI below $112,408.25; the engine removes only the traditional 401(k) deferrals from wages. It then added a $2,354.82 single-return tax for the dependent, but the output is the one tax unit's liability, where the dependent is claimed (earning a $500 ODC) and the dependent's wages are not taxed." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It subtracted $8,389 of ESI premiums from each earner's wages and summed a married-filing-jointly return with a separate single return for the dependent, without the $500 ODC. The correct tax unit is the joint return alone: AGI $112,408.25 with no ESI reduction, $80,208.25 taxable, and $8,628.99 after the ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It cut joint AGI to $95,630 by subtracting the ESI premiums, which do not reduce wages in this computation, and then added $2,309.98 of tax from a separate single return for the dependent. The claimed dependent's wages are not part of the tax unit's liability; the joint return alone yields $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired after 2025 and applied personal exemptions and a $15,850 standard deduction for married filing jointly. OBBBA made the TCJA rate schedule permanent, with a $32,200 2026 joint standard deduction and no exemptions. It also subtracted ESI premiums from wages and added a separate tax for the dependent instead of claiming the $500 ODC on the single joint return." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It built taxable income from a $163,156 income base that includes the dependent's $45,000 wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, and omitted the $500 ODC. The joint return counts only the head's and spouse's income, which gives $80,208.25 of taxable income." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 parameters (personal exemptions and a lower standard deduction), although OBBBA made the TCJA brackets and the $32,200 joint standard deduction permanent. It also added a separate return's tax for the dependent instead of claiming that person as a dependent and taking the $500 ODC on the one joint return." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used 'under 2026 law' personal exemptions from an assumed TCJA sunset and subtracted ESI premiums to reach joint AGI of $95,630. It also added $3,006.65 of tax on a separate single return for the dependent. The correct computation has AGI of $112,408.25, the $32,200 standard deduction, TCJA brackets, and a $500 ODC, which gives $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $11,048 is about $2,419 above the correct $8,628.99 and closely matches pooling all three earners' wages net of ESI premiums into one joint return (about $100,041 taxable, $11,009 after the ODC). Only the head's and spouse's income belongs on the return, which gives $80,208.25 of taxable income." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It subtracted ESI premiums from wages in addition to 401(k) and IRA contributions, and it summed the tax from a separate return for the dependent with the joint return's tax. The claimed dependent's wages generate no liability in this tax unit, and ESI premiums do not reduce wages, so the joint return alone yields $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It computed tax for two filing units, a joint return plus a single return for the dependent, and summed them. The household is one tax unit in which the disabled 27-year-old is a claimed dependent whose wages are not taxed on the joint return and who generates a $500 ODC, giving $8,628.99." +us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It reached a household AGI of about $157,598 by including the dependent's $45,000 wages, and it subtracted personal exemptions that no longer exist under the permanent TCJA/OBBBA rules. The joint return's AGI is $112,408.25, and taxable income after the $32,200 standard deduction is $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It correctly claimed the $500 ODC on the joint return but then added $3,353.74 of tax from a separate return for the dependent; the claimed dependent's wages are not part of this tax unit's liability. It also used a $30,840 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800, which overstated the joint tax ($8,811 versus $8,628.99)." +us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It treated the 27-year-old as a self-supporting separate filer, so it dropped the $500 ODC and added $3,210 of that person's own tax, but a permanently disabled child is a qualifying child regardless of income. It also deducted the full $3,000 of student loan interest instead of the $2,500 cap." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and retirement deductions eliminate all tax. However, AGI of $112,408.25 minus the $32,200 standard deduction leaves $80,208.25 of taxable income, which produces $9,128.99 of tax, or $8,628.99 after the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced income by 'itemized medical' and ESI-premium effects. Its $6,347 is about $2,282 below the correct amount, which matches removing roughly $19,000 (the $16,778 of ESI premiums plus the listed medical costs) at 12%. The ESI premiums do not reduce wages, and the approximately $2,200 of medical costs is far below the 7.5%-of-AGI floor under a standard-deduction return." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It put all three earners' wages on the joint return and subtracted all three ESI premiums, which produced $133,775 of wages and $132,241 of AGI. Only the head's and spouse's income counts, and ESI premiums do not reduce wages, so AGI is $112,408.25 and taxable income is $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 of taxable income includes the dependent's $45,000 wages, which are not part of the joint return. It also applied no nonrefundable credit, missing the $500 ODC for the disabled adult child. The correct result is $80,208.25 taxable, $9,128.99 of tax, and $8,628.99 after the credit." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It taxed household wages from all three earners on the joint return, pushing taxable income into the 22% bracket at about $125,000. The joint return includes only the head's and spouse's income, which gives $80,208.25 of taxable income entirely within the 10%/12% brackets." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its stated joint taxable income of $85,208 is exactly $5,000 above the correct $80,208.25 (AGI $112,408.25 minus the $32,200 standard deduction), which is equivalent to applying only a $27,200 deduction. The extra $5,000 taxed at 12% adds $600, turning the correct $8,628.99 into $9,228.96." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It combined all three workers' wages into one joint return and subtracted all three ESI premiums, reaching $132,241 of AGI. The dependent's wages are excluded from the tax unit's gross income and ESI premiums do not reduce wages, so AGI is $112,408.25." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It computed the joint tax correctly ($9,128.96) but treated the 27-year-old as a separate filer, adding $3,220 of their tax and dropping the $500 ODC. The disabled adult child is a claimed dependent in the single tax unit, so the answer is $9,128.99 minus $500." +us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,778 of listed employer-sponsored insurance premiums from the head's and spouse's wages, which lowered AGI to $95,630. The engine removes only the $4,058.12 of traditional 401(k) deferrals from wages, so AGI is $112,408.25. The extra $16,778 exclusion at 12% accounts exactly for the $2,013.39 shortfall." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It based the tax on 'household wages', pooling the dependent's $45,000 into the taxed income. The joint return taxes only the head's and spouse's wages net of 401(k) plus interest, which gives $80,208.25 of taxable income and $8,628.99 after the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules (personal exemptions and a 15% bracket), although OBBBA made the TCJA brackets and the $32,200 joint standard deduction permanent for 2026. It also added $4,846 of tax from a separate single return for the dependent instead of claiming the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a sunset-law $16,600 standard deduction, $5,300 exemptions, and a 15% bracket, but 2026 runs on the permanent TCJA schedule: a $32,200 standard deduction, 10% to $24,800, then 12%. It also added $4,130 from a separate single return for the dependent and omitted the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It added $3,224 of tax from a separate single return for the dependent and omitted the $500 ODC to which the claimed disabled adult child entitles the joint return. It also subtracted a $173 overtime-premium deduction, although neither earner works over 40 hours at a straight-time hourly rate." +us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'post-TCJA 2026 brackets' with a $30,750 standard deduction, which overstated the joint tax at $11,001. It then added $3,820 of tax from a separate single return for the dependent. Under the permanent TCJA schedule, taxable income of $80,208.25 yields $9,128.99, less the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($80,208 taxable, $9,129 of tax) but also added $3,220 from a separate single return for the 27-year-old and applied no ODC. The disabled adult child is a dependent in the same tax unit, so the answer is $9,128.99 minus the $500 credit for other dependents." +us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It treated all three members as one joint filer with $163,000 of wages, including the dependent's $45,000, and it treated the traditional IRA deductions as fully phased out. Only the head's and spouse's income is on the return, and at that income (about $115,000) the $190 IRA deduction is fully allowed, which gives AGI of $112,408.25." +us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with no derivation. The joint return has $80,208.25 of taxable income (AGI $112,408.25 minus the $32,200 standard deduction), which produces $9,128.99 of tax and $8,628.99 after the $500 ODC." +us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It added the dependent's $45,000 wages to the joint return's income, which gave AGI of $157,408 and taxable income of $125,208. The joint return counts only the head's and spouse's income, so AGI is $112,408.25 and taxable income is $80,208.25." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted $16,778 of ESI premiums from wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, used a $30,900 standard deduction, and denied the $500 ODC for the disabled adult child. It then submitted $7,625.75, which matches none of its own intermediate results ($7,248.40 or $9,345)." +us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It counted all $163,000 of wages, including the dependent's, deducted the full $3,000 of student loan interest, and wrongly asserted that the credit for other dependents is excluded from this metric, although it is a nonrefundable credit that is subtracted. It then submitted $3,778, which contradicts its own computed $13,491 liability." us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model derived every per-person component correctly ($3,286 + $768.50 + $4,030 + $942.50 + $2,790 + $652.50), which sums to exactly $12,469.50, then wrote a total of $11,469.50 — a $1,000 slip in the addition — and submitted a third figure, $12,348.76, that matches neither its components nor its own stated total. The rule work (no ESI reduction of the FICA base, no Additional Medicare Tax below $200,000, no Missouri employee payroll tax) was right; the failure is purely in summing and transcribing its own numbers." -us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model's reasoning is correct end to end — $163,000 of wages, 6.2% Social Security = $10,106, 1.45% Medicare = $2,363.50, no Additional Medicare Tax, no Missouri employee payroll tax — and it explicitly states the total as $12,469.50, but it submitted $12,487.50, transposing the digits of its own answer. The error is entirely in transcribing the final value into the outputs field." -us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model listed the six correct components ($3,286.00, $768.50, $4,030.00, $942.50, $2,790.00, $652.50) and correctly ruled out both the Additional Medicare Tax and any Missouri employee payroll tax, but its stated sum of $12,622.90 overshoots the true total of $12,469.50 by $153.40. The rule application was right and the addition of its own six terms was wrong." -us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model treated each worker's $8,389 employer-sponsored insurance premium as a Section 125 pre-tax exclusion and cut the FICA base from $163,000 to $137,833 before applying 7.65%, producing $10,544.22. PolicyEngine's employee payroll tax base is gross wages and salaries with no reduction for ESI premiums, so the correct base is the full $163,000 and the correct tax is $12,469.50." -us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model named the correct rule and the correct base — 6.2% plus 1.45% on $163,000 of household wages — which evaluates to $12,469.50, but submitted $12,481.00, $11.50 high. It never carried out the multiplication it described; the figure it reported corresponds to no stated intermediate step." -us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model deducted the three $8,389 employer-sponsored insurance premiums as Section 125 wages excluded from FICA, shrinking the base to $137,833 and yielding $10,544 at 7.65%. PolicyEngine applies the 6.2% Social Security and 1.45% Medicare rates to the full $163,000 of gross wages and salaries, with ESI premiums not removed from the payroll tax base, giving $12,469.50." -us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced each earner's wages by the $8,389 ESI premium ($44,611, $56,611, $36,611) as a pre-tax cafeteria-plan exclusion before applying 7.65%, arriving at $10,544.22. The employee payroll tax base is unreduced gross wages — $53,000, $65,000, and $45,000 — so Social Security is $10,106 and Medicare is $2,363.50 for a total of $12,469.50." -us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no derivation, only an asserted total of $12,654. Applying 6.2% and 1.45% to the household's $163,000 of gross wages yields $12,469.50; the submitted figure corresponds to a base of about $165,412, $2,412 more wages than the household reports, and no computation the model offers supports it." -us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model netted the $8,389 employer-sponsored insurance premium out of each worker's wages ($44,611, $56,611, $36,611) as pre-tax Section 125 compensation and taxed the resulting $137,833 at 7.65% for $10,544.22. PolicyEngine taxes the full $163,000 of gross wages and salaries, giving $12,469.50." -us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule — Social Security and Medicare on the $53,000, $65,000, and $45,000 wages with no Additional Medicare Tax — but reported $12,300.30, which is $169.20 short of the $12,469.50 that 7.65% of $163,000 produces. It supplied no intermediate figures, and its answer corresponds to a base of roughly $160,788 rather than the $163,000 it identified." -us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model wrote the correct expression, 0.062 × $163,000 + 0.0145 × $163,000, which evaluates to $12,469.50, then reported $9,024 — a number its own formula cannot produce. That figure tracks 7.65% of roughly $118,000, the head-plus-spouse wages with the 27-year-old dependent's $45,000 dropped out of the household base." -us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model excluded the three $8,389 ESI premiums from FICA wages as pre-tax cafeteria-plan contributions, taxing $137,833 at 7.65% for $10,544.22. Employer-sponsored insurance premiums do not reduce PolicyEngine's employee payroll tax base, which stays at the $163,000 of gross wages and produces $10,106 of Social Security plus $2,363.50 of Medicare, or $12,469.50." -us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model applied 7.65% to $137,833 — gross wages less the three $8,389 employer-sponsored insurance premiums treated as pre-tax — for $10,544.22. Its handling of the wage ceiling, the Additional Medicare Tax, and the absence of a Missouri employee payroll tax was right; the single error is removing ESI premiums from a payroll tax base that PolicyEngine keeps at the full $163,000." -us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive reasoning exists to evaluate. The required output — 6.2% plus 1.45% on the household's $163,000 of gross wages, or $12,469.50 — was simply absent from the submission." -us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with no derivation for a household with three W-2 earners reporting $53,000, $65,000, and $45,000 in gross wages. Employee Social Security at 6.2% and Medicare at 1.45% on those wages are mandatory and produce $10,106 and $2,363.50 respectively; a zero answer is consistent with never computing the item at all." -us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model computed each earner's 7.65% correctly ($4,054.50, $4,972.50, $3,442.50), correctly noted the $176,100 wage base is not binding and no Additional Medicare Tax applies, and stated the total as $12,469.50 — then submitted $12,553.50, an $84 transcription error away from its own stated answer." -us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Reached Missouri taxable income of about $79,800, within $25 of the correct $79,776.80, then ran one graduated schedule over the combined amount instead of Missouri's married filing combined method, which taxes each spouse's share separately ($1,453.04 + $1,936.62) and costs about $180 less. It also never subtracted Missouri's federal income tax deduction of $431.45, which is 5% of the $8,629 federal liability in the $100,001-$125,000 Missouri AGI band." -us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Built Missouri AGI of $156,408 by pulling Dependent 1's $45,000 of wages into the joint return; the couple's Missouri AGI is $112,408.25 and the 27-year-old's wages generate no Missouri liability in this household. It compounded that by declaring the federal income tax deduction phased out above $125,000 when Missouri AGI of $112,408.25 sits in the 5% band worth $431.45, and by running one bracket schedule on combined income rather than the per-spouse computation." -us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used $119,156 of AGI with no reduction for the $4,058 of traditional 401(k) deferrals or the $2,689.62 of above-the-line deductions, applied a $27,900 MFJ standard deduction instead of Missouri's conforming $32,200, and then abandoned the bracket schedule for an invented 4.04% flat effective rate. The correct chain leaves $79,776.80 taxed on two separate spouse schedules for $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Computed AGI as $116,466, explicitly declining to subtract the $4,058 of traditional 401(k) deferrals that reduce W-2 wages (the reference gross income of $115,097.88 is wages net of those deferrals), and used a guessed $30,000 Missouri standard deduction instead of $32,200. It also skipped the $431.45 federal income tax deduction and ran a single graduated schedule over combined income rather than the two married-filing-combined spouse schedules." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Deducted the full $3,000 of student loan interest rather than the $2,500 statutory cap that yields $2,499.62 here, used a $31,500 standard deduction instead of $32,200, and dismissed Missouri's federal income tax deduction as near zero when it is $431.45. Its own bracket arithmetic produced about $3,640, which it then inflated to $4,126 with an unexplained adjustment, while the per-spouse computation on $79,776.80 gives $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"Its joint-unit figure of about $3,400 tracks the reference, but it then added roughly $1,400 of Missouri tax for the 27-year-old dependent. Dependent 1's $45,000 of wages sit outside the tax unit's $115,097.88 of gross income and produce no Missouri liability, so the household total is the couple's $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Subtracted the $16,778 of employer-sponsored insurance premiums from wages as a Section 125 exclusion, when the reference gross income of $115,097.88 reduces wages only by the $4,058 of traditional 401(k) deferrals, then deducted a $4,400 Missouri personal exemption that no longer exists and the entire $7,339 federal income tax liability instead of the 5%-band amount of $431.45. Those over-deductions drove taxable income to $53,391 against the correct $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Its own stated subtractions (wages net of 401(k) plus $1,156 of interest, less a $30,000 standard deduction) produce about $85,000, yet it asserted roughly $102,000 of Missouri taxable income and taxed that on one schedule. The correct chain is Missouri AGI of $112,408.25 less the $32,200 standard deduction and the $431.45 federal income tax deduction, giving $79,776.80 taxed per spouse for $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"Started from $160,098 of AGI, which folds Dependent 1's $45,000 of wages into the joint return, then subtracted three $2,100 Missouri personal exemptions when Missouri's personal exemption is $0, and applied a 4.5% flat rate. The unit's Missouri AGI is $112,408.25, the deductions are $32,200 plus $431.45, and the top rate is 4.7% applied separately to each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"Used tax year 2024 Missouri brackets and standard deductions rather than 2026's $32,200 conforming deduction and 4.7% top rate, and added $1,056.06 of tax for the 27-year-old as a separate single filer. Dependent 1 generates no Missouri liability in this household, so the household figure is the couple's $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"Aggregated $99,241 of Missouri taxable income across two filing units, $19,464 above the correct $79,776.80, by taxing Dependent 1's wages alongside the couple's, and used a 4.6% top rate instead of 4.7%. It also omitted the $431.45 federal income tax deduction available in the $100,001-$125,000 Missouri AGI band." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Treated Missouri's federal income tax deduction as a $10,000 MFJ cap on federal taxes paid; the deduction is a percentage of federal liability that drops to 5% for Missouri AGI between $100,001 and $125,000, giving $431.45 on the $8,629 federal liability. It also applied a 4.8% top rate instead of 4.7% and taxed the dependent as a separate Missouri filer, whose liability here is $0." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"Submitted a bare number with no derivation; $6,055 corresponds to roughly $129,000 of taxable income run through a single Missouri schedule, which is the full $164,156 of household income less about a standard deduction. The couple's taxable income is $79,776.80 after the $32,200 standard deduction and $431.45 federal income tax deduction, and Dependent 1's wages sit outside that base entirely." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied a 4.5% rate when Missouri's 2026 top rate is 4.7%, and did so on a base of about $88,900, roughly $9,100 above the correct $79,776.80 of Missouri taxable income. It never took the $431.45 federal income tax deduction and never split income across the two married-filing-combined spouse schedules that yield $1,453.04 and $1,936.62." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Named the right ingredients but its $4,166 corresponds to about $92,400 of Missouri taxable income on a single bracket run, $12,600 above the correct $79,776.80. It carried too much income past the $32,200 standard deduction, omitted the $431.45 federal income tax deduction, and taxed the combined amount on one schedule instead of the two spouse schedules." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave a bare $5,321, which corresponds to about $117,000 of Missouri taxable income, essentially the unit's $115,097.88 of gross income with no standard deduction applied. Missouri AGI of $112,408.25 is reduced by the $32,200 conforming standard deduction and $431.45 of federal income tax before the 4.7% schedules apply, leaving $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $3,421 corresponds to about $80,400 of Missouri taxable income, meaning it stopped after the $32,200 standard deduction and never subtracted the $431.45 federal income tax deduction, which is 5% of the $8,629 federal liability in the $100,001-$125,000 Missouri AGI band. The correct base of $79,776.80 taxed per spouse yields $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Submitted $4,850 with no derivation; that figure corresponds to about $107,000 of Missouri taxable income, roughly $27,000 above the correct $79,776.80. It carried income into the Missouri base without the full $32,200 standard deduction and the $431.45 federal income tax deduction, and taxed the result on a single schedule." -us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $5,250 corresponds to about $115,400 of Missouri taxable income, which is the unit's gross income before any deduction. Missouri AGI of $112,408.25 less the $32,200 standard deduction and the $431.45 federal income tax deduction leaves $79,776.80, taxed per spouse for $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"Applied tax year 2024 standard deductions of $29,200 MFJ and $14,600 single instead of 2026's $32,200, taxed the whole base at a flat 4.7% with no graduated lower brackets, and added $1,428 of Missouri tax for Dependent 1 as a separate single filer. Missouri taxes this couple under married filing combined on $79,776.80 split between the spouses, and the dependent contributes $0." -us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"Its couple base of $79,708 lands within $70 of the correct $79,776.80, but it applied a flat 4.7% to the whole amount instead of running the graduated Missouri schedule separately for each spouse, producing $3,746 rather than $1,453.04 + $1,936.62. It then added $1,356 for the 27-year-old dependent, who owes no Missouri tax in this household." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction and allowances fully offset the household's wages and interest. Missouri AGI is $112,408.25, and the $32,200 conforming standard deduction plus the $431.45 federal income tax deduction leave $79,776.80 of taxable income, which produces $3,389.65 under Missouri's schedules." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Submitted an unreconciled approximation with no bracket work; $3,427 sits $37 above the reference and matches a base that stopped at the $32,200 standard deduction without absorbing the $431.45 Missouri federal income tax deduction. The correct base is $79,776.80, run through the two married-filing-combined spouse schedules for $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Built a $157,598 federal AGI that includes Dependent 1's $45,000 of wages, then subtracted $5,400 of Missouri personal exemptions when Missouri's exemption is $0, plus a $1,000 health-insurance subtraction, and applied a 4.5% top rate. The joint unit's Missouri AGI is $112,408.25 and its taxable income $79,776.80, taxed at a 4.7% top rate on each spouse's share." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 Missouri taxable base exceeds the correct $79,776.80 by $44,931, almost exactly Dependent 1's $45,000 of wages, which never enter the couple's return. It also omitted the $431.45 federal income tax deduction and taxed the combined amount on a single bracket schedule." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Declared the household above the range for Missouri's federal income tax deduction, when Missouri AGI of $112,408.25 falls inside the $100,001-$125,000 band where 5% of the $8,629 federal liability, or $431.45, is deductible. Its $5,720 implies about $125,000 of taxable income, which folds Dependent 1's wages into the joint base against the correct $79,776.80." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Followed the right structure but subtracted Missouri personal exemptions on top of the standard and federal income tax deductions, and Missouri's personal exemption is $0. That phantom allowance of roughly $4,200 cut the base below the correct $79,776.80 and left the tax $197 short of $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Stated that no federal-tax deduction applies at this AGI, when Missouri AGI of $112,408.25 sits in the 5% band and yields $431.45, and carried $99,041 into the Missouri base, $19,264 above the correct $79,776.80, while adding a $1,000 health-insurance deduction the reference does not use. Its 4.7% top rate was right; the base and the single-schedule run were not." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave a one-line estimate; $3,780 corresponds to about $84,200 of Missouri taxable income on a single bracket run, roughly $4,400 above the correct $79,776.80. It stopped at the standard deduction, skipping the $431.45 federal income tax deduction and the per-spouse married-filing-combined split worth about $180." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"Carried $95,365 of taxable income for the couple, $15,588 above the correct $79,776.80, by omitting the $2,689.62 of above-the-line deductions and the $431.45 federal income tax deduction, and used a 4.8% top rate instead of 4.7%. It then added $1,547 for the dependent's own return, which contributes $0 to this household's Missouri tax." -us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"Its couple base of $95,808 overstates the correct $79,776.80 by $16,031 because it deducted only the standard deduction from federal AGI, skipping the $2,689.62 of above-the-line adjustments and the $431.45 federal income tax deduction. It then added $1,555 of tax for Dependent 1, who owes no Missouri tax here." -us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Applied a 4.95% top-rate schedule when Missouri's 2026 top rate is 4.7%, which is why its $3,930 for the parents overstates their $3,389.65, and it took no federal income tax deduction and no per-spouse bracket split. It then added $1,354 for the dependent as a separate filer, whose Missouri liability is $0." -us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"Its joint-unit estimate of $3,227 falls $163 short of the couple's $3,389.65, and it then added $1,047 for the 27-year-old as a separate Missouri filer. Dependent 1's wages stay outside the tax unit's $115,097.88 of gross income and generate no Missouri liability." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Applied Missouri's pre-reform schedule of ten small brackets topping out at 5.3%, a $13,000 standard deduction, and $2,000 personal exemptions, and claimed a 35% federal income tax deduction. For 2026 the standard deduction is $32,200, personal exemptions are $0, the top rate is 4.7%, and the federal income tax deduction rate at $112,408.25 of Missouri AGI is 5%, worth $431.45." -us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"Started from a household federal AGI of $161,656 that includes Dependent 1's $45,000 of wages, and declared the federal income tax deduction fully phased out when the joint unit's Missouri AGI of $112,408.25 sits in the 5% band worth $431.45. Its $32,200 standard deduction was right, but the $129,456 base it produced exceeds the correct $79,776.80 by $49,679." -us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Submitted $0 with no derivation. Missouri AGI of $112,408.25 exceeds the $32,200 standard deduction and the $431.45 federal income tax deduction by $79,776.80 of taxable income, which produces $3,389.65 under the per-spouse married-filing-combined schedules." -us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"Used $157,408 of AGI, which includes Dependent 1's $45,000 of wages, and subtracted $3,600 of personal exemptions at $1,200 each when Missouri's personal exemption is $0. Its $32,200 standard deduction and 4.7% rate were right, but the resulting $121,608 base overstates the correct $79,776.80 by $41,831 and it took no federal income tax deduction." -us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"Summed all three earners' wages into one $164,156 federal AGI, used a $26,600 standard deduction and a 4.95% flat rate, and then reported $11,145.20 after its own arithmetic produced $6,809, so the submitted figure follows from neither path. The couple's Missouri taxable income is $79,776.80 after the $32,200 standard deduction and $431.45 federal income tax deduction, taxed per spouse for $3,389.65." -us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"Started Missouri from a federal taxable income of about $124,942 rather than Missouri AGI of $112,408.25, so the $32,200 standard deduction never came off the correct base and Dependent 1's wages were swept in, and it used a 4.5% top rate instead of 4.7%. The correct base is $79,776.80, taxed on each spouse's share for $3,389.65." +us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"Its per-person figures were all correct ($3,286/$768.50, $4,030/$942.50, $2,790/$652.50), and they add up to $12,469.50. It summed them to $11,469.50, a $1,000 addition error, and then submitted $12,348.76, which matches neither its own sum nor any calculation it showed." +us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"It correctly computed $10,106 Social Security plus $2,363.50 Medicare = $12,469.50. It then submitted $12,487.50, so the output field is $18 off from its own correct total." +us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"It listed the correct 6.2% and 1.45% amounts for all three earners, which add up to $12,469.50. It summed them wrongly to $12,622.90, overstating the total by $153.40." +us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated each worker's $8,389 of employer-sponsored insurance premiums as a pre-tax Section 125 deduction and cut FICA wages to $137,833. No fact makes those premiums salary-reduction contributions, so the FICA base is the full $163,000 of gross wages, and 7.65% of that is $12,469.50." +us,scenario_093,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted $8,389 of employer-sponsored insurance premiums from each worker's wages as if they were pre-tax cafeteria-plan deductions, giving a $137,833 base and $10,544.22. The FICA base is the full $163,000 of listed gross wages, and 7.65% of that is $12,469.50." +us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the right method, 6.2% plus 1.45% of $163,000 in total wages, which equals exactly $12,469.50. It submitted $12,481, an $11.50 arithmetic error." +us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It explicitly treated the three $8,389 employer-sponsored insurance premiums ($25,167 in total) as Section 125 exclusions and taxed only $137,833. No listed fact designates those premiums as pre-tax salary reductions, so the full $163,000 is taxable and the tax is $12,469.50." +us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced each worker's FICA wages by $8,389 of employer-sponsored insurance premiums ($44,611, $56,611, $36,611) and got $10,544.22. The premiums are not salary-reduction contributions, so the listed gross wages of $53,000, $65,000 and $45,000 are the FICA base, and the correct total is $12,469.50." +us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It showed no working. Its $12,654 total implies a 7.65% base of about $165,412, which matches no combination of the listed wages. The correct total is 7.65% of $163,000 = $12,469.50, so its answer is $184.50 too high." +us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted $8,389 of employer-sponsored insurance premiums from each worker's wages as pre-tax health insurance, taxing only $137,833 and getting $10,544.22. Nothing in the facts makes those premiums a cafeteria-plan salary reduction, so FICA applies to the full $163,000, giving $12,469.50." +us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly said Social Security and Medicare apply to the full $53,000, $65,000 and $45,000 with no Additional Medicare Tax, which gives $12,469.50. It reported $12,300.30 instead, a $169.20 arithmetic shortfall." +us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its formula, 0.062 × $163,000 + 0.0145 × $163,000, is correct and equals $10,106 + $2,363.50 = $12,469.50. It evaluated the formula as $9,024, a pure arithmetic error." +us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It defined FICA wages as gross wages minus pre-tax employer-sponsored insurance premiums ($137,833 in total) and got $10,544.22. The premiums are not designated as Section 125 salary reductions, so the FICA base is the full $163,000 and the tax is $12,469.50." +us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 7.65% to $137,833, which is wages after subtracting $8,389 per worker of employer-sponsored insurance premiums as pre-tax deductions. No listed fact makes those premiums cafeteria-plan contributions, so FICA applies to the full $163,000 of gross wages, giving $12,469.50." +us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It netted the employer-plan premiums ($8,389 per worker) out of household wages, got a $137,833 FICA base and reported $10,544.22. The listed gross wages of $163,000 are the payroll-tax base because the premiums are not salary-reduction contributions, so the correct answer is $12,469.50." +us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value and no explanation, so there is no substantive answer to grade against the $12,469.50 of employee FICA on $163,000 of wages." +us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"It reported $0 of payroll tax with no working. That ignores the 6.2% Social Security and 1.45% Medicare employee taxes on $163,000 of wages across three earners, which total $12,469.50." +us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed 7.65% for each earner ($4,054.50 + $4,972.50 + $3,442.50) and itself wrote the total as $12,469.50. It then submitted $12,553.50, so the output field is $84 off from its own correct total." +us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It deducted the full $3,000 of student loan interest instead of the $2,500 cap, and it skipped Missouri's federal income tax deduction: 5% of the $8,629 federal liability, or $431.45, for MO AGI in the $100,001–$125,000 band. It also ran the graduated schedule once on the joint ~$79,800. Missouri's combined return instead splits taxable income by income percentage ($34,743.94 / $45,032.86) and taxes each spouse's share separately, which uses the low brackets twice and saves about $181." +us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It put the dependent's wages into the couple's MO AGI ($156,408 instead of the spouses' $112,408.25). That pushed AGI over $125,000 and wrongly eliminated the $431.45 federal-tax deduction. It then taxed the inflated total on one schedule instead of taxing each spouse's income-percentage share of $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from $119,156 of gross income without subtracting the $4,058 of 401(k) deferrals, the $190 IRA deduction or the $2,500 of student loan interest. It used a stale $27,900 standard deduction instead of 2026's $32,200 and replaced Missouri's graduated schedule (4.7% top rate) with an invented flat effective rate of about 4.04%. It also left out the $431.45 federal-tax deduction and the per-spouse split of $79,776.80 of taxable income." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It never subtracted the $4,058 of traditional 401(k) deferrals (its AGI of $116,466 = $118,000 + $1,156 − $190 − $2,500). It also used a $30,000 standard deduction instead of $32,200 and skipped the $431.45 federal-tax deduction, which overstated taxable income at $86,466 against the correct $79,776.80. It then applied one schedule to the joint amount instead of taxing each spouse's income-percentage share separately." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It deducted $3,000 of student loan interest instead of the $2,500 cap, used $31,500 instead of the $32,200 standard deduction, and set the federal-tax deduction to zero when it is $431.45 (5% of federal liability in the $100,001–$125,000 MO AGI band). Its own bracket arithmetic came to about $3,640, which it then pushed up to $4,126 with no basis. The correct method splits $79,776.80 between the spouses by income percentage and taxes each share, giving $1,452.33 + $1,935.91." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It added about $1,400 of Missouri tax on a separate return for the dependent's $45,000 of wages. The household is one tax unit with the disabled adult claimed as a dependent, so no dependent tax arises. The couple alone owes $3,388.25 after splitting $79,776.80 by income percentage and taxing each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It folded the dependent's $45,000 of wages into the couple's Missouri AGI ($157,408), which pushed AGI above $125,000 and wiped out the federal-tax deduction. The couple's MO AGI is the spouses' $112,408.25 only. That keeps a $431.45 federal-tax deduction (5% band) and leaves $79,776.80 of taxable income, taxed separately per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted $16,778 of employer-sponsored insurance premiums from wages as if they were pre-tax, took a $4,400 personal exemption that Missouri does not have, and deducted 100% of an estimated $7,339 of federal tax instead of the percentage-limited deduction (5% of $8,629 = $431.45 at MO AGI of $112,408.25). It also deducted the full $3,000 of student loan interest instead of the $2,500 cap, used $30,000 instead of the $32,200 standard deduction, and applied a flat 4.7% instead of the graduated schedule per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It settled on about $102,000 of Missouri taxable income, more than $22,000 above the correct $79,776.80, even though the components it listed net to far less. It skipped the $2,500 student-loan and $190 IRA subtractions, used $30,000 instead of $32,200 for the standard deduction, and left out the $431.45 federal-tax deduction. It also taxed the joint total on one schedule instead of taxing each spouse's income-percentage share." +us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It got MO AGI ($112,408) and the $32,200 standard deduction right but left out Missouri's federal income tax deduction (5% of the $8,629 federal liability, or $431.45, in the $100,001–$125,000 band), leaving taxable income at $80,208 instead of $79,776.80. It then applied the graduated schedule once to the joint total. Missouri's combined return allocates taxable income by income percentage ($34,743.94 and $45,032.86) and taxes each share separately, which runs the low brackets twice and cuts about $181 from the tax." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It included the dependent's wages in a $160,098 AGI and subtracted $6,300 of personal exemptions that Missouri does not provide. It also used a $30,600 standard deduction instead of $32,200 and a flat 4.5% rate instead of the 2026 graduated schedule with its 4.7% top rate. The correct base is the spouses' $112,408.25 less $32,200 and $431.45, or $79,776.80, taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It added a separate single-filer return for the dependent ($1,056.06), even though the household is one tax unit and the dependent owes no Missouri tax. It also priced the joint return with 2024 brackets and the 2024 $29,200 standard deduction instead of 2026's $32,200. The couple's correct tax is $3,388.25, from taxing each spouse's income-percentage share of $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It totaled $99,241 of taxable income across the couple and a separately taxed dependent, bringing in the dependent's $45,000 of wages, which generate no Missouri tax in this unit. It also applied a 4.6% top rate instead of the 4.7% in Missouri's 2026 schedule. Only the spouses' $79,776.80 is taxable, split by income percentage and taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It cut the couple's AGI to $95,630 by treating $16,778 of employer-sponsored insurance premiums as pre-tax wage reductions. That also moved the couple into the 15% federal-tax-deduction band; the correct MO AGI is $112,408.25, with a 5% deduction of $431.45. It then added a separate $801.81 single-filer tax for the dependent, who belongs to the couple's tax unit and owes no Missouri tax." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It applied a flat 4.8% rate (the 2024 top rate) instead of Missouri's 2026 graduated schedule topping at 4.7%. It also computed tax for both an MFJ unit and a single unit for the dependent, citing both the $10,000 MFJ and $5,000 single federal-tax-deduction caps. Only the couple is taxed: $79,776.80 split between the spouses by income percentage yields $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $6,055 corresponds to about $128,800 of income at 4.7%, which matches taxing the dependent's $45,000 of wages together with the couple's income. The correct computation taxes only the spouses' $79,776.80 ($112,408.25 − $32,200 − $431.45), split by income percentage and taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It taxed the household's combined income, including the dependent's $45,000 of wages, at a rough flat 4.5%. Missouri's 2026 schedule has a 4.7% top rate with graduated lower brackets. Only the spouses' $79,776.80 of taxable income is taxed, split between them by income percentage." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It applied multiple standard deductions and added tax on a separate return for the dependent's $45,000 of wages, a return the one-tax-unit household does not produce. The couple alone owes $3,388.25 after the $32,200 standard deduction, the $431.45 federal-tax deduction and applying the brackets per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $5,321 implies about $117,000 of taxable income at the 4.7% margin, which matches including the dependent's $45,000 of wages. The correct base is the spouses' $79,776.80, split by income percentage and taxed per spouse for $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It named the right components but came out $32.75 high, which at the 4.7% marginal rate equals about $697 of overstated taxable income. The correct base is $112,408.25 − $32,200 standard deduction − $431.45 federal-tax deduction = $79,776.80. Split $34,743.94 / $45,032.86 and taxed per spouse, that gives $1,452.33 + $1,935.91." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It summed Missouri tax over several household tax units, adding tax on the dependent's $45,000 of wages; $4,850 exceeds the couple-only $3,388.25 by $1,461.75. The dependent belongs to the couple's unit and generates no Missouri tax." +us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It gave no derivation. $5,250 implies about $115,500 of taxable income at 4.7%, which matches taxing the dependent's $45,000 of wages along with the couple's income. Only the spouses' $79,776.80 is taxed, per spouse, for $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It treated the dependent as a separate single filer and taxed $30,400 of the dependent's income. It used 2024 standard deductions ($29,200 MFJ) instead of 2026's $32,200, skipped the $190 IRA deduction and the $431.45 federal-tax deduction, and applied a flat 4.7% with no lower brackets. The couple's $79,776.80, split by income percentage and taxed per spouse, yields $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It added $1,356 of single-filer tax for the dependent, who belongs to the couple's tax unit and owes no Missouri tax, and applied a flat 4.7% instead of the graduated schedule. It also deducted $3,000 of student loan interest instead of the $2,500 cap and left out the $431.45 federal-tax deduction and the per-spouse split." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and allowances wipe out Missouri taxable income. In fact the couple's $112,408.25 of MO AGI exceeds the $32,200 standard deduction plus the $431.45 federal-tax deduction by $79,776.80, which produces $3,388.25 of tax under the 2026 brackets applied per spouse. No Missouri nonrefundable credit offsets it." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It offered an approximation with no computation. $3,427 is $38.75 above the correct amount, which equals about $824 of overstated taxable income at the 4.7% margin. The correct base is $79,776.80 ($112,408.25 − $32,200 − $431.45), split by income percentage and taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It built a $157,598 AGI that includes the dependent's $45,000 of wages, subtracted $5,400 of personal and dependent exemptions that Missouri does not allow, and used a 4.5% top rate instead of 2026's 4.7%. The spouses' MO AGI is $112,408.25; after the $32,200 standard deduction and the $431.45 federal-tax deduction, $79,776.80 is split between them and taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It reached $124,708 of taxable income by including the dependent's $45,000 of wages in the couple's base. The spouses' MO AGI of $112,408.25 keeps a $431.45 federal-tax deduction and yields $79,776.80 of taxable income. It also applied one schedule to the joint total instead of taxing each spouse's income-percentage share." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It put the couple above the income range for a federal-tax deduction, which only happens if the dependent's $45,000 of wages is folded into AGI. The spouses' MO AGI of $112,408.25 sits in the 5% band ($100,001–$125,000), which gives a $431.45 deduction. Taxing all household income on one schedule inflated the result to $5,720." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted personal exemptions that Missouri does not provide, which put taxable income about $4,166 below the correct $79,776.80 and the tax $195.81 low. The only deductions from the $112,408.25 of MO AGI are the $32,200 standard deduction and the $431.45 federal-tax deduction." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used $99,041 of Missouri taxable income, $19,264 above the correct $79,776.80. It also allowed no federal-tax deduction, although the spouses' MO AGI of $112,408.25 falls in the 5% band for a $431.45 deduction. It then taxed $99,041 on a single schedule instead of splitting taxable income between the spouses by income percentage." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It taxed only the spouses but understated taxable income by about $5,622, landing $264.25 low. The federal-tax deduction is only 5% of the $8,629 federal liability ($431.45) because MO AGI of $112,408.25 is in the $100,001–$125,000 band. That leaves $79,776.80, taxed per spouse for $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It taxed the couple on $80,208 and added a separate return for the dependent on about $28,900, but the dependent belongs to the couple's tax unit and owes no Missouri tax. The couple's base also leaves out the $431.45 federal-tax deduction, and Missouri taxes each spouse's income-percentage share rather than the joint total." +us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $61,776.10 of taxable income plus the $32,200 standard deduction and its $1,653.90 federal-tax deduction implies an MO AGI of $95,630. That means it subtracted the $16,778 of employer-sponsored insurance premiums from wages as pre-tax. Without that exclusion, MO AGI is $112,408.25, which drops the federal-tax deduction to the 5% band ($431.45) and leaves $79,776.80, taxed per spouse for $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It stated only that the tax came after the standard deduction; $3,780 corresponds to about $84,300 of taxable income on one schedule. The correct computation subtracts both the $32,200 standard deduction and the $431.45 federal-tax deduction from $112,408.25. It then splits the resulting $79,776.80 by income percentage and taxes each spouse's share separately." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It filed a separate single return for the dependent (about $1,547), overstated the couple's taxable income at $95,365 against $79,776.80, and used a 4.8% top rate instead of 2026's 4.7%. The dependent belongs to the couple's tax unit and owes no Missouri tax, and the couple's tax is $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It added a $1,555 single-filer return for the dependent, who belongs to the couple's tax unit and owes no Missouri tax. It also overstated the couple's taxable income at $95,808 against the correct $79,776.80 and left out the $431.45 federal-tax deduction. It then applied the schedule to the joint total instead of taxing each spouse's income-percentage share." +us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It taxed the dependent as a separate single filer ($1,196 on $28,900), although the dependent belongs to the couple's tax unit and generates no Missouri tax. For the couple it left out the $431.45 federal-tax deduction and applied the schedule once to $80,035 instead of splitting taxable income by income percentage and taxing each spouse's share." +us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It used a 4.95% top rate (the 2023 rate) instead of 2026's 4.7%. It added a separate $1,354 return for the dependent, who belongs to the couple's tax unit and owes no Missouri tax. It also skipped the $431.45 federal-tax deduction and the per-spouse split of $79,776.80." +us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It added a separate single-unit tax of $1,047 for the dependent, but the household is one tax unit and the dependent's wages generate no Missouri tax. The joint piece is $3,388.25: $79,776.80 split by income percentage and taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used obsolete Missouri brackets with 5.3–5.5% rates instead of 2026's schedule topping at 4.7%, and subtracted $6,000 of personal exemptions that Missouri does not allow. It applied the 35% federal-tax-deduction rate (the AGI ≤ $25,000 band) instead of 5% for the $100,001–$125,000 band. It also deducted $3,000 of student loan interest instead of the $2,500 cap and used $29,900 instead of the $32,200 standard deduction, then nudged the result up for 'indexation'." +us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in the couple's AGI ($161,656), which wrongly eliminated the federal-tax deduction and inflated taxable income to $129,456. The spouses' MO AGI is $112,408.25, giving $79,776.80 of taxable income taxed per spouse for $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 with no reasoning. The couple has $79,776.80 of Missouri taxable income ($112,408.25 MO AGI − $32,200 standard deduction − $431.45 federal-tax deduction) and no Missouri nonrefundable credits, so the tax is $1,452.33 + $1,935.91 = $3,388.25." +us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It used a $157,408 AGI that includes the dependent's $45,000 of wages and subtracted $3,600 of exemptions that Missouri does not provide. The spouses' MO AGI of $112,408.25 keeps a $431.45 federal-tax deduction and yields $79,776.80 of taxable income, taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in AGI ($164,156) and skipped the 401(k), IRA and student-loan subtractions. It used a $26,600 standard deduction instead of $32,200 and assumed a 4.95% rate. It then reported $11,145.20, which contradicts its own $6,809 calculation; the correct base is the spouses' $79,776.80, taxed per spouse." +us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"Its $124,942 starting point equals $157,142 − $32,200, which means the dependent's $45,000 of wages was folded into the couple's base. It then applied a 4.5% rate instead of the 2026 schedule topping at 4.7%. Only the spouses' $79,776.80 is taxable, after a $431.45 federal-tax deduction that its inflated AGI eliminated, and it is taxed per spouse." us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI." us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero." us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets." @@ -6969,27 +7656,32 @@ us,scenario_095,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model i us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an incorrect 200%-of-FPL broad-based categorical eligibility threshold and then invented a $95 monthly New Jersey minimum SNAP benefit. The household’s $29,605 income exceeds the applicable gross-income limit, and the minimum allotment applies only to an otherwise eligible household." us,scenario_095,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the household’s countable income “very low” without reconciling that claim with the listed $29,605 annual income. That income exceeds the applicable one-person gross-income limit, so the $500 asset balance cannot establish eligibility and the correct allotment calculation yields $0." us,scenario_095,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly treated $2,467 in monthly gross income as below New Jersey’s applicable broad-based categorical eligibility limit. Because the household fails the gross-income screen, it is ineligible and cannot receive the $24 monthly minimum allotment, which is reserved for otherwise eligible households." -us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly excluded Social Security, applied the age-62+ retirement exclusion to the full $11,385 401(k), and reduced the remaining $4,198 by the $1,000 personal plus $1,000 age-65 exemptions, but then taxed the $2,198 residual at 1.4% instead of applying N.J.S.A. 54A:2-1.1, which imposes no New Jersey tax at all when NJ gross income is at or below $20,000 for a qualifying surviving spouse ($10,000 single). Its own computed NJ gross income of $4,198 after exclusions sits far below that threshold, so the correct liability is $0.00, not $30.77." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model reached NJ gross income of $4,198 after the retirement exclusion, took a $88.34 medical deduction and a $1,000 personal exemption, and taxed $3,109.66 at 1.4%, never applying New Jersey's statutory zero-tax provision (N.J.S.A. 54A:2-1.1) that exempts filers with gross income at or below $20,000 for a surviving spouse ($10,000 single). It also dropped the additional $1,000 age-65 exemption, but the decisive omission is the threshold exemption, which makes the liability $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model never applied New Jersey's age-62+ pension/retirement income exclusion, leaving the entire $11,385 401(k) distribution in the base and taxing $13,092.10 at 1.4% for $183.29. Excluding that distribution brings NJ gross income to $4,198, which falls under the N.J.S.A. 54A:2-1.1 zero-tax threshold of $20,000 for a surviving spouse ($10,000 single), so no tax is imposed; the model also started from a federal-style AGI of $15,286 that subtracts half of self-employment tax, a deduction New Jersey does not allow." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model built the base correctly — Social Security excluded, the full $11,385 401(k) removed by the pension exclusion, $2,000 of exemptions and a $188.34 medical deduction — and then taxed $2,009.66 at 1.4%, skipping New Jersey's threshold exemption under N.J.S.A. 54A:2-1.1, which zeroes the tax outright when NJ gross income is at or below $20,000 for a surviving spouse ($10,000 single). With NJ gross income of $4,198 after exclusions, the correct answer is $0.00; it additionally counted the $100 of over-the-counter health expenses as deductible medical costs." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model excluded Social Security and the full $11,385 401(k) distribution, applied the $1,000 personal and $1,000 age-65 exemptions, and taxed the $2,198 remainder at 1.4%, never reaching New Jersey's statutory exemption in N.J.S.A. 54A:2-1.1 that imposes no tax when NJ gross income is at or below $20,000 for a qualifying surviving spouse ($10,000 single). Its own $4,198 of post-exclusion NJ gross income is under that threshold, making the liability $0.00 rather than $30.77." -us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model correctly reduced NJ income to $4,198 via the Social Security exclusion and the full pension exclusion, subtracted $2,000 of personal and senior exemptions, and multiplied $2,198 by the 1.4% bottom bracket. It missed N.J.S.A. 54A:2-1.1, which exempts the taxpayer entirely — no bracket is applied — when NJ gross income is at or below the $20,000 surviving-spouse threshold ($10,000 single), so the tax is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model derived NJ taxable income of $2,198 ($4,198 self-employment income less $2,000 of personal and age-65 exemptions, with the 401(k) fully excluded) and taxed it at 1.4%. It never applied New Jersey's zero-tax threshold under N.J.S.A. 54A:2-1.1, which imposes no gross income tax when NJ gross income is at or below $20,000 for a surviving spouse ($10,000 single); at $4,198 of NJ gross income the household owes $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model excluded Social Security and the entire $11,385 401(k) distribution, then taxed roughly $3,110-$3,198 of remaining self-employment income at 1.4% after only the $1,000 personal exemption, omitting the $1,000 age-65 exemption. The controlling error is skipping N.J.S.A. 54A:2-1.1, which exempts filers with NJ gross income at or below $20,000 (surviving spouse) or $10,000 (single) from tax entirely — at $4,198 the correct liability is $0.00, not $44." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model excluded Social Security and the $11,385 distribution, subtracted $2,000 of exemptions and about $316 of medical expenses over the 2% floor, and taxed the $1,882 residual at 1.4% for $26. New Jersey imposes no tax at all on this filer under N.J.S.A. 54A:2-1.1 because NJ gross income of $4,198 is at or below the $20,000 surviving-spouse threshold ($10,000 single), so the bottom bracket is never reached and the answer is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied the Social Security exclusion, the age-qualified retirement income exclusion for the $11,385 distribution, and the personal, age, and medical deductions, then ran the leftover self-employment income through the 1.4% bracket for $28.14. It omitted New Jersey's statutory zero-tax provision (N.J.S.A. 54A:2-1.1), under which NJ gross income of $4,198 — below the $20,000 surviving-spouse threshold and the $10,000 single threshold — produces no tax liability at all, so the value is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model invented a partial ""low-income tax reduction"" that trimmed the liability to $18.46 instead of applying New Jersey's actual low-income rule, N.J.S.A. 54A:2-1.1, which is an all-or-nothing exemption: with NJ gross income at or below $20,000 for a qualifying surviving spouse ($10,000 single), no gross income tax is imposed. After the Social Security and $11,385 retirement exclusions, NJ gross income is $4,198, so the full exemption applies and the tax is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model correctly reduced NJ gross income to $4,198 (Social Security exempt, 401(k) fully excluded under the age-62+ pension exclusion), subtracted $316 of excess medical expenses and $2,000 of exemptions, and taxed $1,882 at 1.4%. It failed to apply N.J.S.A. 54A:2-1.1, which exempts the household from New Jersey income tax entirely at that gross income level (threshold $20,000 for a surviving spouse, $10,000 single), making the correct value $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model computed NJ gross income of $4,198, applied $2,000 of personal and age exemptions plus the excess-medical deduction, and taxed the ~$1,882 remainder at 1.4% for $26. New Jersey's statutory exemption in N.J.S.A. 54A:2-1.1 imposes no tax when NJ gross income does not exceed $20,000 for a qualifying surviving spouse ($10,000 single), so the $4,198 base produces $0.00 and no bracket is applied." -us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model taxed $2,901 at 1.75%, a rate that only applies to New Jersey taxable income above $20,000, and omitted the $1,000 age-65 exemption while starting from a federal AGI of $15,286 that nets out half of self-employment tax. Above all it skipped N.J.S.A. 54A:2-1.1, under which NJ gross income of $4,198 after the pension exclusion is at or below the $20,000 surviving-spouse threshold ($10,000 single) and no tax is imposed, so the correct answer is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model correctly applied the $75,000 retirement exclusion to the $11,385 distribution, the $2,000 of exemptions, and the ~$188 medical deduction, then taxed $2,010 at 1.4%. It never applied New Jersey's zero-tax threshold under N.J.S.A. 54A:2-1.1, which exempts a filer whose NJ gross income is at or below $20,000 as a surviving spouse ($10,000 single); with $4,198 of post-exclusion gross income, the liability is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate. The correct derivation excludes the $14,022 of Social Security and the full $11,385 401(k) distribution under New Jersey's age-62+ retirement income exclusion, leaving $4,198 of NJ gross income that falls under the N.J.S.A. 54A:2-1.1 zero-tax threshold and yields $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model deducted $297 for half of self-employment tax and a $1,000 ""standard deduction"" — New Jersey allows neither; it has a $1,000 personal exemption plus a $1,000 age-65 exemption and no adjustment for half of SE tax — then taxed $2,901 at 1.4%. The decisive miss is N.J.S.A. 54A:2-1.1: with NJ gross income of $4,198 after the retirement exclusion, at or below the $20,000 surviving-spouse threshold ($10,000 single), New Jersey imposes no tax and the value is $0.00." -us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model used a $10,000 retirement/pension exclusion — the pre-2017 New Jersey cap — instead of the $75,000 available to a qualifying surviving spouse aged 62+, leaving $1,385 of the 401(k) distribution in the base and producing $3,394.66 taxed at 1.4%. With the full $11,385 excluded, NJ gross income is $4,198, at or below the $20,000 surviving-spouse threshold in N.J.S.A. 54A:2-1.1, under which no New Jersey income tax is imposed, so the correct value is $0.00." -us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Treated the ANCHOR property-tax relief rebate as a refundable New Jersey income tax credit and hard-coded its $1,500 homeowner amount for income under $150,000. ANCHOR is a property-tax relief payment outside the gross income tax's refundable credit structure, so it never enters state_refundable_credits, and the household has $0 property tax and $0 rent — a $31,000 mortgage balance is not a property tax payment — so no New Jersey property-tax benefit accrues at all. The credits that do belong in this output (NJ EITC match, NJ CTC, refundable NJ CDCC) are each $0 for a childless 73-year-old with no care expenses." -us,scenario_095,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Built the NJ EITC off a self-constructed ""age-adjusted federal credit"" instead of the federal EITC as computed; the state credit is 40% of the federal EITC amount, which is $0 here because the no-qualifying-child EITC is unavailable above age 64 and this filer is 73. It compounded the error by using self-employment income less half the self-employment tax as the earned income base — that subtraction belongs to AGI, not to EITC earned income — and its $119.38 is exactly 40% of a roughly $298 childless credit phased down against AGI of about $15,300, a credit the age ceiling eliminates entirely." -us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_refundable_credits, so no state credit computation was submitted rather than a substantive miscalculation. The correct derivation yields $0: the NJ EITC is 40% of a federal EITC that is $0 for a 73-year-old with no qualifying child, and the NJ Child Tax Credit and refundable NJ child and dependent care credit are both $0 with no children and no care expenses." +us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly excluded Social Security and the full $11,385 401(k) distribution, leaving $4,198 of NJ gross income. It then subtracted $2,000 of exemptions and taxed $2,198 at 1.4% to get $31. It never applied New Jersey's no-tax rule: NJ gross income of $4,198 is below the $10,000 single / $20,000 qualifying-widow(er) threshold, so the tax is $0." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"After the full 401(k) exclusion it taxed the remaining $4,198 of self-employment income, less an $88 medical deduction and only a $1,000 exemption (it dropped the age-65 exemption), at 1.4% to get $44. It missed that NJ gross income of $4,198 after the pension exclusion is under New Jersey's $10,000/$20,000 threshold, below which no tax is owed." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It started from federal AGI of $15,286, deducting half of SE tax, which is not an NJ deduction. It never applied the age-62+ pension exclusion to the $11,385 401(k) distribution and taxed $13,092 at 1.4% to get $183.29. With the exclusion applied, NJ gross income is $4,198. Even without it, $15,583 is under the $20,000 qualifying-widow(er) no-tax threshold, so the tax is $0." +us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It excluded Social Security and the 401(k) distribution, then taxed $4,198 less $2,000 of exemptions at 1.4% to get $30.77. It never applied New Jersey's rule that NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)) owes no tax, and $4,198 is below both." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the pension exclusion, the $2,000 of exemptions and a $188 medical deduction, then taxed $2,010 at 1.4% to get $28.14. It skipped the NJ gross-income threshold: after the pension exclusion, NJ gross income is $4,198. That is below the $10,000/$20,000 no-tax threshold, so the liability is $0." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It explicitly found only $4,198 of NJ gross income left after excluding Social Security and the 401(k) distribution. It still subtracted $2,000 of exemptions and taxed $2,198 at 1.4% to get $30.77. It missed that NJ gross income that low, under $10,000 single or $20,000 qualifying widow(er), carries zero New Jersey tax." +us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It excluded the full $11,385 pension, took $2,000 of exemptions and taxed $2,198 at 1.4% to get $30.77. It never applied New Jersey's no-tax threshold for NJ gross income at or below $10,000/$20,000, and the post-exclusion $4,198 is below it." +us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It correctly excluded Social Security and the 401(k) distribution, then taxed $4,198 less $2,000 of exemptions at the 1.4% bracket to get $30.77. It left out New Jersey's rule that NJ gross income of $10,000 or less (single) or $20,000 or less (qualifying widow(er)) owes no tax, so the answer is $0." +us,scenario_095,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It excluded the 401(k) distribution, then taxed roughly $3,110-$3,198 at 1.4% to get $44, having used only the $1,000 personal exemption plus a small medical deduction and omitted the age-65 exemption. It missed that NJ gross income of $4,198 after the pension exclusion is below the $10,000/$20,000 threshold, so no NJ tax is owed." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It excluded Social Security and the pension, then taxed $4,198 of net profit less $2,000 of exemptions and a $316 medical deduction at 1.4% to get $26. It never applied New Jersey's zero-tax rule for NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)), which covers this $4,198." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"After the retirement-income exclusion it taxed the remaining self-employment income, less exemptions and medical deductions, to get $28.14. It skipped the NJ gross-income threshold: $4,198 of post-exclusion NJ gross income is under $10,000/$20,000, so the liability before refundable credits is $0." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It excluded the retirement income, applied the regular and age-65 exemptions, and treated a 'low-income tax reduction' as only partly cutting the tax, to $18.46. New Jersey's low-income rule removes the tax entirely when NJ gross income is at or below $10,000 (single) or $20,000 (qualifying widow(er)), and $4,198 qualifies, so the tax is $0." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It excluded the $11,385 401(k) distribution, then taxed $4,198 of business income less $2,000 of exemptions at 1.4% to get $31. It never applied New Jersey's no-tax threshold: NJ gross income of $4,198 is below $10,000/$20,000, so no tax is due." +us,scenario_095,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It excluded Social Security and the retirement distribution, then taxed the remaining business income, less exemptions and a medical deduction, to get $29.54. It left out New Jersey's rule that NJ gross income at or below $10,000 (single) or $20,000 (qualifying widow(er)) is not taxed, which zeroes this $4,198 of NJ gross income." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly computed NJ gross income of $4,198, then subtracted a $316 medical deduction and $2,000 of exemptions and taxed $1,882 at 1.4% to get $26. It did not recognize that NJ gross income that low, under the $10,000 single / $20,000 qualifying-widow(er) threshold, carries no New Jersey income tax." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It stated that NJ gross income is $4,198, then subtracted exemptions and medical expenses and taxed $1,882 at 1.4% to get $26. It never checked New Jersey's gross-income threshold, below which ($10,000 single, $20,000 qualifying widow(er)) the tax is zero, so the answer is $0." +us,scenario_095,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It computed NJ gross income of $4,198, then subtracted $2,000 of exemptions and $316.04 of medical expenses and taxed $1,881.96 at 1.4% to get $26.35. It missed that New Jersey imposes no tax when NJ gross income is $10,000 or less (single) or $20,000 or less (qualifying widow(er)), and $4,198 is below both." +us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It started from federal AGI, which wrongly subtracts half of SE tax, and took only a $1,000 exemption. It then taxed $2,901 at 1.75%, even though the first NJ bracket is 1.4%, to get $51. Beyond those errors, it missed that NJ gross income of $4,198 after the pension exclusion is below the $10,000/$20,000 threshold, below which the tax is $0." +us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It fully excluded the 401(k) distribution, took $2,000 of exemptions and a $188 medical deduction, and taxed about $2,010 at 1.4% to get $28. It never applied New Jersey's no-tax threshold for NJ gross income at or below $10,000/$20,000, which the post-exclusion $4,198 falls under." +us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score. The correct derivation excludes Social Security and the $11,385 401(k) distribution, leaving $4,198 of NJ gross income. That is below New Jersey's $10,000/$20,000 no-tax threshold, so the answer is $0." +us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It subtracted half of SE tax, which is not an NJ deduction, and took a $1,000 'standard deduction' in place of the $2,000 personal and age-65 exemptions. It then taxed $2,901 at 1.4% to get $41. It missed that NJ gross income of $4,198 after the pension exclusion is below New Jersey's $10,000/$20,000 threshold, so no tax is owed." +us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It capped the retirement exclusion at $10,000 instead of fully excluding the $11,385 401(k) distribution under the age-62+ pension exclusion (worth up to $75,000 single). It then taxed $3,394.66 at 1.4% to get $47.53. Once the full exclusion is applied, NJ gross income of $4,198 is under the $10,000/$20,000 threshold, so the tax is $0." +us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Counted the NJ ANCHOR homeowner benefit as a $1,500 refundable income tax credit. ANCHOR is a separate property-tax relief payment, not part of the NJ-1040 refundable credits (NJ EITC, NJ Child Tax Credit, property tax credit). A mortgage balance is not property tax paid, and unlisted property taxes are $0. The model also skipped the NJ EITC, which phases out to $0 at this household's AGI of about $15,286." +us,scenario_095,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Correctly found the age-65+ childless NJ EITC pathway and the $298.45 phase-in amount (7.65% × $3,901 net SE earnings), but then took 40% of that without applying the phase-out. The phase-out uses the greater of AGI or earned income, and AGI is about $15,286 because it includes the $11,385 401(k) distribution. With the non-joint threshold of about $11k, the reduction of 7.65% × (AGI − threshold) exceeds $298.45, so the hypothetical federal credit and the NJ EITC are both $0." +us,scenario_095,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Applied 40% to the phase-in amount alone ($298.45 = 7.65% × $3,901 of earnings after the deductible half of SE tax) and never ran the phase-out. The phase-out runs on AGI of about $15,286, which includes the $11,385 401(k) distribution. Because that distribution alone is above the roughly $11k non-joint threshold, the reduction exceeds the phase-in amount and the hypothetical federal credit, and therefore the NJ EITC, is $0." +us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for state_refundable_credits, so there was nothing to score. The correct answer is $0: the age-65+ childless NJ EITC is fully phased out at AGI of about $15,286, and with no property tax or rent there is no NJ property tax credit." us,scenario_095,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_098,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $30,595 of Social Security as taxable when it set AGI to $35,035. After the farm rental loss offsets most of the pension, provisional income is below the Social Security taxation threshold, leaving $4,440 of AGI that the standard deduction fully eliminates." us,scenario_098,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted all Social Security and the $1,100 energy subsidy as taxable income, omitted the $11,100 farm rental loss, and used an inapplicable $13,500 standard deduction. The farm loss leaves $4,440 of non-Social-Security income, Social Security remains nontaxable under the provisional-income test, and the standard deduction reduces taxable income to zero; its claimed $950 age credit does not repair those upstream errors." @@ -7032,127 +7724,141 @@ us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible." us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility." us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It priced the CDCC at the pre-OBBBA 20% rate ($6,000 x 20% = $1,200) instead of the 32% rate that applies at $159,585 of AGI ($1,920), and stacked a $1,575 auto-loan-interest deduction on top of the standard deduction that the correct deduction chain does not contain. It then discarded its own worked result of $10,965 and submitted $12,103 with no arithmetic supporting the $1,138 jump." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the credits exactly right ($4,400 CTC plus $1,920 CDCC at 32%) but deducted only the $32,200 MFJ standard deduction, omitting the $2,000 charitable deduction available to non-itemizers in 2026 that raises the deduction to $34,200. That single omission (plus capping the educator expense at $300 instead of $338) left taxable income at $127,422 rather than $125,385.05, and the resulting $448.25 of extra tax is precisely its entire gap from $10,679.75." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built AGI of $165,502 without subtracting the $3,087.20 of traditional 401(k) deferrals or the $2,999.76 capital-loss deduction on the $6,478 net loss, then applied the 2024-era $29,200 MFJ standard deduction instead of the $34,200 2026 standard-plus-charitable amount. It also skipped the CDCC entirely and invented a $3,174 partial CTC in place of the $4,400 nonrefundable CTC ($2,200 per child) plus $1,920 CDCC." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It used a $2,000-per-child CTC ($4,000) rather than the 2026 amount of $2,200 per child ($4,400), and the pre-OBBBA 20% CDCC rate ($1,200) rather than the 32% rate at this AGI ($1,920), for $5,200 of credits instead of $6,320. It compounded that with a $31,500 standard deduction that omits both 2026 indexation to $32,200 and the $2,000 non-itemizer charitable deduction, and finally submitted $12,404 rather than the $12,661 its own arithmetic produced." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning worked all the way to $12,421 and it then submitted $18,962, a figure $6,541 higher that exceeds even its own $17,621 estimate of tax before credits — the answer contradicts every credit it said it applied. The underlying computation was also wrong on three counts: a $31,500 standard deduction instead of $34,200, a $4,000 CTC instead of $4,400, and a 20% CDCC ($1,200) instead of 32% ($1,920)." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"On its own taxable income of roughly $127,200 the 2026 MFJ schedule (10% to $24,800, 12% to $100,800, 22% above) produces about $17,400, but it asserted $19,650 — a $2,250 bracket error. It then applied only $4,000 of CTC instead of $4,400 at $2,200 per child and zeroed the CDCC, when $6,000 of the $12,740 childcare expenses at the 32% 2026 rate yields a $1,920 nonrefundable credit." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It split the CTC into a $1,700-per-child 'refundable portion' and a $300-per-child 'nonrefundable portion,' claiming only $600 of nonrefundable CTC; the nonrefundable CTC is the full per-child credit absorbed against liability, which here is $4,400 ($2,200 x 2), with refundability only limiting what survives beyond tax. It also used the 20% CDCC rate ($1,200 instead of $1,920), a $30,000 standard deduction instead of $34,200, and disallowed the $144 traditional IRA deduction that the correct AGI includes." -us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It reached AGI of $159,584 — within a dollar of the reference — then submitted $24,500, a number $4,600 above even its own $19,900 estimate of tax before any credits, justified by AMT and NIIT add-backs that do not apply to a $159,585-AGI wage household with no preference items and no net investment income above the $250,000 MFJ threshold. Its underlying chain also used a $30,300 standard deduction rather than $34,200 and a 20% CDCC rather than 32%." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It added the $540 short-term gain to income while dropping the $7,018 long-term loss entirely, forfeiting the $2,999.76 capital-loss deduction, and used a $30,900 standard deduction instead of the $34,200 standard-plus-charitable amount. Its credits were $1,800 rather than $6,320 because it claimed only a $600 'nonrefundable portion' of the CTC instead of the full $4,400 and a 20% CDCC ($1,200) instead of the 32% $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It computed 2026 under a TCJA sunset that never happened: four personal exemptions at $5,061 ($20,244), itemized deductions of $25,814, and a CTC phased down to $750 under the repealed $110,000 MFJ threshold. OBBBA made the exemption repeal, the $32,200 standard deduction, the $400,000 CTC phaseout threshold, and the $2,200 per-child credit permanent, and it further understated AGI at $134,987 by excluding employer-sponsored insurance premiums that never reduce the $161,912.80 of employment income." -us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA structure to 2026 — $21,200 of personal exemptions plus itemized deductions — and cut AGI to $135,069 by excluding the employer-sponsored insurance premiums under a Section 125 exclusion, when the only pre-tax reduction to the $165,000 of wages is the $3,087.20 of traditional 401(k) deferrals. Personal exemptions remain repealed in 2026, the $34,200 standard-plus-charitable deduction beats its $29,806 of itemized deductions, and its $2,000 CTC understates the $4,400 available at $2,200 per child." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It got AGI right at $159,584 but then applied repealed pre-TCJA law: $20,200 of personal exemptions and $28,971 of itemized deductions, driving taxable income to $110,413 instead of $125,385.05 against the $34,200 standard-plus-charitable deduction. Its credits were also pre-TCJA — a $1,000-per-child CTC ($2,000) and a 20% CDCC ($1,200) instead of $4,400 and $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Its $164,481 AGI subtracts only the $3,088 of 401(k) deferrals, ignoring the $2,999.76 capital-loss deduction, the $338 educator expense, and the $144 traditional IRA that bring AGI to $159,585.05. More decisively, $17,822 sits above the $16,999.75 of tax before credits, so it subtracted none of the $6,320 in nonrefundable credits ($4,400 CTC plus $1,920 CDCC) the output definition requires." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset — $21,200 of personal exemptions, itemizing at $31,022, and a CTC cut to $700 by the repealed $110,000 MFJ phaseout — when OBBBA permanently keeps exemptions repealed, sets the deduction floor at $34,200 here, and phases the $2,200-per-child CTC only above $400,000. It also understated AGI at $135,169 by excluding employer-sponsored insurance premiums from the $161,912.80 of employment income." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA 2026 law — $20,200 of personal exemptions, $30,150 of itemized deductions, and a CTC phased down to $750 — and an AGI near $135,000 that improperly excludes employer-sponsored insurance premiums from wages. Under actual 2026 law AGI is $159,585.05, the $34,200 standard-plus-charitable deduction beats itemizing, exemptions are zero, and the credits are $4,400 of CTC plus a 32% CDCC of $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It supplied no derivation, only a restatement of its number. The correct chain is $16,999.75 of tax on $125,385.05 of taxable income less $4,400 of CTC and $1,920 of CDCC; its $11,463 sits $783 high, which is the $720 shortfall from pricing the CDCC at the pre-OBBBA 20% rate ($1,200) rather than the 32% rate plus a small overstatement of taxable income." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It named the right two credits but showed no computation, and its $11,105 is $425 above the reference — the tax at the 22% marginal rate on roughly $2,000 of income that should have been removed by the 2026 non-itemizer charitable deduction that lifts the MFJ deduction from $32,200 to $34,200. Everything else in its stated structure ($4,400 CTC, $1,920 CDCC against $16,999.75) reconciles once that deduction is restored." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It states it applied itemized deductions plus personal exemptions for 2026, a structure OBBBA permanently repealed; exemptions are zero and the household's itemizable total of roughly $27,500 (SALT plus charity net of the new 0.5%-of-AGI charitable floor, with $11,400 of medical costs below the $11,969 7.5% floor) falls short of the $34,200 standard-plus-charitable deduction. Its $9,877 is $803 below the reference, consistent with deducting about $3,650 more than the correct $34,200." -us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It identified the correct credit pair but showed no arithmetic; its $10,817 is $137.25 high, which at the 22% marginal rate is $624 of taxable income above the correct $125,385.05. That is a shortfall in the deduction chain — the $3,481.76 of above-the-line items ($2,999.76 capital loss, $338 educator expense, $144 traditional IRA) plus the $34,200 standard-plus-charitable deduction — not a credit error." -us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It fabricated a $15,600 nonrefundable CTC for two children when the 2026 credit is $2,200 per child for $4,400, capped the CDCC expense base at $3,000 as if there were one qualifying child (the two-child cap is $6,000) and applied 20% rather than the 32% 2026 rate, and claimed a $1,404 Saver's Credit at an AGI of $159,585 that is far above the roughly $79,000 MFJ ceiling. It also added the $6,478 net capital loss into income rather than deducting the $2,999.76 allowed against ordinary income." -us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It claimed only an $800 'nonrefundable portion' of the CTC when the entire $4,400 ($2,200 x 2) is absorbed nonrefundably against a $16,999.75 liability, and used the 20% CDCC rate ($1,200) instead of the 32% rate at this AGI ($1,920) — a $4,520 credit shortfall. It also stopped at the $32,200 standard deduction without the $2,000 non-itemizer charitable deduction that brings the total to $34,200." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $17,278 exceeds the $16,999.75 of tax before credits, so despite asserting that a large nonrefundable child-related reduction applies, it subtracted none of the $6,320 in credits — $4,400 of nonrefundable CTC at $2,200 per child and $1,920 of CDCC (32% of the $6,000 two-child expense cap). The requested output is explicitly after nonrefundable credits including CDCC and the nonrefundable CTC." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Working back from $16,999.75 of pre-credit tax, its $14,696 reflects about $2,304 of nonrefundable credits rather than the $6,320 actually available. It never claimed the $1,920 CDCC (32% of the $6,000 two-child expense cap on $12,740 of childcare) and used far less than the $4,400 nonrefundable CTC for the 9- and 4-year-olds." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It applied the pre-OBBBA 20% CDCC rate for $1,200 instead of the 32% rate that yields $1,920 on the $6,000 expense cap, a $720 shortfall, and deducted only the $32,200 standard deduction while omitting the $2,000 non-itemizer charitable deduction, leaving taxable income at $127,566 rather than $125,385.05. Those two errors account for the full $1,200.25 gap; its $4,400 CTC was correct." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It used the correct $4,400 CTC but claimed a $1,200 CDCC, the pre-OBBBA 20% rate, when the 2026 rate at $159,585 of AGI is 32% and yields $1,920 on the $6,000 two-child expense cap. That $720 credit shortfall, offset slightly by an understated pre-credit tax, is its entire $509.25 gap." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Its $127,528 of taxable income omits the $2,000 non-itemizer charitable deduction that raises the 2026 MFJ deduction from $32,200 to $34,200, and it claimed a $1,200 CDCC at the pre-OBBBA 20% rate rather than $1,920 at the 32% rate. Those two errors — $471 of extra tax and $720 of missing credit — sum to its $1,191.45 gap." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It used a $2,000-per-child CTC ($4,000) instead of the 2026 $2,200 per child ($4,400) and a $1,200 CDCC at the 20% rate instead of $1,920 at 32%, a $1,120 credit shortfall. It also itemized rather than taking the $34,200 standard-plus-charitable deduction, which exceeds the household's roughly $27,500 of itemizable SALT and charity." -us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It had every 2026 parameter right — the $2,000 non-itemizer charitable deduction on top of the standard deduction, the $4,400 CTC, and the $1,920 CDCC — but set AGI at $159,728 by treating the $144 of traditional IRA contributions ($72 each) as nondeductible under the active-participant phaseout, when they are deductible above the line. That $144 of extra taxable income at the 22% marginal rate is exactly its $31.45 gap." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave a bare estimate with no computation; $14,000 is the correct $16,999.75 of pre-credit tax less exactly $3,000 of credits, half the $6,320 actually available. It dropped the $1,920 CDCC (32% of the $6,000 two-child cap) and understated the nonrefundable CTC, which is $4,400 at $2,200 per child for the 9- and 4-year-olds." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It computed 2026 under restored pre-2018 law — $21,200 of personal exemptions, itemizing, 2017-style brackets, and a CTC treated as fully phased out — when OBBBA permanently keeps exemptions repealed, sets the 10/12/22% schedule, and phases the $2,200-per-child CTC only above $400,000, leaving $4,400 fully usable here. It also claimed only the $1,200 CDCC at 20% instead of $1,920 at the 32% 2026 rate." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied the repealed pre-TCJA regime — a $16,600 standard deduction, $21,200 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC fully phased out above $110,000 of AGI — producing taxable income of $109,776 instead of $125,385.05. Under 2026 law the deduction is $34,200 ($32,200 plus the $2,000 non-itemizer charitable deduction), exemptions are zero, the CTC is $4,400, and the CDCC is $1,920 at the 32% rate rather than the $1,200 it claimed." -us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It taxed $110,683 of ordinary income under pre-TCJA 10/15/25% brackets built on an AGI of $134,987 that excludes employer-sponsored insurance premiums from the $161,912.80 of employment income, and phased the CTC down to $751 using the repealed $110,000 MFJ threshold. The 2026 chain is $159,585.05 of AGI less $34,200, taxed at 10/12/22%, with a full $4,400 CTC and a 32% CDCC of $1,920 rather than its $1,200." -us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It claimed $5,600 of credits — the correct $4,400 CTC plus a $1,200 CDCC at the pre-OBBBA 20% rate instead of $1,920 at the 32% 2026 rate — and deducted only the $32,200 standard deduction without the $2,000 non-itemizer charitable deduction that brings the total to $34,200. Those two errors, plus a slightly overstated bracket tax, produce its $1,228 gap." -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the required key never reached the outputs object. This is a missing submission rather than a substantive tax error." -us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the required key never reached the outputs object. This is a missing submission rather than a substantive tax error." -us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 on the assertion that charitable deductions and nonrefundable credits wipe out the liability, and its explanation breaks off mid-sentence with no arithmetic. The $12,621 of charitable giving does not even displace the $34,200 standard-plus-charitable deduction, and the $6,320 of nonrefundable credits offsets only part of the $16,999.75 of tax on $125,385.05 of taxable income." -us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used the correct $4,400 CTC but a $1,200 CDCC at the pre-OBBBA 20% rate rather than $1,920 at the 32% 2026 rate, and took only the $32,200 standard deduction without the $2,000 non-itemizer charitable deduction, leaving taxable income at $127,384. Its $17,852 of pre-credit tax also overstates the $16,999.75 that the 10/12/22% MFJ schedule produces on the correct base." -us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning ended at $11,730 and it submitted $24,711.50, a figure that exceeds even its own $17,004 estimate of tax before credits, so the answer is disconnected from every step it worked. The underlying chain was also wrong: it subtracted the full $7,018 long-term loss in reaching AGI of $156,588 instead of applying the $2,999.76 capital-loss limitation, and used a $4,000 CTC and a 20% CDCC instead of $4,400 and $1,920." -us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It claimed only the $1,200 CDCC and no CTC at all, forgoing the $4,400 nonrefundable credit for the 9- and 4-year-olds at $2,200 each. It also fabricated a $4,160 educator-expense deduction in place of the actual $338, used $3,244 of 401(k) deferrals rather than $3,087.20, took no capital-loss deduction on the $6,478 net loss, and applied a $29,994 standard deduction instead of $34,200 — leaving AGI at $164,894 against the correct $159,585.05." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at the old 20% rate ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It deducted $1,575 of auto-loan interest although no qualifying new, U.S.-assembled vehicle is listed, and it used a $32,600 standard deduction instead of $32,200. Its own arithmetic reached $10,965, yet it submitted $12,103, which none of its steps support." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its $32,200 standard deduction and $6,320 of credits ($4,400 CTC + $1,920 CDCC) were right. It left out OBBBA's $2,000 joint deduction for non-itemizers' cash gifts, which the couple's $11,933 of cash donations supports. It also capped the $338 educator expense at $300, so taxable income was $127,422 instead of $125,385.05, overstating tax by about $448." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It built AGI of $165,502 without removing the $3,088 of traditional 401(k) deferrals from wages or taking the $3,000 net capital loss deduction. It used the 2024 $29,200 standard deduction instead of 2026's $32,200 plus the $2,000 non-itemizer charitable deduction. It then subtracted a made-up $3,174 CTC with no CDCC, instead of the full $4,400 CTC and a $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-OBBBA credit amounts: a $2,000-per-child CTC instead of $2,200 and a 20% CDCC instead of the 32% rate at this AGI, giving $5,200 of credits instead of $6,320. It also used a $31,500 standard deduction and made-up bracket thresholds ($24,150/$98,150) instead of $32,200 and $24,800/$100,800, and it left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Its own work ended at $12,421, but it submitted $18,962, which is more than its own $17,621 pre-credit tax, so the submitted value subtracts no credits. The work itself counted the $128 of qualified dividends twice and used a $31,500 standard deduction without the $2,000 non-itemizer charitable deduction. It also used a $2,000-per-child CTC and a 20% CDCC instead of $2,200 and 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It put tax at about $19,650 on roughly $127,200 of taxable income, about $2,200 more than the 2026 brackets (10% to $24,800, 12% to $100,800, then 22%) produce. It then subtracted only a $4,000 CTC ($2,000 per child instead of $2,200) and zeroed out the CDCC. With two working parents and $12,740 of childcare for two children, the CDCC is $1,920 (32% of $6,000). It also left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its credits ($4,400 CTC and $1,920 CDCC) and bracket method were right. However, it deducted $1,575 of auto-loan interest with no listed fact showing a qualifying new, U.S.-assembled vehicle, and it left out the $2,000 non-itemizer charitable deduction that the $11,933 of cash gifts supports. With the $300 educator cap, taxable income was $125,847 instead of $125,385.05, adding $101.63 of tax." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It treated only $300 per child ($600) of the CTC as nonrefundable. In fact the whole CTC is applied against tax first, and only the unused remainder goes to the refundable ACTC, so with about $17,000 of tax the full $4,400 ($2,200 per child under OBBBA) is used. It also took the CDCC at 20% instead of 32% and used 2025 bracket thresholds. Its standard deduction was $30,000 instead of $32,200, and it had no $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It estimated about $19,900 of tax before credits. Instead of subtracting credits, it submitted $24,500, citing an AMT add-back that does not apply to someone taking the standard deduction at this income. The correct steps are the $32,200 standard deduction plus the $2,000 charitable deduction (not its $30,300), $16,999.75 of tax, then subtracting the $4,400 CTC and $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also swapped the $2,000 non-itemizer charitable deduction for a $1,575 auto-loan interest deduction that no qualifying-vehicle fact supports, and it disallowed the IRA deduction. Together these overstate the result by about $853." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It counted only $600 of the CTC as nonrefundable, but the full $4,400 ($2,200 per child) is applied against its roughly $17,000 of tax before any refundable portion is figured. It dropped the $7,018 long-term capital loss, missing the $3,000 net capital loss deduction. It also used a 20% CDCC, a $30,900 standard deduction, and no $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied expired pre-TCJA law: four $5,061 personal exemptions, itemizing, 15/25% brackets, and a $1,000-per-child CTC reduced to $750. OBBBA made TCJA permanent, so 2026 has no personal exemptions, a $32,200 standard deduction, 10/12/22% brackets, and a $2,200-per-child CTC. It also subtracted the $24,597 of employer-paid health premiums from wages, putting AGI at $134,987 instead of $159,585.05; its closeness to the right answer comes from errors that cancel out." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It used TCJA-sunset rules ($21,200 of personal exemptions, itemized deductions, and a phased-down CTC) instead of OBBBA's permanent 2026 rules: a $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and a $2,200-per-child CTC. It also treated employer-paid health premiums as a pre-tax wage reduction, cutting AGI to $135,069 instead of $159,585.05, and it took the CDCC at 20% instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,700 standard deduction instead of 2026's $32,200 and left out the $2,000 non-itemizer charitable deduction. It then subtracted a $2,000-per-child CTC ($4,000) and a 20% CDCC ($1,200) instead of OBBBA's $2,200 per child and 32% rate ($6,320 total), overstating the result by about $2,088." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and applied $20,200 of personal exemptions, itemized deductions, and a $1,000-per-child CTC. OBBBA made the TCJA framework permanent, so 2026 uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, no exemptions, a $2,200-per-child CTC, and a 32% CDCC ($1,920), not 20%." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It reported AGI of $164,481 without the $3,000 net capital loss deduction. Its $17,822 is essentially tax before credits: it never subtracted the $4,400 CTC and $1,920 CDCC that bring $16,999.75 down to $10,679.75." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied the TCJA sunset ($21,200 of personal exemptions, itemizing, and a $1,000 CTC reduced to $700), although OBBBA made TCJA permanent with a $32,200 standard deduction and a $2,200-per-child CTC that phases out only above $400,000. It subtracted employer-paid health premiums from wages, getting AGI of $135,169 instead of $159,585.05, and it took the CDCC at 20% instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules ($20,200 of personal exemptions, itemizing, and a $1,000 CTC reduced to $750) instead of OBBBA's permanent rules: a $32,200 standard deduction plus the $2,000 charitable deduction and a $2,200-per-child CTC. It also reduced AGI to about $135,000 by subtracting employer-paid health premiums, and it took the CDCC at 20% instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working; the correct result is $16,999.75 of tax on $125,385.05 of taxable income minus the $4,400 CTC and $1,920 CDCC. Its $11,463 is $783.25 too high, which matches taking the CDCC at the pre-OBBBA 20% rate ($1,200 instead of $1,920, +$720) plus about $290 of extra taxable income." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $11,105 is $425.25 above the correct $10,679.75, which is 22% of about $1,933. That matches applying the full $6,320 of credits while leaving out OBBBA's $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It computed tax after itemized deductions and personal exemptions, which is expired pre-TCJA law. For 2026, OBBBA permanently repealed personal exemptions, and the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction beats itemizing. The credits are the $2,200-per-child CTC and the 32% CDCC, giving $16,999.75 minus $6,320." +us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no working. Its $10,817 is $137.25 above the correct $10,679.75, which equals about $624 of extra taxable income with the correct $6,320 of credits. That matches small deduction slips such as disallowing the $144 IRA deduction, capping the educator expense at $300, and swapping auto-loan interest for the $2,000 charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It made up a $15,600 nonrefundable CTC; the 2026 CTC is $2,200 × 2 = $4,400. It limited the CDCC to one child's $3,000 at 20%, although two children qualify for the $6,000 cap at 32%. It claimed a $1,404 saver's credit even though joint AGI near $160,000 is far above that credit's limit, and it never removed the 401(k) deferrals or took the capital loss deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only an $800 nonrefundable portion of the CTC, but the full $4,400 ($2,200 per child) is applied against its roughly $17,500 of tax before any refundable portion is figured. It also took the CDCC at 20% instead of 32% and left out the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. Its $17,278 is higher than the correct $16,999.75 of tax before credits, so in effect it subtracted none of the $4,400 CTC and $1,920 CDCC." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no calculation. Its $14,696 implies only about $2,300 of credits subtracted from roughly $17,000 of tax, instead of the full $4,400 CTC plus $1,920 CDCC ($6,320)." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also left out the $2,000 non-itemizer charitable deduction, so taxable income was $127,566 instead of $125,385.05. These two misses make up its $1,200 overstatement." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $1,200 CDCC (20% of $6,000) instead of $1,920. Under OBBBA the rate is 32% at $159,585 joint AGI: 35% minus one point per $4,000 over $150,000. That overstates the result by $720, partly offset by tax before credits about $211 below the correct $16,999.75." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% ($1,920). It also left out the $2,000 non-itemizer charitable deduction and the $144 IRA deduction, so taxable income was $127,528 instead of $125,385.05." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It used a $2,000-per-child CTC ($4,000) instead of OBBBA's $2,200 ($4,400) and a 20% CDCC ($1,200) instead of 32% ($1,920). Its roughly $128,000 of taxable income also leaves out the $2,000 non-itemizer charitable deduction that brings taxable income to $125,385.05." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Everything else matched, but it disallowed the $144 traditional IRA deduction. That put AGI at $159,728 instead of $159,585.05, which reflects about $482 of adjustments (the $338 educator expense plus the $144 IRA contributions). The extra $143 of taxable income taxed at 22% is its $31.45 overstatement." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its tax of about $18,360 on $127,566 of taxable income is roughly $880 more than the 2026 brackets (10% to $24,800, 12% to $100,800, then 22%) produce. It also left out the $2,000 non-itemizer charitable deduction and took the CDCC at 20% ($1,200) instead of 32% ($1,920)." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It left out OBBBA's $2,000 non-itemizer charitable deduction, so taxable income was $127,384 instead of $125,385.05. It also took the CDCC at 20% ($1,200) instead of the 32% rate that applies at $159,585 joint AGI ($1,920)." +us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It cut the OBBBA CDCC rate by one point per $2,000 over $150,000, reaching 30% ($1,800). For joint returns the step is $4,000, which gives 32% ($1,920). It also disallowed the $144 traditional IRA deduction, adding $31.45 of tax." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Its rough $14,000 guess implies only about $3,000 of credits subtracted from the $16,999.75 of tax before credits. The full $4,400 CTC ($2,200 × 2) and $1,920 CDCC (32% of $6,000) apply, giving $10,679.75." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law: $21,200 of personal exemptions, 2017-style brackets, a 10% medical floor, and a fully phased-out $1,000 CTC. OBBBA permanently kept the TCJA rates and set the CTC at $2,200 per child with a $400,000 joint phase-out threshold. It then subtracted only a 20% CDCC ($1,200) instead of the $4,400 CTC plus a 32% CDCC ($1,920)." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as after the TCJA sunset: personal exemptions, a $16,600 standard deduction, 10/15/25% brackets, and a CTC phased out above $110,000. OBBBA made the TCJA framework permanent, with a $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, 10/12/22% brackets, and a $2,200-per-child CTC that is untouched below $400,000. It also took the CDCC at 20% instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA law: four $5,400 personal exemptions, itemizing, 10/15/25% brackets, and a CTC fully phased out above $110,000. OBBBA made TCJA permanent, so 2026 uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction and a $2,200-per-child CTC ($4,400). It also used a 20% CDCC instead of 32% ($1,920)." +us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income on 10/15/25% pre-TCJA brackets with a $1,000-per-child CTC reduced to $751, although OBBBA made the TCJA brackets and the $2,200-per-child CTC permanent. It also subtracted employer-paid health premiums from wages, putting AGI at $134,987 instead of $159,585.05, and it took the CDCC at 20% instead of 32%." +us,scenario_099,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% at $159,585 joint AGI ($1,920). It also used the $32,200 standard deduction without the extra $2,000 non-itemizer charitable deduction, so its tax before credits of about $17,508 is about $500 too high." +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is nothing to score. The correct figure is $16,999.75 of tax minus $6,320 of credits." +us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is nothing to score. The correct figure is $16,999.75 of tax minus $6,320 of credits." +us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It assumed charitable and other itemized deductions wipe out the tax. Those deductions ($8,654 of property tax plus CA income tax, $12,621 of gifts less the 0.5%-of-AGI floor, and medical costs below the 7.5% floor) fall short of the $32,200 standard deduction. Taxable income is $125,385.05 with $16,999.75 of tax, leaving $10,679.75 after $6,320 of credits." +us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It took the CDCC at 20% ($1,200) instead of OBBBA's 32% ($1,920). It also used 2025 bracket thresholds ($23,850/$96,950) instead of 2026's $24,800/$100,800 and left out the $2,000 non-itemizer charitable deduction, overstating tax before credits by about $850." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Its own work reached $11,730, but it submitted $24,711.50, which no step supports. The work itself subtracted the whole $6,478 net capital loss instead of the $3,000 limit and used a $2,000-per-child CTC and a 20% CDCC. It also used a $32,600 standard deduction without the $2,000 non-itemizer charitable deduction." +us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $4,400 child tax credit and applied only a $1,200 CDCC. It also made up a $4,160 educator expense (the actual figure is $338) and skipped the $3,000 capital loss deduction. Its $29,994 standard deduction should have been $32,200 plus the $2,000 non-itemizer charitable deduction." us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Its own reasoning computed pre-credit federal tax of about $17,200 and stated that the $4,000 CTC reduces it to about $13,200, which under §24(d) leaves zero credit unused by the liability limitation, then abandoned that conclusion and reported $1,200 as an unexplained residual. It applied only the earned-income prong (15% × ($165,000 − $2,500) = $24,375) and never the controlling ceiling that the ACTC cannot exceed the CTC disallowed by tax liability. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200." us,scenario_099,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the statutory per-child refundable cap ($1,700 × 2 = $3,400) as an automatic refund for any filer with earnings above the $2,500 floor, skipping §24(d)'s requirement that the ACTC equals only the CTC remaining after the credit is applied against tax liability. With roughly $17,400 of federal income tax before credits, the full $4,400 of 2026 CTC is absorbed nonrefundably, so the refundable amount is $0. It also used the 2025 $1,700 cap instead of the 2026 indexed figure." us,scenario_099,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly identified the OBBBA 2026 $2,200 per-child CTC and the $1,800 per-child refundable cap, then treated that cap as a guaranteed refund because earnings exceed the $2,500 threshold and AGI is under the $400,000 phase-out. The refundable portion is the lesser of the earned-income formula, the per-child cap, and the CTC left over after offsetting tax; on about $127,000 of taxable income the roughly $17,400 tax consumes all $4,400 of CTC nonrefundably, leaving $0." us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the model performed no substantive computation to evaluate. The correct derivation gives $0: about $17,400 of pre-credit federal tax fully absorbs the $4,400 CTC under the §24(d) liability limitation, and $159,676 of income is far past the two-child MFJ EITC phase-out." us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so no substantive reasoning exists to fault. The correct derivation gives $0: the roughly $17,400 of federal tax before credits leaves no CTC unused for the ACTC, and EITC is zero at $159,676 of income." us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It invented a ""$300 Other Dependent Credit portion"" for two children aged 9 and 4 who are qualifying children; the §24(h)(4) other-dependent credit is $500, applies only to dependents who are not qualifying children, and is wholly nonrefundable, so it can never enter federal_refundable_credits. Its own explanation states that pre-credit tax absorbs the nonrefundable credits and no ACTC remains — a derivation that yields $0 — yet it submitted $600." -us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model applied only the federal NSLP income tests — 130% FPG for free and 185% for reduced-price — and stopped once household income of ~$166,694 cleared them. It missed California's Universal Meals Program (Education Code 49501.5), which requires every LEA to serve free breakfast and lunch to all enrolled K-12 students regardless of income, setting school_meal_tier to FREE for the 9-year-old and producing $1,130.96 of annual free-meal support." -us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model tested only two pathways — the 130% FPG income limit and categorical eligibility via SNAP/TANF — both of which correctly fail here (fpg_ratio 5.03, categorical False). It omitted the third pathway that governs this household: California's statewide universal free school meals program, which assigns the FREE tier to all K-12 students irrespective of the FPG ratio." -us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model computed ~$166,694 of gross income against 130% FPG and additionally invoked an asset test on the $2,230,000 stock and $28,035 bank balances; school meal eligibility has no asset test, and neither test decides this case. California's Universal Meals Program grants free meals to every enrolled K-12 student regardless of income or resources, which sets the FREE tier for the 9-year-old and yields $1,130.96." -us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model described 185% FPL as the threshold for free meals — 185% is the reduced-price cutoff, with 130% governing free — and then rejected the household on income alone. The controlling rule is California's universal free school meals mandate, which makes the K-12 child free-tier at an FPG ratio of 5.03 and returns positive free-meal support." -us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model treated the federal 130% FPG income test as the sole determinant and concluded no free-meal support flows to a household with $165,000 of wages plus investment income. It never applied California's Universal Meals Program, the state rule that overrides the income tier and provides free breakfast and lunch to all K-12 enrollees, giving the 9-year-old $7.15 per meal and $1,130.96 for the year." -us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model checked the two federal pathways — the 130% FPG income limit and categorical eligibility — and correctly found both unmet, then stopped. California's statewide universal free meals program is the operative pathway here: it sets school_meal_tier to FREE for the enrolled K-12 child without regard to the 5.03 FPG ratio." -us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the National School Lunch Program's 130% FPG free-meal limit (~$42,900 for four) and ruled the household out on income. It missed that California funds universal free meals for all K-12 students statewide, so the FREE tier attaches to the 9-year-old at any income level and the household receives $1,130.96." -us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model cited federal free/reduced thresholds of roughly $40,560 and $57,720 for a family of four and rejected the household on its $165,000-plus income. Those federal limits are superseded in California by the Universal Meals Program, which provides free meals to every enrolled K-12 student regardless of the FPG ratio, so the correct answer is positive free-meal support of $1,130.96." -us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model reduced the question to the federal 130% FPG free-meal limit and answered from the income comparison alone. California's universal free school meals program overrides that income test for all enrolled K-12 students, setting the FREE tier for the 9-year-old at an FPG ratio of 5.03." -us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model's entire derivation was 'income > 130% FPL,' which decides only the federal income pathway. The determinative rule is California's statewide universal free meals mandate, which grants free breakfast and lunch to every K-12 enrollee irrespective of income and produces $1,130.96 of annual support here." -us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model applied the 130% FPG income screen and inferred zero free-meal support from the household's income. It omitted California's Universal Meals Program, the state pathway that assigns the FREE school_meal_tier to all enrolled K-12 students regardless of the 5.03 FPG ratio." -us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model asserted only that household income exceeds the free-meal threshold, applying the federal income test as if it were the sole gate. California provides universal free school meals to all K-12 students, which sets the FREE tier for the 9-year-old and yields positive annual free-meal support." -us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the federal 130% FPG limit and declared both children ineligible. It missed California's Universal Meals Program, which makes the enrolled K-12 child free-tier regardless of income; the 4-year-old contributes nothing because he is not in K-12, and the household benefit of $1,130.96 comes from the 9-year-old alone." -us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave only an income-threshold rationale, treating the federal free-meal income limit as dispositive. The state pathway controls: California's universal free school meals program serves all K-12 enrollees without an income test, so the household's FPG ratio of 5.03 does not defeat eligibility." -us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model rejected eligibility purely on the federal income threshold for free meals. California's statewide universal meals mandate supersedes that threshold for enrolled K-12 students, setting school_meal_tier to FREE and producing $1,130.96 in annual subsidy at $7.15 per meal for the 9-year-old." -us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model applied the 130% federal poverty guideline test and stopped there. It never reached California's Universal Meals Program, the rule that grants free meals to every K-12 student in the state regardless of the household's 5.03 FPG ratio." -us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model's only stated basis was that income exceeds the free-meal income threshold, an exclusively federal test. California's universal free school meals program assigns the FREE tier to the enrolled 9-year-old without regard to income, so the household receives positive free-meal support." -us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model computed an AGI of about $156,588 and compared it to 130% FPG; school meal countable income is $166,154 against a $33,000 guideline for four, but neither figure decides the case. California's Universal Meals Program provides free meals to all enrolled K-12 students regardless of the FPG ratio, making the household free-meal eligible." -us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model applied the ~$43,000 (130% FPG) free-meal income limit to a ~$160,000 income and concluded no support. The controlling rule is California's statewide universal free school meals program, which sets the FREE tier for the enrolled K-12 child irrespective of income and yields $1,130.96." -us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"The model relied solely on the federal free-meal income thresholds and expected no benefit at this income. California's Universal Meals Program grants free breakfast and lunch to every enrolled K-12 student without an income test, so the 9-year-old is free-tier and the household receives positive annual support." -us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model reasoned that no fact in the prompt signals free-meal eligibility and that wage income likely exceeds the thresholds. The dispositive fact is present — state CA plus a K-12-aged child — because California's universal free school meals program makes school_meal_tier FREE for all enrolled K-12 students regardless of income. -us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model enumerated exactly two pathways, the 130%-of-poverty income limit and categorical eligibility, both correctly failing here. It omitted the state-universal pathway: California serves free meals to all K-12 students statewide, which sets the FREE tier at an FPG ratio of 5.03 and returns $1,130.96." -us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income threshold and concluded no positive support. California's Universal Meals Program overrides that income screen for enrolled K-12 students, so the 9-year-old qualifies for free meals at $7.15 per meal and the household total is $1,130.96." -us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model's sole basis was that household income far exceeds the federal school-meal income threshold. That test is not the operative one in California, where the state's universal free meals program grants the FREE tier to every K-12 enrollee regardless of income." -us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model treated high income as automatically disqualifying, applying the federal NSLP income test alone. California's universal free school meals program has no income test for enrolled K-12 students, so the 9-year-old is free-tier and the household receives $1,130.96 of annual free-meal support." -us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model checked the 130% FPG income limit and categorical eligibility, both correctly failing, and inferred a zero from PolicyEngine. It skipped the state-universal branch of the school_meal_tier calculation: California's universal free meals program forces the FREE tier for all K-12 students regardless of the 5.03 FPG ratio." -us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model applied the ~$43,000 130%-FPG free-meal cutoff for a family of four and rejected the household on income. California's Education Code universal meals mandate supplies free breakfast and lunch to every enrolled K-12 student without regard to income, making the household eligible." -us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model correctly established that income exceeds 130% FPG and that SNAP, TANF, and foster-care categorical eligibility are all absent, but its pathway list was incomplete. California's statewide universal free school meals program is a separate pathway that sets school_meal_tier to FREE for all enrolled K-12 students, producing $1,130.96 of support here." -us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model submitted no value and no explanation for free_school_meals_eligible, so the required binary never reached the outputs object and the answer was scored as missing rather than as a substantive determination. The correct submission is 1, because California's universal free school meals program sets the FREE tier for the enrolled 9-year-old regardless of the household's 5.03 FPG ratio." -us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"The model applied the federal free-meal income limits alone and rejected the household. California's Universal Meals Program provides free meals to all K-12 enrollees with no income limit, so the FREE tier applies and the household's annual free-meal subsidy is $1,130.96." -us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model applied the 130% FPG (~$43,000) free-meal test to a ~$160,000 income and predicted a PolicyEngine zero. It missed the state-level override: California's universal free school meals program sets school_meal_tier to FREE for every enrolled K-12 student regardless of the FPG ratio." -us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model named California explicitly yet still applied only the federal 130% FPG threshold for a household of four. California's Universal Meals Program requires free breakfast and lunch for all enrolled K-12 students statewide, which makes the 9-year-old free-tier at an FPG ratio of 5.03 and yields positive free-meal support." -us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model gave only the federal income-threshold rationale for a zero. The state pathway is decisive: California's universal free school meals program grants the FREE tier to every enrolled K-12 student without an income test, so the household receives $1,130.96 of annual free-meal support." +us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Compared about $166,694 of gross income to the 130% FPL federal free-meal limit and stopped there. It never applied California's universal school meals program, which gives free meals to the 9-year-old K-12 child at any income." +us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Treated the 130% FPL income test and categorical eligibility as the only ways to qualify for free meals. It missed California's universal free school meals program, which covers every enrolled K-12 student regardless of income." +us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL income test and also cited asset holdings, although the school meal programs have no asset test. It overlooked California's universal school meals program, under which the 9-year-old K-12 child receives free meals regardless of income." +us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Used the 185% FPL threshold as the free-meal limit, but 185% is actually the reduced-price cutoff. More importantly, it ignored California's universal school meals program, which makes the K-12 child eligible for free meals at any income." +us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Relied only on the federal 130% FPL free-meal income test. It never considered that California provides universal free school meals to every enrolled K-12 student, so the 9-year-old qualifies despite $165,000 in wages." +us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Considered only the 130% FPL income test and categorical eligibility. It omitted California's universal free school meals program, which gives the K-12 child free meals regardless of income." +us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Denied eligibility on the 130% FPL income test alone. It missed California's universal school meals program, which overrides the income test and makes the 9-year-old K-12 child eligible for free meals." +us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the NSLP rule of income at or below 130% FPL (about $42,900) as the sole test. It overlooked California's universal school meals program, which gives free meals to all enrolled K-12 students at any income." +us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Compared $165,000 in wages to the federal 130%/185% free and reduced-price thresholds and never applied California's universal school meals program. That program gives the 9-year-old K-12 child free meals regardless of household income." +us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,Used the 130% FPL federal free-meal threshold as decisive and ignored California's universal free school meals program. Under that program the K-12 child is eligible at any income. +us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Applied only the 130% FPL free-meal limit. It missed California's universal school meals program, which makes the enrolled K-12 child eligible for free meals regardless of income." +us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Denied eligibility because income exceeds 130% FPL and did not account for California's universal free school meals program. That program covers every K-12 student, including the 9-year-old, at any income." +us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Treated income above 130% FPL as disqualifying. It never applied California's universal school meals program, which supersedes the federal income test and gives the K-12 child free meals." +us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,Relied on the federal 130% FPL free-meal cutoff alone and missed California's universal free school meals program. Under that program the 9-year-old K-12 child qualifies regardless of income. +us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Denied eligibility on an income threshold. It ignored that California's universal school meals program has no income threshold and gives free meals to every enrolled K-12 student, including Child 1." +us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,Applied the 130% FPL federal free-meal test and never applied California's universal school meals program. That program makes the 9-year-old K-12 child eligible for free meals at any income. +us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Used an income threshold to deny free meals. It missed California's universal free school meals program, which drops the income test entirely for enrolled K-12 students." +us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,Denied eligibility on the federal income threshold and overlooked California's universal school meals program. That program provides free meals to the K-12 child regardless of household income. +us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"Treated the 130% FPL federal free-meal limit as decisive. It never considered California's universal free school meals program, which covers every enrolled K-12 student at any income." +us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Denied free meals on an income threshold. It missed California's universal school meals program, under which the 9-year-old K-12 child qualifies for free meals with no income test." +us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Compared about $156,588 of AGI to 130% FPL and denied eligibility. It never applied California's universal school meals program, which makes the K-12 child eligible for free meals regardless of income." +us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Applied the 130% FPL limit (about $43,000) as the only route to free meals and missed California's universal free school meals program. That program covers the 9-year-old K-12 student at any income." +us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"Denied eligibility because income exceeds the free-meal thresholds. It ignored California's universal school meals program, which has no income threshold for enrolled K-12 students." +us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed wage income above federal thresholds makes the household ineligible. It never applied California's universal free school meals program, under which PolicyEngine gives the K-12 child the FREE tier regardless of income." +us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL free-meal limit and checked for categorical eligibility, but missed California's universal school meals program. That program makes the K-12 child eligible for free meals at any income." +us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Denied free meals on an income threshold and overlooked California's universal school meals program. That program gives every enrolled K-12 student, including Child 1, free meals with no income test." +us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Relied on the federal free-meal income threshold. It missed California's universal free school meals program, which makes the 9-year-old K-12 child eligible regardless of income." +us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied only the federal free-meal income threshold and ignored California's universal school meals program. Under that program PolicyEngine gives the K-12 child positive free-meal support at any income. +us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,Treated high income as disqualifying. It missed that California's universal school meals program provides free meals to all enrolled K-12 students regardless of income. +us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Considered only the 130% FPL test and categorical eligibility. It never applied California's universal free school meals program, which gives the K-12 child free meals at any income." +us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked the 130% FPL limit and categorical certification through SNAP, TANF, or Medicaid, but overlooked California's universal school meals program. That program is a separate route that makes the 9-year-old K-12 child eligible regardless of income." +us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Applied the 130% FPL (about $43,000) federal free-meal limit as decisive. It missed California's universal free school meals program, which covers the enrolled K-12 child at any income." +us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Tested income against 130% of the 2026 poverty guideline and looked for categorical factors such as SNAP, TANF, or foster care. It omitted California's universal school meals program, which gives free meals to the K-12 child regardless of income." +us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no parseable value or explanation for free_school_meals_eligible, so there was no answer to score. The correct derivation yields 1, because California's universal school meals program makes the 9-year-old K-12 child eligible for free meals." +us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"Denied eligibility on free-meal income limits. It never applied California's universal school meals program, which has no income limit for enrolled K-12 students." +us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Treated income at or below 130% FPL (about $43,000) as a requirement for free meals. It missed California's universal free school meals program, under which the K-12 child qualifies at any income." +us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"Named California but still applied only the federal 130% FPL test. It missed California's universal school meals program, which gives every enrolled K-12 student free meals regardless of household income." +us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,Denied eligibility on a free-meal income threshold and overlooked California's universal school meals program. That program makes the 9-year-old K-12 child eligible for free meals at any income. us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Correctly identified CA SDI as an uncapped mandatory employee tax but applied the 2025 rate of 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145), landing on $14,602.50 in its own reasoning. It then submitted $14,118, a figure matching neither its stated total nor any component sum, compounding a rate error with a transcription error." -us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Computed only federal employee FICA (6.2% + 1.45% on $165,000 = $12,622.50) and omitted California's mandatory employee SDI contribution entirely, despite the output definition explicitly including mandatory employee state payroll taxes. The missing 1.3% uncapped SDI on $165,000 is exactly the $2,145 shortfall." -us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Asserted that ""California has no state-mandated employee payroll tax,"" conflating SDI with state income tax withholding. California imposes a mandatory employee SDI contribution at 1.3% of all wages with no taxable wage ceiling for 2026, worth $2,145 here, which the model dropped to return federal FICA alone." -us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied a stale 1.2% CA SDI rate rather than the 2026 rate of 1.3%, reaching $14,602.50 in its own recomputation. It then submitted $13,197.50, which corresponds to no step in its derivation, so the answer is wrong on both the SDI parameter and the reported total." -us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Computed CA SDI at a stale 1.1% ($1,815) and then discarded it on the reasoning that ""payroll_tax here focuses on federal FICA per typical benchmark,"" submitting $12,622.50. The output definition includes mandatory employee state payroll taxes, and 2026 CA SDI is 1.3% uncapped = $2,145." -us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Reported only Social Security ($10,230) and Medicare ($2,392.50), omitting the mandatory California employee SDI contribution of 1.3% on all $165,000 of wages ($2,145). It also truncated Medicare to $2,392 rather than $2,392.50." -us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly recognized that SB 951 removed the CA SDI wage cap and that SDI belongs in the total, but used a 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145). Every other component matched the reference, so the entire $330 gap is the SDI rate." -us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"First declared CA SDI ""voluntary/not modeled here as mandatory,"" then computed it at a stale 1.1%, then abandoned both paths and submitted $12,903.50, a number equal to neither its federal-only figure ($12,622.50) nor its SDI-inclusive figure ($14,437.50). CA SDI is a mandatory employee contribution at 1.3% uncapped in 2026, worth $2,145." -us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Included CA SDI as a mandatory employee tax but priced it at approximately 1.1% ($1,815) rather than the 2026 rate of 1.3% ($2,145). Its federal FICA of $12,622.50 was exact, so the shortfall is purely the SDI rate." -us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Subtracted the $24,597 of employer-sponsored insurance premiums from wages to get a FICA base of $140,403, but PolicyEngine applies employee payroll taxes to the full $165,000 of gross employment income. It compounded this with a 1.1% CA SDI rate instead of the 2026 rate of 1.3%." -us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Reduced the payroll tax base to $140,403 by netting out $24,597 of employer-sponsored insurance premiums, when the base is the full $165,000 of gross wages. It also used a 1.1% CA SDI rate rather than the 2026 rate of 1.3%, understating SDI by $600." -us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Structured the calculation correctly as Social Security plus Medicare plus CA SDI on $165,000, but applied a 1.1% SDI rate ($1,815) instead of the 2026 rate of 1.3% ($2,145). The $330 error is entirely the stale SDI parameter." -us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Listed only Social Security and Medicare on $165,000 and omitted California's mandatory employee SDI contribution, which the payroll_tax definition explicitly includes. CA SDI at 1.3% uncapped on $165,000 adds the missing $2,145." -us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Applied FICA to $140,403 after deducting $24,597 of employer-sponsored insurance premiums, whereas the payroll tax base is the full $165,000 of gross wages. It then applied CA SDI at 1.1% to that same reduced base instead of 1.3% on gross wages, propagating the base error into the state component." -us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Included all three correct components on the correct $165,000 base but used a 1.1% CA SDI rate ($1,815) rather than the 2026 rate of 1.3% ($2,145). The answer is short by exactly the 0.2-point rate difference on $165,000." -us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported ""Social Security and Medicare"" only, omitting the mandatory CA employee SDI contribution of $2,145. Its $12,963 does not even reconcile to federal FICA, which is 7.65% of $165,000 = $12,622.50, so it overstated the federal piece by $340.50 while dropping the state piece entirely." -us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Named Social Security, Medicare and CA SDI as the components, but its $13,280 total leaves only $657.50 above exact federal FICA of $12,622.50, an implied SDI rate near 0.40% rather than the 1.3% uncapped 2026 rate. It priced SDI at less than a third of the $2,145 actually owed." -us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 is exactly 8.85% of $140,403, i.e. FICA plus a 1.2% SDI rate applied to wages after subtracting $24,597 of employer-sponsored insurance premiums. Both moves are wrong: the base is the full $165,000 of gross wages, and the 2026 CA SDI rate is 1.3%." -us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Identified the right three components, but its $14,103 total sits only $1,480.50 above federal FICA of $12,622.50, an implied SDI rate of about 0.90% instead of the 2026 CA rate of 1.3%. Correct SDI on the uncapped $165,000 base is $2,145." -us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"Applied the Social Security wage base to the household's combined $165,000 rather than per worker, producing spurious capped figures of $10,918.20 and $10,465.20 when neither earner approaches the cap and the correct tax is 6.2% x $165,000 = $10,230. It omitted CA SDI entirely and then submitted $12,632.40, which matches none of its own three stated totals." -us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"Correctly kept 401(k) deferrals in the payroll tax base and correctly treated CA SDI as mandatory and uncapped, but used the 2025 rate of 1.2% ($1,980) rather than the 2026 rate of 1.3% ($2,145). The $165 gap is exactly the one-year rate increase." -us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Stated that ""both adults have wage earnings subject to employee FICA only,"" omitting California's mandatory employee SDI contribution of 1.3% on all $165,000 of wages. That omission is the full $2,145 shortfall, and it also truncated the total from $12,622.50 to $12,622." -us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Limited the calculation to Social Security and Medicare, dropping the $2,145 of mandatory CA employee SDI. Its $13,577 does not reconcile to its own stated method either, since 7.65% of $165,000 is $12,622.50, so it overstated federal FICA by $954.50." -us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Assembled the correct three components on the correct $165,000 base but estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). Every federal figure matched the reference exactly." -us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"Claimed CA SDI of ""approximately 1.2%"" but its $14,312.40 total is only $1,689.90 above federal FICA of $12,622.50, an implied SDI rate of 1.02%. The 2026 CA SDI charge on $165,000 of uncapped wages is $2,145 at 1.3%." -us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"Reproduced Social Security ($10,230) and Medicare ($2,392.50) exactly but estimated CA SDI at $1,980, i.e. the 2025 rate of 1.2%, rather than $2,145 at the 2026 rate of 1.3%. The entire $165 miss is the SDI rate parameter." -us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"Used a stale 1.2% CA SDI rate to get ""about $1,980"" instead of $2,145 at the 2026 rate of 1.3% on the uncapped $165,000 base. It additionally rounded Medicare to $2,393 and the total to a whole dollar." -us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Reported only ""employee SS 6.2% and Medicare 1.45%"" and omitted California's mandatory employee SDI, which the payroll_tax definition explicitly requires. Adding 1.3% on the uncapped $165,000 wage base contributes the missing $2,145." -us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Correctly included CA SDI on uncapped wages but applied a 1.1% rate ($1,815) instead of the 2026 rate of 1.3% ($2,145). Federal FICA of $12,622.50 matched the reference exactly, leaving the $330 gap entirely attributable to the stale SDI rate." -us,scenario_099,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Treated CA SDI as mandatory and uncapped, the correct structure, but priced it at the 2025 rate of 1.2% ($1,980) rather than the 2026 rate of 1.3% ($2,145). Its federal FICA of $12,622.50 was exact." -us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Cut the FICA wage base from $165,000 to $140,403 by subtracting $24,597 of employer-sponsored health premiums, when PolicyEngine applies payroll taxes to full gross employment income. It then omitted CA SDI altogether, so it missed both $1,881.67 of federal FICA and the entire $2,145 of state SDI." -us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"Built the correct three-component structure on the correct $165,000 base but applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). It also rounded its own $14,602.50 up to $14,603." -us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required submit_outputs key was absent rather than substantively wrong. The correct answer is $10,230 Social Security plus $2,392.50 Medicare plus $2,145 of CA SDI at 1.3% on $165,000 = $14,767.50." -us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model produced no parseable payroll_tax value or explanation, so this is a contract failure rather than a computation error. The correct derivation is 6.2% and 1.45% on $165,000 of wages plus uncapped CA employee SDI at 1.3%, totaling $14,767.50." -us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Its own reasoning derived the employee-side total of $12,622.50 ($6,502.50 + $6,120), then it submitted $25,290, essentially the combined employer-plus-employee 15.3% on $165,000, which the prompt expressly excludes. It also omitted the $2,145 of mandatory CA employee SDI." -us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Stated flatly that ""no modeled employee-side state payroll tax applies"" in California, when CA imposes a mandatory employee SDI contribution at 1.3% of all wages with no taxable wage ceiling in 2026. That omission of $2,145 is the sole difference from the reference." -us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Recognized that the prompt's ""mandatory employee state payroll taxes"" language pulls CA SDI in, computed it at a stale 1.1% with a nonexistent $153,164 wage cap, reached $14,437.50, then reversed itself and submitted the federal-only $12,622.50. CA SDI for 2026 is 1.3% on all wages with no cap, or $2,145." -us,scenario_099,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"Applied CA SDI to the head's $85,000 only, ignoring the spouse's $80,000, and even for the head reported $472.20, which is 0.56% rather than the 1.2% it claimed or the correct 2026 rate of 1.3%. Correct SDI is 1.3% on the household's full $165,000 of uncapped wages = $2,145." +us,scenario_099,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). Its own sum came to $14,602.50, but it submitted $14,118, which no step of its derivation supports." +us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($10,230 SS + $2,392.50 Medicare) and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145. The prompt explicitly includes that tax in payroll_tax." +us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed California has no state-mandated employee payroll tax, so it dropped CA SDI. That tax is 1.3% of all wages in 2026 ($2,145), and leaving it out left only FICA of $12,622.50." +us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a CA SDI rate of 1.2% ($1,980) instead of 2026's 1.3% ($2,145). Its own recomputed total was $14,602.50, yet it submitted $13,197.50, an arithmetic inconsistency." +us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It computed CA SDI at 1.1% ($1,815) and then dropped it on the assumption that the benchmark 'focuses on federal FICA'. The prompt defines payroll_tax to include mandatory employee state payroll taxes, and the correct SDI is 1.3% × $165,000 = $2,145." +us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It reported only SS ($10,230) and Medicare, truncating Medicare to $2,392. It left out California's mandatory 1.3% employee SDI ($2,145) entirely." +us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly treated CA SDI as uncapped but applied the 2024 rate of 1.1% ($1,815). The 2026 rate is 1.3% ($2,145), so its answer is $330 short." +us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It first called CA SDI voluntary, then briefly added it at 1.1%, and finally submitted $12,903.50, a figure none of its steps produce. The correct answer adds mandatory SDI at 1.3% of $165,000 ($2,145) to FICA of $12,622.50." +us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145) and rounded the result to $14,438." +us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the employer-sponsored insurance premiums ($24,597) from gross wages as pre-tax deductions, cutting the FICA/SDI base to $140,403. These are listed employer premiums, not employee salary reductions, so the base is the full $165,000. It also used a 1.1% SDI rate instead of 1.3%." +us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It treated the $24,597 of employer-sponsored insurance premiums as pre-tax salary reductions and taxed only $140,403 of wages. SS, Medicare and SDI all apply to the full $165,000 of gross wages. It also used a 1.1% SDI rate instead of 2026's 1.3%." +us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145), which leaves it $165 short." +us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a CA SDI rate of 1.1% ($1,815) instead of the 2026 rate of 1.3% on $165,000 ($2,145)." +us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare. It left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145, which the payroll_tax definition includes." +us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted $24,597 of employer-sponsored insurance premiums from gross wages as if they were pre-tax employee deductions, taxing only $140,403. The FICA and SDI base is the full $165,000. It also used a 1.1% SDI rate instead of 1.3%." +us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 uncapped rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"By its own description it included only Social Security and Medicare, leaving out California's mandatory 1.3% SDI ($2,145). Its $12,963 does not even match correct FICA of $12,622.50 on $165,000 of wages." +us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It said it included CA SDI, but full-wage FICA of $12,622.50 leaves only $657.50 of SDI in its $13,280 total. Correct SDI is 1.3% × $165,000 = $2,145, so it badly understated the state disability tax." +us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 equals exactly 8.85% × $140,403. That means it took $24,597 of employer-sponsored insurance premiums out of the wage base and used a 1.2% SDI rate. The correct result is 7.65% FICA plus 1.3% SDI on the full $165,000." +us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $14,103 leaves about $1,480 for SDI above full-wage FICA of $12,622.50, roughly a 0.9% rate. The 2026 CA SDI rate is 1.3%, which gives $2,145." +us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It applied 6.2% to the wage base cap ($176,100) instead of to actual wages, then landed on an unexplained $12,632.40. Both earners are under the cap, so SS is $10,230. It also left out California's mandatory 1.3% SDI ($2,145) entirely." +us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"It correctly treated CA SDI as mandatory and uncapped but used a 1.2% rate ($1,980) instead of 2026's 1.3% ($2,145)." +us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It treated both adults as subject to employee FICA only and left out California's mandatory employee SDI of 1.3% × $165,000 = $2,145. It also truncated the FICA total to $12,622." +us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It counted only employee FICA and left out California's mandatory 1.3% SDI ($2,145). Its $13,577 also overstates FICA, which is exactly $12,622.50 (7.65% of $165,000)." +us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"It estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It stated a CA SDI rate of about 1.2% instead of the 2026 rate of 1.3%. Its $14,312.40 does not even match its own inputs, since FICA of $12,622.50 plus 1.2% SDI ($1,980) is $14,602.50." +us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"It estimated CA SDI at $1,980 (1.2% of wages) instead of the 2026 rate of 1.3%, which gives $2,145." +us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used CA SDI of about $1,980 (a 1.2% rate) instead of 1.3% × $165,000 = $2,145, then rounded to $14,603." +us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It computed only employee SS and Medicare (about $12,623) and left out California's mandatory 1.3% employee SDI of $2,145." +us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"It applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"It correctly added mandatory CA SDI but used a 1.2% rate ($1,980) instead of 2026's 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"It treated CA SDI as uncapped but applied 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)." +us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $24,597 of employer-sponsored insurance premiums from gross wages and computed FICA on only $140,403 instead of the full $165,000. It also left out California's mandatory 1.3% SDI ($2,145) entirely." +us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"It applied CA SDI at 1.2% ($1,980) instead of 2026's 1.3% ($2,145) and rounded the total to $14,603." +us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The correct answer is FICA of $12,622.50 plus CA SDI of $2,145, for $14,767.50." +us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation. The correct answer is FICA of $12,622.50 plus CA SDI of $2,145, for $14,767.50." +us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Its per-person employee FICA ($6,502.50 + $6,120 = $12,622.50) was right, but it reported $25,290, roughly double. That number is consistent with adding the employer share, which the prompt excludes. It also left out California's mandatory 1.3% SDI ($2,145)." +us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that no employee-side state payroll tax applies and stopped at FICA of $12,622.50. California's SDI is a mandatory employee payroll tax of 1.3% on all wages in 2026 ($2,145)." +us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It recognized that the prompt's definition includes mandatory CA SDI and computed it, though at an outdated 1.1% rate with a wage cap that no longer exists. It then went back and submitted FICA only ($12,622.50), leaving out the 1.3% SDI of $2,145." +us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied CA SDI to the head's wages only, ignoring the spouse's $80,000. It also got the arithmetic wrong: 1.2% of $85,000 is $1,020, not $472.20. Correct SDI is 1.3% of both earners' $165,000, which is $2,145." us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -7162,45 +7868,52 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_ us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Built a near-correct base — CA AGI of $160,066 and $21,275 of itemized deductions — then ran $138,791 through a bracket schedule that produced $6,940 of tentative tax, where the indexed 2026 MFJ brackets yield $5,778.21 on $139,108.47; that $1,160 rate overstatement carries directly into the answer. It compounded this by dropping the $338 educator and $144 IRA above-the-line deductions from CA AGI and by deducting the full $12,621 of donations instead of $11,823.08 after the 2026 0.5%-of-AGI charitable floor." -us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Added roughly $4,406 of medical expenses to its itemized total; the allowed CA itemized deduction here is $20,476.57 — real estate taxes of $8,653.50 plus floor-reduced charity of $11,823.08 — with no medical component, because the 7.5%-of-AGI threshold of $11,968.88 leaves nothing deductible. That phantom deduction cut taxable income to $134,241 instead of $139,108.47 and the tentative tax to $5,507 instead of $5,778.21." -us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used a $4,754 married-filing-jointly standard deduction when California's 2026 joint standard deduction is $11,412, and never itemized at all, discarding the $20,476.57 of real estate tax and charitable deductions the household claims. It also omitted the $1,010 of interest, $144 of dividends and the $3,000 capital-loss offset from income, and replaced the $1,284.46 of CA exemption credits with an invented $146 child credit plus a $543 'educator expense benefit'." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Derived $4,680 in its own work (about $6,200 of tentative tax less roughly $1,520 of credits) and then discarded that figure to submit $5,959 with no supporting arithmetic. Its inputs were also off in both directions: the 2026 MFJ brackets give $5,778.21 on $139,108.47, not $6,200, and the CA exemption credits total $1,284.46, not $1,520." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Added the $338 educator expense back to CA AGI and then added it a second time, reaching $163,010 instead of $159,585.05, and placed $141,735 of taxable income in California's 9.3% bracket to get roughly $7,973 of tentative tax. The 2026 joint 9.3% bracket begins near $149,000, so $139,108.47 tops out at 8% and yields $5,778.21 — a $2,200 rate overstatement that swamps its otherwise reasonable $1,256 credit estimate." -us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Estimated $7,700 of tentative CA tax on roughly $138,000 of taxable income, about $1,900 above the $5,778.21 the 2026 MFJ brackets produce on $139,108.47 — an average rate of 5.6% where the correct one is 4.15%. It then listed four exemption credits of $150, $150, $500 and $500 but subtracted only $450 of that $1,300, against an actual $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Reconstructed the base almost exactly, reaching $138,791 of taxable income against $139,108.47, then subtracted $1,800 of exemption credits by valuing all four at about $450 each; California's 2026 credits run roughly $157 per personal exemption and $485 per dependent, totaling $1,284.46. That $516 over-credit, partly offset by a $211 bracket overstatement, produced the $305 shortfall." -us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Applied the federal $10,000 SALT cap to the California return and never settled a deduction figure, guessing '$18,000-$20,000' where CA allows the full $8,653.50 of real estate taxes plus $11,823.08 of floor-reduced charity for $20,476.57, then asserted taxable income of about $150,000 instead of $139,108.47. Its explanation claims to subtract exemption credits yet reports $6,800 both before and after, so the $1,284.46 of credits never came out." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Claimed a $2,752 medical itemized deduction that the 7.5%-of-AGI floor of $11,968.88 eliminates entirely, and set CA AGI at $163,306, $3,721 above the $159,585.05 that results from removing the $3,088 of 401(k) deferrals and capping the capital loss at $3,000. It then used $6,042 of tentative tax where the brackets give $5,778.21 and about $1,000 of exemption credits against the actual $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its implied income base of $134,987 is exactly CA AGI of $159,585.05 less $24,597 — the head's $18,708 and spouse's $5,889 of employer-sponsored insurance premiums — so it excluded ESI premiums from wages that the $165,000 gross figure already reports net of. That drove taxable income to $112,436 instead of $139,108.47 and tentative tax to $4,551.40 instead of $5,778.21, even though its $1,288 credit figure was within $4 of the correct $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Matched the $1,284.46 of exemption credits and lost the case entirely on the base: its roughly $135,069 AGI is CA AGI of $159,585.05 minus the $24,597 of employer-sponsored insurance premiums that gross wages already exclude, and its $27,545 of itemized deductions exceeds the allowed $20,476.57 by adding medical costs the 7.5%-of-AGI floor disallows. Taxable income came out at $107,524 against $139,108.47, so tentative tax was $3,545 rather than $5,778.21." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Structured the computation correctly and mis-set both California parameters: it applied about $1,180 of personal and dependent exemption credits against the actual $1,284.46, and its implied tentative tax of roughly $5,845 sits $67 above the $5,778.21 the 2026 joint brackets produce on $139,108.47 of taxable income." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"$7,845 is what California's 2026 joint brackets produce on the entire undeducted $159,585.05 of CA AGI, so it applied no deduction at all — the $8,653.50 of real estate taxes and $11,823.08 of floor-reduced charity total $20,476.57 and bring taxable income to $139,108.47 — and it also never subtracted the $1,284.46 of exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $135,169 CA AGI is the correct $159,585.05 less roughly the $24,597 of employer-sponsored insurance premiums, which are already excluded from the $165,000 of reported wages, and it then took $26,138 of itemized deductions where California allows $20,476.57. Taxable income of $109,031 instead of $139,108.47 dropped tentative tax to about $3,900 against $5,778.21, and it applied only about $600 of the $1,284.46 in exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Gave no figures; $2,570 corresponds to a taxable base near $105,000 once the $1,284.46 of exemption credits are applied, roughly $34,000 below the $139,108.47 that CA AGI of $159,585.05 less $20,476.57 of itemized deductions produces. A gap that size is consistent with stripping the $24,597 of employer-sponsored insurance premiums out of already-net wages and adding a medical deduction the 7.5%-of-AGI floor disallows." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Asserted $8,725 with no derivation, a figure that exceeds even the tax California's 2026 joint brackets produce on the entire undeducted $159,585.05 of CA AGI. It therefore taxed gross income with neither the $20,476.57 itemized deduction that brings taxable income to $139,108.47 nor the $1,284.46 of personal and dependent exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Submitted a bare figure $366 below the reference: $4,128 requires either $5,412 of tentative tax — a taxable base about $4,575 under the correct $139,108.47 — or $1,650 of exemption credits against the actual $1,284.46. A $4,575 base reduction is consistent with allowing a medical itemized deduction, which the 7.5%-of-AGI floor of $11,968.88 disallows in full here." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Named the right structure — CA itemized deductions and exemption credits — but landed $438 low: $4,056 implies $5,340 of tentative tax where the 2026 joint brackets give $5,778.21 on $139,108.47 of taxable income, the result of over-indexing the bracket thresholds by roughly 4%." -us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"$5,245 is the tax that results from taking California's $11,412 joint standard deduction instead of the $20,476.57 of itemized deductions: $148,173 of taxable income yields about $6,530 of bracket tax, and the $1,284.46 of exemption credits leaves roughly $5,246. Itemizing wins by $9,065 here because CA imposes no SALT cap on the $8,653.50 of real estate taxes and allows $11,823.08 of charity after the 0.5%-of-AGI floor." -us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Asserted that charitable donations are deductible only to the extent they exceed the standard deduction and so refused to itemize, after mis-stating California's joint standard deduction as $6,566 when it is $11,412 and the itemized total is $20,476.57. It also subtracted the $338 educator expense as though it were a credit rather than an AGI adjustment, and then submitted $3,202.96 while its own explanation derives $5,559." -us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Took an approximate $11,500 standard deduction rather than the larger $20,476.57 of itemized deductions, leaving about $149,000 of taxable income against $139,108.47 and roughly $6,770 of tentative tax against $5,778.21. It then applied only about $620 of exemption credits versus $1,284.46 and added a $240 California child and dependent care credit that is unavailable, since CA's CDCC is limited to AGI of $100,000 or less and this household is at $159,585.05." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Submitted $6,713 with no computation, $2,219 above the reference. That level is consistent with a run that neither itemized the $20,476.57 of real estate tax and floor-reduced charitable deductions — $9,065 more than California's $11,412 joint standard deduction — nor subtracted the $1,284.46 of personal and dependent exemption credits from the $5,778.21 of bracket tax." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Gave no derivation; $4,931 implies $6,215 of tentative tax net of the $1,284.46 of exemption credits, $437 above the $5,778.21 that the indexed 2026 joint brackets produce on $139,108.47 of taxable income. That overshoot corresponds to bracket thresholds set roughly 5% below their 2026 inflation-adjusted levels." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Subtracted 'health-premium exclusions' from wages that are already reported net of them, putting CA AGI at $134,323 instead of $159,585.05 — a $25,262 cut close to the $24,597 of employer-sponsored premiums — and then reported $6,882 of tax on its own $101,811 base, which California's joint brackets tax at about $3,100. The submitted number follows neither the correct chain ($139,108.47 taxable, $5,778.21 of tax, $1,284.46 of credits) nor its own arithmetic." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Stated that it applied the California standard deduction; itemizing is the larger deduction here, with $8,653.50 of real estate taxes (no SALT cap in CA) plus $11,823.08 of charity after the 2026 0.5%-of-AGI floor totaling $20,476.57 against an $11,412 standard deduction — a $9,065 base difference worth about $725 at the 8% marginal rate. That plus a bracket estimate roughly $280 high accounts for the $996 overstatement." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Deducted the full $12,621 of donations rather than $11,823.08 after the 2026 0.5%-of-AGI charitable floor, and used $160,066 of CA AGI by omitting the $144 IRA and $338 educator above-the-line deductions that bring it to $159,585.05. Those two offsetting errors left it at $138,791 of taxable income, and it then read a bracket tax below the $5,778.21 the 2026 joint schedule produces." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Submitted $4,250 with no derivation, $244 below the correct chain of $139,108.47 of taxable income, $5,778.21 of bracket tax and $1,284.46 of exemption credits. The shortfall is consistent with deducting the full $12,621 of donations instead of $11,823.08 after the 2026 0.5%-of-AGI charitable floor and over-sizing the four California exemption credits." -us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its stated inputs isolate the errors: CA AGI of $160,066 omits the $144 IRA and $338 educator above-the-line deductions that bring it to $159,585.05, and its $21,275 of itemized deductions uses the full $12,621 of charity rather than $11,823.08 after the 2026 0.5%-of-AGI floor, for $20,476.57. Its $5,871.38 of tentative tax also exceeds the $5,778.21 the brackets give, and its $1,256 of credits falls $28 short of $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Wrote a single line — 'CA state income tax after deductions on ~165k income approx 4.5k' — and rounded to $4,500 without deriving CA AGI of $159,585.05, the $20,476.57 of itemized deductions, the $5,778.21 of bracket tax on $139,108.47, or the $1,284.46 of exemption credits. The unworked round number misses by $6.26." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Deducted the full $12,621 of donations instead of $11,823.08 after the 2026 0.5%-of-AGI charitable floor and set CA AGI at $159,728, giving $138,453 of taxable income against $139,108.47. It then read about $5,890 of tentative tax off its bracket schedule where its own base implies roughly $5,726 and the correct base gives $5,778.21, a $164 rate overstatement against a $1,290 credit estimate that was nearly right." -us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Deducted the full $12,621 of charity rather than $11,823.08 after the 2026 0.5%-of-AGI floor and used $160,066 of CA AGI, omitting the $144 IRA and $338 educator adjustments — two errors that nearly cancel — then read $5,743 of tentative tax and $1,246 of exemption credits where the correct figures are $5,778.21 and $1,284.46. The residual bracket and credit rounding leaves it $3.26 off." -us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Stopped its bracket walk in the 6% band on $110,811 of taxable income, about $28,300 below the $139,108.47 that CA AGI of $159,585.05 less $20,476.57 of deductions produces — a gap the size of excluding the $24,597 of employer-sponsored premiums already netted out of wages — and then declared that 'no nonrefundable credits apply at this income level.' California's two personal and two dependent exemption credits total $1,284.46 and apply in full here; they phase out only far above this household's AGI." -us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Built the base within $620 of the reference at $138,491 of taxable income, then applied only $576 of exemption credits where California's two personal plus two dependent credits total $1,284.46 — a $708 under-credit. Its $5,866 of tentative tax also exceeds the $5,778.21 the 2026 joint brackets produce." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing was submitted against the required $4,493.74. The failure is a missing output rather than a substantive computation error." -us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing was submitted against the required $4,493.74. The failure is a missing output rather than a substantive computation error." -us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Submitted $0 with a truncated one-line explanation. A California joint return with $159,585.05 of AGI and $20,476.57 of itemized deductions has $139,108.47 of taxable income and $5,778.21 of bracket tax; zero liability would require taxable income small enough for the $1,284.46 of exemption credits to erase the tax, roughly under $40,000." -us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Asserted that claiming the federal standard deduction forces the California standard deduction; California lets a filer itemize on Schedule CA regardless of the federal election, and here $8,653.50 of real estate taxes plus $11,823.08 of floor-reduced charity total $20,476.57 against an $11,412 standard deduction. That forced roughly $147,800 of taxable income instead of $139,108.47 and about $6,800 of tentative tax instead of $5,778.21, while its $1,260 credit estimate was close to the actual $1,284.46." -us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Set CA AGI at $159,676 by netting the entire $7,018 long-term capital loss instead of applying the $3,000 annual capital-loss limitation and by never removing the $3,088 of traditional 401(k) deferrals from wages; the correct AGI is $159,585.05. It also deducted the full $12,621 of donations rather than $11,823.08 after the 0.5%-of-AGI floor, stated that 'no nonrefundable credits apply' where the exemption credits total $1,284.46, and cited a 9.3% band of $131,294-$157,559 that belongs to the single-filer schedule, not the joint one." -us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Applied the federal married-filing-jointly standard deduction of $34,000 on the California return, where the standard deduction is $11,412 and the household itemizes $20,476.57, then computed $8,100.31 of tax on its own $130,894 base when CA's joint brackets produce about $5,100 there. It closed with a $41.16 renter's credit in place of the $1,284.46 of exemption credits; California's nonrefundable renter's credit is $120 and is unavailable at this household's $159,585.05 of AGI." +us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It got taxable income roughly right (~$138,791 after $21,275 itemized) but priced it at ≈$6,940, about $1,160 more than the frozen reference's $5,778 and about $1,070 more than the $5,871 that the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, give on $138,791. Both that schedule and the engine's projected one keep ~$139k in the 8% bracket. It then reported $6,222, which contradicts its own post-credit figure of $5,684; its charity deduction without the 0.5%-of-AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor." +us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a ~$4,406 medical deduction by counting the $2,500 premium twice per adult (both premium fields) and adding over-the-counter items. The real deductible medical costs ($5,000 of premiums plus $5,000 of other medical) are below the 7.5%-of-AGI floor of ~$11,969, so the medical deduction is zero. Itemized deductions are $20,477 (charity after the 0.5% AGI floor plus property tax), not $25,681, which drove taxable income down to ~$134,241." +us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used a $4,754 standard deduction instead of itemizing $20,477 of charity (after the 0.5% AGI floor) and property tax, which exceeds the $11,412 CA MFJ standard deduction. It also deducted unreimbursed employee expenses above the line. In place of the $1,284.46 personal/dependent exemption credits it subtracted an invented $146 'child tax credit' and a $543 educator-expense 'credit'." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income (~$138,309) was close to the frozen reference's $139,108, but it guessed bracket tax at ~$6,200, above both the $5,778 the frozen reference's projected 2026 CA MFJ schedule produces and the $5,897 that the 2025 schedule, which the release's California convention (c_ca_hold_2025) holds for 2026, gives on $139,108. It then raised its own ~$4,680 net to $5,959 by 'adjusting for SDI'. SDI is a payroll tax and is not part of state income tax." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It started from $165,000 of gross wages without excluding the $3,088 of traditional 401(k) deferrals and added back the educator deduction, which inflated CA AGI to ~$163,010 instead of $159,585. It then priced $141,735 as if it were deep in the 9.3% bracket (~$7,973). Under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected 2026 MFJ schedule, income near $139k sits in the 8% bracket (for $5,778 of tax in the frozen reference)." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of ~$138,000 was about right, but it priced it at ~$7,700, roughly $1,900 above the frozen reference's projected 2026 CA MFJ schedule ($5,778 at $139,108) and $1,800 above the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,897 there). It then subtracted only ~$450 of exemption credits, although its own listed components add up to ~$1,300. The personal-plus-dependent credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection)." +us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity instead of $11,823 after the 0.5%-of-AGI charitable floor, and it left the $338 educator deduction out of CA AGI. Taxable income came to $138,647 instead of $139,108, so its bracket tax of $5,729 fell short of $5,778 and the final answer landed $53 low." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It valued each adult's personal exemption credit at ~$450 (the dependent amount) instead of about $150, which put total credits at $1,800 instead of $1,284.46. It also priced $138,791 at $5,989 using guessed brackets. Its taxable income omitted the 0.5% AGI charitable floor and the IRA/educator adjustments, and the overstated credits account for most of the $305 shortfall." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It listed the personal and dependent exemption credits but never subtracted them: its reported $6,800 equals its own pre-credit tax, while the credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference). It also guessed taxable income at ~$150,000 instead of $139,108 by under-counting itemized deductions, and it applied a federal-style $10k SALT cap that California does not have." +us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added a medical deduction of about $4.4k, counting the $2,500 premium twice per adult plus over-the-counter items. The household's actual deductible medical costs ($10,000) are below the 7.5%-of-AGI floor, so medical adds nothing. Itemized deductions are $20,477, not ~$25.7k, and taxable income is $139,108, not ~$134k." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It left the $3,088 of 401(k) deferrals in CA AGI ($163,306 instead of $159,585). It then added a $2,752 medical deduction built on double-counted premiums, although deductible medical costs are below the 7.5% floor. It overpriced the bracket tax at ~$6,042 and subtracted only ~$1,000 of dependent credits, leaving out the two personal exemption credits (the total is $1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026, and $1,284.46 in the frozen reference)." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $24,597 of employer-sponsored insurance premiums from income, cutting CA AGI to ~$134,987; the listed wages are already the taxable gross, so CA AGI is $159,585. The lower AGI shrank the 7.5% medical floor and produced a $1,276 medical deduction that should not exist. Taxable income came out at $112,436 instead of $139,108." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $24,597 of employer-sponsored insurance premiums from AGI (~$135,069 instead of $159,585). Against that reduced floor it claimed a ~$6,270 medical deduction by counting the $2,500 premium twice per adult plus over-the-counter items. With no medical deduction, itemized deductions are $20,477 and taxable income is $139,108, not $107,524." +us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $4,395 medical deduction by measuring $16,400 of spending (the $2,500 premium counted twice per adult, plus over-the-counter items) against a $12,005 floor. The actual $10,000 of premiums and medical costs is below 7.5% of AGI, so itemized deductions are only charity after the 0.5% AGI floor plus property tax ($20,477). That makes taxable income $139,108, not $134,396." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted about $1,180 of personal and dependent exemption credits instead of $1,284.46. Its answer implies ~$5,845 of pre-credit tax, above the $5,778 the frozen reference's projected 2026 CA MFJ brackets produce on $139,108 of taxable income but in line with the $5,844 behind the exclusion's corrected value ($4,588.48 plus the $1,256 of 2025 exemption credits that the release's California convention, c_ca_hold_2025, holds for 2026); against that value its $76.52 overstatement is the credit shortfall." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $7,845 matches CA bracket tax on roughly $163k of income, meaning it taxed near-gross income with no itemized deduction and no exemption credits. The frozen reference subtracts $20,477 of itemized deductions to reach $139,108 of taxable income, then $5,778 of tax on the engine's projected 2026 schedule less $1,284.46 of exemption credits; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $24,597 of employer-sponsored insurance premiums from CA AGI ($135,169 instead of $159,585). It then claimed a ~$4,860 medical deduction that the 7.5%-of-AGI floor eliminates, and it subtracted only ~$600 of exemption credits instead of $1,284.46." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no numbers. $2,570 plus the $1,284 exemption credits implies ~$3,850 of pre-credit tax, which corresponds to taxable income near $108k. That is about $31k below the correct $139,108, the same shortfall that comes from excluding the $24,597 of employer-sponsored insurance premiums from income and claiming a medical deduction the 7.5% floor eliminates." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $8,725 is more than the CA bracket tax on the household's entire ~$165k of gross wages, so it applied rates with no itemized deduction and no exemption credits. The frozen reference computes $139,108 taxable, $5,778 of tax, less $1,284.46 of credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,128 is $366 low, which matches adding a ~$4.5k medical deduction (taxable income ~$134.5k). The household's deductible medical costs are below the 7.5%-of-AGI floor, so itemized deductions stop at $20,477 and taxable income is $139,108." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,056 is $438 low, which equals 8% of about $5.5k in excess deductions, the pattern produced by claiming a medical deduction the 7.5%-of-AGI floor eliminates. The only allowable itemized deductions are charity after the 0.5% AGI floor ($11,823) and property tax ($8,654)." +us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $5,245 matches taking the $11,412 CA standard deduction (taxable ~$148,173, tax ~$6,500, less $1,284 of credits ≈ $5,220) instead of itemizing $20,477 of charity and property taxes." +us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the charitable gifts did not exceed its $6,566 standard deduction. In fact charity after the 0.5% AGI floor ($11,823) plus $8,654 of property tax totals $20,477, well above the $11,412 CA MFJ standard deduction, so the household itemizes. It also subtracted the $338 educator expense as if it were a credit, left out the $1,284.46 exemption credits, and reported $3,202.96, which matches none of its own intermediate figures." +us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the ~$11,500 standard deduction instead of itemizing $20,477 (charity after the 0.5% AGI floor plus $8,654 property tax), which overstated taxable income by ~$9k. It used ~$620 of exemption credits instead of $1,284.46. It also claimed a ~$240 CA child and dependent care credit, which is unavailable above $100,000 of AGI." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $6,713 matches bracket tax on ~$150k of income with no exemption credits, which means it used the standard deduction instead of $20,477 of itemized deductions and never subtracted the $1,284.46 personal and dependent exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,931 is $437 high, close to the result of pricing ~$139k of taxable income with 2024 CA brackets that were not indexed. Those brackets start 9.3% at $136,700, whereas under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected schedule, this income stays in the 8% bracket (the frozen reference gives $5,778 of tax before its $1,284.46 credits)." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted health insurance premiums from income (CA AGI $134,323 instead of $159,585). It also treated personal and dependent exemptions as deductions from taxable income rather than as the $1,284.46 of credits. It then reported $6,882, about double the ~$3,300 of CA tax its own $101,811 of taxable income produces." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the CA standard deduction ($11,412) instead of itemizing. Charity after the 0.5% AGI floor ($11,823) plus property tax ($8,654) totals $20,477, so its taxable income was about $9k too high." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity and left the $144 IRA and $338 educator deductions out of CA AGI ($160,066), putting taxable income at $138,791; that is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity), where the frozen reference applies the $798 floor and both deductions ($159,585 of CA AGI, $139,108 taxable). Its bracket and credit estimates then understated net tax by ~$86 against the frozen reference's projected 2026 schedule ($5,778 of tax, $1,284.46 of credits) and by ~$233 against the 2025 schedule and credits that the release's California convention (c_ca_hold_2025) holds for 2026 ($4,615.38 on $138,791), which is its whole gap to the fully law-correct $4,615.48 that also adds back the educator expense." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. $4,250 is $244 below $5,778 − $1,284.46, which requires about $3k more in deductions than the correct $20,477 itemized total. That excess comes from deducting charity without the 0.5%-of-AGI floor and claiming medical costs the 7.5%-of-AGI floor eliminates." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It priced taxable income with the 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and used the 2025 exemption credits ($1,256); the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. It also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It itemized charity and property tax, which leaves ~$5,800 of bracket tax, but its $5,138 answer implies only about $660 of exemption credits. The two personal and two dependent exemption credits total $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection). It also deducted full charity without the 0.5%-of-AGI floor." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It deducted full charity without the 0.5% AGI floor and dropped the educator deduction (taxable income $138,647 instead of $139,108). It then priced that income with the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, giving ~$5,860 of pre-credit tax where the frozen reference's projected 2026 schedule gives $5,778 on $139,108. That schedule difference drives its $72 overstatement against the frozen reference; against the exclusion's corrected value, $4,588.48, it is $22.48 low, mainly because its answer then implies about $1,294 of exemption credits where the convention's 2025 amounts give $1,256." +us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used the 'latest published' 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and the 2025 exemption credits of $1,256; the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. To reach that taxable income it also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a round ~$4,500 guess based on '~165k income' without working out itemized deductions, brackets or exemption credits. The frozen reference computes $139,108.47 taxable, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income ($138,453) is $655 low because it deducted full charity without the 0.5%-of-AGI floor. It then priced that income at ~$5,890, about $160 more than the frozen reference's projected 2026 CA MFJ brackets produce (~$5,726) and about $46 more than the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives (~$5,844), for a net overstatement of $106 against the frozen reference." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity without the 0.5%-of-AGI floor and left out the IRA and educator adjustments, understating taxable income by $317 ($138,791). It used ~$1,246 of exemption credits instead of $1,284.46. The errors mostly offset and left it $3.26 high." +us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $12,621 of charity instead of $11,823 after the 0.5%-of-AGI floor and left out the educator deduction. That understated taxable income by $617 ($138,491 instead of $139,108), and the answer landed $67 low." +us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $110,811 is about $28k too low, the shortfall produced by excluding the $24,597 of employer-sponsored insurance premiums from income. It also stated that no nonrefundable credits apply, but CA's personal and dependent exemption credits ($1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026; $1,284.46 in the frozen reference) are not reduced at ~$160k of AGI." +us,scenario_099,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted only $576 of exemption credits instead of $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 (two personal credits of $153 and two dependent credits of $475; the frozen reference's projection gives $1,284.46). It also priced its $138,491 of taxable income at $5,866, above what either that 2025 schedule (~$5,847) or the frozen reference's projected schedule gives; its charity deduction without the 0.5% AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor." +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits. The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48." +us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits. The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48." +us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 without any computation, as if the household owed no CA income tax. In fact $139,108 of taxable income generates $5,897 of bracket tax under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,778 on the frozen reference's projected schedule), and the $1,256 of exemption credits ($1,284.46 in the frozen reference) offsets only part of it." +us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It asserted that California requires the standard deduction when the federal standard deduction is claimed. California lets filers itemize regardless of the federal election, and $20,477 of itemized charity (after the 0.5% AGI floor) and property tax beats the $11,412 standard deduction. Its taxable income (~$147,800) was therefore about $8.7k too high." +us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It never excluded the $3,088 of 401(k) deferrals or the $144 IRA deduction, and it netted the full $6,478 capital loss instead of limiting it to $3,000. It then priced $138,063 at only $4,301.50, although the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,813 (the frozen reference's projected 2026 brackets, about $5,700). It also left out the $1,284.46 of personal and dependent exemption credits, claiming none apply." +us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It invented a $34,000 standard deduction instead of itemizing $20,477, and it priced $130,894 at $8,100 when the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,240 at that income. It left out the $1,284.46 of exemption credits and subtracted a $41 renter's credit that the household's ~$160k AGI makes unavailable." us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -7244,39 +7957,41 @@ us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_out us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model described a child aged 5 as “under 5,” which is the exact categorical error. Falling below the income threshold does not establish WIC eligibility for a child who has aged out." us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model interpreted “up to age 5” as including children who are already 5. WIC child categorical eligibility ends at the fifth birthday, regardless of the household's low income." us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model included age 5 in the WIC child category and then relied on the income limit. Because the child is not under five, the categorical test fails before income can establish eligibility." -us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its explanation derives both components correctly — EITC of 40% x $5,915 = $2,366 and refundable CTC of 15% x ($5,915 - $2,500) = $512 — and states the total as $2,878, matching the reference. It then submitted 6425 in the value field, a number unrelated to its own arithmetic, breaking the explanation-matches-value contract rather than making a tax error." -us,scenario_100,federal_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It reduced EITC earned income by the $3,859 traditional 401(k) deferral to $2,056, but the earned-income base for both the EITC and the refundable CTC is the full $5,915 of gross wages, giving a 40% phase-in credit of $2,365.89. That same erroneous base pushed earnings below $2,500 and zeroed the refundable CTC, which is in fact 15% x ($5,915 - $2,500) = $512.21." -us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It computed both pieces correctly — EITC 40% x $5,915 = $2,366 and refundable CTC 15% x ($5,915 - $2,500) = $512.25 — then discarded its own ACTC figure for an unsupported '$834 approximation for two children' and rounded the total to $3,200. The 15%-of-excess-earnings limitation is a single tax-unit computation with no per-child enlargement at this income level." -us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"After correctly deriving EITC $2,366 and ACTC $512 (total $2,878), it applied a nonexistent 'AGI test uses smaller of' haircut and cut the EITC to $1,906. The AGI comparison governs only the EITC phase-out; at $5,915 of earnings the household is in the 40% phase-in, where the credit equals 40% of earned income with no AGI-based reduction." -us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived EITC $2,366 and ACTC $512, summing to the correct $2,878, then shaved the total to $2,330 on the stated ground that the traditional 401(k) and IRA contributions lower AGI. A lower AGI does not reduce the EITC inside the phase-in range, and the refundable CTC is already fixed at 15% of earnings above $2,500." -us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It reached the correct $2,878 and then added the $710 FLSA overtime premium to earnings, raising the EITC to $2,905 and the total to $3,417. The prompt states that gross wage and salary amounts already include overtime pay, so the overtime premium is not incremental earned income on top of the $5,915." -us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its text derives EITC $2,366 and ACTC $512 and then overrides that with $6,295 attributed to 'IRS EITC tables,' a figure close to the two-child maximum credit. At $5,915 of earnings the household is in the 40% phase-in, so the maximum credit is unreachable and the correct total is $2,878.10." -us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used earned income of $6,625 — the $5,915 of wages plus the $710 FLSA overtime premium — inflating the EITC to $2,650 and the refundable CTC to $618.75. Gross wages already include overtime pay, so the base is $5,915 and the credits are $2,365.89 and $512.21." -us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut earned income to $928 by subtracting the $3,859 401(k) deferral and both $564 health-premium lines from the $5,915 of wages. Neither the elective deferral nor the health premiums reduce the earned-income base for the EITC or the refundable CTC, so the credits are $2,365.89 and $512.21, not $371 and $0." -us,scenario_100,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,859 traditional 401(k) deferral from wages to get a $2,056 earned-income base and a $822.40 EITC. Earned income for the EITC is the full $5,915 of gross wages ($2,365.89 at the 40% two-child phase-in), and that same $5,915 clears the $2,500 refundable-CTC threshold, producing $512.21 rather than the $0 it asserted." -us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It got the EITC right at 40% x $5,915 = $2,366 but applied a pre-TCJA $3,000 refundable-CTC earnings threshold, producing $437.25. The threshold in force for 2026 is $2,500, so the refundable CTC is 15% x ($5,915 - $2,500) = $512.21 and the total is $2,878.10." -us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It asserted 'the full Earned Income Tax Credit' plus a refundable CTC without applying either earnings limitation, landing on $6,848 — a figure consistent with the two-child maximum EITC plus a large ACTC. At $5,915 of earnings both credits are earnings-limited: the EITC is 40% of earned income ($2,365.89) and the refundable CTC is 15% of earnings above $2,500 ($512.21)." -us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It treated the $20,644 employer-sponsored insurance premium and the other premium lines as pre-tax payroll reductions that zero out wages, concluding earned income is $0. Those premium inputs do not reduce the $5,915 of gross wages that serve as earned income for the EITC and refundable CTC, which yield $2,365.89 and $512.21." -us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It claimed pre-tax health insurance premiums reduce taxable earned income to $0 and therefore denied both credits. The health-premium inputs leave the $5,915 of gross wages intact as the earned-income base, so the household receives the 40% two-child EITC phase-in of $2,365.89 plus $512.21 of refundable CTC." -us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It asserted zero refundable credits with no derivation. A head-of-household filer with two qualifying children and $5,915 of wages sits in the EITC's 40% two-child phase-in ($2,365.89) and above the $2,500 refundable-CTC earnings threshold ($512.21); the zero is consistent with treating the household as having no qualifying earned income at all." -us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its EITC of $2,366 is right, but it computed the refundable CTC as 15% of earned income above $3,000, the pre-TCJA threshold, giving $437.25. The 2026 threshold is $2,500, making the refundable CTC $512.21 and the total $2,878.10." -us,scenario_100,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its explanation computes $2,803.25 while the submitted value is 0, so the answer contradicts its own text; and the $2,803.25 it argued for rests on the pre-TCJA $3,000 refundable-CTC earnings threshold. The 2026 threshold is $2,500, so the refundable CTC is $512.21 and the correct total is $2,878.10." -us,scenario_100,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied 'pre-TCJA expiration rules' with a $3,000 refundable-CTC earnings threshold, yielding $437.25 on top of a correct $2,366 EITC. The threshold for 2026 is $2,500, so the refundable CTC is 15% x ($5,915 - $2,500) = $512.21 and the total is $2,878.10." -us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It treated the $1,700-per-child refundable cap as the amount actually received, claiming $3,400 of refundable CTC, and never applied the binding limit of 15% of earnings above $2,500, which is $512.21. It also computed the EITC off AGI of $1,876 (after the 401(k) deduction) for '$600' instead of 40% of the $5,915 earned income, which is $2,365.89." -us,scenario_100,federal_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It invented a '$1,650 per-child minimum' refundable CTC floor under a supposed OBBBA schedule and took $3,300 instead of the earnings-limited amount. The refundable CTC has no per-child floor: it is the lesser of 15% of earnings above $2,500 ($512.21) and the per-child refundable cap, so the total is $2,878.10, not $5,666." -us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It asserted that income is 'too low for positive refundable federal credits,' inverting the phase-in structure. The EITC has no minimum-earnings floor, so $5,915 of wages with two qualifying children produces $2,365.89 at the 40% phase-in rate, plus $512.21 of refundable CTC from the earnings above $2,500." -us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It declared that low wage income triggers no refundable credits, which reverses how the EITC works: the credit rises from the first dollar of earnings at the 40% two-child rate, giving $2,365.89 here. The refundable CTC adds 15% x ($5,915 - $2,500) = $512.21 for a total of $2,878.10." -us,scenario_100,federal_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It based the EITC on $2,056 of wages after subtracting the $3,859 traditional 401(k) contribution, producing $822.40. The elective deferral lowers AGI but leaves the earned-income base at $5,915, so the EITC is $2,365.89 and earnings clear the $2,500 refundable-CTC threshold for a further $512.21." -us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,other,False,"It asserted a zero phase-in amount with no computation. The two-child EITC phase-in rate applies from the first dollar of earned income, so $5,915 of wages yields $2,365.89, and the refundable CTC adds 15% x ($5,915 - $2,500) = $512.21." -us,scenario_100,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It reduced EITC earned income by the $3,859 401(k) deferral to $2,056 and then applied a $3,000 refundable-CTC threshold. Earned income is the full $5,915 of wages, giving an EITC of $2,365.89, and the refundable-CTC threshold is $2,500, giving $512.21." -us,scenario_100,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its EITC of $2,366 on the correct $5,915 earned-income base is right, but it computed the refundable CTC as 15% x ($5,915 - $3,000) = $437 using the pre-TCJA threshold. The 2026 threshold is $2,500, so the refundable CTC is $512.21 and the total is $2,878.10." -us,scenario_100,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It equated EITC earned income with W-2 Box 1 after the traditional 401(k) deferral, using $2,056 and getting $822. The earned-income base is the full $5,915 of gross wages, so the EITC is $2,365.89 and earnings exceed the $2,500 refundable-CTC threshold, adding $512.21 rather than the $0 it claimed." -us,scenario_100,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted both the $3,859 401(k) and $180 IRA contributions from wages to get $1,876 of earned income and a $750 EITC. Retirement contributions reduce AGI, not the earned-income base for the EITC or the refundable CTC, so the correct amounts are $2,365.89 and $512.21." -us,scenario_100,federal_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It set EITC earned income to W-2 Box 1 of $2,056 after the $3,859 pre-tax deferral, producing $822.40 and a $0 refundable CTC. The earned-income base is the full $5,915 of gross wages, giving $2,365.89 of EITC and, with earnings above the $2,500 threshold, $512.21 of refundable CTC." -us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Its text estimated an EITC of about $6,500 and an ACTC of about $2,300, ignoring that at $5,915 of earnings the EITC equals 40% of earned income ($2,365.89) and the refundable CTC equals 15% of earnings above $2,500 ($512.21). It then submitted 0, contradicting its own reasoning that the household qualifies for substantial credits." -us,scenario_100,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It computed the refundable CTC limitation correctly at 15% x ($5,915 - $2,500) = $512.25 and then doubled it to $1,024.50 by applying it per child. That 15%-of-excess-earnings limitation is computed once for the tax unit, with the $1,700-per-child amount acting only as a ceiling, so the refundable CTC is $512.21 and the total $2,878.10." -us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It computed the correct 40% phase-in EITC of $2,366 and then replaced it with an invented $3,730, while treating $1,800 per child ($3,600) as automatically refundable. The refundable CTC is limited to 15% of earnings above $2,500, which is $512.21 here, so the total is $2,878.10 rather than $7,330." -us,scenario_100,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It claimed the EITC and refundable CTC are 'capped by tax liability,' which is the defining property of nonrefundable credits; both pay out in full with zero income tax liability. With $5,915 of wages and two qualifying children the household receives $2,365.89 of EITC plus $512.21 of refundable CTC, totaling $2,878.10." +us,scenario_100,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"Its own reasoning reached the right total: EITC of 40% × $5,915 = $2,366 plus refundable CTC of 15% × ($5,915 − $2,500) = $512, which is $2,878. It then submitted 6,425, a number that no step in its reasoning supports. The error is in the value it submitted, not in the method." +us,scenario_100,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It subtracted the $3,859 traditional 401(k) deferral from wages and used $2,056 as earned income. EITC earned income is actually gross wages of $5,915, so its EITC came out at $822.40 instead of $2,366. It also wrongly zeroed the refundable CTC, which is 15% × ($5,915 − $2,500) = $512." +us,scenario_100,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It correctly worked out EITC of $2,366 and refundable CTC of 15% × $3,415 = $512.25. It then dropped the refundable CTC figure and swapped in an unexplained '$834 approximation', giving $3,200 instead of $2,878." +us,scenario_100,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It computed the correct 40% × $5,915 = $2,366 EITC, then cut it to $1,906 by misapplying the rule that compares credits on AGI and on earned income. That comparison only applies once AGI reaches the phase-out range. In the phase-in range the credit is simply 40% of earned income, even when AGI is lower." +us,scenario_100,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It derived EITC of $2,366 and refundable CTC of $512, which total $2,878. It then 'reduced slightly' the total to $2,330 on the mistaken idea that the 401(k) and IRA deductions trim the credits. In PolicyEngine, neither contribution reduces the earned income behind either credit." +us,scenario_100,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It treated the $710 FLSA overtime premium as extra earnings on top of the $5,915 wages. The prompt says that wage total already includes overtime. It also inflated EITC to $2,905, more than 40% of even the double-counted $6,625. The correct figures are EITC of 40% × $5,915 = $2,366 plus $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It took earned income as wages net of the 401(k) deferral, $5,915 − $3,859 = $2,056. EITC earned income is actually the gross $5,915. As a result it produced $822.40 of EITC instead of $2,366 and wrongly zeroed the $512 refundable CTC." +us,scenario_100,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It correctly computed EITC of $2,366 and refundable CTC of $512. It then threw that out and submitted $6,295, citing 'IRS EITC tables'. That figure is close to the maximum-credit amounts, which do not apply here: at $5,915 of earnings the household is in the 40% phase-in range, not at the plateau." +us,scenario_100,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It added the $710 FLSA overtime premium to the $5,915 wages to get $6,625 of earned income. The prompt states the wage total already includes overtime. The double count inflated EITC to $2,650 and refundable CTC to $618.75, instead of $2,366 and $512." +us,scenario_100,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It subtracted the $3,859 traditional 401(k) deferral and both $564 health premium amounts from wages, leaving $928 of earned income. EITC earned income is gross wages of $5,915. That cut EITC to $371 instead of $2,366 and wrongly eliminated the $512 refundable CTC." +us,scenario_100,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It reduced EITC earned income by the $3,859 traditional 401(k) deferral to $2,056. The correct base is gross wages of $5,915, so it got $822.40 of EITC instead of $2,366. It also dropped the $512 refundable CTC, which is 15% × ($5,915 − $2,500)." +us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It got EITC of $2,366 right but used a $3,000 earnings threshold for the refundable CTC, assuming the pre-TCJA rules return in 2026. OBBBA made the $2,500 threshold permanent. The correct refundable CTC is 15% × ($5,915 − $2,500) = $512, not $437.25." +us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It gave the household roughly the maximum two-child EITC plus a large refundable CTC, $6,848 in total. At $5,915 of earnings the EITC is still phasing in, at 40% × $5,915 = $2,366. The refundable CTC is limited to 15% × ($5,915 − $2,500) = $512." +us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It treated the $20,644 of employer-sponsored insurance premiums as a pre-tax deduction that wiped out wages, and so used zero earned income. PolicyEngine does not reduce EITC or CTC earned income by those premiums; the base is the $5,915 of gross wages. That base gives $2,366 of EITC and $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It netted the health insurance premiums against wages to reach zero earned income and denied both credits. Earned income for the EITC and the refundable CTC is gross wages of $5,915. That yields $2,366 of EITC and $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It returned zero refundable credits with no working. A head-of-household filer with two qualifying children and $5,915 of earnings qualifies for 40% × $5,915 = $2,366 of EITC plus 15% × ($5,915 − $2,500) = $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It computed EITC correctly at $2,366 but used a $3,000 earnings threshold for the refundable CTC. The 2026 threshold is $2,500, made permanent by OBBBA. That is why it got $437.25 instead of $512 and a total of $2,803.25." +us,scenario_100,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It submitted 0 even though its explanation concludes 'value = 2803.25', so the submitted value contradicts its own reasoning. That 2,803.25 also used the wrong $3,000 refundable CTC threshold instead of the 2026 $2,500 threshold." +us,scenario_100,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It applied the pre-TCJA $3,000 earnings threshold, assuming the TCJA rules expire in 2026. OBBBA kept the $2,500 threshold permanently. Its refundable CTC was therefore $437.25 instead of $512, and its total $2,803.25 instead of $2,878." +us,scenario_100,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It granted the full $1,700-per-child refundable cap, $3,400, and ignored the earned-income limit of 15% × ($5,915 − $2,500) = $512. It also estimated EITC at about $600 from $1,876 of AGI. The EITC phases in on earned income of $5,915, giving $2,366." +us,scenario_100,federal_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It invented a guaranteed minimum refundable CTC of $1,650 per child, $3,300 in total, under OBBBA. The refundable portion is actually the lesser of the per-child cap and 15% of earnings above $2,500, with no floor. Here that is $512, which puts the total at $2,878, not $5,666." +us,scenario_100,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It judged income too low for any refundable credit, but the EITC phases in from the first dollar of earnings. With $5,915 of earnings, EITC is 40% × $5,915 = $2,366. Refundable CTC is 15% × ($5,915 − $2,500) = $512." +us,scenario_100,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It returned zero, missing that low earnings are exactly the EITC phase-in range. With two children and $5,915 of earnings, EITC is $2,366. Earnings above $2,500 also produce $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It based the EITC on $2,056 of wages after the $3,859 traditional 401(k) contribution instead of gross wages of $5,915. That produced $822.40 instead of $2,366. It also wrongly concluded earnings were below the refundable CTC threshold, losing the $512." +us,scenario_100,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It subtracted the traditional 401(k) contribution from wages and used $2,056 as earned income. PolicyEngine uses gross wages of $5,915 for both credits. That base yields $2,366 of EITC and 15% × $3,415 = $512 of refundable CTC, not $822.40 and zero." +us,scenario_100,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It claimed the phase-in amount was zero. The two-child EITC phases in at 40% of earned income from the first dollar, which on $5,915 gives $2,366. Refundable CTC adds another 15% × ($5,915 − $2,500) = $512." +us,scenario_100,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It took EITC earned income as W-2 wages net of the 401(k) deferral, $2,056, instead of gross wages of $5,915, and so got $822 of EITC instead of $2,366. It also cited a $3,000 refundable CTC threshold instead of the 2026 $2,500, compounding its wrong zero for the $512 refundable CTC." +us,scenario_100,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It computed EITC correctly at $2,366, but based the refundable CTC on a $3,000 earnings threshold. For 2026 the threshold is $2,500, so the refundable CTC is 15% × $3,415 = $512, not $437, and the total is $2,878." +us,scenario_100,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It used W-2 Box 1 wages after the traditional 401(k) deferral, $2,056, as EITC earned income. PolicyEngine counts gross wages of $5,915. The result was $822 instead of $2,366 of EITC and $0 instead of $512 of refundable CTC." +us,scenario_100,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It took earned income to be wages minus both the 401(k) deferral and the IRA contribution, $1,876, which is AGI rather than earned income. IRA contributions are above-the-line adjustments that never reduce earned income, and the base here is $5,915. That base gives $2,366 of EITC and $512 of refundable CTC, not $750 and zero." +us,scenario_100,federal_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It equated EITC earned income with the $2,056 of W-2 Box 1 wages left after the 401(k) deferral, instead of the $5,915 of gross wages that PolicyEngine uses. That undercut EITC to $822.40 instead of $2,366 and wrongly eliminated the $512 refundable CTC." +us,scenario_100,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"Its reasoning estimated EITC of about $6,500 and refundable CTC of about $2,300, yet it submitted 0, so the value contradicts its own analysis. Its estimates were also wrong: at $5,915 of earnings the EITC is in the 40% phase-in, at $2,366, and the refundable CTC is limited to $512." +us,scenario_100,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It applied the 15% × ($5,915 − $2,500) earned-income formula once per child and got $1,024.50. That formula is a single family-level limit on the whole refundable CTC. The refundable CTC is $512 in total, making the total $2,878." +us,scenario_100,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It treated the CTC as refundable up to $1,800 per child, ignoring the earned-income limit of 15% × ($5,915 − $2,500) = $512. It then replaced its own correct $2,366 EITC with an arbitrary $3,730. Those two errors inflate the total to $7,330." +us,scenario_100,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It claimed the EITC and refundable CTC are capped by tax liability. Refundable credits are paid out regardless of liability; that is what makes them refundable. The household receives $2,366 of EITC and $512 of refundable CTC." us,scenario_100,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility with no derivation, never applying either free-tier pathway: SNAP receipt of $8,625.89 categorically confers free meals, and countable income of $11,979.68 (wages $5,914.72 plus TANF $6,064.96) is 44% of the three-person federal poverty guideline, far below the 130% free threshold. Its 0 is consistent with treating SNAP/TANF as absent because receipt is not a listed input fact, even though both are derived outputs of this same household. The income test alone settles it: the $5,915 of wages is about 22% of the poverty guideline, so the household lands in the free tier with two school-age children (ages 6 and 5) generating $7.15 per child per school day, $2,261.92 annually." us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model gave a circular non-answer — that the benchmark does not indicate positive free-meal support — instead of computing the free-tier tests, so it applied neither SNAP categorical eligibility ($8,625.89 of SNAP automatically confers free meals) nor the 130%-of-poverty income test, which the household passes at 44% FPG on countable income of $11,979.68. Montana has no universal free-meals policy, so the tier turns entirely on those two tests, both of which this household clears. The correct derivation pays $7.15 per school day for each of the two school-age children (ages 6 and 5) for $2,261.92 annually; the model's 0 corresponds to no test being run at all." us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model evaluated the head only under an expansion/disability pathway and missed the parent/caretaker category, whose Montana income limit is 100% FPL and which the head satisfies at 0.07 × FPL. It also used an erroneous $3,627 annual 138%-FPL threshold and incorrectly imposed an asset limit on MAGI-based Medicaid eligibility." @@ -7298,45 +8013,52 @@ us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identi us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($5,915 × 0.062 = $366.73 and $5,915 × 0.0145 = $85.78, totaling $452.51) and then discarded that result, submitting $477 on the invented rationale of 'rounding to annual benchmark.' No rounding, annualization, or state payroll-tax add-on applies here; the answer its own arithmetic supports is $452.48." us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support." us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals." -us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Counted only the $492.92 monthly wage as SNAP income, so it never added the household's TANF cash grant to countable unearned income (gross is $998.31/month) and it zeroed the $453.64 excess shelter deduction. Its $89 net income understates the true $237.09, cutting the 30% household contribution from $71.10 to $27/month, and it held the maximum allotment flat at $785 for all twelve months instead of stepping to $802.60 later in 2026." -us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Built annual net income of $1,040 from wages alone less standard, 20% earned-income, and excess-medical deductions, omitting both the TANF grant that raises countable gross to $998.31/month and the $453.64/month excess shelter deduction. That yields a $312 annual household contribution against the correct $853.20, and it annualized a flat $785 allotment rather than the $785/$802.60 calendar-2026 blend." -us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Worked through the gross-income and resource tests, confirmed the household qualifies, then abandoned the deduction arithmetic entirely and submitted a 'conservative' $1,908 that no step in its own reasoning produces. The derivation it dropped is the blended annual maximum near $9,479 less 30% of $237.09 monthly net income." -us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Computed $9,108 from its own deduction chain and then discarded it with an unexplained downward trim to $7,320 justified by 'shelter deduction absence' — the household has a $453.64/month excess shelter deduction, which raises the benefit rather than cutting it. What actually reduces the benefit is the TANF grant it never counted, which leaves $237.09 of net income and a $71.10 monthly contribution." -us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Treated the standard and 20% earned-income deductions as the only offsets to $5,915 of wages, omitting the TANF grant from countable gross income and the $453.64/month excess shelter deduction, then applied an unexplained trim from its own $8,496 to $8,316. Correct net income is $237.09/month from $998.31 gross less $761.21 in deductions." -us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Skipped the net-income computation and asserted a near-maximum allotment from wages alone, never counting the TANF cash grant that puts countable gross at $998.31/month and net at $237.09. It also used a flat $785 monthly maximum instead of the calendar-2026 blend that reaches $802.60 in later months." -us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Settled on $72.67 monthly net income built from wages, the standard deduction, the 20% earned-income deduction, and an excess medical deduction, omitting the TANF grant (countable gross is $998.31) and the $453.64 excess shelter deduction. That makes the household contribution $21.80/month instead of $71.10, and it paid a flat $784 allotment for twelve months." -us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Counted the $20,644 employer-sponsored insurance premium as a deductible medical expense to force net income to zero, then reported $13,700 — above its own stated $766/month maximum times twelve ($9,192) and above the true annual ceiling near $9,479. Net income is $237.09/month because the TANF grant is countable unearned income, so the benefit is capped well below its figure." -us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Added the $710 FLSA overtime premium on top of the $5,915 wage total that already includes overtime, inflating monthly earnings to $552.08, and then omitted both the TANF grant from unearned income and the $453.64 excess shelter deduction. It also used a $797 flat maximum allotment rather than $785 stepping to $802.60." -us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Counted a TANF grant as unearned income but at $458.54/month rather than the $505.39 that brings countable gross to $998.31, and dropped the $453.64 excess shelter deduction entirely. Its net income of $602.21 overstates the true $237.09, inflating the household contribution by about $110/month." -us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted TANF at only $335.31/month (countable gross is $998.31, not $828.23) and omitted the $453.64 excess shelter deduction, leaving net income of $390.98 against the true $237.09. Its explanation's own total of $7,808.48 also conflicts with the $9,037.20 submitted." -us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Derived $54/month net income from the wage figure plus a $140 excess medical deduction, never adding the TANF grant to countable gross ($998.31/month) and never applying the $453.64 excess shelter deduction. It then paid a flat $781 allotment for twelve months instead of the $785/$802.60 blend, so the household contribution came to $16/month rather than $71.10." -us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Provided no computation and submitted $9,408, which is twelve months of maximum allotment with essentially no household contribution. Net income of $237.09/month — produced by counting the TANF grant as unearned income against $761.21 of deductions — withholds $71.10 every month." -us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Its $6,128 contradicts its own stated premise of a near-maximum allotment: $6,128/12 is $510.67/month against a three-person maximum of $785 rising to $802.60, implying a household contribution near $274/month. The actual contribution is $71.10, being 30% of $237.09 net income." -us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted TANF as unearned income but omitted the $453.64 excess shelter deduction, so its $604.75 monthly benefit implies net income near $544 rather than $237.09. It also used a $768 maximum allotment instead of $785 stepping to $802.60." -us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Returned zero on asserted income and resource ineligibility. Gross income of $998.31/month is 45% of the three-person poverty guideline, $2,800 in bank assets sits within the limit, and TANF non-cash receipt confers categorical eligibility outright, so the household draws $713.90–$733.60 per month." -us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Built net income from the $492.92 monthly wage plus standard and medical deductions, omitting the TANF grant from countable gross income and the $453.64 excess shelter deduction that PolicyEngine applies. It then annualized a flat maximum allotment rather than the mid-year step from $785 to $802.60." -us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave no deduction arithmetic and submitted a value inconsistent with its own explanation's $9,072. Both figures assume a near-maximum allotment with no household contribution, while countable net income of $237.09/month — the TANF grant counts as unearned income — withholds $71.10 monthly." -us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Stopped at wages less the standard, 20% earned-income, and medical deductions for $97 net income, omitting the TANF grant in countable gross ($998.31) and the $453.64 excess shelter deduction. It also used a $768 flat maximum instead of the $785/$802.60 calendar-2026 blend." -us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Deducted the $20,644 employer-sponsored insurance premium as a medical expense to drive net income to zero and then invented an $800/month maximum allotment. The disabled head's SNAP medical deduction covers his out-of-pocket premiums and expenses, and net income is $237.09/month because the TANF grant is countable unearned income, so the benefit is $713.90–$733.60 monthly." -us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Used an $800/month maximum allotment that does not apply to a three-person household in the contiguous states ($785, rising to $802.60 later in 2026) and computed $179.34 net income from wages less standard and earned-income deductions only. The correct net is $237.09, from $998.31 gross including the TANF grant less $761.21 of deductions including $453.64 of excess shelter." -us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Submitted $12,588, which is $1,049/month and exceeds the three-person maximum allotment of $785 ($802.60 later in the year) that it claims to be paying. The annual ceiling is near $9,479 before subtracting the $853.20 household contribution from $237.09 monthly net income." -us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned zero because no rent or utility figures were listed, treating missing shelter data as disqualifying. Shelter costs are not an eligibility condition; the household passes the gross, net, and asset tests with $998.31 gross and $237.09 net monthly income and is categorically eligible through TANF non-cash status." -us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Claimed the head's medical and health-insurance expenses drive countable net income to zero and paid a flat $767/month maximum for twelve months. Net income is $237.09/month because the TANF grant counts as unearned income, and the maximum is $785 rising to $802.60, so $71.10 is withheld each month." -us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Paid the full $785 three-person maximum for twelve months with no household contribution at all, treating countable earnings as negligible. Counting the TANF grant leaves $237.09 of net income after $761.21 in deductions, withholding $71.10/month, and the allotment rises to $802.60 later in 2026." -us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Assumed the disabled head's medical premiums and expenses reduce net SNAP income to nothing and paid the $785 maximum for all twelve months. The TANF grant is countable unearned income, leaving $237.09 net and a $71.10 monthly contribution, and the maximum steps to $802.60 in later months." -us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Assumed SSI income the household does not receive and produced $533/month, which implies net income near $840 against the true $237.09. The household's unearned income is the TANF grant, and net income is $998.31 gross less $761.21 of standard, earned-income, and excess shelter deductions." -us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Paid a flat $785 maximum for twelve months on the premise that the head's listed insurance and medical expenses eliminate countable net income. The TANF grant leaves $237.09 of net income and a $71.10 monthly contribution, and the maximum allotment rises to $802.60 later in the year." -us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Returned zero with no computation, treating the listed inputs as producing no SNAP. The household clears the gross ($998.31, 45% of the poverty guideline), net ($237.09), and asset tests and is categorically eligible through TANF non-cash status, receiving $713.90–$733.60 per month." -us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Built annual net income of $668 from wages less standard, earned-income, and excess-medical deductions, omitting both the TANF grant from countable gross and the $453.64/month excess shelter deduction. It also used a $766 monthly maximum instead of $785 rising to $802.60." -us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Drove net income to zero with the excess medical deduction and paid twelve months at a $766 maximum. The TANF grant is countable unearned income, leaving $237.09 net and a $71.10 monthly contribution, and the three-person maximum is $785 stepping to $802.60." -us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Reported $805 — a monthly allotment figure — as the annual benefit, a twelve-fold period error. Its zero net income is also wrong: counting the TANF grant leaves $237.09 countable each month after the $209 standard, $98.58 earned-income, and $453.64 excess shelter deductions." -us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"Set net income near $50/month from wages, the standard deduction, the 20% earnings deduction, and a medical deduction, omitting the TANF grant from countable gross ($998.31) and the $453.64 excess shelter deduction. It then paid a flat $774 allotment for twelve months rather than the $785/$802.60 blend." -us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for snap, so no substantive computation reached the grader. The requested derivation is the blended annual maximum near $9,479 less the $853.20 household contribution taken as 30% of $237.09 monthly net income." -us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Paid twelve months at $785 on the premise that the head's listed premiums and medical expenses reduce net SNAP income to zero. The TANF grant is countable unearned income, leaving $237.09 net and $71.10 withheld each month, and the maximum allotment rises to $802.60 later in 2026." -us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted 100% of its $393 net income from the $766 maximum instead of the statutory 30%, cutting the monthly benefit to $377. The benefit formula withholds 30% of net income, which on the correct $237.09 net is $71.10 against a $785/$802.60 maximum." -us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Reduced net income to about $3/month by applying an excess medical deduction to wage income alone, omitting the TANF grant that puts countable gross at $998.31 and net at $237.09. It therefore paid essentially the full $785 maximum for twelve months rather than $713.90–$733.60." -us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium to the $5,915 wage total that already includes overtime, and its $12,444 contradicts its own arithmetic ($766 × 12 = $9,192) while exceeding the true annual ceiling near $9,479. The benefit is the $785/$802.60 maximum less $71.10 per month for 30% of $237.09 net income." -us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Declared the household resource-ineligible and returned zero. The $2,800 bank balance and $1,981 vehicle are within SNAP resource limits, and TANF non-cash receipt confers categorical eligibility that waives the asset test entirely, so the household receives $713.90–$733.60 per month." +us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It counted only the $493/month of wages and left out the $505.39/month of Montana TANF cash that SNAP counts as unearned income. It also took a $94 excess medical deduction, but the head is not SNAP-disabled because they receive no SSI or SSDI. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), and its $89 net income understates the $691 that correction gives (the frozen reference's is $237.09). Its own $758 × 12 is $9,096, not the $9,046 it submitted." +us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It computed net income from wages alone, leaving out the ~$505/month of TANF cash, and subtracted a $1,124/year medical deduction the head doesn't qualify for (no SSI or SSDI, so not SNAP-disabled). It also used a $214 standard deduction instead of $209. It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. It then applied $785 × 12; holding the $785 maximum for October–December matches the release's SNAP convention (c_snap_hold_fy2026), and the frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule." +us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It concluded the household was eligible but never applied the benefit formula (maximum allotment minus 30% of net income), and submitted an arbitrary 'conservative' $1,908 (~$159/month). The frozen reference's steps are $998.31 gross (wages plus TANF), minus the $209 standard, $98.58 earned-income and $453.64 excess shelter deductions, giving $237.09 net and $713.90/month for January–September. Its $453.64 comes from engine defect r30_snap_heat_and_eat_sua: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. The record's correction takes no utility-allowance shelter deduction and gives $691 net ($998.31 − $209 − $98.58, rounded), a $208 contribution and $577/month." +us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It took a $141/month medical deduction for a head who receives no SSI or SSDI and so is not SNAP-disabled, and left out the ~$505/month of TANF income. It also used the FY2025 $768 maximum instead of $785. It then cut its own $9,108 result down to $7,320 because there was no shelter deduction. But a missing deduction can't lower a benefit already computed without it. Taking no shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 utility-allowance shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended." +us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It used the outdated $766 maximum allotment instead of FY2026's $785 and worked from wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then reduced its own $8,496 to $8,316 for 'medical and shelter considerations', even though extra deductions raise the benefit rather than lower it." +us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It counted only wages and applied only the standard and earned-income deductions, leaving out the $505.39/month of TANF income. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Its $8,300 (~$692/month) is below even its own wages-only formula ($785 − 30% × $185 ≈ $730). The frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It subtracted a $94/month disabled-member medical deduction, but the head receives no SSI or SSDI and so is not SNAP-disabled. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF from gross income. That produced $92 net income and $757.50/month for all 12 months, against the frozen reference's $237.09 net income, $713.90 for January–September and $733.60 for October–December; the $733.60 uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It used a $228 standard deduction instead of the FY2026 3-person $209, and took a $93.67 medical deduction for a head who is not SNAP-disabled (no SSI or SSDI). It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it counted wages only, leaving out $505.39/month of TANF. That gives $72.67 net instead of the $691 the correction gives (the frozen reference has $237.09), with a $784 maximum applied flat for 12 months." +us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It treated the $20,644 employer-sponsored premium as an out-of-pocket medical deduction to push net income to zero, although the head isn't SNAP-disabled and that premium isn't a household expense. It then submitted $13,700, which is more than its own 12 × $766 = $9,192 ceiling. SNAP can never exceed the 3-person maximum allotment for the year." +us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It used a disabled-member medical deduction to bring net income down to $0–$50, but the head receives no SSI or SSDI and gets no SNAP medical deduction. It also left out the $505.39/month of TANF income, which with no utility-allowance shelter deduction gives $691 net and a $208 monthly contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium on top of $5,915 in wages, although the wage total already includes overtime. It also subtracted a $100.67 medical deduction for a head who is not SNAP-disabled and left out the $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It used a $797 maximum and a $221 standard deduction instead of $785 and $209." +us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It correctly counted TANF as unearned income, though at $458.54/month rather than $505.39, and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). Its net income came to $602, against $691 under that correction and the frozen reference's $237.09. It also took a $46.67 medical deduction for a head who is not SNAP-disabled and used the FY2025 $204 standard deduction and $768 maximum instead of $209 and $785." +us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It submitted $9,037.20 while its explanation computes $7,808.48. That explanation understated TANF ($335/month vs $505.39) and took a $140.67 medical deduction for a head who is not SNAP-disabled. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also used a $198 standard deduction and $768 maximum instead of $209 and $785." +us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"It took a $53.67 medical deduction for a head who receives no SSI or SSDI and so is not SNAP-disabled. It counted wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then applied an $805 maximum to all 12 months, although the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 $785 maximum for all 12 months of 2026." +us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It deducted $140/month of the head's medical costs, but the head is not SNAP-disabled because they receive no SSI or SSDI. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out $505.39/month of TANF from gross income. That gives $54 net income instead of the $691 the correction gives (the frozen reference has $237.09), with a $781 maximum instead of $785." +us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $9,408 is $784 × 12, a full maximum allotment with no household contribution. It ignored the 30% of net income that must be subtracted: $208/month on the $691 of net income from wages plus TANF under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10/month on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It said the household gets near the maximum allotment but submitted $6,128 ($510.67/month), which is more than $270/month below the $785 maximum and contradicts its own reasoning. The record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, gives $691 net income, a $208 contribution and $577/month; the frozen reference's $237.09 net income, $71.10 contribution and $713.90/month for January–September also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It counted TANF as unearned income and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended); its implied net income was ~$544 and its benefit $604.75/month. It also applied a medical deduction for a head who is not SNAP-disabled and used the FY2025 $768 maximum instead of $785." +us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It said the household fails SNAP income and resource limits, but gross income is only 45% of the poverty guideline. The household is also broad-based categorically eligible through TANF non-cash status, and $2,800 in bank assets is within limits, so the benefit is positive." +us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it applied a disabled-head medical deduction the head doesn't qualify for (no SSI or SSDI) and computed income from wages only, leaving out $505.39/month of TANF. The result was $753/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It submitted $9,216 ($768 × 12, the FY2025 maximum with no contribution) while its explanation says $9,072. It never subtracted 30% of net income: $691 from wages plus TANF after the standard and earned-income deductions under the record's correction of engine defect r30_snap_heat_and_eat_sua, or the frozen reference's $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used the FY2025 $204 standard deduction and $768 maximum. That gave $97 net income instead of the $691 the correction gives (the frozen reference has $237.09) and $739/month applied flat for 12 months." +us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It counted the $20,644 employer-sponsored premium as a deductible medical expense of a disabled member to push net income to zero. The head isn't SNAP-disabled without SSI or SSDI, and SNAP income also includes $505.39/month of TANF, giving $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It used an $800 maximum instead of FY2026's $785." +us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It correctly took no medical deduction and, as the record's correction of engine defect r30_snap_heat_and_eat_sua does, no utility-allowance shelter deduction (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF income and used a $215 standard deduction instead of $209. It also applied an $800 maximum to all 12 months instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; at $785 the frozen reference pays $713.90/month for January–September." +us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"It said it used the 3-person maximum allotment, but $12,588 is $1,049/month, well above the FY2026 3-person maximum of $785 ($9,420/year). It also ignored the monthly contribution: $208 on $691 of net income under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10 on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It set SNAP to zero because no rent or utility costs were listed, but SNAP eligibility doesn't depend on shelter costs. This household has gross income at 45% of the poverty guideline, and the frozen reference's $237.09 net income gives $713.90–$733.60 per month; the $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It treated the head's health-insurance premiums, including the employer-sponsored plan, as a deductible medical expense of a disabled member to zero out net income. The head receives no SSI or SSDI and gets no SNAP medical deduction, and wages plus $505.39/month of TANF leave $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also used a $767 maximum instead of $785." +us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It assumed countable net income was zero and paid the $785 maximum for all 12 months. It left out the $505.39/month of TANF that brings net income to $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, which requires a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It used the head's premiums and medical costs as a disabled-member deduction to wipe out net income, but the head receives no SSI or SSDI and is not SNAP-disabled. It also ignored $505.39/month of TANF income, so under the record's correction of engine defect r30_snap_heat_and_eat_sua net income is $691 and the benefit $577/month, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It counted SSI as unearned income, but SSI is zero because the head's $2,800 bank balance exceeds the $2,000 SSI resource limit. The actual unearned income is $505.39/month of TANF; the record's correction of engine defect r30_snap_heat_and_eat_sua takes no utility-allowance shelter deduction, while the frozen reference offsets the TANF with a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also applied a medical deduction the non-SNAP-disabled head doesn't qualify for, which put its $6,400 far below the frozen reference's $8,625.89, which uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 all year." +us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It said the disabled head's insurance and medical expenses eliminate countable net income, but the head receives no SSI or SSDI and gets no SNAP medical deduction. With $505.39/month of TANF, net income is $691 and the contribution $208/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, so the benefit is below the $785 maximum; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"It computed income from wages only with no shelter costs, leaving out $505.39/month of TANF; taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua, since the frozen reference's $453.64 comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It then applied a single $710.92 monthly amount to all 12 months; the frozen reference's rise to $733.60/month for October–December 2026 uses the engine's projected FY2027 parameters, which the release's SNAP convention (c_snap_hold_fy2026) replaces with the FY2026 schedule all year." +us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted TANF cash as income and applied only the earned-income and standard deductions, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference also takes a $453.64 excess shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). The correction gives $691 net income and $577/month, $41 below its $618; at the $785 maximum, $618 implies about $555 of net income, so its estimated TANF was about $135/month below the $505.39. Measured against the frozen reference, leaving out the $453.64 deduction raised net income, and its benefit came to $618/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"It counted TANF income and explicitly assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance excess shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also took a medical deduction for a head who is not SNAP-disabled, and ended at $647/month instead of the frozen reference's $713.90 (January–September) and $733.60 (October–December); the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It returned $0 without doing any SNAP calculation, although the household has gross income at 45% of the poverty guideline and broad-based categorical eligibility; the record's correction of engine defect r30_snap_heat_and_eat_sua gives $691 net income and $577/month, and the frozen reference, whose $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), has $237.09 net income. That yields the frozen reference's $713.90–$733.60 per month; its $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." +us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted $1,688/year of medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used an outdated $198 standard deduction and $766 maximum instead of $209 and $785. That produced $668/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month)." +us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It used an excess medical deduction for the 'disabled' head to bring net income to $0, but the head receives no SSI or SSDI and is not SNAP-disabled. With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also paid the outdated $766 maximum for all 12 months instead of $785 minus the $208 contribution under that correction (the frozen reference's is $71.10)." +us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it subtracted a $1,688/year disabled-member medical deduction the head doesn't qualify for (no SSI or SSDI) and counted wages only, leaving out $505.39/month of TANF, which gave $536/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month). It also used a $787 maximum instead of FY2026's $785." +us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"It reported $805, a single month's maximum allotment, as the annual benefit instead of adding up 12 monthly allotments. It also used a $182 medical deduction the non-SNAP-disabled head doesn't get to assume zero net income, when TANF-inclusive net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"It applied a disabled-member medical deduction that brought net income to ~$50, but the head receives no SSI or SSDI and is not SNAP-disabled. It also left out $505.39/month of TANF, which with wages gives $691 net and a $208 contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so there was no answer to score against the frozen reference's $8,625.89 annual benefit." +us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"It deducted the head's premiums and medical expenses as a disabled-member medical deduction to set net income to zero, but the head receives no SSI or SSDI and is not SNAP-disabled. With $505.39/month of TANF, net income is $691, so the benefit is $577/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It subtracted the full ~$393 net income from the $766 maximum ($377/month) instead of 30% of net income. It also skipped the $209 standard deduction, left out TANF, and used an outdated $766 maximum instead of $785. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It deducted $182/month of medical costs, including both $564 premium entries and Child 2's $500, as a disabled-member medical deduction. The head receives no SSI or SSDI and is not SNAP-disabled, so there is no medical deduction. With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, not $3; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended)." +us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium to wages that already include overtime and used the outdated $766 maximum. It then submitted $12,444, which is more than its own 12 × $766 = $9,192 ceiling, even though it said it was adjusting the benefit downward." +us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It said resources exceed SNAP limits, but the $2,800 bank balance is under the federal $3,000 resource limit. The household is also broad-based categorically eligible through TANF non-cash status, and vehicle value is excluded, so SNAP is positive, not zero: the frozen reference gives $8,625.89, using the engine's projected FY2027 schedule for October–December, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year." us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head's generic disability fact as satisfying SSI's categorical disability requirement and proceeded directly to the earned-income calculation. PolicyEngine's SSI-specific aged/blind/disabled indicator is false for the head, so no SSI benefit-rate or income-exclusion calculation applies; its submitted value also contradicts its own stated $9,480 calculation." us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the head's generic disability status with SSI categorical eligibility. The head's SSI-specific aged/blind/disabled indicator is false, so the earned-income reduction it applied is irrelevant and SSI is zero." us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-qualified disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI's aged, blind, or qualifying-disabled criterion, so the calculation stops before income exclusions or the federal benefit rate are applied." @@ -7346,118 +8068,135 @@ us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract." us,scenario_100,ssi,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed the 46-year-old head's generic disability status met SSI's qualifying-disability test and then calculated countable wages. The SSI-specific categorical indicator is false, so the head is ineligible regardless of the wage calculation; neither child qualifies either." us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the generic disability input as SSI categorical eligibility and therefore performed an income-reduction calculation that should never occur. It also added the separately listed overtime premium to annual gross wages and invented an IRWE adjustment despite instructions not to infer unlisted expenses, but categorical ineligibility alone makes SSI zero." -us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It stripped the $3,859 traditional 401(k) deferral out of EITC earned income, thrashing between bases of $2,056 and $2,646; PolicyEngine's EITC earned income is the full $5,915 of gross wages, so the federal credit is the 40% phase-in value of $2,365.89. It compounded that with a 10% Montana match instead of the 20% rate in force for 2026, then submitted $265, a figure none of its own arithmetic produces." -us,scenario_100,state_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its $822.40 federal EITC is 40% of $5,915 minus the $3,859 traditional 401(k) deferral; elective deferrals do not reduce EITC earned income, so the federal credit is 40% x $5,915 = $2,365.89. It then applied a 10% Montana match rather than the 20% rate for 2026, understating the credit nearly sixfold." -us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted Montana has no refundable individual income tax credit; Montana pays a refundable EITC equal to a percentage of the federal credit, 20% for 2026, which is $473.18 on this household's $2,365.89 federal EITC. Its filing-threshold argument is beside the point: the credit is refundable and is paid on a filed return regardless of liability, and the benchmark instructs that filing and take-up be assumed." -us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It computed the federal EITC correctly as the 40% phase-in on $5,915 (~$2,366) and correctly counted both children as qualifying, then applied a 10% Montana match instead of the 20% rate for 2026, deriving $237. It then submitted $297, a number its own reasoning never produces." -us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly identified the Montana credit as a percentage of the federal EITC and confirmed two qualifying children with a positive federal credit, then abandoned the computation and submitted 0. Montana's EITC is refundable at 20% of federal for 2026, so the low state liability it treated as disqualifying does not reduce the $473.18 payout." -us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It claimed the Montana EITC is nonrefundable and therefore zeroed out by the absence of state income tax liability. The Montana EITC is refundable, so the full 20% match on the $2,365.89 federal credit — $473.18 — is paid to this filer." -us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It got the federal EITC right at $2,366 (40% phase-in on the full $5,915, correctly declining to net out the 401(k) deferral) and correctly concluded the Montana credit is refundable, then applied the superseded 10% match rate. Montana's 2026 rate is 20% of the federal EITC, giving $473.18 rather than $237." -us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It asserted the Montana EITC is nonrefundable for 2026 and therefore worth nothing to a filer with no Montana liability. The credit is refundable at 20% of the federal EITC, and this head of household with two qualifying children and $5,915 of wages has a $2,365.89 federal credit, so $473.18 is paid out." -us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It invented both inputs: a 12% Montana match rate and a $2,650 federal EITC. Montana matches at 20% for 2026 and the federal credit is the 40% phase-in on $5,915, $2,365.89, giving $473.18." -us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Its $371 federal EITC is on the scale of the childless-worker schedule, not the two-child schedule; with children aged 6 and 5 in the household the phase-in rate is 40% of $5,915, giving $2,365.89. It then matched at 10% instead of Montana's 20%, understating the state credit more than twelvefold." -us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It fabricated a $1,200 Montana refundable child tax credit for the five-year-old — Montana has enacted no state child credit — and computed its EITC component as 3% (the rate that applied through 2023) of an $822.40 federal credit that had the $3,859 401(k) deferral netted out of earned income. The correct chain is $5,915 x 40% = $2,365.89 federal, matched at Montana's 20% rate, for $473.18; it also submitted $24.67 while its explanation totaled $1,224.67." -us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It computed the federal EITC exactly right at $2,366 and correctly treated the Montana credit as refundable, then matched at 10%. Montana's 2026 EITC is 20% of the federal credit, so the answer is $473.18." -us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It asserted without derivation that no state refundable credit applies, ignoring Montana's refundable EITC. The filer's $5,915 of wages with two qualifying children puts the federal credit in the 40% phase-in range at $2,365.89, and Montana's 20% match pays $473.18." -us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It denied Montana offers a refundable EITC and asserted the household has zero earned income, contradicting the stated $5,915 of gross wages. Montana's EITC is refundable at 20% of the federal credit, and the federal credit on $5,915 with two qualifying children is $2,365.89, so the state credit is $473.18." -us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It set the federal EITC to zero and propagated that to the state credit. With two qualifying children, $5,915 of wages and AGI far below the phase-out start, the federal credit is the full 40% phase-in amount of $2,365.89, and Montana's refundable 20% match is $473.18." -us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave a bare assertion of zero with no derivation. The correct computation is a $2,365.89 federal EITC (40% phase-in on $5,915 with two qualifying children) times Montana's refundable 20% match, $473.18; a zero is consistent only with denying Montana's EITC exists or treating it as nonrefundable." -us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its explanation applied Montana's 3% match rate, which was replaced after 2023, to the federal EITC to get $70.98, and it submitted 0 alongside that explanation. Montana's 2026 credit is 20% of the $2,365.89 federal EITC, $473.18." -us,scenario_100,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It claimed Montana refundable credits are not modeled in PolicyEngine and returned 0. PolicyEngine computes the Montana EITC as a percentage of the federal credit — 20% of $2,365.89 for 2026 — giving $473.18." -us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It stated Montana levies no individual income tax and therefore has no refundable income tax credits. Montana taxes individual income and pays a refundable EITC at 20% of the federal credit, which here is $2,365.89 (40% phase-in on $5,915), yielding $473.18." -us,scenario_100,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used Montana's pre-2024 3% match rate on an otherwise correct $2,366 federal EITC. The 2026 rate is 20% of the federal credit, so the refundable Montana credit is $473.18, not $70.98." -us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It concluded from the household facts that no Montana refundable credit is indicated, without testing the EITC pathway. Wages of $5,915 with two children aged 6 and 5 generate a $2,365.89 federal EITC, and Montana's refundable state EITC pays 20% of that, $473.18." -us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It asserted the benchmark yields no positive Montana credit for a low-income profile rather than computing one. Low earnings are exactly what maximizes the phase-in: $5,915 x 40% = $2,365.89 federal EITC, matched at Montana's refundable 20% rate for $473.18." -us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It put the federal EITC near the two-child plateau at ~$4,207, a level requiring roughly $10,500 of earned income; at $5,915 the credit is still climbing the 40% phase-in at $2,365.89. Its 10% Montana match instead of the 20% rate partly offset that overstatement, landing at $421 rather than $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It had the federal EITC right at $2,366 but applied a 10% Montana match. Montana's 2026 EITC rate is 20% of the federal credit, giving $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It added a $1,200 refundable Montana young-child credit that does not exist in Montana law on top of a 10% match of the $2,366 federal EITC. Montana's only refundable credit here is the EITC at 20% of federal, $473.18." -us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It stacked a nonexistent $1,200 Montana young-child credit for the five-year-old onto a 10% state EITC match. Montana has no state child credit, and its refundable EITC matches the $2,365.89 federal credit at 20%, for $473.18." -us,scenario_100,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied Montana's correct 20% match rate but to an $822.40 federal EITC computed on wages net of the $3,859 traditional 401(k) deferral. EITC earned income here is the full $5,915 of gross wages, so the federal credit is $2,365.89 and the Montana credit $473.18." -us,scenario_100,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It asserted with no derivation that no Montana refundable credit applies. Montana's refundable EITC pays 20% of the federal credit, which is $2,365.89 for this head of household with two qualifying children and $5,915 of wages, giving $473.18." -us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It stated Montana offers no state EITC and looked instead for property-tax and elderly credits that are irrelevant to a 46-year-old filer. Montana has a refundable EITC set at 20% of the federal credit for 2026, so the $2,365.89 federal credit produces $473.18." -us,scenario_100,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It denied the existence of a Montana state EITC and evaluated only elderly/disabled property-tax credits. The Montana EITC exists and is refundable at 20% of the federal credit, paying $473.18 on this household's $2,365.89 federal EITC." -us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It returned a bare zero with no analysis of the Montana EITC pathway. The correct derivation is the 40% federal phase-in on $5,915 ($2,365.89) times Montana's refundable 20% match, $473.18." -us,scenario_100,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It fabricated a $1,200 Montana refundable child tax credit and pegged the federal EITC at roughly $750, about a third of the 40% phase-in value of $2,365.89 on $5,915 of wages, then matched at 10%. Montana's only refundable credit for this household is the EITC at 20% of federal, $473.18." -us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It denied Montana offers a state EITC or any refundable credit on these facts. Montana's EITC is refundable at 20% of the federal credit for 2026, and the $5,915 of wages with two qualifying children yields a $2,365.89 federal credit, so the state credit is $473.18." -us,scenario_100,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It computed the federal EITC correctly at $2,366 and correctly ruled out other Montana refundable credits, then applied a 10% match. Montana's 2026 rate is 20% of the federal EITC, giving $473.18." -us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It asserted Montana has no state income tax and therefore no refundable income tax credits. Montana imposes an individual income tax and pays a refundable EITC of 20% of the federal credit, $473.18 on this household's $2,365.89 federal EITC." -us,scenario_100,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It derived the federal EITC exactly right at $2,366.00 but applied a 10% Montana match. The Montana EITC rate for 2026 is 20% of the federal credit, so the refundable amount is $473.18." -us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It denied Montana has an earned income tax credit and reasoned that no Montana liability means no credit. Montana's EITC exists and is refundable, paying 20% of the $2,365.89 federal credit — $473.18 — irrespective of state tax liability." -us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It returned zero with the bare claim that no Montana refundable credit is applicable or needed. The Montana refundable EITC applies here at 20% of the federal credit, which is the 40% phase-in on $5,915 of wages with two qualifying children, giving $473.18." -us,scenario_100,tanf,claude-fable-5,llm_error,state_local_rule,False,"It applied Montana's $200-plus-25%-of-remainder disregard correctly, reaching about $220 of countable monthly income and a positive grant, then discarded that result and submitted $0 on the false claim that earnings exceed the net income standard; countable income of $219.69 sits $505.41 below Montana's three-person payment standard. It also priced the grant off the frozen $588 maximum rather than the 2026 standard of $725.10, so even its abandoned arithmetic understated the benefit." -us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It asserted Montana TANF is not computed for this household and returned $0 without running any eligibility or benefit step. Montana's formula pays the three-person payment standard of $725.10 less countable earnings of $219.69 after the $200-plus-25% disregard, or $505.41 monthly." -us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It applied a $2,000 TANF asset limit to a household it counted as four people; Montana's TANF resource limit is $3,000 and the assistance unit is three (head plus two children), so the $2,800 bank balance passes. It also compared gross annual wages of $5,915 to the income limit instead of the post-disregard countable figure of $219.69 per month, which is far below the $725.10 payment standard." -us,scenario_100,tanf,claude-opus-4.7,llm_error,categorical_eligibility,False,"It conditioned eligibility on work-participation and application steps that do not enter the benefit computation, then submitted $0 after acknowledging earnings are minimal and assets within limits. The operative test is the $200-plus-25% disregard leaving $219.69 countable against the $725.10 three-person payment standard, which pays $505.41 monthly." -us,scenario_100,tanf,claude-opus-4.8,llm_error,state_local_rule,False,"It stated a $588 monthly maximum grant and then reported a $595 monthly benefit, so it never subtracted countable income from the payment standard and produced a grant larger than its own stated maximum. The correct step is $725.10 minus $219.69 of countable earnings, or $505.41 per month." -us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It returned $0 by bare assertion, citing no payment standard, disregard, or income test. Montana's three-person standard of $725.10 less $219.69 of countable earnings pays $505.41 monthly for this household." -us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It disqualified the household with a roughly $1,000 countable-liquid-asset limit; Montana's TANF resource limit is $3,000, so the $2,800 bank balance passes alongside the excluded vehicle. With resources clear, the grant is the $725.10 three-person payment standard less $219.69 of countable earnings, or $505.41 monthly." -us,scenario_100,tanf,claude-sonnet-5,llm_error,state_local_rule,False,"It used a $327 monthly three-person payment standard, under half Montana's 2026 standard of $725.10, and then paid essentially that full amount. Subtracting the $219.69 of countable earnings from $725.10 gives $505.41 monthly, not $327." -us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to the $2,800 bank balance to reach $4,781 of countable resources; Montana excludes the household vehicle from the TANF resource test, leaving $2,800 against the $3,000 limit. The household is resource-eligible and receives $725.10 less $219.69 of countable earnings, or $505.41 monthly." -us,scenario_100,tanf,deepseek-v4-pro,llm_error,state_local_rule,False,"It used a $605 three-person payment standard instead of Montana's $725.10 and disregarded 50% of the post-$200 remainder instead of 25%, yielding $146.46 countable rather than $219.69. Both errors compound into a $458.54 monthly grant instead of $505.41." -us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,other,False,"It submitted $0 in the outputs field while its own explanation computed $4,023.75, contradicting itself on the sign of eligibility. That explanation was also wrong on the standard, using $555 per month rather than Montana's $725.10, which with $219.69 of countable earnings pays $505.41 monthly." -us,scenario_100,tanf,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied Montana's $200-plus-25% disregard correctly to reach $220 of countable income but paired it with the frozen $588 payment standard instead of the 2026 three-person standard of $725.10, understating the monthly grant by $137. It then padded its own $4,416 result to $4,419 with no rule behind the change." -us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared the household fails Montana's requirements without naming an income limit, resource limit, or payment standard. Countable earnings of $219.69 fall $505.41 below the $725.10 three-person payment standard, so the household qualifies for $6,064.96 annually." -us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,asset_resource,False,"It assumed SSI receipt suppresses the TANF grant; the head's $2,800 bank balance exceeds SSI's $2,000 resource limit, so SSI is $0 and no unearned income enters the TANF income test. The only countable income is $219.69 of post-disregard earnings against the $725.10 payment standard." -us,scenario_100,tanf,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied a $200-plus-50% earned income disregard, leaving $146 countable, and used a $588 payment standard. Montana disregards $200 plus 25% of the remainder for $219.69 countable, and its three-person standard is $725.10, giving $505.41 monthly." -us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It stated a bare $0 with no eligibility test or benefit formula. Montana pays the $725.10 three-person payment standard less $219.69 of countable earnings, or $6,064.96 for the year." -us,scenario_100,tanf,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $5,298.48 is twelve months of $588 less $146.46, so it used the frozen $588 payment standard with a 50% disregard on the post-$200 remainder. Montana's disregard is 25% of that remainder, giving $219.69 countable, and its 2026 three-person standard is $725.10, for $505.41 monthly." -us,scenario_100,tanf,gemini-3.7-flash,llm_error,other,False,"Its explanation found the household income-eligible and computed $4,824, yet it submitted $0 as the value, contradicting its own finding. The correct grant is $725.10 less $219.69 of countable earnings for twelve months, or $6,064.96." -us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,It claimed Montana TANF is not modeled and returned $0 without computing anything. Montana's grant formula applies here: the $725.10 three-person payment standard minus $219.69 of countable earnings after the $200-plus-25% disregard. -us,scenario_100,tanf,glm-5.2,llm_error,state_local_rule,False,"It added the $710 FLSA overtime premium to the $5,915 of wages that already contain it, compared that inflated gross directly to the payment standard without applying the $200-plus-25% earned income disregard, and asserted a work-hours failure for a head working 55 hours per week. Countable income is $219.69 per month against a $725.10 standard, so the grant is $505.41 monthly." -us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,It asserted Montana's FAIM program produces no positive benefit for this household and skipped the computation entirely. The three-person payment standard of $725.10 less $219.69 of countable earnings pays $505.41 monthly. -us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It concluded TANF is not payable from the stated facts without applying an income test, treating the absence of an explicit benefit-receipt fact as ineligibility. The facts given — two dependent children, $5,915 of wages, $2,800 in resources — produce $219.69 of countable income against Montana's $725.10 payment standard and a $6,064.96 annual grant." -us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required an explicit eligibility trigger in the facts and returned $0 when none appeared, never running Montana's income test. A three-person unit with $492.92 of monthly earnings has $219.69 countable after the $200-plus-25% disregard and receives $505.41 monthly against the $725.10 standard." -us,scenario_100,tanf,gpt-5.5,llm_error,state_local_rule,False,"It used a $588 three-person payment standard and an ad-hoc $150 countable-income estimate. Montana's 2026 standard is $725.10 and its $200-plus-25% disregard leaves exactly $219.69 countable, producing a $505.41 monthly grant rather than $438." -us,scenario_100,tanf,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $359.50 monthly grant implies a three-person payment standard near $579, well under Montana's $725.10. Subtracting the $219.69 of countable earnings from the correct standard yields $505.41 per month, or $6,064.96 annually." -us,scenario_100,tanf,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $4,419.75 is exactly twelve months of $588 minus $219.6875, so it applied Montana's $200-plus-25% disregard correctly but anchored on the frozen $588 three-person maximum. The 2026 standard is $725.10, which pays $505.41 monthly." -us,scenario_100,tanf,gpt-5.6-terra,llm_error,asset_resource,False,"It imputed SSI as countable unearned income and used it to zero out the grant; the head's $2,800 bank balance exceeds SSI's $2,000 resource limit, so SSI is $0 for this household. Earnings alone leave $219.69 countable against the $725.10 three-person payment standard, paying $505.41 monthly." -us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"It reproduced Montana's structure exactly — the $200 monthly plus 25% disregard leaving $2,636.25 of annual countable earnings, resources under $3,000, vehicle excluded — but rounded the three-person payment standard to $725 instead of the 2026 value of $725.10. That single rounding of the standard accounts for the entire $1.21 annual shortfall." -us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It returned $0 stating no amount could be derived from the listed inputs, skipping Montana's benefit formula. Those inputs are sufficient: $492.92 monthly wages less the $200-plus-25% disregard gives $219.69 countable, and $725.10 minus that is $505.41 per month." -us,scenario_100,tanf,grok-4.5,llm_error,state_local_rule,False,"It applied Montana's $200-plus-25% disregard correctly to reach $220 countable but used a $588 three-person payment standard rather than the 2026 standard of $725.10. That $137 gap in the standard is the whole error, taking the monthly grant from $505.41 down to $368." -us,scenario_100,tanf,grok-4.6,llm_error,state_local_rule,False,"Its disregard arithmetic is right — $493 monthly earnings less $200 and 25% of the remainder leaves about $220 countable — but its $588 payment standard is the frozen maximum rather than Montana's 2026 three-person standard of $725.10. The correct grant is $505.41 monthly, or $6,064.96 annually." -us,scenario_100,tanf,grok-build-0.1,llm_error,state_local_rule,False,"It used a $675 monthly three-person maximum instead of Montana's $725.10 while applying the $200-plus-25% disregard correctly to reach $220 countable. Its own $455 monthly figure also annualizes to $5,460, not the $5,464 submitted; the correct grant is $505.41 monthly." -us,scenario_100,tanf,inkling,llm_error,state_local_rule,False,"It applied Montana's $200-plus-25% earned income disregard correctly for about $220 of countable income, then subtracted it from a $588 payment standard instead of the 2026 three-person standard of $725.10. Using the correct standard gives $505.41 monthly and $6,064.96 for the year." -us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for tanf, so no substantive computation was submitted. The required derivation is Montana's $725.10 three-person payment standard minus $219.69 of countable earnings after the $200-plus-25% disregard, or $6,064.96 annually." -us,scenario_100,tanf,kimi-k3,llm_error,state_local_rule,False,"Montana's FAIM cash grant equals the three-person payment standard of $725.10 per month minus countable income, and countable income here is $219.69 — $492.92 of monthly wages less the $200 flat disregard and 25% of the remainder — giving $505.41 per month and $6,064.96 for the year. The household clears every gate: two dependent children, a $2,800 bank balance under Montana's $3,000 TANF resource limit, an excluded vehicle, and no unearned income, since that same $2,800 exceeds SSI's $2,000 resource limit and zeroes SSI. The wrong answers split two ways: declaring ineligibility on invented asset limits, imputed SSI income, double-counted overtime premium, or ""not modeled"" assertions; and pricing the grant off the frozen $588 three-person maximum (or a 50% disregard) instead of the 2026 standard of $725.10." -us,scenario_100,tanf,minimax-m3,llm_error,state_local_rule,False,It characterized Montana grants as roughly $50–$200 per month and zeroed out by earned income; Montana's three-person payment standard is $725.10 per month and the $200-plus-25% disregard leaves only $219.69 countable. The household therefore receives $505.41 monthly rather than $0. -us,scenario_100,tanf,ox-alpha,llm_error,asset_resource,False,"It imputed about $780 per month of SSI as countable unearned income; the head's $2,800 bank balance exceeds SSI's $2,000 resource limit, so SSI is $0 here. It also used a $614 payment standard rather than Montana's $725.10, against which the $219.69 of countable earnings leaves a $505.41 monthly grant." -us,scenario_100,tanf,qwen-3.7-max,llm_error,state_local_rule,False,"It added the $710 FLSA overtime premium to wages that already include it and tested the resulting $6,625 gross against a self-invented $6,600–$7,200 annual income limit. Montana instead compares post-disregard countable income of $219.69 per month to the $725.10 three-person payment standard, paying $505.41 monthly." -us,scenario_100,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"It asserted income is too high for Montana TANF without stating a threshold or applying the earned income disregard. Monthly earnings of $492.92 reduce to $219.69 countable and fall $505.41 short of the $725.10 three-person payment standard, so the household receives $6,064.96." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It reached the correct $101,130 AGI and $16,100 standard deduction, then subtracted a $1,400 qualified-auto-loan-interest deduction that is worth $0 here because the qualified-vehicle status inputs are unlisted and therefore false (and which would in any case phase down by $200 per $1,000 of MAGI above $100,000). It then submitted $12,064, a figure neither of the two bracket computations in its own reasoning ($13,111 and $13,058) produced." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Every 2026 parameter it used is right — $101,130 AGI, $16,100 standard deduction, 10/12/22 brackets at $12,400 and $50,400 — but it subtracted an extra $1,000 auto-loan-interest deduction (the $1,400 phased down by $400 for the $1,130 of MAGI over $100,000). That deduction is $0 because the qualified-vehicle status flags are unlisted and therefore false; the $220 shortfall is exactly 22% of the $1,000 it deducted." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $700 of veterans benefits into gross income, which are excluded from AGI, and applied the 2024 single standard deduction of $14,600 instead of the 2026 figure of $16,100, producing $87,230 of taxable income against the correct $85,030. Its bracket arithmetic then ran 12% across $75,630 — far past the top of the 12% band — and it submitted $11,435 rather than the $10,637.50 that same arithmetic yielded." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning is fully correct: $101,130 AGI, $16,100 standard deduction, $85,030 taxable income, and $1,240 + $4,560 + $7,618.60 = $13,418.60 under the $12,400/$50,400 breakpoints. It then discarded that exact result as 'rounding to bracket estimates' and submitted $14,716, so the failure is entirely at the reporting step rather than in the tax computation." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It derived the correct $85,030 taxable income and computed $1,240 + $4,560 + $7,619 ≈ $13,419 with the right $12,400/$50,400 breakpoints, then abandoned that figure and submitted $13,876 with no derivation behind it. The submitted number corresponds to roughly $87,110 of taxable income, which none of its stated inputs produce." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It treated 2026 as 2025 grown by inflation, cycling through $15,350, $15,390, and finally a $15,750 standard deduction with invented breakpoints of $12,300 and $49,950, rather than the OBBBA 2026 single standard deduction of $16,100 and breakpoints of $12,400 and $50,400. Its medical-expense and capital-loss handling are correct, so the entire $124 error comes from the guessed deduction amount and bracket edges." -us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It got the correct $16,100 standard deduction and $85,030 taxable income, then applied fabricated 2026 breakpoints of $12,150 and $49,400 instead of $12,400 and $50,400, giving $13,523.60. It then subtracted a further unexplained 'standard deduction indexing' adjustment to report $13,322, double-counting an indexation it had already applied." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its income side is exact ($101,130 AGI after the $1,870 capital loss), but it used a $15,600 standard deduction and bracket edges of $12,175 and $49,550 instead of $16,100, $12,400, and $50,400. That understates the standard deduction by $500 and shifts $850 of income from the 12% band into the 22% band, producing $13,618 instead of $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $3,840 of health-insurance premiums from wages as a further exclusion; the $103,000 gross wage figure is already the taxable amount, and those premiums enter only as itemized medical expenses, which fall below the 7.5%-of-AGI floor of $7,584.75. Combined with a $15,400 standard deduction instead of $16,100, it understated taxable income by $3,140." -us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA rate structure expires for 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption, and pre-TCJA 10/15/25 rates. OBBBA made the TCJA structure permanent, so 2026 carries a $16,100 standard deduction, no personal exemption, and 10/12/22 rates breaking at $12,400 and $50,400." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the TCJA-sunset parameters — an $8,300 standard deduction plus a $5,300 personal exemption at 10/15/25 rates — when 2026 uses the OBBBA-permanent $16,100 standard deduction, no exemption, and 10/12/22 rates. It compounded that by subtracting $1,920 of employer-plan premiums from the $103,000 of wages, which is already the taxable wage figure." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It dropped the $1,870 long-term capital loss entirely and taxed the full $103,000 of wages, and used a $15,300 standard deduction rather than $16,100, arriving at $87,700 of taxable income against the correct $85,030. Those two omissions add $2,670 to taxable income, which at the 22% marginal rate accounts for the overstatement." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It stopped at the correct $101,130 AGI and asserted a liability of 'around $15,400' without ever naming a standard deduction or applying a bracket schedule. That figure corresponds to about $94,000 of taxable income, i.e. a deduction near $7,100, whereas the $16,100 single standard deduction leaves $85,030 and $13,418.60 under the $12,400/$50,400 breakpoints." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expire for 2026 and applied an $8,250 standard deduction plus a $5,300 personal exemption with reverted 10/15/25 brackets. The 2026 schedule is the OBBBA-permanent one — $16,100 standard deduction, no personal exemption, 10/12/22 rates — so its taxable income of $87,580 overstates the correct $85,030 and its top rate is 3 points too high." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Its explanation breaks off mid-derivation without ever fixing a standard deduction or applying brackets, then asserts $11,102. The correct chain — $101,130 AGI, $16,100 standard deduction, $85,030 taxable, 10/12/22 at $12,400/$50,400 — gives $13,418.60; $11,102 implies about $74,500 of taxable income, a deduction near $26,600 that no provision supplies." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset law for 2026: an $8,300 standard deduction, a $5,300 personal exemption, and pre-TCJA brackets breaking at $11,000 and $44,700 with a 15% and 25% rate. OBBBA keeps the TCJA structure in 2026, so the correct parameters are a $16,100 standard deduction, no exemption, and 10/12/22 rates at $12,400 and $50,400." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It computed the correct $101,130 AGI and then subtracted an $8,300 standard deduction plus a $5,300 personal exemption on the assumption that TCJA lapses in 2026. Both are wrong for 2026 under OBBBA — the standard deduction is $16,100 and personal exemptions remain repealed — and the pre-TCJA 15%/25% rates it applied are replaced by 12%/22%." -us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 parameters unchanged: a $15,000 standard deduction and breakpoints of $11,925 and $48,475, instead of the 2026 values of $16,100, $12,400, and $50,400. The $1,100 of extra taxable income and the earlier start of the 22% band produce $13,862.60 rather than $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI and exclusion reasoning are exact, but it invented indexed 2026 parameters — a $15,450 standard deduction with breakpoints at $12,283 and $49,929 — instead of the published $16,100, $12,400, and $50,400. The $650 understated deduction and the $471 earlier start of the 22% band account for the $192 overstatement." -us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the correct 2026 bracket breakpoints ($12,400 and $50,400) and correct AGI, but used a $15,800 standard deduction instead of $16,100, leaving $85,330 of taxable income rather than $85,030. The entire $66 error is 22% of that $300 deduction shortfall." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It described 'taxable pension income' as part of the base, pulling the $2,308 of tax-exempt private pension into income that is excluded from AGI, and never stated a standard deduction or bracket application. Its $14,498 corresponds to about $89,930 of taxable income, roughly $4,900 above the correct $85,030 that $101,130 AGI less the $16,100 standard deduction produces." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted $9,587 with an explanation that recites the definition of the output and shows no deduction amount, bracket, or arithmetic. The correct derivation — $101,130 AGI, $16,100 standard deduction, $85,030 taxable, 10/12/22 at $12,400/$50,400 — gives $13,418.60; $9,587 implies about $67,600 of taxable income, a deduction near $33,500 that no provision supports." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got AGI right at $101,130 and then applied 'scheduled 2026 current-law' parameters consisting of a standard deduction plus a personal exemption, reaching about $87,380 of taxable income. Personal exemptions remain repealed in 2026 and the standard deduction is $16,100, which leaves $85,030 and $13,418.60 at 10/12/22 rates rather than the pre-TCJA rates its $16,203 reflects." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It claimed a 'phased-down qualified auto-loan-interest deduction' on top of the $16,100 standard deduction, reducing taxable income to $84,030. That deduction is $0 because the qualified-vehicle status inputs are unlisted and therefore false; its $220 shortfall is exactly 22% of the $1,000 it deducted." -us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its AGI, standard deduction, and 2026 brackets are all correct, but it subtracted an additional $1,000 auto-loan-interest deduction ($1,400 less the $400 phaseout for MAGI above $100,000) to reach $84,030 of taxable income instead of $85,030. The deduction is unavailable because the qualified-vehicle status flags are unlisted and therefore false, and 22% of the $1,000 is the entire $220 error." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a guessed $15,300 standard deduction rather than $16,100, overstating taxable income by $800, and then applied unstated 'projected' brackets that give $13,774 where the $12,400/$50,400 schedule on its own $85,830 figure yields $13,594.60. Both the deduction amount and the bracket edges depart from the published 2026 parameters." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a 'post-TCJA sunset' 2026 with an ~$8,400 standard deduction plus a $5,300 personal exemption taxed at 10/15/25. OBBBA made the TCJA structure permanent, so 2026 provides a $16,100 standard deduction, no personal exemption, and 10/12/22 rates, which turn its $87,430 taxable income into $85,030 and its $16,200 into $13,418.60." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It carried the 2025 parameters into 2026: a $15,000 standard deduction with breakpoints of $11,925 and $48,475 instead of $16,100, $12,400, and $50,400. That leaves $86,130 of taxable income rather than $85,030 and starts the 22% band $1,925 too early, giving $13,863." -us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It explicitly applied 'reverted TCJA law' for 2026 — an $8,300 standard deduction plus a $5,300 personal exemption, with 10/15/25 rates breaking at $12,100 and $48,800. The 2026 law is the OBBBA-permanent TCJA structure: $16,100 standard deduction, no personal exemption, and 10/12/22 rates at $12,400 and $50,400." -us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It set up the computation exactly right — $101,130 AGI, $16,100 standard deduction, $85,030 taxable income, 2026 10/12/22 brackets — but reported an approximate $13,426 instead of applying the $12,400 and $50,400 breakpoints precisely. The exact application, $1,240 + $4,560 + 0.22 × $34,630, is $13,418.60, so the $7.40 gap is a rounded bracket edge in the final step." -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this key, so no tax computation was submitted to evaluate. The failure is the absent output itself rather than a substantive error in deductions, brackets, or credits." -us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,174 auto-loan-interest deduction, computed by reducing $1,400 by a linear 20% of the $1,130 of AGI above $100,000; the statutory phaseout is $200 per full or partial $1,000 of excess, and the deduction is $0 in any case because the qualified-vehicle status inputs are unlisted and therefore false. Its taxable income of $83,856 is $1,174 below the correct $85,030, and 22% of that gap is the entire $258.28 error." -us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Its explanation states that the standard deduction and 2026 brackets 'leave positive tax before credits,' then submits $0 on the circular ground that 'the numeric answer returned is zero.' No nonrefundable credit exists for this childless single filer, so the correct value is the $13,418.60 its own described method produces on $85,030 of taxable income." -us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and invented breakpoints of $12,150 and $49,000 instead of the 2026 values of $16,100, $12,400, and $50,400, computing $13,641 on $85,380 of taxable income. It then submitted $14,827, a number its own bracket arithmetic never produced." -us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It subtracted $4,824 of employee payroll tax as a 'nonrefundable credit,' which does not exist in the federal income tax, and it also refused the $1,870 capital loss against wages by offsetting it against the tax-exempt pension while using a $13,600 standard deduction instead of $16,100. Removing the invented payroll-tax credit and applying the correct $16,100 deduction to $101,130 of AGI gives $85,030 taxable and $13,418.60." +us,scenario_100,state_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It used a 10% Montana EITC rate instead of the 2026 rate of 20%. It also switched between several earned-income bases and never settled on the gross $5,915 wage base; the correct federal EITC on that base is $2,365.89. Its final $265 is 10% of $2,646, a number with the 40% federal phase-in never applied, so both the federal base and the state rate are wrong." +us,scenario_100,state_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed the federal EITC as $822.40, which is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The EITC base is gross earned wages of $5,915, which gives $2,365.89. It then applied the outdated 10% Montana rate instead of the 2026 rate of 20%." +us,scenario_100,state_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It said Montana has no earned-income-based refundable credit. In fact Montana's refundable EITC equals 20% of the federal EITC in 2026. Being below the filing threshold does not block a refundable credit. The $2,365.89 federal EITC yields $473.18." +us,scenario_100,state_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It correctly got a federal EITC of about $2,366 but applied a 10% Montana rate instead of the 2026 rate of 20%. It then submitted $297 even though its own 10% arithmetic gave $237. The correct amount is 20% × $2,365.89 = $473.18." +us,scenario_100,state_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It recognized that the federal EITC is positive and that Montana's EITC is a percentage of it, but still reported $0 as a 'conservative' estimate. Montana's EITC is refundable and pays 20% of the $2,365.89 federal EITC, which is $473.18, even when state tax liability is zero." +us,scenario_100,state_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It treated the Montana EITC as nonrefundable and zeroed it out because tax liability is zero. The Montana EITC is refundable and pays 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its federal EITC of $822.40 is 40% of wages after subtracting the traditional 401(k) deferral. The EITC base is gross wages of $5,915, which gives $2,365.89. It then applied the outdated 10% Montana rate instead of 20%, so it got $82.24 instead of $473.18." +us,scenario_100,state_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It correctly computed a federal EITC of about $2,366 and correctly treated the Montana EITC as refundable. However, it applied the 10% rate enacted for 2024 instead of the 20% rate that applies in 2026, so it got $237 instead of $473.18." +us,scenario_100,state_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It said the Montana EITC is nonrefundable and answered $0. The Montana EITC is refundable, at 20% of the $2,365.89 federal EITC in 2026, which is $473.18." +us,scenario_100,state_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of about $2,366 but applied the old 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It applied a 12% Montana rate, which has never been Montana's EITC rate; the 2026 rate is 20%. It also used a $2,650 federal EITC, which does not match 40% × $5,915 = $2,365.89." +us,scenario_100,state_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It computed a federal EITC of only $371, far below the two-child 40% phase-in result of $2,365.89 on $5,915 of earnings. It also applied a 10% Montana rate instead of 20%, so it got $37 instead of $473.18." +us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It applied the repealed 3% Montana EITC rate to a federal EITC of $822.40, which was reduced for the 401(k) deferral. The correct figures are 20% of $2,365.89. It also added a $1,200 young-child credit that is not part of this household's Montana refundable credits. Its submitted $24.67 covers only the 3% EITC piece." +us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It said no Montana refundable credits apply and overlooked Montana's refundable EITC. That credit pays 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It correctly identified the refundable Montana EITC and the $2,366 federal EITC but used the old 10% rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It found no state refundable credits and missed Montana's refundable EITC. That credit pays 20% of the $2,365.89 federal EITC earned on $5,915 of wages with two qualifying children." +us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It said Montana has no refundable EITC and that the household has zero earned income. The head earns $5,915 in wages, which produces a $2,365.89 federal EITC. Montana's refundable EITC pays 20% of that, which is $473.18." +us,scenario_100,state_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It set the federal EITC to $0 and carried that zero into Montana. A head of household with two qualifying children under 19 and $5,915 of earned income is in the 40% phase-in, which gives $2,365.89. Montana's 20% EITC on that is $473.18." +us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no reasoning and answered $0. The correct figure is 20% of the $2,365.89 federal EITC, which is $473.18. A $0 answer matches ignoring Montana's refundable EITC entirely." +us,scenario_100,state_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"Its reasoning applied the repealed 3% Montana EITC rate and got $70.98. It then submitted $0, which contradicts that reasoning. The 2026 rate is 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It claimed PolicyEngine does not model Montana refundable credits. The Montana refundable EITC is computed at 20% of the federal EITC, which gives $473.18 here." +us,scenario_100,state_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It claimed Montana has no state individual income tax. Montana levies an income tax and provides a refundable EITC equal to 20% of the federal EITC for 2026, which is $473.18 here." +us,scenario_100,state_refundable_credits,glm-5.3,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of $2,366 but applied the original 3% Montana rate. That rate was raised to 10% for 2024 and to 20% for 2026, so it got $70.98 instead of $473.18." +us,scenario_100,state_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It found no refundable Montana credits and overlooked the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It assumed a low-income household gets no Montana refundable credits. Low earned income is exactly what produces the federal EITC of $2,365.89, and Montana refunds 20% of it, which is $473.18." +us,scenario_100,state_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It overstated the federal EITC at $4,207. The two-child 40% phase-in on $5,915 of earnings gives only $2,365.89. It also applied a 10% Montana rate instead of 20%." +us,scenario_100,state_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It used the correct federal EITC of about $2,366 but the 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"It added a $1,200 young-child credit for the age-5 child, but this household's 2026 Montana refundable credits consist solely of the state EITC. It also applied a 10% EITC rate instead of 20%, getting $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It included a $1,200 young-child credit that is not part of this household's 2026 Montana refundable credits, which are only the state EITC. It also used a 10% rate on the $2,366 federal EITC instead of 20%, which would give $473.18." +us,scenario_100,state_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It used the correct 20% Montana rate but applied it to an $822.40 federal EITC. That figure is 40% of wages after subtracting the $3,859 traditional 401(k) deferral. The federal EITC base is gross wages of $5,915, which gives $2,365.89, so the credit is $473.18." +us,scenario_100,state_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"Its state EITC of $473.20 matches, but it added $1,800 of child tax credit for the two children. This household's 2026 Montana refundable credits consist solely of the state EITC, so the total inflated to $2,273.20." +us,scenario_100,state_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It applied the correct 20% rate to an $822.40 federal EITC, which was computed on wages after subtracting the traditional 401(k) deferral. The EITC earned-income base is gross wages of $5,915, which gives a $2,365.89 federal EITC and a $473.18 Montana EITC." +us,scenario_100,state_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said no Montana refundable credits apply and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It asserted that Montana offers no state EITC. Montana has a refundable EITC, 20% of the federal EITC for 2026, worth $473.18 here." +us,scenario_100,state_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It stated Montana has no state EITC and looked only at elderly and property-tax credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It ruled out credits based on age and rent and claimed Montana has no EITC for this case. Montana's refundable EITC applies to any federal EITC recipient and pays 20% of $2,365.89, which is $473.18." +us,scenario_100,state_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It said no refundable Montana credits apply and omitted the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It added a $1,200 child tax credit for the 5-year-old, but this household's 2026 Montana refundable credits are only the state EITC. It also used a federal EITC of about $750 instead of $2,365.89, and a 10% rate instead of 20%." +us,scenario_100,state_refundable_credits,kimi-k2.6,reference_engine_defect,credit_phaseout,False,"It stated Montana has no state EITC and looked only for rent or property-tax credits. Montana's refundable EITC pays 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,state_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used the correct $2,366 federal EITC but the 10% Montana rate instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It claimed Montana has no state income tax. Montana taxes income and provides a refundable EITC equal to 20% of the federal EITC, which is $473.18 here." +us,scenario_100,state_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It correctly took the federal EITC as $2,366 but applied the 10% rate that ended after 2025 instead of the 2026 rate of 20%. That gave $236.60 instead of $473.18." +us,scenario_100,state_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It asserted that Montana has no earned income tax credit and used zero tax liability as the reason for $0. Montana's EITC is refundable, pays 20% of the $2,365.89 federal EITC, and does not depend on tax liability." +us,scenario_100,state_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It found no applicable Montana refundable credits and missed the refundable state EITC. That credit is 20% of the $2,365.89 federal EITC, which is $473.18." +us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found countable earnings of about $220 a month, below the payment standard, which makes the family eligible. It then reversed itself and submitted $0. The correct benefit is the $725.10 three-person payment standard minus $219.69, which is $505.41 a month or $6,064.96 a year." +us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It claimed Montana TANF is not modeled and returned $0. In fact the family qualifies categorically with two minor children. Paying the $725.10 standard less $219.69 of countable earnings gives $6,064.96 a year." +us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It sized the unit at four instead of three and applied a $2,000 asset limit. Montana's TANF resource limit is $3,000, so the $2,800 bank balance passes. It also compared gross wages to an income limit without the $200 + 25% earnings disregard, which leaves only $219.69 a month countable." +us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"It asserted without calculating that countable income after disregards and a low payment standard would zero out the benefit. Countable earnings are only $219.69 a month against a $725.10 standard. Resources of $2,800 are under the $3,000 limit with the vehicle excluded, so the benefit is $505.41 a month." +us,scenario_100,tanf,claude-opus-4.8,llm_error,other,False,"It assumed a roughly $588 maximum, then submitted $595 a month, which is above its own maximum, so it never subtracted countable earnings. The correct grant is the $725.10 standard minus $219.69 of countable earnings, which is $505.41 a month or $6,064.96 a year, not $7,140." +us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It asserted PolicyEngine gives no Montana TANF benefit for this household and returned $0. Under the $725.10 standard, the family with two minor children and $219.69 of countable earnings receives $505.41 a month." +us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"It used an estimated $730 payment standard and rounded countable earnings to $220. PolicyEngine's three-person standard is $725.10 and exact countable earnings are $219.69, giving $505.41 a month. That makes the correct annual amount $6,064.96, not $6,120." +us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It applied a roughly $1,000 liquid-asset limit and ruled the family ineligible because of the $2,800 bank balance. Montana's TANF resource limit is $3,000, so the family passes. The benefit is $725.10 minus $219.69 of countable earnings." +us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327 monthly payment standard and paid that full amount as the grant. The actual standard for three is $725.10. The correct benefit is that standard minus $219.69 of countable earnings, which is $505.41 a month or $6,064.96 a year." +us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The family has two minor children, passes the $3,000 resource test, and receives the $725.10 standard less $219.69 of countable earnings." +us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to the $2,800 bank balance, giving $4,781 of countable resources. Montana excludes the household vehicle, so countable resources are $2,800, which is under the $3,000 limit. The family is therefore eligible for $505.41 a month." +us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $605 maximum grant and a $200 + 50% earnings disregard. Montana disregards $200 + 25%, which leaves $219.69 countable, not $146.46. The payment standard is $725.10, so the benefit is $505.41 a month, not $458.54." +us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It submitted $0 even though its explanation computed $4,023.75. That computation also used a $555 payment standard instead of the $725.10 three-person standard. The correct benefit is $725.10 minus $219.69, which is $505.41 a month or $6,064.96 a year." +us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 maximum and a $200 + 50% earnings disregard. Montana's standard for three is $725.10 and the disregard is $200 + 25%, which leaves $219.69 countable. The benefit is $505.41 a month, not $353.54." +us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the correct $200 + 25% disregard, but used the outdated $588 payment standard instead of PolicyEngine's 2026 standard of $725.10. That understates the grant by about $137 a month: $368 instead of $505.41." +us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared the household ineligible without naming any failed test. The family has two minor children, $2,800 of resources under the $3,000 limit, and $219.69 of countable earnings against a $725.10 standard. That qualifies it for $505.41 a month." +us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It assumed the head receives SSI and zeroed TANF on that basis. SSI is $0 because the $2,800 bank balance exceeds SSI's $2,000 resource limit, so no SSI offset or exclusion applies. The family receives the $725.10 standard minus $219.69." +us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $5,298 answer matches a $588 payment standard with a $200 + 50% disregard. Montana disregards $200 + 25%, leaving $219.69 countable, and the 2026 standard for three is $725.10. The correct benefit is $505.41 a month." +us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 without reasoning, which treats the family as ineligible. Two minor children, resources under $3,000, and $219.69 of countable earnings against the $725.10 standard produce $6,064.96 a year." +us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $5,298.48 equals $588 minus $146.46 times 12, which is an outdated $588 standard with a $200 + 50% disregard. The correct figures are a $725.10 standard and a $200 + 25% disregard ($219.69 countable), giving $505.41 a month." +us,scenario_100,tanf,gemini-3.7-flash,llm_error,thresholds_rates,False,"It submitted $0 while its explanation said the household qualifies and computed $4,824. That $402-a-month figure is also about $103 short of the correct $505.41, which is the $725.10 standard less $219.69 of countable earnings." +us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It asserted Montana TANF is not modeled and returned $0. The eligible three-person family receives the $725.10 standard minus $219.69 of countable earnings, which is $6,064.96 a year." +us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $710 FLSA overtime premium on top of wages that already include overtime. It then compared the full $6,625 to the payment standard without the $200 + 25% earnings disregard, and invoked a work requirement that 55 hours a week satisfies. Correct countable earnings are $219.69 a month against a $725.10 standard." +us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,"It asserted PolicyEngine gives no positive Montana TANF benefit here. The family passes the categorical, $3,000 resource, and income tests, and receives $725.10 minus $219.69, which is $505.41 a month." +us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated TANF as not payable because receipt was not listed, but the prompt assumes take-up. The family with two minor children receives the $725.10 standard less $219.69 of countable earnings, which is $6,064.96 a year." +us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,It claimed no TANF-qualifying circumstances exist. Two dependent children aged 6 and 5 in a low-earning single-parent household satisfy Montana's categorical requirement. The benefit is $505.41 a month. +us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"It estimated $150 of countable income, which is consistent with a 50% disregard, and a grant of $438. Montana disregards $200 + 25%, which leaves $219.69 countable, and the three-person standard is $725.10. The correct benefit is $505.41 a month." +us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50 monthly grant reflects a payment standard near the outdated $588. The 2026 three-person standard is $725.10, and subtracting $219.69 of countable earnings gives $505.41 a month." +us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,419.75 equals $588 minus $219.69 times 12, so it used the outdated $588 payment standard. The 2026 standard is $725.10, which yields $6,064.96 a year." +us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It counted SSI income against TANF. SSI is $0 because the $2,800 bank balance exceeds SSI's $2,000 resource limit. The only countable income is $219.69 of earnings against a $725.10 standard." +us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"It used $725 as the three-person payment standard instead of the exact $725.10 ($8,701.21 a year). That understated the annual benefit by $1.21: $6,063.75 instead of $6,064.96." +us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,298 matches a $588 standard with a $200 + 50% disregard. Montana disregards $200 + 25%, leaving $219.69 countable, and the 2026 standard is $725.10. The correct benefit is $505.41 a month." +us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25 monthly grant is $588 minus $219.69, so it used the outdated $588 payment standard rather than the 2026 standard of $725.10. The correct grant is $505.41 a month." +us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,"It applied the correct $200 + 25% disregard but used a $588 three-person payment standard instead of $725.10. That understates the benefit by about $1,645 a year." +us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It computed nothing and returned $0. The eligible family receives the $725.10 standard minus $219.69 of countable earnings, which is $6,064.96 a year." +us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"It used the outdated $588 payment standard. With the $725.10 standard for 2026, $725.10 minus $219.69 of countable earnings is $505.41 a month, not $368." +us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It used a $588 payment standard instead of $725.10. With $219.69 of countable earnings the grant is $505.41 a month, not $368." +us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"It got the countable earnings exactly right at $219.69, but subtracted them from an outdated $588 standard instead of $725.10. That produced $4,419.75 instead of $6,064.96." +us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,It used a $675 maximum for a family of three instead of the $725.10 standard. That left a $455 grant in place of $505.41 a month. +us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It used the outdated $588 maximum grant rather than the 2026 three-person standard of $725.10. After subtracting $219.69 of countable earnings, the correct grant is $505.41 a month, not about $368." +us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no TANF value or explanation, so there was no answer to score against the $6,064.96 reference." +us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"It handled the resource test and the $200 + 25% disregard correctly ($219.69 countable), but used a $588 grant level instead of the $725.10 three-person standard. The result understates the annual benefit by $1,645.21." +us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana grants are only $50 to $200 a month and that earnings wipe them out. The three-person standard is $725.10, so $219.69 of countable earnings leaves a $505.41 monthly grant." +us,scenario_100,tanf,ox-alpha,llm_error,categorical_eligibility,False,"It imputed about $780 a month of SSI as unearned income. SSI is $0 because the $2,800 bank balance exceeds the $2,000 SSI resource limit, so the only countable income is $219.69 of earnings against a $725.10 standard." +us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,It added the $710 overtime premium to wages that already include it. It then compared gross annual income to the payment standard without applying the $200 + 25% monthly earnings disregard. Countable earnings are only $219.69 a month against a $725.10 standard. +us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"It declared income too high without applying the $200 + 25% disregard. That disregard reduces $492.92 of monthly wages to $219.69 countable, well below the $725.10 standard, so the benefit is $505.41 a month." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the full $1,400 of auto loan interest as an OBBBA qualified vehicle-loan deduction. No qualifying vehicle is established, so taxable income is $85,030, not $83,630. It then submitted $12,064, which matches none of its own computations ($13,111 or $13,058), and in the second pass it swapped in wrong bracket thresholds ($12,525/$50,900) for the correct $12,400/$50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted $1,000 of auto loan interest, taking the $400 phaseout off the $1,400 of interest rather than off the $10,000 cap. The listed interest is not qualified passenger-vehicle loan interest, so no such deduction applies. Its brackets were correct, but taxable income of $84,030 instead of $85,030 understates tax by 22% x $1,000 = $220." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It added the nontaxable $700 of veterans benefits to AGI and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It taxed everything above the 10% band at 12%, ignoring the 22% bracket above $50,400. It then submitted $11,435, which does not even match its own $10,637.50 figure." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It derived the exact correct figure: $85,030 taxable income and $1,240 + $4,560 + $7,618.60 = $13,418.60. It then discarded that result and submitted $14,716 under the label 'rounding to bracket estimates', which no computation supports." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It computed the correct taxable income of $85,030 and the correct bracket tax of about $13,419. It then submitted $13,876, which none of its stated bracket arithmetic supports." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the full $1,400 of auto loan interest as an OBBBA qualified vehicle-loan deduction. It applied the $400 phaseout correctly against the $10,000 cap, but the listed interest is not established as qualified new, U.S.-assembled vehicle loan interest. Taxable income is therefore $85,030, not $83,630, and the extra deduction cut tax by 22% x $1,400 = $308." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It settled on a $15,750 standard deduction (the 2025 amount) instead of the 2026 amount of $16,100. It also used invented brackets (10% to $12,300, 12% to $49,950) in place of 10% to $12,400 and 12% to $50,400. That yields $85,380 of taxable income and $13,543 of tax, instead of $85,030 and $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used understated bracket thresholds of $12,150 and $49,400 instead of $12,400 and $50,400, producing $13,523.60. It then submitted an unsupported $13,322 'after adjusting for standard deduction indexing', although it had already applied the deduction." +us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It took a $1,174 auto-loan-interest deduction, reducing the $1,400 of interest by a phaseout. The listed interest is not qualified passenger-vehicle loan interest, so no deduction applies, and taxable income is $85,030 rather than $83,856. That understated tax by roughly 22% x $1,174." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,600 standard deduction instead of $16,100, and brackets of 10% to $12,175 and 12% to $49,550 instead of $12,400 and $50,400. That inflated taxable income to $85,530 and tax to $13,618." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted $3,840 of health premiums from wages as a pre-tax exclusion by double-counting the two $1,920 premium fields. The $103,000 is gross taxable wages and no premium exclusion applies. It also used a $15,400 standard deduction and $12,200/$49,700 brackets instead of $16,100 and $12,400/$50,400." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied an $8,300 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, no personal exemption and 10/12/22% brackets." +us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of the 2026 amount of $16,100, and brackets of 10% to $12,200 and 12% to $49,575 instead of $12,400 and $50,400. That produced $85,780 of taxable income and $13,670.10 of tax." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,300 standard deduction, a $5,300 personal exemption and 10/15/25% brackets instead of OBBBA's permanent $16,100 standard deduction and 10/12/22% brackets. It also wrongly subtracted $1,920 of premiums from the gross taxable wages." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It started from $103,000 of wages without subtracting the $1,870 capital loss, and it used a $15,300 standard deduction instead of $16,100. That gave $87,700 of taxable income instead of $85,030. Its $13,385 also does not follow from the correct 2026 brackets on its own taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It stated the correct AGI of $101,130 but never applied the $16,100 standard deduction or the 10%/$12,400, 12%/$50,400, 22% brackets. It submitted a round guess of $15,400 instead of computing $13,418.60 on $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and used an $8,250 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA permanent: the $16,100 standard deduction applies, there is no personal exemption, and 10/12/22% rates apply." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It cited only a vague standard deduction of about $15,000+ and never applied the 2026 figures ($16,100 deduction; 10% to $12,400, 12% to $50,400, then 22%). Its $11,102 corresponds to roughly $74,500 of taxable income, far below the correct $85,030." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired and applied an $8,300 standard deduction, a $5,300 personal exemption and outdated 10/15/25% brackets ($11,000/$44,700). OBBBA made the $16,100 standard deduction and the 10/12/22% brackets permanent." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and subtracted an $8,300 standard deduction plus a $5,300 personal exemption, then applied pre-TCJA rates. Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets apply." +us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket thresholds ($11,925/$48,475). The correct 2026 figures are a $16,100 deduction and $12,400/$50,400 thresholds, which give $85,030 of taxable income instead of $86,130." +us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $15,450 standard deduction and self-inflated brackets ($12,283/$49,929) instead of the 2026 figures of $16,100 and $12,400/$50,400. That overstated taxable income at $85,680 and gave tax of $13,611.04." +us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its brackets were correct, but it used a $15,800 standard deduction instead of the 2026 amount of $16,100. That overstated taxable income by $300 ($85,330 vs $85,030) and tax by 22% x $300 = $66." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated the tax-exempt $2,308 pension as taxable and described 'small taxable capital gains' where there is a $1,870 net loss that reduces AGI to $101,130. That inflated taxable income above $85,030 and produced $14,498." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It invoked itemized medical adjustments that do not apply: $2,520 of medical costs sit far below the 7.5%-of-AGI floor, and the $16,100 standard deduction exceeds itemized deductions. Its $9,587 corresponds to taxable income around $67,000, far below the correct $85,030." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the scheduled TCJA-sunset parameters (a small standard deduction plus a personal exemption, about $87,380 of taxable income, pre-TCJA rates). Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets give $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted a phased-down $1,000 qualified auto-loan-interest deduction, reaching $84,030 of taxable income. The listed auto loan interest is not qualified passenger-vehicle loan interest, so taxable income stays $85,030, and its answer is $220 (22% x $1,000) too low." +us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,000 auto-loan-interest deduction after phaseout. No qualifying new, U.S.-assembled vehicle loan is established, so no OBBBA vehicle-interest deduction applies. Taxable income is $85,030, not $84,030, and tax is $13,418.60." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 amount of $16,100, giving $85,830 of taxable income. It then applied unspecified 'projected' brackets that overshoot even that base: $13,774 versus $13,594.60 under the correct 10%/$12,400, 12%/$50,400, 22% schedule." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026 and used an $8,400 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and 2025 bracket thresholds ($11,925/$48,475) instead of the 2026 figures of $16,100 and $12,400/$50,400. That gave $86,130 of taxable income and $13,863 of tax." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used bracket thresholds of $12,200 and $49,575 instead of the 2026 thresholds of $12,400 and $50,400. Too much income fell in the 22% band, producing $13,505.10." +us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted TCJA law' with an $8,300 standard deduction, a $5,300 exemption and 10/15/25% brackets. Under OBBBA-permanent law the $16,100 standard deduction and 10/12/22% brackets apply to $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but reported $13,426 instead of the exact bracket result $1,240 + $4,560 + $7,618.60 = $13,418.60. The $7.40 overstatement comes from bracket thresholds slightly off the 2026 $12,400/$50,400 schedule." +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $13,418.60 of tax on $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,174 auto-loan-interest deduction, reducing the $1,400 of interest by 20% of AGI over $100,000. The listed interest is not qualified passenger-vehicle loan interest, so no deduction applies, and taxable income is $85,030 rather than $83,856." +us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Its own reasoning acknowledged positive tax on $103,000 of wages after the standard deduction, yet it submitted $0 by deferring to 'the numeric answer returned'. It never computed the $13,418.60 bracket tax on $85,030 of taxable income." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction instead of $16,100 and brackets of $12,150/$49,000 instead of $12,400/$50,400, which gave $13,641. It then submitted $14,827, which contradicts that computation." +us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It ignored the $1,870 capital-loss deduction and used a $13,600 standard deduction instead of $16,100. It taxed everything above $12,400 at 12%, ignoring the 22% bracket above $50,400. It then subtracted $4,824 of payroll tax as a nonrefundable credit, but no such credit exists." us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,386 of Social Security tax and $1,493.50 of Medicare tax, then submitted $8,149.50 despite its own arithmetic totaling $7,879.50. Its unexplained $270 addition is not produced by a wage-base cap, rounding, Additional Medicare Tax, or Texas employee payroll tax." us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated total employee payroll tax as $7,879.50 but submitted $7,854.00. The $25.50 reduction has no basis in any payroll-tax component stated in its reasoning." @@ -7467,147 +8206,120 @@ us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated ordinary withholding of employee FICA taxes as though it eliminated annual payroll-tax liability. W-2 wages of $103,000 generate $6,386 of employee Social Security tax and $1,493.50 of employee Medicare tax regardless of employer-sponsored insurance or Texas's lack of a state payroll tax." us,scenario_101,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly summed Social Security and Medicare taxes to $7,879.50, then added an unexplained $152. No Additional Medicare Tax or mandatory Texas employee payroll tax applies, so that extra amount has no payroll-tax source." us,scenario_101,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invented a Texas individual income tax liability by applying an unstated tax rate to taxable income. Texas imposes no individual state income tax, so the correct state income tax before refundable credits is $0, not $6,035." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Excluded the spouse's $7,989 employer-sponsored insurance premium from wages a second time as a cafeteria-plan exclusion, when the listed gross wages are already the taxable employment income. It then submitted $5,459, which matches neither its own $6,092 no-exclusion branch nor its $5,133 exclusion branch; the correct path ($32,200 standard deduction, $0.20 QBI deduction, 10% to $24,800 then 12%) gives $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"Subtracted an assumed $2,810 of wage withholding from its computed liability; withholding is a tax payment, not a nonrefundable credit, and never reduces income tax before refundable credits. It compounded this by using the 2024 MFJ standard deduction of $29,200 instead of the 2026 figure of $32,200, so its pre-credit figure of $5,620 was already wrong before the withholding subtraction." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a $31,500 standard deduction and a 10%-bracket ceiling of $24,000 instead of the 2026 values of $32,200 and $24,800, producing $6,204, and then submitted $5,764 — a figure its own bracket arithmetic never generates. The statutory parameters give taxable income of $55,000.80 and tax of $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own arithmetic — $32,300 standard deduction, 10% to $24,800, 12% above — produced $6,092, but it submitted $6,633, abandoning the computation it had just completed twice. With the statutory $32,200 deduction and the $0.20 QBI deduction the same method lands on $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Reproduced the reference computation essentially exactly — $87,201 gross, $32,200 MFJ standard deduction, $2,480 at 10% on the first $24,800 plus 12% on the remainder, tax '≈6,104' — and then submitted $5,983 instead of the $6,104.10 it had just derived. The submitted number follows from none of its stated deductions or rates." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Anchored on the pre-OBBBA $30,000 MFJ standard deduction and inflation-projected it to $30,800, missing the 2026 statutory $32,200, and set the 10% ceiling at $24,500 rather than $24,800. The $1,400 deduction shortfall taxed at 12% ($168) plus the $6 bracket effect accounts for the entire $174 overstatement; its medical-itemization screen and no-credits conclusion were correct." -us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Computed $6,056 using a $32,600 standard deduction with correct 10%/12% breakpoints, then inflated the submission to $6,800 for a claimed 'small SE tax adjustment,' which on $1 of self-employment income is a fraction of a cent and in fact produces a $0.20 QBI deduction that lowers tax. The statutory $32,200 deduction with its own method yields $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used a $30,600 standard deduction and a $24,000 top of the 10% bracket instead of the 2026 values of $32,200 and $24,800. The $1,600 of overstated taxable income at 12% plus the $800 of bracket shift at 2% produces the $208 excess over $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Applied 2025 parameters unchanged — a $30,000 standard deduction and a $23,850 top of the 10% bracket — instead of the 2026 MFJ standard deduction of $32,200 and $24,800 breakpoint, and omitted the $0.20 QBI deduction on the $1 of self-employment income. Its correct conclusion that no elderly/disabled credit applies did not offset the $283 deduction and bracket error." -us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA structure lapses for 2026, restoring a $16,600 standard deduction, $10,600 of personal exemptions, and a 15% second bracket. OBBBA made the enlarged standard deduction permanent ($32,200 MFJ in 2026), personal exemptions remain repealed, and the second bracket is 12%, so the correct taxable income is $55,000.80 rather than $60,001." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Stacked three errors: it subtracted the $7,989 employer-sponsored premium from wages that were already stated gross, applied a lapsed-TCJA $16,000 standard deduction with $10,300 of personal exemptions, and used a 15% second bracket. Under 2026 law the full $87,201 is reduced only by the $32,200 standard deduction and $0.20 QBI deduction, with 12% above $24,800." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Stated a $30,000 standard deduction and $57,201 of taxable income, which under the 2026 MFJ schedule yields about $6,387, but submitted $4,124 — a number its own stated inputs cannot produce under any bracket set. The statutory $32,200 deduction plus $0.20 QBI deduction gives $55,000.80 of taxable income and $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Built the calculation on a TCJA sunset — a $15,850 standard deduction plus $10,100 of personal exemptions and pre-TCJA rates — and additionally removed the $7,989 ESI premium from the stated gross wages. For 2026 the MFJ standard deduction is $32,200, personal exemptions are repealed, the second bracket is 12% above $24,800, and no ESI exclusion applies to the wage figures given." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Assumed post-TCJA-expiration law with a $16,000 standard deduction and a 15% bracket above $22,000, and subtracted the $7,989 ESI premium from the already-gross wages to reach a $79,212 AGI. The 2026 schedule applies a $32,200 standard deduction and 12% above $24,800 to the full $87,201, giving $6,104.10 rather than $8,381.80." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Asserted $0 with no derivation at all. Taxable income is $55,000.80 after the $32,200 standard deduction and $0.20 QBI deduction, which falls entirely in the 10% and 12% MFJ brackets for $6,104.10; a zero liability on $87,201 of wage income is consistent only with performing no bracket computation." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the $7,989 employer-sponsored insurance premium from wages that are already stated gross — an exclusion worth $958.68 of tax at the 12% rate — and then used a $30,800 standard deduction with the 2025 $23,850 bracket top instead of $32,200 and $24,800. The correct base is the full $87,201." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Computed the liability from 'standard deduction and personal exemptions under 2026 law,' i.e. a lapsed-TCJA structure with exemptions restored and a 15% second bracket. Personal exemptions remain repealed for 2026, the MFJ standard deduction is $32,200, and the top rate reached here is 12%, giving $6,104.10 on $55,000.80 of taxable income." -us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Used the 2025 MFJ standard deduction of $30,000 and the 2025 $23,850 top of the 10% bracket rather than the 2026 figures of $32,200 and $24,800, and omitted the $0.20 QBI deduction. That inflates taxable income by $2,200 and tax by exactly the $283 gap between $6,387.12 and $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Carried the computation to $6,398.12 using a $30,000 standard deduction and a $23,800 bracket top, correctly found no CTC or CDCC applies, and then submitted $1,737 — a value unrelated to its own arithmetic. With the 2026 $32,200 standard deduction and $0.20 QBI deduction the same method gives $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Gave no deduction figure, no brackets, and no arithmetic, asserting the household 'owes about $3,357.' Taxable income is $55,000.80 after the $32,200 standard deduction and $0.20 QBI deduction, and the 2026 MFJ rate schedule ($2,480 at 10% plus 12% on $30,200.80) yields $6,104.10; $3,357 corresponds to no 2026 deduction and rate combination." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Asserted zero liability without applying any rate schedule, reasoning only that no child, dependent-care, or other nonrefundable credits were specified. Absent credits, $87,201 of income less the $32,200 standard deduction and $0.20 QBI deduction leaves $55,000.80 taxed at 10% and 12% for $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Excluded the spouse's $7,989 employer-sponsored insurance premium from gross income before the standard deduction: $79,212 − $32,200 = $47,012, taxed at 10% to $24,800 plus 12% above, is precisely its $5,145.44. The listed gross wages are already taxable employment income, and that single exclusion removes exactly the $958.66 separating it from $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Executed the 2026 schedule correctly — $32,200 standard deduction, $0.20 QBI deduction, 10% to $24,800 then 12% — but ran it on $79,212 after subtracting the spouse's $7,989 ESI premium from wages already stated gross. That lone exclusion costs exactly $958.68 of tax at 12% and is the entire gap between $5,145.42 and $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Used a $30,000 standard deduction, implied by its '~$57,200' taxable income, instead of the 2026 MFJ figure of $32,200, and then reported a rounded $6,200 that its own taxable income does not produce (the 2026 brackets give $6,368 on $57,200). The statutory deduction leaves $55,000.80 and $6,104.10 of tax." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Projected the standard deduction as $30,840 by inflating the pre-OBBBA $30,000 base and put the 10% ceiling at $24,518, rather than the statutory 2026 values of $32,200 and $24,800. The $1,360 deduction shortfall at 12% plus the small bracket shift is the whole $169 overstatement." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Assumed TCJA expiration for 2026, using a $16,550 standard deduction, $10,600 of personal exemptions, and a 15% second bracket. OBBBA made the larger standard deduction permanent at $32,200 for MFJ in 2026 with personal exemptions still repealed and a 12% second bracket, so taxable income is $55,000.80, not $60,051." -us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the spouse's $7,989 ESI premium from the stated gross wages to reach a $79,212 AGI and then applied a $30,800 standard deduction with a $24,500 bracket top. The wages given are already the taxable employment income, and 2026 uses a $32,200 standard deduction with the 10% bracket ending at $24,800, so the base is $87,201 and the tax $6,104.10." -us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so the required key was absent from its outputs object. No substantive tax computation exists to evaluate." -us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Estimated the 2026 MFJ standard deduction at $30,750 and the 10% ceiling at $24,450 rather than the statutory $32,200 and $24,800; the $1,450 deduction shortfall at 12% plus the bracket shift produces essentially all of the $181 overstatement. Its medical-expense screen against the 7.5% AGI floor and its no-CTC/no-CDCC conclusion were correct." -us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Subtracted a $600 Child and Dependent Care Credit from its pre-credit figure although the household consists of two 26-year-old spouses with no qualifying individual and no care expenses, so the CDCC is $0. It also used a $33,500 standard deduction instead of $32,200, and even its own stated figures give $6,324 − $600 = $5,724, not the $3,602 submitted." -us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"Invented two qualifying children and a $4,000 Child Tax Credit for a household listing only two 26-year-old spouses and no dependents, so no CTC exists. Its stated pre-credit tax of $8,631 also does not follow from its own $57,201 taxable income under the 2026 MFJ schedule (which gives $6,387), and the statutory $32,200 deduction with no dependents yields $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed about $6,092 using a $32,300 standard deduction ($100 above the actual $32,200, with no QBI deduction), and about $5,133 after wrongly excluding the ESI premium. It then submitted $5,459, which matches neither calculation. The correct figure on $55,000.80 of taxable income is $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It used the 2024 standard deduction of $29,200 instead of 2026's $32,200. It then subtracted an invented $2,810 of 'withholding' from the tax. Withholding is a payment, not a nonrefundable credit, so it never reduces income tax before refundable credits." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $31,500 and a $24,000 10% bracket instead of 2026's $32,200 and $24,800, which gave $6,204. It then submitted $5,764, which does not follow from its own arithmetic." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It worked through the brackets to about $6,092, using a $32,300 standard deduction instead of $32,200 and no QBI deduction. It then submitted $6,633, which contradicts its own computation; the correct figure is $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It followed the correct derivation ($32,200 standard deduction, about $55,001 taxable income, $2,480 + $3,624 = $6,104) and then submitted a 'rounded estimate' of $5,983. That unexplained $121 reduction has no basis in any credit or deduction." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It inflated the pre-OBBBA 2025 figures ($30,000 standard deduction, $23,850 10% bracket) by 2.7% to $30,800 and $24,500. It missed OBBBA raising the 2026 amounts to $32,200 and $24,800, which overstated taxable income by $1,400." +us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It used a $32,600 standard deduction instead of $32,200 and reached $6,056. It then added an invented 'SE tax adjustment' to get $6,800. Self-employment tax is not part of income tax, and $1 of self-employment income is below the $400 threshold, so there is no SE tax at all." +us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $30,600 standard deduction and a $24,000 10% bracket instead of 2026's OBBBA amounts of $32,200 and $24,800. That overstated taxable income by $1,600 and gave $6,312." +us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $30,000 and the 2025 10% bracket top of $23,850, not 2026's $32,200 and $24,800. That overstated taxable income by $2,200 and tax by $283." +us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied a $16,600 standard deduction, $10,600 of personal exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction, no personal exemptions, and 10%/12% brackets." +us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,500 10% bracket, which are inflation-projected pre-OBBBA values. The 2026 amounts are $32,200 and $24,800, so it overstated taxable income by $1,400." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset ($16,000 standard deduction, $10,300 of exemptions, a 15% bracket) even though OBBBA made the $32,200 standard deduction and 10%/12% brackets permanent. It also subtracted the $7,989 ESI premium from AGI even though the stated gross wages are fully taxable." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the pre-OBBBA $30,000 standard deduction instead of $32,200. It then reported $4,124 on $57,201 of taxable income, which is far below what even its own inputs give under the 10%/12% brackets (about $6,387)." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied the pre-TCJA 2026 structure ($15,850 standard deduction plus $10,100 of personal exemptions, 15% bracket) that OBBBA repealed. It also subtracted the $7,989 ESI premium from AGI, which wrongly reduced taxable wages." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used post-sunset brackets (10% to $22,000, then 15%) and a $16,000 standard deduction. OBBBA's permanent 2026 schedule is a $32,200 standard deduction with 10% to $24,800 and 12% above. It also wrongly subtracted the $7,989 ESI premium from AGI." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It reported $0 with no derivation. After the $32,200 standard deduction, the couple has $55,000.80 of taxable income and no nonrefundable credits to offset the resulting $6,104.10 of tax." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium as a pre-tax exclusion, lowering AGI to $79,212, although the stated gross wages are fully taxable. It also used a $30,800 standard deduction and a $23,850 10% bracket instead of 2026's $32,200 and $24,800." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied personal exemptions and the post-sunset rate schedule to 2026. OBBBA made the TCJA's zero-exemption, $32,200 standard deduction and 10%/12% bracket structure permanent, so its $7,915.15 overstates tax." +us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It applied the pre-OBBBA 2025 standard deduction of $30,000 and the 2025 10% bracket top of $23,850 instead of 2026's $32,200 and $24,800. That overstated tax at $6,387.12." +us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It computed $6,398.12 using the outdated $30,000 standard deduction and a $23,800 10% bracket. It then submitted $1,737, which contradicts its own calculation and is backed by no credit." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It said no nonrefundable credits apply, yet reported $3,357, roughly half the correct liability. Applying the $32,200 standard deduction and the 10%/12% brackets to $87,201 of income gives $6,104.10, so its figure understates taxable income or bracket tax with no supporting rule." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It reported zero tax on $87,200 of wages while listing no credits. The $32,200 standard deduction shelters only part of the income and leaves $55,000.80 taxable, which produces $6,104.10 of tax." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It treated the spouse's $7,989 ESI premium as a pre-tax salary reduction and cut AGI to $79,212. The prompt lists gross wages and no cafeteria-plan election, so wages stay fully taxable. That exclusion alone lowered tax by $958.68 to $5,145.44." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It excluded the $7,989 ESI premium from wages as if it were a pre-tax cafeteria-plan deduction, although the stated gross wages are fully taxable. It also claimed a half-SE-tax deduction even though $1 of self-employment income is below the $400 threshold." +us,scenario_102,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It got the $32,200 standard deduction and $0.20 QBI deduction right, but subtracted the $7,989 ESI premium as a pretax employer-plan exclusion and reduced AGI to $79,212. The stated gross wages are fully taxable, so taxable income is $55,000.80 and tax is $6,104.10." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its taxable income of about $57,200 implies a pre-OBBBA $30,000 standard deduction instead of 2026's $32,200. The $6,200 is a rounded figure not tied to the 2026 brackets (10% to $24,800, 12% above)." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected the pre-OBBBA 2025 values forward to a $30,840 standard deduction and a $24,518 10% bracket. The actual 2026 figures are $32,200 and $24,800, so it overstated taxable income by about $1,360." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired, with a $16,550 standard deduction, $10,600 of personal exemptions and a 15% bracket. OBBBA made the TCJA structure permanent, so 2026 uses a $32,200 standard deduction and 10%/12% brackets with no exemptions." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,500 10% bracket, which are projected pre-OBBBA values. OBBBA set 2026 at $32,200 and $24,800, so it overstated taxable income by $1,400 and tax by $174." +us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $7,989 ESI premium from wages to reach a $79,212 AGI, although the stated gross wages are fully taxable. It also used a $30,800 standard deduction instead of 2026's $32,200." +us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for this variable, so nothing substantive can be scored." +us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used an estimated $30,750 standard deduction and a $24,450 10% bracket instead of OBBBA's 2026 amounts of $32,200 and $24,800. That overstated taxable income by $1,450." +us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed a Child and Dependent Care Credit, but the household has no qualifying dependent and no care expenses, so the CDCC is zero. It also used an inflated $33,500 standard deduction instead of $32,200. Its final $3,602 does not even equal its own $6,324 minus $600." +us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It invented two qualifying children and subtracted a $4,000 Child Tax Credit, but the household is a childless couple with no CTC. It also used the outdated $30,000 standard deduction and overstated pre-credit tax at $8,631." us,scenario_102,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a $3,995 refundable disability-related credit even after correctly ruling out EITC, refundable CTC, and the American Opportunity Credit. The Credit for the Elderly or the Disabled is nonrefundable, disability alone does not establish it for these working 26-year-olds, and no listed fact generates any refundable component." us,scenario_102,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract." us,scenario_102,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented a medically needy spend-down pathway in the PolicyEngine determination and improperly treated rent as a Medicaid spend-down expense. The head qualifies for no Medicaid category, and the household's $87,201 MAGI exceeds every applicable income threshold, so the listed expenses and assets do not change the result." us,scenario_102,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model incorrectly declared $41,201 within North Carolina's income limit for a disabled adult and evaluated only the head's earnings instead of the applicable household income. The trace uses household MAGI of $87,201, equal to 4.03 times FPL, and assigns no Medicaid eligibility category." -us,scenario_102,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model invoked the under-65 disability route to Medicare from the bare `is_disabled` flag, skipping its actual prerequisite of 24 months of SSDI/RRB disability entitlement (or ESRD/ALS), none of which is listed and all of which the prompt directs be treated as absent. It also ignored that Head's $41,200 in wages exceeds the substantial-gainful-activity level that would permit SSDI entitlement in the first place. PolicyEngine's medicare_eligible turns solely on age ≥ 65, and Head is 26, so the answer is 0." -us,scenario_102,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model asserted a PolicyEngine rule that does not exist — that disabled individuals are Medicare eligible regardless of age — when medicare_eligible is computed from the age ≥ 65 threshold alone and takes no disability input. Head's age of 26 fails that single test, and no SSDI entitlement, ESRD, or ALS fact is present that would trigger the statutory under-65 pathway either, so the answer is 0." -us,scenario_102,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model attributed a disability-based Medicare eligibility rule to PolicyEngine; PolicyEngine's medicare_eligible depends only on whether age is at least 65, and Head is 26. The disability flag drives SSI and disability-related deductions in the model, not Medicare, and the statutory under-65 route would still require 24 months of SSDI entitlement that Head's $41,200 in wages precludes, so the answer is 0." -us,scenario_102,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied a nonexistent 'PolicyEngine disability-based Medicare eligibility rule' and stopped there, never testing the one criterion the variable actually uses — age ≥ 65 — which Head fails at 26. Nothing in the household establishes the 24-month SSDI entitlement, ESRD, or ALS that the real under-65 pathway requires, so the answer is 0." -us,scenario_102,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as a disjunction, 'age 65 or disabled,' and satisfied it through the second branch; PolicyEngine's medicare_eligible has no disability branch at all and returns True only for age ≥ 65. Head is 26 with $41,200 in wages — above the substantial-gainful-activity level that gates the SSDI entitlement underpinning the statutory under-65 route — so the answer is 0." -us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model repeatedly used 0.145 instead of 0.0145 for Medicare, incorrectly considered subtracting unlisted pretax premium contributions from FICA wages, and then submitted $7,149.68 without deriving it. Applying 6.2% and 1.45% to the full $87,200 of wages yields $6,670.80." -us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly computed $3,151.80 plus $3,519.00 as $6,670.80 but submitted $6,672.30. Its output is a $1.50 transcription or arithmetic error that contradicts its own derivation." -us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly derived Social Security tax of $5,406.40 and Medicare tax of $1,264.40 but rounded their exact $6,670.80 sum to $6,672.00. Payroll tax is an exact annual amount here, so that unsupported approximation caused the error." -us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model listed all four correct person-level components and explicitly summed them to $6,670.80, then submitted $6,664.60. The submitted value is inconsistent with its own correct arithmetic by $6.20." -us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the spouse’s $7,989 employer-sponsored insurance premium from FICA-taxable wages even though the prompt does not state that this amount is an employee pretax salary reduction. Payroll tax applies to the full listed wages of $87,200, producing $6,670.80 rather than $6,059.64." -us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced the spouse’s payroll-tax wage base from $46,000 to $38,011 by treating the listed employer-sponsored insurance premium as a pretax employee contribution. With no such employee deduction specified, both spouses’ full wages are subject to 7.65% FICA, totaling $6,670.80." -us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly deducted $7,989 of employer-sponsored insurance premiums from the spouse’s FICA wages. The stated gross wages remain the payroll-tax base, so 7.65% applies to $87,200 and yields $6,670.80." -us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the $7,989 employer-sponsored insurance premium as an employee pretax payroll deduction and taxed only $38,011 of the spouse’s $46,000 wages. The full listed wages are subject to employee Social Security and Medicare taxes, yielding $6,670.80." -us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"The model correctly calculated $5,406.40 of Social Security tax and $1,264.40 of Medicare tax and correctly stated their sum as $6,670.80, but submitted $6,667.80. The final value is a $3 transcription error." -us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct formula, but its multiplication is wrong: $87,200 × 0.0765 equals $6,670.80, not $7,063.00. No other payroll-tax component accounts for the difference." -us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model reduced wage compensation by the spouse’s $7,989 employer-sponsored insurance premium without a stated employee pretax contribution. Using the full $87,200 wage base gives $5,406.40 of Social Security tax and $1,264.40 of Medicare tax, totaling $6,670.80." -us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model netted the spouse’s $7,989 employer-sponsored insurance premium against wages for FICA purposes. Because the prompt supplies no employee pretax salary reduction, payroll taxes apply to the full $46,000 of spouse wages and total $6,670.80 for the household." -us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly removed the $7,989 employer-sponsored insurance premium from the spouse’s FICA wage base and then rounded its result. Applying 7.65% to the full $87,200 of listed wages yields exactly $6,670.80." -us,scenario_102,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required output contract." -us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $5,406.40 of Social Security tax and $1,264.40 of Medicare tax, whose exact sum is $6,670.80, but replaced that result with an unrelated $6,539.00 submission. The claimed rounding cannot transform $6,670.80 into $6,539.00." -us,scenario_102,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model truncated total wages from $87,200 to $87,000 when calculating both Social Security and Medicare taxes, then submitted $6,681.00 even though its stated rounded components sum to $6,656.00. Using the exact wage inputs produces $6,670.80." +us,scenario_102,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model said disabled people can qualify for Medicare and marked the 26-year-old head eligible. It skipped the rule that disability-based Medicare starts only after 24 months of SSDI entitlement. The head has no SSDI, and $41,200 in wages rules SSDI out. At age 26, the head fails the age-65 test, so the answer is 0." +us,scenario_102,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model claimed PolicyEngine treats disabled people as Medicare eligible, which turned the disability flag into automatic eligibility. Under-65 Medicare requires 24 months of SSDI receipt (or ESRD/ALS). The head has zero SSDI and earns $41,200 in wages, so only the age-65 test applies, and a 26-year-old fails it." +us,scenario_102,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model saw the head was under 65 but still granted eligibility because of the disability flag. It ignored that the disability route requires 24 months of SSDI entitlement. The head has no SSDI and earns $41,200, well above substantial gainful activity, so the head is not Medicare eligible." +us,scenario_102,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model assumed the listed disability makes the head eligible under PolicyEngine's rules despite being under 65. Disability-based Medicare requires 24 months of SSDI receipt, and the head has zero SSDI plus $41,200 in wages. With no SSDI, the age-65 test is the only pathway, and a 26-year-old fails it." +us,scenario_102,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model's disability-based Medicare rule was triggered by the disability flag alone. That skips the 24-month SSDI entitlement requirement. The head has no SSDI and earns $41,200 in wages, so the disability pathway is closed, and at 26 the head also fails the age-65 test." +us,scenario_102,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model's modeled disability pathway was triggered just by the head's disability flag. That pathway only applies after 24 months of SSDI entitlement. The head has zero SSDI and $41,200 in wages, so the only test left is age 65 or older, which a 26-year-old fails." +us,scenario_102,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as 'at least 65 or disabled', but PolicyEngine's actual rule is age 65 or older, or 24 months of SSDI receipt (or ESRD/ALS). The head has no SSDI, and $41,200 in wages is far above substantial gainful activity. The disability flag alone does not qualify a 26-year-old, so the answer is 0." +us,scenario_102,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used 14.5% instead of the 1.45% employee Medicare rate, which gave $12,644 of Medicare tax. It then switched between subtracting the $8,389 of premiums and not subtracting them, and submitted $7,149.68, a figure that none of its own calculations produce. The correct figure is 7.65% of the full $87,200 of gross wages, which is $6,670.80." +us,scenario_102,payroll_tax,claude-opus-4.8,llm_error,other,False,"It correctly calculated $3,151.80 + $3,519.00 = $6,670.80 at 7.65% of each spouse's gross wages. It then submitted $6,672.30, a transcription error that doesn't match its own total." +us,scenario_102,payroll_tax,claude-opus-5,llm_error,other,False,"It correctly found Social Security tax of $5,406.40 and Medicare tax of $1,264.40, which add to $6,670.80. It then rounded to 'about $6,671' and submitted $6,672, which is neither the exact sum nor a correct rounding of it." +us,scenario_102,payroll_tax,claude-sonnet-5,llm_error,other,False,"It correctly broke out both spouses' taxes ($2,554.40 + $597.40 + $2,852.00 + $667.00) and stated the total as $6,670.80. It then submitted $6,664.60, a transcription error that contradicts its own sum." +us,scenario_102,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It treated the spouse's $7,989 employer-sponsored insurance premium as a pre-tax salary reduction and used $79,211 as the FICA base. FICA applies to the full listed gross wages of $87,200, which gives $6,670.80." +us,scenario_102,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It cut the spouse's FICA wages to $38,011 by subtracting the $7,989 employer-sponsored insurance premium. The spouse's full $46,000 of gross wages is subject to FICA, which gives $3,519.00 for the spouse and $6,670.80 for the household." +us,scenario_102,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $7,989 employer-sponsored insurance premium as pre-tax and used $79,211 of FICA-taxable wages. The listed gross wages of $41,200 and $46,000 are the payroll tax base with no premium exclusion, so 7.65% × $87,200 = $6,670.80." +us,scenario_102,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It subtracted the $7,989 employer-sponsored insurance premium from the spouse's wages to get a $38,011 FICA base. The employee Social Security and Medicare taxes apply to the spouse's full $46,000, which gives $6,670.80 for the household." +us,scenario_102,payroll_tax,glm-5.2,llm_error,other,False,"It correctly calculated $5,406.40 + $1,264.40 = $6,670.80 on $87,200 of wages. It then submitted $6,667.80, a digit transcription error." +us,scenario_102,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It used the correct base and rate, (41,200 + 46,000) × 0.0765, but got the multiplication wrong and reported $7,063. The product is $6,670.80." +us,scenario_102,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It calculated FICA on $79,211 after subtracting the spouse's $7,989 employer-sponsored premium as if it were pre-tax. FICA applies to the full $87,200 of gross wages, which gives $6,670.80." +us,scenario_102,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It reduced the spouse's wages by the $7,989 employer-sponsored premium, treating it as a pre-tax salary reduction. Employee FICA applies to the spouse's full $46,000 of gross wages, which gives $6,670.80 for the household." +us,scenario_102,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It applied 7.65% to $79,211 after subtracting the $7,989 employer-plan premium as a pre-tax deduction. The payroll tax base is the full $87,200 of listed gross wages, which gives $6,670.80." +us,scenario_102,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It subtracted the $7,989 employer-sponsored insurance premium from the spouse's wages, used a $79,211 FICA base, and rounded the result to $6,060. The spouse's full $46,000 is FICA wages, so the household total is 7.65% × $87,200 = $6,670.80." +us,scenario_102,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score. The correct result is 7.65% of the $87,200 of combined gross wages, which is $6,670.80." +us,scenario_102,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly found Social Security tax of $5,406.40 and Medicare tax of $1,264.40, a total of about $6,671. It then submitted $6,539, a number its own arithmetic does not support." +us,scenario_102,payroll_tax,qwen3.8-max,llm_error,other,False,"It used $87,000 of total wages instead of $41,200 + $46,000 = $87,200. It then added its own $5,394 and $1,262 wrongly and submitted $6,681 instead of their $6,656 sum. The correct total on $87,200 is $6,670.80." us,scenario_102,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a $31 minimum or rounding treatment after acknowledging that $1 of self-employment income produces approximately zero tax. Net earnings at this level do not reach the self-employment-tax filing threshold, so the liability is $0." us,scenario_102,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested self_employment_tax output entirely. The listed $1 of self-employment income is below the threshold for self-employment tax, yielding $0." -us,scenario_102,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model asserted that ""in PolicyEngine's simplified rule, individuals who are disabled qualify for Medicare regardless of age,"" inventing a disability override that PolicyEngine's Medicare variable does not contain — that variable tests age 65 or older, and the disability flag routes only to SSI, disabled-household SNAP treatment, and medical-expense deductions. It also skipped the statutory under-65 pathway's actual gate, 24 months of entitlement to Social Security disability benefits (or ESRD/ALS), which the household cannot satisfy because it reports no Social Security disability income and unlisted inputs are zero." -us,scenario_102,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model claimed ""PolicyEngine treats disabled individuals as Medicare eligible even though Spouse is under age 65,"" substituting a fabricated disability rule for PolicyEngine's actual age-65 threshold. Disability alone never confers Medicare: entitlement under 65 requires 24 months of Social Security disability benefit receipt or an ESRD/ALS diagnosis, and with the Spouse at age 26 and zero Social Security disability income the age test is the only test in play and it fails." -us,scenario_102,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model wrote that ""Under the PolicyEngine eligibility rule, Spouse's listed disability makes Spouse eligible for Medicare despite being under age 65,"" treating the disability flag as an exception that overrides the age condition when PolicyEngine's Medicare variable is the age-65-or-older test with no disability branch. The real under-65 route requires 24 months of Social Security disability entitlement or ESRD/ALS, neither of which is present in a household with no Social Security disability income." -us,scenario_102,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model invoked ""PolicyEngine's disability-based Medicare eligibility rule,"" a rule that does not exist — PolicyEngine determines Medicare eligibility solely by whether the person is at least 65, and the Spouse is 26. It never checked the condition that actually opens Medicare before 65, namely 24 months of Social Security disability benefit entitlement or an ESRD/ALS diagnosis, both absent from a household whose unlisted disability-benefit inputs are zero." -us,scenario_102,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as a disjunction — ""eligible for Medicare if they are at least age 65 or are disabled"" — and satisfied it through the disability leg; PolicyEngine's Medicare variable has no disability leg, only the age-65 threshold, which a 26-year-old fails. Adding disability as an independent qualifier collapses the statutory requirement of 24 months of Social Security disability entitlement (or ESRD/ALS) into the bare impairment flag, and this household reports no Social Security disability benefits." -us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed the correct chain in its reasoning — $87,201 AGI less the $25,500 MFJ standard deduction, times NC's 3.99% 2026 rate, equals $2,462 — then discarded it over an invented revenue-trigger contingency and submitted $2,755, a figure it rationalized as 4.25% on a $64,830 base it never derived from any household fact. NC's 3.99% rate applies for 2026 unconditionally, and the $25,500 deduction it had already used is correct, so its abandoned computation was the answer." -us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It asserted that North Carolina has no state income tax for 2026, confusing NC with a no-income-tax state. NC levies a flat individual income tax on federal AGI less the state standard deduction, so $61,701 of NC taxable income produces $2,461.87 at the 2026 rate of 3.99%." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 flat rate of 4.25% instead of the 3.99% rate in effect for 2026, producing $2,622 on the correct $61,701 base, then reported $2,587 as a 'rounded estimate' with no stated basis. Its AGI of $87,201 and $25,500 MFJ standard deduction were both right; the stale rate is the entire substantive error." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning twice derived the correct result — $61,701 × 3.99% = $2,462 — and it then submitted $3,389, which no step in its own work produces. The submitted number implies a 5.49% effective rate on the taxable income it had already computed, while NC's 2026 flat rate is 3.99%." -us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stated the correct derivation (AGI $87,201, $25,500 MFJ standard deduction, $61,701 taxable, ~$2,462 at 3.99%) and then overrode it with $2,792 attributed to unnamed 'PolicyEngine NC parameters.' That $2,792 is exactly 4.25% of $65,701 — the pre-2026 rate applied over the stale $21,500 deduction — not the 2026 parameters it had already named correctly." -us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used two stale parameters simultaneously: the 4.25% rate rather than 2026's 3.99%, and a $21,500 MFJ standard deduction rather than $25,500, yielding $65,701 × 4.25% = $2,792.29. Its AGI treatment was correct — it rightly kept the $7,989 ESI premiums out of the subtraction — so replacing both stale values gives $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It built the correct $61,701 taxable base but applied 4.25% instead of 2026's 3.99% to get $2,622.29, then abandoned even that and submitted $3,846 on the strength of unspecified adjustments for 'NC's treatment of self-employment.' The $1 of self-employment income shifts NC tax by four cents, and NC grants no personal exemption that changes the base further." -us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It substituted a $30,000 federal-style MFJ standard deduction for North Carolina's own $25,500 statutory deduction and used a 4.0% rate instead of the 3.99% flat rate, giving $57,201 × 4% = $2,288. NC's deduction is a fixed state amount and does not track the federal figure." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used the correct 3.99% rate and $25,500 deduction but started from a $79,212 AGI, which is $87,201 minus the spouse's $7,989 employer-sponsored insurance premiums. The stated gross wages are the tax-reportable amount and any cafeteria-plan exclusion is already reflected in them, so NC's base is $87,201 and the tax is $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It applied the stale 4.25% rate and, despite claiming a standard deduction, submitted $3,717 — 4.25% of roughly $87,459, essentially undeducted AGI. The 2026 computation removes the $25,500 MFJ standard deduction first and applies 3.99%, giving $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its rate (3.99%) and deduction ($25,500) were correct, but it set AGI at $79,212 — the $87,201 total less the spouse's $7,989 ESI premiums. Employer-sponsored premiums are not a subtraction from stated gross wages for NC purposes, so the taxable base is $61,701, not $53,712." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It named the correct 3.99% rate and $25,500 MFJ standard deduction but used a $79,212 federal AGI, which is $87,201 with the spouse's $7,989 ESI premiums removed. That subtraction double-counts a pre-tax exclusion already embedded in the stated wages, understating taxable income by $7,989 and the tax by $318.76." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It applied 4.25% rather than 2026's 3.99%, and applied it to $79,700 — its $3,387.25 is exactly 4.25% of that — a base from which no $25,500 standard deduction was removed. The correct base is $87,201 − $25,500 = $61,701." -us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the correct 3.99% rate and $25,500 MFJ deduction but began from a $79,212 NC AGI, which is the household's $87,201 less the spouse's $7,989 employer-sponsored insurance premiums. NC starts from federal AGI, in which the stated gross wages already reflect any pre-tax premium exclusion, so nothing further comes out." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no derivation, and its $2,551 is 4.135% of the correct $61,701 base — a rate sitting between the repealed 4.25% and the enacted 3.99%, so it never committed to NC's 2026 statutory flat rate. Applying 3.99% to $87,201 less the $25,500 MFJ standard deduction gives $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It concluded that without itemized deductions or credits no positive NC liability arises, inverting the state's structure: NC imposes a flat 3.99% tax on federal AGI less the $25,500 MFJ standard deduction, with no further zero-bracket. Nothing in the household facts — the $1 of self-employment income, the medical expenses, the ESI premiums — eliminates a liability on $61,701 of taxable income." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the correct 3.99% rate and $25,500 MFJ standard deduction to a $79,212 state income figure, which is $87,201 with the spouse's $7,989 ESI premiums stripped out. That premium amount is not deductible from the stated gross wages for NC, so taxable income is $61,701 and the tax is $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It explicitly applied a 'pre-tax insurance exclusion,' subtracting the spouse's $7,989 employer-sponsored premiums to reach $79,212 before the $25,500 deduction. The stated $46,000 gross wage is already the post-exclusion tax-reportable figure, so applying the exclusion again removed $7,989 from the NC base and $318.77 from the tax; its rate (3.99%), deduction, and half-SE-tax treatment were otherwise right." -us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no derivation, and its $2,800 is 4.25% of $65,701 — AGI less the stale $21,500 standard deduction — rounded to the nearest hundred. Both inputs are pre-2026: the 2026 flat rate is 3.99% and the MFJ standard deduction is $25,500, giving $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It invented a $26,400 'projected' 2026 MFJ standard deduction and a 4% rate. NC's MFJ standard deduction is a fixed statutory $25,500 for 2026 and is not inflation-indexed, and the flat rate is 3.99%, so the base is $61,701 rather than $60,801 and the tax is $2,461.87 rather than $2,432." -us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It stated that no standard deduction or subtraction applies and applied 4% to a $79,212 base, compounding two errors: removing the spouse's $7,989 ESI premiums from stated gross wages, and denying the $25,500 NC MFJ standard deduction that NC allows against federal AGI. The correct computation is $87,201 − $25,500 = $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so no substantive computation was submitted for scoring. The required chain is $87,201 federal AGI less NC's $25,500 MFJ standard deduction, times the 2026 flat rate of 3.99%, equal to $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 4.25% rate along with a fabricated ~$23,400 MFJ standard deduction; the 2026 statutory values are 3.99% and $25,500. Both errors push upward, giving $63,801 × 4.25% = $2,712 instead of $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a 4.5% rate — two rate reductions stale — and settled on a fabricated $23,850 MFJ standard deduction after first trying $21,500; the 2026 values are 3.99% and $25,500. Its medical-expense itemization detour changed nothing because it correctly concluded the standard deduction dominates, so the stale rate and invented deduction are the whole error: $63,351 × 4.5% = $2,850.80 instead of $61,701 × 3.99% = $2,461.87." -us,scenario_102,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It subtracted a $30,000 federal standard deduction instead of North Carolina's own $25,500 and applied an invented 4.49% rate rather than the 2026 flat 3.99%. That produced $57,201 × 4.49% ≈ $2,568, reported as $2,567, against the correct $61,701 × 3.99% = $2,461.87." +us,scenario_102,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model said PolicyEngine grants Medicare to any disabled person regardless of age. In fact, the disability pathway under 65 requires 24 months of SSDI entitlement (or ESRD/ALS), and the 26-year-old spouse receives no SSDI. With the age-65 test failed and no SSDI history, the spouse is not eligible, so the answer should have been 0." +us,scenario_102,spouse_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model treated the 'is disabled' flag as enough for Medicare. Under-65 eligibility actually requires 24 months of SSDI receipt (or ESRD/ALS), and the spouse has $0 SSDI. The 26-year-old spouse fails both the age-65 test and the SSDI-duration test, so the answer should have been 0." +us,scenario_102,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model stated that PolicyEngine treats disabled people as Medicare eligible even under 65. It skipped the rule that disability-based Medicare requires 24 months of SSDI entitlement. The spouse has no SSDI income and is 26, so neither the age pathway nor the SSDI pathway applies and eligibility is 0." +us,scenario_102,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model let the spouse's listed disability override the age-65 requirement. The actual under-65 pathway requires 24 months of SSDI benefits (or ESRD/ALS), and the household reports $0 SSDI. The spouse is therefore ineligible at age 26." +us,scenario_102,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model used a disability-based Medicare rule keyed only to the disability flag. That pathway requires 24 months of SSDI entitlement, and the spouse receives no SSDI. At age 26 with no SSDI history and no ESRD/ALS, the spouse fails every Medicare pathway, so the answer should have been 0." +us,scenario_102,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model called on the 'modeled disability-based Medicare pathway' but treated the disability flag alone as satisfying it. That pathway turns on 24 months of SSDI receipt, and the spouse's SSDI is $0. The 26-year-old spouse is therefore not Medicare eligible." +us,scenario_102,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model stated the rule as 'age 65 or disabled,' which drops the requirement that the disability pathway runs through 24 months of SSDI entitlement (or ESRD/ALS). The spouse is 26 and receives no SSDI, so neither condition is met and eligibility is 0." +us,scenario_102,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reached the correct $61,701 × 3.99% ≈ $2,462. It then wrongly claimed the 2026 rate of 3.99% depends on revenue triggers, switched to 4.25%, and applied that to an invented $64,830 base to get $2,755. The 3.99% rate is set by statute for 2026, and the base is $87,201 minus the $25,500 NC joint standard deduction." +us,scenario_102,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed North Carolina has no state income tax. In fact NC levies a 3.99% flat individual income tax in 2026. Applied to $61,701 of NC taxable income ($87,201 AGI minus the $25,500 joint standard deduction), that gives $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It got the base right ($87,201 − $25,500 = $61,701) but used the 2025 rate of 4.25% instead of the 2026 statutory rate of 3.99%. It then submitted $2,587, which does not even match its own $2,622 result. The correct figure is $61,701 × 3.99% = $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its reasoning twice arrives at the correct $61,701 × 3.99% ≈ $2,462, but it submitted $3,389, which no step of its derivation produces. That number amounts to taxing about $79,700 at 4.25%, which drops most of the $25,500 joint standard deduction and uses the superseded 2025 rate." +us,scenario_102,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It computed the correct ~$2,462 from 3.99% of $61,700, then replaced it with $2,792. That number equals $65,701 × 4.25%, which means it used the outdated $21,500 joint standard deduction and the 2025 rate instead of NC's $25,500 deduction and the 3.99% rate for 2026." +us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $21,500 as the NC married-filing-jointly standard deduction, but that deduction has been $25,500 since 2022. It also used a 4.25% rate instead of the 3.99% statutory rate for 2026, so it taxed $65,701 at 4.25% rather than $61,701 at 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the correct $61,701 taxable income but applied the 2025 rate of 4.25% (giving $2,622.29) instead of 3.99%. It then inflated the result to $3,846 with an unexplained 'adjustment' that has no basis in NC law." +us,scenario_102,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It subtracted a $30,000 standard deduction, which is the federal joint figure, instead of NC's $25,500. It also taxed at 4.0% instead of the 3.99% rate for 2026, so it computed $57,201 × 4% = $2,288 rather than $61,701 × 3.99% = $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The rate (3.99%) and joint standard deduction ($25,500) were right. The error was using $79,212 as AGI: it subtracted the spouse's $7,989 employer-sponsored insurance premium from wages as a pre-tax exclusion. The listed gross wages are the wage base, so federal AGI, NC's starting point, is $87,201 and taxable income is $61,701." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"$3,717 is about 4.25% of the full ~$87,000 of gross income, which means it both skipped the $25,500 NC joint standard deduction and used the 2025 rate. The correct calculation taxes $61,701 at the 3.99% rate for 2026." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from $79,212 of AGI, treating the spouse's $7,989 employer-sponsored insurance premium as excluded from wages. Federal AGI is $87,201 (all listed wages plus the $1 of self-employment income), so NC taxable income is $61,701, not $53,712, and 3.99% of it is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used $79,212 as federal AGI by subtracting the spouse's $7,989 employer-sponsored insurance premium from gross wages. No such exclusion applies to the listed wages, so AGI is $87,201. After the $25,500 deduction, $61,701 is taxed at 3.99%, which is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It applied the 2025 rate of 4.25%, and its $3,387.25 implies a taxable base of $79,700. That is far above the correct $61,701 ($87,201 AGI minus the $25,500 NC joint standard deduction), which is taxed at 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It began from $79,212 of AGI, treating the spouse's $7,989 employer-sponsored insurance premium as a pre-tax reduction of wages. AGI is actually $87,201, so taxable income after the $25,500 joint standard deduction is $61,701, and 3.99% of it is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave $2,551 with no derivation, about $89 more than the correct $61,701 × 3.99% = $2,461.87. That means it did not apply NC's 3.99% flat rate for 2026 to AGI of $87,201 minus the $25,500 joint standard deduction." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It said this couple has no positive NC liability. But $87,201 of AGI minus the $25,500 joint standard deduction leaves $61,701 of NC taxable income, taxed at the 3.99% flat rate, and no NC nonrefundable credit offsets the resulting $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used $79,212 as NC starting income by excluding the spouse's $7,989 employer-sponsored insurance premium from wages. Federal AGI, NC's starting point, is the full $87,201, so taxable income is $61,701 and the tax is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It applied a 'pre-tax insurance exclusion' that took the spouse's $7,989 employer-sponsored insurance premium out of wages, cutting AGI to about $79,212. The listed gross wages go into AGI in full ($87,201), so NC taxable income is $61,701, and 3.99% of it is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It began from $79,212 of AGI, excluding the spouse's $7,989 employer-sponsored insurance premium from wages. Federal AGI is $87,201, so NC taxable income after the $25,500 joint standard deduction is $61,701, not $53,712, and the 3.99% tax is $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its ~$2,800 matches 4.25% of $65,701, meaning it used the outdated $21,500 joint standard deduction and the 2025 rate. The correct calculation uses NC's $25,500 deduction and the 3.99% rate for 2026: $61,701 × 3.99% = $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected a $26,400 NC joint standard deduction, but the statutory amount is fixed at $25,500. It also used a 4% rate instead of 3.99%, so it computed $60,801 × 4% = $2,432 rather than $61,701 × 3.99% = $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It said no standard deduction applies, but NC allows a $25,500 standard deduction for joint filers. It also cut AGI to $79,212 by excluding the spouse's $7,989 employer-sponsored insurance premium, and taxed at 4% instead of 3.99%. The correct calculation is ($87,201 − $25,500) × 3.99% = $2,461.87." +us,scenario_102,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for state_income_tax_before_refundable_credits, so there was no answer to score against the $2,461.87 correct amount." +us,scenario_102,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a ~$23,400 NC joint standard deduction instead of the statutory $25,500, and the 2025 rate of 4.25% instead of the 3.99% rate for 2026. It taxed $63,801 at 4.25% rather than $61,701 at 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2024 rate of 4.5% instead of the 3.99% rate for 2026, and a $23,850 standard deduction instead of NC's $25,500 joint amount. It taxed $63,351 at 4.5% rather than $61,701 at 3.99%." +us,scenario_102,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It subtracted the $30,000 federal standard deduction instead of NC's own $25,500 joint deduction, and applied a nonexistent 4.49% rate instead of the 3.99% rate for 2026. It computed on $57,201 rather than $61,701 × 3.99% = $2,461.87." us,scenario_102,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_104,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented automatic eligibility through New York’s medically needy spend-down pathway after its own income calculation exceeded the aged/disabled limit. PolicyEngine assigns no Medicaid category, so neither age nor the existence of a spend-down program converts this household into an eligible Medicaid case." us,scenario_104,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 72 plus Social Security below an approximate aged/disabled threshold as sufficient and categorically removed all $20,520 of veterans benefits from its income test. The applicable category determination yields NONE, with no SSI-based or other aged pathway establishing eligibility." us,scenario_104,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly calculated net income of about $1,956 per month and explicitly found that it exceeded the elderly household's net-income limit, but then awarded $2,052 instead of enforcing the failed eligibility test. The minimum allotment applies only to an eligible one- or two-person household; it cannot replace a failed net-income test." us,scenario_104,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model did not carry the listed $8,389 annual employer-sponsored insurance premium through the elderly medical-expense deduction and then asserted an unsupported $196 monthly benefit. With all $34,616 of Social Security and veterans income counted, the allowable deductions still leave net income above the one-person limit, while the $540 monthly rent is below half of adjusted income and creates no excess-shelter deduction." -us,scenario_104,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It reached New York's real property tax credit but killed it with a fabricated rent screen, asserting that $540 average monthly rent produces no credit for a 65+ renter. The credit treats 25% of the $6,484 rent ($1,621) as deemed real property tax, and with countable NY household gross income of $14,096 (the Social Security; veterans benefits are not counted) the excess property tax reaches the age-65-or-over credit maximum of $375." -us,scenario_104,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It enumerated only earnings- and child-based New York credits (EITC, Empire State Child Credit) and never considered the IT-214 real property tax credit, which is the refundable renter/senior credit that supplies the entire $375. That credit keys off rent and age, not earned income or dependents: 25% of the $6,484 rent is deemed real property tax, and a 72-year-old with $14,096 of countable household gross income receives the $375 senior maximum." -us,scenario_104,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It declared the state EITC to be New York's primary refundable individual credit and swept the remainder into child-care and NYC credits, omitting the IT-214 real property tax credit entirely. That credit converts 25% of the $6,484 annual rent into deemed property tax and pays the 72-year-old head the age-65+ maximum of $375." -us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It computed the renter's deemed property tax correctly ($1,621 = 25% of $6,484) but disqualified the household by adding the $20,520 of veterans benefits to Social Security for a $34,616 household gross income against the $18,000 IT-214 limit; veterans benefits are outside NY household gross income, leaving $14,096, which qualifies. It then submitted an unsupported $63 that follows from no credit formula at all, rather than the $375 age-65+ maximum its own $1,621 property-tax figure produces." -us,scenario_104,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It reasoned that zero NY taxable income eliminates the ""real property tax credit basis,"" conflating a refundable credit with a liability offset. The IT-214 credit is paid regardless of tax liability and is triggered by low household gross income plus rent: $14,096 countable income and $6,484 rent for a 72-year-old yield the $375 senior maximum." -us,scenario_104,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It named the real property tax credit and then dismissed it on ""zero AGI,"" inverting the rule — low household gross income is the qualifying condition for IT-214, not a disqualifier. With $14,096 of countable household gross income, $6,484 of rent generating $1,621 of deemed property tax, and age 72, the credit pays its $375 senior maximum." -us,scenario_104,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It identified the right credit and the right $18,000 household gross income test but built the income figure by summing Social Security and veterans benefits to $34,616. Veterans benefits are excluded from NY household gross income for IT-214, so countable income is $14,096, under the limit, and the 72-year-old renter receives the $375 senior maximum." -us,scenario_104,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It waved off the real property tax credit as one that ""phases out or requires filing thresholds not met by this non-taxable income,"" treating taxable income as a gate. IT-214 is a standalone refundable credit claimed by a 65+ renter with household gross income at or below $18,000; $14,096 of countable income and $6,484 of rent produce the $375 senior maximum." -us,scenario_104,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It equated New York's refundable credits with the state EITC and concluded that zero taxable and earned income ends the analysis. The IT-214 real property tax credit requires neither earnings nor tax liability: 25% of the $6,484 rent as deemed property tax against $14,096 of countable household gross income gives the age-72 head the $375 senior maximum." -us,scenario_104,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It checked only the NY EITC's earned-income requirement and asserted no other refundable credit applies, missing the IT-214 real property tax credit for renters age 65+. That credit pays $375 here on $6,484 of rent and $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It located the real property tax credit but disqualified the household on the income test, which requires counting the $20,520 of veterans benefits toward household gross income. NY household gross income for IT-214 counts the $14,096 of Social Security and not the veterans benefits, leaving income under the $18,000 limit and yielding the $375 age-65+ maximum." -us,scenario_104,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied an earned-income-or-dependents screen to all New York refundable credits, a test the IT-214 real property tax credit does not use. The credit is keyed to rent and age: $6,484 of rent yields $1,621 of deemed property tax, and the 72-year-old head with $14,096 of countable household gross income receives the $375 senior maximum." -us,scenario_104,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It asserted non-qualification with no analysis, and its zero is consistent with considering only earnings- and child-conditioned NY credits. The IT-214 real property tax credit applies to this 72-year-old renter with $6,484 of rent and $14,096 of countable household gross income and pays the $375 senior maximum." -us,scenario_104,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It correctly cited the IT-214 $18,000 household gross income limit but computed gross income including the $20,520 of veterans benefits. Veterans benefits fall outside NY household gross income, so the counted figure is the $14,096 of Social Security, the household clears the limit, and the age-65+ credit maximum of $375 applies." -us,scenario_104,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It gave a bare non-qualification assertion, skipping the IT-214 real property tax credit entirely. A 72-year-old NY renter paying $6,484 with $14,096 of countable household gross income receives the $375 senior maximum." -us,scenario_104,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It stated flatly that no NY refundable credits exist for this household without testing any specific credit. New York's refundable real property tax credit pays this 72-year-old renter $375 on $6,484 of rent and $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It named IT-214 and rejected it on gross income exceeding the limit, which is true only if the $20,520 of veterans benefits is counted. NY household gross income for this credit is the $14,096 of Social Security, below the $18,000 ceiling, so the age-65+ maximum of $375 applies." -us,scenario_104,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It offered a one-line non-qualification with no credit-by-credit test. The IT-214 real property tax credit turns 25% of the $6,484 rent into deemed property tax and, against $14,096 of countable household gross income at age 72, pays the $375 senior maximum." -us,scenario_104,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It concluded there is ""no qualifying income or dependents"" for NY refundable credits, applying an earnings/dependents frame that IT-214 does not use. Low household gross income plus rent is the qualifying pattern, and $14,096 of countable income with $6,484 of rent at age 72 yields $375." -us,scenario_104,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It invented a minimum-rent requirement, claiming $6,484 is ""below the threshold required"" for the real property tax credit. The credit has no rent floor; it deems 25% of rent ($1,621) to be real property tax, and against $14,096 of countable household gross income the 72-year-old head receives the $375 age-65+ maximum." -us,scenario_104,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It reasoned from zero NY income tax liability and the absence of earnings or children to zero refundable credits, testing only the EITC and child credit. The refundable IT-214 real property tax credit is paid without any liability and delivers $375 to this 72-year-old renter with $6,484 of rent and $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It said the facts indicate no refundable NY credit, overlooking that age 72 plus $6,484 of pre-subsidy rent is precisely the fact pattern IT-214 rewards. With $14,096 of countable household gross income the credit reaches its age-65+ maximum of $375." -us,scenario_104,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It scanned only for earned-income-related refundable credits and reported none, never testing the real property tax credit. That credit is triggered here by the renter's $6,484 rent and the head's age of 72, paying the $375 senior maximum on $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It dismissed the senior/renter credit on a rent screen, reasoning that $6,484 annual rent implies an average monthly rent above New York's renter limit. The credit computes from that rent — 25% of $6,484 is $1,621 of deemed real property tax — and with countable household gross income of $14,096 the 72-year-old head receives the $375 age-65+ maximum." -us,scenario_104,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It asserted the facts establish no refundable NY credit eligibility, ignoring the two facts that establish it: age 72 and $6,484 of rent. IT-214 pays the $375 senior maximum on those facts with $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It looked for earnings, children, or another ""listed basis"" and found none, missing that rent plus age 65+ is itself the basis for New York's real property tax credit. On $6,484 of rent and $14,096 of countable household gross income the credit is $375." -us,scenario_104,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It reasoned that an elderly one-person household with no earnings or children shows no refundable-credit eligibility, which inverts the IT-214 profile — elderly, low household gross income, and paying rent is exactly who the credit targets. The result is $375, the age-65+ maximum." -us,scenario_104,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It applied the real property tax credit's income limit but included the $20,520 of veterans benefits along with Social Security in the comparison. NY household gross income for IT-214 counts only the $14,096 of Social Security here, clearing the $18,000 limit, so the 72-year-old renter receives the $375 maximum." -us,scenario_104,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It reduced the question to qualifying income and children, never reaching the IT-214 real property tax credit. That credit deems 25% of the $6,484 rent to be property tax and pays the age-72 head $375 on $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It correctly invoked the $18,000 renter-credit threshold but measured it against gross income that includes the $20,520 of veterans benefits. Veterans benefits are outside NY household gross income for this credit, leaving $14,096 and producing the $375 age-65+ maximum." -us,scenario_104,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It rejected the IT-214 renter credit on the income limit, which fails only when the $20,520 of veterans benefits is added to Social Security. Counting the $14,096 of Social Security alone puts the household under the $18,000 limit, and the 72-year-old renter with $6,484 of rent receives $375." -us,scenario_104,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It tested only the Empire State Child Credit and generic ""other eligibility criteria,"" never reaching the real property tax credit. Age 72 plus $6,484 of rent and $14,096 of countable household gross income produce the credit's $375 senior maximum." -us,scenario_104,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It limited the analysis to the state EITC and Empire State Child Credit, both keyed to earnings and dependents, and stopped there. New York's refundable real property tax credit needs neither and pays this 72-year-old renter $375 on $6,484 of rent and $14,096 of countable household gross income." -us,scenario_104,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It reached the real property tax credit last in its list and rejected it on income limits, a conclusion that requires counting the $20,520 of veterans benefits as household gross income. Only the $14,096 of Social Security counts, so the household is under the $18,000 limit and the age-65+ credit maximum of $375 applies." -us,scenario_104,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It stated the arithmetic explicitly — $34,616 against the $18,000 renter real-property-tax-credit limit — by adding veterans benefits to Social Security. NY household gross income for IT-214 excludes the $20,520 of veterans benefits, leaving $14,096 and a qualifying 72-year-old renter who receives the $375 maximum." -us,scenario_104,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It asserted that a single elderly household with no earned income or children gets no state refundable credits, omitting the rent- and age-based IT-214 credit. With $6,484 of rent, age 72, and $14,096 of countable household gross income, that credit is $375." -us,scenario_104,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It disqualified the renter's real property tax credit by comparing ""household resources"" of ~$34,616 — Social Security plus veterans benefits — to the $18,000 limit. Veterans benefits are not part of NY household gross income for this credit, so the $14,096 of Social Security is the counted amount and the 72-year-old renter receives $375." -us,scenario_104,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It named the Real Property Tax Credit but asserted it ""requires earned income or dependent children,"" which is false — the credit is conditioned on household gross income at or below $18,000, rent or property tax paid, and age. The 72-year-old head with $6,484 of rent and $14,096 of countable household gross income receives the $375 age-65+ maximum." -us,scenario_104,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It claimed there is ""no qualifying rent"" and no NY tax liability, when $6,484 of pre-subsidy rent is exactly the qualifying input and the credit is refundable regardless of liability. Deeming 25% of that rent to be real property tax against $14,096 of countable household gross income yields the $375 senior maximum." -us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $6,948 of disability benefits to gross income and to provisional income, pushing provisional income to $35,145 and taxable Social Security to $5,473 instead of $1,598, and inflating AGI to $26,263 against the correct $15,439.75. It also used a $15,400 standard deduction plus a $2,000 aged addition ($17,400) rather than the 2026 figures of $16,100 + $2,050, and omitted the $6,000 OBBBA senior deduction entirely. With disability benefits excluded, AGI of $15,440 falls below even its own $17,400 stack, so tax is $0 rather than $886." -us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It built a $15,600 deduction stack out of a $10,300 standard deduction and a $5,300 personal exemption; personal exemptions are zero and the 2026 single stack is $16,100 basic + $2,050 age-65 addition + $6,000 OBBBA senior deduction = $24,150, which exceeds even its own inflated AGI of $17,177.50 and yields $0. It also added half the disability benefits ($3,474) into provisional income to reach $31,671, doubling taxable Social Security to $3,335.50 when disability benefits are outside the provisional-income base and the correct taxable amount is $1,598.50." -us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $15,650 deduction stack ($8,300 standard + $2,050 aged + $5,300 personal exemption) is wrong on two counts: the 2026 single standard deduction is $16,100, and personal exemptions are zero, replaced for this taxpayer by the $6,000 OBBBA senior deduction it never applied. The correct $24,150 stack exceeds even its own overstated AGI of $17,177.50 — itself inflated by folding half the disability benefits into provisional income to get taxable Social Security of $3,335.50 instead of $1,598.50 — leaving $0 of taxable income rather than $1,527.50." -us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It claimed roughly $15,450 of deductions built on a personal exemption, which is zero for 2026, and it never applied the $6,000 OBBBA age-65 senior deduction; the correct stack is $16,100 + $2,050 + $6,000 = $24,150. That stack alone wipes out its own $17,178 AGI, and that AGI is itself overstated because it counted taxable Social Security of $3,336 by pulling half the $6,948 disability benefits into provisional income when the correct taxable amount is $1,598.50 on AGI of $15,439.75." -us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $811.33 equals 10% of $8,113, which is AGI of $26,263 less a $18,150 deduction ($16,100 + $2,050) — so it got the basic and aged deductions right but counted the $6,948 of non-taxable disability benefits in gross income and in provisional income, taking taxable Social Security to $5,473 under the 85% tier instead of $1,598.50. Excluding disability benefits gives AGI of $15,439.75, below even its own $18,150 deduction, so tax is $0; it additionally omitted the $6,000 OBBBA senior deduction." -us,scenario_107,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so no substantive computation was submitted. The required derivation is AGI of $15,439.75 (wages $6,366 + pension $7,476 + taxable Social Security $1,598, with disability benefits excluded) against a 2026 deduction stack of $16,100 + $2,050 + $6,000 = $24,150, giving $0 of taxable income and $0 of tax." -us,scenario_107,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It named the right deduction stack — standard, age-65 addition, and the senior deduction, $24,150 total — but its $171.25 implies $1,712.50 of taxable income and therefore AGI of $25,862.50, which is exactly the correct $15,439.75 plus the $6,948 of disability benefits plus the $3,474 of extra taxable Social Security that counting those benefits in provisional income produces. Disability benefits are outside AGI and outside the provisional-income base, so AGI is $15,439.75, below the $24,150 stack, and the tax is $0." -us,scenario_107,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It explicitly put the $6,948 of disability benefits into gross income and into combined income, reaching $35,145 of provisional income and $5,473 of taxable Social Security under the 85% tier rather than $1,598.50 under the 50% tier, giving AGI of $26,263 instead of $15,439.75. It also projected a single $17,375 aged standard deduction instead of $16,100 + $2,050 and omitted the $6,000 OBBBA senior deduction; on the correct AGI, even its own $17,375 figure leaves $0 of taxable income." -us,scenario_107,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It assembled the correct 2026 deduction stack of $24,150 ($16,100 + $2,050 + $6,000 senior deduction), so its entire $211 stems from treating the $6,948 of disability benefits as taxable income and as part of provisional income, which raised taxable Social Security to $5,473 and AGI to $26,263. Disability benefits are excluded from AGI, leaving $15,439.75 against the $24,150 stack and $0 of taxable income." +us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"Its own reasoning ruled out the Real Property Tax Credit because household gross income of about $34,616 exceeds the $18,000 limit, and it found no other refundable credit without earned income or dependents. That leaves $0. It then added an invented $63 labeled 'household credit / other minor refundable amounts', but the NY household credit is nonrefundable and can only offset tax. With zero NY tax liability, it contributes nothing to state_refundable_credits." +us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $6,948 of disability benefits as gross income and in provisional income, which raised taxable Social Security to $5,473 and AGI to $26,263, when the correct AGI is $15,439.75. It also used wrong 2026 deduction amounts ($15,400 + $2,000 instead of $16,100 + $2,050) and left out the $6,000 senior deduction. Its taxable income of $8,863 comes from both errors, but the correct AGI is fully offset by the standard deduction alone." +us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It used a pre-TCJA deduction setup: a $10,300 standard deduction plus a $5,300 personal exemption. The personal exemption is $0 for 2026 because the TCJA suspension was made permanent. The correct deductions are $16,100 standard, $2,050 for age 65+ and the $6,000 senior deduction. It also added the disability benefits to Social Security benefits in combined income ($31,671), which pushed taxable Social Security up to $3,335.50. Even its own AGI of $17,177.50 is below the correct $24,150 of deductions, so the tax is $0." +us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA-sunset structure: an $8,300 standard deduction plus a $5,300 personal exemption. For 2026 the single standard deduction is $16,100 and the personal exemption is $0. It also left out the $6,000 senior deduction for filers 65 and older. With the correct $18,150 age-adjusted standard deduction, its $17,177.50 AGI has no taxable income, so the tax is $0." +us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It built its roughly $15,450 of deductions from the basic standard deduction plus a personal exemption, a pre-TCJA structure. The 2026 personal exemption is $0, while the single standard deduction is $16,100, plus $2,050 for age 65+ and the $6,000 senior deduction. Its roughly $17,178 AGI is below the correct deductions, so taxable income and tax are both $0, not the $1,728 it taxed at 10%." +us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $811.33 equals 10% of $8,113.25, which is $26,263.25 minus $18,150. So it counted the $6,948 of disability benefits in AGI, which also raised taxable Social Security to $5,473.25, and it left out the $6,000 senior deduction. PolicyEngine excludes the disability benefits, which gives AGI of $15,439.75, below the $18,150 age-adjusted standard deduction, so tax is $0." +us,scenario_107,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was no numeric answer to score. The correct derivation is AGI of $15,439.75 against an $18,150 age-adjusted standard deduction, for $0 of tax." +us,scenario_107,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It said it applied the $16,100 standard deduction, the $2,050 age-65 addition and the $6,000 senior deduction, yet still found about $1,712.50 of taxable income. That requires an AGI of about $25,860, which only happens if the $6,948 of disability benefits are counted in gross income and provisional income. Without them, AGI is $15,439.75, below even the base standard deduction, so tax is $0. Its $171.25 also does not match the $211 that the taxable-disability reading gives, which points to a separate error in its taxable Social Security calculation." +us,scenario_107,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its deductions were correct: $16,100 standard, the age-65 addition and the $6,000 senior deduction, totaling $24,150. Its only divergence is that it explicitly assumed the $6,948 of disability benefits are taxable, which raised taxable Social Security to $5,473 and left $2,113 taxable. One reading of the prompt treats the benefits as employer-funded and taxable under IRC section 105. The other, which PolicyEngine uses, excludes them from gross income. Under that reading AGI is $15,439.75 and tax is $0." +us,scenario_107,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the correct $18,150 age-adjusted standard deduction and the $6,000 senior deduction. However, it put the $6,948 of disability benefits into gross income and provisional income, which made taxable Social Security $5,473.25 and taxable income $2,113.25. The prompt does not say who paid the premiums, which decides whether the benefits are taxable (IRC section 105) or excluded (section 104). PolicyEngine excludes them, which gives AGI of $15,439.75 and tax of $0." +us,scenario_107,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted the $6,948 of disability benefits in gross income, which gave AGI of $26,263 and taxable Social Security of $5,473 under the 85% formula. It also projected a $17,375 age-65 standard deduction instead of the actual 2026 $18,150 ($16,100 + $2,050) and left out the $6,000 senior deduction. With the disability benefits excluded, AGI is $15,439.75, below the standard deduction, so tax is $0." +us,scenario_107,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly subtracted $16,100 + $2,050 + $6,000 = $24,150. Its $211 comes only from treating the $6,948 of disability benefits as taxable, which gave AGI of about $26,263. The prompt does not say whether the benefits were funded by an employer (taxable) or with after-tax money (excluded). PolicyEngine excludes them, so AGI is $15,439.75 and tax is $0." us,scenario_107,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for federal_refundable_credits, violating the required output contract." us,scenario_107,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly carried the temporary 2021 removal of the childless EITC upper age limit into 2026, even though a 75-year-old is outside the applicable age range. It also understated AGI by omitting the taxable pension and applicable taxable Social Security income, placing the filer inside the phaseout range when the household’s income exceeds the childless EITC cutoff." us,scenario_107,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age 65+, $2,000 of assets, and $49,500 of annual income as sufficient for Ohio's aged Medicaid pathway without applying its actual income standard. The head qualifies through no Medicaid category, so satisfying the immigration rule and asserting that assets are below a limit cannot establish eligibility." @@ -7627,20 +8339,21 @@ us,scenario_107,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813, us,scenario_107,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate to $3,966 of taxable income, missing Ohio's 0% bracket covering the first $26,050 of taxable nonbusiness income, under which this return owes nothing before credits. It also invented a $25,000 age-65 retirement income exclusion that Ohio does not have — Ohio grants a retirement income tax credit capped at $200, not a deduction — so both the deduction and the 2.75% multiplication are unsupported." us,scenario_107,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model taxed $14,159 at 2.75%, ignoring Ohio's zero bracket on taxable nonbusiness income up to $26,050, which by itself makes the liability $0. It also started from an AGI of $16,559 because it never applied Ohio's subtraction for the federally taxable portion of Social Security benefits, and it computed the retirement income credit as 1.5% of pension income instead of Ohio's statutory tabular amount ($130 for $7,476 of retirement income)." us,scenario_107,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It added the $10,392 of non-Social-Security survivor benefits into gross income, but those benefits are not an IRS gross income source, so AGI is the $7,656 of wages alone rather than its $18,048. It then deducted only $17,000 (a 2025-era $15,000 base plus a $2,000 aged addition), omitting both the 2026 OBBBA amounts ($16,100 standard deduction plus $2,050 aged addition) and the $6,000 senior deduction, whose $24,150 total zeroes taxable income even against its own overstated AGI." -us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It included the $10,392 survivor benefits in AGI, which excludes them from nothing — they are outside IRS gross income entirely, leaving AGI at the $7,656 of wages. Its $17,750 deduction ($15,650 + $2,100) also omits the $6,000 OBBBA senior deduction available to a filer age 85 with MAGI far below the $75,000 phaseout, and the correct $24,150 of deductions drives taxable income to $0 rather than its $298." -us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It built AGI as $18,048 by treating the $10,392 survivor benefits as taxable income when they are not an IRS gross income source, so AGI is $7,656. Its single 65+ deduction of $16,800 understates the 2026 figure and leaves out the $6,000 senior deduction; the correct $16,100 + $2,050 + $6,000 = $24,150 exceeds even its own AGI, so the $1,248 of taxable income it taxed at 10% does not exist." -us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It labeled the $10,392 survivor benefits 'taxable survivor benefits' and rolled them into AGI, though they are excluded from IRS gross income, leaving AGI at $7,656 of wages. Its $17,450 age-65 standard deduction omits the $6,000 OBBBA senior deduction, and the correct $24,150 total leaves $0 taxable income instead of its $598." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reconstructed 2026 as a TCJA-sunset year, subtracting an $8,300 standard deduction, a $1,950 aged addition, and a $5,050 personal exemption; OBBBA made the TCJA structure permanent, so the personal exemption stays repealed and the 2026 single standard deduction is $16,100 plus a $2,050 aged addition plus the $6,000 senior deduction. It also inflated AGI to $18,048 by counting the $10,392 survivor benefits, which are not IRS gross income, so taxable income is $0 rather than its $2,748." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its explicit premise — 'assuming expiration of the TCJA in 2026' — is wrong: OBBBA made the TCJA individual provisions permanent, so there is no restored personal exemption and the 2026 deductions are $16,100 + $2,050 + the $6,000 senior deduction = $24,150. It compounded this by adding the $10,392 survivor benefits to AGI when they are not an IRS gross income source, so the $2,298 of taxable income behind its $230 is entirely an artifact of both errors." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied a combined standard deduction plus personal exemption of roughly $15,800, but the personal exemption remains repealed in 2026 under OBBBA and the correct deductions are $16,100 + $2,050 aged + $6,000 senior = $24,150. It also counted the $10,392 survivor benefits in AGI when they fall outside IRS gross income, so AGI is $7,656 and taxable income is $0, not its $2,248." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its Social Security analysis is right, but it added the $10,392 of survivor benefits to AGI even though they are not an IRS gross income source, so AGI is the $7,656 of wages. Its $17,700 age-65 standard deduction also omits the $6,000 OBBBA senior deduction; the correct $24,150 of deductions leaves $0 taxable income instead of the $348 it taxed." -us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted 'the 2026 standard deduction and personal exemption totaling $15,600', but OBBBA leaves the personal exemption repealed and sets 2026 deductions at $16,100 standard plus $2,050 aged plus the $6,000 senior deduction. It further overstated AGI by including the $10,392 survivor benefits, which are excluded from IRS gross income, so taxable income is $0 rather than its $2,448." -us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $10,392 survivor pension benefits to AGI when they are not an IRS gross income source, leaving AGI at $7,656 of wages rather than $18,048. Its estimated $17,450 age-65 standard deduction also drops the $6,000 OBBBA senior deduction, and the correct $24,150 total eliminates the $598 of taxable income behind its $59.80." -us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumed 'current-law parameters after TCJA sunset' and restored a ~$5,300 personal exemption alongside an $8,300 standard deduction; OBBBA made the TCJA individual rules permanent, so 2026 gives $16,100 standard, $2,050 aged, and the $6,000 senior deduction. It also treated the $10,392 survivor benefits as taxable when they are outside IRS gross income, so AGI is $7,656 and taxable income is $0, not its $2,398." -us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used 'about $10,350' as the 2026 single standard deduction plus aged addition — the 2016 combined standard-deduction-plus-exemption figure — when the correct 2026 amounts are $16,100 plus a $2,050 aged addition plus the $6,000 senior deduction. Adding the $10,392 survivor benefits to AGI compounded it; those benefits are not IRS gross income, so AGI is $7,656 and the $7,698 of taxable income producing its $770 is fabricated." -us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It deducted a pre-TCJA-style package — a $7,941 standard deduction, a $1,626 aged addition, and a $5,190 personal exemption — when OBBBA keeps the exemption repealed and sets 2026 at $16,100 + $2,050 + the $6,000 senior deduction = $24,150. It also counted the $10,392 survivor benefits in AGI, which excludes them from nothing since they are not IRS gross income, so AGI is $7,656 and taxable income is $0 rather than its $3,291." -us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It put the $10,392 of other survivor benefits into AGI although they are not an IRS gross income source, so AGI is the $7,656 of wages. Its $17,700 single 65+ standard deduction omits the $6,000 OBBBA senior deduction — which this filer qualifies for, with MAGI far below the $75,000 phaseout — and the correct $24,150 leaves $0 taxable income instead of its $348." +us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Counted the $10,392 non-Social Security survivor benefits in AGI ($18,048 instead of the reference's $7,656, which is wages only). It then deliberately fell back to 2025 figures ($15,000 + $2,000 = $17,000) rather than the 2026 single deduction of $16,100 plus the $2,050 aged add-on. It also ignored the $6,000 senior deduction, which together zero out taxable income." +us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Treated the $10,392 survivor benefits as taxable, which inflated AGI to $18,048 against the reference's $7,656. It then used an understated $17,750 deduction ($15,650 + $2,100) and omitted the $6,000 senior deduction for filers 65 and older, leaving a spurious $298 of taxable income." +us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Included the $10,392 other survivor benefits in AGI ($18,048 vs. the reference's wages-only $7,656). It applied an undersized $16,800 single 65+ standard deduction and no $6,000 senior deduction, producing $1,248 of taxable income when the correct figure is $0." +us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Added the $10,392 survivor benefits to AGI ($18,048 instead of $7,656). It used a $17,450 deduction that falls short of the 2026 $16,100 + $2,050 aged amount and left out the $6,000 senior deduction, which produced $598 of phantom taxable income." +us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Counted the $10,392 survivor benefits as taxable income (AGI $18,048 vs. the reference's $7,656). It applied a $17,350 standard deduction below the 2026 single 65+ amount and skipped the $6,000 senior deduction, yielding $698 taxable instead of $0." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA had expired and applied pre-TCJA amounts ($8,300 standard deduction, $1,950 aged add-on, $5,050 personal exemption), even though OBBBA made the TCJA structure permanent. It also counted the $10,392 survivor benefits in AGI ($18,048 vs. $7,656), leaving $2,748 taxable instead of $0." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Explicitly assumed the TCJA expired in 2026 and applied the smaller pre-TCJA standard deduction plus a personal exemption, missing the permanent OBBBA-era $16,100 + $2,050 deduction and the $6,000 senior deduction. It also included the $10,392 survivor benefits in AGI ($18,048 vs. $7,656)." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Added the $10,392 survivor benefits to AGI ($18,048 instead of the reference's $7,656). It used a combined standard deduction and personal exemption of only $15,800, missing the 2026 single deduction of $16,100 plus the $2,050 aged add-on and the $6,000 senior deduction, so it taxed $2,248 that is actually sheltered." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Included the $10,392 survivor benefits in AGI ($18,048 vs. $7,656). It applied a $17,700 single 65+ deduction, below the 2026 $18,150, with no $6,000 senior deduction, leaving $348 of taxable income when the correct amount is $0." +us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Counted the $10,392 survivor benefits as taxable (AGI $18,048 vs. $7,656). It used a combined standard deduction and personal exemption of $15,600, well below the 2026 single 65+ deduction of $18,150, and omitted the $6,000 senior deduction, producing $2,448 of taxable income." +us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Included the $10,392 survivor pension in AGI ($18,048 instead of the wages-only $7,656). It estimated the deduction at $15,400 + $2,050 = $17,450 rather than the 2026 base of $16,100, and ignored the $6,000 senior deduction, which left $598 taxable instead of $0." +us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Assumed a TCJA sunset in 2026 and applied a pre-TCJA personal exemption plus an $8,300 standard deduction and $2,050 elderly add-on, rather than the permanent OBBBA-era $16,100 + $2,050 deduction and the $6,000 senior deduction. It also counted the $10,392 survivor benefits in AGI ($18,048 vs. $7,656)." +us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Applied an assumed post-TCJA-sunset deduction of only $10,350 and dropped even the personal exemption that a sunset would restore. It missed the 2026 single 65+ deduction of $18,150 and the $6,000 senior deduction, and it included the $10,392 survivor benefits in AGI ($18,048 vs. $7,656), producing $7,698 of taxable income." +us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Assumed a TCJA sunset and used pre-TCJA amounts ($7,941 standard deduction, $1,626 aged add-on, $5,190 exemption, $14,757 total) instead of the permanent 2026 $16,100 + $2,050 deduction plus the $6,000 senior deduction. It also added the $10,392 survivor benefits to AGI ($18,048 vs. $7,656)." +us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Included the $10,392 other survivor benefits in AGI ($18,048 vs. the reference's wages-only $7,656). It used a $17,700 single 65+ deduction, below the 2026 $18,150, and no $6,000 senior deduction, leaving $348 taxable and $35 of tax instead of $0." us,scenario_108,federal_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model applied the 7.65% childless-EITC phase-in rate without enforcing the childless claimant age limit. At age 85, the head is ineligible for the childless EITC, so the computed $586 EITC must be $0." us,scenario_108,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits. The required result is $0 because every refundable-credit component evaluates to zero, including EITC for this 85-year-old claimant with no qualifying children." us,scenario_108,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -7662,91 +8375,104 @@ us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identi us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract." us,scenario_108,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_108,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model ran the elderly/disabled 100% FPG net income test ($1,305/month) as a hard eligibility gate and stopped at ""not eligible,"" never applying Wisconsin's broad-based categorical eligibility, under which a TANF-funded non-cash benefit makes the household categorically eligible whenever gross income is under 200% FPG — here $2,581.83 is 1.98 times the $1,304.17 guideline — and waives both the net income and asset tests. Having declared ineligibility it never reached the one- and two-person minimum allotment (8% of the $298 maximum = $23.84/month, $24.37 from October), which is the whole $287.68 answer." -us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It denied eligibility solely because net income of ~$2,214/month exceeds the 100% FPG one-person net limit, missing that Wisconsin confers broad-based categorical eligibility through a TANF-funded non-cash benefit at a 200% FPG gross screen that this household clears at 1.98 times the guideline, which waives the net income test it applied. It therefore never reached the minimum allotment for one-person households ($23.84/month, $24.37 after the October adjustment) that produces the $287.68 annual total." -us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a 130% FPG gross income test and quoted the wrong limit ($1,385/month, when 130% of the $1,304.17 one-person guideline is $1,695), while a household containing an elderly or disabled member is exempt from the gross income test entirely and Wisconsin's applicable screen is the 200% FPG broad-based categorical eligibility limit, which the $2,581.83 gross income clears at 1.98 times the guideline. It also skipped the one-person minimum allotment of $23.84/month that generates the $287.68 answer." -us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"The model derived the correct chain — negative computed allotment, therefore ""minimum benefit for 1-2 person households = ~$23/mo = $276/yr"" — then discarded that result and submitted $2,227, a figure ($185.58/month) that no step of its own arithmetic produces. Its own $276 was short only because it used the unindexed $23 minimum instead of the FY2026 value of 8% of the $298 maximum allotment ($23.84/month, rising to $24.37 in October), which totals $287.68." -us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning concluded ""the household is not eligible for SNAP"" after failing the 100% FPG net income test, missing Wisconsin's broad-based categorical eligibility via TANF-funded non-cash assistance, which applies a 200% FPG gross screen the household clears at 1.98 times the guideline and waives the net income and asset tests. It then submitted $2,376 — $198/month — which contradicts its own ineligibility finding and matches no computation it performed; the correct result is the one-person minimum allotment of $23.84/month ($24.37 from October)." -us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model reached the correct structure — a categorically eligible one-person household whose 30%-of-net contribution ($610.50) exceeds the $298 maximum allotment, leaving the statutory minimum benefit — but priced that minimum at the unindexed $23/month. The FY2026 minimum is 8% of the $298 maximum, $23.84/month through September, rising to $24.37 with the October allotment adjustment, which sums to $287.68 rather than $276." -us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model asserted ""WI does not have broad-based categorical eligibility that eliminates the gross income test for elderly/disabled households""; Wisconsin does confer BBCE through a TANF-funded non-cash benefit with a 200% FPG gross screen and no asset test, and this household's gross income of 1.98 times the $1,304.17 guideline clears it, waiving the 100% FPG net income test the model used to deny eligibility. It then also skipped the one-person minimum allotment of $23.84/month ($24.37 from October) that yields $287.68." -us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model stopped computing and asserted net monthly income ""around $1,700"" and a $164/month allotment, crediting an uncapped excess shelter deduction of roughly $500/month. The actual net income is $2,035.67 — rent of $600 plus the standard utility allowance leaves only about $180/month of excess shelter — so 30% of net income ($610.50) exceeds the $298 maximum allotment and the benefit falls to the one-person minimum of $23.84/month ($24.37 from October), $287.68 for the year." -us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It denied the benefit because net income of ~$2,184/month exceeds the 100% FPG one-person net limit, missing that Wisconsin's broad-based categorical eligibility via TANF-funded non-cash assistance waives that test for a household whose gross income is 1.98 times the poverty guideline, inside the 200% FPG screen. A categorically eligible one-person household with a computed allotment of zero still receives the minimum benefit of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_108,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly found that the expected contribution exceeds the maximum allotment and produces a negative figure, then floored that at $0. Federal rules floor an eligible one- or two-person household at the minimum allotment instead — 8% of the $298 maximum, i.e. $23.84/month and $24.37 after the October adjustment — which is exactly the $287.68 reference." -us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed that 30% of net monthly income exceeds the one-person maximum allotment and concluded the benefit is zero, omitting the minimum-allotment floor that pays eligible one- and two-person households 8% of the maximum allotment regardless of the negative formula result. That floor is $23.84/month for FY2026 and $24.37 from October, producing the $287.68 annual total." -us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model used the exact gross figure of $2,581.83 but called it above the one-person eligibility limit; that amount is 1.98 times the $1,304.17 poverty guideline and sits inside Wisconsin's 200% FPG broad-based categorical eligibility screen, which the TANF-funded non-cash benefit triggers and which waives the net income and asset tests. Its second reason — 30% of net exceeding the maximum allotment — leads to the minimum allotment of $23.84/month ($24.37 from October), not to zero." -us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted the household's income exceeds the eligibility threshold with no computation; gross income of $2,581.83 is 1.98 times the $1,304.17 one-person guideline, within Wisconsin's 200% FPG broad-based categorical eligibility limit conferred by a TANF-funded non-cash benefit, so the household is eligible with assets of $270 and the income tests waived. Its zero corresponds to stopping at an income screen the household passes, in place of the minimum allotment of $23.84/month ($24.37 from October) that totals $287.68." -us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It denied eligibility on the ground that both gross and net income exceed SNAP limits; gross income is 1.98 times the poverty guideline and therefore inside Wisconsin's 200% FPG broad-based categorical eligibility screen, which is triggered by TANF-funded non-cash assistance and waives the net income and asset tests the model relied on. The eligible household then draws the one- and two-person minimum allotment, $23.84/month rising to $24.37 in October, for $287.68." -us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model identified the exact pathway — Wisconsin BBCE at 200% FPG, assets of $270, high net income leaving only the minimum benefit — and then used the stale $23/month minimum allotment. The FY2026 minimum is 8% of the $298 one-person maximum, $23.84/month, and $24.37 for the months after the October allotment adjustment, summing to $287.68 rather than $276." -us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave no derivation, and its $0 is the result of terminating at either the 100% FPG net income test or the negative formula result. Wisconsin's broad-based categorical eligibility, conferred by a TANF-funded non-cash benefit at a 200% FPG gross screen that this household clears at 1.98 times the guideline, waives the net income and asset tests, and the eligible one-person household receives the minimum allotment of $23.84/month ($24.37 from October), $287.68 for the year." -us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It ruled the household out because gross monthly income of $2,581.83 ""exceeds SNAP limits""; that amount is 1.98 times the $1,304.17 guideline and passes Wisconsin's 200% FPG broad-based categorical eligibility screen triggered by TANF-funded non-cash assistance, which also waives the net income test it invoked next. The eligible household receives the one-person minimum allotment of $23.84/month, $24.37 from October, for $287.68." -us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model denied the benefit on the 100% FPG net income test alone, missing Wisconsin's broad-based categorical eligibility through a TANF-funded non-cash benefit, which screens on gross income at 200% FPG — passed here at 1.98 times the guideline — and waives the net income and asset tests. Eligibility then carries the minimum allotment of $23.84/month ($24.37 from October), totaling $287.68." -us,scenario_108,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly found that 30% of net income exceeds the one-person maximum allotment and then returned zero, dropping the minimum-allotment floor that pays eligible one- and two-person households 8% of the maximum allotment. That floor is $23.84/month for FY2026 and $24.37 after the October adjustment, which sums to the $287.68 reference." -us,scenario_108,snap,glm-5.2,llm_error,thresholds_rates,False,"Its arithmetic ends at ""max allotment (~$300) minus 30% of net income (~$659), which is negative, resulting in $0,"" skipping the minimum-allotment rule that pays an eligible one- or two-person household 8% of the maximum allotment when the formula result is zero or negative. For FY2026 that is $23.84/month, $24.37 from October, producing $287.68; the household is eligible through Wisconsin's 200% FPG broad-based categorical eligibility." -us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model rejected eligibility because net income exceeds the ~100% FPG one-person limit and applied a ""capped shelter deduction""; the excess shelter deduction is uncapped for a household containing an elderly or disabled member, and more decisively Wisconsin's broad-based categorical eligibility through TANF-funded non-cash assistance waives the net income test outright for gross income under 200% FPG, which this household meets at 1.98 times the guideline. The eligible household receives the minimum allotment of $23.84/month ($24.37 from October), $287.68 annually." -us,scenario_108,snap,gpt-5.4-mini,llm_error,other,False,"The model reached the right eligibility conclusion but submitted $1,172 — $97.67/month — with no supporting computation of any kind. The correct derivation is that 30% of net income ($610.50) exceeds the $298 maximum allotment, so the household receives the one-person minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_108,snap,gpt-5.4-nano,llm_error,other,False,"The model declined to compute an allotment and defaulted to zero on the grounds that it lacked details on allowable deductions, when the facts given fully determine the result. Wisconsin's 200% FPG broad-based categorical eligibility makes the household eligible at 1.98 times the guideline, the $610.50 expected contribution exceeds the $298 maximum allotment, and the one-person minimum allotment of $23.84/month ($24.37 from October) yields $287.68." -us,scenario_108,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model concluded that income is ""above the amount that could produce a positive allotment"" and returned zero, dropping the rule that an eligible one- or two-person household receives the minimum allotment — 8% of the $298 maximum — whenever the standard formula yields zero or less. That is $23.84/month for FY2026 and $24.37 after the October adjustment, giving the $287.68 reference; the household is eligible via Wisconsin's 200% FPG broad-based categorical eligibility." -us,scenario_108,snap,gpt-5.6-luna,llm_error,other,False,"The model announced an unsupported $103/month allotment from rent, medical expenses, and ""income-based deductions,"" without ever computing the expected contribution. With net income of $2,035.67, the 30% contribution of $610.50 exceeds the $298 maximum allotment, so the benefit is the one-person minimum of $23.84/month ($24.37 from October), $287.68 for the year." -us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It denied the benefit because countable net income stays above the one-person elderly-or-disabled net income limit, missing Wisconsin's broad-based categorical eligibility via a TANF-funded non-cash benefit, which screens gross income at 200% FPG — passed here at 1.98 times the $1,304.17 guideline — and waives that net income test and the asset test. The eligible household then receives the minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the 100% FPG net income limit as dispositive, missing that Wisconsin's TANF-funded non-cash benefit confers broad-based categorical eligibility with a 200% FPG gross screen this household clears at 1.98 times the guideline, waiving the net income and asset tests. Eligibility carries the one- and two-person minimum allotment of $23.84/month ($24.37 from October), which is the $287.68 answer." -us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted income exceeds SNAP limits with no computation; gross income of $2,581.83 is 1.98 times the $1,304.17 one-person guideline and falls inside Wisconsin's 200% FPG broad-based categorical eligibility screen conferred by TANF-funded non-cash assistance, which waives the income and asset tests. The eligible household receives the minimum allotment of $23.84/month, $24.37 from October, for $287.68." -us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"It compared an annual net income of ~$28,000 against ~$16,000 at 100% FPG and returned zero; both figures are off (the engine's net income is $24,428 annually) and, more fundamentally, Wisconsin's broad-based categorical eligibility through a TANF-funded non-cash benefit waives the net income test for any household under 200% FPG gross, which this one meets at 1.98 times the guideline. The eligible household receives the minimum allotment of $23.84/month, rising to $24.37 in October, for $287.68." -us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model recognized the elderly/disabled waiver of the gross income test but then treated the 100% FPG net income test as binding, missing that Wisconsin's broad-based categorical eligibility via TANF-funded non-cash assistance waives the net income test as well for households under 200% FPG gross — satisfied here at 1.98 times the guideline. Eligibility triggers the one-person minimum allotment of $23.84/month ($24.37 from October), $287.68 for the year." -us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"After computing net income of ~$2,218/month it failed the household against the ~$1,305 net income limit, missing Wisconsin's broad-based categorical eligibility through a TANF-funded non-cash benefit, which applies a 200% FPG gross screen the household passes at 1.98 times the guideline and waives the net income test it applied. It also omitted the standard utility allowance from shelter costs and never reached the minimum allotment of $23.84/month ($24.37 from October) that yields $287.68." -us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"The model denied SNAP because net income of ~$2,225/month exceeds 100% FPG, missing that Wisconsin's TANF-funded non-cash benefit confers broad-based categorical eligibility with a 200% FPG gross screen this household clears at 1.98 times the $1,304.17 guideline, waiving the net income and asset tests. An eligible one-person household with a zero computed allotment receives the minimum benefit of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It found the household ineligible on the 100% FPG net income test alone, missing Wisconsin's broad-based categorical eligibility conferred by TANF-funded non-cash assistance, which screens on gross income at 200% FPG — met here at 1.98 times the guideline — and waives the net income and asset tests. The eligible household receives the one- and two-person minimum allotment of $23.84/month ($24.37 from October), which is the $287.68 reference." -us,scenario_108,snap,kimi-k3,llm_error,thresholds_rates,False,"The model handled the gross-test waiver and asset test correctly but ended at ""30% of net income (about $664) far exceeds the one-person maximum allotment, so the benefit is $0,"" omitting the minimum-allotment floor that pays an eligible one- or two-person household 8% of the maximum ($23.84/month for FY2026, $24.37 from October). It also zeroed the excess shelter deduction by leaving out the standard utility allowance; the engine's net income of $2,035.67 reflects about $180/month of excess shelter." -us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model applied a 130% FPG gross income test despite the exemption for households with an elderly or disabled member, and used fabricated limits ($2,010 gross and $1,180 net, when the one-person guideline is $1,304.17/month); Wisconsin's operative screen is the 200% FPG broad-based categorical eligibility limit, which the $2,581.83 gross income passes at 1.98 times the guideline. It also omitted the 20% earned income deduction, inflating net income to $2,346, and never reached the $23.84/month minimum allotment." -us,scenario_108,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model invoked a ""165% FPL gross income test (about $2,150 per month)"" as the eligibility gate; 165% FPG is the threshold under 7 CFR 273.1(b)(7) for treating an elderly and permanently disabled person who lives with others as a separate household, not an eligibility limit, and Wisconsin's applicable screen is the 200% FPG broad-based categorical eligibility limit, which this household's gross income clears at 1.98 times the guideline. The eligible household receives the one-person minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." -us,scenario_108,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"After several restarts the model settled on 30% of net income ($671.87) exceeding the $292 maximum allotment and returned zero, omitting the minimum-allotment floor that pays an eligible one- or two-person household 8% of the FY2026 $298 maximum, i.e. $23.84/month and $24.37 from October. Its intermediate work also invented a $400/month utility allowance and imposed the $672 shelter cap, which does not apply to a household containing an elderly or disabled member." -us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that countable income exceeds the one-person SNAP limit without computation; gross income of $2,581.83 is 1.98 times the $1,304.17 guideline and inside Wisconsin's 200% FPG broad-based categorical eligibility screen triggered by TANF-funded non-cash assistance, which waives the net income and asset tests. The eligible household receives the minimum allotment of $23.84/month, $24.37 from October, for $287.68." -us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"Applied Wisconsin's single-filer standard deduction ($13,940) to a filer whose listed status is qualifying surviving spouse, which draws a much larger sliding-scale deduction that already exceeds the $18,048 of non-Social-Security income. It also invented a '$270 age-65+ additional deduction' — Wisconsin grants no age-based addition to the standard deduction, its age-65 benefit being the $250 addition to the $700 personal exemption, and $270 is the household's bank balance — and omitted the personal exemption entirely, leaving $3,838 of phantom taxable income taxed at 3.5%." -us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"Used the single column of Wisconsin's sliding-scale standard deduction ($13,230, the 2024 single maximum) instead of the surviving-spouse column the stated filing status requires, and folded the $250 age-65 amount into the standard deduction while dropping the $700 personal exemption it attaches to. It then taxed $4,568 of income that the correct deduction eliminates, at a 3.54% bottom rate Wisconsin repealed after tax year 2022 (the bottom rate is 3.50%)." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"Ran the sliding-scale standard deduction off the single-filer schedule for a qualifying surviving spouse and omitted the $700 personal exemption and its $250 age-65 addition. The surviving-spouse deduction alone exceeds the $18,048 of Wisconsin income it started from, so the ~$4,548 of residual taxable income it taxed at 3.5% is an artifact of the wrong deduction column." -us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"Subtracted only the single-filer sliding-scale standard deduction and a personal exemption, when the surviving-spouse standard deduction plus the $700 exemption and $250 age-65 addition already exceed the $18,048 of non-Social-Security income, so pre-credit Wisconsin taxable income and tax are zero. Its residual $70 is the wrong-column liability net of the renter school property tax credit; that credit is nonrefundable and reduces a liability that never existed." -us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Named the surviving-spouse status but pulled single-filer parameters for it: $12,760 is Wisconsin's single maximum standard deduction (a 2023 figure) with the single 12% phaseout above $16,480, while the surviving-spouse column carries a much larger maximum whose phaseout starts well above $18,048, so no phaseout applies and the deduction wipes out taxable income. Its $1,400 exemption also doubles the single $700 personal exemption for a one-person household, and even that overstated exemption still left $4,184 of taxable income taxed at 3.5%." -us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Gave no derivation, but $144.13 backs out exactly to ($7,656 + $10,392 − $13,230 − $700) × 3.5% — Wisconsin's 2024 single-filer maximum standard deduction plus a single $700 personal exemption. It applied the single column to a qualifying surviving spouse and omitted the $250 age-65 exemption addition; the surviving-spouse deduction plus exemptions exceeds $18,048, making the correct pre-refundable-credit liability zero." -us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"Phased the standard deduction down with the single-filer sliding scale (12% of AGI over $16,880) to $13,817, when the surviving-spouse column governs and its phaseout threshold sits far above this household's $18,048 of income, leaving the full deduction intact. It also dropped the $700 personal exemption and $250 age-65 addition and used the 3.54% rate Wisconsin repealed after 2022, taxing $4,231 of income that does not exist." -us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for state_income_tax_before_refundable_credits, so the required output key is absent rather than substantively wrong. No Wisconsin standard deduction, exemption, or rate computation was returned to evaluate." -us,scenario_108,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It built Schedule H household income as wages + Social Security + the separate $10,392 survivor benefits = $30,982 and declared the household over the $24,680 ceiling, when Wisconsin household income here is $7,656 of wages plus $12,934 of tax-exempt Social Security = $20,590, comfortably under the limit. It then submitted $636 in direct contradiction of its own zero-credit conclusion, and used 25% of rent capped at $1,460 instead of the $1,440 of property taxes accrued (20% of $7,200); the correct formula, 80% x ($1,440 - 8.785% x ($20,590 - $8,060)), gives $271.39." -us,scenario_108,state_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $10,392 survivor benefits on top of wages and Social Security to reach $30,982 of homestead household income and killed the credit on the ~$24,680 limit. Schedule H household income is $7,656 of wages plus $12,934 of tax-exempt Social Security = $20,590, which leaves 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It computed the homestead household income base exactly right at $20,590 ($7,656 wages + $12,934 Social Security) and then excluded the credit by classifying it as 'a property tax relief mechanism, not a refundable income tax credit.' The Wisconsin homestead credit is a refundable individual income tax credit claimed on Schedule H and paid regardless of tax liability, so it belongs in state_refundable_credits at 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It inverted the household-income composition, dropping the $12,934 of tax-exempt Social Security that Schedule H counts in full while counting the $10,392 survivor stream that stays out, giving $18,048 instead of $20,590. It also stopped at 'estimated roughly $300' rather than evaluating the statutory reduction, 80% x ($1,440 - 8.785% x (household income - $8,060)), which yields $271.39 on the correct base." -us,scenario_108,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $10,392 survivor benefits alongside wages and the $12,934 of Social Security to reach $30,982 and rejected the credit against the $24,680 income limit. Schedule H household income is $20,590, and the 85-year-old disabled renter clears both the age/disability gate and the income ceiling, earning 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It used a household income of 'about $18,000,' which excludes the $12,934 of tax-exempt Social Security that Wisconsin counts in full on Schedule H while retaining the $10,392 survivor stream that is excluded; the correct base is $20,590. It then rounded to $371 instead of applying the reduction, 80% x ($1,440 property taxes accrued - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It summed wages, Social Security survivor benefits, and the separate $10,392 survivor benefits into $30,982 of homestead household income and disqualified the household at the $24,680 cap. The Schedule H base is $7,656 + $12,934 = $20,590, so the credit computes to 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It tied Wisconsin refundable credits to qualifying children and dismissed the homestead credit as 'already captured elsewhere,' never treating it as a state refundable income tax credit. The homestead credit is the credit built for exactly this claimant: age 62+ or disabled, $7,200 of rent giving $1,440 of property taxes accrued, and $20,590 of household income under the $24,680 limit, producing $271.39." -us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It treated every listed income stream, including the $10,392 survivor benefits, as homestead household income and concluded the household was over the limit. Schedule H household income is $7,656 of wages plus $12,934 of tax-exempt Social Security = $20,590, below the $24,680 ceiling, giving 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It stated homestead household income as $30,982, which requires stacking the $10,392 survivor benefits onto wages and the $12,934 of Social Security. The correct Schedule H base is $20,590, leaving the household eligible for 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It ruled the household over the homestead income limit by counting the $10,392 survivor benefits along with the $7,656 of wages and $12,934 of Social Security. Wisconsin household income here is $20,590, under the $24,680 cap, so the 85-year-old disabled renter receives 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It defined homestead household income as 'AGI plus Social Security' and got $30,982, an amount reachable only by including the $10,392 survivor benefits in AGI. The Schedule H base is $7,656 + $12,934 = $20,590, well under $24,680, yielding 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It asserted blanket ineligibility for Wisconsin refundable credits without applying any homestead rule. The head is 85 and disabled, satisfying the Schedule H claimant test, pays $7,200 of rent giving $1,440 of property taxes accrued, and has $20,590 of household income under the $24,680 limit, so the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It put total household income at $30,982 by adding the $10,392 survivor benefits to wages and Social Security and failed the household on the homestead income limit. Schedule H household income is $20,590, which produces 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It compared $30,982 of income against the $24,680 homestead limit, an income figure that double-counts the $10,392 survivor benefits on top of the $12,934 of Social Security and $7,656 of wages. The correct base of $20,590 clears the limit and produces 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It declared the head ineligible for the homestead credit without testing the Schedule H conditions. Age 85 with a disability satisfies the claimant requirement, $7,200 of rent converts to $1,440 of property taxes accrued, and $20,590 of household income is under the $24,680 ceiling, giving a credit of $271.39." -us,scenario_108,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used $30,982 of household income against the $24,680 homestead threshold, which counts the separate $10,392 survivor benefits that Schedule H household income excludes. On the correct $20,590 base the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It concluded household income exceeds the homestead income limit by summing all three listed income streams, including the $10,392 survivor benefits. Schedule H household income is wages of $7,656 plus tax-exempt Social Security of $12,934 = $20,590, under the $24,680 cap, producing $271.39." -us,scenario_108,state_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It failed the household against the $24,680 homestead limit using an income total that adds the $10,392 survivor benefits to wages and Social Security, and then pivoted to the childless-EITC rule that has no bearing on the homestead credit. Household income of $20,590 leaves the 85-year-old disabled renter with 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It computed $30,982 of household income by stacking the $10,392 survivor benefits onto the $7,656 of wages and $12,934 of Social Security and rejected the homestead credit on the income limit. The Schedule H base is $20,590, giving 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $10,392 survivor benefits as pension income inside homestead household income to reach ~$30,982 and disqualified the household at ~$24,780. That stream sits outside Wisconsin household income, which is $7,656 + $12,934 = $20,590, so the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It reported that no Wisconsin refundable credit is triggered by the listed facts, skipping the homestead credit entirely. The facts that trigger it are all present: age 85, disability, $7,200 of pre-subsidy rent giving $1,440 of property taxes accrued, and $20,590 of household income below the $24,680 limit, for a credit of $271.39." -us,scenario_108,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It searched only for 'refundable EITC equivalents' and concluded nothing applies, never evaluating Wisconsin's renter-based homestead credit. That credit pays this 85-year-old disabled renter 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It disqualified the household by including both the Social Security and the separate $10,392 survivor benefits in homestead household income. Schedule H household income is $7,656 of wages plus $12,934 of tax-exempt Social Security = $20,590, under the $24,680 limit, producing $271.39." -us,scenario_108,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It claimed Wisconsin offers no refundable individual income tax credit for an elderly taxpayer without qualifying children. The homestead credit is precisely that credit, open to claimants age 62+ or disabled with household income under $24,680; at $20,590 of income and $1,440 of property taxes accrued on $7,200 of rent it pays $271.39." -us,scenario_108,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It placed the household above the homestead credit income limit by treating all listed benefit income, including the $10,392 survivor benefits, as household income. The Schedule H base is $20,590, which is under the $24,680 ceiling and yields 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It gave a blanket 'no refundable credit eligibility' for a one-adult household without applying any Wisconsin rule. The homestead credit turns on renter status and age/disability, both satisfied here, and pays $271.39 on $1,440 of property taxes accrued against $20,590 of household income." -us,scenario_108,state_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It compared $30,982 of total income against a '$24,480' homestead ceiling, and that income total includes the $10,392 survivor benefits that Wisconsin household income excludes. On the correct $20,590 base the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It asserted that no state refundable credit qualifies without naming or testing any rule. The Wisconsin homestead credit qualifies this 85-year-old disabled renter: $7,200 of rent gives $1,440 of property taxes accrued and household income of $20,590 is under the $24,680 limit, so the credit is $271.39." -us,scenario_108,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used ~$30,982 of household income against a ~$25,000 limit, an amount that adds the $10,392 survivor benefits to wages and Social Security. Schedule H household income is $20,590, leaving 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It stated homestead household income 'including SS and survivor benefits is about 30982' and put the household over the limit. Only the $7,656 of wages and $12,934 of tax-exempt Social Security enter Wisconsin household income, giving $20,590 and a credit of 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It computed 'AGI + SS = 30982,' which absorbs the separate $10,392 survivor benefits into the homestead income base, and compared it to an inflated ~$26,500 limit. The correct base of $20,590 is under the $24,680 limit and produces $271.39." -us,scenario_108,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It evaluated only the Wisconsin earned income credit and stopped, never considering the homestead credit as a state refundable credit. The homestead credit applies here on age 85 and disability, with $1,440 of property taxes accrued from $7,200 of rent and $20,590 of household income, paying $271.39." -us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for state_refundable_credits, so no computation reached the grader. The required derivation is the Wisconsin homestead credit for this 85-year-old disabled renter: 80% x ($1,440 of property taxes accrued - 8.785% x ($20,590 of household income - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It applied a 'broad household income' of about $30,982 to the homestead income limit, folding in the $10,392 survivor benefits alongside the $12,934 of Social Security. Wisconsin household income is $20,590, under the $24,680 limit, giving 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It reasoned that zero Wisconsin income tax liability implies zero refundable credits. A refundable credit is paid without regard to liability, and the homestead credit is filed on Schedule H by claimants with no tax owed, paying this household 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It built household income of about $30,982 from wages, Social Security survivor benefits, and the separate $10,392 survivor benefits, then failed the household at a ~$25,000 limit. Only the $7,656 of wages and $12,934 of tax-exempt Social Security count, giving $20,590 and a credit of $271.39." -us,scenario_108,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It explicitly summed $7,656 + $12,934 + $10,392 = $30,982 as homestead household income and disqualified the household at the ~$24,680 threshold. The $10,392 survivor stream is outside Wisconsin household income, so the base is $20,590 and the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39." -us,scenario_108,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It conditioned Wisconsin refundable credits on the presence of qualifying children and returned zero. The homestead credit has no child requirement; it pays renters age 62+ or disabled, and here delivers 80% x ($1,440 of property taxes accrued - 8.785% x ($20,590 - $8,060)) = $271.39." +us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income test (~$1,305) to a net income of ~$2,217 and declared the household ineligible. It missed that Wisconsin's TANF non-cash broad-based categorical eligibility waives the net income test. An eligible one-person household whose formula benefit is negative gets the $24 monthly minimum, or $288 a year." +us,scenario_108,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income of ~$2,214 exceeded the 100% FPL net limit. It ignored Wisconsin's broad-based categorical eligibility, which removes the net income test. The categorically eligible one-person household gets the $24 monthly minimum allotment ($288 a year)." +us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It tested countable income against a 130% FPL limit and declared the household ineligible. But elderly/disabled households are not subject to the 130% gross test. Wisconsin's BBCE gross limit is 200% FPL (~$2,608), which the $2,582 gross income passes. The categorical eligibility that follows yields the $24 monthly minimum benefit, $288 a year." +us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"Its own arithmetic showed max allotment minus 30% of net income was negative, and it reached a minimum-benefit answer of $23/month ($276/yr). It then discarded that for an unsupported $2,227 that none of its computations produce. The correct result is the FY2026 minimum of $24/month, $288/yr." +us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Its reasoning applied the 100% FPL net income test (~$1,304) to ~$2,186 net income and concluded the household is ineligible. It never recognized that Wisconsin's BBCE waives that test. It then submitted $2,376, which contradicts its own ineligibility conclusion. The categorically eligible household gets the $24 monthly minimum, $288/yr." +us,scenario_108,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly landed on a minimum-level benefit but used the FY2025 minimum allotment of $23/month ($276/yr). The FY2026 minimum for a one-person household is $24/month ($288/yr). It also overstated the excess medical deduction as ~$465/month. The monthly medical cost is $66.67, so the deduction is only $31.67." +us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It asserted that Wisconsin has no broad-based categorical eligibility and applied the 100% FPL net income test to $2,224 net income, concluding the household is ineligible. Wisconsin does confer categorical eligibility through TANF non-cash benefits, which removes the net income test. The household therefore gets the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It claimed an excess shelter deduction that brought net income down to ~$1,700 and yielded $164/month. But $600 rent is below half of adjusted income, so the shelter deduction is $0. Net income stays near $2,036, and 30% of it ($611) exceeds the $298 maximum, so only the $24 minimum ($288/yr) is paid." +us,scenario_108,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It treated the 100% FPL net income test as binding for the elderly/disabled household and returned $0. Wisconsin's BBCE via TANF non-cash waives that test. The eligible one-person household whose formula benefit is negative receives the $24 monthly minimum, $288/yr." +us,scenario_108,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"It declared the household ineligible because ~$2,184 net income exceeds the 100% FPL net income limit. It missed that Wisconsin's broad-based categorical eligibility removes the net income test. The categorically eligible one-person household receives the $24/month minimum, $288/yr." +us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It stopped at a negative formula benefit and returned $0. It missed that the household is categorically eligible through Wisconsin's TANF non-cash BBCE, and that eligible one- and two-person households get a minimum allotment. The minimum is $24/month, or $288/yr." +us,scenario_108,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It reasoned that 30% of net income exceeds the one-person maximum allotment and set the benefit to $0. It omitted the minimum allotment that SNAP pays to eligible one- and two-person households. This household is categorically eligible through Wisconsin BBCE, so it receives $24/month ($288/yr)." +us,scenario_108,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit for elderly/disabled households and returned $0. Wisconsin's broad-based categorical eligibility waives the net income test for this household. The benefit is therefore the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It said gross income exceeds the limits and that 30% of net income exceeds the maximum allotment, then set the benefit to $0. It missed two things: gross income of $2,581.83 is under Wisconsin's 200% FPL BBCE limit, and categorically eligible one-person households get the $24/month minimum ($288/yr)." +us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared income above the SNAP eligibility threshold without further analysis. The $2,581.83 gross income passes Wisconsin's 200% FPL BBCE limit for this elderly/disabled household. Categorical eligibility waives the net income test, so the household gets the $24/month minimum allotment, $288/yr." +us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It asserted that both gross and net income exceed SNAP limits. It never applied Wisconsin's broad-based categorical eligibility: the 200% FPL gross limit, which the $2,581.83 gross income passes, and no net income test. Under that pathway the one-person household receives the $24 monthly minimum, $288/yr." +us,scenario_108,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly found categorical eligibility through Wisconsin's 200% FPL BBCE and correctly applied the minimum benefit. But it used the FY2025 minimum of $23/month ($276/yr). The FY2026 minimum allotment for a one-person household is $24/month, or $288/yr." +us,scenario_108,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It asserted a zero benefit from income and resources. The $270 in assets pass the asset test, and the $2,581.83 gross income passes Wisconsin's 200% FPL BBCE limit, so the household is categorically eligible. Because the formula benefit is negative, it receives the $24/month minimum, $288/yr." +us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"It treated $2,581.83 of monthly gross income as over the SNAP limits and set the benefit to $0. That income is within Wisconsin's 200% FPL BBCE gross limit, and categorical eligibility removes the net income test. When the formula benefit goes negative, the minimum allotment of $24/month ($288/yr) applies." +us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It applied a net income threshold to rule the household out. Wisconsin's TANF non-cash categorical eligibility waives the net income test for this household. The eligible one-person household receives the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It correctly found that 30% of net income exceeds the one-person maximum allotment, but then set the benefit to $0. It omitted the minimum allotment for eligible one- and two-person households. This household qualifies through Wisconsin BBCE, so the benefit is $24/month, $288/yr." +us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,"It computed max allotment minus 30% of ~$2,196 net income as negative and returned $0. It never applied the minimum allotment for categorically eligible one-person households. With Wisconsin BBCE eligibility, that minimum is $24/month, $288/yr." +us,scenario_108,snap,glm-5.3,llm_error,categorical_eligibility,False,"It ruled the household out because net income is above the 100% FPL net limit. It also wrongly described the shelter deduction as capped, although it is uncapped for elderly/disabled households. Wisconsin's BBCE waives the net income test, so the household gets the $24 monthly minimum, $288/yr." +us,scenario_108,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted a substantial benefit of $1,172/yr without computing net income. After the earned income, standard, and medical deductions, and with no excess shelter deduction, net income is about $2,036. Thirty percent of that ($611) exceeds the $298 maximum, so the benefit is only the $24 monthly minimum, $288/yr." +us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It set SNAP to $0 on the basis of the household's income without applying Wisconsin's broad-based categorical eligibility. That pathway passes the household at 200% FPL gross and waives the net income test. The categorically eligible one-person household receives the $24/month minimum allotment, $288/yr." +us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It concluded that income is too high to produce a positive allotment and returned $0. It omitted the minimum allotment that SNAP guarantees to eligible one- and two-person households. The household is categorically eligible through Wisconsin BBCE, so it receives $24/month, $288/yr." +us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It produced $103/month by overstating the deductions, apparently crediting the $600 rent as a shelter deduction. But rent is below half of adjusted income, so the excess shelter deduction is $0. Net income is about $2,036, 30% of it ($611) exceeds the $298 maximum, and only the $24 minimum ($288/yr) is paid." +us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It declared the household ineligible because net income exceeds the one-person elderly/disabled net income limit. Wisconsin's TANF non-cash categorical eligibility waives the net income test. The household therefore gets the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It applied the elderly/disabled net income limit to rule the household ineligible. Wisconsin's broad-based categorical eligibility removes the net income test. The eligible one-person household's negative formula benefit is raised to the $24 monthly minimum, $288/yr." +us,scenario_108,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It ruled the household out on the SNAP net income limit. Wisconsin's BBCE via TANF non-cash benefits makes this household categorically eligible without a net income test. Its benefit is the $24/month minimum allotment, $288/yr." +us,scenario_108,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It concluded that net income is too high for a one-person allotment and returned $0. It skipped the minimum allotment that SNAP pays to eligible one- and two-person households. The household is categorically eligible through Wisconsin BBCE, so it gets $24/month, $288/yr." +us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"It stated that income exceeds SNAP limits after deductions and returned $0. Wisconsin's broad-based categorical eligibility waives the net income test, and gross income is within its 200% FPL limit. The household receives the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"It compared ~$28,000 of annual net income with the 100% FPL net income limit and returned $0. That test does not apply to a household categorically eligible through Wisconsin's TANF non-cash BBCE. The benefit is the $24/month minimum, $288/yr." +us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"It waived the gross test for the elderly/disabled household but failed it on the 100% FPL net income test. Wisconsin's BBCE removes that net income test. The eligible one-person household therefore receives the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,grok-4.7,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income exceeds the one-person net income limit. It missed Wisconsin's broad-based categorical eligibility, which removes that test. The minimum allotment of $24/month ($288/yr) then applies." +us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"It computed ~$2,218 net income against the ~$1,305 100% FPL net limit and returned $0. It did not recognize that Wisconsin's TANF non-cash categorical eligibility waives the net income test. The eligible household receives the $24/month minimum, $288/yr." +us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,"It failed the household because ~$2,225 net income exceeds the ~$1,300 100% FPL limit. Wisconsin's broad-based categorical eligibility waives the net income test. The eligible household gets the $24 monthly minimum allotment, $288/yr." +us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit for elderly/disabled households and declared the household ineligible. Wisconsin's TANF non-cash BBCE makes the household categorically eligible without that test. The benefit is the $24/month minimum allotment, $288/yr." +us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"It correctly found that 30% of ~$2,214 net income far exceeds the maximum allotment, but then set the benefit to $0. It omitted the minimum allotment for eligible one- and two-person households. With categorical eligibility through Wisconsin BBCE, the household receives $24/month, $288/yr." +us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross test, which elderly/disabled households are exempt from, and the 100% FPL net test to rule the household out. It also left out the earned income deduction. Wisconsin's BBCE sets a 200% FPL gross limit, which the $2,581.83 gross income passes, and has no net test. The eligible household gets the $24/month minimum, $288/yr." +us,scenario_108,snap,ox-alpha,llm_error,categorical_eligibility,False,"It applied a 165% FPL gross income test, the separate-purchase-and-preparation rule for elderly/disabled members, and ruled the household ineligible. The household is categorically eligible through Wisconsin's BBCE, whose 200% FPL gross limit (~$2,608) the $2,582 gross income passes. It gets the $24/month minimum, $288/yr." +us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Its reasoning noted that a contribution above the maximum allotment means SNAP 'would be minimum,' but its final answer treated the household as not eligible and submitted $0. Along the way it also invented a ~$400 utility allowance that the facts do not support. Categorical eligibility through Wisconsin BBCE yields the $24/month minimum allotment, $288/yr." +us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It declared countable income above the one-person SNAP limit and returned $0. Wisconsin's broad-based categorical eligibility applies a 200% FPL gross limit, which this household passes, and no net income test. The eligible household receives the $24 monthly minimum allotment, $288/yr." +us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model stopped at a gross tax of $134.33 (3.5% of $3,838) and never subtracted Wisconsin's nonrefundable renter school property tax credit. On the $7,200 rent that credit is 12% of $1,440-$1,800, or $172.80-$216, which wipes out the tax. It also used a made-up $270 age deduction instead of Wisconsin's $700 personal exemption plus the $250 exemption for age 65 and over." +us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model said outright that there were 'no other nonrefundable credits', which skips the renter school property tax credit. That credit is 12% of 20-25% of the $7,200 rent, or $172.80-$216, and it takes the $161.71 gross tax to zero. It also used an old 3.54% bottom rate instead of the current 3.50% and left out the $700 and $250 exemptions." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the bottom rate to about $4,548 of taxable income and reported the $159 gross tax as the final answer. It never subtracted the nonrefundable renter school property tax credit ($172.80-$216 on $7,200 rent), which brings Wisconsin tax before refundable credits to $0." +us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model did apply the renter school property tax credit but made it too small, leaving $70 of tax. The credit is 12% of rent treated as property tax (20-25% of the $7,200 rent, or $1,440-$1,800), which is $172.80-$216. That is more than the roughly $110-$150 gross tax, so the correct liability is zero." +us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model worked from an outdated $12,760 standard deduction with a phaseout and a $1,400 exemption, reached $146.45 of gross tax, and stopped. It never applied the nonrefundable renter school property tax credit ($172.80-$216 on $7,200 rent), which reduces the tax to $0." +us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model's reasoning covers only the standard deduction and personal exemption, and it reported $144.13 of gross tax at the bottom bracket as the answer. It left out the nonrefundable renter school property tax credit (12% of 20-25% of the $7,200 rent, or $172.80-$216), which is larger than that tax and brings it to $0." +us,scenario_108,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model taxed $3,538 at 3.5% to get $124 and stopped. It never subtracted Wisconsin's nonrefundable renter school property tax credit, which is $172.80-$216 on $7,200 of rent and is larger than $124, so the tax before refundable credits is $0." +us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model said outright that there were 'no nonrefundable state credits', so it skipped the renter school property tax credit ($172.80-$216 on $7,200 rent) that takes its $150 tax to zero. It also used an old 3.54% bottom rate instead of 3.50% and left out the $700 and $250 exemptions." +us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to grade. The correct derivation takes a small gross Wisconsin tax and applies the nonrefundable renter school property tax credit, which brings it to $0." +us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 in homestead income and concluded the credit was zero. It then submitted an unsupported $636 'partial credit', using 25% of rent capped at $1,460 instead of the 20% rent factor ($1,440). With homestead income of wages plus Social Security ($20,590), the formula 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) gives $271.39." +us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits in Wisconsin homestead household income, reaching $30,982, and zeroed the credit at the $24,680 limit. Homestead income here is wages plus nontaxable Social Security, or $20,590. That is under the cap and yields a $271.39 credit on $1,440 of rent-based property taxes." +us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It computed the correct $20,590 household income but dismissed the Wisconsin homestead credit as 'property tax relief, not a refundable income tax credit'. The homestead credit is a refundable credit claimed with the Wisconsin return (Schedule H) and counts toward state refundable credits. Applying it gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It used the correct $1,440 rent-based property tax but built household income as wages plus survivor benefits ($18,048). It dropped the $12,934 of Social Security, which homestead income includes, and added survivor benefits, which it does not. It also eyeballed $300 instead of applying 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security ($30,982 total) and ruled the homestead credit out at the $24,680 limit. Homestead household income is $7,656 + $12,934 = $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It used a household income of about $18,000, which matches wages plus survivor benefits. That base excludes the $12,934 of Social Security that homestead income includes and wrongly counts the survivor benefits. Its $371 does not follow from any consistent formula. The correct $20,590 base gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It set homestead household income at $30,982 by including the $10,392 survivor benefits, so the household appeared to exceed the $24,680 limit. Homestead income is wages plus Social Security ($20,590), which yields a $271.39 credit." +us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It explicitly added 'other survivor benefits ($10,392)' to wages and Social Security, reaching $30,982, and zeroed the homestead credit. Homestead household income excludes that input and equals $20,590. That is below the cap and produces 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It named the homestead credit but never computed it, claiming no refundable-credit trigger without children. An 85-year-old disabled renter with $7,200 of rent and $20,590 of homestead income qualifies. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used the correct $20,590 income but set rent-based property taxes at 25% of rent capped at $1,460, instead of 20% of rent ($1,440). It also used an 8.65% reduction rate above $8,000 instead of 8.785% above $8,060. Its 0.8 × (1,460 − 0.0865 × 12,590) = $297 should be 0.8 × ($1,440 − 0.08785 × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It declared household income above the homestead limit, which requires adding the $10,392 survivor benefits. Homestead household income is wages plus Social Security ($20,590), under the $24,680 cap. The credit is $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income by counting the $10,392 survivor benefits and zeroed the credit. The correct homestead income is $7,656 + $12,934 = $20,590. That gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It concluded household income exceeds the homestead limit by folding in the $10,392 survivor benefits. Homestead income counts only wages and Social Security here ($20,590), which is below $24,680. That produces a $271.39 credit." +us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It returned $0 with no analysis and never applied the Wisconsin homestead credit. The claimant is 85, disabled, pays $7,200 in rent and has $20,590 of homestead income. Those facts yield 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as part of AGI, getting $30,982 of 'AGI plus Social Security', and applied the $24,680 cutoff. AGI is only the $7,656 of wages, so homestead income is $20,590. The credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It asserted no Wisconsin refundable-credit eligibility without testing the homestead credit. An elderly, disabled renter with $20,590 of homestead income and $1,440 of rent-based property taxes receives $271.39." +us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It computed $30,982 of household income by including the $10,392 survivor benefits and ruled out the homestead credit. Homestead income is wages plus Social Security ($20,590), under the limit. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, reaching $30,982 against the $24,680 limit. Homestead household income is $20,590 (wages plus Social Security), so the credit is $271.39 rather than $0." +us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It declared the head ineligible for the homestead credit without any income test. The head is 85 and disabled, pays rent, and has $20,590 of homestead income, below $24,680. The credit is $271.39." +us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It used $30,982 of total household income, including the $10,392 survivor benefits, and applied the $24,680 threshold. Homestead income is only the $7,656 of wages plus $12,934 of Social Security, or $20,590. That yields $271.39." +us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Its 'income exceeds the homestead limit' conclusion depends on adding the $10,392 survivor benefits. Homestead household income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It placed household income above the $24,680 homestead limit by counting the survivor benefits. Homestead income is wages plus Social Security ($20,590), which yields a $271.39 credit on $1,440 of rent-based property taxes." +us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It used $30,982 of household income by adding the $10,392 survivor benefits and zeroed the homestead credit. The correct homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits as pension income, reaching $30,982 against a ~$24,780 limit. Homestead household income here is wages plus Social Security only ($20,590). The credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It found no Wisconsin refundable credit and never considered the homestead credit. The 85-year-old disabled renter with $7,200 of rent and $20,590 of homestead income receives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It looked only for EITC-like credits and omitted the refundable Wisconsin homestead credit. With rent-based property taxes of $1,440 and homestead income of $20,590, that credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It included the listed survivor benefits with Social Security and wages, which pushed household income over the homestead limit. Homestead income excludes the $10,392 survivor-benefits input and equals $20,590. The credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It concluded no refundable credit applies to an elderly taxpayer without children and overlooked the homestead credit. That credit does not require children and pays $271.39 on $1,440 of rent-based property taxes and $20,590 of household income." +us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It placed income above the homestead limit, which reflects counting the $10,392 survivor benefits. Homestead household income is $7,656 + $12,934 = $20,590. That gives 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It found no refundable-credit eligibility for a one-adult household and never evaluated the homestead credit. The head is 85, disabled and renting, with $20,590 of homestead income, which yields $271.39." +us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It used $30,982 of total income, including the $10,392 survivor benefits, against a $24,480 ceiling. Homestead household income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It ruled out the homestead credit on income by counting the survivor benefits. Homestead income is wages plus Social Security ($20,590), below $24,680. The credit is $271.39." +us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It treated 'total household income', including the $10,392 survivor benefits, as the homestead income base. The base is $20,590, and the formula 0.8 × ($1,440 − 8.785% × $12,530) gives $271.39." +us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It concluded total household income exceeds the homestead limit by including the $10,392 survivor benefits. Homestead income is $7,656 + $12,934 = $20,590, which is under the cap. The credit is $271.39." +us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It found no qualifying state refundable credit and skipped the Wisconsin homestead credit. The household qualifies with $1,440 of rent-based property taxes and $20,590 of homestead income, for a $271.39 credit." +us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used ~$30,982 of household income by adding the $10,392 survivor benefits and compared it to a ~$25,000 limit. Homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It counted Social Security and the survivor benefits for $30,982 of homestead income and zeroed the credit. The $10,392 survivor-benefits input is not homestead income, so income is $20,590. The credit is $271.39." +us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly summed wages, Social Security survivor benefits and other survivor benefits to exceed the homestead limit. Homestead household income excludes the $10,392 survivor benefits, giving $20,590 and a credit of $271.39." +us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It treated the survivor benefits as AGI, computing 'AGI + SS = $30,982' against a ~$26,500 limit. AGI is only the $7,656 of wages, so homestead income is $20,590. The credit is 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It evaluated only the Wisconsin EITC and never applied the refundable homestead credit. For this 85-year-old disabled renter with $20,590 of homestead income, that credit is $271.39." +us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_refundable_credits, so there is no answer to score against the $271.39 homestead credit." +us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a 'broad' household income of $30,982 that includes the $10,392 survivor benefits and ruled out the homestead credit. Homestead income is wages plus Social Security ($20,590), yielding $271.39." +us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,"It reasoned that zero Wisconsin tax liability means no refundable credits. Refundable credits are paid regardless of liability, and the homestead credit here is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39." +us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits along with wages and Social Security ($30,982) and ruled out the homestead credit. Homestead income is $20,590, below the $24,680 cap. The credit is $271.39." +us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It explicitly added the $10,392 survivor benefits to wages and Social Security survivor benefits for $30,982 and applied the $24,680 limit. Homestead household income is $20,590. That yields 0.8 × ($1,440 − 8.785% × $12,530) = $271.39." +us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It reasoned only from the lack of qualifying children and omitted the homestead credit, which needs no children. An 85-year-old disabled renter with $20,590 of homestead income receives $271.39." us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP." us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP." us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band." @@ -7785,42 +8511,46 @@ us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_elig us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified the infant category and computed income at roughly 0.79 FPL against a ~200% FPL Florida infant limit, then reversed its own conclusion by imposing a Florida asset test and requiring Medicaid expansion or confirmed enrollment. MAGI-based children's categories carry no asset or resource test, non-expansion status restricts only the adult expansion group, and CHIP is residual coverage for children who fail Medicaid rather than a substitute for it — so the infant category grants eligibility, and the question asked for eligibility rather than enrollment." us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed WIC status as disqualifying instead of computing eligibility from the supplied facts. The age-zero child satisfies WIC's child category, and the five-person household's income is below the 185%-of-poverty income limit, so the correct eligibility output is Yes." us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an overstated projected standard deduction, incorrectly concluded taxable income and pre-credit tax were zero, and then subtracted an invented $284 of “unused” CTC. The correct computation produces income tax that uses $2,792.91 of the $6,600 CTC as a nonrefundable offset, while the remaining $3,807.09 is refundable; unused nonrefundable credits never make this output negative." -us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the answer exactly right in its own reasoning — EITC $8,231 plus ACTC 0.15 × ($27,880 − $2,500) = $3,807, summing to $12,038 — and then discarded that result by rounding the submitted value to $12,000. The $38 error is purely a self-inflicted rounding of a correct computation." -us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model zeroed out the EITC by asserting the household's income exceeded the MFJ three-child phase-out range 'approximately $50,162', when $27,880.57 of earned income is below the phase-out start and yields the full $8,231 maximum. It then computed the ACTC correctly at 15% × ($28,350 − $2,500) = $3,877.50 but submitted an unexplained $2,500 instead, discarding both its own ACTC figure and the entire $8,231 EITC." -us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model reached the correct components in its scratch work — EITC $8,231 and ACTC min($5,100, 0.15 × $25,380) = $3,807, and even wrote 'sum: 8,231+3,807 = 12,038' — then submitted $10,379, a figure matching neither that sum nor its stated fallback of $11,853. The failure is abandonment of a correct derivation at the submission step." -us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the stale 2025 three-child EITC maximum of $8,046 instead of the 2026 value of $8,231, and computed the ACTC base as net SE earnings $27,705 (the 92.35% figure) rather than earned income net of half the SE tax, $27,880.57, yielding $3,781 instead of $3,807.09. It then submitted an ACTC of $4,416 that appears nowhere in its own arithmetic, inflating the total to $12,462." -us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model placed the household in 'early phase-out' and cut the EITC to $5,035, when earned income of $27,880.57 is below the 2026 MFJ three-child phase-out start and the full $8,231 maximum applies. It also used gross $30,000 as the ACTC base, giving 0.15 × $27,500 = $4,125 rather than the correct $3,807.09 computed from earned income net of the deductible half of SE tax." -us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied the EITC phase-out against gross receipts of $30,000 using a 2025-vintage MFJ threshold of $26,511, subtracting roughly $735–$945; the correct earned income of $27,880.57 (gross SE income less half the SE tax) is below the 2026 MFJ phase-out start, so the undiminished $8,231 applies. It then submitted $7,594 — an EITC-only figure — after computing a total of $11,594 in its own reasoning, dropping the entire $3,807.09 refundable CTC from the answer." -us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model asserted an EITC of 'approximately $3,015' for a joint filer with three children at $27,881 of earned income by invoking a phase-out that does not reach this household; the full 2026 maximum of $8,231 applies. It compounded this by treating the ACTC as the full $1,700 × 3 = $5,100 per-child cap while acknowledging the 15%-of-excess-earnings limit of $3,807, when that earned-income limit binds and caps the refundable CTC at $3,807.09." -us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model computed the refundable CTC as 15% × ($30,000 − $2,500) = $4,125 using gross self-employment receipts, failing to reduce earned income by the deductible half of self-employment tax ($2,119.43), which gives the correct base of $27,880.57 and an ACTC of $3,807.09. Its EITC of $8,215 also undershot the 2026 three-child maximum of $8,231." -us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a flat refundable CTC of $3,000 — the obsolete $1,000-per-child refundable amount — instead of the statutory 15% of earned income above $2,500, which yields $3,807.09 here and sits below the $1,700-per-child refundable cap. Its EITC of $8,026 also fell short of the 2026 three-child maximum of $8,231." -us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model built the answer correctly — full-plateau EITC plus 15% × ($27,881 − $2,500) = $3,807 refundable CTC — but overstated the 2026 MFJ three-child EITC maximum as $8,261 rather than $8,231, a $30 uprating error that accounts for essentially the entire $29.91 shortfall from $12,038.09." -us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model treated the Additional Child Tax Credit as a flat $1,000 per child ($3,000), the pre-2018 refundable amount, instead of 15% of earned income above $2,500, which yields $3,807.09 against a $1,700-per-child cap. Its EITC estimate of $8,290 also exceeded the 2026 three-child maximum of $8,231." -us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model reported an EITC of $3,588 — less than half the $8,231 that a joint filer with three children and $27,880.57 of earned income receives at the plateau — and a refundable CTC of $1,674 in place of 15% × ($27,880.57 − $2,500) = $3,807.09. Both figures are consistent with applying a steep phase-out that does not reach this income level and with ignoring the earned-income ACTC formula." -us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model paired a roughly correct maximum EITC ($8,250 vs. $8,231) with a flat $3,000 Additional Child Tax Credit at $1,000 per child, rather than the 15%-of-earnings-above-$2,500 formula that yields $3,807.09 for earned income of $27,880.57. The $807 ACTC shortfall is the bulk of its error." -us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a maximum ACTC of $1,000 per child ($3,000 total), an obsolete refundable cap; the 2026 refundable CTC is 15% of earned income above $2,500, capped at $1,700 per child, giving $3,807.09 here. Its EITC of $8,240 also missed the $8,231 maximum." -us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"The model gave no derivation, only naming the CTC and EITC as the components, and submitted $7,502 — a figure below even the $8,231 EITC alone, so it accounts for neither the full three-child EITC nor any refundable CTC. The correct build is $8,231 of EITC plus 15% × ($27,880.57 − $2,500) = $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a flat $3,000 refundable Additional Child Tax Credit ($1,000 per child) instead of 15% of earned income above $2,500, which yields $3,807.09 for earned income of $27,880.57 and stays under the $1,700-per-child refundable cap. Its EITC of $8,219 also fell $12 short of the 2026 three-child maximum." -us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model gave no component breakdown, and its $11,046 total is consistent with the stale 2025 three-child EITC maximum of $8,046 plus a flat $3,000 refundable CTC at $1,000 per child. The correct figures are the 2026 maximum EITC of $8,231 and a refundable CTC of 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model explicitly applied '$1,000 per child' refundability for a $3,000 Additional Child Tax Credit, when the 2026 refundable CTC is 15% of earned income over $2,500 — $3,807.09 here, below the $1,700-per-child cap. Its EITC of $8,207 also undershot the $8,231 maximum." -us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a 2026 EITC maximum of $8,025.75, essentially the stale 2025 three-child figure, rather than $8,231, and computed the ACTC from net SE earnings of $27,705 (the 92.35% amount) instead of earned income net of half the SE tax, $27,880.57, giving $3,780.75 rather than $3,807.09. Both understatements combine to the $231.59 shortfall." -us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model applied the self-employment adjustment twice, taking 92.35% of $30,000 to get $27,705 and then subtracting the deductible half of SE tax again to reach an earned income of $25,586; earned income for both credits is $30,000 − $2,119.43 = $27,880.57. That double deduction drove its ACTC to $3,463 instead of $3,807.09, and it also used a stale $8,042 EITC maximum in place of $8,231." -us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model performed no computation, stating only that refundable child-related credits 'dominate,' and submitted a round $13,400 that exceeds the sum of the maximum three-child EITC ($8,231) and the full $1,700-per-child refundable CTC cap ($5,100) less the binding earned-income limit. The correct total is $8,231 plus 15% × ($27,880.57 − $2,500) = $3,807.09." -us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model refused to compute any refundable credit on the ground that no wage or employment information was given, when net self-employment earnings are earned income for both the EITC and the Additional Child Tax Credit under IRC §32(c)(2) and §24(d). That categorical exclusion dropped the entire $8,231 EITC and $3,807.09 refundable CTC." -us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model got the EITC exactly right at $8,231 but applied a refundable Additional Child Tax Credit of '$1,000 for each of three children,' the obsolete refundable amount, instead of 15% of earned income above $2,500. The correct ACTC is 0.15 × ($27,880.57 − $2,500) = $3,807.09, making the $807 difference its entire error." -us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used the correct $8,231 EITC but computed the refundable CTC as 15% × ($30,000 − $2,500) = $4,125 from gross self-employment receipts, omitting the deductible half of self-employment tax ($2,119.43). Earned income of $27,880.57 gives $3,807.09, so the missing adjustment overstates the total by $317.91." -us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model paired the correct $8,231 EITC with an ACTC of $3,781, computed as 15% of net SE earnings ($30,000 × 0.9235 = $27,705) above $2,500. The correct earned-income base is gross SE income less half the SE tax, $27,880.57, giving $3,807.09 — the 92.35% haircut belongs to the SE-tax computation, not to the ACTC earned-income base." -us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the correct $8,231 EITC but set the ACTC at $4,125, which is 15% × ($30,000 − $2,500) using gross self-employment receipts as earned income. Reducing earned income by the deductible half of SE tax ($2,119.43) gives the correct base of $27,880.57 and an ACTC of $3,807.09." -us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model reported zero refundable credits, denying both the EITC and the Additional Child Tax Credit to a joint filer with three qualifying children and $30,000 of self-employment income. Net self-employment earnings are earned income for both credits, producing $8,231 of EITC at the plateau and $3,807.09 of refundable CTC." -us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model asserted a 'post-TCJA 2026' CTC of $1,000 per child with a $3,000 refundable cap and a $3,000 earned-income floor; the 2026 refundable CTC is 15% of earned income above $2,500 with a $1,700-per-child cap, so the earned-income limit binds at $3,807.09 rather than $3,000. Its EITC of $8,226 also fell $5 short of the $8,231 maximum." -us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied a $1,000-per-child CTC with a $3,000 refundable ceiling and a $3,000 earned-income floor, rather than the 2026 rule of 15% of earned income above $2,500 capped at $1,700 per child, which yields $3,807.09 as the binding amount. Its EITC of $8,247 also overshot the $8,231 maximum." -us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped the refundable CTC at $3,000 by treating the credit as $1,000 per child and using a $3,000 earned-income floor, when the refundable amount is 15% of earned income above $2,500 up to $1,700 per child — $3,807.09 here, since the earned-income limit binds below the $5,100 cap. Its EITC of $8,245 also exceeded the $8,231 maximum." -us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model computed the refundable CTC correctly at $3,807 via 15% of earned income above $2,500, but inflated the 2026 MFJ three-child EITC plateau to 'about $8,271' instead of $8,231. That $40 uprating overshoot is the whole of its $39.91 error." -us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The model supplied no explanation, and its $6,695 is below the $8,231 EITC that a joint filer with three children and $27,880.57 of earned income receives at the plateau, so it captures neither the full EITC nor any of the $3,807.09 refundable CTC. The number is consistent with a phased-out EITC estimate submitted alone, with the Additional Child Tax Credit omitted entirely." -us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model cut the EITC to $3,588 by invoking a phase-out starting around $29,600 against gross receipts, when earned income of $27,880.57 sits on the plateau and yields the full $8,231. It then submitted $4,288, a value matching neither its own $8,688 running total nor either of its stated components." -us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model invented a '$24,000 MFJ three-child phaseout start' and reduced the EITC by 21.06% of the excess to $7,414, when $27,880.57 of earned income is below the 2026 phase-out threshold and the full $8,231 applies. It also claimed the full $1,700 × 3 = $5,100 refundable CTC while the 15%-of-earnings-above-$2,500 limit binds at $3,807.09, and its submitted figure matches neither the correct total nor the $12,514 its own reasoning stated." -us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model fabricated an MFJ three-child EITC phase-out start of $24,000–$25,500 and phased the credit down to a final $5,475, when earned income of $27,880.57 is below the 2026 threshold and the full $8,231 maximum applies. It also computed the ACTC as 15% × ($30,000 − $2,500) = $4,125 from gross receipts rather than $3,807.09 from earned income net of half the SE tax." -us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model declared 'zero additional refundable CTC because the CTC is not refundable in this estimate,' dropping the $3,807.09 Additional Child Tax Credit that IRC §24(d) makes refundable at 15% of earned income above $2,500 — which is the whole credit here, since the household's tax liability is zero. It also used the stale 2025 three-child EITC maximum of $8,046 rather than the 2026 value of $8,231." +us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"It derived the exact components: EITC $8,231 plus ACTC of 15% × ($27,880 − $2,500) = $3,807, for $12,038. It then rounded the submitted value to $12,000, throwing away the precision of its own correct computation." +us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It denied the EITC entirely by claiming $28k of income 'exceeds' a phase-out range that it said ends near $50,162. In fact $27,881 of earned income is on the plateau, so the full $8,231 applies. It also computed a refundable CTC of $3,877.50 but submitted an unsupported $2,500." +us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own recomputation reached the correct $8,231 EITC + $3,807 ACTC = $12,038. It then submitted an unexplained $10,379 that matches none of its intermediate figures." +us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2025 EITC maximum of $8,046 instead of the 2026 figure of $8,231. It first computed ACTC on the $27,705 net-SE base (0.9235 × $30,000) rather than on earned income after the half-SE-tax deduction ($27,881). It then replaced that $3,781 with an unsupported $4,416 for the submitted total." +us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated $30,000 as falling in the EITC phase-out and cut the credit to about $5,035, but $27,881 of earned income is below the MFJ phase-out start, so the full $8,231 applies. It also computed ACTC on gross $30,000 ($4,125) instead of on $27,881 net of half SE tax ($3,807)." +us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It phased down the EITC using a $26,511 threshold applied to gross $30,000, which gave about $7,469. The 2026 MFJ phase-out starts above the $27,881 of earned income, so the full $8,231 applies. It also used gross $30,000 for the ACTC ($4,125 instead of $3,807). It then submitted $7,594, which drops $4,000 from its own stated $11,594 total." +us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It counted the ACTC at the full $5,100 cap even though it had just computed the 15% earned-income limit at $3,807, which is the binding amount. It also cut the EITC to about $3,015 through a phase-out that does not apply at $27,881 of earned income, where the plateau credit is $8,231." +us,scenario_109,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It computed the ACTC as 15% of gross $30,000 above $2,500 ($4,125). Earned income for self-employment is net earnings minus the deductible half of SE tax ($27,881), which yields $3,807. It also used an EITC maximum of $8,215 instead of the 2026 figure of $8,231." +us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $3,000 refundable CTC ($1,000 per child, pre-TCJA law). In 2026 the refundable CTC is 15% of earned income over $2,500, capped at $1,700 per child, which gives $3,807 here. Its EITC of $8,026 is also below the 2026 maximum of $8,231." +us,scenario_109,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its ACTC of $3,807, computed on $27,881 of earned income, is correct. It overstated the 2026 three-child EITC maximum as $8,261 instead of $8,231, which leaves the total $30 high." +us,scenario_109,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It computed the ACTC on gross $30,000 ($4,125) without subtracting the deductible half of SE tax, which leaves $27,881 of earned income and a $3,807 ACTC. It also overstated the 2026 EITC maximum as $8,290 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a $1,000-per-child refundable CTC ($3,000) instead of the 2026 formula: 15% of ($27,881 − $2,500) = $3,807, under the $1,700-per-child cap. It also overstated the EITC maximum as $8,290 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It reduced the EITC to $3,588 as if the household were deep in the phase-out, but $27,881 of earned income is on the three-child plateau, which pays $8,231. It also understated the refundable CTC as $1,674 instead of 15% × ($27,881 − $2,500) = $3,807." +us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $1,000-per-child ACTC ($3,000) instead of the 2026 formula, 15% of earned income over $2,500 capped at $1,700 per child, which gives $3,807. It also estimated the EITC maximum at $8,250 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took the ACTC as a flat $1,000 per child ($3,000). The 2026 refundable CTC is 15% of ($27,881 − $2,500) = $3,807, under the $1,700-per-child cap. Its EITC maximum of $8,240 is also above the 2026 figure of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no computation, and its $7,502 is below the $8,231 plateau EITC alone. That means it reduced the EITC for a household whose $27,881 of earned income is below the MFJ phase-out start, and it left out the $3,807 refundable CTC." +us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $3,000 refundable CTC ($1,000 per child) instead of the 2026 ACTC formula: 15% of ($27,881 − $2,500) = $3,807, capped at $1,700 per child. It also understated the EITC maximum as $8,219 instead of $8,231." +us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $11,046 total equals the 2025 three-child EITC maximum of $8,046 plus a $1,000-per-child ACTC of $3,000. The correct 2026 figures are an EITC of $8,231 and a refundable CTC of 15% × ($27,881 − $2,500) = $3,807." +us,scenario_109,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a refundable CTC of $1,000 per child ($3,000) instead of the 2026 formula: 15% of earned income over $2,500 = $3,807, below the $1,700-per-child cap. It also used an EITC maximum of $8,207 instead of $8,231." +us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used $27,705 (0.9235 × $30,000) as earned income. For self-employment, earned income is net SE earnings minus the deductible half of SE tax: $30,000 − $2,119 = $27,881. That left its ACTC at $3,780.75 instead of $3,807. It also used an EITC maximum of $8,025.75 instead of the 2026 figure of $8,231." +us,scenario_109,federal_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It subtracted the half-SE-tax deduction from the already-reduced $27,705 net-earnings base, which double-counts the reduction and gives $25,586. Earned income is $30,000 − $2,119 = $27,881, which yields a $3,807 ACTC instead of its $3,463. It also used an EITC maximum of $8,042 instead of $8,231." +us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $13,400 matches a roughly $8,300 EITC plus the full $5,100 refundable CTC cap ($1,700 × 3). It ignored the ACTC earned-income phase-in, which limits the refundable CTC to 15% of ($27,881 − $2,500) = $3,807." +us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned zero on the grounds that there were no wages. But $30,000 of self-employment income is earned income for both the EITC and the ACTC. That qualifies the household for the $8,231 plateau EITC and a $3,807 refundable CTC." +us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It had the EITC right at $8,231 but applied a $1,000-per-child refundable CTC ($3,000). The 2026 ACTC is 15% of ($27,881 − $2,500) = $3,807, capped at $1,700 per child." +us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but ran the ACTC formula on gross $30,000 ($4,125). SE earned income must be reduced by the deductible half of SE tax to $27,881, which gives $3,807." +us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but based the ACTC on $27,705 (0.9235 × $30,000), which gave $3,781. Earned income for the credit is $30,000 minus the $2,119 deductible half of SE tax, i.e. $27,881, which yields $3,807." +us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but computed the ACTC on gross $30,000 ($4,125). It did not subtract the deductible half of SE tax, which brings earned income to $27,881 and the ACTC to $3,807." +us,scenario_109,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It had the EITC right at $8,231 but applied 15% over $2,500 to gross $30,000 ($4,125). Earned income for self-employment is net of the deductible half of SE tax ($27,881), which gives an ACTC of $3,807." +us,scenario_109,federal_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its refundable CTC of $3,807.09 is exactly right. It overstated the 2026 three-child EITC maximum as $8,427 instead of $8,231, which leaves the total $196 high." +us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It computed no refundable credits at all. Self-employment income of $30,000 is earned income, which qualifies this MFJ three-child family for the $8,231 plateau EITC and a $3,807 refundable CTC." +us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA expired and used a $1,000-per-child ACTC with a $3,000 earnings threshold. For 2026 the refundable CTC is 15% of earned income over $2,500, capped at $1,700 per child, which gives $3,807 on $27,881. Its EITC maximum of $8,226 is also slightly below $8,231." +us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It treated the 2026 CTC as $1,000 per child with a $3,000 earnings threshold, giving $3,000. The applicable ACTC is 15% × ($27,881 − $2,500) = $3,807 under a $1,700-per-child cap. It also used an EITC maximum of $8,247 instead of $8,231." +us,scenario_109,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It had the EITC right at $8,231 but explicitly assumed TCJA expired, which led it to a $1,000-per-child refundable CTC and a $3,000 earnings threshold. The 2026 rules keep the $2,500 threshold, the 15% rate and a $1,700-per-child refundable cap, which gives $3,807." +us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used pre-TCJA ACTC parameters: $1,000 per child and a $3,000 threshold, capping the credit at $3,000. The 2026 refundable CTC is 15% of ($27,881 − $2,500) = $3,807. Its EITC maximum of $8,245 is also above $8,231." +us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its ACTC of about $3,807 is correct. It overstated the 2026 three-child EITC maximum as $8,271 instead of $8,231, which leaves the total $40 high." +us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,False,"It gave no explanation, and its $6,695 is below the $8,231 plateau EITC alone. That means it reduced the EITC as if $27,881 of earned income were in the MFJ phase-out, and it did not add the $3,807 refundable CTC." +us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It assumed an MFJ phase-out start near $29,600 and cut the EITC to $3,588, but $27,881 of earned income is on the plateau and gets the full $8,231. It first set the ACTC at the full $5,100 cap, ignoring the 15% earned-income limit of $3,807, then submitted an unsupported $4,288 total." +us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It claimed the 15% earnings test supports the full $5,100 refundable CTC. In fact 15% × ($27,881 − $2,500) = $3,807, and that binds. It also phased the EITC down from a nonexistent ~$24,000 MFJ threshold to $7,414, although $27,881 is on the plateau at $8,231. Its $12,013.75 submission is not the $12,514 its own explanation states." +us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It phased the EITC down using an assumed $24,000–$25,500 MFJ threshold, then replaced that with an arbitrary $5,475. The household's $27,881 of earned income is below the MFJ phase-out start, so the full $8,231 applies. It also computed the ACTC on gross $30,000 ($4,125) instead of on $27,881 ($3,807)." +us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated the CTC as nonrefundable and set the ACTC to zero. The household has no income tax liability and $27,881 of earned income, so it receives a refundable CTC of 15% × ($27,881 − $2,500) = $3,807. It also used the 2025 EITC maximum of $8,046 instead of the 2026 figure of $8,231." us,scenario_109,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted ineligibility without running either qualifying pathway: SNAP receipt of $8,020.55 confers categorical free-meal status by itself, and the household's $30,000 income is 78% of the 2026 five-person federal poverty guideline, far below the 130% free-meal threshold. Its bare 0 is consistent with demanding an explicit school-enrollment fact instead of deriving two K-12 students from the stated ages of 9 and 6, and Florida has no universal free-meal program that would make the answer turn on state policy." us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated school-meal eligibility as an unlisted fact to be zeroed out under the 'treat unlisted household facts as false' instruction, so it never applied the SNAP categorical qualifier or the 130% poverty-guideline income test. Ages 9 and 6 establish two children in K-12, the household's $8,020.55 SNAP benefit is a standalone categorical pathway to free meals, and $30,000 is 78% of the five-person poverty guideline — each route independently yields the FREE tier and $2,261.92 of annual support." us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as offering Medicaid expansion to low-income adults and therefore used low MAGI as sufficient for eligibility. Florida has no expansion pathway for this age-30 head, and the head fits no other categorical pathway, producing medicaid_category NONE and ineligibility." @@ -7834,45 +8564,47 @@ us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model set up 30000 × 0.9235 × 0.153 correctly but evaluated it as 4234 when the product is 4238.87, understating the tax by $4.87 — its number implies a net-earnings base of $27,673 instead of $27,705. It then compounded the confusion with an irrelevant Additional Medicare Tax digression and a stray discarded figure of 2793 before committing to the miscomputed 4234." us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,other,False,"The model named the correct base (92.35% of $30,000) and the correct combined 15.3% Social Security and Medicare rate, but reported the product as $4,241 when $27,705 × 0.153 = $4,238.87, which rounds to $4,239. The $2.13 overstatement is a multiplication slip presented as a rounding, equivalent to using a net-earnings base of $27,719." us,scenario_109,self_employment_tax,minimax-m3,llm_error,other,False,"The model computed every component correctly — $27,705 net earnings, 12.4% Social Security of $3,435.42, 2.9% Medicare of $803.45 — and stated the total as approximately $4,238, then submitted $4,233, contradicting its own arithmetic by $5. Its own component figures sum to $4,238.87, so the submitted value is a transcription error rather than a rule or base mistake." -us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a ~$225 standard deduction and the FY2025 five-person maximum allotment of $1,158 instead of the FY2026 $261 and $1,183, then abandoned its own $625.50/month result and submitted $3,768 ($314/month) as a benefit ""after payroll/SE adjustments"" — SNAP counts gross self-employment income less the 20% earned-income deduction and applies no payroll or SE-tax offset. The correct derivation is $661.30/month through September and $689.62/month from October." -us,scenario_109,snap,claude-fable-5.1,llm_error,period_annualization,False,"Every FY2026 parameter and deduction step is right — $261 standard deduction, $1,739 net, $1,183 allotment, $661.30/month, no excess shelter deduction — but it multiplied the September figure by twelve instead of switching to the October 1 fiscal-year update. From October the standard deduction rises to $266.85 and the allotment to $1,209.52, giving $689.62/month for the last quarter and $8,020.55 for the year." -us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used $1,316 as the five-person maximum allotment (the FY2026 value is $1,183), reduced SNAP gross income by half the self-employment tax when SNAP counts the full $2,500/month less the 20% earned-income deduction, and then submitted $244/month, which its own stated inputs ($1,316 − 0.3 × $1,830 = $767) contradict. The FY2026 computation yields $661.30/month through September and $689.62/month thereafter." -us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"Its deduction chain was nearly right ($2,500 − $500 earned-income − standard deduction, no excess shelter because $800 rent is below half of adjusted income), but it plugged in $1,613/month as the five-person FY2026 maximum allotment when the actual figure is $1,183 ($1,209.52 from October). That $430/month inflation alone accounts for the $13,016 answer versus $8,020.55." -us,scenario_109,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It granted a shelter deduction that drove net income down to ~$1,470 when $800/month rent is below half of the $1,739 adjusted income, so the excess shelter deduction is $0, and it used the FY2025 $1,158 allotment rather than the FY2026 $1,183. It then submitted $11,436 ($953/month), which contradicts its own stated $717/month figure." -us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It asserted an excess shelter deduction from the $9,600 rent, but $800/month is below half of the $1,739 adjusted income ($869.50), so no shelter deduction applies. Its $424/month figure implies a maximum allotment near $946, far below the FY2026 five-person $1,183, against a correct $661.30/month through September and $689.62/month afterward." -us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly zeroed the excess shelter deduction and computed the 30% contribution, but used $228 as the standard deduction (the FY2026 five-person value is $261) and, believing the FY2025 five-person allotment was $967, projected a $1,017/month FY2026 maximum — the FY2025 figure is $1,158 and the FY2026 figure is $1,183. The $166/month allotment shortfall drives its $5,820 answer." -us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six household members instead of five and declared the household over both income tests, contradicting its own arithmetic ($2,309 gross against a $4,590 limit it itself cited). The five-person household's $2,500 gross is below the 130% FPL limit of about $4,079/month and its $1,739 net is far below the ~$3,138/month net limit, so SNAP is $8,020.55, not zero." -us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person $261, leaving net income of $1,650 rather than $1,739, and paired it with a $1,185 allotment across all twelve months. That overstates the monthly benefit by about $29 through September and understates the October–December allotment of $1,209.52, landing at $8,280 instead of $8,020.55." -us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction and a $1,184 maximum allotment rather than the FY2026 $261 and $1,183, producing $654/month instead of $661.30, and applied that single figure to all twelve months. The October 1 uprating to a $266.85 standard deduction and $1,209.52 allotment raises the last three months to $689.62 each." -us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its shelter logic was right ($800 rent below the $878 half-of-adjusted-income threshold), but it used a $244 standard deduction and the FY2025 five-person maximum allotment of $1,158 instead of the FY2026 $261 and $1,183, yielding $631/month against the correct $661.30. It also held that figure flat through the October uprating to $1,209.52 and $689.62/month." -us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its $695/month figure requires a maximum allotment near $1,220; the FY2026 five-person allotment is $1,183 through September and $1,209.52 from October. With the $261 standard deduction leaving $1,739 net, the benefit is $661.30/month for nine months and $689.62 for three, totaling $8,020.55." -us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no parameter arithmetic and submitted $765/month, which is consistent with granting roughly $346/month of extra deductions on top of the earned-income and standard deductions. The $800/month rent generates no excess shelter deduction because it falls below half of the $1,739 adjusted income, so the benefit is $1,183 − $521.70 = $661.30 through September and $689.62 from October." -us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly applied an excess shelter deduction, but $800/month rent is below half of the $1,739 adjusted income ($869.50), so the excess shelter deduction is $0 and no utility allowance applies since no utility expense is listed. Its $786/month implies about $416/month of phantom shelter deduction against a correct $661.30 through September and $689.62 from October." -us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a standard deduction of about $250 (net income $1,750 rather than $1,739) and an implied maximum allotment near $1,190 instead of the FY2026 $1,183, giving $665/month. It then held that single figure across all twelve months, missing the October 1 rise to a $266.85 standard deduction and a $1,209.52 allotment worth $689.62/month." -us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,other,False,"It performed no computation and defaulted to zero. The five-person household clears the 130% FPL gross test ($2,500 against about $4,079/month), the net test ($1,739 against about $3,138/month), and the asset test ($900), so the FY2026 formula yields $661.30/month through September, $689.62/month from October, and $8,020.55 for the year." -us,scenario_109,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It applied the $800 rent as a deduction outright, but SNAP deducts only shelter costs exceeding half of adjusted income, and $800 is below the $869.50 threshold, so the shelter deduction is zero. Its $752.50/month implies net income of about $1,435 rather than the correct $1,739, which yields $661.30/month through September and $689.62/month afterward." -us,scenario_109,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It supplied no derivation; its $732/month implies net income of about $1,503, which is consistent with a roughly $236/month excess shelter deduction. The $800 rent is below half of the $1,739 adjusted income, so net income stays at $1,739 and the benefit is $1,183 − $521.70 = $661.30 through September and $1,209.52 − $519.90 = $689.62 from October." -us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $675/month is consistent with a standard deduction near $307 against the $1,183 allotment; the FY2026 five-person standard deduction is $261, leaving $1,739 net and $661.30/month. It also applied one flat monthly figure across the year, skipping the October uprating to $266.85 and $1,209.52 that produces $689.62/month." -us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"Its shelter reasoning was right, but it used a $236 standard deduction and a $1,155 maximum allotment rather than the FY2026 five-person $261 and $1,183, producing $625.80/month against the correct $661.30. It also carried that figure through October–December, where the parameters rise to $266.85 and $1,209.52 for a $689.62 monthly benefit." -us,scenario_109,snap,glm-5.3,llm_error,period_annualization,False,"It identified Florida's 200% FPL broad-based categorical eligibility and correctly zeroed the shelter deduction, arriving at about $657/month with rounded parameters ($258 standard deduction, $1,180 allotment versus the exact $261 and $1,183). It then multiplied by twelve, skipping the October 1 switch to a $266.85 standard deduction and $1,209.52 allotment worth $689.62/month." -us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It described ""high shelter costs"" when $800/month rent falls below half of the $1,739 adjusted income and generates no deduction, and its $484.67/month implies a maximum allotment near $1,006 — roughly the four-person figure rather than the five-person $1,183. The correct benefit is $661.30/month through September and $689.62/month from October." -us,scenario_109,snap,gpt-5.4-nano,llm_error,other,False,"It declined to compute and submitted zero on an assumption of no take-up, contradicting the prompt's instruction to assume program take-up. The five-person household passes the gross, net, asset, and work-requirement tests, and the FY2026 formula yields $661.30/month through September and $689.62/month from October for $8,020.55." -us,scenario_109,snap,gpt-5.5,llm_error,period_annualization,False,"It nailed the FY2026 parameters — $261 standard deduction, $1,739 net income, $1,183 allotment, $661/month, no excess shelter deduction — and then multiplied by twelve. October through December run on the uprated $266.85 standard deduction and $1,209.52 allotment, producing $689.62/month and lifting the annual total from $7,932 to $8,020.55." -us,scenario_109,snap,gpt-5.6-luna,llm_error,period_annualization,False,"It reached $659/month, within $2.30 of the correct $661.30 September figure, and applied it uniformly across twelve months. The October 1 fiscal-year update raises the standard deduction to $266.85 and the maximum allotment to $1,209.52, giving $689.62 for each of the final three months." -us,scenario_109,snap,gpt-5.6-sol,llm_error,period_annualization,False,"It correctly found no excess shelter deduction and a $661/month benefit under the FY2026 parameters, then multiplied that single month by twelve. From October the standard deduction is $266.85 and the maximum allotment $1,209.52, producing $689.62/month and a $8,020.55 annual total." -us,scenario_109,snap,gpt-5.6-terra,llm_error,period_annualization,False,"It computed about $659/month against the correct $661.30 and held it flat for the whole calendar year. The last quarter falls in the next fiscal year, where the $266.85 standard deduction and $1,209.52 maximum allotment yield $689.62/month, raising the total to $8,020.55." -us,scenario_109,snap,gpt-6-astra,llm_error,period_annualization,False,"Its arithmetic is exactly the reference's January–September calculation — $2,500 − $500 − $261 = $1,739 net, $1,183 − 30% = $661.30 — but it annualized as 12 × $661.30 = $7,935.60. The October 1 uprating to a $266.85 standard deduction and $1,209.52 allotment adds $28.32/month for three months, producing $8,020.55." -us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It denied eligibility on income and assets, but $2,500/month gross is below the 130% FPL limit of about $4,079 for five people (and far below Florida's 200% FPL broad-based categorical eligibility threshold), and $900 in bank assets is below the $3,000 resource limit. The household qualifies every month, at $661.30 through September and $689.62 from October." -us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected a $1,213/month maximum allotment for the whole year when the FY2026 five-person figure is $1,183 through September, and used $1,744 net income instead of $1,739 from the $261 standard deduction. Its flat $690/month overstates the first nine months by $28.70 each while matching only the October–December level of $689.62." -us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It approximated the maximum allotment at $1,190 and net income at $1,740 and applied one flat $668/month figure to the calendar year, where the actual parameters are $1,183 and $1,739 net through September ($661.30/month) and $1,209.52 with $1,733.15 net from October ($689.62/month). The compensating errors landed within $5 of the reference without reproducing either fiscal-year regime." -us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used the FY2025 five-person standard deduction of $254 and a $1,189 allotment rather than the FY2026 $261 and $1,183, giving $667/month against the correct $661.30. It then applied that figure to all twelve months, skipping the October uprating to $266.85 and $1,209.52 that produces $689.62/month." -us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It approximated the standard deduction at $250 and the maximum allotment at $1,190 instead of the FY2026 $261 and $1,183, producing $665/month against $661.30, and it held that single figure across the calendar year. October through December use the uprated $266.85 deduction and $1,209.52 allotment for $689.62/month." -us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $300/month standard deduction ($3,600 annually) and a projected $1,220 maximum allotment instead of the FY2026 five-person $261 and $1,183, giving $710/month. The correct figures are $661.30/month through September and $689.62/month from October under the uprated $266.85 deduction and $1,209.52 allotment." -us,scenario_109,snap,kimi-k3,llm_error,period_annualization,False,"It reproduced the reference's January–September computation exactly — $261 standard deduction, $1,739 adjusted income, no excess shelter deduction, $1,183 allotment, $661/month — and then annualized by multiplying by twelve. The October 1 parameter update ($266.85 standard deduction, $1,209.52 allotment) raises the final three months to $689.62 each." -us,scenario_109,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed a capped excess shelter deduction reduced net income to about zero, when the $800/month rent is below half of the $1,739 adjusted income and produces no shelter deduction at all, and it used a $1,151 allotment rather than the FY2026 $1,183. Its submitted $489/month matches neither its own zero-net-income premise (which would pay the full allotment) nor the correct $661.30 and $689.62." -us,scenario_109,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added a ~$400 Florida standard utility allowance that the household does not have — no utility expense is listed, and unlisted inputs are zero — manufacturing a $356 excess shelter deduction, and it used a $306 standard deduction instead of $261. With net income correctly at $1,739 the benefit is $661.30/month through September and $689.62/month from October, and its explanation's $9,400 also conflicts with the $7,920 value it submitted." -us,scenario_109,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After applying the 20% earned-income deduction it jumped straight to paying the maximum allotment, never subtracting the 30% expected contribution from net income. With $1,739 net that contribution is $521.70/month, so the benefit is $1,183 − $521.70 = $661.30 through September and $689.62 from October, not the $962/month maximum it awarded." -us,scenario_109,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated countable net income as zero after the self-employment and standard deductions and paid a full maximum allotment, and it used the four-person figure ($998) for a five-person household. Net income is $2,500 − $500 earned-income deduction − $261 standard deduction = $1,739, so $521.70/month is subtracted from the $1,183 five-person allotment, giving $661.30/month through September and $689.62/month from October." +us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model first got about $625 a month using stale FY2025 figures: a $225 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It then cut the benefit to $314 a month by subtracting 'payroll/SE adjustments', but SNAP does not deduct self-employment tax from countable income. The correct result is $1,183 − $522 = $661 a month." +us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"The inputs were right ($1,739 net income, $1,183 maximum), but the model left the 30% contribution at $521.70. SNAP rounds it up to the next whole dollar ($522) under 7 CFR 273.10(e)(2)(ii)(A). That makes the monthly benefit $661 rather than $661.30 and the annual total $7,932, not $7,936." +us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model subtracted half of the self-employment tax from gross income, but SNAP counts self-employment income before income tax adjustments. It also used a $1,316 maximum instead of $1,183. Its $244 monthly benefit does not follow from the maximum-minus-30%-of-net formula, which gives $1,183 − $522 = $661." +us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The method was right, but the model used a $1,613 maximum allotment for five, well above the actual FY2026 figure of $1,183. It also used a $240 standard deduction instead of $261. The result was a $1,085 monthly benefit instead of $661." +us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model cut income by 7.65% for self-employment tax and assumed a rent-based shelter deduction lowered net income. SNAP does not deduct self-employment tax. Rent of $800 is below half of adjusted income ($869.50), so the excess shelter deduction is zero. It also used the FY2025 $1,158 maximum instead of $1,183." +us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model claimed an excess shelter deduction for the $9,600 rent. None exists: $800 of rent is below half of the $1,739 adjusted income. Even so, its $424 monthly result is lower than the correct $661, so it does not follow the $1,183 − 30% of $1,739 formula at all." +us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the FY2026 maximum for five at $1,017 instead of the actual $1,183. It also used the FY2025 four-person standard deduction ($228) instead of the $261 five-person figure. The result was $485 a month instead of $661." +us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model counted six people instead of five and cut income by 7.65% for self-employment tax. It then ruled the household ineligible even though its own numbers passed both tests: $2,309 of gross income under a $4,590 limit and net income under a $3,532 limit. The household passes both tests and gets $661 a month." +us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model used the right $1,739 net income and $1,183 maximum. It did not round the $521.70 contribution up to $522 as 7 CFR 273.10(e)(2)(ii)(A) requires, so it annualized $661.30 into $7,936 instead of $661 × 12 = $7,932." +us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model used a $350 standard deduction instead of the FY2026 five-person figure of $261. That understated net income ($1,650 instead of $1,739). Combined with a $1,185 maximum instead of $1,183, it gave $690 a month instead of $661." +us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $232 standard deduction instead of $261, which overstated net income ($1,768 instead of $1,739). It also used a $1,184 maximum instead of $1,183, giving $654 a month instead of $661." +us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used stale figures: a $244 standard deduction instead of $261 and the FY2025 $1,158 maximum instead of the FY2026 $1,183. The result was $631 a month instead of $661." +us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The $1,183 maximum was right, but the model used a $250 standard deduction instead of $261. Net income came out at $1,750 instead of $1,739 and the contribution at $525 instead of $522, giving $658 a month instead of $661." +us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model's $1,750 net income implies a $250 standard deduction instead of $261. Its $695 monthly benefit implies a maximum of about $1,220 instead of $1,183. Both errors are parameter mistakes on top of the correct $2,500 − $500 − $261 method." +us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $765 monthly benefit implies net income of about $1,393, roughly $346 below the correct $1,739. That matches adding an excess shelter deduction or utility allowance. Neither applies: rent of $800 is below half of adjusted income and no utility costs are listed." +us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly applied an excess shelter deduction, which pulled net income down to about $1,323 and the benefit up to $786 a month. Shelter costs of $800 are below half of the $1,739 adjusted income ($869.50), and no utilities are listed, so the shelter deduction is zero and the benefit is $661." +us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model's $1,750 net income implies a $250 standard deduction instead of $261. Its $665 monthly benefit implies a maximum of about $1,190 instead of $1,183. The correct figures give $1,183 − $522 = $661." +us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model answered $0 with no calculation. The household passes the gross income test ($2,500 a month against a $3,137.50 guideline), the net income test ($1,739) and the asset test ($900). The benefit is $1,183 − $522 = $661 a month." +us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated the $800 rent as producing a deduction, and its $752.50 monthly benefit implies net income of about $1,435. Rent is only deductible above half of adjusted income ($869.50), so this household's shelter deduction is zero. Net income stays at $1,739." +us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $732 monthly benefit implies net income of about $1,503, around $236 below the correct $1,739. That matches applying a shelter deduction the household does not qualify for, since $800 of rent is below half of adjusted income ($869.50)." +us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $675 monthly benefit is $14 above the correct $661. That requires either overstating the $1,183 FY2026 maximum (about $1,197) or deducting about $46 more than the $761 allowed. The $800 rent it cites produces no shelter deduction." +us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used a $236 standard deduction instead of $261 and a $1,155 maximum instead of $1,183. It also did not round the 30% contribution up to a whole dollar, which gave $625.80 a month instead of $661." +us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"The model used a $258 standard deduction instead of $261 and a $1,180 maximum instead of $1,183. That gave about $657 a month instead of $661." +us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model claimed very low net income and high shelter costs, but those would raise the benefit above $661, not lower it to about $485. Its number matches subtracting 30% of income from the FY2025 $1,158 maximum without taking the $500 earned income deduction. The correct figure is $1,183 − 30% of $1,739." +us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model assumed no take-up and returned $0, even though the prompt says to assume take-up whenever it is required. It also skipped the income calculation, which shows the household eligible for $661 a month." +us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $659 monthly benefit is exactly the $1,183 maximum minus 30% of $1,746, which comes from using the FY2025 $254 standard deduction instead of the FY2026 $261. The correct net income is $1,739 and the benefit is $661." +us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $659 monthly benefit is exactly $1,183 minus 30% of $1,746, which uses the FY2025 $254 standard deduction instead of the FY2026 $261. The correct net income is $1,739 and the benefit is $661." +us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"The deductions and maximum were right, but the model kept the contribution at $521.70. 7 CFR 273.10(e)(2)(ii)(A) rounds it up to $522, so the benefit is $661 a month and $7,932 a year, not $661.30 and $7,935.60." +us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $8,045 works out to about $670.42 a month. That is not a whole-dollar SNAP amount, and it is $9.42 above the correct $1,183 − $522 = $661. It reflects an overstated maximum or understated standard deduction plus no rounding of the contribution." +us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model's $1,739 net income and $1,183 maximum were correct. It did not round the $521.70 contribution up to $522, so it reported $661.30 a month ($7,935.60 a year) instead of $661 ($7,932)." +us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"The model said income and assets exceed SNAP limits. Gross income of $2,500 a month is well below 130% of the $3,137.50 guideline, net income of $1,739 is below 100%, and $900 in assets is below the asset limit. The household is eligible for $661 a month." +us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected a $1,213 maximum instead of the published FY2026 $1,183. It also used a standard deduction that gave $1,744 of net income instead of $1,739. The result was $690 a month instead of $661." +us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"The model's net income ($1,740) was essentially right, but it used a maximum of about $1,190 instead of $1,183. That gave $668 a month instead of $661." +us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"The model used FY2025 figures: a $254 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. It also annualized without rounding the contribution, giving $7,610.40 instead of $7,932." +us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,189 maximum instead of $1,183 and the FY2025 $254 standard deduction. It also made an arithmetic slip: $2,000 − $254 is $1,746, not the $1,740 it wrote. The result was $667 a month instead of $661." +us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"The model used a standard deduction of about $250 instead of $261, which put net income at $1,750. It also used a maximum of about $1,190 instead of $1,183. That gave $665 a month instead of $661." +us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $300 monthly standard deduction instead of $261, which put net income at $1,700. It also projected a $1,220 maximum instead of $1,183. That gave $710 a month instead of $661." +us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"The model used a $1,151 maximum instead of $1,183 and first said net income was about $0. Its final $489 benefit implies a contribution of about $662, or net income near $2,207. That matches skipping the $500 earned income deduction; the correct net income is $1,739." +us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"The model's explanation added a Florida utility allowance of about $400 even though no utility costs are listed, and used a $306 standard deduction instead of $261. That created a $356 shelter deduction and a $9,400 total. The submitted $7,920 contradicts that explanation and is still $12 below the correct $661 × 12." +us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model awarded the full maximum allotment as if net income were zero. It skipped subtracting the 30% contribution ($522) from $1,739 of net income. It also used a $962 maximum instead of the FY2026 five-person $1,183." +us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model used a four-person household when there are five people: two adults and three children. It also treated net income as zero, but after the $500 earned income and $261 standard deductions it is $1,739. Its $998 figure is the four-person maximum, not the five-person $1,183." us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base." us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis." us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold." @@ -7886,100 +8618,115 @@ us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligib us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category." us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false." us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added the $6,323 of estate income to AGI, denied the $198.35 traditional IRA above-the-line deduction on a phase-out argument, and took neither the $387 non-itemizer cash charitable deduction nor the $1,264.59 QBI deduction that the estate income generates, which is why its own arithmetic lands at $25,841 rather than $23,897.44. It then submitted $21,120, a figure no step in its derivation produces." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Correctly paired the $16,100 standard deduction with the $387 non-itemizer cash charitable deduction and used the right 2026 brackets, but added the $6,323 of estate income to AGI, denied the $198.35 traditional IRA deduction, and omitted the $1,264.59 QBI deduction equal to 20% of that estate income. Those three omissions inflate ordinary taxable income by $7,786 and tax by $1,851." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used a $15,000 single standard deduction instead of the 2026 figure of $16,100, included the $6,323 of estate income in AGI, and skipped both the $387 non-itemizer charitable deduction and the $1,264.59 QBI deduction. It then cut its own $27,137 result to $25,837 by subtracting roughly $1,300 of nonrefundable credits it never named, and this childless single filer qualifies for none." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Included the $6,323 of estate income in AGI, denied the $198.35 IRA deduction, and claimed no QBI or non-itemizer charitable deduction, overstating ordinary taxable income by $8,173. It also understated the 2026 single bracket thresholds as $12,150/$49,400/$105,450 instead of $12,400/$50,400/$105,700, pushing $7,509 of income into the 24% bracket that belongs at 22%." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Allowed the $198.35 IRA deduction but added the $6,323 of estate income to AGI and omitted both the $1,264.59 QBI deduction that income generates and the $387 non-itemizer charitable deduction. It also set the 22% bracket ceiling at about $107,500 rather than $105,700, understating the ordinary-rate tax on the income it did count." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its own stated components — about $20,000 of ordinary tax plus $6,133 of preferential-rate tax — sum to roughly $26,133, yet it submitted $30,150, a number produced by no step it wrote. The underlying derivation also added the $6,323 of estate income to AGI and left out the $1,264.59 QBI deduction and the $387 non-itemizer charitable deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Included the $6,323 of estate income in AGI and omitted both the $1,264.59 QBI deduction and the $387 non-itemizer charitable deduction, which alone accounts for $1,670 of the gap. It compounded this by inventing 2026 parameters through inflation-scaling 2025 figures — a $15,750 standard deduction and self-derived $12,150/$49,200/$104,900 brackets — instead of the $16,100 deduction and $12,400/$50,400/$105,700 thresholds." -us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Assumed a 2026 TCJA sunset restoring the 2%-of-AGI miscellaneous itemized deduction and deducted $5,331 of the $8,730 of unreimbursed employee business expenses; that deduction is repealed, and the $16,100 standard deduction plus the $387 non-itemizer charitable deduction applies instead. It also included the $6,323 of estate income in AGI, then abandoned its own $26,044.70 total for an unexplained $20,200." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used a $15,000 standard deduction rather than $16,100, added the $6,323 of estate income to AGI, and took neither the $1,264.59 QBI deduction on that income nor the $387 non-itemizer charitable deduction, overstating taxable income by $9,075. Its ordinary-bracket arithmetic on that inflated base produced $20,174 where the reference charges $17,764.88." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied a pre-2018 regime — a $6,800 standard deduction, a $4,850 personal exemption, brackets running to 28%, and $5,469 of repealed 2%-floor employee business expenses — and subtracted the $6,889 of employer-sponsored insurance premiums from wages that are already excluded from the $95,755.10 of employment income. It then ran the ordinary rate schedule across the entire $141,820 of taxable income including the $40,884 of qualified dividends and long-term gains, and taxed that same $40,884 again at 15%." -us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Applied a lapsed pre-TCJA structure — an $8,650 standard deduction, a $5,450 personal exemption, and $20,588.36 of itemized deductions — where 2026 gives the $16,100 standard deduction plus the $387 non-itemizer charitable and $1,264.59 QBI deductions. It also subtracted the $6,889 of employer-sponsored insurance premiums from wages a second time, since PolicyEngine's $95,755.10 of employment income already nets only the $4,244.90 401(k) deferral." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed a 2026 TCJA sunset and subtracted a $5,050 personal exemption plus $5,331 of unreimbursed employee business expenses above a 2% AGI floor; both are repealed, and itemized deductions of $10,915 lose to the $16,100 standard deduction. It also included the $6,323 of estate income in AGI while omitting the $1,264.59 QBI deduction that income generates." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave no derivation; the correct path is AGI of $163,421.64 less $17,752.06 of deductions, giving $145,669.58 of taxable income taxed at $17,764.88 on ordinary income plus $6,132.56 on $40,883.77 of preferential income. Its $19,688 matches ordinary-rate tax alone on roughly $113,000, with the qualified dividends and long-term gains taxed at 0% when ordinary income of about $105,000 sits far above the ~$49,000 zero-rate ceiling and places all of them in the 15% band." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Explicitly assumed TCJA sunset provisions for 2026 and subtracted personal exemptions, which are repealed, and subtracted the pre-tax health premiums from wages on top of the 401(k) deferral even though employment income is already $95,755.10. The 2026 build uses no personal exemption and offsets AGI of $163,421.64 with the $16,100 standard deduction, the $387 non-itemizer charitable deduction, and the $1,264.59 QBI deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Itemized $20,096 by folding in Ohio state income tax as SALT and $5,331 of 2%-floor unreimbursed employee expenses, then subtracted a $5,050 personal exemption; the miscellaneous itemized deduction and the personal exemption are both repealed, and real itemized deductions of $10,915 lose to the $16,100 standard deduction. It also added the $6,323 of estate income to AGI while omitting the $1,264.59 QBI deduction it generates." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Supplied no computation at all; the correct derivation splits $145,669.58 of taxable income into $104,785.81 of ordinary income taxed at $17,764.88 and $40,883.77 of preferential income taxed at 15% for $6,132.56. Its $19,679 corresponds to ordinary-rate tax alone, with the entire 15% capital gains and qualified dividend charge dropped." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Built AGI of $163,054 by subtracting the $6,889 of employer-sponsored insurance premiums from wages while adding the $6,323 of estate income; the correct $163,421.64 uses $95,755.10 of employment income net only of the 401(k) deferral, excludes estate income, and subtracts the $198.35 IRA deduction. It also named no deduction amounts and took no $1,264.59 QBI deduction, so its $22,415 undershoots by $1,482." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Used the same $163,054 AGI built by double-subtracting the $6,889 of ESI premiums from wages while including the $6,323 of estate income, and specified no deductions at all. The correct AGI of $163,421.64 excludes estate income, and taxable income of $145,669.58 after the $16,100 standard, $387 charitable, and $1,264.59 QBI deductions yields $17,764.88 of ordinary tax plus $6,132.56 of preferential tax." -us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Added the $6,323 of estate income to AGI, used a $15,350 standard deduction instead of $16,100, and took neither the $387 non-itemizer charitable deduction nor the $1,264.59 QBI deduction, overstating ordinary taxable income by $8,923. Its preferential-rate tax of $6,132.60 matches the reference, so the entire $2,172 gap sits in the ordinary-rate base." -us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Included the $6,323 of estate income in AGI and omitted both the $387 non-itemizer charitable deduction and the $1,264.59 QBI deduction, adding $7,975 to ordinary taxable income. It compounded that with 2025 parameters — a $15,400 standard deduction and $11,925/$48,475/$103,350 thresholds instead of $16,100 and $12,400/$50,400/$105,700." -us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Stacked the $40,884 of qualified dividends and long-term gains at the bottom of the rate schedule and declared them all inside the 0% capital gains band; they stack on top of about $105,000 of ordinary income and fall entirely in the 15% band, so it omitted $6,132.56 of preferential-rate tax outright. It also added the $6,323 of estate income to AGI while omitting the $1,264.59 QBI and $198.35 IRA deductions." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its own explanation states that the federal income tax remains positive before refundable credits, yet it submitted 0. Taxable income of $145,669.58 generates $17,764.88 of ordinary tax plus $6,132.56 on $40,883.77 of qualified dividends and long-term gains, and no nonrefundable credit is available to this childless single filer." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Provided no derivation; its $16,811 equals the 2026 ordinary-rate tax on roughly $100,000 — the wage income alone. That drops the $21,301 of taxable interest and $5,662 of non-qualified dividends from the ordinary base and charges no separate 15% tax on the $40,883.77 of qualified dividends and long-term gains." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Used the right $16,100 standard deduction and the right 2026 brackets, but added the $6,323 of estate income to AGI, denied the $198.35 traditional IRA above-the-line deduction, and claimed neither the $387 non-itemizer cash charitable deduction nor the $1,264.59 QBI deduction equal to 20% of that estate income. Those four omissions add $8,173 to taxable income and exactly the $1,943 of tax separating $25,841 from $23,897.44." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Picked up the $387 non-itemizer cash charitable deduction but still added the $6,323 of estate income to AGI, omitted the $1,264.59 QBI deduction that income generates, and dropped the $198.35 traditional IRA above-the-line deduction. Its $25,611 reflects an ordinary-rate base overstated by roughly $7,786." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Estimated AGI at $169,943 by including the $6,323 of estate income and refusing the $198.35 traditional IRA deduction, then applied only the $16,100 standard deduction without the $387 non-itemizer charitable deduction or the $1,264.59 QBI deduction. Taxable income of $153,843 instead of $145,669.58 raises ordinary tax by $1,943." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Set AGI at $169,943 by counting the $6,323 of estate income and skipping the $198.35 traditional IRA above-the-line deduction, and deducted only the $16,100 standard amount, leaving out the $387 non-itemizer charitable deduction and the $1,264.59 QBI deduction. Its preferential-rate tax matches the reference, so the entire $1,943.32 excess is ordinary-rate tax on an $8,173 overstated base." -us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Subtracted the $6,889 of employer-sponsored insurance premiums from wages on top of the 401(k) deferral, when employment income is $95,755.10 net of the deferral only, and then invented a $9,000 qualified-overtime deduction; the facts give gross wages inclusive of overtime with a $40 straight-time rate and no qualified overtime premium, so no overtime deduction arises. It also omitted the $1,264.59 QBI deduction on the estate income and the $198.35 IRA deduction." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Offered no computation and rested on itemized deductions including medical expenses plus unspecified nonrefundable credits; medical costs of about $2,300 fall far below the 7.5%-of-AGI floor of roughly $12,257, itemized deductions of $10,915 lose to the $16,100 standard deduction, and no nonrefundable credit applies to this childless single filer. The correct build — AGI $163,421.64 less $17,752.06 of deductions — gives $17,764.88 of ordinary tax plus $6,132.56 of preferential tax." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed restored pre-TCJA brackets for 2026 along with an approximately $5,300 personal exemption and $5,331 of 2%-floor unreimbursed employee expenses; personal exemptions and miscellaneous itemized deductions remain repealed and the ordinary schedule is 10/12/22/24% over $12,400/$50,400/$105,700. It also added the $6,323 of estate income to AGI while omitting the $1,264.59 QBI deduction that income generates." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied a restored 10/15/25% pre-TCJA rate schedule with a $5,300 personal exemption and a 2%-floor deduction for $5,331 of employee business expenses; 2026 uses 10/12/22/24% brackets, no personal exemption, and no miscellaneous itemized deduction. It also itemized $20,151 by counting $6,167 of SALT including Ohio income tax, when the $16,100 standard deduction plus the $387 non-itemizer charitable deduction and $1,264.59 QBI deduction is the correct offset." -us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Used post-sunset 10/15/25% brackets and a $5,387 personal exemption, and itemized $20,136 including $5,335 of repealed 2%-floor employee expenses and $3,886 of Ohio income tax in SALT. For 2026 there is no personal exemption and no miscellaneous itemized deduction, and the offsets are the $16,100 standard deduction, $387 non-itemizer charitable deduction, and $1,264.59 QBI deduction against an AGI that excludes the $6,323 of estate income." -us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Itemized $21,146 by adding roughly $7,162 of SALT including Ohio income tax and $5,331 of unreimbursed employee business expenses after a 2% floor; that miscellaneous itemized deduction is repealed and actual itemized deductions of $10,915 lose to the $16,100 standard deduction plus the $387 non-itemizer charitable deduction. It also included the $6,323 of estate income in AGI and omitted the $1,264.59 QBI and $198.35 IRA deductions." -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for this variable, so nothing was scored against the reference. The correct computation is AGI of $163,421.64 less $17,752.06 of deductions, giving $145,669.58 of taxable income taxed at $17,764.88 on ordinary income plus $6,132.56 on $40,883.77 of preferential income, for $23,897.44." -us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Invented an $8,425.50 phased qualified-overtime deduction from a $10,400 estimated premium; the facts state gross wages already include overtime at a $40 straight-time rate with no qualified overtime premium listed, so no overtime deduction is allowed. That fictitious deduction nearly offset its two opposite errors — adding the $6,323 of estate income to AGI and omitting the $1,264.59 QBI deduction that income generates — leaving it $184.17 short." -us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Asserted without any computation that PolicyEngine returns 0 for this variable. The filer has $145,669.58 of taxable income producing $17,764.88 of ordinary-rate tax and $6,132.56 on $40,883.77 of qualified dividends and long-term gains, and no nonrefundable credit offsets any part of it." -us,scenario_110,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Applied the correct $16,100 standard deduction, the $387 non-itemizer charitable deduction, and the right 2026 brackets, but added the $6,323 of estate income to AGI, denied the $198.35 traditional IRA above-the-line deduction, and omitted the $1,264.59 QBI deduction that the estate income generates. Its $6,133 of preferential-rate tax matches the reference, so all $1,851 of the gap is ordinary-rate tax on an overstated base." -us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Never subtracted the $4,245 traditional 401(k) deferral from wages and used a $15,750 standard deduction instead of $16,100, while also including the $6,323 of estate income and skipping the QBI and IRA deductions. It then submitted $43,879.20 even though its own arithmetic terminated at $26,973, so the reported figure follows from nothing it wrote." -us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Reported AGI of $113,617 while claiming it includes estate income, dividends, capital gains and interest that in fact total $163,421.64, and computed a $297.87 QBI deduction rather than 20% of the $6,323 of estate income ($1,264.59). It then reported $23,648.06 of tax on $103,986.13 of taxable income — that base yields about $14,727 — and subtracted $194.77 of nonrefundable credits that do not exist for this filer." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own computation reached $25,840.76 (AGI $169,943 including estate income, $16,100 standard deduction, 15% on $40,884), yet it submitted $21,120, a figure no step in its reasoning produces. It also included the $6,323 estate income that the reference leaves out of AGI, and it skipped the $387 nonitemizer charitable, $198.35 IRA and $1,264.59 QBI deductions that the reference applies." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction. The reference's AGI of $163,421.64 leaves out estate income and nets a $198.35 IRA deduction. The model also took no QBI deduction, versus the reference's $1,264.59. Its $16,100 standard deduction and $387 nonitemizer charitable deduction match the reference, so the whole $1,850.56 gap comes from how estate income, the IRA and QBI were treated." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It used the pre-OBBBA $15,000 standard deduction instead of the 2026 figure of $16,100 and included the $6,323 estate income. It then subtracted an arbitrary $1,300 of nonrefundable credits from its own $27,137, although this childless filer has no such credits. The reference instead applies $16,100 plus the $387 charitable and $1,264.59 QBI deductions to an AGI of $163,421.64." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction, whereas the reference AGI of $163,421.64 excludes estate income. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. On top of that, it used guessed brackets (12,150/49,400/105,450) instead of the 2026 thresholds of 12,400/50,400/105,700." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,745 after the IRA deduction) and omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. The reference's AGI of $163,421.64 excludes estate income and its taxable income is $145,669.58. The model also put the top of the 22% bracket at $107,500 instead of the 2026 figure of $105,700." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Its components (about $20,000 of ordinary tax plus $6,133 of 15% preferential tax) add up to roughly $26,133, but it stacked the preferential tax again on a total that already included it and submitted $30,150. It also included the $6,323 estate income and omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions that the reference applies." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It took a $10,400 qualified overtime deduction by assuming 10 overtime hours a week from the 50-hour schedule. No overtime premium is listed, the reference applies no overtime deduction, and the model also ignored the phase-down above $150,000 MAGI. It also included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions; these errors partly cancelled and left it $533.42 below the reference." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,750 standard deduction and brackets it inflated itself (12,150/49,200/104,900) instead of the 2026 $16,100 deduction and 12,400/50,400/105,700 brackets. It included the $6,323 estate income that the reference keeps out of its $163,421.64 AGI, and it omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It computed $26,044.70 by itemizing with a $5,331 2%-floor miscellaneous deduction for unreimbursed employee expenses, a deduction that OBBBA permanently eliminated. It then cut the result to $20,200 with no supporting calculation. The reference applies the $16,100 standard deduction plus the $387 charitable and $1,264.59 QBI deductions to an AGI of $163,421.64." +us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and disallowed the IRA deduction, whereas the reference AGI of $163,421.64 excludes estate income and nets a $198.35 IRA deduction. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions. As a result, $7,259 of its ordinary income falls in the 24% bracket, while the reference's $104,785.81 of ordinary income stays entirely within 22%." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used the outdated $15,000 standard deduction instead of the 2026 figure of $16,100. It also overstated ordinary tax at $20,174 on $113,861; the 2026 brackets give $19,924.64. It included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions that the reference applies." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied a personal exemption, a $6,800 standard deduction, 2%-floor miscellaneous deductions and a 28% bracket. OBBBA made the TCJA rates, the $16,100 standard deduction and the repeal of exemptions permanent. It also reduced wages by ESI premiums the reference does not subtract, and its $32,490 ordinary tax on about $101,000 of ordinary income is far higher than any 2026 schedule produces." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: an $8,650 standard deduction, a $5,450 personal exemption, and itemized deductions that included 2%-floor employee expenses. OBBBA permanently keeps the $16,100 standard deduction and the repeal of exemptions for 2026. It also subtracted the $6,889 ESI premiums from wages, whereas the reference's employment income is $95,755.10, net of only the 401(k) deferral." +us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. The reference's AGI of $163,421.64 excludes estate income, and its taxable income of $145,669.58 reflects all three deductions. The model's taxable income was $8,173 higher, which pushed $7,259 of ordinary income into the 24% bracket." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and claimed a $5,050 personal exemption and $5,331 of 2%-floor miscellaneous employee expenses, both of which OBBBA eliminated permanently. The reference uses the $16,100 standard deduction, the $387 nonitemizer charitable deduction and the $1,264.59 QBI deduction. The model also included the $6,323 estate income that the reference leaves out of AGI." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Its $19,688 matches the tax on ordinary income alone, leaving the $40,884 of qualified dividends and long-term gains untaxed. Ordinary taxable income is well above the $49,450 0% threshold, so all of that preferential income is taxed at 15% ($6,132.56 in the reference)." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used itemized deductions and personal exemptions under pre-TCJA brackets. OBBBA permanently kept the $16,100 standard deduction, the TCJA brackets and the repeal of exemptions. It also subtracted pre-tax health premiums from wages, whereas the reference's employment income is $95,755.10, net of only the 401(k) deferral." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: a $5,050 personal exemption, 2%-floor unreimbursed employee expenses and pre-TCJA brackets. The permanent OBBBA rules for 2026 instead give a $16,100 standard deduction and 10/12/22/24% brackets. It also included the $6,323 estate income that the reference leaves out of its $163,421.64 AGI." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $19,679 approximates the tax on ordinary income alone and leaves out the 15% tax on $40,884 of qualified dividends and long-term gains ($6,132.56 in the reference). That 15% rate applies because ordinary taxable income is above the $49,450 0% threshold." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $6,889 employer-sponsored insurance premiums to get an AGI of $163,054. The reference keeps employment income at $95,755.10, net of only the 401(k) deferral, and instead excludes the $6,323 estate income. Its $22,415 is $1,482 below the reference, so it also took deductions beyond the reference's $16,100 standard, $387 charitable and $1,264.59 QBI amounts." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages to get an AGI of $163,054. The reference instead keeps employment income at $95,755.10 and excludes the $6,323 estate income. Its $22,765 is $1,132 short of the reference, so it also took deductions beyond the reference's $16,100 standard, $387 charitable and $1,264.59 QBI amounts." +us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,350 standard deduction instead of the 2026 figure of $16,100 and included the $6,323 estate income, giving taxable income of $154,593 against the reference's $145,669.58. It also omitted the $387 nonitemizer charitable and $1,264.59 QBI deductions." +us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,400 standard deduction and the 2025 bracket thresholds (11,925/48,475/103,350) instead of the 2026 $16,100 deduction and 12,400/50,400/105,700 brackets. It also included the $6,323 estate income and omitted the $387 charitable and $1,264.59 QBI deductions that the reference applies." +us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $40,884 of qualified dividends and long-term gains at 0%, even though its $112,959 of ordinary taxable income is already above the $49,450 0% threshold. All of that income belongs in the 15% bracket ($6,132.56 in the reference). It also included the $6,323 estate income and skipped the $387 charitable and $1,264.59 QBI deductions." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its own reasoning says federal tax stays positive for this high-income filer, yet it submitted $0. The reference taxes $104,785.81 of ordinary income ($17,764.88) and $40,883.77 of preferential income at 15% ($6,132.56)." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation and applied unspecified itemized medical and charity assumptions to reach $16,811, which is $7,086 below the reference. Itemized deductions fall short of the $16,100 standard deduction, and the reference's total is $17,764.88 of ordinary tax plus $6,132.56 of 15% tax on the $40,883.77 of preferential income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. The reference's AGI of $163,421.64 excludes estate income, and its taxable income is $145,669.58. This left $7,259 of the model's ordinary income in the 24% bracket, while the reference's ordinary income stays entirely within 22%." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI and took no QBI deduction, whereas the reference's AGI of $163,421.64 excludes estate income and its deductions include $1,264.59 of QBI and the $198.35 IRA deduction. Its $387 nonitemizer charitable deduction matches the reference." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages and claimed a $9,000 qualified overtime deduction based on an overtime premium that was never listed. The reference keeps employment income at $95,755.10 and applies no overtime deduction. The model's $163,054 AGI also includes the $6,323 estate income that the reference excludes, and it omitted the $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI and took none of the IRA ($198.35), nonitemizer charitable ($387) or QBI ($1,264.59) deductions. That left taxable income at $153,843, against the reference's $145,669.58, whose AGI of $163,421.64 excludes estate income." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It claimed an $8,400 qualified overtime deduction by inferring an overtime premium from the 50-hour schedule. No overtime premium is listed, and the reference applies no overtime deduction. It also included the $6,323 estate income in AGI and omitted the $387 charitable and $1,264.59 QBI deductions; these errors partly cancelled and left it $49.86 below the reference." +us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It excluded the $6,889 ESI premiums from wages to reach an AGI of $163,054. The reference keeps employment income at $95,755.10 and instead leaves the $6,323 estate income out of AGI. The model also omitted the $198.35 IRA and $1,264.59 QBI deductions, so its taxable income is $897 above the reference's." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded guess of $24,500 with no computation and relied on 'standard nonrefundable credits' that do not exist for this childless filer. The reference total is $17,764.88 of ordinary tax plus $6,132.56 of preferential-rate tax, with no credits." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired: it itemized with 2%-floor employee expenses, took a $5,300 personal exemption and used restored pre-TCJA brackets. OBBBA permanently keeps the $16,100 standard deduction, the repeal of exemptions and the 10/12/22/24% brackets for 2026. It also included the $6,323 estate income that the reference excludes." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It applied the law that would have followed a TCJA expiration: 10/15/25% brackets, a $5,300 personal exemption and $5,331 of 2%-floor employee expenses. The permanent OBBBA rules for 2026 instead give a $16,100 standard deduction and brackets that keep the reference's $104,785.81 of ordinary income within 22%. It also included the $6,323 estate income that the reference excludes." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied the rules that would have followed a TCJA expiration: a $5,450 personal exemption, 10/15/25% brackets and $5,331 of 2%-floor employee expenses. OBBBA made the $16,100 standard deduction, the TCJA brackets and the repeal of exemptions permanent for 2026. It also included the $6,323 estate income that the reference leaves out of its $163,421.64 AGI." +us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired and used a $5,387 personal exemption, 10/15/25% brackets and a $5,335 miscellaneous deduction subject to the 2% floor. OBBBA permanently eliminated all of these for 2026, and the reference instead uses the $16,100 standard deduction plus the $387 charitable and $1,264.59 QBI deductions. The model also included the $6,323 estate income that the reference excludes." +us,scenario_110,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It itemized $21,146 by including $5,331 of 2%-floor unreimbursed employee expenses, a miscellaneous deduction that OBBBA permanently eliminated. Without it, itemized deductions fall below the $16,100 standard deduction the reference uses. It also included the $6,323 estate income and omitted the $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed an $8,425.50 qualified overtime deduction from an estimated $10,400 premium that is not among the listed inputs; the reference applies no overtime deduction. It also included the $6,323 estate income in taxable income and omitted the $1,264.59 QBI deduction." +us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed without any computation that PolicyEngine returned $0. The reference taxes $145,669.58 of taxable income, $104,785.81 of it at ordinary rates and $40,883.77 at 15%, for $23,897.44." +us,scenario_110,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It included the $6,323 estate income in AGI ($169,943) and took no IRA or QBI deduction. The reference's AGI of $163,421.64 excludes estate income and nets a $198.35 IRA deduction, and the reference also takes $1,264.59 of QBI. The model's $16,100 standard deduction and $387 nonitemizer charitable deduction match the reference." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Its own reasoning totals $26,973, yet it submitted $43,879.20, a figure no step produces. The computation also skipped the $4,245 401(k) exclusion and used guessed brackets and a $15,750 standard deduction instead of the 2026 figure of $16,100." +us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It understated AGI at $113,617 by dropping about $50,000 of the listed interest, dividend and capital gain income; the reference AGI is $163,421.64. It then applied a $297.87 QBI deduction instead of the reference's $1,264.59 and subtracted $194.77 of nonrefundable credits for a filer who has none." us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model shrank the FICA base to $93,111 by subtracting the $6,889 employer-sponsored insurance premium as a Section 125 pre-tax exclusion, but the stated $100,000 of gross wages and salaries is itself the payroll tax base and nothing in the facts establishes a cafeteria-plan salary reduction. Applying 6.2% and 1.45% to the full $100,000 yields $6,200 + $1,450 = $7,650; its $7,122.99 is exactly 7.65% of the $6,889-reduced base." -us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model deducted the $6,889 employer-sponsored insurance premium from wages to reach a $93,111 FICA base, treating a health premium input as a wage exclusion; the payroll tax base is the stated $100,000 of gross wages. Correct computation is 6.2% × $100,000 = $6,200 plus 1.45% × $100,000 = $1,450 = $7,650, and its $7,122.99 is 7.65% of the improperly reduced base." -us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model applied the correct 7.65% combined employee rate but to $93,111 instead of $100,000, removing the $6,889 employer-sponsored insurance premium from the wage base. Gross wages of $100,000 are the FICA base in full, so the tax is $6,200 Social Security plus $1,450 Medicare = $7,650; the $527 shortfall is 7.65% of the premium it excluded." -us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model recharacterized the $6,889 employer-sponsored insurance premium as a pre-tax salary reduction and taxed only $93,111, producing $5,773 Social Security and $1,350 Medicare. The base is the stated $100,000 of gross wages, which is under both the $168,600 Social Security cap and the $200,000 Additional Medicare threshold, giving $6,200 + $1,450 = $7,650." -us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model netted the $6,889 health premium out of wages before applying the 6.2% and 1.45% rates, taxing $93,111 rather than the stated $100,000 of gross wages and salaries. The correct employee payroll tax is $6,200 + $1,450 = $7,650; $7,122.99 is precisely 7.65% of its understated base." -us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model treated the $6,889 employer-sponsored insurance premium as excluded from FICA wages, computing 7.65% on $93,111. Gross wages of $100,000 are the payroll tax base with no premium offset, so Social Security is $6,200 and Medicare $1,450, totaling $7,650." -us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule set — 6.2% on wages up to the wage base, 1.45% on all wages, no Ohio employee payroll tax — which on $100,000 of wages produces $6,200 + $1,450 = $7,650, but it submitted $8,210, a figure equal to 7.65% of $107,320 that appears nowhere in its own derivation. It never carried its stated computation into the answer field, inflating the base by $7,320 beyond the wages given." -us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model subtracted the $4,245 traditional 401(k) elective deferral from wages to get a $95,755 FICA base, but elective deferrals to a traditional 401(k) are included in Social Security and Medicare wages even though they are excluded from income tax. Taxing the full $100,000 gives $6,200 + $1,450 = $7,650; $7,325.26 is 7.65% of the deferral-reduced base." -us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model excluded the $6,889 employer-sponsored insurance premium as pretax and applied 7.65% to $93,111, correctly ruling out Additional Medicare Tax and Ohio employee payroll taxes but on the wrong base. The stated $100,000 of gross wages is the FICA base, yielding $6,200 Social Security and $1,450 Medicare for $7,650." -us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly kept 401(k) contributions in the FICA base but then removed the $6,889 employer-sponsored insurance premium, taxing $93,111. The premium input does not reduce wages for payroll tax purposes; 6.2% and 1.45% of the full $100,000 give $6,200 + $1,450 = $7,650, $527 above its answer." -us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so the required key was absent from its outputs object rather than wrong on the merits. The requested computation is 6.2% × $100,000 = $6,200 plus 1.45% × $100,000 = $1,450 = $7,650." -us,scenario_110,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model asserted a zero result as if quoting an engine output instead of applying the payroll tax rules to the household's $100,000 of wages, zeroing out a base that is fully taxable. Employee Social Security of $6,200 and Medicare of $1,450 are owed on those wages, totaling $7,650." -us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning derived the reference exactly — $6,200 Social Security on wages under the 2026 wage base plus $1,450 Medicare, no Additional Medicare Tax, no Ohio employee payroll tax, total $7,650 — yet it submitted $9,351.90, a number produced by no step of that derivation. The failure is the answer field contradicting its own completed arithmetic." -us,scenario_110,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model computed the two correct components, $6,200 Social Security and $1,450 Medicare, then added $252.55 of Additional Medicare Tax on a nonexistent excess over the $200,000 single-filer threshold; wages are $100,000, so the 0.9% surtax base is zero. Dropping the invented surtax leaves $7,650." +us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $6,889 employer-sponsored insurance premium from gross wages as a pre-tax cafeteria-plan deduction, giving $93,111 of FICA wages. No Section 125 plan is listed, and PolicyEngine does not reduce the FICA base for this premium. The FICA base is the full $100,000, so the tax is $6,200 + $1,450 = $7,650." +us,scenario_110,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"The model reduced FICA wages to $93,111 by subtracting the $6,889 employer-sponsored insurance premium as though it were a Section 125 salary reduction. Payroll tax applies to the full $100,000 of gross wages. That gives $6,200 of Social Security tax plus $1,450 of Medicare tax, or $7,650." +us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model applied 7.65% to $93,111 after subtracting the $6,889 employer-sponsored insurance premium. The premium input is not a pre-tax cafeteria-plan reduction of FICA wages. Applying 7.65% to the full $100,000 gives $7,650." +us,scenario_110,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model treated the $6,889 employer-sponsored insurance premium as a pre-tax health deduction and computed FICA on $93,111. The correct base is the full $100,000 of gross wages, which gives $6,200 of Social Security tax plus $1,450 of Medicare tax, or $7,650." +us,scenario_110,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model shrank the FICA base to $93,111 by subtracting the $6,889 employer-sponsored health premium, a Section 125 exclusion the household facts never establish. With 6.2% and 1.45% applied to the full $100,000 of wages, the total is $7,650." +us,scenario_110,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model excluded the $6,889 employer-sponsored insurance premium from FICA wages as pre-tax health insurance. PolicyEngine taxes the full $100,000 of gross wages at 7.65%, which gives $7,650 rather than $7,122.99." +us,scenario_110,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model's stated method, 6.2% Social Security plus 1.45% Medicare on wages with no Ohio payroll tax, yields exactly $6,200 + $1,450 = $7,650 on $100,000 of wages. Its submitted $8,210 is $560 higher than its own method produces. The extra amount comes from a calculation error, since nothing else in its method applies." +us,scenario_110,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model subtracted the $4,245 traditional 401(k) deferral and computed FICA on $95,755. Elective 401(k) deferrals remain FICA wages under IRC §3121(v)(1)(A). The full $100,000 is taxable, which gives $7,650." +us,scenario_110,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model applied 7.65% to $93,111 after treating the $6,889 employer-sponsored insurance premium as a pre-tax deduction from FICA wages. PolicyEngine's payroll-tax base is the full $100,000 of gross wages, so the correct total is $7,650." +us,scenario_110,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model correctly kept the traditional 401(k) deferral in FICA wages. It still subtracted the $6,889 employer-plan premium as a pre-tax cafeteria-plan exclusion, giving $93,111. The FICA base is the full $100,000 of gross wages, which yields $7,650." +us,scenario_110,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model excluded the $6,889 employer-sponsored health premium from FICA wages and taxed only $93,111. Gross wages of $100,000 are fully subject to 6.2% Social Security and 1.45% Medicare tax, which gives $7,650." +us,scenario_110,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so no answer could be scored. The correct derivation is 7.65% of the $100,000 in wages, or $7,650." +us,scenario_110,payroll_tax,minimax-m3,llm_error,other,False,"The model claimed PolicyEngine returned $0 and never applied FICA to the head's $100,000 of wages. Employee Social Security tax of $6,200 and Medicare tax of $1,450 apply to those wages, for a total of $7,650." +us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The explanation correctly derives $6,200 + $1,450 = $7,650, but the model submitted $9,351.90, which contradicts its own stated total. The error lies in the submitted value, not in the tax rules it described." +us,scenario_110,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model added $252.55 of Additional Medicare Tax. That 0.9% tax applies only to Medicare wages and self-employment income above $200,000 for a single filer, and investment income does not count toward that threshold. With $100,000 of wages none is owed, so the total is $7,650." us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own worksheet produced $4,265.55, and it then submitted $5,052, a figure its stated derivation never generates. It also carried the pre-2026 $360.69 base amount rather than the 2026 schedule's $332, allowed the $198 traditional IRA deduction that is fully phased out for a workplace-plan participant at this MAGI, and taxed unadjusted federal AGI of $169,745 instead of Ohio AGI of $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied 2.75% to the excess over $26,050 but dropped the $332 base amount that Ohio's 2026 schedule adds to that excess, and it treated unadjusted federal AGI of $169,943 as Ohio AGI instead of $163,421.64. The errors partly cancel: the $6,521.36 AGI overstatement adds $179.34 of tax while the missing base subtracts $332." -us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the federal $15,000 single standard deduction from the Ohio base; Ohio starts at Ohio AGI of $163,421.64 and subtracts only the $1,900 personal exemption, giving $161,521.64. It then reported $3,645 with no rate schedule applied — the 2026 schedule ($332 plus 2.75% of the excess over $26,050) on even its own $154,745 yields $3,871.11." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It invented indexed brackets (0% to $27,350, 2.75% to $103,500, 3.5% above), where Ohio's 2026 schedule is $332 plus 2.75% of Ohio taxable income over a fixed $26,050 with no higher bracket, computed $4,328 from that structure, then discarded it and submitted $2,900 with no derivation." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It taxed the entire amount above $26,050 at 3.5%, the 2024 top marginal rate that Ohio replaced for 2026 with a single 2.75% rate plus a $332 base, reaching $5,325, then subtracted an unexplained $808 to submit $4,517. It also used an $1,850 exemption instead of the $1,900 allowed at OAGI above $80,000 and never applied the Ohio adjustments that bring the base to $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It asserted a 3.5% tier above ~$100,000 that Ohio does not have in 2026 and a ~$2,500 exemption rather than the $1,900 tier for OAGI above $80,000, and it subtracted the $954 tax-exempt interest from AGI that federal AGI already excludes. It performed no arithmetic: its own stated schedule on the ~$166,200 base it described yields about $4,350, and the actual schedule yields $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It taxed everything above $26,050 at 3.5%, the pre-2026 top rate, which alone overstates the tax by about $685 relative to the 2026 schedule's $332 plus 2.75%, and it invented $10,915 of Ohio itemized deductions for mortgage interest, real estate taxes and charity, deductions Ohio's return does not provide in place of the $1,900 exemption. It also subtracted the $20 personal exemption credit, which Ohio limits to OAGI of $30,000 or less." -us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It added the $954 tax-exempt interest back into the Ohio base, invented a ~3.99% top bracket and a retirement income credit for a household with no retirement income, and submitted $3,200 without arithmetic — its own described schedule on ~$168,000 produces about $4,414. Ohio's 2026 tax on the actual $161,521.64 base is $332 plus 2.75% of $135,471.64, or $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied Ohio's pre-2023 multi-bracket table (2.765%/3.302%/3.688%) to $169,745 and denied the personal exemption as fully phased out; Ohio's exemption is $1,900 for OAGI above $80,000 and never reaches zero. The 2026 schedule is $332 plus 2.75% of taxable income above $26,050, which on $161,521.64 gives $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It applied 2.75% to the whole base, discarding Ohio's zero bracket on the first $26,050 and the $332 base amount that accompanies it, denied the $1,900 personal exemption, and added the $954 tax-exempt interest. Taxing all $164,008 at 2.75% overstates the liability by $452.75." -us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It kept a 3.5% bracket on income above $100,000, which Ohio's 2026 schedule does not contain — the rate above $26,050 is a single 2.75% plus a $332 base — and it added the $954 tax-exempt interest to Ohio AGI. Its own $161,910 of taxable income under the 2026 schedule yields $4,068.15, and the correct $161,521.64 base yields $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used a $2,500 personal exemption instead of Ohio's $1,900 tier for OAGI above $80,000 and split the base at $100,000 with 3.50% above, a bracket Ohio repealed for 2026. The actual $332-plus-2.75% schedule on $161,521.64 gives $4,057.47, $336.53 below its answer." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It cited a standard deduction Ohio does not grant and applied no Ohio rate schedule. Its $2,981 corresponds to Ohio taxable income of about $122,400 under the 2026 schedule, roughly $39,100 below the household's $161,521.64, so it left most of the $21,301 interest, $15,694 dividends and $30,870 capital gains out of the Ohio base." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Its $4,241 is the pre-2026 graduated computation — 2.75% on $26,050 to $100,000 plus 3.5% above $100,000 — applied to $163,054 with no personal exemption. Ohio's 2026 schedule has one rate above $26,050 (2.75%, plus a $332 base) and allows the $1,900 exemption, producing $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It relied on a top rate of approximately 3.5% that Ohio replaced for 2026 with a flat 2.75% above $26,050 plus a $332 base, and started from federal AGI of $169,943 rather than Ohio AGI of $163,421.64. It also did no arithmetic: its own description of the schedule on $169,943 yields about $4,480, not the $3,850 submitted." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no computation. Its $3,280 corresponds to Ohio taxable income near $133,250 under the 2026 schedule, about $28,270 short of the household's $161,521.64, so part of the interest, dividend and capital gain income was omitted from the Ohio base; the correct tax is $332 plus 2.75% of $135,471.64." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $4,240.52 is exactly 2.75% of $73,950 plus 3.5% of $63,054 — the repealed pre-2026 two-rate structure applied to federal AGI of $163,054 with no personal exemption. For 2026 the rate above $26,050 is a flat 2.75% with a $332 base and the $1,900 exemption applies, giving $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no computation, and $3,986 matches a bare 2.75%-above-$26,050 calculation with no $332 base amount and no personal exemption on roughly $171,000 of income. The Ohio base is $163,421.64 less the $1,900 exemption, and the 2026 schedule adds $332 to 2.75% of the excess over $26,050." -us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It denied the personal exemption on the theory that none is allowed above $80,000 of AGI; $1,900 is precisely Ohio's exemption amount in that OAGI tier. It also omitted the schedule's $332 base amount and used federal AGI of $169,943 instead of Ohio AGI of $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It declared the $2,400 exemption fully phased out, when Ohio's exemption steps down to $1,900 for OAGI above $80,000 and stays there, and it applied 3.5% to income above $100,000, a bracket absent from Ohio's 2026 schedule. The correct computation is $332 plus 2.75% of ($161,521.64 − $26,050) = $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It taxed $169,943 with a 3.5% bracket above $100,000 that Ohio's 2026 schedule does not include and claimed the personal exemption was fully phased out; the exemption is $1,900 at OAGI above $80,000. Using the 2026 schedule ($332 plus 2.75% above $26,050) on $161,521.64 gives $4,057.47, $424 below its answer." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It submitted 0 while its own explanation states that Ohio tax is positive at this income level, a self-contradictory non-answer. Ohio taxes the household's $161,521.64 of taxable income at $332 plus 2.75% of the excess over $26,050, or $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted federal itemized items — mortgage interest and charitable contributions — from the Ohio base, which is Ohio AGI of $163,421.64 less only the $1,900 exemption, and it identified no Ohio rate schedule. Its $5,625 is close to taxing that entire base at 3.5%, a rate Ohio dropped for 2026, and overstates the correct $4,057.47 by $1,567.53." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It had the $1,900 exemption and the 2.75% rate above $26,050 right but omitted the $332 base amount Ohio's 2026 schedule adds, and it used federal AGI of $169,943 as the Ohio starting point instead of Ohio AGI of $163,421.64. The missing base costs $332 and the inflated AGI adds back $179.34." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $4,418 tracks the repealed 2024 table — $2,033.63 plus 3.5% of income above $100,000 — on about $168,000 of taxable income. Ohio's 2026 schedule taxes the excess over $26,050 at a single 2.75% plus $332, and the Ohio base is $161,521.64, giving $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied 2.75% above the $26,050 zero bracket but left out the $332 base amount in Ohio's 2026 schedule, and it used unadjusted federal AGI of $169,943 rather than Ohio AGI of $163,421.64: $3,905 is 2.75% of ($169,943 − $1,900 − $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It used an $1,850 exemption where Ohio allows $1,900 for OAGI above $80,000, omitted the $332 base amount, and taxed unadjusted federal AGI of $169,943 instead of Ohio AGI of $163,421.64. Its arithmetic, 2.75% of $142,043, is internally consistent but built on all three wrong inputs." -us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It reproduced the 2026 Ohio schedule ($332 plus 2.75% of the excess over $26,050) and the $1,900 exemption correctly, but it added the $954 tax-exempt interest as an Ohio addback on top of a $163,054 starting AGI, putting Ohio taxable income at $162,108 instead of $161,521.64. That $586.36 overstatement of the base is worth exactly the $16.13 by which its answer exceeds the reference." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It showed no computation and credited modest credits that do not exist here — Ohio's $20 exemption credit is capped at $30,000 of OAGI and the joint filing credit requires a joint return. Its $3,200 corresponds to Ohio taxable income near $130,340, about $31,180 below the household's $161,521.64, the size of the $30,852 of long-term capital gains it left out of the base." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Its $4,415 is $2,033.63 plus 3.5% of the excess over $100,000 — Ohio's 2024 schedule — applied to $169,943 less the $1,900 exemption. For 2026 the only rate above $26,050 is 2.75% with a $332 base, and Ohio AGI is $163,421.64, so the tax is $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used the correct 2.75% rate and $1,900 exemption but omitted the $332 base amount in Ohio's 2026 schedule and treated federal AGI of $169,943 as Ohio AGI rather than $163,421.64, leaving it $152.47 short." -us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It claimed the $2,400 exemption, which Ohio allows only at OAGI of $40,000 or less — this household's tier is $1,900 — omitted the schedule's $332 base amount, and started from $169,745 of federal AGI by allowing the $198 traditional IRA deduction that is fully phased out at this MAGI with workplace-plan coverage." -us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It used the 2024 table's construction, $2,394 on the first $100,000 plus roughly 3.5% on the excess; both that base amount and the 3.5% bracket are gone in 2026, where the schedule is $332 plus 2.75% of the excess over $26,050. On the correct $161,521.64 base that gives $4,057.47, $700.53 below its answer." -us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation for state_income_tax_before_refundable_credits was returned, so no Ohio computation was submitted against the $4,057.47 reference ($332 plus 2.75% of $161,521.64 less $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It omitted the $332 base amount in Ohio's 2026 schedule and then subtracted a $20 personal exemption credit that Ohio allows only when OAGI is $30,000 or less. It also started from $169,745 of federal AGI, allowing the fully phased-out $198 IRA deduction, rather than Ohio AGI of $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It asserted that the engine returns zero Ohio tax. Ohio taxes this household's $161,521.64 of taxable income — Ohio AGI of $163,421.64 less the $1,900 personal exemption, the only subtraction available — at $332 plus 2.75% of the excess over $26,050, giving $4,057.47." -us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It used a $1,700 exemption, which is not one of Ohio's three tiers of $2,400/$2,150/$1,900 (this household qualifies for $1,900), and applied 2.75% above $26,050 without the schedule's $332 base amount, on unadjusted federal AGI of $169,943 instead of Ohio AGI of $163,421.64." -us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It built a $174,944 base by adding the $954 tax-exempt interest and never subtracting the $4,245 traditional 401(k) deferral from wages, applied the repealed pre-2023 brackets (2.765%/3.225%), and invented a $250 personal exemption credit where Ohio's is $20 and capped at $30,000 of OAGI. It then cut its own $4,170.50 to $3,148.93 with an unexplained rate reduction; the correct computation is $332 plus 2.75% of ($161,521.64 − $26,050)." -us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported Ohio taxable income of about $96,517 after a standard deduction Ohio does not provide; the base is Ohio AGI of $163,421.64 less the $1,900 exemption, $161,521.64, so it understated the base by roughly $65,000. It also applied progressive rates where Ohio's 2026 schedule is a single 2.75% rate above $26,050 plus a $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own working reached $4,265.55, using the old $360.69 base, a $1,700 exemption and a $169,745 AGI that left the $6,889 pre-tax employer premiums in wages. It then submitted $5,052, which no step of its reasoning supports. The correct 2026 figure is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It taxed income above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI instead of $163,421.64, because it did not take the $6,889 pre-tax employer premiums out of wages, so taxable income came to $168,043 instead of $161,521.64. The two errors partly cancel, and the answer ends up $152 low." +us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a $15,000 federal standard deduction, but Ohio has none; the only reduction from Ohio AGI at this income is the $1,900 personal exemption. It then guessed from vague 'progressive brackets' instead of using the 2026 schedule of $332 + 2.75% above $26,050. Its $169,745 AGI also left the $6,889 pre-tax employer premiums in wages." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used an old two-bracket schedule (2.75% up to about $103,500, 3.5% above) to reach about $4,328, then dropped that result for an unexplained $2,900. The 2026 schedule has no 3.5% bracket: tax is $332 + 2.75% × $135,471.64 = $4,057.47. Its $167,895 taxable income was also too high because it left the $6,889 pre-tax employer premiums in wages." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It taxed all income above $26,050 at 3.5% on top of the old $360.69 base, reached $5,325, and then submitted an unrelated $4,517. The 2026 rate above $26,050 is 2.75% plus a $332 base. Its $167,895 taxable income also kept the $6,889 pre-tax employer premiums in AGI and used a $1,850 exemption instead of the $1,900 tier." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It used the old 2.75%/3.5% schedule and took a $2,500 exemption instead of $1,900. It also subtracted the $954 of tax-exempt interest from an AGI that never included it. Its $3,900 equals 2.75% of about $141,800 above $26,050 with no $332 base, taxed on an AGI that still includes the $6,889 pre-tax employer premiums." +us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,795 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also used a $169,745 AGI, which leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $10,915 of itemized deductions, but Ohio allows none; only the $1,900 personal exemption applies. It used a 3.5% rate above $26,050 when the 2026 rule is $332 + 2.75%. It also took the $20 exemption credit, which is only available when the Ohio income tax base is under $30,000." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It invented a 3.99% bracket, used the old 2.75%/3.5% schedule, and claimed a retirement income credit even though the filer is 51 with no retirement income. It then submitted $3,200, which is below even its own bracket math. The 2026 tax is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It denied the personal exemption, but Ohio allows $1,900 at MAGI above $80,000. It also left out the $332 base and taxed $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the full $169,745 with no personal exemption, claiming a standard deduction and exemption were phased out. Ohio has no standard deduction, and its exemption is still $1,900 at this income. It also used an old multi-bracket schedule (2.765%/3.302%/3.688%) instead of the 2026 schedule of $332 + a single 2.75% rate above $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied 2.75% to every dollar of $164,008, ignoring the 0% bracket on the first $26,050 and the $332 base, and it gave no $1,900 personal exemption. The 2026 computation is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Its $161,910 of taxable income was close to the correct $161,521.64, but it used the old 2024 schedule (2.75% up to $100,000, 3.5% above) with no base amount. The 2026 schedule taxes all income above $26,050 at 2.75% plus a $332 base, which gives about $4,068 even on the model's own taxable income." +us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used the $2,400 exemption, which only applies at MAGI of $40,000 or less, instead of $1,900. It kept Ohio AGI at $169,943 instead of $163,421.64. It also applied a made-up three-tier schedule when the 2026 Ohio tax is $332 + 2.75% above $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It applied the 2024 schedule (2.75% up to $100,000, 3.5% above) without base amounts and took a $2,500 exemption instead of $1,900. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums. The 2026 rule is $332 + 2.75% on everything above $26,050." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted 'standard deductions', but Ohio has none; the only reduction from the $163,421.64 Ohio AGI is the $1,900 exemption. Under the correct $332 + 2.75% schedule, its $2,981 implies about $39,000 less taxable income than the correct $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It started from a $163,054 AGI, close to the $163,421.64 Ohio AGI, but skipped the $1,900 personal exemption. It then used the old 2.75%/3.5% schedule with no base: $2,033.63 + 3.5% × $63,054 ≈ $4,241. The 2026 schedule has no 3.5% bracket; the tax is $332 + 2.75% above $26,050 on $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It started from $169,943, which leaves the $6,889 pre-tax employer premiums in wages, and used an old graduated schedule topping out at 3.5%. For 2026, Ohio replaced that with $332 + a single 2.75% rate above $26,050. Its $3,850 also does not follow from its own stated brackets, which give about $4,400." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47, and $3,280 is $777 short. That implies about $28,300 less taxable income, which fits subtracting deductions Ohio does not allow; Ohio permits only the $1,900 exemption here." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,240.52 is exactly $2,033.63 + 3.5% × ($163,054 − $100,000). It taxed a $163,054 AGI with no $1,900 exemption under the old 2.75%/3.5% schedule. The 2026 tax is $332 + 2.75% above $26,050 on $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47. Its $3,986 equals 2.75% of about $144,950 above $26,050 with no $332 base, which fits taxing an overstated income near $171,000 that keeps the $6,889 pre-tax employer premiums in AGI and skips the $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It wrongly said no personal exemption is allowed above $80,000; Ohio allows $1,900 at that income. It also left out the $332 base and taxed $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It said the personal exemption is fully phased out, but Ohio still allows $1,900 at MAGI above $80,000. It kept Ohio AGI at $169,943 instead of $163,421.64 and applied the 2024 2.75%/3.5% schedule. For 2026, all income above $26,050 is taxed at 2.75% plus a $332 base." +us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $169,943 with no personal exemption under the old 2.75%/3.5% schedule. The 2026 rule is $332 + 2.75% above $26,050, applied to $161,521.64 ($163,421.64 Ohio AGI less the $1,900 exemption)." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It acknowledged that Ohio tax is positive at this income but refused to compute it and entered $0. Applying the $1,900 exemption and the 2026 schedule of $332 + 2.75% above $26,050 to $163,421.64 of Ohio AGI gives $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted itemized deductions (mortgage interest and charitable gifts), which Ohio does not allow, and still reported $5,625. Even a flat 2.75% on the whole $169,943 comes to only $4,673, so its figure implies a rate well above Ohio's single 2026 rate of 2.75%. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,993 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its $4,418 matches the old 2024 schedule ($2,033.63 on the first $100,000 plus 3.5% above) applied to about $168,100 of taxable income. That income comes from a $169,943 AGI that includes the $6,889 pre-tax employer premiums. The 2026 schedule is $332 + 2.75% above $26,050 on $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied only 2.75% above the zero bracket and left out the $332 base in the 2026 Ohio schedule. Its $3,905 corresponds to $168,043 of taxable income, built from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base and used a $1,850 exemption instead of the $1,900 tier for MAGI above $80,000. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $1,900 exemption and the $332 + 2.75% schedule correctly. However, it built Ohio AGI as $163,054 plus the full $954 of tax-exempt interest, giving $164,008, which is $586.36 above the $163,421.64 Ohio AGI. That put taxable income at $162,108 instead of $161,521.64 and the tax $16.13 too high." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $169,943 with no personal exemption under the old 2.75%/3.5% schedule. The 2026 rule is $332 + a single 2.75% rate above $26,050, applied to $161,521.64 of taxable income after the $1,900 exemption." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It skipped the $1,900 personal exemption and left out the $332 base, taxing $169,943 at 2.75% above $26,050. That AGI still includes the $6,889 pre-tax employer premiums, so it is well above the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It applied the $1,900 exemption and the $332 + 2.75% schedule correctly, but treated Ohio AGI as equal to its $163,054 federal AGI. That left out the net +$367.64 of Ohio adjustments that bring Ohio AGI to $163,421.64. Taxable income came to $161,154 instead of $161,521.64, and the tax is $10.11 too low." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation and claimed 'modest credits', but no Ohio nonrefundable credit applies here; the $20 exemption credit requires an Ohio income tax base under $30,000. The correct result is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47, so $3,200 is $857 short." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly used the 2024 schedule (2.75% up to $100,000, 3.5% above) with no base amounts on $168,043. The 2026 schedule has no 3.5% bracket; it is $332 + 2.75% above $26,050. Its $169,943 AGI also leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $141,993 above $26,050 at 2.75% but left out the $332 base in the 2026 Ohio schedule. It also started from a $169,943 AGI that leaves the $6,889 pre-tax employer premiums in wages, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It applied the old 2.75%/3.5% schedule to $168,997: $2,033.63 + 3.5% × $68,997 ≈ $4,449. The 2026 rule is $332 + a single 2.75% rate above $26,050. Its income base also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base and used the $2,400 exemption, which only applies at MAGI of $40,000 or less, instead of $1,900. It also started from a $169,745 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2024 schedule: about $2,394 of tax on the first $100,000 plus 3.5% on the excess. For 2026, Ohio replaced that with $332 + a single 2.75% rate above $26,050. Its taxable income of about $167,500 also came from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of $161,521.64." +us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base in the 2026 schedule and subtracted the $20 exemption credit, which is only available when the Ohio income tax base is under $30,000. It also started from a $169,745 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It made up a claim that PolicyEngine returns $0 and did no computation. Ohio AGI of $163,421.64, less the $1,900 exemption, taxed at $332 + 2.75% above $26,050, gives $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $332 base in the 2026 schedule and took a $1,700 exemption instead of the $1,900 tier for MAGI above $80,000. It also started from a $169,943 AGI that includes the $6,889 pre-tax employer premiums, instead of the $163,421.64 Ohio AGI." +us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It counted the $954 of tax-exempt interest in AGI and did not subtract the $4,245 401(k) deferral from gross wages. It then used made-up 2.765%/3.225% brackets, an invented $250 exemption credit and an unexplained further 'graduated credit', ending at $3,148.93. The 2026 tax is $332 + 2.75% × ($161,521.64 − $26,050) = $4,057.47." +us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It cut Ohio taxable income to about $96,517 by subtracting 'the standard deduction and exemptions', but Ohio has no standard deduction and allows only a $1,900 exemption. The correct taxable income is $161,521.64, and $332 + 2.75% above $26,050 gives $4,057.47." us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate." us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount." @@ -7996,14 +8743,16 @@ us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_ra us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model characterized countable income as low after unspecified exclusions and then approved ABD Medicaid without computing the controlling income test. The spouse’s disability and low stated resources do not independently establish eligibility, and no ABD or other non-MAGI category is satisfied." us,scenario_111,spouse_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model understated Medicaid MAGI by counting only the $18,240 IRA distribution and excluding all Social Security on the ground that none was federally taxable. The eligibility computation produces MAGI equal to 2.02 times FPL, which is above the 138% FPL adult expansion limit." us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the spouse’s lack of personally assigned income as zero income for ABD Medicaid and also understated household MAGI as $18,240. The applicable calculations yield MAGI of 2.02 times FPL and do not place the spouse in an ABD or any other Medicaid category; excluding the vehicle and noting $840 of bank assets does not establish eligibility." -us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"Asserted that ""disabled individuals qualify for Medicare regardless of age,"" which is neither the statutory rule nor PolicyEngine's implementation. The under-65 pathway requires 24 months of entitlement to SSDI (or an ESRD/ALS diagnosis), and the spouse has $0 Social Security disability income under the treat-unlisted-inputs-as-zero instruction, so only the age-65 test remains and 49 fails it." -us,scenario_111,spouse_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"Invoked ""the disability pathway"" as though the `is_disabled` demographic flag were itself the trigger. That flag drives SSI, the SNAP elderly/disabled deductions, and EITC rules; Medicare's disability route runs through 24 months of SSDI entitlement, which a spouse with zero Social Security disability income does not have, leaving age >= 65 as the binding test the 49-year-old fails." -us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"Stated that ""Medicare eligibility extends to those under 65 who are disabled"" and stopped there, skipping the benefit-receipt condition that gates that extension: 24 months of SSDI entitlement, or ESRD/ALS. With no Social Security disability income listed for the spouse, the only satisfiable branch of PolicyEngine's `is_medicare_eligible` is the age-65 threshold, which 49 does not meet." -us,scenario_111,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"Named the correct mechanism — Medicare ""via Social Security disability benefits"" — then submitted 1 without checking whether the spouse actually receives those benefits. The household lists no Social Security disability income for the spouse, so the SSDI-entitlement branch is unsatisfied and the age-65 threshold decides the 49-year-old's eligibility as False." -us,scenario_111,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"Attributed to PolicyEngine a ""disability-based eligibility"" rule keyed on the `is_disabled` input. PolicyEngine keys Medicare on the age-65 threshold and on entitlement to Social Security disability benefits (the 24-month SSDI waiting period), and the spouse satisfies neither: age 49 with $0 Social Security disability income." -us,scenario_111,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"Read ""Spouse is disabled"" as sufficient under ""the modeled rules,"" collapsing a two-part condition — a disability determination plus 24 months of SSDI entitlement (or ESRD/ALS) — into the single demographic flag. With the spouse's Social Security disability income at zero, PolicyEngine falls back to the age-65 threshold and returns False." -us,scenario_111,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"Cited a ""PolicyEngine disability-based Medicare eligibility rule"" fired by the is_disabled flag alone. Medicare's under-65 route requires 24 months of SSDI entitlement or an ESRD/ALS diagnosis, none of which is present in this household, so the age-65 threshold governs and the 49-year-old spouse is not eligible." -us,scenario_111,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"Correctly identified the age-65 threshold and that the spouse falls below it, then overrode that with the claim that ""PolicyEngine treats individuals with a disability as eligible for Medicare regardless of age."" The disability pathway requires 24 months of SSDI entitlement, which a spouse with $0 Social Security disability income does not have, so the age test alone decides and yields False." +us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model claimed that disabled people qualify for Medicare regardless of age. Under-65 disability eligibility actually requires 24 months of Social Security disability benefit entitlement. The 49-year-old spouse has no SSDI income or SSDI months listed, so the correct answer is not eligible." +us,scenario_111,spouse_medicare_eligible,claude-fable-5.1,llm_error,age_disability,False,"The model applied the Medicare disability pathway using only the disabled flag. That pathway requires 24 months of SSDI entitlement, and the spouse has zero Social Security disability benefits, so the pathway does not apply." +us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model stated that Medicare covers disabled people under 65 and stopped there. It never applied the requirement of 24 months of Social Security disability benefit receipt, which the 49-year-old spouse (with no SSDI listed) does not meet." +us,scenario_111,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model correctly said under-65 Medicare comes through Social Security disability benefits. It then assumed eligibility from the disabled flag anyway, even though the spouse has $0 SSDI and no months of disability-benefit receipt. That means the 24-month entitlement test fails." +us,scenario_111,spouse_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model described PolicyEngine's disability-based Medicare rule as triggered by disability status alone. The rule actually requires at least 24 months of Social Security disability receipt, and the spouse has none." +us,scenario_111,spouse_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"The model equated being disabled with Medicare eligibility. Under 65, Medicare requires 24 months of SSDI entitlement (or ESRD or ALS), and none of these apply to the 49-year-old spouse, who has no Social Security disability income." +us,scenario_111,spouse_medicare_eligible,gpt-5.5,llm_error,age_disability,False,The model invoked disability-based Medicare eligibility for the under-65 spouse without checking its gating condition: 24 months of Social Security disability benefit entitlement. The spouse fails that condition because no SSDI is listed. +us,scenario_111,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model treated the disabled flag as enough under PolicyEngine's disability-based Medicare rule. That rule keys on at least 24 months of Social Security disability receipt, and the spouse has zero months, so the spouse is not eligible." +us,scenario_111,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied the modeled Medicare disability rule to the 49-year-old spouse based only on disability status. It skipped the rule's requirement of 24 months of SSDI entitlement, which fails because the spouse has no Social Security disability benefits." +us,scenario_111,spouse_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,"The model correctly noted that the spouse is below the age-65 threshold. It then claimed PolicyEngine treats any disabled person as Medicare-eligible regardless of age. The under-65 pathway actually requires 24 months of Social Security disability benefit receipt, which the spouse (with no SSDI) lacks." us,scenario_112,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model explicitly built AGI as ""wages $5,923 + farm rent $1,920 + financial assistance $13,000 - $3,000"" = $17,843, treating the $13,000 financial-assistance receipt as taxable ordinary income; financial assistance is a non-taxable cash transfer excluded from gross income and AGI, so true AGI is $4,843. Removing that $13,000 leaves AGI below the ~$16,100 single standard deduction, so taxable income is $0 and the 10%-bracket computation that produced $174 never arises." us,scenario_112,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model stated its ordinary income base as ""wages, farm-rent income, and financial assistance,"" including the $13,000 financial assistance in taxable income even though that receipt is a non-taxable transfer excluded from gross income and AGI. With only $5,923 of wages plus $1,920 of farm rent less the $3,000 allowable capital loss, AGI is $4,843 — under the 2026 single standard deduction — so taxable income is $0 and the $1,743 it ran through the 10% bracket to reach $174.30 is entirely the misclassified assistance." us,scenario_112,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model executed the correct derivation in full - childless EITC phase-in of 7.65% on $5,923 of earned income, AGI of $4,843 below the phase-out start, investment income below the disqualification limit - and stated the answer as approximately $453, then submitted 191. The submitted number corresponds to no step in its own arithmetic; it discarded its correct $453.10 result at the point of transcription." @@ -8044,146 +8793,168 @@ us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base, us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings." us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies." us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271." -us,scenario_112,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted the $1,920 farm rental income in SNAP gross income, reaching $1,737/month against a 130% limit of $1,696 and declaring a gross-test failure; farm rent is not a countable SNAP income source, so countable gross is $18,923 ($1,576.90/month, 121% of the $1,304.17 guideline). The same $160/month inflated its BBCE net income to $1,429 against the $1,305 net limit, where the correct net is $1,269.19 (97%), and it therefore never reached the one- and two-person minimum allotment of $23.84/month that produces the $287.68." -us,scenario_112,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"Included the $1,920 farm rent in countable income, pushing net income to $1,429 versus the correct $1,269.19 (97% of the $1,304.17 guideline), then treated ""30% of net exceeds the $298 maximum allotment"" as ending the calculation at zero. Eligible one- and two-person households receive the minimum allotment — 8% of the one-person maximum, $23.84/month rising to $24.37 in October — which is exactly when the computed benefit falls to zero, giving $287.68." -us,scenario_112,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Summed all $20,843 of listed income, including $1,920 of farm rental income that SNAP does not count, and ruled the household over the one-person gross limit at $1,737/month; countable gross is $18,923 ($1,576.90/month, 121% of the guideline), which passes, and Texas broad-based categorical eligibility waives both the gross test and the asset test it invoked. Having declared ineligibility, it never applied the one-person minimum allotment that generates $287.68." -us,scenario_112,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Counted the $1,920 farm rent as SNAP gross income to reach $1,737/month and compared it to an understated 130% limit of $1,650 (the 2026 one-person figure is $1,695.42); excluding farm rent leaves countable gross of $1,576.90, 121% of the guideline, passing both the 130% test and Texas's 165% BBCE limit. The eligible household's benefit is the $23.84/month minimum allotment, not zero." -us,scenario_112,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Excluded the $13,000 of financial assistance as non-countable gift income; SNAP counts it as unearned income, so countable gross is $18,923 and net income is $1,269.19/month, not the near-zero figure that produced its near-maximum allotment. With the 30% expected contribution at $380.70 above the $298 maximum allotment, the formula benefit is zero and the household receives only the one- and two-person minimum allotment — $287.68 for the year, not $2,304." -us,scenario_112,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Dropped the $13,000 of financial assistance from countable income entirely and counted only wages plus farm rent (~$654/month), yielding a near-maximum $275/month benefit. The countable set is the reverse: financial assistance counts as unearned income and farm rent does not, so net income is $1,269.19/month, its 30% contribution of $380.70 exceeds the $298 maximum allotment, and the household drops to the $23.84/month minimum allotment ($287.68 annually)." -us,scenario_112,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Added the $1,920 farm rent to countable income for $20,843 and compared it to an understated 130% limit of $19,584 (the 2026 one-person figure is $20,345); farm rental income is not countable, leaving $18,923 of gross income that passes even the straight 130% test, and Texas BBCE raises the applicable limit to 165% of the guideline. It stopped at ineligibility and so never applied the one-person minimum allotment of $23.84/month." -us,scenario_112,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Treated wages, farm rent, and the $13,000 financial assistance as countable gross income exceeding a ~$1,632/month limit; farm rental income is excluded from SNAP countable income, so gross is $1,576.90/month — 121% of the $1,304.17 guideline — against a correct 130% limit of $1,695.42. Both income tests pass and the eligible one-person household receives the minimum allotment, $287.68 for the year." -us,scenario_112,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Used the correct 130% threshold of ~$20,345 but inflated countable income to $20,843 by including $1,920 of farm rental income, which SNAP does not count; the countable total is $18,923, below the limit, so the household passes the gross test at 121% of the guideline. Eligible, it receives the one- and two-person minimum allotment of $23.84/month, $287.68 for the year." -us,scenario_112,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Included the $1,920 farm rent to reach $1,737/month against a $1,672 limit; countable SNAP gross income is wages plus financial assistance only — $1,576.90/month, 121% of the $1,304.17 guideline — and the 2026 one-person 130% limit is $1,695.42, so the gross test passes. The eligible household's allotment is the $23.84/month minimum, $287.68 annually." -us,scenario_112,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Correctly found that 30% of net income exceeds the one-person maximum allotment but treated that as producing a zero benefit. The minimum allotment for eligible one- and two-person households — 8% of the maximum, $23.84/month and $24.37 after the October adjustment — applies precisely when the computed benefit falls to zero, giving $287.68." -us,scenario_112,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $1,440.20 net income includes the $160/month of farm rental income that SNAP does not count; the correct net is $1,269.19, which is 97% of the $1,304.17 guideline and passes the net income test. It also asserted that a categorically eligible household must clear the net test to receive even the minimum allotment — this household clears it, and the one-person minimum allotment of $23.84/month yields $287.68." -us,scenario_112,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave no derivation; its zero is the output of stopping at max(0, $298 − 30% of net income), which is zero here because the $380.70 expected contribution exceeds the $298 one-person maximum allotment. The household passes the gross test at 121% and the net test at 97% of the guideline, so the one- and two-person minimum allotment of $23.84/month ($24.37 from October) applies, giving $287.68." -us,scenario_112,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly computed that 30% of net income exceeds the one-person maximum allotment but stopped there at zero. The minimum allotment for eligible one- and two-person households applies exactly in that case, so the benefit is $23.84/month rising to $24.37 in October — $287.68 for the year." -us,scenario_112,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Inflated net income to $1,440 by counting $1,920 of non-countable farm rental income, then treated the resulting $432 expected contribution exceeding the maximum allotment as producing zero. Net income is $1,269.19 with a $380.70 contribution, which still exceeds the $298 maximum, and the one- and two-person minimum allotment of $23.84/month then applies, giving $287.68." -us,scenario_112,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Asserted a zero benefit with no derivation; that value is the output of the benefit formula max(0, $298 − $380.70) with no floor applied. The eligible one-person household — gross income 121% and net income 97% of the guideline — receives the minimum allotment of $23.84/month, $24.37 after the October adjustment, totaling $287.68." -us,scenario_112,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Stopped at the zero produced by 30% of net income exceeding the one-person maximum allotment. The household passes the gross (121%) and net (97%) income tests and is also categorically eligible through TANF non-cash assistance, so the minimum allotment of $23.84/month applies, $287.68 annually." -us,scenario_112,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Asserted that income exceeds the net income limit and reduces the allotment to zero; countable net income is $1,269.19, 97% of the $1,304.17 guideline, because farm rental income is not countable. The eligible one-person household receives the minimum allotment of $23.84/month, $287.68 for the year." -us,scenario_112,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Reached zero from 30% of net income exceeding the one-person maximum allotment and also invoked vehicle/asset resources; the $10 in bank accounts passes the asset test, categorical eligibility waives it in any case, and the minimum allotment of $23.84/month takes over once the computed benefit hits zero, giving $287.68." -us,scenario_112,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"Counted $1,920 of farm rental income, inflating net income to $1,434 and failing the net test against its $1,330 limit; SNAP does not count farm rent, so net income is $1,269.19 — 97% of the $1,304.17 guideline — and the test passes. The eligible one-person household then receives the minimum allotment of $23.84/month, $287.68 for the year." -us,scenario_112,snap,glm-5.3,llm_error,thresholds_rates,False,"Used a 130% gross limit of ~$17,000 for one person when the 2026 figure is $20,345, and treated the $11,838 vehicle as busting the asset test; countable gross income is $18,923 (farm rent is not countable), the household's $10 in bank accounts passes the asset test, and Texas categorical eligibility waives it. Eligible, the household receives the $23.84/month minimum allotment, $287.68 annually." -us,scenario_112,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"Asserted without derivation that income and resources are too high. Countable gross income is $1,576.90/month (121% of the $1,304.17 guideline) and net income $1,269.19 (97%), so both tests pass; the zero is the unfloored formula benefit, and the one- and two-person minimum allotment of $23.84/month produces $287.68." -us,scenario_112,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"Declared household size and eligibility inputs indeterminate and defaulted to zero; the facts specify a single adult, which is a one-person SNAP unit with a $1,304.17 monthly guideline and a $298 maximum allotment. Gross income of $1,576.90 and net income of $1,269.19 clear the 130% and 100% tests, and the one-person minimum allotment of $23.84/month ($24.37 after October) gives $287.68." -us,scenario_112,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Counted the $1,920 farm rental income to reach $20,843 of gross income above the 130% limit; farm rent is not countable SNAP income, so gross is $18,923, under the $20,345 limit and far under Texas's 165% BBCE limit. The eligible household receives the one-person minimum allotment of $23.84/month, $287.68 annually." -us,scenario_112,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Blamed the $13,000 financial assistance for busting the one-person gross limit, but that assistance is counted in the reference and the household still passes: with the non-countable $1,920 farm rent removed, gross is $18,923 ($1,576.90/month, 121% of the guideline) against a $20,345 limit. Eligible, the household receives the $23.84/month minimum allotment, $287.68 for the year." -us,scenario_112,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Included farm rental income in countable income, pushing net income above the 100% guideline limit; excluding it leaves net income of $1,269.19 — 97% of the $1,304.17 guideline — so the net test passes with no shelter or medical deduction needed. The $380.70 expected contribution then exceeds the $298 maximum allotment, and the one-person minimum allotment of $23.84/month gives $287.68." -us,scenario_112,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Added the $1,920 farm rental income to countable income to exceed the one-person gross limit; SNAP counts the financial assistance but not farm rent, leaving $18,923 of gross income ($1,576.90/month, 121% of the guideline) below the $20,345 limit. The eligible household receives the one- and two-person minimum allotment of $23.84/month, $287.68 annually." -us,scenario_112,snap,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Counted farm rental income alongside wages and financial assistance, pushing post-deduction income above the one-person net limit; farm rent is not countable, so net income is $1,269.19, 97% of the $1,304.17 guideline, and the net test passes without any shelter or elderly/disabled medical deduction. The eligible household then receives the $23.84/month minimum allotment, $287.68 for the year." -us,scenario_112,snap,grok-4.3,llm_error,thresholds_rates,False,"Offered no derivation beyond a generic income/asset assertion; countable gross income is $1,576.90/month (121% of the guideline), net income $1,269.19 (97%), and $10 in bank accounts passes the asset test, so the household is eligible. Its zero equals the unfloored formula benefit max(0, $298 − $380.70); the one- and two-person minimum allotment of $23.84/month gives $287.68." -us,scenario_112,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Asserted a disjunctive gross-or-net failure without computing either; removing the non-countable $1,920 farm rent leaves gross income of $1,576.90/month (121% of the guideline) and net income of $1,269.19 (97%), so both tests pass. The eligible one-person household receives the minimum allotment of $23.84/month, $287.68 annually." -us,scenario_112,snap,grok-4.6,llm_error,thresholds_rates,False,"Reached zero because 30% of net income exceeds the one-person maximum allotment; that is precisely where the minimum allotment takes over. Eligible one- and two-person households receive 8% of the one-person maximum — $23.84/month, $24.37 from October — which is the $287.68 reference." -us,scenario_112,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Included $1,920 of farm rental income to reach $20,843 against a ~$20,700 limit; SNAP does not count farm rent, so countable gross is $18,923, under the $20,345 one-person 130% limit. Eligibility established, the household receives the minimum allotment of $23.84/month, $287.68 for the year." -us,scenario_112,snap,inkling,llm_error,taxable_income_or_deductions,False,"Counted farm rent along with wages and the $13,000 assistance to exceed the 130% FPL gross limit; farm rental income is not countable, leaving $1,576.90/month — 121% of the $1,304.17 guideline — against a $1,695.42 limit. The eligible one-person household receives the minimum allotment of $23.84/month, $287.68 annually." -us,scenario_112,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Applied the 130% gross test to $20,843 that includes $1,920 of non-countable farm rental income; countable gross is $18,923, which clears the $20,345 limit outright, and Texas broad-based categorical eligibility through TANF non-cash assistance raises the limit to 165% regardless of the elderly/disabled status it cited. The eligible one-person household receives the minimum allotment, $287.68 for the year." -us,scenario_112,snap,kimi-k3,llm_error,categorical_eligibility,False,"Denied categorical eligibility on the ground that the household receives neither TANF cash nor SSI; categorical eligibility runs through TANF non-cash assistance under Texas BBCE, and the household passes both income tests on its own (121% gross, 97% net) once the non-countable farm rent is excluded. Its ""expected contribution eliminates any allotment"" step also skips the minimum allotment guaranteed to eligible one- and two-person households, $23.84/month here." -us,scenario_112,snap,minimax-m3,llm_error,household_unit_or_filing_status,False,"Treated the absence of additional members and dependents as disqualifying; a single adult is a one-person SNAP household with a $1,304.17 monthly guideline, a $298 maximum allotment, and a $23.84 minimum allotment. Its gross income of $1,576.90 and net income of $1,269.19 clear both tests, so the annual benefit is $287.68." -us,scenario_112,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"Used the correct ~$1,695 gross threshold but inflated gross income to $1,737/month by adding $1,920 of farm rental income that SNAP does not count; the countable figure is $1,576.90, 121% of the guideline, which passes. The eligible one-person household then receives the minimum allotment of $23.84/month, $287.68 annually." -us,scenario_112,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"Twice computed a negative formula benefit and floored it at zero, then discarded that and submitted $3,576 — exactly twelve months of the $298 maximum allotment — invoking a shelter deduction the facts do not include. The floor is not the maximum allotment but the one- and two-person minimum allotment of $23.84/month, rising to $24.37 after the October adjustment, totaling $287.68." -us,scenario_112,snap,qwen3.8-max,llm_error,thresholds_rates,False,"Asserted ineligibility with no derivation; countable gross income of $1,576.90/month is 121% of the $1,304.17 guideline, net income of $1,269.19 is 97%, and $10 in bank accounts passes the asset test, so the household is eligible on every test. Its zero matches the unfloored benefit formula, while the one- and two-person minimum allotment of $23.84/month produces $287.68." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It built SALT from the $4,064 real estate tax alone, omitting the $5,367.59 of Virginia income tax that brings SALT to $9,431.59, and skipped the 0.5%-of-AGI charitable floor, so its itemized total was $19,518 rather than $24,787.23. It then discarded its own $11,889 bracket computation and submitted $8,971 with no supporting derivation; the correct stack of $24,787.23 itemized plus $4,872.35 QBI plus $4,105.65 senior leaves $72,807.33 taxable and $10,729.61 of tax." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the structure right — 0.5% charitable floor, phased $4,105.65 senior deduction, $4,872.35 QBI, itemizing over the $18,150 standard deduction — but understated two components: Virginia income tax in SALT at $3,524 instead of $5,367.59, and medical at $4,007 instead of $4,441.86. Those two shortfalls total $2,278 of taxable income, which at 22% is exactly the $501 by which its $11,231 exceeds $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $19,828.80 of taxable Social Security out of income entirely (summing only $86,744) and applied a $28,300 single standard deduction that does not exist for 2026 — the figure is $16,100 plus the $2,050 aged addition — while asserting that $18,300 of itemized deductions 'exceeds' $28,300. It then added $336 of Medicare payroll tax to income tax and jumped from its own $8,268 to $10,268 citing withholding and safe-harbor considerations, which have no bearing on tax liability." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It claimed the senior deduction at the full $6,000 without the 6% phase-down over $75,000 of MAGI (which leaves $4,105.65), computed medical from the $10,000 'other medical' figure while dropping the $2,000 of OTC health expenses, and left the $5,367.59 of Virginia income tax out of SALT. It then abandoned its own $10,711 result and submitted $5,489 with no derivation at all." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It fabricated roughly $4,500 of mortgage interest from the $95,618 loan balance, though mortgage interest is an unlisted input and therefore zero, and it omitted both the $4,105.65 senior deduction and the $5,367.59 of Virginia income tax within SALT. It then reduced its own computed $11,802 to $11,210 'after rounding/SS taxable adjustments' without any calculation behind the change." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the $6,000 senior deduction as an add-on available only with the standard deduction, concluding that $24,150 beat its understated $17,650 itemized figure; the senior deduction stacks on itemized deductions, and with Virginia income tax in SALT and the $2,000 OTC costs in medical, itemized deductions total $24,787.23. It also subtracted about $1,721 of half-SE tax when above-the-line deductions here total $22, then submitted $6,900 against its own roughly $11,900 computation." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction outright, invented $6,693 of mortgage interest from the loan balance (an unlisted input, hence zero), left Virginia income tax out of SALT, and ignored the 0.5%-of-AGI charitable floor. Those errors partly offset to leave taxable income of $75,490 instead of $72,807.33, and it then applied guessed brackets ($12,247/$49,784) rather than the 2026 $12,400/$50,400 breakpoints." -us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and confined SALT to the $4,064 real estate tax, leaving out $5,367.59 of Virginia income tax, and applied no 0.5% charitable floor. Its $19,518 of itemized deductions versus the correct $24,787.23 plus the missing senior deduction pushed taxable income to $82,183 instead of $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It reduced AGI by $1,721 of half self-employment tax, when the partnership/S-corp income here generates no such above-the-line deduction (total above-the-line deductions are $22), and it computed medical from the $10,000 base while ignoring the $2,000 of OTC health expenses. It also left Virginia income tax out of SALT and omitted the $4,105.65 senior deduction, producing $82,357 of taxable income against the correct $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction, capped medical at $2,007.04 by excluding the $2,000 of OTC health expenses from the $12,000 base (correct: $4,441.86), and understated SALT at $7,615.53 versus $9,431.59, partly offset by skipping the 0.5% charitable floor. Those four differences sum to $7,823 of excess taxable income, exactly the gap between its $80,630.83 and the correct $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a repealed pre-TCJA structure for 2026 — a $5,350 personal exemption and 10/15/25% brackets — and consequently allowed no §199A QBI deduction and no senior deduction. For 2026 there is no personal exemption, the single schedule is 10% to $12,400, 12% to $50,400, and 22% above, and the deduction stack includes $4,872.35 of QBI plus $4,105.65 of senior deduction." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied sunset-law parameters — a $5,000 personal exemption and 10/15/25% brackets — and also cut AGI to $104,872 with a $1,721 half-SE-tax adjustment that does not apply (above-the-line deductions total $22). Under 2026 law the $72,807.33 of taxable income is taxed at 10/12/22% for $10,729.61, with no exemption and with the $4,872.35 QBI and $4,105.65 senior deductions in the stack." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated gross income of $110,094 includes the full $23,328 of Social Security rather than the 85% maximum of $19,828.80, and its deduction list names only real estate taxes, charitable gifts and medical expenses — no Virginia income tax in SALT, no $4,872.35 QBI deduction, and no $4,105.65 senior deduction. Its $13,919 on roughly $81,300 of taxable income also requires a 25% bracket that the 2026 single schedule does not contain." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It stated outright that it computed the tax 'under 2026 pre-TCJA law' with personal exemptions, applying repealed 10/15/25% brackets and omitting both the $4,872.35 QBI deduction and the $4,105.65 senior deduction. For 2026 there is no personal exemption and the tax on $72,807.33 of taxable income is $5,800 + 22% × $22,407.33 = $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It subtracted a $5,050 personal exemption and applied 'sunset' brackets, neither of which exists for 2026, and it itemized only $19,256 by leaving Virginia income tax out of SALT and the senior deduction out entirely. On its own $77,395 of taxable income the 2026 schedule gives $11,739, not the $14,029 its 25%-bracket arithmetic produced." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It returned a bare figure that restates the question with no derivation. The correct path — AGI of $106,572.56 less $24,787.23 of itemized deductions, $4,872.35 of QBI, and the phased $4,105.65 senior deduction — gives $72,807.33 of taxable income and $10,729.61 of tax; its $9,334 corresponds to taxable income near $66,460, about $6,300 more deduction than the law allows, the size of an unphased $6,000 senior deduction stacked on an already-complete deduction total." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its itemized components track the right structure (medical above the 7.5% floor, real estate tax, state income tax, charitable) and it applied the $4,872.40 QBI deduction, but it omitted the $4,105.65 senior deduction entirely and understated the itemized total at $22,018.04 versus $24,787.23. That left $79,682.36 of taxable income instead of $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It elected the $18,150 standard deduction (base plus the aged addition) when itemized deductions of $24,787.23 are larger, and it omitted the $4,105.65 senior deduction. AGI of $106,572.56 less $18,150 and $4,872.35 of QBI is $83,550 of taxable income — precisely the base that produces the $13,104 it submitted." -us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It allowed no QBI deduction and no senior deduction, and its $17,518 of itemized deductions used medical of only $2,007 (the $10,000 base, excluding the $2,000 of OTC health expenses) and SALT of $4,064 with no Virginia income tax. Dropping $4,872.35 of QBI, $4,105.65 of senior deduction, and $7,269 of itemized deductions left taxable income near $89,055 rather than $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It asserted that 'in 2026 (post-TCJA), there is no QBI deduction' and applied a $5,500 personal exemption with 10/15/25% brackets and uncapped SALT. For 2026 the §199A deduction of $4,872.35 applies, there is no personal exemption, the senior deduction adds $4,105.65, and the single rate schedule is 10/12/22% with $12,400 and $50,400 breakpoints." -us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the correct 2026 bracket structure and the $4,872 QBI deduction but itemized only $17,518: SALT of $4,064 with no Virginia income tax, medical of $2,007 computed on the $10,000 base while ignoring the $2,000 of OTC health expenses, and charitable gifts with no 0.5%-of-AGI floor. It also omitted the $4,105.65 senior deduction, leaving $84,183 of taxable income against the correct $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction plus the age-65 addition offsets taxable income to zero; those amounts total $18,150 against AGI of $106,572.56, which includes $22,700 of wages, $24,361.77 of partnership income, $30,000 of pension, $9,600 of IRA distributions, and $19,828.80 of taxable Social Security. Even on the standard-deduction path the tax is $11,969.80, and on the correct itemizing path it is $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It described the income as 'mainly from Social Security and modest taxable income,' when $86,744 of wages, partnership income, pension, IRA distributions and interest drive provisional income to $98,408 and make 85% of Social Security ($19,828.80) taxable. Against $18,150 of standard deduction or $33,765.23 of the correct deduction stack, taxable income is $72,807.33 and the tax is $10,729.61, not zero." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It subtracted a '2026 personal exemption' and applied post-2025 sunset brackets, and its $17,518 of itemized deductions excluded Virginia income tax from SALT and the $2,000 of OTC health expenses from medical, with no QBI or senior deduction. For 2026 there is no personal exemption, and the deduction stack of $24,787.23 itemized plus $4,872.35 QBI plus $4,105.65 senior leaves $72,807.33 taxed at 10/12/22%." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It reduced AGI by a deductible half of estimated self-employment tax, when above-the-line deductions here total $22, and stacked the age-adjusted standard deduction with an unphased senior deduction; its stated $67,020 of taxable income implies roughly $39,550 of deductions against the correct $33,765.23. The senior deduction phases down 6% of MAGI over $75,000 to $4,105.65, and itemizing at $24,787.23 — not the $18,150 standard deduction — is the correct path." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It took the standard-deduction path — $18,150 plus the $1,000 non-itemizer charitable deduction — with the senior deduction and QBI figured correctly, reaching $78,445 of taxable income. Itemized deductions of $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59 including $5,367.59 of Virginia income tax, medical $4,441.86) exceed the standard deduction, and its $11,970 is exactly the standard-deduction alternative, $1,240.19 above the itemizing result." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its deduction list names itemized deductions, QBI, and 'the age-65 additional deduction' — the $2,050 standard-deduction add-on, which is unavailable to an itemizer — while omitting the $4,105.65 senior deduction that stacks on itemized deductions. Its $11,540 implies taxable income near $76,491, about $3,684 above the correct $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It handled the charitable 0.5% floor, the $4,105.63 phased senior deduction, and the $4,872.40 QBI deduction correctly, but its itemized total of $20,639.33 falls $4,147.90 short of $24,787.23 because it understated the Virginia income tax inside SALT (about $5,718 versus $9,431.59) and medical ($4,007 versus $4,441.86). At 22% that shortfall is $912.59 — exactly the amount by which its $11,642.20 exceeds $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, only a bare figure. The correct computation — $106,572.56 of AGI less $24,787.23 itemized, $4,872.35 QBI, and $4,105.65 senior deduction — gives $72,807.33 taxable and $10,729.61 of tax; its $10,234 implies taxable income near $70,555, roughly $2,250 low, consistent with claiming the senior deduction at the unphased $6,000 instead of $4,105.65 and skipping the 0.5%-of-AGI charitable floor." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a repealed pre-TCJA regime for 2026 — a $5,468 personal exemption, uncapped SALT, and 'restored' 10/15/25% brackets — and allowed no QBI or senior deduction. The 2026 single schedule is 10% to $12,400, 12% to $50,400, and 22% above, with no personal exemption, $4,872.35 of QBI and $4,105.65 of senior deduction in the stack." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It stated 'No QBI (expired)' and applied a $5,300 personal exemption with restored 10/15/25% brackets. The §199A deduction of $4,872.35 applies for 2026, there is no personal exemption, the senior deduction adds $4,105.65, and tax on the resulting $72,807.33 is $5,800 + 22% × $22,407.33 = $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used the right bracket structure and the $4,872 QBI deduction but omitted the $4,105.65 senior deduction, understated the Virginia income tax inside SALT at $2,746 versus $5,367.59, and applied no 0.5%-of-AGI charitable floor. Its $22,264 of itemized deductions plus the missing senior deduction left $79,437 of taxable income instead of $72,807.33." -us,scenario_114,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It subtracted a roughly $5,300 personal exemption and applied 'pre-TCJA' brackets, neither of which exists for 2026, and reduced AGI to about $104,852 with a half-SE-tax adjustment when above-the-line deductions total $22. It also allowed no QBI or senior deduction; the correct stack of $33,765.23 leaves $72,807.33 taxed at 10/12/22% for $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for this variable, so there is no substantive computation to evaluate. The required derivation is AGI of $106,572.56 less $24,787.23 of itemized deductions, $4,872.35 of QBI, and $4,105.65 of senior deduction, giving $72,807.33 of taxable income and $10,729.61 of tax at 2026 single rates." -us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed itemized deductions as $17,518 — SALT of $4,064 with no Virginia income tax and medical based on the $10,000 figure without the $2,000 of OTC health expenses — so it took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction. Correct itemized deductions of $24,787.23 exceed the standard deduction, and its $11,969.85 is exactly the standard-deduction alternative, $1,240.19 above the itemizing result of $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It acknowledged positive taxable income and that itemized deductions exceed the standard deduction, then submitted zero on the assertion that nonrefundable credits erase the liability. No nonrefundable credit is available to this single 69-year-old filer with $106,572.56 of AGI, and tax on the resulting $72,807.33 of taxable income is $10,729.61." -us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It compared a $22,256 standard-plus-senior total against an understated itemized figure of about $19,500 and chose the standard deduction; the $4,105.65 senior deduction stacks on itemized deductions either way, so the comparison is $24,787.23 of itemized deductions against $18,150. Its itemized figure omitted the $5,367.59 of Virginia income tax in SALT and the $2,000 of OTC health expenses in medical, and the resulting $79,445 of taxable income is $6,638 too high." -us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded the $19,828.80 of taxable Social Security from income entirely and used a $15,750 standard deduction (a stale figure it decomposed as $14,600 plus $1,150) instead of the $24,787.23 of itemized deductions and the $4,105.65 senior deduction. Its reported $15,654 is also inconsistent with its own $66,122 of taxable income, which yields $9,259 under the 2026 single schedule." -us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reported AGI of $80,164 while listing components that sum to $106,572.80, invented an exclusion of 'the first $400 of capital loss' as a rounding artifact, and took a $20,400 standard deduction over the larger $24,787.23 of itemized deductions while quoting two different QBI figures ($3,550 and $4,450) against the correct $4,872.35. Its submitted $5,290.25 also contradicts its own stated $7,761.45 of regular tax." +us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"It counted the $160/month of farm rent, which gave $1,737/month gross, and failed a 130% gross test that Texas replaces with its 165% BBCE limit; its fallback net figure of about $1,429 also includes the farm rent. The reference's countable income is wages plus financial assistance ($1,576.92), so net income is $1,269.19, below the $1,304 limit, and the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"It applied Texas BBCE correctly but counted the $160/month of farm rent, which raised net income to about $1,429, above the 100% poverty-guideline limit. The reference's income excludes farm rent, so net income is $1,269.19 and the test is passed. A one-person household that passes still gets the $23.84 minimum allotment even when 30% of net income is more than the $298 maximum." +us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $1,737/month gross and failed a gross income limit, ignoring Texas's 165% BBCE limit. The reference's gross income is $1,576.92 without farm rent (121% of the poverty guideline); net income of $1,269 passes the net test, and the one-person minimum allotment of $23.84/month is paid." +us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"It added the $1,920 of farm rent to reach $1,737/month and failed a 130% gross test of about $1,650, which Texas BBCE replaces with 165%. The reference's countable income leaves out farm rent, net income of $1,269 is below the $1,304 limit, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"It treated the $13,000 of financial assistance as a non-countable gift, but it is countable unearned income ($1,083.33/month), which puts net income at $1,269.19. At that level, 30% of net income ($380.76) is more than the $298 maximum, so only the one-person minimum allotment of $23.84/month is paid, not about $192/month." +us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"It left the $13,000 of financial assistance out of countable income and got a near-maximum benefit of $275/month. Once the $1,083.33/month of assistance is counted, net income is $1,269.19, 30% of it is more than the $298 maximum, and the household receives only the $23.84/month minimum allotment." +us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"It assumed the financial assistance is not counted and got a net income of $346 and a benefit of $194/month. SNAP counts the $1,083.33/month of financial assistance as unearned income, which makes net income $1,269.19; the $380.76 expected contribution is more than the $298 maximum, leaving only the $23.84/month minimum allotment." +us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $20,843/year and applied a 130% gross limit of about $19,584, ignoring Texas's 165% BBCE limit. The reference's countable income is $18,923/year without farm rent; it passes the gross and net tests (net income is $1,269/month), and the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and applied a 130% gross limit (about $1,632/month) instead of Texas's 165% BBCE limit. The reference's countable income, without farm rent, is $1,576.92 gross and $1,269.19 net; this passes the 100% net test and yields the $23.84/month minimum allotment." +us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"It treated the $13,000 of financial assistance as not countable and got $194/month. Counting the $1,083.33/month of assistance as unearned income puts net income at $1,269.19; 30% of that is more than the $298 maximum, so only the $23.84/month minimum allotment is paid." +us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"It included the $1,920 of farm rent to reach $20,843 and tested that against the 130% limit, not Texas's 165% BBCE limit. The reference's countable income is $18,923/year without farm rent, net income of $1,269/month passes the net test, and the minimum allotment of $23.84/month applies." +us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Its $1,737/month gross figure includes farm rent, and it applied a 130% gross test in place of Texas's 165% BBCE limit. The reference counts $1,576.92/month without farm rent, which passes the net test at $1,269.19, and it pays the one-person minimum allotment of $23.84/month." +us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and concluded that because 30% of net income is more than the maximum allotment the benefit is $0, never applying the minimum allotment for one-person households. Without farm rent, net income is $1,269.19, below the $1,304 limit, so the household receives the $23.84/month minimum." +us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"It computed an annual net income of $6,454 and compared the annual 30% contribution ($1,936) with a monthly maximum allotment, mixing annual and monthly figures. The correct monthly net income is $1,269.19; its $380.76 contribution is more than the $298 maximum, and the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Its net income of $1,440.20 includes the $160/month of farm rent, which puts it over the net limit. The reference's countable income leaves out farm rent, so net income is $1,269.19; that passes the 100% net test, and the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Its $0 answer matches counting every listed income item, including farm rent, which makes the household fail the net test. The reference's countable income is wages plus financial assistance ($1,576.92/month); net income of $1,269.19 passes, and the $23.84/month minimum allotment is paid." +us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"It reasoned that because 30% of net income is more than the maximum, the benefit is $0; this skips the minimum allotment and counts the farm rent. Without farm rent, net income is $1,269.19, below the $1,304 limit, so the one-person household receives the $23.84/month minimum allotment." +us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"It computed net income of about $1,440, including farm rent, and set the benefit to $0 because 30% of net income is more than the maximum, never applying the minimum allotment. The reference's net income without farm rent is $1,269.19, which passes the net test and yields $23.84/month." +us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Its bare $0 answer matches counting farm rent, which fails the net income test. The reference's countable income excludes farm rent, net income of $1,269.19 is below the $1,304 limit, and the $23.84/month one-person minimum allotment applies." +us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"It set SNAP to $0 because 30% of net income is more than the maximum allotment, ignoring the minimum allotment and counting farm rent. The reference's net income is $1,269.19 without farm rent, which passes the net test, so the $23.84/month minimum allotment is paid." +us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"It found net income above the limit by counting all listed income, including farm rent. The reference's countable income leaves out farm rent, so net income is $1,269.19, below the $1,304.17 limit, and the household receives the $23.84/month minimum allotment." +us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"It cited vehicle and asset resources, although Texas BBCE waives the asset test, and treated 30% of net income being more than the maximum as a zero benefit while counting farm rent. The reference's net income without farm rent is $1,269.19, which passes the net test, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Its $1,434 net income includes the $160/month of farm rent, which is above the net limit. The reference's countable income is $1,576.92 gross and $1,269.19 net without farm rent, which passes the net test, and the one-person minimum allotment of $23.84/month is paid." +us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent, used a gross limit of about $17,000 in place of Texas's 165% BBCE limit, and treated the $11,838 vehicle as disqualifying even though BBCE waives the asset test. The reference's gross income without farm rent is $18,923/year, net income of $1,269/month passes, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Its $0 answer rests on income and resources being too high, which matches counting farm rent and applying an asset test that Texas BBCE waives. The reference's countable income leaves out farm rent, net income of $1,269.19 passes, and the minimum allotment of $23.84/month is paid." +us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"It refused to compute SNAP, claiming the household facts lacked eligibility indicators, and defaulted to $0. The facts fully determine the result: gross income of $1,576.92, net income of $1,269.19 below the $1,304 limit, a $380.76 expected contribution above the $298 maximum, and therefore the $23.84/month one-person minimum allotment." +us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"It counted the $1,920 of farm rent to reach $20,843 and failed a gross limit, without applying Texas's 165% BBCE threshold. The reference's countable income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"It failed the household on a gross limit after including farm rent, ignoring Texas's 165% BBCE limit. Without farm rent, gross income is 121% of the poverty guideline and net income of $1,269.19 passes the 100% test, so the one-person minimum allotment of $23.84/month is paid." +us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent, which pushed net income above the 100% limit. The reference's countable income excludes farm rent, so net income is $1,269.19 and below the $1,304.17 limit, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"It included farm rent and failed a gross income limit instead of applying Texas's 165% BBCE threshold. The reference's gross income is $1,576.92 without farm rent, net income of $1,269.19 passes the net test, and the household receives the $23.84/month minimum allotment." +us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent in net income and found it above the 100% limit. Without farm rent, net income is $1,269.19, which passes the net test, so the one-person household receives the $23.84/month minimum allotment." +us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Counting farm rent put its net income above the limit. The reference's countable income is wages plus financial assistance, giving net income of $1,269.19 below the $1,304.17 limit, and the $23.84/month minimum allotment is paid." +us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and concluded that income was too high for any benefit, never applying the one-person minimum allotment. Without farm rent, net income is $1,269.19, which passes the net test, and the household receives $23.84/month." +us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Including farm rent put its net income too high for a benefit. The reference's net income without farm rent is $1,269.19, below the $1,304 limit, and because 30% of it is more than the $298 maximum, the one-person minimum allotment of $23.84/month is paid." +us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"It blamed income and assets for the $0, but Texas BBCE waives the asset test. Its answer matches counting farm rent, whereas the reference's net income without farm rent is $1,269.19, which passes the net test, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent together with the financial assistance and found the income limits exceeded. The reference's countable income leaves out farm rent, giving gross income of $1,576.92 (121% of the poverty guideline) and net income of $1,269.19 (97%), and pays the $23.84/month minimum allotment." +us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent and set SNAP to $0 because 30% of net income is more than the maximum, never applying the minimum allotment. The reference's net income is $1,269.19 without farm rent, which passes the net test, so the one-person minimum of $23.84/month is paid." +us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent in its $20,843 income total, which put net income above the 100% limit. Without farm rent, net income is $1,269.19, below the $1,304.17 limit, and the $23.84/month minimum allotment applies." +us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent to reach $20,843 and compared it with a 130% limit (about $20,700) instead of Texas's 165% BBCE limit. The reference's gross income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment is paid." +us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"It applied a 130% gross limit to income that includes farm rent and ignored Texas's 165% BBCE threshold. The reference's countable income without farm rent passes both tests (net income is $1,269.19), and the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"It counted the farm rent to reach $20,843 and applied the 130% gross limit, missing Texas's 165% BBCE limit. The reference's gross income is $18,923/year without farm rent, net income of $1,269/month passes, and the $23.84/month minimum allotment is paid." +us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"It denied categorical eligibility, even though Texas BBCE confers it through TANF non-cash assistance, and counted farm rent, which pushed net income too high. Without farm rent, net income is $1,269.19 and passes the net test, so the one-person minimum allotment of $23.84/month applies." +us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"It claimed that a single adult with wage income does not qualify for SNAP; no such exclusion exists. The household passes the gross test (121% of the poverty guideline) and the net test ($1,269.19 against $1,304.17), and it receives the $23.84/month one-person minimum allotment." +us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"It counted farm rent to reach $1,737/month and failed a 130% limit of about $1,695, missing Texas's 165% BBCE limit. The reference's gross income is $1,576.92 without farm rent, net income of $1,269.19 passes, and the minimum allotment of $23.84/month is paid." +us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"It correctly found that 30% of net income is more than the maximum, then threw that result out and substituted $3,576 by assuming a shelter deduction for which no housing costs were listed. With no shelter deduction, net income is $1,269.19, which is below the $1,304 limit, so the correct outcome is the one-person minimum allotment of $23.84/month, not a large benefit." +us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Its bare finding of ineligibility matches counting farm rent, which fails the net test. The reference's countable income leaves out farm rent, net income of $1,269.19 passes the 100% test, and the one-person minimum allotment of $23.84/month applies." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It itemized only $19,518. It left state income tax out of SALT ($4,064 instead of $9,431.59), skipped the 0.5%-of-AGI charitable floor, and counted $2,000 of non-deductible OTC purchases instead of the $2,434.80 Medicare Part B premium, so taxable income came to $78,077 rather than $72,807.33. It then computed about $11,889 at the correct 2026 brackets but replaced that with $8,971 through an arbitrary 'bracket inflation' adjustment that no rule supports." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its structure is right (itemize, $4,106 senior deduction, $4,872 QBI, 2026 brackets), but it put only $3,524 of state income tax into SALT ($7,588 versus $9,431.59). It also built medical from the $2,000 of non-deductible OTC purchases while leaving out the $2,434.80 Medicare Part B premium ($4,007 versus $4,441.86). Together these understate itemized deductions by about $2,277, so taxable income rises to $75,085 and tax rises by 22% of that gap." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $19,828.80 of taxable Social Security out of income, invented a $28,300 aged standard deduction, used 2024 brackets, and dropped the QBI and senior deductions. It then added $336 of Medicare payroll tax and an arbitrary $2,000 'safe harbor' adjustment to reach $10,268. Neither payroll tax nor withholding belongs in income tax before refundable credits; the correct tax comes from $72,807.33 of taxable income after itemizing $24,787.23." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its intermediate work entered SALT at the $10,000 cap instead of the $9,431.59 actually paid, dropped the 0.5%-of-AGI charitable floor, and took the full $6,000 senior deduction without the 6% phase-down on MAGI above $75,000 (correct amount $4,105.65), reaching about $10,711. It then submitted $5,489, a figure that matches none of the steps it computed." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It imputed about $4,500 of mortgage interest from the loan balance although no interest was listed, so mortgage interest is $0. It also left state income tax out of SALT, skipped the charitable floor, and omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. It computed $11,802 and then trimmed it to $11,210 with an unexplained 'rounding' adjustment instead of taxing the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It subtracted half of a self-employment tax (~$1,721) that does not exist here: the partnership/S-corp income carries no SE tax. It also undercounted itemized deductions (no state income tax in SALT; medical limited to $10,000 without the $2,434.80 Part B premium), so it chose the standard deduction with an unphased $6,000 senior deduction. It then cut its own ~$11,900 estimate to $6,900 by citing nonrefundable credits this household does not have." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It followed the correct structure but put only about $3,555 of state income tax into SALT ($7,618.50 versus $9,431.59). It also computed medical at $4,007 by counting the $2,000 of non-deductible OTC purchases instead of the $2,434.80 Medicare Part B premium (correct $4,441.86). Itemized deductions came out $2,247.63 short ($22,539.60 vs $24,787.23), and 22% of that gap is exactly its $494.49 overstatement." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It explicitly set aside the OBBBA senior deduction ($4,105.65 after phase-down), imputed $6,693 of mortgage interest from the loan balance even though no interest was listed, left state income tax out of SALT, and skipped the 0.5% charitable floor. It also used inflation-projected 2025 brackets (12% ending near $49,784) instead of the 2026 thresholds of $12,400 and $50,400, ending at $75,490 of taxable income instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction entirely. It itemized only $19,518, with SALT limited to the $4,064 real estate tax (no state income tax; correct SALT $9,431.59) and no 0.5% charitable floor, giving $82,183 of taxable income instead of $72,807.33. It also used guessed brackets (10% to $12,150, 12% to $49,400) instead of 2026's $12,400 and $50,400." +us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It applied the senior deduction ($4,106), QBI ($4,872), and 2026 brackets correctly, but its itemized total of about $21,188 is $3,599 below the correct $24,787.23. The correct components are charitable $10,913.78, SALT $9,431.59 (including $5,367.59 of state income tax), and medical $4,441.86 (including the $2,434.80 Medicare Part B premium). The resulting $76,407 of taxable income, instead of $72,807.33, adds about $792 of tax at 22%." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted half of a nonexistent self-employment tax ($1,721) from AGI, although the partnership/S-corp income carries no SE tax here. It also omitted the $4,105.65 senior deduction and itemized only $17,645 (no state income tax in SALT, no 0.5% charitable floor, medical without the $2,434.80 Part B premium). Its taxable income of $82,357 is about $9,550 above the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and skipped the 0.5%-of-AGI charitable floor. It also understated state income tax in SALT ($7,615.53 vs $9,431.59) and held medical to $2,007.04 by ignoring the $2,434.80 Medicare Part B premium (correct $4,441.86). On top of that it used wrong brackets (10% to $12,200, 12% to $49,400 instead of $12,400 and $50,400), producing $80,630.83 of taxable income." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the pre-TCJA sunset schedule, with a $5,350 personal exemption and 10%/15%/25% brackets. OBBBA made the TCJA rate structure permanent, so for 2026 personal exemptions stay at zero and the brackets are 10%/12%/22%. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $79,372.69 of taxable income implies itemized deductions of about $22,328 plus QBI, with no senior deduction. It therefore omitted the $4,105.65 OBBBA senior deduction, which itemizers receive, and understated itemized deductions by about $2,460 against the correct $24,787.23. The resulting $6,565 excess over the correct $72,807.33 of taxable income drives its higher tax." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used pre-TCJA law, with a $5,000 personal exemption and 10%/15%/25% brackets, even though OBBBA made the TCJA brackets permanent with no personal exemption. It also dropped the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Finally, it subtracted half of a nonexistent self-employment tax ($1,721), although the partnership/S-corp income carries no SE tax." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $110,094 of gross income counts 100% of the $23,328 of Social Security instead of the 85% maximum ($19,828.80) and ignores the $22 capital loss. Its roughly $81,300 of taxable income omits the $4,105.65 senior deduction and understates itemized deductions (correct $24,787.23). Its $13,919 also exceeds the 2026-bracket tax on $81,300 (about $12,598), so it applied the wrong rate schedule as well." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed tax under 'pre-TCJA' 2026 law with personal exemptions, but OBBBA made the TCJA brackets and the zero personal exemption permanent. Its derivation also includes neither the $4,872.35 QBI deduction nor the $4,105.65 senior deduction. The correct answer is the 10%/12%/22% tax on $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied the TCJA sunset, with a $5,050 personal exemption and the pre-TCJA 'sunset' brackets, although OBBBA made the TCJA rate schedule permanent with no personal exemption. It also omitted the $4,105.65 senior deduction and itemized only $19,256 versus the correct $24,787.23; SALT alone is $9,431.59 because it includes $5,367.59 of state income tax." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct computation is $106,572.56 AGI less $24,787.23 itemized, $4,872.35 QBI and $4,105.65 senior deductions, which leaves $72,807.33 taxed at $10,729.61. Its $9,334 matches about $66,460 of taxable income at the 2026 brackets, so it overstated total deductions by roughly $6,350." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($6,000 reduced by 6% of MAGI over $75,000, to $4,105.65), which itemizers also claim. Its itemized total of $22,018.04 also falls $2,769 short of the correct $24,787.23. Together these leave taxable income at $79,682.36, $6,875 above the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction instead of itemizing, although itemized deductions total $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59, medical $4,441.86). It also omitted the $4,105.65 senior deduction. Its $13,104.09 matches the tax on about $83,550 of taxable income (AGI less $18,150 and QBI only), $10,743 above the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It omitted both the $4,872.35 QBI deduction and the $4,105.65 senior deduction. It also itemized only $17,518: SALT limited to the $4,064 real estate tax instead of $9,431.59, no 0.5% charitable floor, and $2,007 of medical that leaves out the $2,434.80 Medicare Part B premium. That puts taxable income near $89,055 instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It asserted that QBI and the TCJA rates expired after 2025, so it applied a $5,500 personal exemption and 10%/15%/25% brackets. OBBBA made both the TCJA rate structure and the §199A deduction permanent, so the $4,872.35 QBI deduction applies, personal exemptions are zero, and the brackets are 10%/12%/22%. It also omitted the $4,105.65 senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction. It also itemized only $17,518 by excluding state income tax from SALT ($4,064 vs $9,431.59), skipping the 0.5% charitable floor, and limiting medical to $2,007 without the $2,434.80 Medicare Part B premium. Its $84,183 of taxable income is $11,376 above the correct $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and age-65 add-on wipe out all taxable income, ignoring the $106,572.56 of AGI from wages, partnership income, pension, IRA and 85% of Social Security. Even after $33,765.23 of deductions, $72,807.33 of taxable income remains and generates $10,729.61 at the 2026 brackets." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated the filer as living mainly on Social Security with modest taxable income. That overlooks $86,744 of wage, partnership, pension, IRA and interest income, which makes 85% of benefits taxable and puts AGI at $106,572.56. After $33,765.23 of total deductions, $72,807.33 remains taxable at rates up to 22%." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the TCJA sunset, with a personal exemption and the pre-2018 brackets, even though OBBBA made the TCJA brackets permanent and personal exemptions stay at zero in 2026. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Its itemized total was only $17,518 (no state income tax in SALT, no Part B premium in medical, no 0.5% charitable floor)." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It deducted half of a self-employment tax that does not exist, since the partnership/S-corp income carries no SE tax here, and it took the standard-deduction path instead of itemizing $24,787.23. Its $67,020 of taxable income is $5,787 below the correct $72,807.33, so its total deductions were overstated even though it chose the smaller deduction path." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction plus the $1,000 non-itemizer charitable deduction, which reproduces the standard-deduction alternative of $11,969.80. Itemized deductions of $24,787.23 exceed that $19,150: charitable $10,913.78 after the 0.5% floor, SALT $9,431.59 (real estate plus state income tax), and medical $4,441.86 (including the $2,434.80 Medicare Part B premium). Itemizing cuts tax by $1,240.19." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $11,540 matches about $76,490 of taxable income at the 2026 brackets, $3,684 above the correct $72,807.33. That means its itemized deductions fell about $3,684 short of the correct $24,787.23, which is charitable $10,913.78, SALT $9,431.59 (including $5,367.59 of state income tax), and medical $4,441.86 (including the $2,434.80 Medicare Part B premium)." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied the phased senior deduction ($4,105.63), QBI ($4,872.40), and 2026 brackets correctly but itemized only $20,639.33. With charitable at about $10,914 after the floor, its SALT plus medical comes to roughly $9,725, versus the correct $13,873.45 (SALT $9,431.59 including $5,367.59 of state income tax; medical $4,441.86 including the $2,434.80 Medicare Part B premium). The $4,148 shortfall taxed at 22% is exactly its $912.59 overstatement." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. Its implied itemized deductions of about $21,026 also fall $3,761 short of the correct $24,787.23. Taxable income of $80,675 instead of $72,807.33 produces its $1,731 overstatement at 22%." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It stated AGI as $96,572.80, dropping $10,000 from the $106,572.80 its own listed income sums to. That also inflated the senior deduction to $4,705.63, when the correct phase-down on $106,572.56 of MAGI gives $4,105.65. It then took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction instead of itemizing $24,787.23." +us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It applied the phased senior deduction, the $4,872.40 QBI deduction, and the 2026 brackets correctly, but its itemized deductions come to about $20,640 against the correct $24,787.23. The correct itemized total is charitable $10,913.78, SALT $9,431.59 (including $5,367.59 of state income tax), and medical $4,441.86 ($10,000 of other expenses plus the $2,434.80 Medicare Part B premium, with OTC purchases excluded). The $4,147 shortfall taxed at 22% is its $912.48 overstatement." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct computation reaches $72,807.33 of taxable income and $10,729.61 of tax, while its $10,234 matches about $70,555 of taxable income at the 2026 brackets. It therefore claimed roughly $2,250 more deductions than the correct $33,765.23 (itemized $24,787.23, QBI $4,872.35, senior $4,105.65)." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset law, with a $5,468 personal exemption and 10%/15%/25% brackets, although OBBBA made the TCJA brackets permanent with no personal exemption for 2026. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset took effect, using a $5,300 personal exemption and 10%/15%/25% brackets and declaring QBI expired. OBBBA made both the TCJA rate schedule and the §199A deduction permanent. As a result it dropped the $4,872.35 QBI deduction and the $4,105.65 senior deduction, and taxed $78,246 at the wrong rates." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It took the $18,150 aged standard deduction on the belief that it beats itemizing, but itemized deductions total $24,787.23 (charitable $10,913.78 after the 0.5% floor, SALT $9,431.59 including state income tax, medical $4,441.86 including the Medicare Part B premium). Even on the standard path it omitted the $1,000 non-itemizer charitable deduction, leaving $79,444.77 of taxable income instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It omitted the $4,105.65 OBBBA senior deduction, which itemizers also receive. It also put only $2,746 of state income tax into SALT ($6,810 versus $9,431.59) and skipped the 0.5%-of-AGI charitable floor. Taxable income of $79,437 instead of $72,807.33 drives its higher tax." +us,scenario_114,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied pre-TCJA brackets and a $5,300 personal exemption, although OBBBA made the TCJA brackets permanent and personal exemptions remain zero. It also omitted the $4,872.35 QBI deduction and the $4,105.65 senior deduction. Finally, it cut AGI to $104,852 by subtracting half of a self-employment tax the partnership/S-corp income does not incur." +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It computed itemized deductions of only $17,518 by excluding state income tax from SALT ($4,064 instead of $9,431.59), skipping the 0.5% charitable floor, and limiting medical to $2,007 without the $2,434.80 Medicare Part B premium. That led it to choose the standard deduction plus the $1,000 non-itemizer charitable deduction. That path yields $11,969.80, which is $1,240.19 more than itemizing the correct $24,787.23." +us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It acknowledged that taxable income is positive and that itemizing beats the standard deduction, then reported $0 without identifying any credit. With AGI of $106,572.56 and total deductions of $33,765.23, taxable income is $72,807.33, which produces $10,729.61 at the 2026 brackets with no nonrefundable credit to offset it." +us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It estimated itemized deductions at about $19,500 and chose the standard deduction, but itemized deductions total $24,787.23 (charitable $10,913.78, SALT $9,431.59 including $5,367.59 of state income tax, medical $4,441.86 including the $2,434.80 Medicare Part B premium). On its standard path it also omitted the $1,000 non-itemizer charitable deduction, leaving $79,445 of taxable income instead of $72,807.33." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left Social Security out of income, although 85% ($19,828.80) is taxable. It also used an outdated $15,750 aged standard deduction and skipped both itemizing and the senior deduction. Its reported $15,654 is not even the tax on its own $66,122 of taxable income, which the 2026 brackets tax at about $9,258." +us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It stated AGI as $80,164 although its own listed items sum to $106,572.80, cut QBI to $3,550, and took a $20,400 standard deduction instead of itemizing $24,787.23. It then subtracted about $2,471 from its $7,761.45 tax by treating the QBI deduction and charitable amounts as nonrefundable credits, which they are not." us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295." us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract." us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component." us,scenario_114,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly derived $1,736.55, then replaced it with $1,481.30 through an unsupported “program simplification/rounding” adjustment. No rounding or simplification changes the exact $1,407.40 and $329.15 components." us,scenario_114,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract." us,scenario_114,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly summed federal employee Social Security and Medicare taxes to $1,736.55, then added $39.15 as a purported Virginia state payroll tax. Virginia contributes no mandatory employee payroll-tax component to this output." -us,scenario_114,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model stated it ""treated"" the $24,362 partnership/S-corp income as self-employment earnings, applying 0.9235 and the full 15.3% rate to reach $3,441.90. Partnership/S-corp income is a distinct PolicyEngine input that feeds taxable income only; SE tax is assessed on self_employment_income, which is unlisted here and therefore 0, consistent with the exclusion of S-corp distributive shares (Rev. Rul. 59-221) and limited-partner shares (IRC §1402(a)(13)) from net earnings from self-employment. It also checked the wage base against combined wages, a step that is irrelevant once the SE base is zero." -us,scenario_114,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model placed the $24,362 partnership/S-corp income into the self-employment tax base, which excludes S-corp distributive shares and limited-partner shares; the household's self_employment_income is unlisted and thus 0, so no SE tax is assessed. It compounded the error by applying the 0.9235 factor twice ($24,362 × 0.9235 × 0.9235 × 0.153), inventing a second reduction that does not exist in the §1402(a)(12) computation, which is why its $3,440.29 sits below even its own first-pass $3,445.12." -us,scenario_114,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model explicitly treated partnership/S-corp income as self-employment earnings, computing $22,498 of net earnings and splitting it into 12.4% Social Security plus 2.9% Medicare for $3,442. Self-employment tax attaches only to net earnings from self-employment; a distributive share reported as partnership/S-corp income is not that base under Rev. Rul. 59-221 and IRC §1402(a)(13), and the household lists no self-employment income, so the correct liability is $0." -us,scenario_114,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model assumed the $24,362 was a ""general partner share"" and therefore SE-taxable, but general-partner status is an unlisted fact the prompt directs it to treat as false, and IRC §1402(a)(13) excludes a limited partner's distributive share while Rev. Rul. 59-221 excludes S-corp shares entirely. Having invented that status, it correctly applied 92.35% × 15.3% to reach $3,442.22 on a base that should have been the unlisted, and therefore zero, self-employment income." -us,scenario_114,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model rested its entire calculation on the premise that the partnership income is ""likely active partnership income subject to SE tax,"" supplying an active-participation fact the household never lists and that the prompt requires be treated as false. With self_employment_income at 0, SE tax is $0; the model instead produced $3,441.62 from its own arithmetic and then submitted $3,442.65, a figure its stated steps do not yield." -us,scenario_114,self_employment_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model asserted flatly that ""partnership/S-corp business income 24,362 is self-employment income"" and applied 0.9235 × 15.3% for $3,442. That equates a distributive share with net earnings from self-employment, which Rev. Rul. 59-221 and IRC §1402(a)(13) foreclose; the household reports no self-employment income, so the SE tax base is zero." -us,scenario_114,self_employment_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model declared ""partnership income $24,362 is self-employment income"" and computed 15.3% × 92.35% × $24,362 = $3,442.31. Partnership/S-corp income is a separate input that reaches taxable income but not the SE tax base; with self_employment_income unlisted and therefore 0, no self-employment tax is assessed." -us,scenario_114,self_employment_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied 15.3% of 92.35% of the $24,362 partnership income with no examination of whether that income constitutes net earnings from self-employment. S-corp distributive shares are excluded from SE tax by Rev. Rul. 59-221 and limited-partner shares by IRC §1402(a)(13); the household's self-employment income is unlisted and therefore 0, making the correct answer $0 rather than $3,442.27." -us,scenario_114,self_employment_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model began from the same wrong base — partnership/S-corp income ""treated here as self-employment income"" — and then layered on two fabricated rules: an ""AGI limitation"" capping the SE tax deduction at 80% and a ""Virginia state tax offset"" to SE tax; neither exists in IRC §164(f) or Virginia law, and SE tax is a federal liability unaffected by state income tax. Its own chain ended at $2,836.18 while it submitted $3,722.20, a number no step in its reasoning produces; the correct base is the unlisted, zero self-employment income." -us,scenario_114,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model identified ""net self-employment income"" as the partnership/S-corp income of $24,362, a base that excludes S-corp and limited-partner distributive shares under Rev. Rul. 59-221 and IRC §1402(a)(13). Its submitted $3,464.10 does not even match its own described formula (92.35% × 15.3% × $24,362 = $3,442.24), overstating by $21.86 on top of a base that should be $0." -us,scenario_114,self_employment_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model applied self-employment tax to the $24,362 partnership income to reach approximately $3,442, without testing whether a partnership/S-corp distributive share is net earnings from self-employment. It is not — Rev. Rul. 59-221 excludes S-corp shares and IRC §1402(a)(13) excludes limited-partner shares — and the household lists no self-employment income, so the SE tax base is zero." -us,scenario_114,self_employment_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model computed 15.3% of 92.35% of the $24,362 partnership income and checked it against the Social Security wage base, treating the distributive share as net earnings from self-employment. Partnership/S-corp income feeds taxable income but not the SE tax base in PolicyEngine, and self_employment_income is unlisted and therefore 0, so the liability is $0 rather than $3,442." -us,scenario_114,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model gave only ""On partnership income after deductions,"" showing it applied the SE tax to the $24,362 partnership/S-corp figure; $3,447 is what the roughly 14.13% effective SE rate yields on a base near $24,400. The correct derivation uses self_employment_income, which is unlisted and therefore 0, and a partnership/S-corp distributive share is not net earnings from self-employment under Rev. Rul. 59-221 and IRC §1402(a)(13)." -us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"The model converted the $24,362 partnership/S-corp income into $22,498.31 of net earnings and applied the 12.4% plus 2.9% combined rate for $3,442.24. That treats a distributive share as net earnings from self-employment, which S-corp treatment (Rev. Rul. 59-221) and the limited-partner exclusion (IRC §1402(a)(13)) rule out; with no self-employment income listed, the base and the tax are both zero." -us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for self_employment_tax, so the required key was absent from the submitted outputs. This is a contract failure in producing an answer, not a substantive misapplication of the self-employment tax rules." -us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model stated ""partnership or S-corp income of $24,362 is treated as self-employment income"" and derived $3,442.62 twice, then submitted $3,459.40, a value none of its steps produce. Both the base and the submitted number are wrong: partnership/S-corp distributive shares are outside the SE tax base under Rev. Rul. 59-221 and IRC §1402(a)(13), and the household's self-employment income is unlisted and therefore 0." -us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model applied SE tax to the $24,362 partnership/S-corp income and then added a phantom ""additional Medicare-equivalent portion on the remaining amount above the Social Security wage base,"" even though combined wages and business income of about $47,000 fall roughly $130,000 below the 2026 wage base, so no such tier exists. The base itself is wrong: self-employment tax reaches only net earnings from self-employment, which is 0 for this household, making the answer $0 rather than $3,486.63." -us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own walk reached VA taxable income of about $66,296 and tax of about $3,555, then it submitted $2,597, a figure its reasoning never derives and which it excused as ""slight variation in taxable SS and medical deduction."" It also omitted the $800 age-65 exemption, keeping only the $930 personal exemption, and understated itemized deductions at $19,518 versus the allowed $19,419.94 offsetting portion of that gap." -us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied Virginia's married-filing-jointly $75,000 age-deduction threshold to a single filer, leaving a residual $256 age deduction where the $50,000 single threshold zeroes the entire $12,000 deduction at AFAGI $86,743.77. It also carried itemized deductions of $18,985 instead of $19,419.94, so its taxable income of $65,773 sits $179 above the correct $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It summed the income items to a federal AGI of $27,752 when wages, partnership income, IRA distributions, pension, interest, and the capital loss alone total $86,744 before adding $19,828.80 of taxable Social Security, and it treated the $11,447 of charitable donations and $230 of employee business expenses as above-the-line reductions to AGI rather than itemized deductions. It then applied a $28,000 federal standard deduction to Virginia, which instead allows $19,419.94 of itemized deductions plus $1,730 of exemptions against VA AGI of $86,743.77, leaving $65,593.83 taxable rather than zero." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It substituted an $8,750 Virginia standard deduction and no exemptions for the $19,419.94 of itemized deductions plus $930 personal and $800 aged exemptions, computing $4,227 twice in its reasoning, and then submitted $1,854, a number nothing in its derivation produces. The deduction substitution alone inflated taxable income by $12,400 above the correct $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It ran the computation on the $8,500 standard deduction and $930 exemption to get $4,188, recognized that itemizing wins, and then applied an ad hoc $174 haircut instead of actually deducting the $19,419.94 itemized total and the $800 age-65 exemption. Carrying the itemized figure through gives taxable income of $65,593.83 and tax of $3,514.15, $500 below its estimate." -us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used Virginia's $8,500 standard deduction instead of the $19,419.94 of itemized deductions generated by $11,447 of charitable gifts, $4,064 of real estate taxes, and medical costs above the 7.5%-of-AGI floor, and then shaved the result by an unquantified ""personal exemptions credit"" rather than subtracting the $930 and $800 exemptions from taxable income. Its ~$76,500 taxable income overstates the correct $65,593.83 by about $10,900." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It treated Virginia's $930 personal and $800 age-65 exemptions as nonrefundable credits and subtracted the full $1,730 from tax; Virginia subtracts those amounts from taxable income, where they are worth $99.48 at the 5.75% rate. It compounded this by using a $9,000 standard deduction instead of the $19,419.94 itemized total, so its $77,744 taxable income exceeds the correct $65,593.83 by $12,150." -us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It granted the full $12,000 age deduction on the claim that income was ""under threshold,"" when Virginia reduces the deduction dollar-for-dollar above $50,000 of AFAGI and AFAGI here is $86,743.77, zeroing it. It also omitted the $930 and $800 exemptions and then submitted $2,450 against its own $2,947.50 computation." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It deducted a roughly $4,800 Virginia standard deduction plus an $800 aged addition instead of the $19,419.94 of itemized deductions Virginia allows this federal itemizer, and started from a federal AGI of $104,874 rather than $106,572.56. Its $78,515 taxable income exceeds the correct $65,593.83 by $12,921, which at 5.75% is the entire $743 overstatement." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It allowed the full $12,000 age deduction that Virginia phases out dollar-for-dollar above $50,000 of AFAGI and therefore zeroes at $86,743.77, and offset that with an $8,500 standard deduction and no exemptions in place of $19,419.94 of itemized deductions plus $1,730 of exemptions. The two errors partially cancel, leaving taxable income of $66,244 versus the correct $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its structure is right — VA AGI $86,743.77 after the Social Security subtraction, exemptions of $1,730, and the age deduction correctly zeroed — but its itemized total of $19,518.04 overstates the allowable $19,419.94 by $98.10 in the medical component computed against the 7.5%-of-AGI floor. That leaves taxable income of $65,495.96 instead of $65,593.83 and tax $5.63 light." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It inflated itemized deductions to $23,948.60, $4,528.66 above the $19,419.94 Virginia allows, and kept only the $800 aged exemption while dropping the $930 personal exemption. Its starting VA AGI was also about $1,701 below the correct $86,743.77, so it taxed $60,294.40 instead of $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,180 back-solves to VA taxable income of $77,174, which is VA AGI of $86,743.77 less roughly $9,570 of allowances — the Virginia standard-deduction path. The correct derivation subtracts $19,419.94 of itemized deductions and $1,730 of personal and aged exemptions to reach $65,593.83, $11,580 lower." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,726 back-solves to taxable income of about $69,278, which is VA AGI $86,743.77 less roughly $17,466 — the itemized total that results from computing medical expenses without the $2,000 of over-the-counter health costs and then skipping the $930 and $800 exemptions. Virginia's allowable $19,419.94 of itemized deductions plus $1,730 of exemptions gives $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It limited itemized deductions to $15,511 — charitable $11,447 plus real estate taxes $4,064 — dropping the medical expenses deductible above the 7.5%-of-AGI floor, which are the remaining $3,908.94 of the allowable $19,419.94. It also used $1,600 of exemptions instead of the $930 personal plus $800 aged amounts, taxing $69,633 against the correct $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The submitted $5,617 back-solves to Virginia taxable income of about $102,165, essentially the full federal AGI of $106,572.56 less a small allowance. It never applied Virginia's $19,828.80 subtraction for federally taxable Social Security, nor the $19,419.94 of itemized deductions and $1,730 of exemptions that bring taxable income to $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded the $2,000 of over-the-counter health expenses from deductible medical costs, producing state itemized deductions of $17,518.04 rather than the allowable $19,419.94. That single omission raised taxable income from $65,593.83 to $67,495.96 and accounts for the entire $109.37 overstatement of tax." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,623.52 is exactly Virginia's tax on $67,495.96, i.e. VA AGI $86,743.77 less $17,518.04 of itemized deductions and $1,730 of exemptions — the itemized total obtained by omitting the $2,000 of over-the-counter health expenses from deductible medical costs. Counting them yields $19,419.94 of itemized deductions, taxable income of $65,593.83, and $3,514.15 of tax." -us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It stated itemized deductions of $17,518, the total that results from computing medical expenses on the $10,000 of other medical costs alone and dropping the $2,000 of over-the-counter health expenses. Virginia's allowable itemized total is $19,419.94, so taxable income is $65,593.83 rather than the $67,496 it taxed." -us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The method it narrated — VA AGI $86,744 less a $9,000 standard deduction and $930 exemption at 2%–5.75% — produces about $4,200, yet it submitted $14,036.62, which exceeds 5.75% of the entire $106,572.56 federal AGI and therefore cannot come from any Virginia bracket application. It also invoked a 65+ ""pension income deduction"" Virginia does not have; the applicable $12,000 age deduction is fully phased out at AFAGI $86,743.77." -us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the $8,500 standard deduction rather than the $19,419.94 itemized total, took a $930 age-65 addition where Virginia's aged exemption is $800, and applied a fabricated bracket schedule of ""$270 on the first $8,000"" instead of the $720 accumulated through $17,000 (2% of $3,000, 3% of the next $2,000, 5% of the next $12,000). Its taxable income of $76,384 exceeds the correct $65,593.83 by $10,790." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted the household is ""fully sheltered by the standard deduction/exemptions"" without computing anything. VA AGI of $86,743.77 minus the largest available allowances — $19,419.94 of itemized deductions plus $1,730 of exemptions — still leaves $65,593.83 of taxable income, taxed at $720 plus 5.75% of the excess over $17,000." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It characterized a filer with $106,572.56 of federal AGI as having low taxable income because the income is ""mostly retirement,"" but Virginia's only retirement relief here is the $19,828.80 Social Security subtraction and an age deduction fully phased out at AFAGI $86,743.77 — the $9,600 IRA and $30,000 pension remain fully taxable. Taxable income is $65,593.83, not an amount below the deduction floor." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Every step matches the reference except the itemized total: it used $17,518, the figure obtained by deducting only the $10,000 of other medical expenses above the 7.5%-of-AGI floor and excluding the $2,000 of over-the-counter health expenses. Virginia's allowable itemized deductions are $19,419.94, giving taxable income of $65,593.83 rather than $67,496." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It applied ""the available age deduction,"" but Virginia reduces the $12,000 age deduction dollar-for-dollar for AFAGI above the $50,000 single threshold and AFAGI here is $86,743.77, zeroing it. The $51,647 of taxable income it taxed falls $13,947 below the correct $65,593.83, roughly the phantom age deduction plus a further deduction overstatement." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied an $8,750 standard deduction and only the $930 personal exemption, taxing $77,064. Virginia allows this federal itemizer $19,419.94 of itemized deductions and the additional $800 age-65 exemption, bringing taxable income to $65,593.83 — $11,470 lower." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $4,658 back-solves to Virginia taxable income of about $85,487, which is VA AGI of $86,743.77 with barely $1,257 of allowances. It never applied the $19,419.94 of itemized deductions or the $930 personal and $800 aged exemptions that bring taxable income to $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its VA AGI of $86,744, $1,730 of exemptions, and full phase-out of the age deduction are correct, but its itemized deductions of $16,985.18 fall $2,434.76 short of the allowable $19,419.94, chiefly by excluding the $2,000 of over-the-counter health expenses from deductible medical costs. That leaves taxable income of $68,028.82 instead of $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted ""VA standard deduction and exemptions offset tax"" with no computation. After Virginia's $19,828.80 Social Security subtraction, $19,419.94 of itemized deductions, and $1,730 of exemptions, $65,593.83 of taxable income remains from VA AGI of $86,743.77, taxed at $720 plus 5.75% of the excess over $17,000." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It reached the correct VA AGI of $86,744, correctly zeroed the age deduction, and used the right $1,730 of exemptions, but set itemized deductions at $17,518 — the total that drops the $2,000 of over-the-counter health expenses from the medical deduction. Virginia's allowable $19,419.94 gives taxable income of $65,593.83, not the $67,496 it taxed." -us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Every structural step is right — VA AGI $86,743.77, age deduction zeroed by the $50,000 phase-out, exemptions of $930 plus $800 — but its itemized total of $19,518 overstates the allowable $19,419.94 by $98.10 in the medical component net of the 7.5%-of-AGI floor. Taxable income is $65,593.83, not $65,496." -us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It took the full $12,000 age deduction that Virginia phases out to zero at AFAGI $86,743.77, omitted the $930 and $800 exemptions, and then applied a 5.3% top marginal rate instead of Virginia's 5.75%. Its taxable income of $55,226 falls $10,368 below the correct $65,593.83, and the rate error removes a further $172." -us,scenario_114,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It taxed $64,446 while its own stated components — federal AGI $106,573 less $19,829 of Social Security, $19,647 of itemized deductions, and a $930 exemption — arithmetically give $66,167, and it recognized only the $930 personal exemption while Virginia also grants an $800 age-65 exemption. The correct chain from VA AGI $86,743.77 less $19,419.94 itemized and $1,730 exemptions is $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the submission never reached a Virginia computation. The required derivation is VA AGI $86,743.77 less $19,419.94 of itemized deductions and $1,730 of exemptions, giving $65,593.83 taxed at $720 plus 5.75% of the excess over $17,000 = $3,514.15." -us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It asserted the filer does not itemize federally and applied Virginia's $8,500 standard deduction, but $11,447 of charitable gifts, $4,064 of real estate taxes, and the medical costs above the 7.5%-of-AGI floor make itemizing the larger federal deduction and unlock Virginia itemized deductions of $19,419.94. It also used a $930 age-65 addition where Virginia's aged exemption is $800, taxing $76,384 instead of $65,593.83." -us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It labeled a household with about $110,072 of income a ""low-income retiree"" and asserted that deductions plus a $930 exemption drive the liability to zero. Virginia taxable income is $65,593.83 after the $19,828.80 Social Security subtraction, $19,419.94 of itemized deductions, and $930 plus $800 of exemptions, producing $3,514.15 of tax." -us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It used the Social-Security-excluded AFAGI of $86,744 only to test the age deduction and never applied Virginia's $19,828.80 subtraction for federally taxable Social Security to VA AGI itself, taxing federal AGI of $106,573 less a $9,500 standard deduction. It also chose the standard deduction over the $19,419.94 itemized total and skipped the $1,730 of exemptions, so its $97,073 taxable income exceeds the correct $65,593.83 by $31,479." -us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It removed Social Security twice — leaving the $19,828.80 taxable portion out of federal AGI and then subtracting the full $23,328 again as a Virginia exemption — and took the $12,000 age deduction that phases out to zero at AFAGI $86,743.77, along with a nonexistent $1,000 Virginia standard deduction and a $1,721.31 half-SE-tax adjustment. It then submitted $2,453.71 against its own $2,542.44 computation." -us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It started from a federal AGI of $80,164 rather than $106,572.56, applied the $12,000 age deduction that Virginia zeroes at AFAGI $86,743.77, and used a $9,000 standard deduction in place of the $19,419.94 itemized total. It then subtracted $745.50 of Virginia nonrefundable credits, none of which any listed fact supports, from an already understated $3,274.34." +us,scenario_114,self_employment_tax,claude-fable-5,llm_error,payroll_tax_base,False,"Counted the $24,362 partnership/S-corp income as net earnings from self-employment and taxed 92.35% of it at 15.3%. S-corp income and a non-general-partner distributive share are not SE earnings. The household has no self-employment income, so SE tax is $0." +us,scenario_114,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as SE income, but that pass-through income is not net earnings from self-employment, so SE tax is $0. It also applied the 0.9235 factor twice (24,362 × 0.9235 × 0.9235 × 0.153). This mixes up the SE income-tax deduction with the net-earnings reduction." +us,scenario_114,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"Treated the partnership/S-corp income as SE earnings and computed 12.4% plus 2.9% on 92.35% of $24,362. That pass-through income does not create net earnings from self-employment when general-partner status is not given, and S-corp income never does. SE tax is $0." +us,scenario_114,self_employment_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"Assumed the $24,362 was a general partner's share subject to SE tax and taxed 92.35% of it at 15.3%. The prompt gives no general-partner status, and unlisted statuses are false. S-corp income is never SE income, so the SE earnings base is $0 and SE tax is $0." +us,scenario_114,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"Guessed that the partnership/S-corp income was 'likely active partnership income subject to SE tax' and taxed 92.35% of it at 15.3%. It then padded its own $3,441.62 result to $3,442.65. The pass-through amount is not net earnings from self-employment, so SE tax is $0." +us,scenario_114,self_employment_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"Called the $24,362 partnership/S-corp income 'self-employment income' and applied 15.3% to 92.35% of it. That income is not in the SE earnings base, so SE tax is $0." +us,scenario_114,self_employment_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Asserted that the partnership income is self-employment income and computed 15.3% × 92.35% × $24,362. That pass-through income is not net earnings from self-employment, so the household's SE tax is $0." +us,scenario_114,self_employment_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"Applied 15.3% to 92.35% of the $24,362 partnership/S-corp income as if it were SE earnings. S-corp income and a limited or unspecified partner's share are excluded from net earnings from self-employment, so SE tax is $0." +us,scenario_114,self_employment_tax,glm-5.2,llm_error,payroll_tax_base,False,"Treated the partnership/S-corp income as SE earnings, which is wrong because it is not net earnings from self-employment, so SE tax is $0. It then invented an '80% AGI limitation' and a 'Virginia state tax offset' that do not exist in SE tax law. Those raised its own $3,441.96 figure to $3,722.20." +us,scenario_114,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as net SE income and applied the combined Social Security and Medicare rate to 92.35% of it. Its $3,464.10 does not even match the $3,442 that calculation gives. The pass-through income is not SE earnings, so SE tax is $0." +us,scenario_114,self_employment_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"Applied SE tax to the $24,362 partnership income, which assumes it is a general partner's net earnings from self-employment. No such status is given and S-corp income is never SE income, so the SE earnings base is $0 and SE tax is $0." +us,scenario_114,self_employment_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"Computed 15.3% of 92.35% of the $24,362 partnership income as SE tax. That pass-through income is not net earnings from self-employment for this household, so SE tax is $0." +us,scenario_114,self_employment_tax,gpt-6-luna,llm_error,payroll_tax_base,False,"Openly assumed that the $24,362 partnership income is subject to SE tax and applied 15.3% to 92.35% of it. The prompt says unlisted statuses are false, so general-partner status cannot be inferred, and S-corp income is never SE income. SE tax is $0." +us,scenario_114,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"Taxed the partnership income 'after deductions' as SE earnings. Its $3,447 matches 15.3% of roughly 92.35% of $24,362. That pass-through income is not net earnings from self-employment, so SE tax is $0." +us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"Treated the partnership/S-corp income as producing net SE earnings of 92.35% × $24,362 and applied 15.3%. S-corp income and a partner share with no general-partner status are excluded from SE earnings, so SE tax is $0." +us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no self_employment_tax value and no explanation, so there is no substantive calculation to assess. The correct output is $0 because the household has no net earnings from self-employment." +us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as SE income and computed $3,442.62. It then submitted an unexplained $3,459.40. The pass-through income is not net earnings from self-employment, so SE tax is $0." +us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as net SE income, which is the core error because it is not SE earnings, so SE tax is $0. It also added a made-up 'additional Medicare-equivalent portion above the Social Security wage base'. No income here comes near that base, and this pushed its answer to $3,486.63." +us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own path got taxable income of $66,296 and tax of about $3,555, but it then submitted $2,597, an unexplained cut of about $958 with no rule behind it. That path also left out the $800 age-65 exemption and overstated itemized deductions at $19,518 instead of $19,419.94. Fixing those gives taxable income of $65,593.83 and tax of $3,514.15." +us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It phased out the age deduction from the $75,000 married-joint threshold and kept $256. For a single filer the threshold is $50,000, and AFAGI of $86,744 wipes the $12,000 deduction out entirely. It also put itemized deductions at $18,985 by applying a 0.5%-of-AGI charitable floor, $435 below the $19,419.94 Virginia itemized deduction, so its taxable income of $65,773 ends up $179 too high." +us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It shrank $86,744 of non-Social Security income to an 'AGI' of $27,752 and applied a $28,000 federal-style standard deduction. It then claimed zero tax, even though Virginia's 2% bracket starts at the first dollar. The correct base is VA AGI of $86,743.77, minus $19,419.94 itemized deductions and $1,730 exemptions, which leaves $65,593.83 taxable." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning got $4,227, but it submitted $1,854, which nothing in its derivation supports. Even the $4,227 path is wrong: it used an $8,750 standard deduction instead of the $19,419.94 Virginia itemized deductions (the filer itemizes federally), and it left out the $930 personal and $800 age-65 exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It computed tax with the $8,500 standard deduction and only the $930 exemption, then vaguely 'adjusted for itemizing' to $4,014. That figure implies taxable income of about $74,300, meaning total deductions of only about $12,450. The correct figure is $21,149.94 of deductions ($19,419.94 itemized plus $1,730 exemptions); it never computed the itemized total and dropped the $800 aged exemption." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the $8,500 standard deduction instead of Virginia itemized deductions of $19,419.94, which gave taxable income of about $76,500. It then took about $281 off the tax as 'personal exemption credits'. Virginia's $930 personal and $800 aged exemptions are subtractions from taxable income, not credits." +us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,age_disability,False,"It took only the $930 personal exemption and left out the $800 Virginia exemption for a filer 65 or older. It also used $19,518 of itemized deductions instead of $19,419.94. Together these put taxable income at $66,296 instead of $65,593.83, so its tax is about $40 high." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $9,000 standard deduction instead of the $19,419.94 Virginia itemized deductions the filer qualifies for as a federal itemizer. It then subtracted the $930 personal and $800 aged exemptions dollar-for-dollar from tax as credits. They actually reduce taxable income, where they are worth only about $99 at the 5.75% rate." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It allowed the full $12,000 age deduction, but for a single filer it drops $1 for each $1 of AFAGI above $50,000, so AFAGI of $86,744 eliminates it. It also left out the $1,730 in exemptions and then cut its own $2,947.50 result to $2,450 with no supporting step." +us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It put itemized deductions at $17,518 by counting only the $10,000 of other medical expenses and leaving out the $2,000 of over-the-counter health expenses. The Virginia itemized deduction is $19,419.94. It also took only the $930 exemption without the $800 age-65 exemption, so taxable income came out at $68,296 instead of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It understated federal AGI at $104,874 instead of $106,573. It also applied a roughly $4,800 standard deduction plus an $800 'aged addition' instead of the $19,419.94 Virginia itemized deductions, which pushed taxable income to $78,515 instead of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It subtracted the full $12,000 age deduction, but single-filer AFAGI of $86,744 is more than $12,000 over the $50,000 threshold, so the deduction is zero. It also used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and left out the $1,730 in exemptions. Its $66,244 taxable income is close to the correct figure only because these errors offset." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Every step matches the correct derivation (Social Security subtraction, zero age deduction, $1,730 exemptions, brackets) except the itemized total. It used $19,518.04 of Virginia itemized deductions, $98.10 more than the $19,419.94 allowed, which lowers taxable income to $65,495.96 instead of $65,593.83 and tax by $5.63." +us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed Virginia taxable income from full federal AGI of $106,572.80 without subtracting the $19,828.80 of federally taxable Social Security, which Virginia exempts. It also used $17,518 of itemized deductions instead of $19,419.94 and left out the $800 age-65 exemption." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It overstated itemized deductions at $23,948.60, which is $4,528.66 more than the $19,419.94 allowed. It took only the $800 aged exemption and dropped the $930 personal exemption. Its stated taxable income of $60,294.40 does not even match its own inputs ($86,744 − $23,948.60 − $800 = $61,995.40)." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $4,180 implies taxable income of about $77,170. That matches VA AGI of $86,744 minus a roughly $8,500–8,750 standard deduction and a single $930 exemption. The correct deductions are $19,419.94 of Virginia itemized deductions plus $1,730 of personal and age exemptions, which leave $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $4,180 would be the standard-deduction path; its actual $3,726 implies taxable income of about $69,280, meaning only about $17,470 was deducted from VA AGI of $86,744 instead of $21,149.94. That shortfall matches a $17,518 itemized total (counting only the $10,000 of other medical expenses) with the $1,730 in exemptions left out." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $15,511 of itemized deductions covers only charitable gifts and real estate taxes and leaves out the medical expense deduction. That deduction is the $12,000 of medical costs above the 7.5%-of-AGI floor and is part of the $19,419.94 total. It also used a $1,600 exemption instead of Virginia's $930 personal plus $800 age-65 exemptions ($1,730)." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $5,617 implies taxable income of about $102,200. That is federal AGI of $106,573 with no $19,828.80 Social Security subtraction and only about a $4,400 deduction. The correct steps are to subtract Social Security, then $19,419.94 of itemized deductions and $1,730 of exemptions, leaving $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It computed the medical deduction on the $10,000 of other medical expenses only and left out the $2,000 of over-the-counter health expenses. That gave $17,518.04 of itemized deductions instead of $19,419.94. Taxable income came out at $67,495.96 instead of $65,593.83, so the tax is $109 too high." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,623.52 matches itemized deductions of $17,518.04, where the medical deduction counts only the $10,000 of other medical expenses and leaves out the $2,000 of over-the-counter health expenses. The Virginia itemized deduction is $19,419.94, so its taxable income is $1,902 too high." +us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used $17,518 of itemized deductions, with the medical deduction built from the $10,000 of other medical expenses only and the $2,000 of over-the-counter health expenses left out. Against the $19,419.94 Virginia itemized deduction, that makes taxable income $67,496 instead of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"Its described path ($106,573 − $19,829 − $9,000 standard deduction − $930) gives about $76,800 of taxable income and about $4,200 of tax. It instead submitted $14,036.62, which implies roughly $248,000 of taxable income, far more than the household's total income. The path itself also wrongly uses the standard deduction instead of $19,419.94 of itemized deductions and invents a pension deduction." +us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the $8,500 standard deduction instead of $19,419.94 of Virginia itemized deductions and counted a $930 age exemption instead of $800. It also made arithmetic errors: $106,573 − $19,829 comes out as $87,744 instead of $86,744. It then computed tax as $270 plus 5.75% of the amount over $8,000 instead of $720 plus 5.75% of the amount over $17,000." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions shelter all income. After the Social Security subtraction, VA AGI is still $86,743.77, and $19,419.94 of itemized deductions plus $1,730 of exemptions leave $65,593.83 taxable, which produces $3,514.15 of tax." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated retirement income as largely sheltered, but Virginia subtracts only Social Security. The $30,000 pension and $9,600 IRA distributions stay fully taxable, and the $12,000 age deduction is fully phased out at AFAGI of $86,744. That leaves $65,593.83 of taxable income after itemized deductions and exemptions, not $0." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used $17,518 of itemized deductions, computing the medical deduction from the $10,000 of other medical expenses and leaving out the $2,000 of over-the-counter health expenses. The Virginia itemized deduction is $19,419.94, so its taxable income of $67,496 is $1,902 too high." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It applied the $12,000 age deduction as 'available'. For a single filer it is reduced $1 for $1 above $50,000 of AFAGI, so $86,744 of AFAGI eliminates it. That pushed taxable income down to about $51,647 instead of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used an $8,750 standard deduction and only the $930 personal exemption. The filer itemizes federally, so Virginia allows $19,419.94 of itemized deductions plus both the $930 personal and $800 age-65 exemptions. Its taxable income of $77,064 is $11,470 too high." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,658 implies taxable income of about $85,490. That is what results from taking about $21,000 of itemized deductions and exemptions from federal AGI of $106,573 without first subtracting the $19,828.80 of taxable Social Security, which Virginia exempts." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It put itemized deductions at $16,985.18 by applying a 0.5%-of-AGI floor to charitable gifts and counting only the $10,000 of other medical expenses. The Virginia itemized deduction is $19,419.94, so its taxable income of $68,028.82 is $2,435 too high." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $3,397 implies taxable income of about $63,560. That is about $2,035 below the correct $65,593.83, which means it overstated itemized deductions beyond the $19,419.94 allowed, while correctly subtracting Social Security and taking the $1,730 of exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $57,229 means roughly $29,500 was deducted from VA AGI instead of $21,149.94. That matches deducting all $12,000 of medical expenses without the 7.5%-of-AGI floor, which limits the medical deduction to about $4,000 of the $19,419.94 itemized total." +us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used $16,985.18 of itemized deductions, applying a 0.5%-of-AGI charitable floor and counting only the $10,000 of other medical expenses, instead of the $19,419.94 Virginia itemized deduction. Its taxable income is $68,028.82 instead of $65,593.83, and it adds another $0.50 arithmetic slip on top." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted the standard deduction and exemptions offset all tax. After the Social Security subtraction, VA AGI is still $86,743.77, and deductions of $19,419.94 itemized plus $1,730 exemptions leave $65,593.83 taxable, which yields $3,514.15." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used $17,518 of Virginia itemized deductions, with the medical deduction built from the $10,000 of other medical expenses and the $2,000 of over-the-counter health expenses left out, instead of $19,419.94. Its taxable income of $67,496 is $1,902 above the correct $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Everything matches the correct derivation except the itemized total. It used $19,518 of itemized deductions, $98 more than the $19,419.94 Virginia itemized deduction, which lowers taxable income to $65,496 instead of $65,593.83 and tax by about $5." +us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It assumed the filer took the federal standard deduction and so used Virginia's $8,500 standard deduction. In fact, $11,447 of charitable gifts, $4,064 of real estate taxes and the medical deduction make the filer a federal itemizer, so Virginia allows $19,419.94 of itemized deductions. Its taxable income of $76,514 is $10,920 too high." +us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It taxed income above $17,000 at 5.3% instead of Virginia's 5.75% top rate. It also subtracted the $12,000 age deduction, which is fully phased out at a single filer's AFAGI of $86,744, and left out the $1,730 of personal and aged exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its own inputs ($86,744 − $19,647 itemized − $930) give $66,167, yet it taxed $64,446, which is $1,148 below the correct $65,593.83. It also left out the $800 age-65 exemption and overstated itemized deductions compared with $19,419.94." +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for state_income_tax_before_refundable_credits, so there is no substantive computation to evaluate." +us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It assumed the filer took the federal standard deduction and used Virginia's $8,500 standard deduction instead of the $19,419.94 of itemized deductions available to a federal itemizer. It also counted a $930 age-65 exemption instead of $800, which put taxable income at $76,384 instead of $65,593.83." +us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the filer as a low-income retiree with about zero tax, but VA AGI after the Social Security subtraction is $86,743.77. Only $19,419.94 of itemized deductions and $1,730 of exemptions apply, leaving $65,593.83 taxable." +us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It computed Virginia taxable income as federal AGI of $106,573 minus a $9,500 standard deduction and never subtracted the $19,828.80 of taxable Social Security that Virginia exempts. It also used the standard deduction instead of $19,419.94 of itemized deductions and left out the $930 and $800 exemptions." +us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It made several errors. It invented a half-self-employment-tax deduction on partnership income and used a $1,000 standard deduction. It applied the $12,000 age deduction, which is fully phased out at AFAGI of $86,744. It subtracted $23,328 of Social Security even though its AGI already excluded it, a double subtraction. Finally, it submitted $2,453.71, which does not match its own computed $2,542.44." +us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It understated AGI at $80,164, when VA AGI is $86,743.77, and used a $9,000 standard deduction instead of $19,419.94 of itemized deductions. It subtracted a $12,000 age deduction that is fully phased out for a single filer with AFAGI of $86,744, and it took $745.50 of Virginia nonrefundable credits this filer does not qualify for." us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility." us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through." @@ -8196,51 +8967,59 @@ us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility." us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility." us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit." -us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Stated that ""tax-exempt interest is not taxable"" in Alabama, carrying the federal §103 exclusion into a state that defines gross income independently and exempts only U.S.- and Alabama-obligation interest, so it zeroed an Alabama AGI that is actually $4,734. After the $3,000 single standard deduction and $1,500 personal exemption, $234 remains taxable at Alabama's 2% first bracket, producing $4.68 rather than $0." -us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Asserted that federally tax-exempt interest is outside the Alabama base; Alabama does not conform to the federal exclusion and taxes municipal interest other than U.S. and Alabama obligations, so the $4,734 is Alabama AGI in full. Its own two-step logic then fails at the deduction stage: $4,734 less the $3,000 standard deduction and $1,500 exemption leaves $234, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Invoked an ""elderly exemption"" for age 77 on top of the standard deduction; Alabama provides no age-65 or blindness addition to its $3,000 single standard deduction and $1,500 personal exemption, capping total deductions at $4,500. It also kept the $4,734 of tax-exempt interest out of the Alabama base, where Alabama's non-conformity to the federal exclusion puts it in full, leaving $234 taxable and $4.68 of tax at the 2% bracket." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Concluded the head has ""no other taxable income"" because tax-exempt interest ""is also not taxed"" in Alabama, missing that Alabama's separately defined gross income includes interest that is exempt federally unless it comes from U.S. or Alabama obligations. Alabama AGI is therefore $4,734, and after the $4,500 of standard deduction plus personal exemption, $234 is taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Declared Alabama AGI to be $0 by excluding both the Social Security benefits (correct) and the $4,734 of tax-exempt interest (incorrect — Alabama does not adopt the federal §103 exclusion). Carrying the interest into the base yields $4,734 AGI, $234 of taxable income after the $3,000 standard deduction and $1,500 exemption, and $4.68 at the 2% first-bracket rate." -us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Wrote that Alabama ""taxes no tax-exempt interest,"" applying federal conformity that Alabama does not grant; Alabama exempts only U.S.- and Alabama-obligation interest, so the $4,734 is fully taxable at the state level. That single omission is the entire gap: $4,734 − $4,500 in deductions = $234 taxed at 2% = $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Explicitly claimed ""tax-exempt interest is not included in Alabama gross income,"" which reverses Alabama's rule: the state computes gross income under its own statute and exempts only interest on U.S. and Alabama obligations. With $4,734 in the base and $4,500 of standard deduction plus personal exemption, taxable income is $234 and the 2% first bracket yields $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Treated both income items as non-taxable in Alabama; the Social Security exemption is right but the tax-exempt interest exclusion is not, since Alabama does not follow the federal exclusion for non-Alabama municipal interest. Including the $4,734 and subtracting the $3,000 standard deduction and $1,500 exemption leaves $234 taxable, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Relied on ""standard/elderly deductions exceeding any includable amount,"" but Alabama's single-filer deductions here total exactly $4,500 ($3,000 standard plus $1,500 personal exemption) with no elderly add-on, which is less than the $4,734 of Alabama-taxable interest. The $234 remainder is taxed at Alabama's 2% first bracket for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Stated that tax-exempt interest ""is also not taxed"" in Alabama and therefore found no income at all; Alabama's independently defined gross income includes that $4,734 because the state exempts only U.S.- and Alabama-obligation interest. Net of the $4,500 standard deduction plus personal exemption, $234 is taxable at 2%, giving $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Correctly added the tax-exempt interest to the Alabama base but used a fabricated ""$4,500 age 65+ standard deduction""; Alabama's single standard deduction tops out at $3,000 with no age-based increase, so total deductions are $4,500 including the $1,500 exemption, not $6,000. That leaves $234 of taxable income and $4.68 of tax at the 2% rate instead of $0." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Asserted Alabama's combined standard deduction and personal exemption for a single filer over 65 is ""at least $5,000""; the actual figures are $3,000 and $1,500 for $4,500 total, with no over-65 enhancement in Alabama. Since $4,734 of interest exceeds $4,500, $234 is taxable at the 2% first bracket for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Stopped at the Social Security exemption and invoked a nonexistent zero-tax threshold; Alabama taxes from the first dollar of taxable income at 2% and includes the $4,734 of federally tax-exempt interest in its own gross income. After the $4,500 of standard deduction and personal exemption, $234 remains and the tax is $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Reasoned only from the Social Security exemption and never accounted for the $4,734 of tax-exempt interest, which Alabama includes in gross income because it does not adopt the federal exclusion for non-Alabama municipal interest. That interest less $4,500 of standard deduction and personal exemption gives $234 taxable and $4.68 of tax at 2%." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Excluded the tax-exempt interest from Alabama taxable income alongside Social Security; Alabama exempts only interest on U.S. and Alabama obligations, so the $4,734 stays in the state base. Subtracting the $3,000 standard deduction and $1,500 personal exemption leaves $234, taxed at the 2% first bracket for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Claimed taxable income falls ""below the threshold for Alabama state income tax,"" but Alabama has no zero-tax floor above its deductions — the first $500 of taxable income is taxed at 2%. Alabama AGI of $4,734 (tax-exempt interest is in the state base) less $4,500 of deductions leaves $234, so the tax is $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Cited only Alabama's Social Security exemption and concluded there is zero taxable state income, silently dropping the $4,734 of tax-exempt interest that Alabama includes in gross income under its own non-conforming definition. That interest less $4,500 of standard deduction and personal exemption produces $234 taxable and $4.68 at the 2% rate." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Concluded the head has ""no other taxable income"" beyond exempt Social Security, overlooking that the $4,734 of federally tax-exempt interest is fully taxable in Alabama, which exempts only U.S.- and Alabama-obligation interest. Net of $4,500 in deductions, $234 is taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Rested the whole answer on the Alabama Social Security exemption and never brought the $4,734 of tax-exempt interest into the state base, where Alabama's independent gross-income definition places it. Taxable income is $234 after the $3,000 standard deduction and $1,500 exemption, yielding $4.68 at the 2% first bracket." -us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Stated that Alabama exempts ""federally tax-exempt interest,"" which is the opposite of Alabama's rule: the state taxes such interest unless it arises from U.S. or Alabama obligations, so Alabama AGI is $4,734, not $0. Deductions of $3,000 standard plus $1,500 exemption leave $234 taxable and $4.68 of tax at 2%." -us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Declared the tax-exempt interest ""not included in the Alabama AGI base"" before deductions; Alabama does include it, giving AGI of $4,734 rather than $0. The $3,000 standard deduction and $1,500 personal exemption it correctly named then leave $234 taxable, which the 2% first bracket turns into $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Equated ""no taxable wages"" with no Alabama-taxable income, dismissing the $4,734 of tax-exempt interest that Alabama includes in gross income because it does not conform to the federal §103 exclusion. That interest less $4,500 of standard deduction and personal exemption leaves $234, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted without computation that taxable income after deductions is ""effectively 0""; the actual Alabama arithmetic is $4,734 of interest income minus $3,000 standard deduction and $1,500 personal exemption = $234. Alabama's 2% first-bracket rate on that $234 gives $4.68, and its answer is consistent only with omitting the interest or inflating deductions past $4,734." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Concluded no Alabama-taxable income is ""listed"" after deductions, treating the $4,734 of tax-exempt interest as outside the state base; Alabama taxes that interest because it exempts only U.S. and Alabama obligations. With $4,500 of standard deduction plus personal exemption, $234 remains taxable at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Chained Alabama taxable income to the absence of taxable federal income, but Alabama starts from its own gross-income definition rather than federal AGI and therefore includes the $4,734 of federally tax-exempt interest. Less $4,500 in deductions, $234 is taxable and the 2% bracket yields $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Said the reported tax-exempt interest ""does not produce Alabama taxable income,"" reversing Alabama's non-conformity rule under which that $4,734 is fully in state gross income. Subtracting the $3,000 standard deduction and $1,500 personal exemption leaves $234, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Found ""no state-taxable income"" after the Social Security exclusion, ignoring the $4,734 of tax-exempt interest that Alabama includes because it does not adopt the federal exclusion for non-Alabama municipal interest. That base less $4,500 of deductions leaves $234 taxable and $4.68 of tax at Alabama's 2% first bracket." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Stated there is ""no other taxable income"" beyond exempt Social Security, dropping the $4,734 of tax-exempt interest that Alabama taxes under its own gross-income statute. With $3,000 standard deduction and $1,500 personal exemption, $234 is taxable at 2%, giving $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Added the tax-exempt interest back but assumed Alabama's personal exemption plus a single filer age 65+ standard deduction ""fully offset"" $4,734; Alabama's actual total is $4,500 ($3,000 standard, no age enhancement, plus $1,500 exemption). The $234 shortfall is taxed at Alabama's 2% first-bracket rate for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Claimed Alabama taxable income is $0 ""after the standard deduction and personal exemption,"" but those total only $4,500 against $4,734 of Alabama-taxable interest, which the state includes despite its federal exemption. The $234 residual is taxed at 2%, producing $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Hedged the interest into the base but assumed ""sufficient exemptions/deductions"" to zero it out; Alabama's single-filer deductions cap at $3,000 standard plus $1,500 exemption = $4,500, below the $4,734 of interest. The uncovered $234 is taxed at Alabama's 2% first bracket for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of $0 and assumed the standard deduction and personal exemption would erase any added-back interest; Alabama's own gross income includes the full $4,734, and its $3,000 + $1,500 = $4,500 of deductions falls $234 short. Alabama's 2% first-bracket rate on that $234 gives $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Held that neither income item is ""taxable under Alabama law,"" which is wrong for the $4,734 of interest: Alabama exempts only U.S.- and Alabama-obligation interest, not federally exempt municipal interest generally. Alabama AGI of $4,734 less $4,500 of standard deduction and personal exemption leaves $234, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Priced Alabama's single standard deduction plus $1,500 exemption at ""at least $5,500""; the maximum single standard deduction is $3,000, so total deductions are $4,500 against $4,734 of Alabama-taxable interest. The $234 difference is taxed at Alabama's 2% first bracket for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Asserted ""Alabama has no state income tax,"" which is flatly false — Alabama levies a graduated individual income tax of 2%/4%/5% on single filers. Applying it here: $4,734 of state-taxable interest less $3,000 standard deduction and $1,500 personal exemption leaves $234, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Stated that Alabama ""exempts federally tax-exempt interest,"" inverting the state's non-conformity: only U.S. and Alabama obligation interest is exempt, so state AGI is $4,734 rather than $0. After $4,500 of standard deduction and personal exemption, the 2% bracket applies to $234, yielding $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Labeled the $4,734 ""not taxable"" for Alabama on the strength of its federal exemption, missing that Alabama computes gross income under its own statute and taxes that interest. Subtracting the $3,000 standard deduction and $1,500 personal exemption leaves $234 of taxable income, taxed at 2% for $4.68." -us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted Alabama taxable income is $0 after the standard deduction with no supporting arithmetic; the actual computation is $4,734 of state-taxable interest minus a $3,000 standard deduction and $1,500 personal exemption = $234. Alabama's 2% first-bracket rate makes that $4.68, and its $0 is consistent only with excluding the interest from the Alabama base." +us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the $4,734 of federally tax-exempt interest as excluded from Alabama income. Alabama taxes that interest unless it comes from Alabama or U.S. obligations, so Alabama AGI is $4,734. After the $3,000 standard deduction and $1,500 personal exemption, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model said tax-exempt interest is not taxable in Alabama. Alabama does not adopt the federal municipal-interest exclusion, so the $4,734 is Alabama AGI. It exceeds the $4,500 standard deduction plus personal exemption by $234, which is taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest from Alabama income and pointed to unspecified 'elderly exemptions' for age 77. Alabama includes the $4,734 of interest in AGI and has no age-based extra deduction for interest income. The $3,000 standard deduction plus $1,500 exemption leaves $234 taxable at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that tax-exempt interest is not taxed by Alabama. Alabama adds the $4,734 of federally exempt interest to Alabama AGI. Subtracting the $4,500 of standard deduction and exemption leaves $234, and 2% of that is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model set Alabama AGI to $0 by grouping the tax-exempt interest with exempt Social Security. Alabama taxes that interest, so AGI is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234 of taxable income, which produces $4.68 at 2%." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model said Alabama 'taxes no tax-exempt interest'. That is wrong: Alabama includes federally exempt interest from non-Alabama obligations in AGI. The $4,734 minus $4,500 of deductions leaves $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model handled only the Social Security exemption and never counted the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model stated that tax-exempt interest is not included in Alabama gross income. Alabama includes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not subject to Alabama income tax, but Alabama does not follow the federal muni-interest exclusion. The $3,000 standard deduction and $1,500 exemption leave $234 taxable, which yields $4.68 at 2%." +us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model started from a federal AGI of $0 and never added back the $4,734 of federally tax-exempt interest, which Alabama requires. Alabama AGI of $4,734 minus $4,500 of deductions is $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model excluded the tax-exempt interest and relied on nonexistent 'elderly' deductions to reach $0. Alabama includes the $4,734 in AGI. A single filer gets only the $3,000 standard deduction and $1,500 exemption, which leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model asserted that tax-exempt interest is not taxed in Alabama. Alabama taxes federally exempt interest from non-Alabama obligations, so AGI is $4,734. Subtracting $4,500 of deductions leaves $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model correctly added back the interest but used a $4,500 'age 65+' standard deduction. Alabama has no age-65 add-on, and the single standard deduction is $3,000. With the $1,500 exemption, total deductions are $4,500, not $6,000, which leaves $234 taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model claimed Alabama excludes federally tax-exempt interest. Alabama includes it, so AGI is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234 of taxable income, not $0, and tax at 2% is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted the $4,734 but claimed that a single filer over 65 gets at least $5,000 of standard deduction plus exemption. Alabama's single standard deduction is $3,000 and the exemption is $1,500, with no age add-on. That totals $4,500 and leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model looked only at the Social Security exemption and asserted that taxable income was below a threshold. Alabama includes the $4,734 of tax-exempt interest in AGI, and the 2% bracket applies from the first dollar of taxable income. The $234 left after $4,500 of deductions produces $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model addressed only the Social Security exemption and ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stated that tax-exempt interest is excluded in Alabama. Alabama taxes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model claimed income fell below an Alabama tax threshold. Alabama includes the $4,734 of tax-exempt interest in AGI, and its 2% rate applies to the first $500 of taxable income. The $234 left after the $3,000 standard deduction and $1,500 exemption is taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model assumed that exempting Social Security leaves zero Alabama income and never counted the $4,734 of tax-exempt interest, which Alabama taxes. After $4,500 of deductions, $234 remains, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model said there was no other taxable income after Social Security, overlooking that Alabama includes federally tax-exempt interest in AGI. The $4,734 minus the $3,000 standard deduction and $1,500 exemption is $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model exempted Social Security and treated that as the whole calculation, leaving out the $4,734 of tax-exempt interest that Alabama includes in AGI. Taxable income is $234 after $4,500 of deductions, which yields $4.68 at 2%." +us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model asserted that Alabama exempts federally tax-exempt interest. Alabama exempts only interest on Alabama and U.S. obligations, so AGI is $4,734, not $0. The $4,500 of deductions leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model kept the tax-exempt interest out of Alabama AGI. Alabama adds it back, so AGI before deductions is $4,734. The $3,000 standard deduction and $1,500 exemption leave $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model treated wages and similar items as the only Alabama-taxable income and dismissed the tax-exempt interest. Alabama includes the $4,734 in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model called taxable income 'effectively 0' without including the $4,734 of tax-exempt interest, which Alabama taxes. Alabama AGI of $4,734 minus the $3,000 standard deduction and $1,500 exemption is $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model found no taxable Alabama income after exempting Social Security and never counted the $4,734 of federally tax-exempt interest, which Alabama includes in AGI. The $4,500 of deductions leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model based Alabama taxable income on the household's zero federal taxable income. Alabama adds back the $4,734 of federally tax-exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded that the tax-exempt interest produces no Alabama taxable income. Alabama taxes federally exempt interest other than Alabama and U.S. obligations, so AGI is $4,734 and taxable income after $4,500 of deductions is $234. Tax at 2% is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model excluded Social Security but found no other state-taxable income, missing that Alabama includes the $4,734 of federally tax-exempt interest in AGI. The $234 left after $4,500 of deductions is taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model said no other taxable income was reported, treating the $4,734 of tax-exempt interest as outside Alabama AGI. Alabama includes it, and after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model asserted that the tax-exempt interest does not produce Alabama taxable income. Alabama does not follow the federal exclusion, so AGI is $4,734. The $4,500 of deductions leaves $234, which yields $4.68 at 2%." +us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model concluded that the tax-exempt interest produces no Alabama taxable income. Alabama includes federally exempt interest from non-Alabama obligations in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model exempted Social Security and declared that there was no other taxable income, ignoring the $4,734 of tax-exempt interest that Alabama includes in AGI. Taxable income is $234 after $4,500 of deductions, and tax at 2% is $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model allowed for adding back the interest but claimed the standard deduction and exemption for a single filer age 65+ fully offset it. Alabama has no age-65 add-on: the single standard deduction is $3,000 and the exemption is $1,500, so total deductions are $4,500. That leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model began from a federal AGI of $0 and never added back the $4,734 of federally tax-exempt interest, which Alabama requires. Alabama AGI of $4,734 minus $4,500 of deductions leaves $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model required an identified out-of-state bond amount before including any interest, which reverses the default. Alabama includes all federally tax-exempt interest in AGI unless it is shown to come from Alabama or U.S. obligations, and the facts do not show that. The $4,734 minus $4,500 of deductions is $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model asserted that exemptions and deductions would cover the tax-exempt interest if it were included. Alabama does include it, and the single filer's $3,000 standard deduction plus $1,500 exemption total only $4,500 against $4,734 of AGI. The remaining $234 is taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model recognized the add-back but claimed the standard deduction and personal exemption reduce taxable income to $0. For a single filer they total only $4,500: a $3,000 standard deduction and a $1,500 exemption. Against $4,734 of AGI, that leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model treated the $4,734 of tax-exempt interest as not taxable under Alabama law. Alabama includes federally exempt interest other than Alabama and U.S. obligations in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the interest but claimed the low-income single standard deduction plus the $1,500 exemption total at least $5,500. The single standard deduction is capped at $3,000, so the total is $4,500. Against $4,734 of AGI, that leaves $234 taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model stated that Alabama has no state income tax. In fact Alabama levies a graduated income tax starting at 2%. The $4,734 of tax-exempt interest counts toward Alabama AGI, and the $234 left after the $3,000 standard deduction and $1,500 exemption produces $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model claimed that Alabama exempts federally tax-exempt interest. Alabama exempts only interest on Alabama and U.S. obligations, so AGI is $4,734, not $0. After $4,500 of deductions, $234 is taxed at 2%, or $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model labeled the $4,734 of tax-exempt interest as not taxable in Alabama. Alabama includes federally exempt interest in AGI. The $3,000 standard deduction and $1,500 exemption leave $234, taxed at 2% for $4.68." +us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model concluded that taxable income is $0 after the standard deduction without counting the $4,734 of federally tax-exempt interest, which Alabama includes in AGI. The $3,000 standard deduction and $1,500 exemption total only $4,500, leaving $234 taxed at 2%, or $4.68." us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount." us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented expanded or special disabled-adult coverage and treated low assets and potentially favorable treatment of veterans benefits as sufficient. Florida assigns the spouse no Medicaid category, so neither the income comparison nor the $4,870 resource amount creates eligibility." us,scenario_116,spouse_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the disability flag as an automatic Medicaid eligibility pathway. The spouse must first qualify under a Florida Medicaid category, and the engine assigns medicaid_category = NONE." -us,scenario_116,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model manufactured a 24-month SSDI entitlement that the facts never establish: the spouse's only income is $52,800 of veterans benefits, and the prompt's rule that unlisted inputs are 0 sets Title II disability insurance benefits to zero, so no SSDI entitlement exists to start the 24-month Medicare waiting period. Its own reasoning concedes it 'assume[d] the disability has been in place long enough,' which is inference of an unlisted fact the prompt forbids. With no SSDI entitlement, ESRD, or ALS, Medicare eligibility reduces to the age-65 test, and the 42-year-old spouse fails it." -us,scenario_116,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model asserted a 'modeled disability-based Medicare eligibility rule' that PolicyEngine does not have: is_medicare_eligible is keyed to the age-65 threshold, and the is_disabled flag drives SSI and the Medicaid aged/blind/disabled pathway, not Medicare. It mapped the disability boolean straight onto Medicare entitlement without requiring the SSDI receipt that the real 24-month pathway depends on, so it returned 1 for a 42-year-old with zero Social Security disability benefits." -us,scenario_116,spouse_medicare_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model conflated VA disability compensation with SSDI entitlement, claiming that receipt of veterans benefits makes a person Medicare eligible. VA compensation is administered by the Department of Veterans Affairs under Title 38 and confers no Medicare entitlement; the only under-65 routes are 24 months of Title II disability benefits, ALS, or ESRD, none of which the $52,800 in veterans benefits supplies. Applying the actual age-65 test to a 42-year-old yields not eligible." +us,scenario_116,spouse_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly stated that under-65 Medicare requires 24 months of SSDI entitlement. It then assumed the spouse receives SSDI and has met the waiting period, but the household lists no Social Security disability income, so SSDI is 0 and the SSDI-months test fails. It also treated the $52,800 in VA veterans benefits as disability benefits for Medicare purposes, but VA payments are not Title II SSDI and do not start the 24-month clock." +us,scenario_116,spouse_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model applied disability-based Medicare eligibility from the 'is disabled' flag alone. PolicyEngine's rule requires age 65+ or at least 24 months of SSDI receipt. The 42-year-old spouse has no SSDI income or SSDI months (both 0), so the spouse is not Medicare eligible." +us,scenario_116,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model applied disability-based Medicare eligibility because a disability was listed. It skipped PolicyEngine's actual non-age pathway, which is at least 24 months of Social Security disability receipt. The spouse is 42 and has zero SSDI, since only VA veterans benefits are listed, so eligibility is No." +us,scenario_116,spouse_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model claimed that VA disability benefits make a person Medicare eligible. No such pathway exists: VA compensation is not Title II SSDI and does not count toward the 24-month SSDI entitlement that under-65 Medicare requires. The 42-year-old spouse has no SSDI and is under 65, so the spouse is not Medicare eligible." us,scenario_117,child3_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model explicitly recognized that household income exceeded typical Early Head Start limits but then treated age 2 as sufficient for eligibility. It also incorrectly added the separately listed FLSA overtime premium to gross wages even though gross wages already include overtime, but the decisive error was failing to enforce the program's low-income test." us,scenario_117,child3_early_head_start_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied only the under-age-3 requirement and omitted Early Head Start's low-income eligibility test. Child 3 satisfies the age condition, but household income of approximately $250,733 disqualifies the child." us,scenario_117,child3_early_head_start_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model equated meeting the birth-through-age-2 criterion with full Early Head Start eligibility. It failed to apply the poverty-based household income condition, which the approximately $250,733 household income does not satisfy." @@ -8248,49 +9027,57 @@ us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categori us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold." us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies." us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Cut the $25,000 qualified-overtime deduction to $15,185 by phasing it out from $150,000 of MAGI, which is the single-filer threshold; the joint phaseout begins at $300,000, so the full $25,000 survives at MAGI $248,144.53. It also omitted the $2,000 non-itemizer cash-charitable deduction and subtracted a $1,200 CDCC that is zero because Head has no earned income, then hand-adjusted its own $25,791 result up to $25,947." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Applied the overtime deduction phaseout from $150,000 of MAGI (the single threshold; joint starts at $300,000), reducing $25,000 to $14,927. It then itemized $33,088 of charitable-after-the-0.5%-floor plus AR income tax instead of the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction ($34,200 combined), and that itemizing choice led it to tax the $2,589 state refund, which is excluded when the standard deduction is claimed." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Added the $35,525 FLSA overtime premium as income on top of the $248,675 of wages that already contain it, then subtracted the $25,900 charitable gift and $4,752 of suspended employee business expenses above the line, reaching an AGI of $255,607 instead of $248,144.53. It never applied the $25,000 overtime deduction, used a $29,200 standard deduction rather than $32,200, and its $26,809 does not follow from its own $226,407 taxable income, on which the 2026 MFJ schedule yields roughly $39,000." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Fabricated $35,070 of mortgage interest by assuming a 7% rate on the bare $501,000 balance, though no interest amount or rate is given and the prompt directs unlisted numeric inputs to zero, and itemized $70,970 while ignoring the $25,000 qualified-overtime deduction entirely. It also used pre-OBBBA credits ($2,000 CTC plus a $500 other-dependent credit for the 16-year-old rather than $2,200 for each of the three children) and a $600 CDCC that is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Submitted $47,714 while its own derivation ended at $25,972, so the reported value follows from nothing in its arithmetic. That derivation also invented roughly $20,000 of mortgage interest from a bare loan balance, itemized, omitted both the $25,000 overtime deduction and the $2,000 non-itemizer charitable deduction, and used a $2,000-per-child CTC instead of $2,200." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Guessed an approximately $80,000 itemized deduction built on fabricated mortgage interest from the bare $501,000 balance, then deducted the entire $35,525 overtime premium rather than the $25,000 statutory cap, landing near $135,000 of taxable income against the correct $188,944.53. It compounded this with a $1,200 CDCC that is zero because Head has no earned income and a $6,000 CTC instead of $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Correctly declined to invent mortgage interest or SALT and fell back to the standard deduction, but used $30,000 rather than the 2026 MFJ amount of $32,200 and never applied the $25,000 qualified-overtime deduction or the $2,000 non-itemizer charitable deduction, leaving $220,734 of taxable income against $188,944.53. It also taxed the $2,589 state refund and subtracted a $1,200 CDCC that is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Submitted $62,700 after deriving $22,630, so the reported value is untethered from its own computation. That computation also fabricated about $32,000 of mortgage interest from the bare $501,000 balance to itemize $67,900, omitted the $25,000 overtime deduction, and used a $2,000-per-child CTC plus a $1,200 CDCC that is zero without Head's earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Invented $32,565 of mortgage interest and roughly $8,000 of SALT to itemize $67,206 and omitted the $25,000 qualified-overtime deduction, then subtracted only $900 of CTC on an invented nonrefundable split when the full $6,600 (3 × $2,200) offsets tax in this before-refundable-credits measure. It also claimed a $1,200 CDCC that is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Applied a pre-TCJA 10/15/25/28 rate schedule with a $190,750 bracket break to 2026; OBBBA made the 10/12/22 brackets permanent, and the correct tax on $188,944.53 is $30,991.80. It also declared the CTC fully phased out at $250,734 when the 2026 joint phaseout starts at $400,000, so $6,600 is allowed, and it omitted both the $25,000 overtime deduction and the $2,000 non-itemizer charitable deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Excluded the entire $35,525 FLSA overtime premium from gross income; OBBBA grants a deduction capped at $25,000 for joint filers, not a wage exclusion, so AGI remains $248,144.53 rather than its $215,209. It then itemized $36,871 instead of the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, and used $6,000 of CTC rather than 3 × $2,200 = $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Deducted five $5,000 personal exemptions totaling $25,000 and applied estimated pre-TCJA rates; OBBBA keeps the personal exemption repealed for 2026 and the 10/12/22 brackets in force. It also used a $1,000-per-child CTC ($3,000) rather than $2,200 per child ($6,600), added a $1,200 CDCC that is zero without Head's earned income, and omitted the $25,000 overtime deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation; the correct path is $248,144.53 of AGI less $59,200 ($32,200 standard + $25,000 overtime + $2,000 charitable) = $188,944.53 taxable, $30,991.80 of tax, less $6,600 of CTC. Its $46,522 exceeds the roughly $37,500 that 2026 MFJ rates produce with the standard deduction alone, so it is consistent with a reverted pre-TCJA rate schedule applied with no overtime deduction, no non-itemizer charitable deduction, and no full CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Computed explicitly under 'pre-TCJA 2026 rules,' but OBBBA permanently extended the 10/12/22 brackets and the $32,200 MFJ standard deduction and added the $25,000 qualified-overtime deduction, the $2,000 non-itemizer charitable deduction, and the $2,200-per-child CTC. It also applied a CDCC, which is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Ran '2026 post-TCJA expiration rules' with personal exemptions and treated a $1,000-per-child CTC as completely phased out above $248,000 of AGI. For 2026 the CTC is $2,200 per child with the joint phaseout beginning at $400,000, so the household takes the full $6,600, exemptions remain repealed, and the $25,000 overtime and $2,000 non-itemizer charitable deductions it skipped both apply." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation. The correct computation is $188,944.53 of taxable income after the $32,200 standard deduction, $25,000 overtime deduction and $2,000 non-itemizer charitable deduction, taxed at $30,991.80 and reduced by $6,600 of CTC; its $35,283 is consistent with omitting the $25,000 overtime deduction and the $2,000 charitable deduction, taxing the $2,589 state refund, and allowing well under the full $6,600 of CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Asserted a CTC phaseout 'at higher income levels'; the 2026 joint phaseout begins at $400,000 of MAGI, so all three children yield the full $6,600 at an AGI of $248,144.53. Its $26,521.84 sits $2,130 above the reference, the amount produced by dropping the $2,000 non-itemizer charitable deduction and shaving the CTC below $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $41,320 implies about $47,900 of pre-credit tax once the $6,600 CTC is added back, more than the roughly $44,760 that 2026 MFJ rates produce on the entire $248,144.53 AGI with no deduction at all, so it applied an elevated pre-OBBBA rate schedule and took neither the $25,000 overtime deduction nor the $2,000 non-itemizer charitable deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Reached the right AGI of about $248,145 by correctly excluding the $2,589 state refund, but its $33,816 sits $9,424 above the reference. That gap is what omitting the $25,000 qualified-overtime deduction (worth $5,500 at the 22% marginal rate) and the $2,000 non-itemizer charitable deduction produces, together with allowing less than the full $6,600 CTC for the three children." -us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Used an approximate $31,000 standard deduction rather than the 2026 MFJ $32,200, taxed the $2,589 state refund that is excluded because the household does not itemize, and omitted both the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, leaving $219,734 of taxable income against $188,944.53. It also used $6,000 of CTC instead of 3 × $2,200 = $6,600." -us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Itemized about $34,900 of charitable gifts and AR income tax instead of the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction, and never applied the $25,000 qualified-overtime deduction. It then subtracted only $1,500 of CTC on a $500-nonrefundable/$1,700-refundable split when the full $2,200 per child, $6,600 total, is applied against tax in this before-refundable-credits measure, and added a $1,200 CDCC that is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Gave no derivation beyond 'standard deduction' plus child and childcare credits. Its $32,981 exceeds the reference by $8,589, which is what results from omitting the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, taxing the $2,589 state refund, and allowing less than the full $6,600 CTC." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Submitted $0 with no computation for a household with $248,675 of wages and $188,944.53 of taxable income. The 2026 MFJ schedule produces $30,991.80 of tax and the only nonrefundable credit available is $6,600 of CTC, so liability cannot fall below $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"Never applied the $25,000 OBBBA qualified-overtime deduction: its taxable income of $214,996.05 is $26,052 above the correct $188,944.53, essentially the missing $25,000 deduction plus the state refund it taxed. It also itemized charitable, AR income tax and medical instead of the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction, though its $2,200-per-child CTC and zero CDCC were right." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Phased the $25,000 overtime deduction down to roughly $15,200 using the $150,000 single-filer MAGI threshold, when the joint phaseout begins at $300,000 and the full $25,000 applies. It also credited only two children at $2,200 each when the 16-year-old also qualifies for a total of $6,600, and subtracted a dependent-care credit that is zero because Head has no earned income." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Built the correct structure — $25,000 overtime deduction, $32,200 standard deduction, $2,000 non-itemizer charitable deduction, $6,600 CTC — but included the $2,589 state tax refund in AGI, which is excluded because the household claims the standard deduction, and then subtracted a $600 CDCC that is zero because Head has no earned income. Those two errors, worth about $570 up and $600 down, net to its $30 shortfall against $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Itemized roughly $33,900 of charitable-after-floor and AR income tax instead of the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction ($34,200), and taxed the $2,589 state refund, which is excluded precisely because the household does not itemize. Its $25,000 overtime deduction and $6,600 CTC were correct, so those two deduction errors account for the entire $637 excess." -us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Included the $2,589 state tax refund in AGI and itemized charitable gifts after the 0.5% floor plus AR income tax; the household takes the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction, which also makes the refund nontaxable under the tax-benefit rule. Those choices raised taxable income about $3,710 above $188,944.53 and produced the $816 excess over $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Provided no computation at all, only the assertion that high wages produce substantial liability. Its $52,000 exceeds the roughly $44,760 that 2026 MFJ rates produce on the entire $248,144.53 AGI with no deduction and no credit whatsoever, so the number is unreachable once the $59,200 of deductions and $6,600 of CTC are recognized." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Computed under post-TCJA sunset law with personal exemptions and a 25% bracket and declared the CTC fully phased out; OBBBA keeps exemptions repealed, holds the marginal rate here at 22%, and starts the $2,200-per-child CTC phaseout at $400,000 joint, so $6,600 is fully allowed. It also omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied the TCJA-sunset regime in full — five $5,300 personal exemptions, 10/15/25 rates, uncapped SALT — and phased the CTC out above $110,000 of MAGI. For 2026 OBBBA keeps exemptions repealed and the 10/12/22 brackets in force, adds the $25,000 overtime and $2,000 non-itemizer charitable deductions it never took, and phases the $2,200-per-child CTC out only above $400,000 joint, so the full $6,600 applies." -us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Used post-TCJA-expiration mechanics — five $5,184 personal exemptions and 10/15/25 brackets — and subtracted the $8,223 employer-sponsored insurance premium from AGI, though ESI premiums are already excluded from the $248,675 of reported wages and are not a separate above-the-line deduction. It also zeroed the CTC as phased out when $6,600 is allowed below the $400,000 joint threshold, and never applied the $25,000 qualified-overtime deduction." -us,scenario_117,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Itemized $37,873 by including a $3,116 medical deduction and $237 of miscellaneous expenses, when the household's roughly $11,900 of medical costs falls below the 7.5%-of-AGI floor of about $18,611 and miscellaneous itemized deductions remain suspended, instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. Those phantom deductions pushed taxable income to $187,861 against the correct $188,944.53, leaving its tax $229 low." -us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so nothing was submitted against the $24,391.80 reference. The failure is a missing output rather than a substantive tax error." -us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Included the $2,589 state tax refund in AGI and itemized $34,414.79 of AR income tax, charitable gifts after the 0.5% floor, and medical above the 7.5% floor; the household instead takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, a choice that also makes the prior-year refund nontaxable under the tax-benefit rule. The resulting $2,374.68 of extra taxable income at the 22% rate is exactly its $522.43 excess over $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Asserted zero liability with no computation for a household with $248,675 of wages. Taxable income is $188,944.53, the 2026 MFJ schedule yields $30,991.80, and the only nonrefundable credit is $6,600 of CTC, which leaves a floor of $24,391.80." -us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Applied the $25,000 overtime deduction and $6,600 CTC correctly but included the $2,589 state tax refund in AGI, which is excluded because the household claims the standard deduction, and itemized about $34,190 of AR income tax plus charitable gifts without applying the 0.5%-of-AGI charitable floor, instead of the $32,200 standard deduction with the $2,000 non-itemizer charitable deduction. It then rounded its own $24,964 result to a flat $25,000." -us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Fabricated $24,375 of mortgage interest by assuming a 6.5% rate on a supposedly $375,000-capped balance, when no interest or rate is listed and the acquisition-debt limit is $750,000, and itemized $60,275 while never applying the $25,000 qualified-overtime deduction. It also used a $2,000-per-child CTC ($6,000) rather than $2,200 ($6,600) and a $600 CDCC that is zero because Head has no earned income; the phantom interest and the missing overtime deduction largely offset." -us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Treated the $35,525 FLSA overtime premium as excluded from the $248,675 of wages, then invented a '$2,800 additional standard deduction for net overtime' and a '$6,000 overtime deduction credit' in place of the single $25,000 capped qualified-overtime deduction. It further added about $12,000 of tax for a mortgage-interest limitation built on a nonexistent $409,000 acquisition-debt cap, when no mortgage interest enters the return at all." -us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model computed the §24(d) earned-income formula (15% × ($284,200 − $2,500) = $42,255) and then treated its result as a payable amount, never applying the controlling limitation that the ACTC equals only the CTC left unused after the credit offsets income tax liability — with tax liability far exceeding the total CTC for three children, that remainder is $0. It compounded this by double-counting the $35,525 FLSA overtime premium as income on top of the $248,675 wages that already contain it, by using the 2023 MFJ standard deduction of $27,700 for tax year 2026, and by deducting the $4,752 of unreimbursed employee business expenses, a miscellaneous itemized deduction that is disallowed. Its $3,750 answer follows from no stated derivation at all — the reasoning's own numbers ($6,000 cap, $42,255 formula) never produce it." -us,scenario_117,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"The model multiplied the per-child refundable ACTC ceiling by three children ($1,700 × 3 = $5,100) and treated that ceiling as the credit itself, skipping the §24(d) step that limits the refundable portion to the CTC remaining after the nonrefundable credit offsets income tax liability. With this household's federal income tax on roughly $250,700 of income exceeding the entire CTC for three qualifying children, the whole credit is used nonrefundably and nothing spills into the refundable portion." -us,scenario_117,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model stated the rule exactly backwards: it wrote that 'with ample tax liability, the household claims the full refundable CTC (ACTC) portion,' when ample tax liability is precisely what eliminates the ACTC — §24(d) makes the refundable portion equal only to the CTC that cannot be absorbed against income tax liability, so a household owing far more tax than its total CTC for three children gets $0 refundable. It then applied the $1,700 per-child refundable ceiling as a flat entitlement to reach $5,100." -us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so the required key was absent from the submitted outputs object. This is a contract failure in producing a parseable answer, not a substantive misapplication of the CTC refundability or EITC phase-out rules." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Phased the qualified-overtime deduction out from the $150,000 single-filer threshold, which cut it to $15,185. Joint filers phase out only above $300,000 MAGI, so the full $25,000 cap applies at $250,733. It also dropped the taxable $2,589 state refund and skipped the $2,000 non-itemizer charitable deduction. It then subtracted a $1,200 CDCC even though the head's zero earned income sets the §21(d) expense limit to $0." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Phased the overtime deduction out from $150,000, the single-filer threshold, which gave $14,927 instead of the full $25,000 allowed to joint filers below $300,000 MAGI. It also chose to itemize $33,088 by comparing only against the bare $32,200 standard deduction. It missed the $2,000 non-itemizer charitable deduction, which makes the standard path worth $34,200. Taxable income came out at $202,719 instead of $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Added the $35,525 overtime premium on top of $248,675 of wages that already include it. It subtracted the $25,900 charitable gifts and $4,752 of employee business expenses above the line, used a $29,200 standard deduction, and never applied the $25,000 qualified-overtime deduction. Its $26,809 is the pre-credit tax on an inflated $226,407 of taxable income, with none of the $6,600 CTC subtracted." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Invented about $35,070 of mortgage interest from the $501,000 balance; no interest paid was listed, so the deduction is $0. It combined that with a $10,000 SALT figure to itemize and omitted the $25,000 qualified-overtime deduction. It also gave the 16-year-old a $500 other-dependent credit and used $2,000 per child instead of $2,200 for all three under-17 children. It claimed a $600 CDCC despite the head's zero earnings, and its near miss comes from these errors offsetting each other." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Submitted $47,714, which contradicts its own derivation ending near $25,972. That derivation also made up $20,000 of mortgage interest from the balance, omitted the $25,000 qualified-overtime deduction, and used $2,000 rather than $2,200 per child. It claimed a CDCC even though the head has no earned income." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Built an itemized deduction of about $80,000 from imputed mortgage interest (about $32,000, though no interest was listed) and about $40,000 of SALT. It then subtracted the entire $35,525 overtime premium instead of the $25,000 joint cap, pushing taxable income down to about $135,000 instead of $191,533. It also used $2,000 per child and claimed a $1,200 CDCC that the zero-earning head blocks." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Chose to itemize $33,186 (Arkansas income tax plus floored charity) because it beat the bare $32,200 standard deduction. It missed the 2026 non-itemizer charitable deduction of $2,000 for joint filers, which makes the standard path worth $34,200. The $1,014 of lost deductions, taxed at 22%, accounts exactly for its $223 overstatement." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $25,000 qualified-overtime deduction entirely, used the outdated $30,000 standard deduction and 2025 brackets, and took no $2,000 non-itemizer charitable deduction, so it taxed $220,734 instead of $191,533. It also used $2,000 per child instead of $2,200. It subtracted a $1,200 CDCC even though the head's zero earned income caps creditable expenses at $0." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Submitted $62,700, which contradicts its own worked result of $22,630. That worked result also imputed about $32,000 of mortgage interest from a balance with no listed interest, omitted the $25,000 qualified-overtime deduction, and used $2,000 per child. It claimed a $1,200 CDCC that the head's zero earned income rules out." +us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Limited the qualified-overtime deduction to $12,500, the single-filer cap. A joint return gets the $25,000 cap no matter which spouse earned the overtime, and MAGI is below the $300,000 phaseout. It also itemized about $35,400 (including a medical deduction) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, so taxable income was $202,834 instead of $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Imputed $32,565 of mortgage interest from the bare balance and omitted the $25,000 qualified-overtime deduction. It applied only $900 of nonrefundable CTC, although the pre-credit tax absorbs the full $6,600 ($2,200 × 3); a refundable portion exists only when the credit exceeds liability. It also subtracted a $1,200 CDCC that the head's zero earned income disallows." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a pre-TCJA 15%/25%/28% rate schedule and treated the CTC as fully phased out. It ignored the permanent post-2025 regime: 10/12/22% brackets, a $2,200-per-child CTC that phases out only above $400,000 joint MAGI, and the $25,000 qualified-overtime deduction. As a result it taxed $214,093 and subtracted no credit, when the correct figures are $191,533 of taxable income and a $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Excluded the entire $35,525 FLSA overtime premium from AGI instead of taking the below-the-line qualified-overtime deduction capped at $25,000 for joint filers, overstating the deduction by $10,525. It then itemized $36,871, including a $3,905 medical deduction, instead of taking the standard deduction plus the $2,000 non-itemizer charitable deduction. It also used $2,000 per child instead of $2,200." +us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction, using only a $31,500 standard deduction. It counted just $600 of the CTC as nonrefundable. The nonrefundable CTC is the full credit up to tax liability, which here is the entire $6,600 (the model used $6,000), with nothing carved out as refundable." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: five $5,000 personal exemptions, pre-TCJA brackets, and a $1,000-per-child CTC. Post-2025 law keeps the $32,200 standard deduction, 10/12/22% brackets, and a $2,200 CTC, with no exemptions. It also omitted the $25,000 qualified-overtime deduction and claimed a $1,200 CDCC that the zero-earning head blocks." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. The correct path takes the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions to reach $191,533 of taxable income and $31,561 of tax, less the $6,600 CTC. Its $46,522 exceeds even the $37,164 pre-credit tax with the overtime deduction dropped, so it omitted the overtime deduction and never subtracted the CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA 2026 rules instead of the permanent post-2025 regime, so it missed the $2,200-per-child CTC, the $25,000 qualified-overtime deduction, and the $2,000 non-itemizer charitable deduction. It also applied a CDCC, although the head's zero earned income sets the creditable-expense limit to $0." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Used TCJA-expiration rules with personal exemptions and a $1,000 CTC phased out above the old $110,000 threshold. 2026 law gives $2,200 per child and phases out only above $400,000 joint MAGI, so the full $6,600 applies. It also omitted the $25,000 qualified-overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. Its $35,283 is higher than the $30,564 obtained even with the $25,000 qualified-overtime deduction dropped. It therefore omitted the overtime deduction and did not take the full $6,600 CTC off the $31,561 tax on $191,533 of taxable income." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Invoked a CTC phaseout that does not apply, since joint MAGI of $250,733 is far below the $400,000 threshold. The full $6,600 CTC comes off the $31,561 bracket tax on $191,533 of taxable income. Its $26,521.84 keeps about $1,560 of liability that this nonexistent phaseout left uncredited." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. Its $41,320 exceeds the $37,164 pre-credit tax computed with the overtime deduction dropped, so it omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction and subtracted none of the $6,600 CTC. The correct figure is $31,561 of tax on $191,533 of taxable income, less the $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Started from $248,145 AGI, leaving out the taxable $2,589 state refund. Its $33,816 is higher than the $30,564 produced even without the overtime deduction, so it omitted the $25,000 qualified-overtime deduction and did not subtract the full $6,600 CTC." +us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction for the spouse's $35,525 FLSA premium and the $2,000 non-itemizer charitable deduction, so it taxed $219,734 instead of $191,533. It also used $2,000 per child instead of the 2026 CTC of $2,200." +us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction, giving taxable income of $215,834 instead of $191,533. It treated only $500 per child as nonrefundable CTC, although the full $2,200 per child offsets liability and the refundable ACTC applies only to credit that exceeds tax. It also subtracted a $1,200 CDCC that the head's zero earned income disallows." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Its $32,981 is higher than the $30,564 obtained even with no overtime deduction. It therefore never applied the $25,000 qualified-overtime deduction (joint cap, no phaseout below $300,000) or the $2,000 non-itemizer charitable deduction. It also counted a childcare credit that the head's zero earned income disallows." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Reported zero liability, but nothing eliminates the tax. After the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions, taxable income is $191,533 and tax is $31,561. The only nonrefundable credit is the $6,600 CTC, which leaves $24,961." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Omitted the $25,000 qualified-overtime deduction for the spouse's $35,525 FLSA premium (joint cap, MAGI below the $300,000 phaseout), so taxable income stayed at $214,996 instead of $191,533. It also itemized $35,738 instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its value reconstructs exactly: $250,734 − $32,200 standard − $12,500 overtime = $206,034 taxable, then $34,751 of tax less two $2,200 CTCs and a $1,200 CDCC. It applied the single-filer $12,500 overtime cap instead of the $25,000 joint cap and skipped the $2,000 non-itemizer charitable deduction. It also dropped the 16-year-old, who is under 17 and qualifies for the CTC, and claimed a CDCC that the zero-earning head blocks." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Got taxable income ($191,534), bracket tax ($31,561), and the $6,600 CTC right, then subtracted a $600 CDCC. The CDCC is $0 because IRC §21(d) caps creditable expenses at the lower-earning spouse's earned income, and the head has none." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $33,895 (charity plus SALT) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200). That left taxable income about $305 too high and the tax $67 over." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $33,080 (floored charity plus Arkansas income tax) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction worth $34,200. That left taxable income about $1,121 too high and the tax $247 over." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its answer implies itemized deductions of about $35,434, which is $1,234 more than the household can itemize once charity is reduced by the 0.5%-of-AGI floor and medical costs fall under the 7.5% floor. The household takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction instead, so taxable income is $191,533, not about $190,300." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used the $32,200 standard deduction and the full $25,000 overtime deduction but omitted the 2026 non-itemizer charitable deduction of $2,000 for joint filers. That left taxable income at $193,534 and the tax $440 too high." +us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $34,340 (floored charity plus an Arkansas income-tax estimate) instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. Its state-tax figure overstates what can be itemized and pushes the total past $34,200, cutting taxable income $139 below the correct $191,533." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. $52,000 is higher than the $37,164 pre-credit tax even with the overtime deduction dropped, and even above the roughly $45,400 that 2026 brackets would levy on the full $250,733 AGI with no deductions. It therefore omitted the $25,000 qualified-overtime deduction, the $2,000 charitable deduction, and the $6,600 CTC, and used rates above the 2026 schedule." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (personal exemptions, a 25% bracket, and a fully phased-out CTC) instead of post-2025 law. Current law keeps 10/12/22% brackets and a $2,200 CTC that phases out only above $400,000 joint MAGI. It also omitted the $25,000 qualified-overtime deduction and the $2,000 non-itemizer charitable deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly applied TCJA-sunset law: five $5,300 exemptions, uncapped SALT, 15%/25% brackets, and a CTC phased out above $110,000. 2026 law keeps the $32,200 standard deduction, 10/12/22% brackets, and a $2,200 CTC with a $400,000 joint threshold, so the full $6,600 applies. It also omitted the $25,000 qualified-overtime deduction." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Reached the correct $191,534 of taxable income and the $6,600 CTC but computed bracket tax of $31,735 instead of $31,561. The correct sum is 10% of $24,800 ($2,480) + 12% of $76,000 ($9,120) + 22% of $90,734 ($19,961). The $174 arithmetic error carries straight into the answer." +us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted the $8,223 ESI premium from AGI, applied TCJA-sunset personal exemptions and 15%/25% brackets, zeroed the CTC as phased out, and omitted the $25,000 qualified-overtime deduction. Under 2026 law the full $6,600 CTC applies below the $400,000 joint threshold. It also claimed a $1,200 CDCC that the head's zero earned income blocks." +us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Inflated itemized deductions to $37,873. It took the full $25,900 of charity without the 2026 0.5%-of-AGI floor, added a $3,116 medical deduction although total medical costs do not clear the $18,805 (7.5%-of-AGI) floor, and added a $237 miscellaneous deduction that remains suspended. The household takes the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, so taxable income is $191,533, not $187,861." +us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score." +us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $34,415, including a $741 medical deduction that counts the head's $8,223 premium twice; the 'excluding Part B' and 'other' premium lines describe the same premium. Without the double count, medical costs ($11,823) stay under the $18,805 floor, and itemizing ($33,674) loses to the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. That gives $191,533 of taxable income." +us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed that credits offset all liability. The only nonrefundable credit is the $6,600 CTC against $31,561 of tax on $191,533 of taxable income, and no CDCC applies because the head has no earnings. That leaves $24,961." +us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Its derivation reached about $24,980 but it submitted a rounded $25,000. The derivation also itemized the full $25,900 of charity without the 0.5%-of-AGI floor, offset by a low Arkansas-tax guess, instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. It ended up $38.66 above $24,961.34." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Imputed $24,375 of mortgage interest from the balance, though no interest paid was listed, so the deduction is $0. It also took a flat $10,000 SALT and omitted the $25,000 qualified-overtime deduction, then used $2,000 per child plus a $600 CDCC. The invented mortgage interest roughly stood in for the missing overtime deduction, and the invalid $600 CDCC replaced the missing $600 of CTC, so offsetting errors left it only $144 off." +us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Excluded the full $35,525 overtime premium from wages and invented rules that do not exist: a '$2,800 additional standard deduction for net overtime', a SALT-cap 'excess taxes' reduction, an 'overtime deduction credit', and a $409,000 mortgage cap. The correct computation takes the $32,200 standard, $25,000 overtime, and $2,000 charitable deductions to reach $191,533 of taxable income and $31,561 of tax, less the $6,600 CTC." +us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It never applied the rule that the ACTC can refund only the CTC left after offsetting tax liability. A federal tax bill of more than $25,000 absorbs all of this household's CTC, so the ACTC is $0. Instead it invented a $3,750 amount 'limited by the ratio of Additional CTC to total CTC'. Along the way it counted the FLSA overtime premium twice on top of gross wages, used the 2023 $27,700 standard deduction and $2,000-per-child CTC, and said income was above the $400,000 phase-out when AGI is about $250,733." +us,scenario_117,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"It treated the $1,700-per-child refundable cap as an amount paid out automatically ($5,100). It skipped the step that limits the ACTC to CTC left after tax liability. With more than $25,000 of federal income tax, the full $6,600 CTC is used as a nonrefundable credit, so the refundable portion is $0." +us,scenario_117,federal_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It multiplied 3 children by an $1,800 refundable cap to get $5,400. It never limited the ACTC to CTC left after the credit offsets tax liability. The household's federal tax of more than $25,000 absorbs the full $6,600 CTC, so the ACTC is $0. EITC is also $0." +us,scenario_117,federal_refundable_credits,glm-5.3,llm_error,other,False,"It stated the rule backwards, saying that 'with ample tax liability' the household claims the full refundable CTC of $1,700 per child ($5,100). In fact, ample tax liability is exactly what reduces the ACTC to zero: the refundable portion is limited to CTC left after offsetting tax. More than $25,000 of federal tax absorbs the entire $6,600 CTC, so refundable credits are $0." +us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_refundable_credits, so there is no substantive calculation to score. Under the correct derivation, the full $6,600 CTC offsets this household's more-than-$25,000 federal tax, which leaves a $0 ACTC. EITC is $0 as well." us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a household-level “young-child context” pathway and transferred the two-year-old child's WIC category to the 44-year-old head. The head has no listed pregnancy, postpartum, or breastfeeding status and is neither an infant nor a child under five; independently, household income exceeds the 185% poverty limit." us,scenario_117,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"Its own arithmetic reached the reference — 6.2% on the $184,500 2026 Social Security taxable maximum = $11,439 plus 1.45% × $248,675 = $3,605.79, totaling $15,044.79 — and it then overrode that with an unfounded $183,600 projected wage base for a stated $14,988.99. It finally submitted $12,896.44, which no stated step produces: net of the correct $3,605.79 Medicare tax it implies a $149,849 wage base, far below any 2026 figure." us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It added the $35,525 FLSA overtime premium to the $248,675 of gross wages even though the prompt states gross wages already include overtime pay, used the $168,600 2024 taxable maximum instead of 2026's $184,500, and applied the 0.9% Additional Medicare Tax at the $200,000 single-filer threshold when the married-filing-jointly threshold of $250,000 governs and combined wages fall $1,325 below it. The dominant error is a fabricated $17,052 'Arkansas payroll tax' at 6.0%: Arkansas withholding is state income tax, and Arkansas imposes no mandatory employee-side payroll tax, plus an invented $1,680 'additional mandatory AR payroll withholding'." @@ -8320,163 +9107,187 @@ us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It us us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"Its stated components are the reference: $184,500 × 6.2% = $11,439 of Social Security tax, $248,675 × 1.45% = $3,606 of Medicare tax, and no Additional Medicare Tax because combined wages fall below the $250,000 joint threshold — a total of $15,045. It nonetheless submitted $20,603, $5,558 above its own arithmetic, and referenced a phantom '$213,150 of wages' that appears nowhere in the household facts." us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test." -us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It applied Arkansas's ~$2,340 standard deduction and explicitly declared no other adjustments, discarding the $25,900 charitable contribution that produces the only allowable AR itemized deduction ($24,659.28 after the 2026 0.5%-of-AGI floor), so its taxable income of ~$245,805 exceeds the correct $223,485.25 by about $22,300. It then abandoned its own $9,196 result and reported $8,081 by invoking unspecified 'bracket smoothing,' and it never applied the $60 additional tax credit for qualified individuals on top of the $145 personal credits." -us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used an obsolete Arkansas rate schedule containing a 4% middle bracket (2% on $4,500, then 4% on $4,600), which produced $8,586.66 of pre-credit tax on a taxable income of $222,245 — $398 more than the $8,189.05 Arkansas's 2026 graduated schedule yields on the larger $223,485.25 base. It also omitted the 2026 0.5%-of-AGI charitable floor ($1,240.72) and stopped at $145 of personal credits, missing the $60 additional tax credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium to the $248,675 of wages, although the premium is a component of that wage total, and further added the $2,589 state tax refund, which Arkansas excludes from its income base of $248,144.53. It then invented an $18,398 self-employment tax subtraction, a 5% top bracket (Arkansas's 2026 top rate is 3.9%), and $1,050 of dependent/education credits in place of the actual $205, while dropping the $25,900 charitable deduction entirely." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It imputed roughly $22,500 of mortgage interest from the $501,000 loan balance even though no interest amount appears in the facts and unlisted numerics are zero, inflating itemized deductions to ~$48,400 against the correct $24,659.28 (charitable only, net of the 0.5%-of-AGI floor). Its own arithmetic then produced ~$7,645, yet it submitted $11,264, a figure that follows from no step in its derivation." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It added the $2,589 state tax refund to the Arkansas base and imputed ~$20,040 of mortgage interest from the loan balance, deductions of ~$48,000 against the correct $24,659.28. It then stated taxable income of ~$202,000 with a 3.9% top rate — which yields under $7,900 — but submitted $11,820, implying a 5.9% effective rate that its own schedule never produces." -us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It built a deduction stack of charitable, mortgage interest, and medical expenses to reach ~$190,000 of taxable income, but mortgage interest is zero (only the $501,000 balance is listed) and the household's ~$11,800 of medical costs fall far below the 7.5%-of-AGI floor of $18,610, leaving charitable-only deductions of $24,659.28 and taxable income of $223,485.25. Understating the base by ~$33,000 cost about $1,290 of tax, and it applied only $145 of credits instead of $205." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted the $35,525 FLSA overtime premium from Arkansas income as an exempt item, but that premium is already inside the $248,675 of reported wages and Arkansas's 2026 base includes it, so its AGI of $215,209 understates the correct $248,144.53 by $32,936. It then added a $10,000 SALT deduction (Arkansas itemized deductions exclude state income tax and no property tax is listed), a $448 unreimbursed-employee-expense deduction, and a $240 Arkansas child-care credit that is zero because the head has no earned income and the federal CDCC is therefore zero." -us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It imputed ~$20,040 of mortgage interest from the $501,000 balance rather than treating the unlisted interest as zero, and its own graduated-rate work produced ~$8,086. It then submitted $12,500, an amount unconnected to its $202,205 taxable income and 3.9% top rate, and applied none of the $205 in Arkansas nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It applied a 4.9% top rate that Arkansas repealed; the 2026 schedule tops out at 3.9% and produces $8,189.05 on the correct $223,485.25 taxable income. It compounded this with a taxable base of ~$184,000 that overstates deductions by roughly $39,000 and with the assertion that no nonrefundable credits offset the tax, when $145 of personal credits and a $60 additional credit apply." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It reached the correct Arkansas AGI of $248,145 but then took a $5,000 standard deduction instead of itemizing the $25,900 charitable contribution, whose $24,659.28 net-of-floor amount is the deduction Arkansas allows, leaving taxable income $19,660 too high. It also subtracted no credits, omitting the $145 personal credits and the $60 additional tax credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It claimed itemized deductions of $29,805.33 — $5,146 more than the $24,659.28 Arkansas allows once the 2026 0.5%-of-AGI charitable floor is applied and mortgage interest, medical, and misc expenses are excluded — and then reported taxable income of $185,403.68, which is not $248,144.53 minus its own deduction ($218,339.20). Its base is roughly $38,000 below the correct $223,485.25, understating tax by about $1,120 before credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from $250,734 by including the $2,589 state tax refund, which Arkansas excludes, and deducted $26,641 by adding a $741 medical amount that fails the 7.5%-of-AGI floor of $18,610 while omitting the 2026 0.5%-of-AGI charitable floor. Its $224,093 taxable income and $8,413.63 pre-credit tax exceed the correct $223,485.25 and $8,189.05, and it took only $145 of credits, missing the $60 additional tax credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation, and $10,560 corresponds to a rate near 4.4% — Arkansas's repealed pre-2025 top rate — applied to income with essentially no charitable deduction. Arkansas's 2026 schedule yields an average rate of 3.66% on the correct $223,485.25 taxable income ($8,189.05), and $205 of nonrefundable credits then apply." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $8,768 equals exactly 3.9% of $224,834, i.e. $250,734 of income (including the $2,589 state refund Arkansas excludes) less the full $25,900 charitable contribution without the 2026 0.5%-of-AGI floor. Applying the flat top rate to every dollar ignores Arkansas's graduated schedule, which taxes the correct $223,485.25 base at an average 3.66% for $8,189.05, and it subtracted none of the $205 in nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $8,623 is 3.9% of $224,834 less $145, meaning it included the $2,589 state refund Arkansas excludes, deducted the full $25,900 without the 2026 0.5%-of-AGI charitable floor, and applied the top rate flat to every dollar instead of Arkansas's graduated schedule, which yields $8,189.05 on $223,485.25. It also stopped at the $145 personal credits and omitted the $60 additional tax credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation, and $10,879 corresponds to a rate near 4.4% — Arkansas's repealed pre-2025 top rate — applied to income with no charitable deduction taken. The 2026 schedule produces $8,189.05 on the $223,485.25 taxable income that remains after the $24,659.28 charitable deduction, and $205 of nonrefundable credits reduce it to $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its $8,234.12 matches the graduated-schedule tax on a base of about $224,800 — the $250,734 income including the $2,589 state refund Arkansas excludes, less the full $25,900 charitable contribution with no 2026 0.5%-of-AGI floor — with no credits subtracted at all. The $205 of Arkansas nonrefundable credits ($29 per filer and dependent plus $60 additional) account for most of its $250 overstatement." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $9,260 equals 3.9% of about $237,400, a base that leaves roughly $14,000 of the $24,659.28 charitable itemized deduction untaken and applies the top rate flat rather than through Arkansas's graduated schedule, which produces $8,189.05 on $223,485.25. It also subtracted none of the $205 in personal and additional nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It stated that it applied the Arkansas standard deduction, forgoing the $24,659.28 charitable itemized deduction that dwarfs Arkansas's roughly $4,600 joint standard deduction and leaving its taxable base about $20,000 above $223,485.25. It also applied the 3.9% top rate flat rather than Arkansas's graduated schedule, which yields an average 3.66% on the correct base." -us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It invented a bracket schedule topping at 4.7% and 4.4% rates; Arkansas's 2026 schedule tops at 3.9% and produces $8,189.05 on $223,485.25, while its schedule produced $8,928.98 on a smaller $222,245 base. It also omitted the 2026 0.5%-of-AGI charitable floor and subtracted none of the $205 in personal and additional nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It stated that no Arkansas nonrefundable credits apply, but the household receives $145 in personal credits ($29 for each of the two adults and three children) plus a $60 additional credit for qualified individuals, which is the step that carries $8,189.05 down to $7,984.05. It also included the $2,589 state refund Arkansas excludes and allowed a roughly $1,000 medical deduction that fails the 7.5%-of-AGI floor of $18,610, while omitting the 0.5%-of-AGI charitable floor." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It characterized the Arkansas liability as 'modest' and submitted $1,005, which is 0.45% of the $223,485.25 taxable income; Arkansas's schedule reaches a 3.9% top rate and produces $8,189.05 before the $205 of nonrefundable credits. Its answer is off by a factor of about eight and rests on no computation of the base or the rate schedule." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It asserted that deductions and offsets zero out the liability, but the only Arkansas deduction available is the $24,659.28 charitable contribution net of the 2026 0.5%-of-AGI floor, leaving $223,485.25 of taxable income and $8,189.05 of tax before $205 of credits. A zero answer requires deductions nearly ten times what this household has." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It correctly excluded the state refund and itemized the charitable contribution, but omitted the 2026 0.5%-of-AGI floor ($1,240.72) and then applied a rate schedule that produced $8,496.66 on $222,245 — $308 more than the $8,189.05 Arkansas's graduated 2026 schedule yields on the larger $223,485.25 base. It also used $160 of credits rather than the actual $205 ($145 personal plus $60 additional)." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It took 'the available standard and personal deductions' rather than itemizing, leaving its base roughly $20,000 above the $223,485.25 that results after the $24,659.28 charitable deduction. It also declared no applicable Arkansas nonrefundable credit, though $145 of personal credits and a $60 additional credit for qualified individuals reduce the tax to $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $8,107 equals the graduated-schedule tax on a base about $1,600 above the correct $223,485.25 less only the $145 of personal credits, so it left the $60 additional tax credit for qualified individuals unapplied. The excess base comes from not reducing the $25,900 charitable deduction by exactly the 2026 0.5%-of-AGI floor of $1,240.72 against the $248,144.53 AGI." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It identified the right structure — charitable itemized deduction, graduated Arkansas rates, dependent-based nonrefundable credits — but estimated rather than computed the components, landing $11.26 high on a base about $290 above $223,485.25. The exact path is AGI $248,144.53 (wages $248,675 less the $530.47 capital loss, state refund excluded), less $24,659.28 of charitable deduction net of the 0.5%-of-AGI floor, giving $8,189.05 of tax less $205 of credits." -us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It got the base construction nearly right but deducted the full $25,900 without the 2026 0.5%-of-AGI charitable floor and then applied a rate schedule that produced $8,578.26 on $222,245 — $389 above the $8,189.05 Arkansas's graduated 2026 schedule yields on the larger $223,485.25 base, because it effectively taxed all income at the top rate rather than running the 0%/2%/3% lower brackets. It also stopped at $145 of credits, missing the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It submitted a round $8,000 with no rate schedule, deduction, or credit computation behind it. The Arkansas path is $248,144.53 of AGI less $24,659.28 of charitable deduction, $8,189.05 of graduated-schedule tax, and $205 of personal and additional nonrefundable credits, for $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a top marginal rate 'near 4.4%,' Arkansas's repealed pre-2025 rate; the 2026 schedule tops at 3.9% and yields $8,189.05 on $223,485.25, an average rate of 3.66%. It also subtracted none of the $205 in personal and additional nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It concluded that no Arkansas nonrefundable credits apply at this income, but the $29-per-person personal credit ($145 for the two adults and three children) and the $60 additional credit for qualified individuals are not income-tested here and account for $205 of its $451 overstatement. It also included the $2,589 state refund Arkansas excludes, allowed $741 of medical expenses that fail the 7.5%-of-AGI floor of $18,610, and omitted the 2026 0.5%-of-AGI charitable floor." -us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It treated Arkansas 2026 as a flat 3.9% tax ($216,611 × 0.039 = $8,448), ignoring the graduated schedule whose 0%/2%/3% lower brackets bring the tax on $223,485.25 down to $8,189.05, an average rate of 3.66%. It also used an unexplained $242,511 AGI instead of $248,144.53, omitted the 0.5%-of-AGI charitable floor, and applied none of the $205 in nonrefundable credits." -us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied approximately 3.9% flat to its base to get about $8,600, ignoring Arkansas's graduated schedule, which taxes the correct $223,485.25 at an average 3.66% for $8,189.05. It also built the base from 'standard deduction/exemptions' rather than the $24,659.28 charitable itemized deduction and subtracted only about $100 of credits instead of the $205 actually allowed." -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so the submission failed the output contract rather than the tax computation. The required entry is $7,984.05." -us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It applied only the Arkansas joint standard deduction, arriving at $245,654 of taxable income, when the $25,900 charitable contribution — $24,659.28 after the 2026 0.5%-of-AGI floor — is the deduction that brings the base to $223,485.25. That $22,169 excess base drives its $9,172.51 pre-credit figure above $8,189.05, and it took only the $145 personal credits, missing the $60 additional credit for qualified individuals." -us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It asserted no Arkansas liability without any computation; $248,144.53 of AGI less the $24,659.28 charitable deduction leaves $223,485.25 taxable, producing $8,189.05 of tax and $7,984.05 after the $205 of nonrefundable credits. Nothing in the facts — no exemption, exclusion, or credit — zeroes a $223,485 Arkansas base." -us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It used a personal credit of about $31 per person and never applied the $60 additional tax credit for qualified individuals; Arkansas's credit is $29 each, totaling $145, plus the $60, for $205. It also included the $2,589 state refund Arkansas excludes from income, omitted the 2026 0.5%-of-AGI charitable floor, and applied a flat ~3.8% instead of the graduated schedule that yields $8,189.05 on $223,485.25." -us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium and the $2,589 state refund to the $248,675 of wages to reach $286,259, but the premium is a component of those wages and Arkansas excludes the refund, leaving $248,144.53 of AGI after the $530.47 capital loss. It then chose the standard-deduction branch at a repealed 4.4% flat rate over its own itemized computation, and it never reduced the charitable deduction by the 2026 0.5%-of-AGI floor or applied the $60 additional credit." -us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Its own arithmetic produced $7,127, yet it submitted $5,105, a number that appears nowhere in its derivation. The derivation itself used an unexplained $213,150 base and a $28,867 'standard/filing deduction' instead of $248,144.53 less the $24,659.28 charitable deduction, and it took only a $60 credit while omitting the $145 of personal credits." +us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took a roughly $2,340 standard deduction and never itemized the charitable deduction ($25,900 less the 0.5%-of-AGI floor, or $24,646.33), so its taxable income was about $245,805 instead of $223,498.20. It then cut its own ~$9,196 figure to $8,081 with an unexplained 'bracket smoothing' step, and it never applied the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity without the 2026 floor of 0.5% of AGI, giving $222,245 instead of $223,498.20. It then used a 2%/4%/3.9% schedule that produced $8,586.66, well above the $8,189.53 the 2026 schedule gives. It also subtracted only the $145 in personal credits and missed the $60 additional tax credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages even though wages already include it. It also started from federal taxable income, included the state refund, and invented $18,398 in self-employment tax. It then used a 5% top bracket and $1,050 of made-up credits, instead of $223,498.20 of taxable income taxed at $8,189.53 less $205 in credits." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $22,500 of mortgage interest from the $501,000 loan balance, although no interest paid is listed, so it should be $0. That pushed itemized deductions to about $48,400 instead of $24,646.33. It then submitted $11,264, which contradicts its own estimate of about $7,645." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $2,589 state refund as Arkansas income and imputed about $20,040 of mortgage interest when none is listed, putting itemized deductions near $48,000 instead of $24,646.33. Its $11,820 does not follow from its own ~$202,000 taxable income at a 3.9% top rate." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added invented mortgage-interest and medical deductions to charity, which cut taxable income to about $190,000 instead of $223,498.20. Medical expenses fall below the floor, and no mortgage interest is listed. It also subtracted only $145 in credits and missed the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left the $2,589 state refund in Arkansas income and took the full $25,900 in charity without the 0.5%-of-AGI floor, reaching $224,834 of net income instead of $223,498.20. It taxed that under a 3.9%-top schedule to get $8,685, above the $8,189.53 the 2026 schedule gives. It also subtracted only $145 and omitted the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It subtracted the $35,525 FLSA overtime premium as exempt under Arkansas law, but the engine's Arkansas AGI is $248,144.53 with no overtime exclusion. It also added a $10,000 SALT deduction even though no state or local taxes paid are listed, plus $448 of employee business expenses. Finally, it claimed a $240 Arkansas child-care credit even though the federal credit is $0 because the head has no earned income." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It imputed about $20,040 of mortgage interest that is not in the facts, reached a pre-credit tax of about $8,086, and then raised the answer to $12,500 with no computation behind the change. It also never subtracted the $145 in personal credits or the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction and taxed about $245,000, when itemizing the charitable deduction ($24,646.33 after the 0.5%-of-AGI floor) brings taxable income down to $223,498.20. It also applied a 3.9% top rate, which overstates tax compared with the engine's 2026 schedule." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an outdated 4.9% top rate on a taxable income of about $184,000, far below the correct $223,498.20. It also stated that no nonrefundable credits apply, so it dropped the $145 in personal credits and the $60 additional credit. The $8,000 is a rough figure built from these offsetting errors." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $5,000 standard deduction instead of itemizing the charitable deduction ($24,646.33), leaving $243,145 of taxable income instead of $223,498.20. It also used a 3.9%-top schedule and subtracted neither the $145 in personal credits nor the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $185,403.68 of taxable income plus $29,805.33 of deductions implies an AGI of $215,209, which is $250,734 minus the $35,525 overtime premium. That means it treated the FLSA overtime premium as excluded from Arkansas income. The correct Arkansas AGI is $248,144.53, and the correct itemized deduction is $24,646.33, giving $223,498.20 of taxable income." +us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the outdated 4.4% top rate with 2%/4% lower brackets and a $4,400 standard deduction on $250,734, which still includes the state refund. It never itemized the $24,646.33 charitable deduction and applied no personal credits or the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from federal AGI of $250,734, leaving in the $2,589 state refund that Arkansas excludes. It itemized $26,641, meaning full charity with no 0.5% floor plus a medical amount that does not apply, and taxed $224,093 under a 3.9%-top schedule to get $8,413.63 instead of $8,189.53. It subtracted only $145 and missed the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no working, but $10,560 equals a flat 4.4% on $240,000. That matches the outdated 2023 top rate applied to income with no itemized charitable deduction and no credits, instead of $223,498.20 taxed at $8,189.53 less $205." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"$8,768 equals a flat 3.9% on $224,834, which is $250,734 minus $25,900. That means it kept the state refund in income, took charity without the 0.5%-of-AGI floor, used a flat 3.9% instead of the 2026 schedule, and subtracted none of the $205 in nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"$8,623 is a flat 3.9% on $224,834 ($8,768) minus $145. So it kept the $2,589 state refund in Arkansas income, skipped the 0.5%-of-AGI charitable floor, used a flat 3.9% instead of the 2026 schedule's $8,189.53, and omitted the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no working, but $10,879 is consistent with the outdated 4.4% top rate applied to about $247,000, meaning income with only a standard deduction. The correct steps are to itemize $24,646.33 in charity, tax $223,498.20 under the 2026 schedule for $8,189.53, and subtract $205 in credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no working, and its $8,234.12 is about $50 above the correct result. The correct result comes from taxing $223,498.20 (AGI $248,144.53 less $24,646.33 of floored charity) at $8,189.53 under the 2026 schedule and then subtracting the full $205 in credits, including the $60 additional credit for qualified individuals. Its figure reflects a higher-rate schedule or only $145 in credits." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no working, but $9,260 is about 3.9% of $237,000, which is consistent with taxing income after only a standard deduction. The household should itemize $24,646.33 in charity, which brings taxable income to $223,498.20 and pre-credit tax to $8,189.53." +us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly applied the Arkansas standard deduction instead of itemizing the charitable deduction ($25,900 less the $1,253.67 AGI floor, or $24,646.33), so it taxed about $20,000 too much income. It also used a 3.9% top rate instead of the lower-yielding 2026 schedule." +us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It reached $222,245 of taxable income (full charity with no 0.5% floor) and then applied an invented bracket table with 3.4%, 4.7% and 4.4% rates to get $8,928.98. It also subtracted neither the $145 in personal credits nor the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It kept the state refund in AGI, added a small medical deduction that the floor rules out, and rounded a 3.9%-top calculation to $8,000. It also stated that no Arkansas nonrefundable credits apply, which drops the $145 in personal credits and the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It called the Arkansas tax 'modest' and reported $1,005, far too low. The household has $223,498.20 of Arkansas taxable income, which the 2026 schedule taxes at $8,189.53 before the $205 in nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It reported zero tax by citing unspecified 'high deductions/offsets'. The only Arkansas deduction is $24,646.33 of charity, which leaves $223,498.20 of taxable income and $8,189.53 of tax before the $205 in credits." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It took the full $25,900 charitable deduction without the 0.5%-of-AGI floor, giving $222,245, and applied a 3.9%-top high-income schedule that produced $8,496.66 instead of $8,189.53. It then subtracted $160 in credits instead of $205 ($145 in personal credits plus the $60 additional credit for qualified individuals)." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used 'standard and personal deductions' instead of itemizing $24,646.33 in charity, which overstates taxable income. It also stated that no nonrefundable credit applies, dropping the $145 in personal credits and the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It itemized charity but used a graduated schedule that produces more tax than the 2026 schedule's $8,189.53 on $223,498.20. It subtracted only the small personal and dependent credits and left out the $60 additional credit for qualified individuals, which puts it $122 high." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It came within $10.78 but did not reproduce the exact steps: charity of $25,900 less 0.5% of federal AGI ($1,253.67) gives $24,646.33, taxable income is $223,498.20, and pre-credit tax is $8,189.53. The final step subtracts exactly $205 ($145 in personal credits plus the $60 additional credit)." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity without the 0.5%-of-AGI floor, giving $222,245, and taxed that under a 3.9%-top high-income schedule to about $8,578 instead of $8,189.53. It subtracted only $145 and omitted the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"$9,535 plus $145 equals 3.9% of about $248,200, which is essentially the full Arkansas AGI. So it never itemized the $24,646.33 charitable deduction, used a 3.9% rate instead of the 2026 schedule, and omitted the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a 3.9% top marginal rate to taxable income after charity, which overstates the $8,189.53 the 2026 schedule gives on $223,498.20. It also counted only the personal credits and missed the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It deducted the full $25,900 in charity from $248,145 with no 0.5%-of-AGI floor and applied a 3.9%-top high-income schedule, giving about $8,585 before credits instead of $8,189.53. It subtracted only $145 and left out the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave a round $8,000 with no deduction, bracket or credit steps. The actual result requires $24,646.33 of itemized charity, $223,498.20 of taxable income, $8,189.53 of tax under the 2026 schedule, and $205 in nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied the outdated 4.4% top rate to about $224,000 of taxable income. The 2026 schedule taxes $223,498.20 at only $8,189.53, and the answer also needs the $145 in personal credits and the $60 additional credit subtracted." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It started from federal AGI of $250,734, keeping the state refund that Arkansas excludes. It added a $741 medical deduction and skipped the 0.5% charitable floor, then taxed $224,093 under a 3.9%-top schedule. It also stated that no nonrefundable credits apply, dropping the $145 in personal credits and the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used $223,734 as AGI, which is about $24,400 below the true Arkansas AGI of $248,144.53 (wages less the $530 loss). That drove taxable income down to $197,834 instead of $223,498.20. It also missed the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an unexplained AGI of $242,511 instead of $248,144.53 and took charity without the 0.5% floor. It then applied a flat 3.9% instead of the 2026 schedule and subtracted none of the $205 in nonrefundable credits." +us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It taxed about $221,000 at roughly 3.9%, which overstates tax compared with the 2026 schedule's $8,189.53 on $223,498.20. It also subtracted only small personal credits and missed the $60 additional credit for qualified individuals." +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so the model produced no usable answer." +us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took the joint standard deduction, leaving $245,654 of taxable income, instead of itemizing the $24,646.33 charitable deduction, which gives $223,498.20. It also taxed that base under a 3.9%-top schedule and subtracted only $145, omitting the $60 additional credit." +us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability. The household has $223,498.20 of Arkansas taxable income taxed at $8,189.53, and only $205 of nonrefundable credits offset it." +us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the $2,589 state refund in Arkansas AGI and took full charity without the 0.5%-of-AGI floor, reaching $224,834 of taxable income. It then applied a rate of about 3.8% and subtracted $155 in guessed personal credits. That misses the correct $205 in credits ($145 in personal credits plus the $60 additional credit for qualified individuals)." +us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages, which already include it, and added the state refund, giving an inflated AGI of $286,259. It then taxed income after only a standard deduction at a flat, outdated 4.4%, and dropped the itemized path it had computed itself." +us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It invented a $28,867 'filing deduction' and subtracted only the $60 dependent credit, dropping the $145 in personal credits. It then submitted $5,105, which contradicts its own stated $7,127." us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model stopped at a generic disability-based Medicaid inquiry and failed to apply New York's automatic SSI-recipient pathway. The head meets SSI's aged criterion and passes the $70 resource and $213.33 countable-income tests, producing a $780.67 monthly SSI benefit and automatic Medicaid eligibility." us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base." -us,scenario_118,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted only the $233.33/month Social Security and never counted the $780.67/month SSI this aged, blind, disabled head receives, so gross income was $233 instead of $1,014 and net income collapsed to $0. With SSI counted, deductions of $819.62 leave net income of $194.38 and an allotment of $298 − $58.20 = $239.80/month; it also used a $293 maximum instead of the FY2026 one-person $298." -us,scenario_118,snap,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got gross income right at $1,014 (Social Security plus SSI) but stopped its deduction stack at about $405/month — the $209 standard deduction plus a $196 excess shelter allowance ($598.75 of mortgage interest and property taxes less half of adjusted income) — instead of the $819.62 the household receives. That left net income of $609 rather than $194.38, so it charged an expected contribution of $182 instead of $58.20 and paid $115/month rather than $239.80." -us,scenario_118,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It inverted the effect of the deductions it listed, concluding that shelter and medical expenses would make the household 'exceed the income limit'; deductions reduce countable income, and this unit's $1,014/month gross is 78% of the $1,304.17 FPG while net income is $194.38, passing the gross and net tests. With $70 in assets and SSI-based categorical eligibility, the allotment is $239.80/month, not $0." -us,scenario_118,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated Social Security of $233.33/month as the household's entire income and omitted the $780.67/month SSI, so its uncapped excess shelter deduction drove net income to $0 and it paid the maximum allotment. Counting SSI gives gross $1,014, deductions of $819.62, net income of $194.38, and $298 − $58.20 = $239.80/month; its $293 maximum is also below the FY2026 $298." -us,scenario_118,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $780.67/month SSI and computed net income of $29/month from Social Security alone, and its own arithmetic produced roughly $3,400–$3,504/year before it submitted an underived $2,533. The correct base is gross $1,014 less $819.62 of deductions, giving net income of $194.38 and an allotment of $239.80/month." -us,scenario_118,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted only the $233.33/month Social Security and dropped the $780.67/month SSI, putting net income at $29; it then reported $203/month even though its own stated formula ($298 minus 30% of $29) yields $289. Net income is $194.38, so the expected contribution is $58.20 and the allotment is $239.80/month." -us,scenario_118,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It explicitly asserted 'no SSI' for a 74-year-old blind and disabled head with $2,800 of Social Security and $70 in assets, who in fact receives $780.67/month of SSI, so its gross income was $233.33 and its uncapped shelter deduction zeroed net income. Counting SSI puts net income at $194.38 and the allotment at $239.80/month, and its $292 FY2025 maximum is superseded by the FY2026 $298." -us,scenario_118,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $780.67/month SSI, so its reasoning drove net income to zero and landed on the $292 maximum ($3,504/year), then it submitted $2,555 as an unexplained reduction from that figure. With SSI counted, gross income is $1,014, deductions are $819.62, net income is $194.38, and the benefit is $239.80/month." -us,scenario_118,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used $233/month of Social Security as the household's whole income and never counted the $780.67/month SSI, yielding net income of $34 and a near-maximum $282/month allotment. Gross income is $1,014, deductions total $819.62, net income is $194.38, and the allotment is $298 − $58.20 = $239.80/month." -us,scenario_118,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It asserted $0 net income after deductions by counting only Social Security and omitting the $780.67/month SSI, then applied the FY2025 $292 maximum. Net income is $194.38 once SSI is in the $1,014 gross, and the FY2026 one-person maximum is $298, giving $239.80/month." -us,scenario_118,snap,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It deducted only $408.75/month (a $210 standard deduction plus a $198.75 excess shelter allowance) against the $819.62 this elderly/disabled unit receives, leaving net income of $601.25 instead of $194.38. It also inflated the maximum allotment to $303 rather than $298, so its $122.63/month understates the correct $239.80." -us,scenario_118,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It counted SSI but overstated it at $864.67/month and then wiped out the entire $1,098 gross with a standard deduction, shelter deduction, and an invented standard utility allowance to reach $0 net income. Actual gross is $1,014, total deductions are $819.62, net income is $194.38, and the allotment is $298 − $58.20 = $239.80/month rather than the $292 FY2025 maximum." -us,scenario_118,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It denied any benefit on the ground that income was 'insufficient to generate a SNAP benefit,' reversing the allotment formula: the benefit is the maximum allotment minus 30% of net income, so low income produces a larger benefit, and SNAP has no minimum income. Gross $1,014 less $819.62 of deductions gives net income of $194.38 and an allotment of $239.80/month." -us,scenario_118,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It allowed only $371/month of deductions (a $198 standard deduction plus a $173 excess shelter allowance) against the $819.62 the household receives, leaving net income of $679 instead of $194.38. Its $87/month therefore charges a $204 expected contribution where the correct contribution is $58.20 against a $298 maximum, i.e. $239.80/month." -us,scenario_118,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied New York's NYSNIP/SSI-CAP standardized allotment for SSI recipients living alone, which pays a flat maximum regardless of countable income, instead of the ordinary net-income computation. The applicable calculation counts the $780.67 SSI in $1,014 of gross income, deducts $819.62, and pays $298 − 30% × $194.38 = $239.80/month." -us,scenario_118,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 from '$2,800 of Social Security and allowable deductions' without counting the $780.67/month SSI or running the allotment formula at all. Net income of $194.38 produces an expected contribution of $58.20 against the $298 one-person maximum, so the household receives $239.80/month." -us,scenario_118,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $3,492 is exactly 12 × $291, a maximum allotment paid at zero net income, which is what counting only the $233.33/month Social Security against the standard deduction produces. Counting the $780.67/month SSI raises gross income to $1,014 and leaves net income of $194.38 after $819.62 of deductions, for $239.80/month." -us,scenario_118,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $828/year is $69/month, which implies an expected contribution of $229 and net income of about $763 — roughly $251 of deductions against the $819.62 this elderly/disabled unit actually receives. Correct net income is $194.38, giving a $58.20 contribution and $239.80/month." -us,scenario_118,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Working annually, it deducted only $4,863 (a $2,400 standard deduction plus a $2,463 excess shelter allowance) against the $9,835 of deductions the household receives, leaving annual net income of $6,981 instead of $2,332.56. It also used the FY2025 $3,504 annual maximum rather than $3,576, so its 30% reduction of $2,094 replaced the correct $698.40." -us,scenario_118,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"It built the whole calculation on $233.33/month of Social Security and omitted the $780.67/month SSI, so its uncapped excess shelter deduction pushed net income to $0 and it paid a $295 maximum. Gross income is $1,014, deductions are $819.62, net income is $194.38, and the allotment is $298 − $58.20 = $239.80/month." -us,scenario_118,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $780.67/month SSI and also skipped the excess shelter deduction entirely, computing net income of $26 from Social Security minus the standard deduction alone and paying $288/month. Counting SSI gives $1,014 gross, and the full $819.62 deduction stack leaves net income of $194.38 for a $239.80 monthly allotment." -us,scenario_118,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned $0 claiming the household has 'no expenses or other eligibility facts listed,' though $5,551 of mortgage interest, $1,634 of real estate taxes, $70 in assets, and SSI-conferring age/blindness/disability are all stated. Those facts yield gross income of $1,014, deductions of $819.62, net income of $194.38, and an allotment of $239.80/month." -us,scenario_118,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It declared the facts inadequate and returned $0, ignoring that the listed age, blindness, disability, $2,800 of Social Security, $70 of assets, and shelter costs fully determine the allotment. The household passes the gross ($1,014, 78% of the $1,304.17 FPG), net ($194.38), and asset tests and receives $298 − $58.20 = $239.80/month." -us,scenario_118,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied the $20 general income exclusion, an SSI rule with no counterpart in the SNAP net income calculation, and omitted the $780.67/month SSI to reach $0 net income, then paid a $243/month maximum that is $55 below the FY2026 one-person maximum of $298. Net income is $194.38, so the benefit is $298 − $58.20 = $239.80/month." -us,scenario_118,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $3,488.40 is $290.70/month, implying net income of $24.33 — Social Security alone less the standard deduction — so it never counted the $780.67/month SSI as unearned income. With SSI in gross income of $1,014 and $819.62 of deductions, net income is $194.38 and the allotment is $239.80/month." -us,scenario_118,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It counted Social Security and SSI correctly but allowed only about $405/month of deductions — the standard deduction plus a $196 excess shelter allowance from $598.75 of mortgage interest and property taxes — instead of $819.62, leaving net income near $610. That produced $115/month; the correct net income of $194.38 gives a $58.20 contribution and $239.80/month." -us,scenario_118,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $123/month implies an expected contribution of $175 and net income of $583, meaning it allowed roughly $431 of deductions against the $819.62 this elderly/disabled unit receives on its $598.75 of monthly shelter costs. Net income is $194.38, so the allotment is $298 − $58.20 = $239.80/month." -us,scenario_118,snap,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It got gross income exactly right at $1,014 and used the $209 standard deduction, then explicitly refused any shelter deduction on the ground that mortgage payments were unlisted — the $5,551 of mortgage interest and $1,634 of real estate taxes are listed and are deductible shelter costs, uncapped for an elderly/disabled unit. Dropping that deduction left net income at $805 instead of $194.38, cutting the allotment to $56.50/month instead of $239.80." -us,scenario_118,snap,grok-4.3,llm_error,other,False,"It declined to compute, asserting that no calculation inputs were provided, and returned $0 despite the listed state, age, blindness, disability, Social Security income, $70 of assets, mortgage interest, and real estate taxes. Those inputs give gross income of $1,014, deductions of $819.62, net income of $194.38, and an allotment of $239.80/month." -us,scenario_118,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It recognized SSI receipt and categorical eligibility but set net income at about $735/month, allowing only roughly $280 of deductions instead of the $819.62 available on $598.75 of shelter costs plus the standard deduction. Its $81/month also rests on a $301 maximum; the correct figures are net income $194.38, contribution $58.20, maximum $298, and allotment $239.80/month." -us,scenario_118,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It excluded the $780.67/month SSI from countable income, so the uncapped excess shelter deduction zeroed net income and it paid a full $292 FY2025 maximum. Gross income is $1,014, net income after $819.62 of deductions is $194.38, and the FY2026 allotment is $298 − $58.20 = $239.80/month." -us,scenario_118,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It made two offsetting errors: it omitted the $780.67/month SSI, leaving gross income at $233, and it declared shelter costs 'unlisted so 0' although $5,551 of mortgage interest and $1,634 of real estate taxes are stated. The correct calculation counts $1,014 of gross income and $598.75/month of shelter, giving net income of $194.38 and $239.80/month rather than its $279." -us,scenario_118,snap,inkling,llm_error,taxable_income_or_deductions,False,"Its $3,576 is exactly 12 × the $298 maximum, the result of counting only $233/month of Social Security so the shelter deduction drove net income to $0. Counting the $780.67/month SSI leaves net income of $194.38 after $819.62 of deductions, so the allotment is $239.80/month, not the full maximum." -us,scenario_118,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It used $2,800/year of Social Security as the household's entire income, omitting $780.67/month of SSI, and so reached $0 net income; it then applied a $309 maximum instead of the FY2026 $298. Actual net income is $194.38, giving an expected contribution of $58.20 and a $239.80 monthly allotment." -us,scenario_118,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"It overstated SSI at $867.67/month and then stated there was 'no separate SNAP housing-cost input,' discarding the $5,551 of mortgage interest and $1,634 of real estate taxes that generate the uncapped excess shelter deduction. That left net income at $892 instead of $194.38, so its $30/month replaced the correct $239.80." -us,scenario_118,snap,minimax-m3,llm_error,categorical_eligibility,False,"It denied the benefit on the ground that '$2,800 income is too low to qualify,' but SNAP has no minimum income and the allotment rises as net income falls. This unit also receives $780.67/month of SSI, giving gross income of $1,014, net income of $194.38, and a benefit of $298 − $58.20 = $239.80/month." -us,scenario_118,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It reproduced gross income exactly ($233.33 Social Security plus $780.67 SSI = $1,014) and the $209 standard deduction, but its excess shelter allowance of $196.25 left total deductions at $405.25 instead of the $819.62 this elderly/disabled unit receives. Net income is $194.38, not $608.75, so the expected contribution is $58.20 and the allotment $239.80/month rather than $115.38." -us,scenario_118,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $780.67/month SSI and also never applied the excess shelter deduction for the $598.75/month of mortgage interest and real estate taxes, computing net income of $35.33 from Social Security minus a $198 standard deduction. Correct gross income is $1,014 and correct net income is $194.38, giving $298 − $58.20 = $239.80/month instead of its $280." -us,scenario_118,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It returned $0 for lack of a 'qualifying child or household member,' importing a dependent requirement that SNAP does not have: a single 74-year-old is a valid one-person SNAP unit. This unit passes the gross ($1,014, 78% of FPG), net ($194.38), and asset ($70) tests and is categorically eligible through SSI, yielding $239.80/month." -us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model's own reasoning reached the reference exactly — 2026 FBR $994/month less $213.33/month countable unearned income equals $780.67/month, or $9,368/year — and it explicitly wrote that figure before submitting $6,144 ($512/month) instead. It abandoned its correct derivation in favor of a discarded intermediate branch built on a $725.33 FBR it had already rejected, so the loss is a submission error, not a rules error." -us,scenario_118,ssi,claude-haiku-4.5,llm_error,period_annualization,False,"The model compared the annual $2,800 Social Security total against a monthly benefit rate ($943/month) and declared the income limit exceeded; the correct comparison is $233.33/month gross unearned income, which after the $20/month general exclusion leaves $213.33/month countable — far below the $994 FBR. SSI is a top-up equal to the FBR minus countable income, not a cliff, so the correct result is $780.67/month or $9,368/year, not $0." -us,scenario_118,ssi,claude-opus-4.7,llm_error,state_local_rule,False,"The model computed federal SSI from a stale $967 FBR and then added a New York State Supplement Program payment of roughly $145/month to reach $10,788. PolicyEngine's `ssi` variable is the federal payment only — the New York SSP is a separate variable and is never folded into `ssi` — and the 2026 FBR is $994/month, so the answer is $994 − $213.33 = $780.67/month, or $9,368." -us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"The model established categorical eligibility, passed the resource test with $70, computed a positive federal payment, and stated in its own explanation that ""estimated annual SSI is positive"" — then submitted 0. Its arithmetic was also off in a second way: the $20 general income exclusion is monthly ($240/year), so countable income is $2,560, not $2,780, and $11,928 − $2,560 = $9,368." -us,scenario_118,ssi,claude-opus-5,llm_error,other,False,"The model's own inputs — an FBR near $1,000/month less $213/month countable income — yield about $787/month, yet it submitted $667/month ($8,004/year) after invoking a New York state supplement that would push the figure up, not down. It applied an unexplained downward adjustment that contradicts its own arithmetic; with the actual 2026 FBR of $994 the payment is $780.67/month, or $9,368/year." -us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the stale $967/month FBR instead of the 2026 rate of $994/month, and applied the $20 general income exclusion once for the whole year ($2,800 − $20 = $2,780) rather than $20 per month ($240/year, giving $2,560 countable). Both errors cut the payment: the correct calculation is $11,928 − $2,560 = $9,368, not $8,824." -us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"The model computed $12,152 − $2,780 = $9,372 and then discarded it, submitting $6,212 as an unexplained ""adjustment for NY state supplement and rounding"" — a supplement would raise the figure, and PolicyEngine's `ssi` excludes the New York SSP entirely. Its inputs were also wrong in both directions ($1,013/month FBR against the actual $994, and a $20 annual rather than $20 monthly exclusion); the correct result is $11,928 − $2,560 = $9,368." -us,scenario_118,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied the correct method — FBR minus countable income of $213.33/month after the $20 monthly general exclusion — but projected the 2026 federal benefit rate at $991/month instead of $994/month. That $3/month shortfall is the entire error: $994 − $213.33 = $780.67/month, or $9,368 rather than $9,332." -us,scenario_118,ssi,deepseek-v4-pro,llm_error,state_local_rule,False,"The model added an estimated $54/month New York state supplement to the federal benefit rate to build a $1,045/month combined payment standard. PolicyEngine's `ssi` variable reports the federal SSI payment alone, and the 2026 FBR is $994/month rather than the $991 used, so the payment is $994 − $213.33 = $780.67/month, or $9,368." -us,scenario_118,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model executed the FBR-minus-countable-income formula correctly, deducting the $20 monthly general exclusion to reach $213.33/month countable, but used a 2026 FBR of $990/month rather than the actual $994/month. The $4/month rate error is the whole gap between its $9,320 and the correct $9,368." -us,scenario_118,ssi,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model built a combined payment standard of $1,078/month by adding an $87/month New York living-alone supplement to a $991/month federal rate. The New York SSP is a distinct PolicyEngine variable and is not part of `ssi`, and the 2026 FBR is $994/month, so the federal-only payment is $994 − $213.33 = $780.67/month, or $9,368 instead of $10,376." -us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,period_annualization,False,"The model asserted that $2,800 of annual Social Security income exceeds the federal benefit rate, comparing an annual income total against a monthly rate; the 2026 annual FBR is $11,928 and monthly countable income is only $213.33 after the $20 general exclusion. SSI reduces the FBR by countable income rather than cutting off at it, giving $9,368, not $0." -us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model added a $1,044/year New York state supplement to a stale $11,316/year federal maximum ($943/month, the 2024 rate) to reach a $12,360 payment standard. PolicyEngine's `ssi` is the federal payment only and the 2026 annual FBR is $11,928, so the correct result is $11,928 − $2,560 = $9,368; its $240 exclusion and $2,560 countable income were right." -us,scenario_118,ssi,gemini-3.5-flash,llm_error,state_local_rule,False,"The model explicitly used the 2024 Federal Benefit Rate of $943/month for a 2026 calculation and then added an $87/month ($1,044/year) New York State Supplement to its $8,756 federal figure. The 2026 FBR is $994/month and `ssi` excludes the New York SSP, so the payment is $780.67/month, or $9,368." -us,scenario_118,ssi,gemini-3.5-flash-lite,llm_error,period_annualization,False,"The model compared the $2,800 annual Social Security total against the monthly federal benefit rate and zeroed out the benefit; monthly gross unearned income is $233.33, and after the $20 monthly general exclusion countable income is $213.33 against a $994 FBR. The benefit is the FBR less countable income — $780.67/month, or $9,368/year." -us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied the $240 annual general income exclusion correctly to reach $2,560 countable income, but its unstated federal maximum implies $983/month ($11,796/year) rather than the 2026 FBR of $994/month ($11,928/year). Subtracting $2,560 from the correct annual FBR gives $9,368, not $9,236." -us,scenario_118,ssi,gemini-3.7-flash,llm_error,state_local_rule,False,"The model correctly derived $2,560 of countable income after the $240 annual general exclusion but then reported a ""combined federal and NY state supplement"" figure of $10,376. PolicyEngine's `ssi` variable carries the federal SSI payment alone, which is $11,928 − $2,560 = $9,368; the New York SSP belongs to a separate variable." -us,scenario_118,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model got every structural step right — categorical eligibility, the $2,000 resource test, and $2,560 countable income after the $240 annual general exclusion — but anchored on the $967/month ($11,604/year) rate instead of the 2026 FBR of $994/month ($11,928/year). Using the correct rate, $11,928 − $2,560 = $9,368 rather than $9,044." -us,scenario_118,ssi,glm-5.2,llm_error,state_local_rule,False,"The model computed a $777.67/month federal payment from a $991 FBR and then added roughly $94/month of New York state supplement for a blind individual living independently. `ssi` in PolicyEngine is the federal payment only, and the 2026 FBR is $994/month, so the answer is $780.67/month, or $9,368 instead of $10,460." -us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model declined to apply the SSI benefit formula at all, asserting the listed facts ""do not establish positive SSI."" The facts given are complete for the computation: age 74 with blindness establishes the aged/blind/disabled category, $70 in bank assets clears the $2,000 resource limit, and $2,800 of unearned income leaves $213.33/month countable after the $20 general exclusion, yielding $994 − $213.33 = $780.67/month, or $9,368." -us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed no SSI-eligibility facts were supplied and defaulted to zero, ignoring that age 74 plus the stated blindness and disability satisfies the aged/blind/disabled category outright and that $70 in bank assets is the full countable-resource figure against the $2,000 limit. Running the standard formula on the given $2,800 of unearned income gives $11,928 − $2,560 = $9,368." -us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"The model applied the $240 annual general income exclusion correctly for $2,560 countable income, but stated a 2026 single FBR of $982.67/month while its submitted $9,332 implies $991/month — neither matches the actual $994/month ($11,928/year). The correct subtraction is $11,928 − $2,560 = $9,368." -us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model used the correct 2026 individual federal maximum of $11,928 but applied the $20 general income exclusion once for the entire year, reaching $2,780 of countable income. The exclusion is $20 per month — $240 per year — so countable income is $2,560 and the benefit is $11,928 − $2,560 = $9,368, not $9,148." -us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted no calculation inputs were available and returned zero without applying the SSI formula, despite the prompt supplying age 74, blindness, disability, $70 in bank assets, and $2,800 of Social Security income — every input the computation requires. Those facts give countable income of $2,560 per year against the $11,928 federal benefit rate, for $9,368." -us,scenario_118,ssi,grok-4.5,llm_error,state_local_rule,False,"The model derived a federal payment of about $778/month from a $991 FBR and then added an $87/month New York living-alone SSP to reach $865/month. PolicyEngine's `ssi` excludes the New York state supplement, and the 2026 FBR is $994/month, so the payment is $780.67/month, or $9,368." -us,scenario_118,ssi,grok-4.6,llm_error,thresholds_rates,False,"The model used the right structure — countable income of $2,800 less the $240 annual general exclusion, subtracted from the individual FBR — but its $9,044 implies the $967/month ($11,604/year) rate rather than the 2026 FBR of $994/month ($11,928/year). The correct result is $11,928 − $2,560 = $9,368." -us,scenario_118,ssi,grok-build-0.1,llm_error,state_local_rule,False,"The model explicitly used the 2024 federal maximum of $943/month ""as a proxy for 2026"" and added an $87/month New York supplement to build a $1,030 payment standard. The 2026 FBR is $994/month and `ssi` reports federal SSI alone, so the benefit is $994 − $213.33 = $780.67/month, or $9,368." -us,scenario_118,ssi,inkling,llm_error,state_local_rule,False,"The model had every federal input exactly right — $994/month FBR, $213.33/month countable income after the $20 general exclusion, $780.67/month payment — and then added roughly $87/month of New York state supplement. The New York SSP is a separate PolicyEngine variable and is not included in `ssi`, so the federal-only answer of $9,368 stands rather than $10,404." -us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied the $240 annual general income exclusion correctly for $2,560 countable income but estimated the 2026 federal benefit rate at $991/month ($11,892/year) instead of $994/month ($11,928/year). That $36 annual rate shortfall accounts for the entire difference between its $9,332 and the correct $9,368." -us,scenario_118,ssi,kimi-k3,llm_error,state_local_rule,False,"The model derived the federal answer exactly — $11,928 annual maximum less $2,560 countable income after the $240 general exclusion, leaving $9,368 — and then added a $1,044 New York living-alone supplement on top. PolicyEngine's `ssi` variable is the federal SSI payment only; the New York SSP is computed separately and must not be added in." -us,scenario_118,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model wrote that $233/month of Social Security ""exceeds"" a $967/month federal benefit rate — a comparison its own numbers contradict — and treated SSI as an income cutoff rather than a top-up. Countable income is $213.33/month after the $20 general exclusion, so the benefit is $994 − $213.33 = $780.67/month, or $9,368." -us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model's explanation computed a positive federal payment of $729.67/month ($8,756/year) and confirmed categorical eligibility, then submitted 0 as the value in direct contradiction of its own text. Its underlying rate was also stale — the 2026 FBR is $994/month, not $943 — so the correct payment is $780.67/month, or $9,368." -us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model conceded the head is blind and disabled but asserted without computation that SSI is not positive. Age 74 with blindness establishes the aged/blind/disabled category, $70 of bank assets clears the $2,000 resource limit, and $2,560 of countable income after the $240 annual general exclusion leaves $11,928 − $2,560 = $9,368." -us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model reached the right credit and then affirmatively denied it, asserting that ""homeowners with these facts do not qualify for a real property tax circuit breaker credit payment here""; NY Tax Law §606(e) covers homeowners whose real property value does not exceed $85,000 (the first mortgage balance is $82,237) and pays on real estate taxes paid, with no rent requirement. It also treated NY AGI of $0 as dispositive, but the credit is refundable and its test is household gross income (Social Security included, $2,800 against an $18,000 ceiling), not NY AGI." -us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed New York offers no general refundable individual credit outside the Empire State EITC, omitting the refundable real property tax credit of §606(e)/Form IT-214, which requires neither earned income nor dependents. Its reliance on Social Security being untaxed by New York is the wrong test: IT-214 counts Social Security in household gross income ($2,800, well under the $18,000 limit) and pays $375 on $1,633.50 of real estate taxes for a 74-year-old homeowner." -us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly identified the senior real property tax credit and even named the $375 maximum, then substituted an invented ""approximate ~$63 based on formula"" instead of running the IT-214 computation. Excess real property tax is $1,633.50 less 6% of household gross income ($168 on $2,800), and the rate factor applied to that $1,466 excess exceeds the ceiling, so the answer is the $375 cap it had already stated." -us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model made the credit liability-limited, reasoning there was ""no liability against which NY refundable credits (e.g., real property tax credit, EITC) would generate a refund."" The §606(e) real property tax credit is refundable and is paid in full when NY tax is zero; with a 74-year-old homeowner and $1,633.50 of real estate taxes, it pays its $375 senior maximum." -us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model asserted the real property tax credit fails on ""$0 taxable NY income and no reported rent."" Zero taxable income is not a bar to a refundable credit, and rent is only the renter branch of IT-214 — the homeowner branch pays on real estate taxes paid ($1,633.50) once household gross income ($2,800, Social Security included) is under $18,000, producing the $375 age-65+ maximum." -us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model executed the IT-214 arithmetic correctly — household gross income $2,800, 6% offset of $168, excess real property tax $1,466 — and then applied the $75 maximum that governs households with no member age 65 or older to a 74-year-old head. §606(e) sets the maximum at $375 when any household member is 65+, and since the computed credit exceeds that ceiling, the answer is $375; the model's reference to ""no qualifying dependents"" is irrelevant to which cap applies, as the cap turns on age, not dependents." -us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model dismissed the senior circuit breaker as ""very small"" and as requiring rent paid, and treated non-taxable Social Security as ending the analysis. The IT-214 homeowner branch applies here on $1,633.50 of real estate taxes with household gross income of $2,800, and the credit is capped rather than negligible: it pays the full $375 age-65+ maximum." -us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model conditioned state refundable credits on there being state tax liability (""given the zero state tax liability""), which inverts the definition of a refundable credit. New York's §606(e) real property tax credit pays out with zero liability, and this 74-year-old homeowner with $1,633.50 of real estate taxes and $2,800 of household gross income receives its $375 senior maximum." -us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model computed the $375 IT-214 senior maximum correctly, then added a $75 ""NY State Household Credit"" to reach $450. The household credit under §606(b) is nonrefundable — it belongs in state_income_tax_before_refundable_credits, not in refundable credits — and it produces nothing here because NY tax is zero, leaving $375." -us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model asserted without analysis that the household fails NY refundable credit eligibility, never testing the §606(e) real property tax credit. That credit's requirements — full-year NY residence, household gross income of $2,800 under the $18,000 ceiling, property value under $85,000, and real property taxes of $1,633.50 exceeding 6% of household gross income — are all met, and the head's age of 74 sets the maximum at $375." -us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model screened only for credits gated on earned income and dependents (Empire State Child Credit, NY EITC) and concluded zero. New York's refundable real property tax credit requires neither: it pays a 74-year-old homeowner with $1,633.50 of real estate taxes and $2,800 of household gross income the $375 age-65+ maximum." -us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model gave a bare denial of all NY refundable credits, omitting §606(e)'s real property tax credit, whose tests are age, household gross income, property value, and property tax paid. On these facts — age 74, $2,800 household gross income, $82,237 mortgage balance, $1,633.50 of real estate taxes — the credit reaches its $375 senior ceiling." -us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model concluded the household qualifies for no refundable New York credit without evaluating the IT-214 real property tax credit. That credit is refundable, homeowner-eligible, and keyed to age 65+ and real estate taxes paid; the computed excess over 6% of a $2,800 household gross income exceeds the ceiling, so the credit is the $375 maximum." -us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model made ""no earned income or qualifying dependents"" the entire eligibility test. The §606(e) real property tax credit has no earned-income or dependent requirement — it pays a 65+ homeowner on property taxes exceeding 6% of household gross income, which here yields the statutory $375 senior maximum." -us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model checked only the Empire State Child Credit and NY EITC before answering zero, never reaching the refundable real property tax credit. With household gross income of $2,800 (Social Security counted), real estate taxes of $1,633.50, and a head aged 74, that credit pays its $375 ceiling." -us,scenario_118,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model identified the right credit but used a 3.5%-of-household-income offset and a $1,000 senior maximum. §606(e) offsets 6% of household gross income ($168 on $2,800) and caps the credit at $375 when a household member is 65 or older, so its $768 result overshoots a binding ceiling that its own formula would have exceeded regardless." -us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model tested only the NY EITC's earned-income requirement and stopped. The refundable real property tax credit under §606(e) requires no earnings at all; on age 74, $2,800 of household gross income, and $1,633.50 of real estate taxes it pays the $375 senior maximum." -us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model declared no refundable NY credit is triggered without naming any credit it tested, missing the IT-214 real property tax credit. Its listed facts — NY residence, age 74, $1,633.50 real estate taxes, $2,800 household gross income, $82,237 property mortgage balance — satisfy every §606(e) condition and produce the $375 age-65+ maximum." -us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model treated the fact pattern as containing no refundable-credit trigger, overlooking that age plus real estate taxes paid is itself the trigger for New York's §606(e) real property tax credit. Excess real property tax of $1,633.50 over 6% of $2,800 household gross income exceeds the senior ceiling, so the credit equals the $375 maximum." -us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model reasoned from the income side only — ""Social Security-only income"" generating no credit — while the operative credit keys off property tax paid, not income earned. New York's refundable real property tax credit uses the $2,800 Social Security as household gross income (under the $18,000 limit) and pays the $375 senior maximum on $1,633.50 of real estate taxes." -us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model substituted the New York household credit ($90 for a qualifying surviving spouse with NY AGI at or below $5,000) for the refundable credit output; that credit is nonrefundable under §606(b), belongs before refundable credits, and yields nothing against zero NY tax. It never evaluated the refundable §606(e) real property tax credit, which pays this 74-year-old homeowner its $375 maximum." -us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model found the right credit and confirmed property taxes were sufficient to reach the cap, then used the $75 ceiling that applies only when no household member is 65 or older. The head is 74, so §606(e) sets the maximum at $375, and that ceiling binds whether household gross income is the $2,800 of Social Security alone or the grossed-up figure the model assumed." -us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model returned a bare ""no qualifying credits"" with no rule cited, skipping New York's refundable real property tax credit. That credit is satisfied on these facts by full-year NY residence, age 74, $2,800 household gross income below $18,000, property value under $85,000, and $1,633.50 of real estate taxes exceeding 6% of household gross income, giving the $375 senior maximum." -us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted the NY EITC is ""NY's main refundable credit"" and, finding federal EITC of $0 and no qualifying children, stopped. New York's §606(e) real property tax credit is refundable and independent of both, paying $375 — the age-65+ maximum — on $1,633.50 of real estate taxes with $2,800 of household gross income." -us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model explicitly contemplated ""property tax credits that apply here"" and rejected them. New York's real property tax credit applies squarely: the head is 74, owns property with an $82,237 mortgage balance (value under the $85,000 limit), paid $1,633.50 in real estate taxes, and has $2,800 of household gross income, which produces the $375 senior maximum." -us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model enumerated only credits gated on children, earnings, tuition, and childcare, then concluded no other NY refundable credit exists. It omitted the §606(e) real property tax credit, whose gates are age 65+, household gross income under $18,000, and real property taxes exceeding 6% of that income — all met here, yielding the $375 cap." -us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model rested on ""no NY tax liability and no qualifying children,"" both of which are irrelevant to New York's refundable real property tax credit. That credit pays out at zero liability and with no children, delivering the $375 age-65+ maximum on $1,633.50 of real estate taxes against $2,800 of household gross income." -us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model limited its search to the NY EITC and refundable CDCC, both of which require earnings, and concluded zero. New York's §606(e) real property tax credit needs no earnings or dependents and pays this 74-year-old homeowner $375, the statutory maximum for households with a member aged 65 or older." -us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model combined two non-tests — no NY taxable income and no earned income or children — to rule out all refundable credits. New York's real property tax credit is refundable irrespective of taxable income and keys on age 74 plus $1,633.50 of real estate taxes against $2,800 of household gross income, producing the $375 senior maximum." -us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model named age as a factor yet still concluded no NY refundable credit applies; age is precisely what qualifies this household, since §606(e) grants a $375 maximum real property tax credit (versus $75) when a household member is 65 or older. With $1,633.50 of real estate taxes exceeding 6% of $2,800 household gross income, the credit reaches that $375 ceiling." -us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"It ran only CHIP's two-prong screen — ""income ~197% FPL is below VA FAMIS limit of 200% FPL and child is under 19"" — and never applied CHIP's targeted-low-income-child requirement that the child be uninsured, which Child 1's employer-sponsored insurance defeats. Its 197% figure comes from dividing the correct $52,569 MAGI by the 2025 rather than the 2026 three-person guideline, but the income margin is not the operative test here." -us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"It cited only age under 19 and household income below Virginia's 205% FAMIS threshold, treating those two conditions as sufficient. It omitted the categorical requirement that a targeted low-income child not be covered under a group health plan, which Child 1's employer-sponsored insurance violates regardless of the income margin." -us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Its reasoning names only age 14, Virginia residence, and ""income within the CHIP (FAMIS) eligibility threshold,"" with no income computation and no coverage test. CHIP requires the child to be uninsured, and Child 1 holds employer-sponsored insurance, so the categorical prong fails before the income prong matters." -us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"It stopped at ""age 14 and household MAGI within Virginia FAMIS income limits (up to 205% FPL),"" which is the age and income screen alone. It never applied CHIP's requirement that the child lack other creditable coverage, and Child 1's employer-sponsored insurance makes him a non-targeted child." -us,scenario_119,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was returned for child1_chip_eligible, so nothing substantive was submitted for scoring. The correct derivation applies CHIP's uninsured requirement to a 14-year-old carrying employer-sponsored insurance and returns 0." -us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,health_coverage,False,"It correctly placed the household above Virginia's child Medicaid limit (143% FPL for ages 6-18) and inside the FAMIS band, then declared eligibility from that gap alone. Sitting in the Medicaid-to-CHIP income gap does not confer CHIP: the child must also be uninsured, and Child 1 has employer-sponsored insurance." -us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"It reasoned from ""under age 19"" plus an ""estimated Virginia CHIP income range"" without computing MAGI or the FPL ratio, and applied no coverage screen. CHIP excludes a child covered under a group health plan, and Child 1's employer-sponsored insurance is exactly that exclusion." -us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,health_coverage,False,"It computed the income ratio essentially correctly (~192% of the three-person guideline, under Virginia's 200% FAMIS limit) and then treated age plus that ratio as dispositive. The determinative step it skipped is CHIP's targeted-low-income-child definition, which excludes a child already covered by employer-sponsored insurance." -us,scenario_119,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"It framed the case as ""above Virginia child Medicaid but below the FAMIS/CHIP limit of about 200-205% FPL, so eligible,"" which is the income-band test alone. CHIP additionally requires the child to be uninsured, and Child 1's employer-sponsored insurance bars enrollment anywhere in that band." -us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,health_coverage,False,"It subtracted the head's $6,389 employer-sponsored insurance premium from income to reach $46,180 (168% FPL), a deduction MAGI does not allow because Section 125 premiums are already outside the $55,000 gross wage input; correct MAGI is $52,569, near 193% FPL. It then used the ESI fact only as an income deduction while ignoring what that same fact actually triggers — CHIP's exclusion of a child covered under a group health plan." -us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for child1_chip_eligible, so no substantive determination was made. The correct derivation fails Child 1 on CHIP's requirement that a targeted low-income child be uninsured, given his employer-sponsored insurance, and returns 0." -us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"It derived the right MAGI of $52,569 for the three-person unit and then concluded eligibility purely from ""under 19"" plus that figure landing ""within Virginia's CHIP income band."" It never applied CHIP's coverage exclusion, and Child 1's employer-sponsored insurance disqualifies him as a targeted low-income child." -us,scenario_119,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"It computed MAGI of $52,569 correctly but divided it by a stale poverty guideline to get ~204% FPL against Virginia's 205% limit, then rested the answer on that one-point margin plus the child age range. The decisive miss is CHIP's categorical requirement that the child not be covered under a group health plan; Child 1 has employer-sponsored insurance, so the income margin is beside the point." +us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $233/month of Social Security and never computed the head's $780.67/month of federal SSI ($994 benefit rate minus $213.33 of countable Social Security). SSI is SNAP unearned income, so gross income is $1,014. The model then added a guessed ~$857 utility allowance to reach $0 net income and paid a $293 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month (Jan–Sep) and $248.58 (Oct–Dec)." +us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly ($1,014 gross) and took the $209 standard deduction, but its shelter costs were only mortgage interest and property tax ($599), giving an excess shelter deduction of $196 and net income of $609. PolicyEngine adds New York's standard utility allowance to shelter costs. That raises the uncapped deduction to about $611 and cuts net income to $194, so the benefit is $239.80/month, not $115." +us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded that the household exceeds SNAP income limits. In fact the household is categorically eligible through SSI, and its $1,014 gross income (78% of FPG) and $194 net income pass every test. The deductions it listed lower net income and raise the benefit to $298 − $58.20 = $239.80/month; they do not eliminate it." +us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as the only income, leaving out the $780.67/month of federal SSI the aged, blind and disabled head receives. It also explicitly refused any utility allowance. Counting SSI gives $1,014 gross income, and with the utility allowance in shelter costs, net income is $194. The benefit is therefore $239.80/month, not the $293 maximum it assumed." +us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and ignored the mortgage-interest and property-tax shelter costs. It then dropped its own ~$3,400 result for an unexplained $2,533. The correct chain is $1,014 gross, minus the $209 standard deduction and the uncapped excess shelter deduction, giving $194 net income and $239.80/month." +us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the head's $780.67/month of SSI out of gross income and skipped the excess shelter deduction, then miscalculated $298 − 0.3 × $29 as $203 (it is $289). With SSI counted and the uncapped shelter deduction including the utility allowance, net income is $194 and the monthly benefit is $239.80." +us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI ($1,014 gross) and the $209 standard deduction, but its shelter costs were only the $599 of mortgage interest and property tax, which left net income at $609. PolicyEngine's shelter costs also include New York's standard utility allowance (about $1,013/month in total). That makes the uncapped excess shelter deduction about $611 and net income $194, so the benefit is $239.80/month, not $115." +us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It stated outright that the head has no SSI, so it counted only $233/month of Social Security and drove net income to $0. An aged, blind and disabled person with $2,800 of Social Security and $70 of assets receives $780.67/month of federal SSI, which SNAP counts as income. It also used the FY2025 $292 maximum instead of FY2026's $298." +us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, leaving out the $780.67/month of SSI, and reasoned its way to the maximum allotment. It then applied an unexplained cut to reach $2,555. With SSI in gross income ($1,014) and the uncapped shelter deduction, net income is $194 and the benefit is $239.80/month for Jan–Sep and $248.58 for Oct–Dec." +us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as total income and ignored the head's $780.67/month of federal SSI. That zeroed net income, so it paid the $298 maximum every month. Counting SSI gives about $194 of net income, a $58.20 expected contribution, and a $239.80 monthly benefit." +us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI. It also took no shelter deduction and used an outdated $199 standard deduction, which produced net income of $34. The correct figure is $194: $1,014 gross, minus $209, minus the uncapped excess shelter deduction." +us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It set net income to $0 by treating Social Security as the only income, leaving out the $780.67/month of SSI, and it paid the FY2025 $292 maximum. With SSI counted, net income is $194 and the monthly benefit is $298 − $58.20 = $239.80." +us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI but used only mortgage interest and property tax as shelter costs, giving a $198.75 excess shelter deduction and $601 of net income. PolicyEngine adds New York's standard utility allowance, which makes the deduction about $611 and net income $194. The model also used a $303 maximum allotment instead of FY2026's $298." +us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly excluded SSI from SNAP income. SSI is countable unearned income for SNAP, so gross income is $1,014, not $233. That mistake zeroed net income and produced a $299 maximum instead of $298 − 0.3 × $194 = $239.80/month." +us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI at $864.67, above the $780.67 federal SSI that PolicyEngine counts, and assumed a utility allowance big enough (over $735) to wipe out net income. PolicyEngine's total shelter costs are about $1,013/month, which leaves $194 of net income and a $58.20 contribution. The model also used the FY2025 $292 maximum." +us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without any calculation, claiming the income was too low to produce a benefit. That is backwards: lower income raises SNAP. The household is categorically eligible through SSI, has $194 of net income after deductions, and receives $239.80/month." +us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $817 (the federal amount is $780.67) and used stale FY2025 parameters: a $198 standard deduction and a $291 maximum. Its shelter costs were only mortgage interest and property tax, which gave $679 of net income. Adding New York's standard utility allowance to shelter costs brings net income to $194 and the benefit to $239.80/month." +us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied New York's NYSNIP standardized benefit for SSI recipients and paid a flat $291 maximum. PolicyEngine uses the regular SNAP formula: the $298 maximum minus 30% of $194 net income, or $239.80/month ($248.58 in Oct–Dec). $291 is also an outdated maximum." +us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It declared a $0 benefit on $2,800 of Social Security without doing the calculation. The household is categorically eligible through SSI and has $194 of net income after the standard and uncapped excess shelter deductions, which gives $239.80/month." +us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Paying the FY2025 $291 maximum all year means it counted only Social Security and zeroed net income. It left out the $780.67/month of SSI, which raises gross income to $1,014 and net income to $194, for a benefit of $239.80/month." +us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"$828 is $69/month, which implies net income of about $760. That fits counting Social Security plus SSI with a state supplement while deducting, at most, shelter costs made up of mortgage interest and property tax. It left New York's standard utility allowance out of shelter costs. With the allowance included, the uncapped excess shelter deduction is about $611 and net income is $194." +us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2025 values: SSI of $753.67/month instead of 2026's $780.67, a $200 standard deduction instead of $209, and the $292 maximum. Its shelter costs were only mortgage interest and property tax. Without New York's standard utility allowance, net income stayed at $582/month instead of $194." +us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $233/month of Social Security and left out the $780.67/month of SSI. That drove net income to $0, and it paid a guessed $295 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month." +us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and skipped the shelter deduction, which produced $26 of net income. Correct gross income is $1,014; after the $209 standard deduction and the uncapped excess shelter deduction, net income is $194 and the benefit is $239.80." +us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It denied any benefit, claiming the household has no expenses. That ignores the $5,551 of mortgage interest and $1,634 of property tax, which are SNAP shelter costs, and the SSI that makes the household categorically eligible. The household nets $194/month and receives $239.80/month." +us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 on the grounds that required facts were missing. The listed Social Security, the resulting SSI, the mortgage interest and the property tax fully determine net income of $194 and a benefit of $239.80/month." +us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as the only income and applied a $20 exclusion that SNAP does not have. It left out the $780.67/month of SSI and zeroed net income, then used a $243 maximum far below FY2026's $298. The two errors partly cancel; the correct figure is $298 − $58.20 = $239.80/month." +us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $2,800 of Social Security, leaving out SSI, and reached a near-maximum $290.70/month. With $780.67/month of SSI counted, gross income is $1,014, net income is $194, and the benefit is $239.80/month." +us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Social Security plus SSI but used only mortgage interest and property tax ($599) as shelter costs, which gave net income of $609 and $115/month. PolicyEngine adds New York's standard utility allowance, which raises the uncapped excess shelter deduction to about $611 and lowers net income to $194. The benefit is $239.80/month." +us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Social Security plus SSI but used only mortgage interest and property tax as shelter costs, arriving at about $123/month. Leaving out New York's standard utility allowance kept net income near $600 instead of $194, so it missed the $239.80 monthly benefit." +us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted $1,014 of income and the $209 standard deduction but took no shelter deduction at all. Mortgage interest and property tax are SNAP shelter costs, and the excess shelter deduction is uncapped for elderly/disabled households. With about $1,013 of shelter costs (including the utility allowance), the deduction is about $611, net income is $194, and the benefit is $239.80/month, not $56.50." +us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the household as having very low income by leaving out the $780.67/month of SSI, and paid the $298 maximum all year. Counting SSI brings net income to $194 and the benefit to $239.80/month for Jan–Sep and $248.58 for Oct–Dec." +us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added New York's $87 state SSI supplement to income ($1,101/month), but PolicyEngine counts only the $780.67 of federal SSI. Its shelter costs were only mortgage interest and property tax, without the standard utility allowance. Together these left net income at $739/month instead of $194." +us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly used Social Security plus federal SSI, the $209 standard deduction and the $298 maximum, but its shelter costs were only mortgage interest and property tax, which gave net income of $609 and $115/month. PolicyEngine adds New York's standard utility allowance, which brings net income to $194 and the benefit to $239.80/month." +us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without doing any calculation. The household is categorically eligible through SSI and nets $194/month after the standard and uncapped excess shelter deductions, which gives $239.80/month." +us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its ~$735 net income fits counting federal SSI plus New York's state supplement and treating only mortgage interest and property tax as shelter costs. It left out the standard utility allowance that brings net income to $194, and it used a $301 maximum instead of $298." +us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It reached $0 net income by counting only Social Security, leaving out the $780.67/month of SSI, and paid the FY2025 $292 maximum. With SSI counted, net income is $194 and the benefit is $239.80/month." +us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly counted $2,800 of Social Security plus $9,368 of SSI and used the $209 standard deduction, but its shelter costs were only the $7,185 of mortgage interest and property tax. That gave $609/month of net income. PolicyEngine adds New York's standard utility allowance, which cuts net income to $194 and raises the benefit to $239.80/month." +us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI. It set shelter costs to zero despite the listed mortgage interest and property tax, and used 2024 parameters ($193 standard deduction, $291 maximum). The correct figures are $1,014 gross, $194 net and $239.80/month." +us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as the only income and left out the $780.67/month of SSI, so shelter costs wiped out net income and it paid the $298 maximum all year. Counting SSI leaves $194 of net income and a $239.80 monthly benefit." +us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, leaving out the $780.67/month of SSI, which zeroed net income. It then used an inflated $309 maximum instead of FY2026's $298. Counting SSI gives $194 of net income and $239.80/month." +us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI plus New York's $87 state supplement ($1,101/month) and took no shelter deduction, claiming there was no housing-cost input. Mortgage interest and property tax are SNAP shelter costs, and with the utility allowance the uncapped excess shelter deduction is about $611. That leaves $194 of net income and a $239.80 benefit, not $30." +us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It said $233/month of income is too low to qualify. SNAP has no minimum income requirement, and lower income raises the benefit. The household is eligible through SSI and receives $298 − $58.20 = $239.80/month." +us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly built $1,014 of gross income, took the $209 standard deduction and applied an uncapped shelter deduction. But its shelter costs were only the $598.75 of mortgage interest and property tax, which gave $608.75 of net income. PolicyEngine adds New York's standard utility allowance, which raises the deduction to about $611, lowers net income to $194, and yields $239.80/month (and $248.58 for Oct–Dec)." +us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and ignored the mortgage-interest and property-tax shelter deduction. It also used stale parameters ($198 standard deduction, $291 maximum). The correct figures are $1,014 gross, $194 net and $239.80/month." +us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It required a qualifying child, but SNAP has no child requirement, and a single elderly, disabled SSI recipient is categorically eligible. The household nets $194/month and receives $239.80/month." +us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"Its reasoning reached the correct federal computation, 12 x ($994 - $213.33) = $9,368, but then dropped it for an incoherent formula (""12*(967-233.33+20)*..."") that mixed in a stray $725.33 FBR. It submitted $6,144, which none of its stated steps support." +us,scenario_118,ssi,claude-haiku-4.5,llm_error,thresholds_rates,False,"It declared the head income-ineligible because $2,800 of Social Security ""exceeds the SSI income limit."" But $2,800 a year is $233.33 a month, and after the $20 exclusion countable income is $213.33 a month, far below the $994 FBR. The correct federal benefit is $780.67 a month ($9,368 a year), not $0." +us,scenario_118,ssi,claude-opus-4.7,llm_error,state_local_rule,False,"It used the 2025 FBR of $967 instead of the 2026 FBR of $994. It then added a NY state supplement (about $145/month) to the ssi amount, although ssi is the federal payment only. The correct figure is 12 x ($994 - $213.33) = $9,368." +us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"Its reasoning computed a positive federal SSI of $8,824 (FBR $11,604 minus $2,780) and called the benefit positive, yet it submitted $0. That reasoning also used the stale $967 FBR and a single annual $20 exclusion instead of $994/month and $240/year." +us,scenario_118,ssi,claude-opus-5,llm_error,other,False,"It stated a roughly $1,000/month FBR minus $213 countable income plus a NY supplement, yet reported $667/month, an arithmetic result lower than the FBR minus income alone. The correct federal benefit is $994 - $213.33 = $780.67/month, or $9,368 a year, with no state supplement." +us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 FBR of $967/month instead of $994. It also applied the $20 general exclusion once per year ($2,800 - $20 = $2,780) instead of monthly ($240/year, leaving $2,560 countable), which gave $11,604 - $2,780 = $8,824 instead of $11,928 - $2,560 = $9,368." +us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"It overstated the FBR at $1,013/month and applied the $20 exclusion only once a year, which gave $9,372. It then cut that to $6,212 by an unexplained ""NY state supplement and rounding"" adjustment. A state supplement never reduces the federal payment, and the correct result is 12 x ($994 - $213.33) = $9,368." +us,scenario_118,ssi,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It computed countable income correctly at $213.33/month but used an FBR of $991 instead of the 2026 value of $994. That gave $777.67/month ($9,332) instead of $780.67/month ($9,368)." +us,scenario_118,ssi,deepseek-v4-pro,llm_error,state_local_rule,False,"It added an estimated $54/month NY state supplement to the maximum payment, but ssi is the federal SSI payment only. It also used an FBR of $991 instead of $994, which produced ($1,045 - $213.33) x 12 = $9,980 instead of ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed countable income correctly at $213.33/month but used an FBR of $990/month instead of the 2026 value of $994. That gave $776.67/month ($9,320) instead of $780.67/month ($9,368)." +us,scenario_118,ssi,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It added a NY state supplement of $87.30/month to a $993 FBR, but ssi is the federal SSI payment only. The correct amount is ($994 - $213.33) x 12 = $9,368, not ($1,080.30 - $213.33) x 12 = $10,404." +us,scenario_118,ssi,gemini-3-flash-preview,llm_error,state_local_rule,False,"It added an $87/month NY living-alone supplement to a $991 FBR, but ssi is the federal SSI payment only. It also understated the 2026 FBR, which is $994. The correct amount is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,period_annualization,False,"It claimed the head's Social Security exceeds the federal maximum benefit rate. That is only true if $2,800 of annual income is compared against a monthly FBR. Monthly Social Security is $233.33, so countable income of $213.33 leaves a federal payment of $780.67/month ($9,368/year)." +us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added a $1,044 NY state supplement to the federal maximum, but ssi is the federal SSI payment only. It also used the 2024 FBR ($11,316/year, $943/month) instead of the 2026 FBR of $11,928, giving $9,800 instead of $11,928 - $2,560 = $9,368." +us,scenario_118,ssi,gemini-3.5-flash,llm_error,state_local_rule,False,"It explicitly used the 2024 FBR of $943/month instead of the 2026 FBR of $994. It then added an $87/month NY state supplement, which is outside the federal ssi amount. The correct figure is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,gemini-3.5-flash-lite,llm_error,period_annualization,False,"It compared $2,800 of annual Social Security against the monthly FBR and concluded that income exceeds it. On a monthly basis, countable income is $233.33 - $20 = $213.33, well under the $994 FBR, for a federal payment of $9,368/year." +us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560, but its $9,236 answer implies an annual FBR of $11,796 ($983/month) instead of the 2026 FBR of $11,928 ($994/month)." +us,scenario_118,ssi,gemini-3.7-flash,llm_error,state_local_rule,False,"It explicitly combined the federal payment with a NY state supplement, but ssi is the federal SSI payment only. Its $10,376 plus $2,560 of countable income implies a $991 FBR plus $87 of SSP, instead of the $994 federal-only rate that gives $9,368." +us,scenario_118,ssi,gemini-3.8-flash,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560 but used the 2025 FBR of $967/month ($11,604/year) instead of the 2026 FBR of $994/month ($11,928/year). That gave $9,044 instead of $9,368." +us,scenario_118,ssi,glm-5.2,llm_error,state_local_rule,False,"It added a $94/month NY supplement for a blind individual to the federal payment, but ssi is the federal SSI payment only. It also projected the 2026 FBR at $991 instead of $994. The correct amount is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the listed facts do not establish SSI. But the head is 74, blind and disabled, has $70 in resources (under the $2,000 limit), and has only $213.33/month of countable income, so the head qualifies for a federal payment of $780.67/month ($9,368/year)." +us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It asserted that no age or disability facts were provided, although the household lists age 74, blind and disabled. It never applied the income test: $994 FBR minus $213.33 of countable Social Security is $9,368/year." +us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"It used a sub-2026 FBR (stated as $982.67/month) instead of the 2026 FBR of $994/month ($11,928/year). Its $9,332 corresponds to an $11,892 annual FBR ($991/month), which does not match its own stated rate, instead of $11,928 - $2,560 = $9,368." +us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"It had the correct $11,928 annual FBR but applied the $20 general exclusion once per year ($2,800 - $20 = $2,780). The exclusion is $20 per month, which leaves $2,560 of countable income, so the benefit is $9,368 instead of $9,148." +us,scenario_118,ssi,gpt-6-sol,llm_error,state_local_rule,False,"It correctly used the $11,928 FBR and $2,560 of countable income, but then added $1,044 of NY state supplementation. The ssi output is the federal payment alone, $9,368." +us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"It claimed there were no inputs to calculate SSI, ignoring the listed aged (74), blind and disabled status, the $70 of resources and the $2,800 of Social Security. Those facts give a federal SSI of ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,grok-4.5,llm_error,state_local_rule,False,"It added an $87/month NY living-alone SSP to a $991 FBR, but ssi is the federal SSI payment only. The 2026 FBR is also $994. The correct amount is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,grok-4.6,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560, but its $9,044 answer implies the 2025 FBR of $11,604 ($967/month) instead of the 2026 FBR of $11,928 ($994/month)." +us,scenario_118,ssi,grok-build-0.1,llm_error,state_local_rule,False,"It used the 2024 FBR of $943 as a proxy for 2026 instead of the 2026 FBR of $994. It then added an $87/month NY supplement, which is outside the federal ssi amount. The correct figure is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,ssi,inkling,llm_error,state_local_rule,False,"It correctly computed a federal payment of about $780/month from the $994 FBR, then added an $87/month NY state supplement. The ssi output excludes the state supplement and equals $780.67 x 12 = $9,368." +us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"It computed countable income correctly at $2,560 but used an estimated FBR of $991/month ($11,892/year) instead of the 2026 FBR of $994/month ($11,928/year). That gave $9,332 instead of $9,368." +us,scenario_118,ssi,kimi-k3,llm_error,state_local_rule,False,"It correctly derived $9,368 of federal SSI ($11,928 - $2,560), then added a $1,044 NY living-alone supplement. The ssi output is the federal payment only." +us,scenario_118,ssi,minimax-m3,llm_error,thresholds_rates,False,"It asserted that $233/month of Social Security exceeds a $967+/month FBR, which reverses the comparison. Countable income of $213.33/month is far below the $994 FBR, which leaves a federal payment of $780.67/month ($9,368/year)." +us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"Its reasoning computed a positive federal SSI of $8,756/year, then submitted $0, contradicting its own calculation. That calculation also used the 2024 FBR of $943 instead of the 2026 FBR of $994, which gives $9,368." +us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"It declared SSI not positive without any income or resource test, despite the listed aged, blind and disabled status, $70 of resources and only $213.33/month of countable income. The correct federal payment is ($994 - $213.33) x 12 = $9,368." +us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model said homeowners with these facts don't qualify for a circuit-breaker payment, but the NY Real Property Tax Credit covers owner-occupants who pay real property taxes. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a 74-year-old gets a flat $375." +us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model looked only at the Empire State EITC and concluded NY has no other refundable credit. It never applied the refundable Real Property Tax Credit, which gives this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum." +us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified the RPTC and the $375 senior maximum, then cut it to an unexplained $63. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375, so nothing reduces it to $63." +us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model assumed a refund from the real property tax credit requires NY tax liability. The RPTC is refundable and is paid even when NY taxable income is zero, giving $375 to this 65+ homeowner." +us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the real property tax credit as needing rent or taxable income. Real property taxes paid by a homeowner qualify directly, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375." +us,scenario_118,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula, getting $452 as 50% of the property taxes above 6% of a $12,168 household gross income. It then capped that at an income-banded $171, but a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375." +us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula to get $733 and then applied a $75 maximum it tied to having no dependents. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the flat $375 for a filer 65 or older applies." +us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model dismissed the senior circuit breaker as negligible and tied it to rent. For homeowners, the RPTC is based on real property taxes paid, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, this 65+ filer receives a flat $375." +us,scenario_118,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model reported the NY household credit, which is nonrefundable and not part of state refundable credits, and explicitly skipped the real property tax credit. The refundable RPTC for this 65+ homeowner is $375." +us,scenario_118,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model assumed zero state tax liability rules out refundable credits. It missed the refundable NY Real Property Tax Credit, which pays this 74-year-old homeowner the $375 elderly maximum." +us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used the repealed household-gross-income rule to reach $375, the right RPTC for a filer 65 or older with federal AGI of $0, but added a $75 NY household credit. That credit is nonrefundable and is not part of state refundable credits, so the total is $375, not $450." +us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model said the household doesn't qualify for any NY refundable credit. It missed the refundable Real Property Tax Credit, which pays a 65+ homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a flat $375." +us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model checked only the Empire State Child Credit and the NY EITC. It never applied the refundable Real Property Tax Credit, which needs neither earned income nor dependents and pays this 74-year-old homeowner $375." +us,scenario_118,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model's $0 leaves out the refundable NY Real Property Tax Credit. For a 74-year-old homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, the credit is a flat $375." +us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model's $0 leaves out the refundable NY Real Property Tax Credit. It applies here: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375. +us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model assumed refundable credits require earned income or dependents. The NY Real Property Tax Credit requires neither and pays this 74-year-old homeowner with $1,634 in property taxes the $375 elderly maximum." +us,scenario_118,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,The model ruled out only the Empire State Child Credit and the NY EITC and never checked the refundable Real Property Tax Credit. That credit gives this low-income 65+ homeowner $375. +us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used the repealed household-gross-income formula, getting $768 (50% of $1,634 minus 3.5% of $2,800), and compared it to a $1,000 senior maximum that does not apply here. A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375." +us,scenario_118,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model checked only the NY EITC. It missed the refundable Real Property Tax Credit, which needs no earned income and pays this 74-year-old homeowner the $375 elderly maximum." +us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model said no NY refundable credit applies. The listed facts (age 74, owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0) qualify for the refundable Real Property Tax Credit at a flat $375 for a filer 65 or older." +us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model concluded no refundable credit applies and missed the NY Real Property Tax Credit. With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375." +us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated income consisting only of Social Security as ruling out any NY refundable credit. The Real Property Tax Credit still applies to this 74-year-old homeowner with federal AGI of $0 and property taxes above 3.5% of it, and pays a flat $375." +us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model reported the $90 NY household credit, which is nonrefundable and not part of state refundable credits. It left out the refundable Real Property Tax Credit, which is $375 for this 65+ homeowner." +us,scenario_118,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model identified the RPTC but used the repealed household-gross-income rule, with an estimated $12,168 of household gross income, and capped the credit at $75 for a homeowner 65 or older. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the credit is a flat $375." +us,scenario_118,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model used an invented formula of 2.5% of real estate taxes. From 2025 the RPTC is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives $375." +us,scenario_118,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model applied an income-banded $171 cap to the elderly RPTC. A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375." +us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model found no qualifying credits and missed the refundable NY Real Property Tax Credit. It pays this 74-year-old homeowner, whose $1,634 in property taxes is above 3.5% of a federal AGI of $0, a flat $375." +us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model treated the NY EITC and the Empire State Child Credit as the only refundable credits. It left out the refundable Real Property Tax Credit, which is $375 for this 65+ homeowner." +us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model asserted that property tax credits don't apply. A 74-year-old owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0, qualifies for the refundable RPTC at a flat $375." +us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model checked the EITC, the child credit, and the tuition and childcare credits, but never the refundable Real Property Tax Credit. The household's real estate taxes and the head's age of 74 qualify it for the $375 maximum." +us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model assumed that having no NY tax liability rules out refundable credits. The Real Property Tax Credit is paid without liability and gives this 65+ homeowner $375. +us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model considered only the NY EITC and the child and dependent care credit, both of which need earnings. It missed the refundable Real Property Tax Credit, which is based on property taxes paid and gives this 74-year-old $375." +us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model found no eligibility factors, overlooking that a 74-year-old homeowner with federal AGI of $0 and $1,634 in real estate taxes, above 3.5% of that AGI, receives the refundable RPTC at a flat $375." +us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model cited the head's income and age as reasons for no credit, but they are what qualify the head for the Real Property Tax Credit: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375." +us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Tested only age under 19 and income of about 197% FPL against the 200% FAMIS limit. It ignored that Child 1 has employer-sponsored insurance, and a child covered under a group health plan is not a CHIP targeted low-income child." +us,scenario_119,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"Correctly placed income (about 197% FPL) under the 200% FAMIS limit plus the 5% disregard. It then stopped there and never applied CHIP's rule excluding children already covered by employer-sponsored insurance, which Child 1 has." +us,scenario_119,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Checked MAGI of $52,569 (about 197% FPL) against the 200% CHIP limit and confirmed the child is not Medicaid-eligible. It skipped the remaining categorical condition: Child 1 is covered by employer-sponsored insurance, which bars CHIP eligibility." +us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"Applied only the under-19 age test and the 205% FPL FAMIS income limit. It overlooked Child 1's employer-sponsored insurance, which removes the child from CHIP's targeted low-income child definition." +us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Treated age 14 plus income within the FAMIS threshold as sufficient. It never applied CHIP's requirement that the child not already be covered by a group health plan, and Child 1 has employer-sponsored insurance." +us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"Screened only age and MAGI against the 205% FPL FAMIS limit. It ignored that Child 1 is enrolled in employer-sponsored insurance, which fails CHIP's no-other-coverage categorical requirement." +us,scenario_119,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value or explanation for child1_chip_eligible, so there is no answer to score. The correct derivation yields 0 because Child 1 has employer-sponsored insurance, which excludes the child from CHIP." +us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,health_coverage,False,"Placed household income above the child Medicaid limit and within Virginia's CHIP range, then concluded eligible. It never applied the CHIP bar on children already covered by employer-sponsored insurance, which Child 1 has." +us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,Relied solely on age under 19 and an estimated income within Virginia's CHIP range. It missed that Child 1's existing employer-sponsored coverage disqualifies the child from CHIP. +us,scenario_119,child1_chip_eligible,gpt-6-luna,llm_error,health_coverage,False,"Checked only age under 19 and MAGI below Virginia's upper CHIP limit. It disregarded the listed employer-sponsored insurance for Child 1, and a group health plan enrollee is not a CHIP targeted low-income child." +us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,health_coverage,False,"Computed MAGI at about 192% FPL, within the 200% FAMIS limit, and stopped. It omitted CHIP's categorical exclusion for children covered by employer-sponsored insurance, which applies to Child 1." +us,scenario_119,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"Correctly placed income between Virginia's child Medicaid limit and the 200-205% FAMIS limit. It never tested whether Child 1 has other coverage, and the child's employer-sponsored insurance makes the child CHIP-ineligible." +us,scenario_119,child1_chip_eligible,grok-4.7,llm_error,health_coverage,False,"Applied the age test and the Medicaid-to-FAMIS income band (about 190-197% FPL against a 200-205% limit). It ignored Child 1's employer-sponsored insurance, which fails CHIP's requirement that the child not be covered under a group health plan." +us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,health_coverage,False,"Understated MAGI at $46,180 (168% FPL) and checked it only against a 143-200% FPL CHIP band. The decisive error is ignoring that Child 1 has employer-sponsored insurance, which excludes the child from CHIP at any income." +us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for child1_chip_eligible, so there is no answer to score. The correct derivation yields 0 because Child 1 has employer-sponsored insurance, which excludes the child from CHIP." +us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"Checked only age under 19 and three-person MAGI of $52,569 against Virginia's CHIP income band. It skipped the categorical rule excluding children already covered by employer-sponsored insurance, which Child 1 has." +us,scenario_119,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"Computed MAGI of $52,569 at about 204% FPL, under the 205% FAMIS limit, and concluded eligible on age and income alone. It never applied CHIP's exclusion of children covered by a group health plan, and Child 1 has employer-sponsored insurance." us,scenario_119,child1_early_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_medicaid_eligible,claude-fable-5,llm_error,health_coverage,False,"The model computed MAGI correctly ($55,800 less $3,231 of traditional 401(k)/IRA contributions ≈ 1.9x FPL) and correctly rejected Virginia's children's Medicaid MAGI limit of 143% + 5% disregard for ages 6-18, then reversed its own conclusion by asserting that PolicyEngine counts coverage up to ~205% FPL as Medicaid. Virginia's coverage above 143% FPL is FAMIS, a separate CHIP program that PolicyEngine scores under child1_chip_eligible and never under is_medicaid_eligible; the CHIP-funded Medicaid expansion for 6-18 year olds stops at the same 143%/148% line the model had already ruled out, so medicaid_category is NONE at 1.92x FPL." @@ -8487,19 +9298,23 @@ us,scenario_119,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missin us,scenario_119,child1_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,child2_chip_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It reached 197% FPL by netting the $3,087 traditional 401(k) and $144 traditional IRA contributions out of countable income and dividing the resulting $52,569 by the 2025 three-person poverty guideline. PolicyEngine counts the household's ~$55,800 of wages plus taxable interest against the 2026 three-person guideline, which lands above the 200% FAMIS ceiling it correctly named, so the right ceiling was fed an income ratio understated by both the retirement deductions and the stale guideline." -us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It cleared the child against a 205% FPL Virginia FAMIS ceiling, stacking the 5-percentage-point MAGI disregard on top of a limit that already incorporates it, and never computed the household's own FPL ratio. Virginia's FAMIS limit for a child aged 6-18 is 200% of the poverty guideline, and the trace's ~$55,800 countable income over the 2026 three-person guideline sits above it." -us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It asserted the household was 'within the CHIP (FAMIS) eligibility threshold' without ever computing income as a percentage of the federal poverty guideline, treating Virginia residence plus age under 19 as decisive. Dividing the trace's ~$55,800 of wages and taxable interest by the 2026 three-person guideline puts the 11-year-old above Virginia's 200% FAMIS ceiling." -us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a 205% FPL Virginia FAMIS ceiling — the 200% limit with the 5-point disregard added a second time — and asserted the household 'falls within' it with no ratio computed. The household's ~$55,800 countable income measured against the 2026 three-person poverty guideline exceeds the 200% limit PolicyEngine applies to children 6-18." -us,scenario_119,child2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was returned for child2_chip_eligible, so nothing substantive was computed. The correct answer of not eligible follows from the household's ~$55,800 of wages and taxable interest exceeding 200% of the 2026 three-person poverty guideline, Virginia's FAMIS ceiling for a child aged 6-18." -us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It stated only that income was 'within the applicable CHIP/FAMIS child income limit for 2026' without naming the limit or computing the household's FPL ratio, resting the answer on the under-19 age test alone. The ratio it skipped — ~$55,800 over the 2026 three-person poverty guideline — is above Virginia's 200% FAMIS ceiling." -us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It substituted an 'estimated Virginia CHIP income range' for the actual test, combining a guessed band with the under-19 age criterion instead of dividing countable income by the poverty guideline. The household's ~$55,800 of wages plus interest is above 200% of the 2026 three-person guideline, Virginia's FAMIS limit for a child aged 6-18." -us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"It computed MAGI at ~192% FPL by subtracting the $3,087 traditional 401(k) and $144 traditional IRA contributions from the $55,000 of wages before dividing by the poverty guideline. PolicyEngine's countable income is the ~$55,800 of gross wages plus taxable interest with no retirement offsets, which exceeds the 200% FAMIS ceiling it otherwise cited correctly." -us,scenario_119,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"It used a fuzzy '200-205%' FAMIS band and computed no FPL ratio at all, inferring eligibility purely from the household sitting above Virginia's 143% child Medicaid limit. Virginia's FAMIS ceiling is 200%, and the household's ~$55,800 countable income over the 2026 three-person poverty guideline is above it, so being above the Medicaid limit places the child outside CHIP rather than inside it." -us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It built a $46,180 MAGI by excluding the $6,389 employer-sponsored insurance premium along with the $3,087 401(k) and $144 IRA contributions, producing 168% FPL. PolicyEngine counts ~$55,800 — gross wages plus taxable interest, with none of those exclusions — which is above Virginia's 200% FAMIS ceiling; the ESI-premium exclusion alone shifted the ratio by more than 20 percentage points." -us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for child2_chip_eligible, so the miss is a contract failure rather than a rule error. The correct result of not eligible comes from ~$55,800 of countable wages and interest exceeding 200% of the 2026 three-person poverty guideline, Virginia's FAMIS ceiling." -us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"It adopted $52,569 — wages net of the traditional 401(k) and IRA contributions plus taxable interest — as the CHIP MAGI and declared it 'within Virginia's CHIP income band' without dividing by the three-person poverty guideline or naming the limit. PolicyEngine's countable income of ~$55,800 exceeds the 200% FAMIS ceiling for an 11-year-old in Virginia." -us,scenario_119,child2_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"It computed ~204% of the three-person poverty guideline and then cleared that against a 205% Virginia ceiling; Virginia's FAMIS limit in PolicyEngine is 200% for children 6-18, so its own ratio already disqualified the child. It arrived at 204% only by pairing an understated $52,569 MAGI (netting the 401(k) and IRA) with a stale, lower poverty guideline — PolicyEngine's ~$55,800 over the 2026 guideline is likewise above 200%." +us,scenario_119,child2_chip_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Put household income at about 197% FPL, compared it with a 200% FAMIS cap and called child2 eligible. PolicyEngine's CHIP income test, run on the roughly $55,800 household income, puts child2 above Virginia's CHIP child limit, so child2 fails CHIP on income even though the age test passes." +us,scenario_119,child2_chip_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Said income was about 197% FPL and inside the FAMIS limit. PolicyEngine's income level for child2, based on roughly $55,800 of household income, exceeds Virginia's CHIP child limit, so the CHIP income test fails and child2 is not eligible." +us,scenario_119,child2_chip_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Correctly found child2 not Medicaid-eligible, but subtracted the 401(k) and IRA contributions to get a MAGI of $52,569 (about 197% FPL) and placed that under a 200% cap. PolicyEngine's CHIP income test, run on the roughly $55,800 household income, puts child2 above Virginia's CHIP child limit, so child2 is ineligible." +us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Said household income was below a 205% FPL FAMIS threshold without showing an income figure. PolicyEngine's income level on the roughly $55,800 household income exceeds Virginia's CHIP child limit, so child2 fails the income test even though the age test passes." +us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income was within Virginia's FAMIS threshold and checked only age and state. PolicyEngine's CHIP income test on the roughly $55,800 household income puts child2 above Virginia's CHIP child limit, so the income test fails and child2 is not eligible." +us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Placed household MAGI under a 205% FPL FAMIS limit and called child2 eligible. PolicyEngine's income level, based on roughly $55,800 of household income, exceeds Virginia's CHIP child limit, so child2 fails CHIP on income." +us,scenario_119,child2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child2_chip_eligible, so there was no answer to score. The correct derivation passes the under-19 age test but fails Virginia's CHIP child income limit on the roughly $55,800 household income, which gives 0." +us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said household income was within Virginia's 2026 CHIP/FAMIS child limit but showed no ratio. PolicyEngine's CHIP income test on the roughly $55,800 household income puts child2 above that limit, so child2 is not CHIP-eligible." +us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Checked the under-19 age test and assumed income fell within Virginia's CHIP range. PolicyEngine's income level on the roughly $55,800 household income exceeds Virginia's CHIP child limit, so child2 fails the income test." +us,scenario_119,child2_chip_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"Said estimated household MAGI was below Virginia's CHIP upper income limit. PolicyEngine's CHIP income test on the roughly $55,800 household income puts child2 above that limit, so child2 is ineligible even though the age test passes." +us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"Computed household MAGI at about 192% FPL and compared it with a 200% FAMIS cap. PolicyEngine's income level for child2, based on roughly $55,800 of household income, exceeds Virginia's CHIP child limit, so child2 fails CHIP on income." +us,scenario_119,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"Correctly put income above Virginia's child Medicaid limit, but wrongly assumed it was below a 200-205% FAMIS limit. On the roughly $55,800 household income, PolicyEngine places child2 above Virginia's CHIP child limit as well, so child2 qualifies for neither program." +us,scenario_119,child2_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"Estimated MAGI at 190-197% FPL, which put child2 above Medicaid but inside a 200-205% FAMIS band. PolicyEngine's CHIP income test on the roughly $55,800 household income puts child2 above Virginia's CHIP child limit, so the FAMIS band does not apply and child2 is ineligible." +us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Cut household income to a MAGI of $46,180 (168% FPL) by subtracting the $6,389 ESI premiums on top of the 401(k) and IRA contributions, then placed it in a 143-200% CHIP band. PolicyEngine's CHIP income test runs on roughly $55,800 of household income, which puts child2 above Virginia's CHIP child limit, so child2 is ineligible." +us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child2_chip_eligible, so there was no answer to score. The correct derivation passes the under-19 age test but fails Virginia's CHIP child income limit on the roughly $55,800 household income, which gives 0." +us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"Used a MAGI of $52,569 for a three-person household, net of the 401(k) and IRA contributions, and placed it inside Virginia's CHIP band. PolicyEngine's income test on the roughly $55,800 household income puts child2 above Virginia's CHIP child limit, so child2 is not eligible." +us,scenario_119,child2_chip_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"Took a MAGI of $52,569 net of the 401(k) and IRA contributions and put it at about 204% of the poverty guideline, just under a 205% FAMIS limit. PolicyEngine's CHIP income test runs on roughly $55,800 of household income, which puts child2 above Virginia's CHIP child limit, so child2 fails on income." us,scenario_119,child2_early_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted Virginia's FAMIS/CHIP ceiling (200%, effectively 205% FPL with the 5% disregard) for the children's MAGI Medicaid limit, which Virginia sets at 143% FPL for ages 6-18. Its own arithmetic put household income above the 200% figure it cited, and it then overrode that with a ""marginal overage"" fudge and a CHIP-to-Medicaid conflation to return 1. Child2's MAGI of 1.92 x FPL exceeds 143% FPL, so no MAGI child category applies and medicaid_category is NONE." @@ -8510,65 +9325,73 @@ us,scenario_119,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missin us,scenario_119,child2_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It reproduced the reference exactly through pre-credit tax ($52,569 AGI, $24,150 HoH standard deduction, ~$3,064 tax) and then inverted the CTC ordering, writing that the refundable ACTC has first claim on the $4,400 total so that ""nonrefundable CTC used ≈ $800."" Section 24 applies the CTC nonrefundably against the entire remaining liability first and only the unused excess ($2,089.77) becomes refundable ACTC, so all $2,310.23 of residual tax is absorbed. It compounded this by using a 20% CDCC rate ($312) instead of the 2026 35% rate ($546) and omitting the $200 saver's credit." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It filed the head as single with a $14,600 standard deduction even though an unmarried taxpayer maintaining a home for two qualifying children files as head of household, which carries the $24,150 2026 standard deduction and produces $28,418.54 of taxable income rather than $37,969. It also invented a rule that employer-sponsored insurance bars the CDCC (the $546 credit turns only on work-related care expenses for a child under 13), and then submitted $2,873 after its own arithmetic produced $324.28." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It treated the 14-year-old as aged out of the CTC and assigned that child the $500 other-dependent credit; the CTC qualifying-child test is age under 17 at year end, so both children qualify at the 2026 amount of $2,200 each, $4,400 total. That $4,400 alone exceeds the $3,056.23 pre-credit tax, and it further understated credits by using a 20% CDCC rate ($312 instead of $546) and omitting the $200 saver's credit." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Its own reasoning reached the right conclusion — ""CTC easily covers remaining $2,896, leaving tax before refundable credits near $0"" — and it then discarded that and submitted $1,568 as a ""partial"" figure with no derivation behind it. Its remaining parameter errors (a $23,000 standard deduction, a 20% CDCC of $312, and no saver's credit) are immaterial, because $4,400 of nonrefundable CTC exceeds every liability figure it computed." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It asserted that only about $800 of the $4,400 CTC is nonrefundable with ""remainder refundable via ACTC,"" reversing the statutory order: the nonrefundable CTC offsets liability first — $2,310.23 here — and only the unused excess of $2,089.77 becomes the refundable ACTC. It also omitted the $546 CDCC (35% of $1,560 of care expenses) and the $200 saver's credit, which together with the CTC exactly exhaust the $3,056.23 pre-credit tax." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It computed the correct $0 answer under current-law parameters and then overrode it on the assumption that TCJA expired for 2026, rebuilding the return with an ~$8,128 standard deduction, three $5,184 personal exemptions, 15% brackets, and a $1,000-per-child CTC. OBBBA made the TCJA structure permanent: the 2026 HoH standard deduction is $24,150, personal exemptions stay repealed, and the CTC is $2,200 per child ($4,400), which with the $546 CDCC and $200 saver's credit fully offsets the $3,056.23 liability." -us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After computing $3,268 of pre-credit tax on a $22,500 HoH standard deduction, it subtracted its own claimed post-CTC ""remainder"" of ~$291 from the liability instead of subtracting the credit itself, yielding $2,977 = $3,268 − $291. Applying the credit correctly, the $4,400 of CTC exceeds the $3,056.23 pre-credit tax (and exceeds its own $3,268 figure), so tax after nonrefundable credits is $0." -us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It applied a $1,000-per-child CTC for a $2,000 total; the 2026 CTC under OBBBA is $2,200 per child, $4,400 for these two children under 17, which by itself exceeds the $3,056.23 pre-credit tax. It also used the 20% CDCC rate ($312) rather than the 2026 35% rate that yields $546, and omitted the $200 saver's credit on the $2,000 capped retirement contributions." -us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"It built the return on ""2026 pre-TCJA HOH regular tax"" of $2,807.85 and a $1,000-per-child CTC totaling $2,000; 2026 uses the extended TCJA structure ($24,150 HoH standard deduction, $3,056.23 pre-credit tax) with a $2,200-per-child CTC totaling $4,400. It identified the $200 saver's credit correctly but used a 20% CDCC ($312) instead of the 35% rate that gives $546." -us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It applied no nonrefundable credit at all — no CTC, no CDCC, no saver's credit — and reported $1,395 as the tax on the $29,813 of taxable income it derived, though HoH 10%/12% brackets give roughly $3,241 on that base. Its AGI line is also internally inconsistent (the stated expression 55,000 − 3,087 − 144 + 800 equals $52,569, not the $51,713 it used), and the $4,400 CTC alone extinguishes the correct $3,056.23 liability." -us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $46,180 by subtracting the $6,389 employer-sponsored insurance premium, which is excluded from the $55,000 wage figure already and is not an above-the-line deduction; correct AGI is $52,568.54 (wages plus $800 interest less $3,087 traditional 401(k) and $144 deductible IRA). It then layered on repealed personal exemptions and a $2,000 total CTC instead of the $4,400 available, plus a 20% CDCC ($312) rather than the 35% rate giving $546." -us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation, and $2,618 is the tax on roughly $24,650 of HoH taxable income with no nonrefundable credit applied — a residual liability that only survives if the $4,400 CTC, the $546 CDCC, and the $200 saver's credit are all ignored. The correct derivation is $3,056.23 of pre-credit tax on $28,418.54 of taxable income, fully offset by $546 + $200 + $2,310.23 of nonrefundable credits." -us,scenario_119,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so the submission never contained a substantive computation to evaluate. The contract required every requested key to appear exactly once with a numeric value and a supporting explanation." -us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Its $6,350 exceeds the household's entire pre-credit liability of $3,056.23 by more than double, so no nonrefundable credit entered the computation and no $24,150 HoH standard deduction reduced the base — the figure tracks roughly 12% of the full $52,568.54 AGI. Correctly applied, the $546 CDCC, $200 saver's credit, and $2,310.23 nonrefundable CTC absorb the tax entirely." -us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It assumed 2026 reverts to a pre-TCJA ""standard deduction/personal exemptions"" regime, producing $2,755 of pre-credit tax and a $2,000 total CTC at $1,000 per child. 2026 law gives a $24,150 HoH standard deduction with exemptions repealed, a $2,200-per-child CTC ($4,400), a 35% CDCC rate ($546), and a $200 saver's credit — nonrefundable credits that exceed the $3,056.23 liability." -us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It deducted mortgage interest that appears nowhere in the facts — only an $88,912 balance is listed, and the prompt directs that unlisted numeric inputs be treated as 0 — and applied credits far short of what is available. Leaving $2,492 of tax standing requires ignoring the $546 CDCC, the $200 saver's credit, and all but a fraction of the $4,400 CTC, any two of which drive the $3,056.23 liability to zero." -us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"It assumed TCJA expiration, using an HoH standard deduction plus three personal exemptions totaling ~$28,850 with 10%/15% brackets and a $1,000-per-child CTC of $2,000. The 2026 CTC is $2,200 per child ($4,400) and the CDCC rate at this AGI is 35% ($546, not $312); its own $2,662 of tax would be zeroed by the CTC alone, and correct parameters give $3,056.23 of tax fully offset by $546 + $200 + $2,310.23." -us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It rebuilt 2026 as pre-TCJA law with a $12,250 HoH standard deduction, three $5,300 personal exemptions, a 15% bracket, and a $1,000-per-child CTC totaling $2,000. Personal exemptions remain repealed, the 2026 HoH standard deduction is $24,150, the CTC is $2,200 per child ($4,400), and the CDCC rate at $52,569 AGI is 35% ($546) — credits that exceed both its $2,788 figure and the correct $3,056.23 liability." -us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, leaving nothing substantive to evaluate. The submission contract required this key with a numeric value and a supporting explanation." -us,scenario_119,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed the head as single with a $16,100 standard deduction; an unmarried taxpayer maintaining a home for two qualifying children files as head of household, whose $24,150 standard deduction cuts taxable income from its $46,469 to $28,418.54 and tax from $5,328 to $3,056.23. Its own correctly sized $4,400 CTC then wipes out that liability even before the $546 CDCC (it used 20%, $312) and the $200 saver's credit it omitted." -us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Declared AGI $52,569 'well above the EITC phase-out for HoH with 2 children' and set EITC to zero; the 2026 head-of-household two-child phase-out runs to about $58,600 and the base is the greater of AGI and earned income, i.e. the full $55,000 of wages, giving $7,316 − 21.06% × ($55,000 − $23,890) = $764.23. It then set the ACTC to the per-child refundable cap ($1,700 × 2 = $3,400) instead of the CTC left unused after tax, which is $4,400 − $2,310.23 = $2,089.77." -us,scenario_119,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Used the correct 2026 two-child parameters ($7,316 maximum, 21.06% rate, ~$23,850 threshold) but applied them to AGI $52,569 — income already net of the $3,087 traditional 401(k) deferral — instead of the $55,000 phase-out base, inflating the EITC to $1,268 against $764.23. It also let $2,514 of the $4,400 CTC be used nonrefundably rather than $2,310.23, shaving a further $204 off the ACTC." -us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Asserted $55,000 is above the EITC phase-out range; the 2026 HoH two-child credit phases out only at about $58,600, leaving $764.23. It then invented an income threshold that kills the refundable CTC — the CTC phase-out for HoH begins at $200,000 — when in fact the $4,400 credit exceeds the $2,310.23 of tax it can offset and the 15% × ($55,000 − $2,500) = $7,875 earnings formula is not binding, so $2,089.77 is refundable." -us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Zeroed the EITC on the claim that the two-child HoH phase-out completes near $52,900; the 2026 completion point is about $58,600 and the base is $55,000, giving $764.23. It also used the repealed $2,000-per-child CTC ($4,000 rather than 2026's $4,400) and let a $3,100 tax absorb it, versus the $2,310.23 actually remaining after the $546 CDCC and $200 saver's credit, producing a $900 ACTC instead of $2,089.77." -us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Applied a $32,300 standard deduction — the 2026 married-filing-jointly figure — to a head-of-household filer whose 2026 deduction is about $24,200, then reported a taxable income of $48,300 that its own ~$55,800 of income minus that deduction cannot produce, inflating tax to '$5,000+' against an actual ~$3,056. That fabricated liability let it absorb the whole child credit, when the 2026 $4,400 CTC leaves $2,089.77 refundable after $2,310.23 of nonrefundable use, and it separately zeroed a $764.23 EITC." -us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Zeroed the EITC as 'far above the HoH limit' though the 2026 two-child phase-out extends to about $58,600 and a $55,000 base yields $764.23. It then set the ACTC to a fabricated $1,800-per-child cap ($3,600); the 2026 refundable cap is $1,700 per child and binds on neither margin — the refundable amount is the $2,089.77 of the $4,400 CTC left after $2,310.23 offsets tax." -us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Phased the EITC from earned income of $51,913 / AGI $52,569 — wages net of the $3,087 401(k) deferral — instead of the $55,000 base, and used the repealed $2,000-per-child CTC with a $3,277 tax absorbing nearly all of it. The 2026 credit is $2,200 per child ($4,400) and only $2,310.23 of tax survives the $546 CDCC and $200 saver's credit, so the ACTC is $2,089.77, not $723, and the EITC is $764.23, not $922." -us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Claimed tax liability fully absorbs the CTC and that income exceeds the EITC phase-out for every filing status. Tax before credits is about $3,056 against a 2026 CTC of $2,200 per child ($4,400), and the $546 CDCC and $200 saver's credit hold nonrefundable CTC use to $2,310.23, leaving $2,089.77 refundable; the HoH two-child EITC extends to roughly $58,600, giving $764.23 at the $55,000 base." -us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Phased the EITC from '$46,180 AGI' — income stripped of the $6,389 employer-sponsored-insurance premium on top of the 401(k) and IRA contributions — producing $3,913 where the $55,000 statutory base yields $764.23. Its $1,597 ACTC likewise comes from the repealed $2,000-per-child credit absorbed by a $2,403 tax, versus $4,400 less $2,310.23 = $2,089.77." -us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Ran the EITC on AGI $52,569 with a $24,871 threshold and $7,264 maximum; the 2026 HoH two-child parameters are $7,316 and $23,890 applied to the $55,000 base (the greater of AGI and earned income), giving $764.23 rather than $1,431. It also set the refundable CTC to zero as 'fully used' nonrefundably, when only $2,310.23 of tax is available against the $4,400 credit, leaving $2,089.77." -us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Computed the EITC on AGI $52,569 instead of the $55,000 phase-out base, overstating it as $1,075.76 against $764.23, and zeroed the refundable CTC on the false premise that the nonrefundable CTC was fully used. The 2026 $4,400 credit exceeds the $2,310.23 of tax remaining after the $546 CDCC and $200 saver's credit, so $2,089.77 is refundable." -us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Phased the EITC from an AGI of $46,180 — income net of the $6,389 ESI premium plus the 401(k) and IRA contributions — to get $2,643, where the $55,000 base gives $764.23. Its $427 ACTC treats a $2,000-per-child credit as almost entirely absorbed; the 2026 credit is $4,400 and only $2,310.23 of tax absorbs it, leaving $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Asserted a flat $3,000 refundable child credit with no derivation, applying neither the 2026 $2,200-per-child amount nor the ACTC structure, and omitting the EITC entirely. The refundable CTC is the $2,089.77 of the $4,400 credit left after $2,310.23 offsets tax, and the $764.23 EITC brings the total to $2,854.01." -us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Its $1,933 EITC implies a phase-out base near $49,400, well below the $55,000 of earned income that actually drives the phase-out, where the credit is $764.23. Its $172 ACTC assumes a $4,000 credit nearly fully absorbed by tax, missing 2026's $2,200-per-child amount and the $546 CDCC and $200 saver's credit that cap nonrefundable CTC use at $2,310.23 and leave $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Ran the EITC on an invented 'earned income of $45,524' when earned income is the full $55,000 of gross wages, producing $1,808 instead of $764.23. It then zeroed the refundable CTC on the claim the credit is fully used nonrefundably, though the $4,400 2026 CTC exceeds the $2,310.23 of tax available, leaving $2,089.77." -us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Returned zero with no derivation. Both components are positive: the 2026 HoH two-child EITC phases out only at about $58,600, so the $55,000 base yields $764.23, and the $4,400 CTC exceeds the $2,310.23 of tax it can offset, leaving $2,089.77 of refundable ACTC." -us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Named the EITC and refundable CTC but showed no computation; the correct components are $764.23 and $2,089.77. Its $2,216.39 falls $637.62 short — the shortfall produced by using the repealed $2,000-per-child credit ($4,000 rather than 2026's $4,400) and phasing the EITC from an income base above the $55,000 of earned income." -us,scenario_119,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Gave no computation behind $1,530.82; the refundable CTC alone is $2,089.77 and the EITC adds $764.23. The figure is consistent with the repealed $2,000-per-child credit absorbed by a tax liability computed without the $546 CDCC and $200 saver's credit that hold nonrefundable CTC use to $2,310.23." -us,scenario_119,federal_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Its $497 EITC implies phasing out from about $56,270 rather than the $55,000 base, where the 2026 HoH two-child formula gives $764.23. Its $1,044 ACTC reflects the repealed $2,000-per-child credit and an overstated absorption; the 2026 $4,400 credit less $2,310.23 of nonrefundable use leaves $2,089.77." -us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"Zeroed the EITC on the claim that earned income of $51,913 — wages net of the 401(k) deferral — exceeds the 2026 HoH two-child phase-out; earned income is the full $55,000, the phase-out runs to about $58,600, and the credit is $764.23. It also used a $4,000 CTC with $2,894 of nonrefundable use, versus $4,400 and $2,310.23, understating the ACTC by $983.77." -us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"Submitted no value or explanation for federal_refundable_credits, so no substantive computation reached the grader. The required amount is the $764.23 EITC plus the $2,089.77 refundable CTC left of the $4,400 2026 credit after $2,310.23 offsets tax." -us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Treated the entire child credit as refundable at '$2,000 per child,' skipping the ACTC structure and the 2026 $2,200-per-child amount. The refundable piece is only what the $4,400 credit leaves after $2,310.23 offsets tax — $2,089.77 — and it omitted the $764.23 EITC entirely." -us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Submitted $190 with no derivation, an order of magnitude below the $2,854.01 owed by the $764.23 EITC and the $2,089.77 refundable CTC. The figure is consistent with treating the EITC as fully phased out and nearly the whole child credit as absorbed by tax, when only $2,310.23 of tax is available against the $4,400 credit." -us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Zeroed the refundable CTC as 'fully used nonrefundably'; the 2026 $4,400 credit exceeds the $2,310.23 of tax left after the $546 CDCC and $200 saver's credit, so $2,089.77 is refundable. It also computed the EITC on $52,569 rather than the $55,000 phase-out base, roughly doubling it to $1,496 against $764.23." -us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Its ~$3,000 EITC requires a phase-out base near $44,500; the base is the $55,000 of earned income, giving $764.23. Its $2,363.10 refundable CTC also overshoots the $2,089.77 that remains after $2,310.23 of the $4,400 credit is used nonrefundably." -us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Its $1,276 EITC is the 2026 formula applied to AGI $52,569 instead of the $55,000 base — the phase-out uses the greater of AGI and earned income, and the 401(k) deferral does not reduce earned income — where the credit is $764.23. Its $1,656 ACTC comes from the repealed $2,000-per-child credit; the 2026 $4,400 less $2,310.23 of nonrefundable use is $2,089.77." -us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,656 refundable child credit reflects the repealed $2,000-per-child amount; the 2026 credit is $2,200 per child, and $4,400 less the $2,310.23 absorbed by tax leaves $2,089.77. That $433.77 shortfall is not offset by its $785 EITC, which itself sits $20.77 above the $764.23 the $55,000 phase-out base produces." -us,scenario_119,federal_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Phased the EITC from '$46,180 AGI' — income stripped of the $6,389 ESI premium and the retirement contributions — producing $2,621.73 where the $55,000 base gives $764.23. It also allowed only $1,543.60 of the child credit to be used nonrefundably instead of $2,310.23, overstating the refundable CTC at $2,856.40 against $2,089.77." -us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"Declared the household above the EITC phase-out with no other refundable credits; the 2026 HoH two-child EITC runs to about $58,600, so $55,000 of earned income yields $764.23, and the $4,400 CTC exceeds the $2,310.23 of tax it can offset, leaving $2,089.77 of refundable ACTC." -us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Applied roughly correct 2026 parameters (max ~$7,330, 21.06%, threshold ~$23,950) to AGI $52,569 instead of the $55,000 phase-out base, yielding $1,300 against $764.23. It then zeroed the refundable CTC by treating a '$2,000' credit as fully absorbed; the 2026 credit is $2,200 per child and only $2,310.23 of tax absorbs it, leaving $2,089.77." -us,scenario_119,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Used a '$1,000-per-child' child tax credit; the 2026 credit is $2,200 per child ($4,400 total), of which $2,310.23 is used nonrefundably and $2,089.77 is refundable as ACTC. It also phased the EITC from AGI $52,569 with a ~$23,200 threshold rather than $55,000 with $23,890, yielding $1,115 instead of $764.23." -us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Its $4,109 EITC implies a phase-out base near $39,100; the base is the $55,000 of earned income, giving $764.23 under the 2026 HoH two-child schedule. Its $584 ACTC also understates the $2,089.77 left of the $4,400 credit after $2,310.23 of nonrefundable use." -us,scenario_119,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"Used the correct $4,400 2026 CTC but let $2,742 of it be absorbed nonrefundably rather than $2,310.23, omitting part of the $546 CDCC and the $200 saver's credit, and phased the EITC from earned income of $51,913 — wages net of the 401(k) deferral — instead of $55,000, giving $1,188 against $764.23. The two offsetting errors landed it $8.01 from the reference." -us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value or explanation for federal_refundable_credits, so no substantive computation reached the grader. The required amount is the $764.23 EITC plus the $2,089.77 refundable CTC." -us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Computed the EITC exactly right at $764.23 but overstated the tax absorbing the child credit, using $3,816.28 of the $4,400 nonrefundably instead of $2,310.23. It applied a $312 CDCC at the pre-2026 20% rate rather than $546 at the 35% rate that applies at this AGI and omitted the $200 saver's credit on the $4,000 of retirement contributions, understating the ACTC by $1,506.05." -us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Claimed $55,000 exceeds the two-child EITC limit — the 2026 HoH phase-out ends near $58,600, giving $764.23 — and that tax liability exceeds the CTC. Tax before credits is about $3,056 against a $4,400 credit, and only $2,310.23 of it is used nonrefundably, leaving $2,089.77 refundable." -us,scenario_119,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Put the two-child EITC phase-out ceiling at '~$48,000+'; the 2026 HoH ceiling is about $58,600, so $55,000 of earned income yields $764.23. It also stated the entire $4,400 CTC was absorbed by tax liability, when only $2,310.23 of tax remains after the $546 CDCC and $200 saver's credit, leaving $2,089.77 refundable." -us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"Ran the EITC on the married-filing-jointly phase-out threshold (~$49,084) for a head-of-household filer whose 2026 threshold is $23,890, and used $55,800 (wages plus interest) as the base rather than the $55,000 of earned income, producing a $6,400 credit against $764.23. Its $1,837 refundable CTC also rests on the repealed $2,000-per-child amount instead of 2026's $4,400 less $2,310.23 = $2,089.77." -us,scenario_119,federal_refundable_credits,qwen3.8-max,llm_error,asset_resource,False,"Disqualified the EITC on the head's $300,000 of stock assets; the EITC disqualified-investment-income test counts income, not asset balances, and the household's $800 of taxable interest sits far below the 2026 limit, so the credit is $764.23. Its $565 ACTC also understates the $2,089.77 of the $4,400 2026 CTC left after $2,310.23 is used nonrefundably." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It got the AGI, the $24,150 standard deduction, and roughly $3,064 of tax right. Then it set aside part of the $4,400 CTC as refundable ACTC and applied only a partial nonrefundable CTC, leaving $1,210. The nonrefundable CTC should absorb all liability left after the other credits. It also used a 20% CDCC ($312) instead of the 2026 35% rate ($546), and it left out the $200 Saver's credit." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It filed as single with a $14,600 standard deduction and single brackets. A parent with two qualifying children files as head of household with the $24,150 deduction. It also wrongly said employer coverage rules out the CDCC. Its own math gave $324.28, but it reported an unexplained $2,873, when the $4,400 CTC plus the CDCC and Saver's credit bring the $3,056 liability to $0." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It treated the 14-year-old as eligible only for the $500 other-dependent credit. Any child under 17 qualifies for the full CTC, which in 2026 is $2,200 per child, so its $2,500 cap should have been $4,400. It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's credit. With a $4,400 CTC, the leftover liability is fully offset, giving $0." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own reasoning says the CTC easily covers the $2,896 left after the CDCC, which leaves tax near $0. It then reported $1,568 'reflecting partial' use without any calculation behind it. The nonrefundable CTC is limited only by the remaining tax, so the answer is $0." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It applied only about $800 of CTC as nonrefundable and counted the rest as refundable ACTC. That reverses the ordering: the nonrefundable CTC first absorbs all remaining liability, and only the leftover credit becomes ACTC. It also left out the CDCC ($546) and the Saver's credit ($200), and its $3,244 tax on $28,069 of taxable income is too high." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It first reached $0, then dropped that answer on the assumption that TCJA expires in 2026. It rebuilt the tax with personal exemptions, 15% brackets, itemized medical deductions, and a $1,000-per-child CTC. OBBBA made the TCJA parameters permanent: head-of-household deduction of $24,150, no exemptions, and a $2,200 CTC per child. It also filed as single instead of head of household." +us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated $3,268 of tax and noted a $4,000 CTC. It then subtracted only about $291 of credit and reported $2,977, even though a CTC larger than the liability reduces tax to zero. It also left out the CDCC and the Saver's credit." +us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It set the CTC at $1,000 per child ($2,000 total), the pre-TCJA amount. The 2026 amount is $2,200 per child ($4,400). It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's credit, so $896.28 of tax remained instead of $0." +us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the pre-TCJA head-of-household rules and a $2,000 total CTC. For 2026, OBBBA keeps the $24,150 standard deduction and a $2,200-per-child CTC ($4,400). It also used a 20% CDCC ($312) instead of 35% ($546). The full $4,400 CTC would have wiped out the remaining $2,310 of liability." +us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: an $11,700 standard deduction, three $5,100 personal exemptions, 15% brackets, and a $2,000 total CTC. OBBBA keeps the $24,150 head-of-household deduction and a $2,200-per-child CTC, which absorbs all liability left after the $546 CDCC and $200 Saver's credit." +us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It miscalculated AGI as $51,713 instead of $52,569. It also reported $1,395, which is not the tax on its own $29,813 of taxable income. It never subtracted the CTC, CDCC, or Saver's credit, which together exceed the $3,056 pre-credit tax and bring it to $0." +us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used personal exemptions, as if TCJA had expired, and a $2,000 total CTC. It also understated AGI at $46,180 instead of $52,569. The 2026 CTC is $2,200 per child ($4,400) and the CDCC is 35% ($546), which fully offset the liability." +us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It describes only taxable wages and interest minus the standard deduction and pre-tax deductions, and it never subtracts any nonrefundable credit. Its $2,618 is therefore a pre-credit figure. The $4,400 CTC, $546 CDCC, and $200 Saver's credit together exceed the $3,056 tax, so the answer is $0." +us,scenario_119,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,It gave no parseable value and no explanation for federal_income_tax_before_refundable_credits. +us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"$6,350 is about twice the $3,056 pre-credit tax at head-of-household rates, and it matches the 2017 single standard deduction rather than any tax amount. This shows it did not compute head-of-household tax and did not subtract the $4,400 nonrefundable CTC, the CDCC, or the Saver's credit, which together reduce tax to $0." +us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It assumed a post-TCJA-expiration 2026 with personal exemptions and a $2,000 total CTC. OBBBA keeps the $24,150 head-of-household deduction and a $2,200-per-child CTC. It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's credit, so $443 remained instead of $0." +us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It deducted mortgage interest even though no interest amount was given, which the prompt says to treat as $0. It then applied only a small part of the CTC and left $2,492. The $4,400 nonrefundable CTC, $546 CDCC, and $200 Saver's credit exceed the $3,056 tax, so the answer is $0." +us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used 2026 rules as if TCJA had expired: head-of-household deduction plus three personal exemptions, 15% brackets, and a $2,000 total CTC. OBBBA keeps a $2,200-per-child CTC ($4,400) and a 35% CDCC ($546), and together they wipe out all liability." +us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,250 post-TCJA head-of-household deduction, three $5,300 personal exemptions, 15% brackets, and a $2,000 total CTC. OBBBA keeps the $24,150 deduction with no exemptions and a $2,200-per-child CTC. That CTC absorbs the $2,310 left after the $546 CDCC and $200 Saver's credit." +us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It gave no parseable value and no explanation for federal_income_tax_before_refundable_credits. +us,scenario_119,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It filed as single with a $16,100 standard deduction and single brackets, which gave $5,328 of tax. A parent with two qualifying children files as head of household with a $24,150 deduction, and that tax is $3,056.23. It also used a 20% CDCC instead of 35% and left out the Saver's credit. Under head-of-household status, its correct $4,400 CTC fully offsets the liability." +us,scenario_119,federal_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It said AGI of $52,569 is above the two-child EITC phase-out and set the EITC to $0. The 2026 phase-out ends near $58,629, and the EITC on $55,000 of earned income is $764.23. For the CTC, it reported the $3,400 per-child refundable cap instead of the unused CTC. Once the $546 CDCC and $200 Saver's Credit are taken first, only $2,310.23 of the $4,400 CTC offsets tax, leaving $2,089.77 refundable." +us,scenario_119,federal_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It phased out the EITC using AGI ($52,569), which gave $1,268. The phase-out base is the greater of AGI and earned income, $55,000, which gives $764.23. It also left out the $200 Saver's Credit when finding the tax the CTC absorbs: $2,514 instead of $2,310.23. That understated the refundable CTC as $1,886 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It said $55,000 is above the two-child EITC phase-out, but the 2026 phase-out runs until about $58,629, so the EITC is $764.23. It also claimed the refundable CTC has an income cutoff at this level. In fact, the additional CTC is simply the $4,400 CTC left over after $2,310.23 of tax, which is $2,089.77, limited only by the $1,700-per-child cap and 15% of earnings over $2,500." +us,scenario_119,federal_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used $2,000 per child ($4,000) instead of the 2026 amount of $2,200 ($4,400). It let about $3,100 of tax absorb the CTC without first subtracting the $546 CDCC and $200 Saver's Credit, so it got $900 instead of $2,089.77. It also put the EITC's full phase-out near $52,900, when the 2026 two-child credit is still $764.23 at $55,000 of earned income." +us,scenario_119,federal_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It estimated taxable income at about $48,300 and tax above $5,000, and concluded the CTC was fully absorbed. With the $24,150 head-of-household standard deduction, taxable income is $28,419 and tax is about $3,056. The $546 CDCC and $200 Saver's Credit cut that to $2,310.23, leaving $2,089.77 of refundable CTC. It also zeroed the EITC, which is $764.23." +us,scenario_119,federal_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It claimed no EITC, but the two-child credit is $764.23 at $55,000 of earned income. It also assumed only about $800 of CTC was used against tax and applied a $1,800-per-child cap. In fact, $2,310.23 of the $4,400 CTC is absorbed, so the refundable part is $2,089.77, which is below the $1,700-per-child ($3,400) cap for 2026." +us,scenario_119,federal_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"It computed the EITC phase-out on AGI of $52,569, which gave $1,276.20. The phase-out uses the greater of AGI and earned income ($55,000), which gives $764.23. It subtracted the CDCC but not the $200 Saver's Credit (10% of $2,000 in contributions) before the CTC, so it put the absorbed tax at $2,510.28 instead of $2,310.23 and the refundable CTC at $1,889.72 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used outdated parameters: a $2,000 CTC per child instead of $2,200, a $22,500 head-of-household standard deduction instead of $24,150, and a 10% bracket ending at $16,550. It skipped the $546 CDCC and $200 Saver's Credit before the CTC, which gave $723 of refundable CTC instead of $2,089.77. It also ran the EITC on 401(k)-reduced earnings of $51,913 with guessed parameters, getting $922 instead of $764.23 (the 2026 credit on $55,000). It then rounded the total to $1,600." +us,scenario_119,federal_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It assumed tax fully absorbs the CTC. After the $546 CDCC and $200 Saver's Credit, tax is only $2,310.23, so $2,089.77 of the $4,400 CTC is refundable. It also said the EITC was fully phased out, but the two-child credit is still $764.23 at $55,000 of earned income." +us,scenario_119,federal_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It computed the EITC on AGI of $52,569 ($1,276) instead of the greater of AGI and earned income, $55,000 ($764.23). It used a 20% CDCC ($312) instead of the 2026 minimum rate of 35% ($546) and left out the $200 Saver's Credit. As a result, too much tax absorbed the CTC, and it got $1,642 of refundable CTC instead of $2,089.77." +us,scenario_119,federal_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It computed the EITC on an AGI of $46,180, which subtracts the $6,389 employer-sponsored insurance premiums, and phased it out from the wrong starting point, getting $3,913. The 2026 two-child credit on $55,000 of earned income, phased out above $23,890, is $764.23. It also used a $4,000 CTC instead of $4,400, so its refundable CTC was $1,597 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used against tax. Only $2,310.23 of the $4,400 CTC offsets tax after the CDCC and Saver's Credit, so $2,089.77 is refundable. It also computed the EITC on AGI of $52,569 with a $7,264 maximum and a $24,871 phase-out start, instead of 2026's $7,316 and $23,890 applied to $55,000 of earned income ($764.23)." +us,scenario_119,federal_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It set the refundable CTC to $0 because it assumed the CTC was fully used. Tax after the CDCC and Saver's Credit is only $2,310.23, so $2,089.77 of the $4,400 CTC is refunded. Its EITC of $1,075.76 was phased out on AGI of $52,569 instead of $55,000 of earned income, which gives $764.23." +us,scenario_119,federal_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It treated the CTC as a single $2,000 credit that is fully used. For two children, the 2026 CTC is $4,400, and only $2,310.23 of it offsets tax, leaving $2,089.77 refundable. Its EITC used a $7,350 maximum, a $22,800 threshold, and AGI of $52,569. The correct parameters are $7,316 and $23,890, applied to $55,000 of earned income, which gives $764.23." +us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It based the EITC on an AGI of $46,180, which subtracts the $6,389 employer-sponsored insurance premiums, and got $2,643. The phase-out base is $55,000 of earned income, which gives $764.23. Its refundable CTC of $427 also let far too much tax absorb the credit: only $2,310.23 of the $4,400 CTC is used, so $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It reported a flat $3,000 as the refundable CTC and left out the EITC. The refundable CTC is $4,400 minus $2,310.23 of tax (after the $546 CDCC and $200 Saver's Credit), which is $2,089.77. The two-child EITC at $55,000 of earned income adds $764.23." +us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It overstated the EITC at about $1,933 by applying the phase-out to income well below the $55,000 earned-income base. That base gives $764.23. It understated the refundable CTC at $172 by letting almost all of the CTC offset tax. After the $546 CDCC and $200 Saver's Credit, only $2,310.23 is absorbed, leaving $2,089.77 refundable." +us,scenario_119,federal_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut earned income to $45,524 by subtracting both the $6,389 employer-sponsored insurance premiums and the $3,087 401(k) deferral, which inflated the EITC to $1,808. On $55,000 of earned income the EITC is $764.23. It also said the CTC was fully used, but $2,089.77 of the $4,400 CTC remains after $2,310.23 of tax." +us,scenario_119,federal_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave $0 with no reasoning. At $55,000 of earned income, the two-child EITC is $764.23, and tax after the CDCC and Saver's Credit ($2,310.23) absorbs only part of the $4,400 CTC. That leaves $2,089.77 of refundable CTC, so its answer drops both credits." +us,scenario_119,federal_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It gave no breakdown, and its $2,216.39 is $637.62 short of the correct total. The correct total is the $764.23 EITC (on $55,000 of earned income, phased out above $23,890) plus $2,089.77 of refundable CTC. The refundable CTC equals $4,400 minus the $2,310.23 of tax left after the $546 CDCC and $200 Saver's Credit, and the model's total understates that combination." +us,scenario_119,federal_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no breakdown, and its $1,530.82 is $1,323.19 short of the correct total. The EITC is $764.23 on $55,000 of earned income. The refundable CTC is $2,089.77 because only $2,310.23 of the $4,400 CTC offsets tax after the $546 CDCC and $200 Saver's Credit. The model let far more of the CTC offset tax than that." +us,scenario_119,federal_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"Its refundable CTC of $1,044 means it let about $3,356 of tax absorb the CTC, rather than subtracting the $546 CDCC and $200 Saver's Credit first. That leaves $2,310.23 absorbed and $2,089.77 refundable. Its EITC of $497 used pre-2026 parameters. The 2026 two-child maximum of $7,316, phased out at 21.06% above $23,890 on $55,000, gives $764.23." +us,scenario_119,federal_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It said earned income of $51,913 was above the EITC phase-out and set the EITC to $0. The 2026 two-child phase-out ends near $58,629, and the credit on $55,000 is $764.23. It also used a $4,000 CTC instead of $4,400 and let $2,894 of tax absorb it instead of $2,310.23, so it got $1,106 of refundable CTC instead of $2,089.77." +us,scenario_119,federal_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there was no answer to score." +us,scenario_119,federal_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"It treated the full $2,000 per child as refundable. In fact, the CTC first offsets the $2,310.23 of tax left after the CDCC and Saver's Credit, and only the $2,089.77 remainder of the $4,400 CTC is refundable, within the $1,700-per-child cap. It also left out the $764.23 two-child EITC." +us,scenario_119,federal_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It gave $190 with no derivation, which leaves out almost all of both credits. The correct total is the $764.23 EITC (on $55,000 of earned income) plus $2,089.77 of refundable CTC ($4,400 CTC minus $2,310.23 of tax after the CDCC and Saver's Credit)." +us,scenario_119,federal_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It said the CTC was fully used against tax. Tax after the $546 CDCC and $200 Saver's Credit is $2,310.23, so $2,089.77 of the $4,400 CTC is refundable. It also phased out the EITC on $52,569, which gave $1,496. The phase-out base is the greater of AGI and earned income ($55,000), which gives $764.23." +us,scenario_119,federal_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"It set the EITC at about $3,000, far above the $764.23 that the 2026 two-child formula gives at $55,000 of earned income. It also set the refundable CTC at $2,363.10, which assumes only $2,036.90 of tax is absorbed. Tax after the CDCC and Saver's Credit is $2,310.23, so the refundable amount is $2,089.77." +us,scenario_119,federal_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"Its $1,656 refundable CTC equals $4,400 minus tax after only a 20% CDCC ($312). The 2026 CDCC rate at this AGI is 35% ($546), and the $200 Saver's Credit also comes before the CTC, so the refundable CTC is $2,089.77. Its $1,276 EITC used AGI of $52,569 instead of $55,000 of earned income, which gives $764.23." +us,scenario_119,federal_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"Its $1,656 refundable CTC comes from subtracting only a 20% CDCC ($312) before the CTC. The 2026 rate at this AGI is 35% ($546), and the $200 Saver's Credit comes first as well. That leaves $2,310.23 of tax absorbing the CTC and $2,089.77 refundable. Its $785 EITC also drifts from the $764.23 given by the $7,316 maximum and $23,890 threshold applied to $55,000." +us,scenario_119,federal_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It subtracted the $6,389 employer-sponsored insurance premiums from AGI, which gave $46,180. That lowered tax after the CDCC and Saver's Credit to $1,543.60, raised the refundable CTC to $2,856.40, and raised the EITC to $2,621.73. With the correct AGI of $52,569 and earned income of $55,000, tax is $2,310.23, the refundable CTC is $2,089.77, and the EITC is $764.23." +us,scenario_119,federal_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It let the full $3,056 of regular tax absorb the CTC, which left $1,344. The $546 CDCC and $200 Saver's Credit are applied first, leaving $2,310.23 of tax and $2,089.77 of refundable CTC. Its $1,246 EITC was phased out on AGI instead of $55,000 of earned income, which gives $764.23." +us,scenario_119,federal_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It computed the refundable CTC as $4,400 minus the full $3,056.28 of regular tax ($1,343.72), leaving out the $546 CDCC and $200 Saver's Credit that come before the CTC. Including them gives $2,089.77. Its EITC of about $1,474 was not phased out on the $55,000 earned-income base, which gives $764.23." +us,scenario_119,federal_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"It ordered the $546 CDCC and $200 Saver's Credit correctly, but it treated the $6,389 employer-sponsored insurance premiums as reducing AGI and earned income to about $46,180. That cut tax to $1,543.60, which inflated the refundable CTC to $2,856.40 and the EITC to $2,623.83. On AGI of $52,569 and earned income of $55,000, the correct amounts are $2,089.77 and $764.23." +us,scenario_119,federal_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It said the household is above the EITC phase-out and has no other refundable credits. The two-child EITC is $764.23 at $55,000 of earned income. The CTC is $4,400, and after $2,310.23 of tax, $2,089.77 of it is refundable." +us,scenario_119,federal_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It treated the CTC as a single $2,000 credit that is fully used. For two children, the 2026 CTC is $4,400, and only $2,310.23 of it offsets tax, leaving $2,089.77 refundable. Its $1,300 EITC used AGI of $52,569 and estimated parameters instead of $55,000 of earned income with the $7,316 maximum and $23,890 threshold, which give $764.23." +us,scenario_119,federal_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used the pre-2018 CTC of $1,000 per child and treated it as fully used. The 2026 CTC is $2,200 per child ($4,400), and $2,089.77 of it is refundable beyond the $2,310.23 of tax. It also phased out the EITC on AGI of $52,569 from a $23,200 threshold, getting $1,115. The correct base is $55,000 with a $23,890 threshold, which gives $764.23." +us,scenario_119,federal_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"Its $806 of unused CTC understates the refundable CTC. After the $546 CDCC and $200 Saver's Credit, only $2,310.23 of the $4,400 CTC is absorbed, so $2,089.77 is refundable. Its $1,301 EITC was phased out on AGI of $52,569 instead of the $55,000 earned-income base, which gives $764.23." +us,scenario_119,federal_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"It swapped the sizes of the two credits. It put the EITC at $4,109, but the 2026 two-child credit at $55,000 of earned income is $764.23. It put the refundable CTC at $584, but with only $2,310.23 of the $4,400 CTC absorbed by tax, $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It phased out the EITC on 401(k)-reduced earnings of $51,913, getting $1,188. The base is $55,000, which gives $764.23. It let $2,742 of tax absorb the CTC by using too small a CDCC and leaving out the $200 Saver's Credit, so its refundable CTC was $1,658 instead of $2,089.77. The two errors partly offset each other, leaving the total $8.01 short." +us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for federal_refundable_credits, so there was no answer to score." +us,scenario_119,federal_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It got the EITC right at $764.23 but computed regular tax of $4,128.28 using the single standard deduction ($16,100) and single brackets. As head of household, the deduction is $24,150 and tax is about $3,056. It also used a 20% CDCC ($312) instead of 35% ($546) and left out the $200 Saver's Credit. Together these left only $583.72 of refundable CTC instead of $2,089.77." +us,scenario_119,federal_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It said $55,000 is above the two-child EITC limit, but the 2026 phase-out runs until about $58,629, so the EITC is $764.23. It also said tax exceeds the CTC. After the $546 CDCC and $200 Saver's Credit, tax is $2,310.23, well below the $4,400 CTC, so $2,089.77 is refundable." +us,scenario_119,federal_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It put the two-child EITC ceiling near $48,000 and zeroed the credit. The 2026 phase-out runs from $23,890 at 21.06% to about $58,629, which gives $764.23 at $55,000. It also claimed tax absorbed the entire $4,400 CTC, but tax after the CDCC and Saver's Credit is $2,310.23, leaving $2,089.77 refundable." +us,scenario_119,federal_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It filed the single parent as married filing jointly, used AGI of $55,800, and applied a made-up $49,084 phase-out start, which inflated the EITC to about $6,400. As head of household, the phase-out starts at $23,890 and applies to $55,000 of earned income, giving $764.23. It also used a $4,000 CTC instead of $4,400, so its refundable CTC was $1,837 instead of $2,089.77." +us,scenario_119,federal_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It denied the EITC because of the $300,000 in stock. The EITC has no asset test; it disqualifies only investment income above about $12,200, and this household has $800 of interest. It qualifies for $764.23. Its $565 refundable CTC also let too much tax absorb the CTC: after the CDCC and Saver's Credit, $2,089.77 of the $4,400 CTC is refundable." us,scenario_119,free_school_meals_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -8578,18 +9401,19 @@ us,scenario_119,head_medicare_eligible,glm-5.3,parse_contract_failure,missing_ou us,scenario_119,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,head_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,local_income_tax,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model reduced the FICA wage base by the $6,389 employer-sponsored insurance premium input, treating it as a Section 125 pre-tax employee salary reduction, and computed 6.2% and 1.45% on $48,611. That premium is employer-paid coverage that leaves the wage base at the full $55,000, so Social Security is $3,410 and Medicare $797.50 for $4,207.50; the model correctly excluded 401(k) contributions from the base and correctly found no Virginia employee payroll tax." -us,scenario_119,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,389 employer-sponsored insurance premium from gross wages to get a $48,611 FICA base, applying a cafeteria-plan exclusion to a premium input that does not reduce Social Security and Medicare wages. On the correct $55,000 base, 6.2% gives $3,410 and 1.45% gives $797.50, totaling $4,207.50 rather than $3,718.74." -us,scenario_119,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model applied the combined 7.65% employee FICA rate to $55,000 minus the $6,389 employer-sponsored insurance premium, treating that premium as a pre-tax salary reduction that shrinks the Social Security and Medicare wage base. The premium is employer-paid and leaves covered wages at $55,000, so the correct computation is 7.65% x $55,000 = $4,207.50." -us,scenario_119,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model declared the $6,389 employer-sponsored insurance premium to be pre-tax and set FICA taxable wages at $48,611, then applied 7.65%. That premium input does not reduce the employee's Social Security and Medicare wage base, which stays at $55,000, producing $3,410 + $797.50 = $4,207.50." -us,scenario_119,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model computed the 6.2% Social Security and 1.45% Medicare components on $48,611 after removing the $6,389 employer-sponsored insurance premium as pre-tax. The employer-paid premium leaves the FICA base at the full $55,000, so the two components are $3,410 and $797.50 for a total of $4,207.50." -us,scenario_119,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model defined FICA-covered wages as $55,000 minus the $6,389 employer-sponsored insurance premium, applying a Section 125 exclusion this premium input does not carry. With the base at the full $55,000, employee Social Security is $3,410 and Medicare is $797.50, giving $4,207.50." -us,scenario_119,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so no substantive computation was submitted. The required derivation is 6.2% x $55,000 = $3,410 plus 1.45% x $55,000 = $797.50, or $4,207.50." -us,scenario_119,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model identified the correct structure — employee Social Security and Medicare on the head's $55,000 of wages with no Additional Medicare Tax — but submitted an unworked estimate of $4,025 instead of performing the multiplication, a figure consistent with a combined rate of 7.32% rather than the statutory 7.65%. Executing its own stated method gives $3,410 + $797.50 = $4,207.50." -us,scenario_119,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model explicitly chose to treat the $6,389 employer-sponsored insurance premium as pretax and applied the 7.65% combined rate to $48,611. It got the two adjacent rules right — traditional 401(k) contributions do not reduce FICA wages and Virginia imposes no mandatory employee payroll tax — but the employer-paid ESI premium likewise leaves the base at $55,000, giving 7.65% x $55,000 = $4,207.50." -us,scenario_119,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model set FICA wages at $48,611 by netting the $6,389 employer-sponsored insurance premium out of gross pay, then took 6.2% and 1.45% of that reduced figure. The correct base is the full $55,000, yielding $3,410 of Social Security tax and $797.50 of Medicare tax for $4,207.50." -us,scenario_119,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model produced no payroll_tax value and no explanation, so nothing substantive was evaluated. The required answer is 6.2% plus 1.45% of the uncapped $55,000 wage base, or $4,207.50." -us,scenario_119,payroll_tax,minimax-m3,llm_error,other,False,"The model derived the reference exactly in its explanation — $55,000 x 6.2% = $3,410 and $55,000 x 1.45% = $797.50 — but submitted $4,205 instead of the $4,207.50 its own arithmetic produces. The failure is a mis-transcription of the summed components, not a rule, base, or rate error." +us,scenario_119,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model subtracted the $6,389 employer-sponsored insurance premium as a pre-tax Section 125 deduction and applied FICA to $48,611. Nothing in the facts says the premium comes out of salary, so FICA applies to all $55,000 of gross wages. The correct total is $3,410 + $797.50 = $4,207.50, not $3,719." +us,scenario_119,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model called the $6,389 employer-sponsored insurance premium a pre-tax deduction and applied the 7.65% FICA rate to $48,611. No salary-reduction arrangement is given, so the Social Security and Medicare base is the full $55,000. That makes the tax $4,207.50." +us,scenario_119,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced gross wages by the $6,389 employer-sponsored insurance premium before applying 7.65%. Nothing in the facts supports a Section 125 exclusion, so FICA wages stay at $55,000. The result is $4,207.50, not $3,718.74." +us,scenario_119,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed the $6,389 employer-sponsored insurance premium was paid pre-tax and took it out of FICA wages. The facts say no such thing, so the 7.65% employee FICA rate applies to all $55,000 of gross wages. That gives $4,207.50." +us,scenario_119,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used $48,611 as taxable wages by subtracting the $6,389 employer-sponsored insurance premium as a pre-tax deduction. The household facts never describe a cafeteria-plan salary reduction. Social Security (6.2%) and Medicare (1.45%) therefore apply to the full $55,000, totaling $4,207.50." +us,scenario_119,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model treated the $6,389 employer-sponsored insurance premium as a pre-tax health deduction, which cut FICA-covered wages to $48,611. No Section 125 arrangement is stated, so the FICA base is the full $55,000. Employee FICA is $4,207.50." +us,scenario_119,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is 7.65% × $55,000 = $4,207.50." +us,scenario_119,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model says FICA applies to $55,000 of wages but reports $4,025 without doing the arithmetic. At 7.65%, $4,025 corresponds to only about $52,614 of wages. The correct amount is 6.2% × $55,000 = $3,410 plus 1.45% × $55,000 = $797.50, or $4,207.50, so the answer is $182.50 too low." +us,scenario_119,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) deferrals in FICA wages but then treated the $6,389 employer-sponsored insurance premium as pretax, taxing only $48,611. No salary-reduction arrangement is stated, so FICA applies to all $55,000 of gross wages. That gives $4,207.50." +us,scenario_119,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $48,611 after subtracting the $6,389 employer-sponsored insurance premium as a pretax deduction. The facts never say the premium is withheld through a Section 125 plan, so the employee Social Security and Medicare base is the full $55,000. Tax owed is $4,207.50." +us,scenario_119,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model computed Social Security ($3,014) and Medicare ($705) on $48,611 of FICA wages, having removed the $6,389 employer-sponsored insurance premium as a pre-tax deduction. No salary reduction is stated, so both taxes apply to $55,000. That gives $3,410 + $797.50 = $4,207.50." +us,scenario_119,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score. The correct derivation is 7.65% × $55,000 = $4,207.50." +us,scenario_119,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed Social Security at $3,410 and Medicare at $797.50, which add to $4,207.50. It then reported $4,205, a figure its own arithmetic doesn't support, which is a $2.50 transcription and rounding error on the final value." us,scenario_119,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It sized the unit as a household of four and compared income to a 185% limit of ~$57,720, but the household is three people — head plus the 14- and 11-year-old children — whose 2026 185% reduced-price limit is about $50,600. Its income figure of $55,800 matches the engine exactly, and against the three-person guideline that is a school_meal_fpg_ratio of 2.04, placing the household in the PAID tier with no SNAP/TANF categorical pathway available." us,scenario_119,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It built a $46,180 income base by subtracting the $6,389 employer-sponsored insurance premium, the $3,087 traditional 401(k) contribution, and the $144 traditional IRA contribution from $55,800; the school-meal income test counts gross wages plus interest with none of those deductions. It compounded this with a stale threshold, $47,767 (185% of the 2024 three-person guideline of $25,820) rather than the 2026 figure of roughly $50,600, and the correct comparison of $55,800 against $50,600 gives a ratio of 2.04 and the PAID tier." us,scenario_119,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It asserted the household was under 185% FPL without ever computing the ratio from the stated $55,000 wages plus $800 interest, and its $47,767 three-person threshold is 185% of the 2024 poverty guideline, not the 2026 guideline whose 185% level is about $50,600. Gross school-meal income of $55,800 exceeds even the stale figure it cited, so the engine's school_meal_fpg_ratio of 2.04 puts the household in the PAID tier with no categorical eligibility." @@ -8601,96 +9425,114 @@ us,scenario_119,self_employment_tax,glm-5.3,parse_contract_failure,missing_outpu us,scenario_119,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,snap,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_119,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It actually derived the correct Virginia chain mid-reasoning — $52,569 AGI less the $8,750 head-of-household standard deduction, $2,790 of exemptions and the $1,560 child and dependent care expense deduction, giving $2,011.94 — then discarded it and submitted $2,296, a figure no step in its own reasoning produces. It also never applied Virginia's 20% nonrefundable EITC, which is $152.85 here (20% of the $764.23 phased-down federal EITC)." -us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Its taxable income of $39,469 and gross tax of $2,011.94 match the reference exactly; the entire $152.85 gap is the credit step. It declared Virginia's EITC ""taken as refundable"" and applied no nonrefundable credit, but the Virginia EITC is claimed either as 20% nonrefundable or 15% refundable, and the 20% nonrefundable branch applies here for $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used a $4,500 Virginia standard deduction — the pre-2019 single amount, not the $8,750 head-of-household figure for 2026 — and dropped the $2,790 of personal exemptions and the $1,560 child and dependent care expense deduction entirely. Its own stated $48,069 taxable income yields $2,506.47 under the brackets it cited, so the submitted $1,888 does not follow from its own derivation either." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It used an $8,500 Virginia standard deduction instead of $8,750 and omitted the $1,560 child and dependent care expense deduction, inflating taxable income to $41,279 and gross tax to $2,116 versus $39,468.54 and $2,011.94. It then shaved the result to $1,971 with unquantified ""small nonrefundable credits"" rather than the specific 20% nonrefundable Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"After computing $2,116 from an $8,500 standard deduction with no child and dependent care expense deduction, it raised the answer to $2,459 by ""rounding"" — a $343 increase with no computational basis. The correct chain is $39,468.54 of taxable income (the $8,750 standard deduction plus $2,790 exemptions plus the $1,560 child care deduction) producing $2,011.94, less the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It used an $8,500 standard deduction and omitted the $1,560 child and dependent care expense deduction to reach $2,116, then increased that to $2,280 while describing the result as ""after nonrefundable credits"" — credits reduce liability. The only nonrefundable credit here is the 20% Virginia EITC of $152.85, which takes the correct $2,011.94 gross tax down to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used a $9,000 Virginia standard deduction rather than the $8,750 head-of-household amount and never took the $1,560 child and dependent care expense deduction, which Virginia subtracts from Virginia AGI for employment-related care costs. It also concluded no nonrefundable credit applied, missing the 20% nonrefundable Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It got the $8,750 standard deduction right but dropped both the $2,790 of personal exemptions and the $1,560 child care expense deduction, reaching $43,819 of taxable income and $2,262.60 of tax. It then subtracted a nonexistent ""$220 low-income/standard nonrefundable credit"" and a 30%-of-expenses child care credit; Virginia grants the child care benefit as a $1,560 income deduction, and the only nonrefundable credit here is the 20% Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its $46,180 AGI subtracts the $6,389 of employer-sponsored insurance premiums a second time; those premiums are already excluded from the $55,000 W-2 wage figure, so Virginia AGI is $52,568.54. It then reduced tax by a ""$1,000 VA child tax credit,"" which Virginia does not have — dependents produce a $930 exemption each, and the only nonrefundable credit is the $152.85 Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It omitted the $2,790 of personal exemptions entirely (taxable income $44,069 = AGI less an $8,500 standard deduction only) and used $8,500 rather than $8,750. It then applied a ""VA CDCC = 40% of federal CDCC"" of $124.80, but Virginia has no dependent care credit: it allows the $1,560 of care expenses as a deduction from Virginia AGI, and its nonrefundable credit here is the 20% Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used an $8,500 standard deduction instead of $8,750 and never applied the $1,560 child and dependent care expense deduction, so its $41,279 taxable income overstates the correct $39,468.54 by $1,810. Its $2,116.04 is exactly gross tax on that inflated base with no credit; the answer requires subtracting the 20% nonrefundable Virginia EITC of $152.85 from the correct $2,011.94." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It set the Virginia EITC at 25% of the federal credit and sized it at $660, implying a $2,640 federal EITC; the rate is 20% and the two-child federal EITC is phased down to $764.23 at $55,800 of household income, so the credit is $152.85. It compounded this with a $34,780 taxable income (nearly $4,700 below the correct $39,468.54) and a state child care credit that Virginia grants as a $1,560 deduction instead." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated $51,713 AGI is $856 below the $52,568.54 the facts produce (wages $55,000 less $3,087 traditional 401(k) and $144 traditional IRA, plus $800 of taxable interest). Its $2,043 corresponds to gross tax on roughly $40,000 of taxable income, meaning it never took the $1,560 child and dependent care expense deduction and never subtracted the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a $46,180 Virginia AGI, double-subtracting the $6,389 of employer-sponsored insurance premiums already excluded from the $55,000 wage figure, and applied a ""$3,000 post-2025 scheduled standard deduction"" where the 2026 head-of-household standard deduction is $8,750. Its $386.60 Virginia EITC implies a $1,933 federal credit against the actual phased-down $764.23, which yields a $152.85 nonrefundable credit." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It built on a $46,180 AGI that double-counts the $6,389 employer-sponsored insurance exclusion, used an $8,500 standard deduction instead of $8,750, and took $2,400 of exemptions rather than $930 × 3 = $2,790. It also replaced the $1,560 child care expense deduction with a $62.40 ""VA Child and Dependent Care Credit"" that does not exist and skipped the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation; $1,974 is gross Virginia tax on about $38,800 of taxable income with no credit applied. The correct chain is $52,568.54 AGI less the $8,750 standard deduction, $1,560 child and dependent care expense deduction and $2,790 exemptions, giving $39,468.54 and $2,011.94 of tax, from which the 20% nonrefundable Virginia EITC of $152.85 is subtracted." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It reduced the tax with a ""nonrefundable state child care credit,"" but Virginia treats the $1,560 of employment-related child care as a deduction from Virginia AGI, not a credit. The one nonrefundable credit available is the 20% Virginia EITC of $152.85 (20% of the $764.23 federal EITC), which brings the correct $2,011.94 gross tax to $1,859.09 rather than its $1,696.28." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no derivation; $1,450.48 implies $561.46 of nonrefundable credits against the correct $2,011.94 gross tax, roughly 3.7 times the single credit that applies. Virginia's only nonrefundable credit here is the EITC at 20% of the $764.23 federal credit, or $152.85, and the $1,560 of child care is a deduction from income, not a further credit." -us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used an $8,000 standard deduction rather than $8,750 and omitted the $1,560 child and dependent care expense deduction, producing $41,779 of taxable income and $2,144.79 of tax instead of $39,468.54 and $2,011.94. It then applied a $78 ""VA CDCC"" that Virginia does not grant as a credit and a $100 Virginia EITC instead of 20% × $764.23 = $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It estimated the federal EITC at $5,669.56 — essentially the two-child maximum — for a household with $52,569 of AGI, where the credit is deep in phaseout at $764.23, so its 20% Virginia nonrefundable credit of $1,133.91 overstates the actual $152.85 by more than sevenfold. It also used an $8,500 standard deduction instead of $8,750 and omitted both the $2,790 of personal exemptions and the $1,560 child care expense deduction." -us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_income_tax_before_refundable_credits, so no substantive computation was submitted. The graded chain is $39,468.54 of Virginia taxable income producing $2,011.94 of tax, less the $152.85 nonrefundable Virginia EITC, for $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $803.75 is Virginia tax on roughly $18,450 of taxable income, less than half the $39,468.54 the facts produce after the $8,750 standard deduction, $2,790 of exemptions and the $1,560 child care expense deduction. It also asserted no state credits applied while the 20% nonrefundable Virginia EITC of $152.85 does, so it never built the Virginia base at all." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $180 is the tax on roughly $6,200 of Virginia taxable income (2% × $3,000 + 3% × $2,000 + 5% × $1,200), applying only the bottom brackets to a small fraction of income. The actual base is $39,468.54, which reaches the 5.75% bracket and produces $2,011.94 before the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It claimed to subtract the standard deduction, exemptions and child-care deduction yet reported $45,219 of taxable income, implying only $7,350 of subtractions instead of the $13,100 available ($8,750 + $2,790 + $1,560), which inflated gross tax to $2,343 against $2,011.94. It then used a $267 Virginia EITC rather than 20% of the $764.23 federal credit, or $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It named the correct three subtractions — Virginia standard deduction, personal exemptions and the qualifying dependent-care subtraction — but its $1,483.02 corresponds to about $30,265 of taxable income, roughly $9,200 below the $39,468.54 those subtractions leave from the $52,568.54 AGI. The correct chain is $2,011.94 of tax less the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its base is exactly right — the $8,750 standard deduction, three $930 exemptions and the $1,560 dependent-care deduction give $39,468.54 and $2,011.94 of tax — and the whole $152.85 error is the credit step. It tested only the Virginia low-income credit and concluded no nonrefundable credit applied, missing that Virginia's EITC can be claimed as a 20% nonrefundable credit, worth $152.85 on the $764.23 federal EITC." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $1,101.67 is the tax on $41,029 — correct AGI less the standard deduction and exemptions but without the $1,560 child care expense deduction — minus $1,000 of ""two $500 nonrefundable child credits."" Virginia has no child tax credit; dependents give a $930 exemption each, and the only nonrefundable credit here is the 20% Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used a $46,180 Virginia AGI, subtracting the $6,389 of employer-sponsored insurance premiums a second time even though they are already excluded from the $55,000 W-2 wage amount, so its $33,080 taxable income falls $6,388.54 short of $39,468.54. Its deduction stack was otherwise correct, and it also omitted the $152.85 nonrefundable Virginia EITC." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation; $1,874 is gross Virginia tax on about $37,050 of taxable income, roughly $2,400 below the $39,468.54 produced by the $8,750 standard deduction, $2,790 of exemptions and the $1,560 dependent-care deduction. It also applied no nonrefundable credit, where the 20% Virginia EITC of $152.85 reduces the correct $2,011.94 to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used an $8,500 standard deduction instead of the $8,750 head-of-household amount and never took the $1,560 child and dependent care expense deduction, leaving $41,279 of taxable income against the correct $39,468.54. Its $2,116 is gross tax on that base with no credit; the 20% nonrefundable Virginia EITC of $152.85 also applies." -us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It applied the right rule — a 20% nonrefundable Virginia EITC — but sized it at $223, implying a $1,115 federal credit; the two-child federal EITC at $52,569 of AGI is phased down to $764.23, giving $152.85. It also used an $8,500 standard deduction rather than $8,750 and omitted the $1,560 child and dependent care expense deduction." -us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its $35,330 taxable income is $4,138.54 below the $39,468.54 that $52,568.54 of AGI leaves after the $8,750 standard deduction, $2,790 of exemptions and the $1,560 dependent-care deduction, and it showed no derivation of that base. It then applied no nonrefundable credit, omitting the 20% Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"Its $41,779 taxable income uses an $8,000 standard deduction instead of $8,750 and omits the $1,560 child and dependent care expense deduction. Its 20% Virginia EITC of $238 implies a $1,190 federal credit, where the phased-down two-child federal EITC at this income is $764.23, giving a $152.85 nonrefundable credit." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so there is no computation to evaluate. The graded chain is $39,468.54 of Virginia taxable income yielding $2,011.94 of tax, less the $152.85 nonrefundable Virginia EITC, for $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It used an $8,500 standard deduction rather than $8,750 and skipped the $1,560 child and dependent care expense deduction, and it explicitly concluded that ""no nonrefundable state credit"" applied. Virginia's EITC is claimable as a 20% nonrefundable credit, worth $152.85 against the $764.23 federal EITC, which is the final step to $1,859.09." -us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a fabricated bracket schedule — 3% from $3,001 to $17,000 and a 5% top rate — where Virginia's 2026 schedule is 2% on the first $3,000, 3% to $5,000, 5% to $17,000 and 5.75% above $17,000, and paired it with a $5,200 standard deduction and $800 exemptions instead of $8,750 and $930 each. Its final $2,867 also exceeds its own computed $2,120 with no stated step producing the increase." -us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used an $8,500 standard deduction instead of the $8,750 head-of-household amount and omitted the $1,560 child and dependent care expense deduction, giving $41,279 rather than $39,468.54 of taxable income. It then stated no nonrefundable credits applied, missing the 20% nonrefundable Virginia EITC of $152.85." -us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It inflated federal AGI to $59,111 by adding employer-paid ESI premiums and an ""imputed value of ESI for children,"" though employer-provided health coverage is excluded from gross income under IRC §106, and it never subtracted the $3,087 traditional 401(k) or $144 traditional IRA contributions. Its own $41,111 taxable income yields $2,046.88 under Virginia's brackets, not the $1,173.89 it submitted." -us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its $50,200 taxable income implies only about $2,400 of subtractions instead of the $13,100 available ($8,750 standard deduction, $2,790 exemptions, $1,560 dependent-care deduction), and Virginia's brackets on $50,200 give $2,628.50, not the $2,024 base tax it stated. The ""$112 credit for taxable income under $50,250"" does not exist in Virginia; the applicable nonrefundable credit is the 20% Virginia EITC of $152.85." -us,scenario_119,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Applied Virginia's 15% refundable EITC option in 2026, but that election is authorized only for taxable years beginning before January 1, 2026 and therefore does not exist in the benchmark year. The model also missed that the refundable option is claimed in lieu of the 20% nonrefundable credit, which this household's Virginia liability on ~$55,800 fully absorbs, so the optimal Virginia EITC election produces $0 of refundable credit either way." -us,scenario_119,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used 20%, which is Virginia's nonrefundable EITC rate — the refundable alternative is 15% and lapsed after tax year 2025 — and treated a nonrefundable credit as producing a refund even though it is capped at Virginia liability. It compounded this with a federal EITC of $3,913, an amount that corresponds to income near the two-child plateau rather than $55,800 of AGI, which sits deep in the phaseout range." -us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Treated Virginia's 20% share of the federal EITC as refundable; that credit is nonrefundable and offsets only Virginia tax, landing in state_income_tax_before_refundable_credits rather than state_refundable_credits. The separate refundable Virginia EITC is 15%, is elected instead of the 20% credit, and covers only taxable years beginning before January 1, 2026, so no refundable Virginia credit exists for this 2026 filer." -us,scenario_119,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Multiplied its federal EITC estimate by Virginia's 20% rate, which is the nonrefundable credit rate; a nonrefundable credit cannot exceed Virginia liability and never appears in state_refundable_credits. The 15% refundable Virginia EITC that the model conflated with it is elected in lieu of the 20% credit and applies only through tax year 2025, leaving $0 for 2026." -us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the correct 15% refundable share but applied it in a year where the election is unavailable: Virginia's refundable EITC covers taxable years beginning on or after January 1, 2022 and before January 1, 2026. Its implied federal EITC of $2,216 also overstates the credit for a head of household with two children at ~$55,800 of AGI, which is near the top of the phaseout range." -us,scenario_119,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value was submitted for state_refundable_credits, so the key is absent from the outputs object and no substantive computation was offered. The correct derivation yields $0: Virginia's 20% EITC credit is nonrefundable and the 15% refundable election does not apply to tax year 2026." -us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied Virginia's 20% EITC rate as if it were refundable, when that credit is nonrefundable and is exhausted against Virginia income tax liability. The refundable Virginia EITC is 15%, is an alternative to the 20% credit rather than an addition to it, and is not available for taxable years beginning in 2026, so the refundable total is $0." -us,scenario_119,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Booked 20% of the federal EITC as a refundable Virginia credit, conflating the nonrefundable 20% credit with the separate 15% refundable election that expired for taxable years beginning on or after January 1, 2026. Its federal EITC input of $2,621.73 is also far above the phased-down two-child credit at ~$55,800 of AGI, compounding a rule error with a phaseout error." -us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Asserted a refundable Virginia earned income credit at 20% of a round $1,300 federal EITC; Virginia's 20% credit is nonrefundable and capped by Virginia liability, which this $55,800 head-of-household return fully covers. The refundable Virginia option is 15%, is elected instead of the 20% credit, and does not apply to tax year 2026." -us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was submitted for state_refundable_credits, leaving the required key missing from the outputs object rather than reflecting a substantive rule error. The correct derivation yields $0: Virginia's 20% EITC credit is nonrefundable and the 15% refundable election is limited to taxable years beginning before January 1, 2026." -us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Applied the 20% rate — Virginia's nonrefundable EITC share — as a refundable credit, so the amount it computed reduces state_income_tax_before_refundable_credits rather than adding to state_refundable_credits. The 15% refundable Virginia EITC it was reaching for is an in-lieu-of election limited to taxable years beginning before January 1, 2026, giving $0 here." +us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,credit_phaseout,False,"It correctly reached Virginia taxable income of $39,469 (AGI less the $8,750 standard deduction, $2,790 of exemptions and the $1,560 child and dependent care deduction) and bracket tax of $2,012. It then dropped that result for an unsupported $2,296 that matches no bracket computation. It also never subtracted Virginia's 20% nonrefundable EITC ($152.85), which reduces tax before refundable credits." +us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,credit_phaseout,False,"It computed taxable income ($39,469) and bracket tax ($2,012) correctly but treated the Virginia EITC as refundable only. Here Virginia's 20%-of-federal EITC is taken as a nonrefundable credit ($152.85 on the $764.23 federal EITC), which lowers tax before refundable credits to $1,859.09." +us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,credit_phaseout,False,"It used the obsolete $4,500 single standard deduction instead of the 2026 $8,750 amount and left out both the three $930 personal exemptions and the $1,560 child and dependent care deduction, so taxable income came out at $48,069 instead of $39,468.54. Its $1,888 does not match bracket tax on its own $48,069 ($2,506), and it applied no nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of the 2026 $8,750 and left out the $1,560 child and dependent care expense deduction, which gave $41,279 of taxable income and $2,116 of tax instead of $39,468.54 and $2,011.94. It then subtracted an unquantified ~$145 'childcare credit' (Virginia has no childcare credit, only the deduction) instead of the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care deduction, reaching $2,116, then raised that to $2,459 with no stated basis. It applied no nonrefundable Virginia EITC, which takes $152.85 off the correct $2,011.94 bracket tax." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and skipped the $1,560 child and dependent care deduction (taxable income $41,279, tax $2,116). It then added unexplained upward 'adjustments' to reach $2,280 instead of subtracting the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,credit_phaseout,False,"Its Virginia taxable income ($39,469) and bracket tax ($2,011.97) match the derivation, so the whole gap is the Virginia EITC. It set that credit at 20% of a $1,276.20 federal EITC phased out on $52,569 of AGI, while the reference's $764.23 federal EITC is phased out on $55,000 of gross wages before the $3,087 traditional 401(k) deferral. That makes the reference's Virginia credit $152.85 instead of the model's $255.24." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,credit_phaseout,False,"It used a $9,000 standard deduction (the 2026 Virginia amount is $8,750) and never applied the $1,560 child and dependent care expense deduction, which gave $40,779 of taxable income and $2,087 of tax. It also said no nonrefundable Virginia credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,credit_phaseout,False,"It subtracted only the $8,750 standard deduction and left out the three $930 exemptions and the $1,560 child and dependent care deduction, so taxable income was $43,819 instead of $39,468.54. It then subtracted an invented $220 low-income adjustment and a federal-style 30% childcare credit instead of the $152.85 nonrefundable EITC. Virginia gives childcare relief only as a deduction, and its low-income credit requires income at or below the poverty line." +us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,credit_phaseout,False,"It applied the $8,750 standard deduction and $2,790 of exemptions but left out the $1,560 child and dependent care expense deduction (taxable income $41,029 instead of $39,468.54; tax $2,102 instead of $2,011.94). It subtracted no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by also subtracting the $6,389 employer-sponsored insurance premiums, but AGI is $52,568.54 (wages plus interest, less only the $3,087 traditional 401(k) and the $144 IRA). It then used a ~$9,470 standard deduction without the exemptions or the $1,560 childcare deduction. Finally it subtracted a $1,000 'Virginia child tax credit' that does not exist instead of the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction and left out the $2,790 of personal exemptions and the $1,560 childcare deduction (taxable income $44,069 instead of $39,468.54). It then subtracted a nonexistent Virginia CDCC set at 40% of the federal credit ($124.80) instead of the 20% nonrefundable Virginia EITC of $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116.04). It applied no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,credit_phaseout,False,"It used the obsolete $4,500 standard deduction instead of $8,750 and left out the $1,560 childcare deduction. It also misapplied the brackets by stretching the 5% band to $22,000 instead of stopping it at $17,000, charging $850 instead of $600 on that band. It applied no nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,credit_phaseout,False,"It understated taxable income at about $34,780 instead of $39,468.54. It subtracted a $78 Virginia child care credit that does not exist (Virginia gives a deduction). It took the Virginia EITC as 25% of an overstated federal EITC ($660) instead of 20% of the $764.23 federal EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,credit_phaseout,False,"It started from $51,713 of AGI instead of $52,568.54. By its own explanation it subtracted only the standard deduction and exemptions, with no $1,560 child and dependent care deduction. Its $2,043 is bracket tax on about $40,010 of taxable income with no $152.85 nonrefundable Virginia EITC subtracted." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 ESI premiums. It applied a $3,000 standard deduction on the false premise that Virginia's deduction reverts after 2025, when the 2026 amount is $8,750. The lower AGI also inflated its federal EITC, so its 20% Virginia EITC came to $386.60 instead of $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,credit_phaseout,False,"It cut AGI to $46,180 by subtracting the $6,389 ESI premiums. It used an $8,500 standard deduction and outdated $800 exemptions ($2,400) instead of $8,750 and 3 × $930, and left out the $1,560 childcare deduction. It then subtracted a nonexistent Virginia CDCC ($62.40) instead of the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,credit_phaseout,False,"It gave no derivation. The correct path is $52,568.54 of AGI less $8,750, $2,790 and $1,560, giving $39,468.54 of taxable income and $2,011.94 of bracket tax, minus the $152.85 nonrefundable Virginia EITC, for $1,859.09. Its $1,974 is bracket tax on about $38,810 of taxable income with nothing subtracted for the Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,credit_phaseout,False,"It claimed a nonrefundable Virginia child care credit, but Virginia gives childcare relief only as the $1,560 child and dependent care expense deduction. The only nonrefundable credit here is the 20% Virginia EITC ($152.85), and its $1,696.28 is $162.81 below the correct $2,011.94 − $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,credit_phaseout,False,"It gave no derivation. Its $1,450.48 is bracket tax on about $29,700 of taxable income, roughly $9,800 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560). It therefore over-deducted from income and understated tax by $408.61 compared with $2,011.94 less the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (tax $2,145 instead of $2,011.94). It then subtracted a nonexistent $78 Virginia CDCC and a $100 Virginia EITC instead of the $152.85 nonrefundable EITC (20% of the $764.23 federal EITC)." +us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,credit_phaseout,False,"It estimated the federal EITC at $5,669.56 without applying the 21.06% phase-out above $23,890, so its 20% Virginia EITC came to $1,133.91 instead of $152.85 (20% of $764.23). It also left out the $2,790 of exemptions and the $1,560 childcare deduction and used $8,500 instead of $8,750, which overstated bracket tax at $2,276.47 against $2,011.94." +us,scenario_119,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,859.09 target." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,credit_phaseout,False,"Its $803.75 is Virginia bracket tax on only about $18,460 of taxable income, roughly $21,000 below the correct $39,468.54 (AGI of $52,568.54 less the $8,750 standard deduction, $2,790 of exemptions and the $1,560 childcare deduction). It applied no nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,credit_phaseout,False,"It reported $180 after 'nonrefundable credits', which means taking about $1,832 of credits off the $2,011.94 bracket tax. The only nonrefundable Virginia credit this household gets is the 20% EITC of $152.85, so it applied roughly $1,680 of nonexistent credits." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,credit_phaseout,False,"It subtracted only about $7,350 from AGI (taxable income $45,219), although the $8,750 standard deduction, $2,790 of exemptions and $1,560 childcare deduction total $13,100 (taxable income $39,468.54). That overstated bracket tax at $2,343 against $2,011.94. Its $267 Virginia EITC is also above the $152.85 that comes from the $764.23 federal EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,credit_phaseout,False,"Its $1,483.02 is bracket tax on about $30,270 of taxable income, roughly $9,200 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560), so it over-reduced income. It also listed no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,credit_phaseout,False,"It got taxable income and bracket tax ($2,012) right but said no nonrefundable low-income credit is available and applied no credit. That misses Virginia's 20% nonrefundable EITC ($152.85 on the $764.23 federal EITC), which lowers tax before refundable credits to $1,859.09." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child and dependent care deduction (taxable income $41,029, tax $2,101.67). It then subtracted two $500 nonrefundable child credits that Virginia does not offer, instead of the $152.85 nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,credit_phaseout,False,"It started from $46,180 of AGI by also subtracting the $6,389 employer-sponsored insurance premiums, but AGI is $52,568.54 (only the $3,087 traditional 401(k) and $144 IRA come out). That put taxable income at $33,080 instead of $39,468.54. It applied no nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,credit_phaseout,False,"It left out the $1,560 child and dependent care expense deduction (bracket tax $2,101.67 instead of $2,011.94). It then subtracted a ~$63 Virginia dependent-care credit that does not exist, instead of the 20% nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,credit_phaseout,False,"It stopped at the standard deduction and exemptions (taxable income $41,029), leaving out the $1,560 child and dependent care expense deduction. It subtracted no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,credit_phaseout,False,"Its $33,080 of taxable income comes from $46,180 of AGI, which means it subtracted the $6,389 ESI premiums; the correct figures are $52,568.54 of AGI and $39,468.54 of taxable income. It also misapplied the brackets, reporting $1,764.60 when its own $33,080 yields $1,644.60, and it applied no nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,credit_phaseout,False,"It listed only the standard deduction and exemptions and no Virginia EITC. Its $1,874 is bracket tax on about $37,070 of taxable income, so it over-deducted by about $2,400 against the correct $39,468.54 while also leaving out the $152.85 nonrefundable EITC; the two errors nearly cancel." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116 instead of $2,011.94). It subtracted no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (tax $2,116 instead of $2,011.94). It then took a $223 Virginia EITC instead of $152.85 (20% of the $764.23 federal EITC)." +us,scenario_119,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,credit_phaseout,False,"It applied the childcare deduction and exemptions but used an $8,500 standard deduction instead of $8,750, giving $2,026 of tax against $2,011.94. Most of its gap is the Virginia EITC, which it put at $260 from a federal EITC phased out on AGI. The reference's $764.23 federal EITC is phased out on $55,000 of gross wages, which gives $152.85." +us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,credit_phaseout,False,"Its $35,330 of taxable income is $4,139 below the correct $39,468.54 (AGI of $52,568.54 less $8,750, $2,790 and $1,560), which understated bracket tax. It subtracted no nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,credit_phaseout,False,"It used an $8,000 standard deduction instead of $8,750 and left out the $1,560 childcare deduction (taxable income $41,779, tax $2,145 instead of $2,011.94). It then subtracted a $238 Virginia EITC instead of $152.85 (20% of the $764.23 federal EITC)." +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $1,859.09 target." +us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116.04). It said no nonrefundable state credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,credit_phaseout,False,"It used a $5,200 standard deduction and invented brackets: 3% up to $17,000 and 5% above, instead of 2%/3%/5% up to $17,000 and 5.75% above. It counted four $800 exemptions for a three-person unit and then reported a net tax ($2,867) higher than its own gross tax ($2,120). It left out the $1,560 childcare deduction and the nonrefundable Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,credit_phaseout,False,"It used an $8,500 standard deduction instead of $8,750 and left out the $1,560 child and dependent care expense deduction (taxable income $41,279, tax $2,116). It said no nonrefundable credit applies, missing the 20% nonrefundable Virginia EITC ($152.85)." +us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,credit_phaseout,False,"It built a federal AGI of $59,111 by adding imputed employer health-coverage value and never subtracting the $3,087 traditional 401(k) or the $144 IRA (AGI is $52,568.54). It cited an $18,000 joint standard deduction for a single-parent filer. Its reported $1,173.89 is not bracket tax on its own $41,111 (about $2,106), and it left out the childcare deduction and the Virginia EITC." +us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,credit_phaseout,False,"It reported $50,200 of taxable income, subtracting only about $2,370 instead of $13,100 ($8,750 + $2,790 + $1,560). It then claimed a $112 credit for taxable income under $50,250 that Virginia does not have (its low-income credit requires income at or below the poverty line) instead of the $152.85 nonrefundable EITC. Its $2,024 base tax also does not match bracket tax on $50,200 ($2,629)." +us,scenario_119,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Put 15% of an assumed ~$1,268 federal EITC into refundable credits. It ignored Virginia's rule that filers take either the refundable or the nonrefundable EITC. With about $2,000 of Virginia liability absorbing the credit, the EITC is claimed as nonrefundable and refundable credits are $0." +us,scenario_119,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Treated Virginia's EITC as a refundable 15% of the ~$1,276 federal EITC. Virginia requires a choice between refundable and nonrefundable. This household's roughly $2,000 state liability fully absorbs the credit, so it goes to the nonrefundable credits and state refundable credits are $0." +us,scenario_119,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Booked 20% of a federal EITC of ~$3,913 as a refundable Virginia credit. That federal figure is far too high: with AGI of ~$52.6k, the phaseout leaves about $1.3k. The model also missed that Virginia's EITC is taken in its nonrefundable form when state liability covers it, which leaves $0 of refundable state credits." +us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied 20% to a ~$1,431 federal EITC and reported the result as refundable. It never applied Virginia's either/or election between refundable and nonrefundable EITC. The household's ~$2,000 Virginia tax fully uses the nonrefundable credit, so refundable credits are $0." +us,scenario_119,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Computed a refundable Virginia EIC of 20% × $1,075.76. Virginia's EITC goes to the nonrefundable credits when state income tax (~$2,000 here) is large enough to absorb it. The EITC therefore reduces pre-refundable state tax, and state_refundable_credits is $0." +us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Reported a refundable Virginia EITC at 15% of an implied ~$2,216 federal EITC. It missed that Virginia filers take either the refundable or the nonrefundable EITC. Because this household's Virginia liability covers the credit, the credit is nonrefundable and refundable state credits are $0." +us,scenario_119,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no parseable value or explanation for state_refundable_credits. The correct answer is $0: Virginia's EITC is claimed as the nonrefundable credit against the household's roughly $2,000 state liability." +us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Put 20% of an understated $785 federal EITC (the correct figure is about $1.3k) into refundable credits. The core error is treating Virginia's EITC as automatically refundable. Virginia's liability absorbs the nonrefundable version, so refundable state credits are $0." +us,scenario_119,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Put 20% of an overstated $2,621.73 federal EITC into refundable credits. The federal EITC at ~$52.6k AGI is about $1.3k. More fundamentally, the model ignored that Virginia's EITC is claimed in nonrefundable form when state tax covers it, which makes state_refundable_credits $0." +us,scenario_119,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Estimated a refundable Virginia EITC of 20% × ~$1,246. The federal base is close, but Virginia makes the filer choose refundable or nonrefundable. With about $2,000 of state liability fully absorbing the credit, it is nonrefundable and refundable credits are $0." +us,scenario_119,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Reported 15% of a $1,474 federal EITC as a refundable Virginia credit. It missed the rule that Virginia's EITC is taken as a nonrefundable credit when the state liability (~$2,000 here) absorbs it, which leaves $0 in state refundable credits." +us,scenario_119,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Applied 20% to an overstated $2,623.83 federal EITC and called it refundable. It ignored Virginia's election between refundable and nonrefundable EITC. Here the credit is claimed against the roughly $2,000 state tax as nonrefundable, so refundable credits are $0." +us,scenario_119,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Reported 20% of a $1,300 federal EITC as a refundable Virginia credit. Virginia's EITC is claimed as the nonrefundable version when state liability covers it, and this household's ~$2,000 Virginia tax does. The credit therefore reduces pre-refundable state tax, and state_refundable_credits is $0." +us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for state_refundable_credits. The correct answer is $0 because Virginia's EITC is taken as a nonrefundable credit against the household's state liability. +us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Put 20% of an understated $764.23 federal EITC into refundable credits. The federal EITC is about $1.3k at this AGI. The decisive error is treating Virginia's EITC as refundable: the household's ~$2,000 Virginia tax absorbs the nonrefundable form, so refundable state credits are $0." us,scenario_119,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It applied the expired $10,000 SALT cap, which discarded roughly $3,900 of the ~$11,896 Connecticut income tax that the 2026 $40,400 cap lets through, deducted the full $3,744 of mortgage interest instead of prorating it to the $750,000 acquisition-debt limit ($2,808), and skipped the 0.5%-of-AGI charitable floor. It then reported $30,238 of bracket tax on its own $200,500 taxable income, $10,480 below the $40,718 the 2026 single schedule (10/12/22/24 with the 24% bracket running to $201,775) produces, and omitted the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $1,750 state and local tax refund to AGI, which PolicyEngine excludes, and dropped the $29 traditional IRA above-the-line deduction, inflating AGI to $234,414 versus $232,635 and taxable income to $198,819 versus $196,918. It also computed the 3.8% NIIT on rental income net of the $2,447 farm-rent loss (~$70) instead of on the full $4,300 of taxable interest plus rental income ($163.40); the farm loss is an above-the-line deduction and does not reduce net investment income." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It used a $30,000 standard deduction (the 2026 single filer aged 76 gets $16,100 + $2,050 = $18,150) and abandoned the $35,717 of itemized deductions that actually control, then applied head-of-household rates to a single filer. Fatally, it subtracted $11,297 of ""estimated tax paid/withholding"" from the computed tax; withholding is a payment, not a nonrefundable credit, and the requested output is liability before refundable credits." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It capped SALT at the expired $10,000 TCJA limit instead of the 2026 $40,400 cap, losing about $3,900 of deductible Connecticut income tax, deducted mortgage interest in full rather than prorating $3,744 by $750,000/$1,000,000 to $2,808, and ignored the 0.5%-of-AGI charitable floor that cuts the $20,164 gift to $19,001. It then computed $40,823 from its own brackets but submitted $38,740, and never added the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It held SALT to the expired $10,000 cap rather than the 2026 $40,400 cap that admits the full $13,908 of Connecticut income tax plus property tax, added the $1,750 state tax refund to AGI, and omitted the 0.5%-of-AGI charitable floor, producing $202,030 of taxable income against the correct $196,918. It also invented bracket edges (22% ending at $107,400, 24% at $205,050) instead of the 2026 thresholds of $105,700 and $201,775, and left out the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It built its SALT deduction from the $2,012 real estate tax alone and never included the roughly $11,896 of Connecticut income tax, which the 2026 $40,400 cap fully admits, and it took the full $3,744 of mortgage interest instead of the $2,808 that survives the $750,000 acquisition-debt proration and skipped the 0.5% charitable floor. Its $25,920 of itemized deductions is $9,800 short of the correct $35,717, and it also omitted the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunset for 2026 and rebuilt the return on pre-TCJA law: 10/15/25/28/33 rate brackets, a $5,050 personal exemption, an uncapped SALT deduction, and $2,966 of unreimbursed employee business expenses as a miscellaneous itemized deduction subject to a 2% floor. All four are wrong for 2026, where TCJA rates are permanent, personal exemptions and miscellaneous itemized deductions remain repealed, and SALT is capped at $40,400. Its submitted $42,046 also matches none of its own computed totals ($47,383 and $47,592)." -us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct $232,635 AGI, then omitted the roughly $11,896 of Connecticut income tax from SALT and skipped the 0.5%-of-AGI charitable floor, arriving at $207,651 of taxable income. It then never applied the rate schedule at all, reporting an ""approximate"" $24,500 where the 2026 single brackets on $207,651 yield about $43,200; the submitted figure is an unanchored guess, roughly 12% of taxable income." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted $7,429 of employer-sponsored insurance premiums from the $165,597 of wages, but that amount is already excluded from the reported W-2 wage figure and is not an above-the-line deduction, so AGI fell to $226,985 instead of $232,635. It also added the $1,750 refund, skipped the 0.5% charitable floor, and its $34,322 is about $4,400 below the $38,726 its own $192,198 taxable income yields under the 2026 brackets; the $163.40 NIIT is missing as well." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed 2026 reverts to pre-TCJA law, claiming a $5,063 personal exemption and $3,114 of miscellaneous itemized deductions after a 2% floor; both remain repealed in 2026, and its pre-TCJA rate schedule is wrong because the 10/12/22/24 brackets are permanent. It also deducted $7,429 of employer-sponsored insurance premiums from wages, which are already excluded from the W-2 amount, and computed NIIT on $1,853 of rental income net of the farm loss rather than on the full $4,300." -us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It claimed a $5,250 personal exemption, which has been repealed since 2018 and stays repealed in 2026, and $42,059 of itemized deductions against the correct $35,717 — an excess consistent with deducting the $7,654 of unreimbursed employee business expenses that are also repealed and taking mortgage interest without the $750,000 debt proration. It then priced the result on pre-TCJA rates, and computed NIIT as $70.41 on farm-loss-netted rental income rather than $163.40 on the full $4,300." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It explicitly assumed TCJA sunset, treating the SALT cap as expired and restoring $2,966 of unreimbursed employee business expenses as a 2%-floor miscellaneous itemized deduction; in 2026 SALT is capped at $40,400 and miscellaneous itemized deductions remain repealed. Its $45,093 of tax on $192,187 of taxable income is a 23.5% effective rate, only reachable with pre-TCJA 28%/33% rates rather than the permanent 24% top bracket that applies here." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its $39,591 is $430 short of the reference. The 2026 brackets put $39,591 of regular tax at about $195,804 of taxable income, roughly $1,114 below the correct $196,918, and the answer contains none of the $163.40 net investment income tax owed on the $4,300 of interest and rental income; it over-deducted by about $1,100 and skipped NIIT entirely." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It deducted the $7,429 of employer-sponsored insurance premiums from wages, cutting AGI to about $226,956 instead of $232,635; those premiums are already excluded from the $165,597 W-2 figure and are not a separate adjustment. It also included ""miscellaneous expenses"" among itemized deductions, which are repealed for 2026, and computed NIIT as about $87 rather than 3.8% of the full $4,300 ($163.40)." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took a $5,000 personal exemption and $3,149 of unreimbursed employee expenses after a 2% floor — both repealed for 2026 — and subtracted $7,429 of employer-sponsored insurance premiums from wages, dropping AGI to $225,235 against the correct $232,635. Its $42,068 of tax on only $180,066 of taxable income is a 23.4% effective rate, which requires pre-TCJA 28%/33% brackets instead of the permanent 24% top rate applicable at this income." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It supplied no derivation, only a restatement of the question. Inverting its $33,814 through the 2026 single brackets implies about $171,700 of taxable income, roughly $25,200 below the correct $196,918 — consistent with subtracting the $7,429 of already-excluded employer insurance premiums and the $7,654 of repealed unreimbursed employee expenses on top of a full charitable deduction with no 0.5% floor — and it includes no NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $7,429 of employer-sponsored insurance premiums from wages, which are already excluded from the $165,597 W-2 amount, and capped SALT at the expired $10,000 limit rather than 2026's $40,400, so its $32,972 of itemized deductions omits roughly $3,900 of deductible Connecticut income tax. The resulting $194,013 taxable income is $2,905 below the correct figure, and it never added the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $37,824 corresponds under the 2026 single brackets to about $188,442 of taxable income, some $8,476 below the correct $196,918 — the size of the $7,654 of repealed unreimbursed employee business expenses plus the missing 0.5%-of-AGI charitable floor — and it carries no net investment income tax on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $33,023.76 implies about $168,441 of taxable income under the 2026 single brackets, roughly $28,500 below the correct $196,918 — consistent with stripping the $7,429 of already-excluded employer insurance premiums and the $7,654 of repealed unreimbursed employee expenses and further understating taxable Social Security or pension income — and it omits the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Its $24,984 of itemized deductions counts only the $2,012 of real estate tax as SALT, omitting the roughly $11,896 of Connecticut income tax that the 2026 $40,400 cap fully admits, and takes the $20,164 charitable gift without the 0.5%-of-AGI floor that reduces it to $19,001. That pushed taxable income to $209,430, into the 32% bracket that the correct $196,918 never reaches, and it added no net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It deducted $7,488 of mortgage interest by adding the tax-unit first and second mortgage interest ($2,976 + $768) to the $3,744 person-level total, double-counting the same interest, when the deductible amount is $2,808 after prorating by the $750,000 acquisition-debt limit against the $1,000,000 of combined balances. It also held SALT to the expired $10,000 cap instead of $40,400, skipped the 0.5% charitable floor, and omitted the $163.40 NIIT; the offsetting errors left $195,012 of taxable income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that ""itemized deductions are limited by the standard deduction,"" inverting the rule: the taxpayer takes the larger of the two, and here the $35,717 of itemized deductions exceeds the $18,150 standard deduction. Its $38,141 corresponds to about $190,431 of taxable income, $6,487 under the correct $196,918, and it includes none of the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It offered no derivation, only a list of considerations, and referred to ""nonrefundable credit-like items"" that do not exist here — no nonrefundable credit applies to this single filer with no dependents. Its $35,233 implies roughly $177,700 of taxable income under the 2026 brackets, about $19,200 below the correct $196,918, and it omits the $163.40 NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It stated it was applying ""estimated 2026 post-TCJA-sunset rules,"" claiming a $5,300 personal exemption and $40,615 of itemized deductions; personal exemptions stay repealed in 2026, miscellaneous itemized deductions remain repealed, and the 10/12/22/24 brackets are permanent rather than reverting to 15/25/28/33. Pricing $188,470 of taxable income at pre-TCJA rates produced $43,335 where the correct $196,918 at 2026 rates yields $39,858 plus $163.40 of NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the 2026 charitable floor and the correct bracket schedule — $210,286.16 of taxable income does yield exactly its $43,747.57 — but overstated taxable income by $13,368, an amount matching the roughly $11,896 of Connecticut income tax it left out of the SALT deduction plus the $1,750 state tax refund it wrongly included in AGI. It also stopped at the regular tax and never added the $163.40 net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It added the $1,750 state and local tax refund to AGI, which PolicyEngine excludes, giving $234,414 instead of $232,635, and computed the net investment income tax as about $70 on rental income net of the $2,447 farm-rent loss rather than $163.40 on the full $4,300 of taxable interest plus rental income. The farm-rent loss is an above-the-line deduction and does not reduce net investment income." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It invoked ""the restored personal exemption"" and priced $203,194 of taxable income at pre-TCJA rates; personal exemptions remain repealed for 2026 and the top rate reached at this income is 24%, not 28% or 33%. It also failed to prorate the $3,744 of mortgage interest to the $750,000 acquisition-debt limit and omitted the 0.5%-of-AGI charitable floor, and the resulting $48,134 overstates the correct $40,021.82 by 20%." -us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted $7,429 of ""pretax employer premiums"" from wages, cutting AGI to $226,985 instead of $232,635; employer-sponsored insurance premiums are already excluded from the reported $165,597 of W-2 wages and are not a further adjustment. Its remaining treatment (charitable floor, mortgage debt limit, no senior bonus deduction) was right, so the $5,650 AGI error plus the omitted $163.40 NIIT accounts for nearly all of the $1,394 shortfall." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave a one-sentence assertion with no computation. Its $39,250 corresponds to about $194,035 of taxable income under the 2026 single brackets, $2,883 below the correct $196,918 — the scale of the omitted 0.5%-of-AGI charitable floor plus an unprorated mortgage interest deduction — and it contains no net investment income tax on the $4,300 of interest and rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied ""post-TCJA-sunset 2026 rules"": a $5,427 personal exemption and $2,966 of unreimbursed employee business expenses after a 2% floor, both repealed for 2026, priced at ""inflation-adjusted pre-TCJA brackets/rates"" instead of the permanent 10/12/22/24 schedule. Taxing $189,528 at pre-TCJA 28%/33% rates yielded $43,663 where $196,918 at 2026 rates gives $39,858 plus $163.40 of NIIT." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It explicitly used ""restored 10/15/25/28% brackets"" and a $5,437 personal exemption; the 2026 schedule is the permanent 10/12/22/24 set with the 24% bracket running to $201,775, and personal exemptions remain repealed. Its $38,146 of itemized deductions also exceeds the correct $35,717 by including repealed miscellaneous itemized deductions, and it added no net investment income tax." -us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied ""2026 federal brackets post-TCJA expiration"" and included $2,966 of unreimbursed employee business expenses after a 2% floor in its $40,043 of itemized deductions; miscellaneous itemized deductions are repealed for 2026 and the pre-TCJA rate schedule does not apply. Its $194,371 of taxable income taxed under 2026 rates would produce about $39,247, so the $45,511 answer comes almost entirely from the wrong rate schedule, and NIIT is missing." -us,scenario_120,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly prorated mortgage interest to $2,808 and computed the $163.40 NIIT, but deducted $2,966 of unreimbursed employee business expenses as a miscellaneous itemized deduction, which is repealed for 2026, took the $20,164 charitable gift without the 0.5%-of-AGI floor that trims it to $19,001, and added the $1,750 state tax refund to AGI. Net of those, its taxable income of $194,235 sits $2,683 below the correct $196,918." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value was returned for federal_income_tax_before_refundable_credits and no explanation accompanied the submission, so the required key never reached the outputs object. This is a contract failure in producing the answer, not a substantive tax computation error." -us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value was returned for federal_income_tax_before_refundable_credits and no explanation accompanied the submission, so the required key never reached the outputs object. This is a contract failure in producing the answer, not a substantive tax computation error." -us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with the bare text ""value = 0"" and no derivation, asserting no federal income tax liability for a single filer with $232,635 of AGI and $196,918 of taxable income after itemizing $35,717. The 2026 single brackets alone produce $39,858.42 on that base before the $163.40 net investment income tax, and no nonrefundable credit is available to this childless filer to offset any of it." -us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It got the 2026 SALT cap, the charitable 0.5% floor, and the bracket schedule right, but deducted the full $3,744 of mortgage interest instead of prorating it by $750,000/$1,000,000 of combined acquisition debt to $2,808, and added the $1,750 state tax refund to AGI. Those left taxable income $767 high, and it stopped at regular tax without the $163.40 net investment income tax on the $4,300 of interest plus rental income." -us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took an $18,525 standard deduction and discarded itemizing entirely, throwing away the $19,001 charitable deduction, $13,908 of SALT, and $2,808 of mortgage interest that total $35,717 and control here. It also included the $1,750 refund, omitted the $617 pre-tax 401(k) reduction to wages, priced the result on 2025 rather than 2026 bracket edges, and added no NIIT, leaving taxable income $17,588 too high." -us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It filed this single, unmarried 76-year-old as married filing separately, using a $16,700 MFS standard deduction and ""2026 separate-filer rate brackets"" instead of the single schedule and $18,150 standard deduction, and then stacked the $20,164 charitable gift on top of the standard deduction rather than choosing the larger of standard or itemized. It further invented a $750 ""retirement income credit"" that does not exist, and its own arithmetic ($45,287 − $750 = $44,537) contradicts the $48,557 it submitted." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"At the 2026 single brackets, its own ~$200,500 of taxable income produces about $40,700 of tax, so reporting $30,238 misapplies the rate schedule by roughly $10,500. It also capped SALT at the old $10,000 instead of deducting the ~$13,908 allowed under the OBBBA's $40,400 cap. It deducted the full $3,744 of mortgage interest instead of the $2,808 allowed on $750k of $1M debt, and it omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It added the $1,750 state tax refund to AGI ($234,414 instead of $232,635), which raised taxable income by about $1,900 and regular tax by about $457. It also computed NIIT on $1,853 after netting the farm rent loss, instead of on the $4,300 of interest and rental income ($163.40). That offsets only $93 of the overstatement." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It took a $30,000 standard deduction instead of itemizing $35,717: $19,001 of charitable gifts after the 0.5% floor, ~$13,908 of SALT including Connecticut income tax, and $2,808 of limited mortgage interest. It counted all $41,900 of Social Security instead of the $35,615 taxable portion. It then subtracted a fabricated $11,297 of withholding, which is a tax payment, not a nonrefundable credit." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own bracket computation gave $40,823 on $200,477 of taxable income, but it reported $38,740 with no credit to justify the $2,083 cut. Its taxable income used the old $10,000 SALT cap instead of the ~$13,908 allowed under the OBBBA's $40,400 cap. It also deducted the full $3,744 of mortgage interest instead of $2,808 under the $750k debt limit, skipped the 0.5%-of-AGI charitable floor, and omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap, skipped the 2026 0.5%-of-AGI charitable floor, never subtracted the $617 401(k) deferral from wages, and added the $1,750 refund. Together these put taxable income at $202,030 instead of $196,918. It also ended the 22% bracket at $107,400 instead of $105,700 and omitted the $163.40 NIIT on $4,300 of interest and rental income." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left Connecticut income tax (~$11,896) out of SALT, deducted the full $3,744 of mortgage interest instead of $2,808 under the $750k/$1M acquisition-debt ratio, and skipped the 0.5% charitable floor, reaching $208,465 of taxable income. At 2026 brackets that income yields about $43,165, so its $40,650 also misapplies the rate schedule. It omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 state tax refund in AGI ($234,385 instead of $232,635), which pushed taxable income to $198,906 and regular tax up by about $477. It also computed NIIT at $70 on $1,853 netted for the farm rent loss, instead of 3.8% of the $4,300 of interest and rental income ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025 and applied pre-TCJA 10/15/25/28/33% brackets, a personal exemption, no SALT cap, and a 2%-floor miscellaneous deduction. The OBBBA made the TCJA rules permanent: the 2026 single schedule is 10/12/22/24%, with no personal exemption and no miscellaneous itemized deductions. It also left Connecticut income tax out of SALT, and its reported $42,046 matches none of its own totals ($49,719, $48,053, $47,592, $47,383)." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached $207,651 of taxable income but reported ~$24,500 of tax, while 2026 single brackets give about $42,900 on that amount, a gross misapplication of the rate schedule. It also left the ~$11,896 of Connecticut income tax out of SALT, although it is deductible under the OBBBA's $40,400 cap. It skipped the 0.5% charitable floor and omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It never subtracted the $617 traditional 401(k) deferral and added the $1,750 state refund, so AGI was $235,031 instead of $232,635 and regular tax was about $656 too high. It then computed NIIT on $1,853 after netting the farm rent loss, instead of on the $4,300 of interest and rental income ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 employer-sponsored insurance premium from wages, but only the $617 traditional 401(k) deferral reduces the stated gross wages. It also understated SALT at $11,815 and skipped the 0.5% charitable floor. Its own $192,198 of taxable income yields about $38,725 at 2026 brackets, so its $34,322 also misapplies the rates, and it omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It treated 2026 as a TCJA-sunset year, with a $5,063 personal exemption, a $3,114 2%-floor miscellaneous deduction, and pre-TCJA brackets. The OBBBA made the TCJA brackets permanent and eliminated personal exemptions and miscellaneous itemized deductions. It also subtracted the $7,429 ESI premium from wages and computed NIIT on $1,853 instead of $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,250 personal exemption, pre-TCJA brackets, and $42,059 of itemized deductions against the correct $35,717. The OBBBA permanently kept the 10/12/22/24% schedule with no personal exemption. It also computed NIIT at $70.41 on investment income netted for the farm rent loss, instead of $163.40 on $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It omitted the $163.40 NIIT on $4,300 of interest and rental income, even though MAGI exceeds the $200,000 threshold. It also added the $1,750 state refund to AGI and deducted charitable gifts without the 2026 0.5%-of-AGI floor, leaving taxable income $1,639 too high at $198,557." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset: no SALT cap, $2,966 of 2%-floor miscellaneous deductions for unreimbursed employee expenses, and pre-TCJA rates. The OBBBA made the TCJA brackets permanent and permanently eliminated miscellaneous itemized deductions. Its $192,187 of taxable income yields about $38,720 at the actual 2026 schedule, so the pre-TCJA rates drove the overstatement; it also omitted NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $39,591 corresponds to about $195,800 of taxable income with no NIIT, which means it overstated deductions by roughly $1,100 against the correct $35,717 itemized total. It also omitted the $163.40 NIIT on $4,300 of interest and rental income." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It reduced wages by the $7,429 ESI premium, when only the $617 401(k) deferral reduces wages. It itemized 2%-floor miscellaneous employee business expenses, which the OBBBA permanently eliminated, and applied sunset-era rates that produced $41,404 of regular tax on an AGI below the correct $232,635. It computed NIIT at $87 instead of 3.8% of $4,300 ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,000 personal exemption, $3,149 of 2%-floor miscellaneous deductions, and pre-TCJA brackets. The OBBBA made the TCJA schedule permanent and removed those deductions. It also reduced wages by the $7,429 ESI premium and computed NIIT on $1,853 instead of $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. At 2026 single brackets, $33,814 implies about $171,700 of taxable income, roughly $25,200 below the correct $196,918, so the model understated income or overstated deductions by that amount. It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, when only the $617 401(k) deferral reduces gross wages. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap and skipped the 0.5% charitable floor, cutting taxable income to $194,013. It omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $37,824 corresponds to about $188,400 of taxable income, roughly $8,500 below the correct $196,918, which matches subtracting the $7,429 ESI premium from wages as its sibling models did. It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 single brackets, $33,024 implies about $168,400 of taxable income, roughly $28,500 below the correct $196,918, so it understated taxable income by that amount. It also omitted the $163.40 NIIT on $4,300 of interest and rental income." +us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left Connecticut income tax (~$11,896) out of SALT and deducted only the $2,012 property tax, so itemized deductions came to $24,984 instead of $35,717 and taxable income of $209,430 crossed into the 32% bracket. It also used 2025 bracket thresholds instead of 2026 ones, added the $1,750 refund, and skipped the 0.5% charitable floor. It omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488: the head's $3,744 is the same interest as the tax unit's $2,976 + $768, and the $750k acquisition-debt limit restricts the deduction to $2,808. It also capped SALT at the old $10,000 instead of the ~$13,908 allowed under the OBBBA's $40,400 cap and skipped the 0.5% charitable floor. It omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation and wrongly said itemized deductions are limited by the standard deduction. At 2026 brackets, $38,141 implies about $189,800 of taxable income, roughly $7,100 below the correct $196,918. It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"At 2026 brackets, $35,233 implies about $177,600 of taxable income, roughly $19,300 below the correct $196,918. Its reasoning invokes a retirement-savings credit that is unavailable at $232,635 of AGI, far above the Saver's Credit income limit. It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-sunset rules: a ~$5,300 personal exemption, $40,615 of itemized deductions, and pre-TCJA rates. The OBBBA made the TCJA brackets permanent with no personal exemption, and the correct 2026 rules give $39,858.42 on $196,918 of taxable income. It also computed NIIT at ~$70 on netted investment income instead of $163.40 on $4,300." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2026 brackets and charitable floor, but its taxable income of $210,286 is $13,368 too high. That matches leaving the ~$11,896 of Connecticut income tax out of SALT and including the $1,750 state refund. The excess pushed $8,511 into the 32% bracket, and it omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used $234,414 of AGI, which includes the $1,750 state refund, and itemized deductions of about $38,600 against the correct $35,717, landing near $195,800 of taxable income. It then computed NIIT at $70 on investment income netted for the farm rent loss, instead of 3.8% of $4,300 ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset and applied a restored personal exemption and pre-TCJA rates, but the OBBBA kept the 10/12/22/24/32% schedule permanent with no personal exemption. Its itemized deductions left out Connecticut income tax, leaving its $203,194 of taxable income about $6,300 above the correct $196,918. It omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, giving AGI of $226,985 instead of $232,635, and omitted the $163.40 NIIT on $4,300 of interest and rental income. Its $38,627.54 corresponds to about $191,800 of taxable income, roughly $5,100 below the correct $196,918." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It overstated itemized deductions at $40,221 against the correct $35,717. With mortgage interest at $2,808, its total implies about $37,400 of charitable gifts plus SALT, against the correct $32,909 ($19,001 after the 0.5% floor plus ~$13,908 SALT). It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It omitted the $163.40 NIIT on $4,300 of interest and rental income, even though MAGI exceeds the $200,000 threshold. Its taxable income of $196,143 is $775 below the correct $196,918: it added the $1,750 refund but overstated itemized deductions by more. Regular tax came to $39,672 instead of $39,858.42." +us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 ESI premium from wages, giving AGI of $226,985 instead of $232,635, when only the $617 401(k) deferral reduces gross wages. Its taxable income of $190,999 is $5,920 short of the correct $196,918, which cost $1,420.68 of regular tax. It also omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. At 2026 brackets, $39,250 corresponds to about $194,400 of taxable income, roughly $2,500 below the correct $196,918, so it overstated deductions. It also omitted the $163.40 NIIT on $4,300 of interest and rental income." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law: a $5,427 personal exemption, $2,966 of 2%-floor employee expenses, the full $3,744 of mortgage interest under the old $1M limit, and pre-TCJA brackets. The OBBBA made permanent the TCJA brackets, the $750k mortgage limit, and the repeal of personal exemptions and miscellaneous deductions. It also omitted NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $190,831 at restored pre-TCJA 10/15/25/28% brackets after a $5,437 personal exemption, but the OBBBA made the TCJA rate schedule permanent with no exemption. At the correct 2026 schedule, the $196,918 of taxable income yields $39,858.42, plus the $163.40 NIIT it omitted." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $20,164 of charitable gifts without the 2026 0.5%-of-AGI floor (the correct amount is $19,001). It overestimated Connecticut income tax, putting SALT at $15,371 instead of ~$13,908. It also computed NIIT on $1,853 after netting the farm rent loss, instead of on $4,300 ($163.40)." +us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025, adding $2,966 of 2%-floor miscellaneous deductions, deducting the full $3,744 of mortgage interest, and applying pre-TCJA brackets. The OBBBA made permanent the TCJA brackets, the $750k mortgage limit, and the elimination of miscellaneous deductions. Its $194,371 of taxable income yields about $39,250 at the actual 2026 schedule, so the pre-TCJA rates caused the overstatement; it also omitted NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of 2%-floor miscellaneous itemized deductions for unreimbursed employee business expenses, which the OBBBA permanently eliminated, and skipped the 2026 0.5%-of-AGI charitable floor. Those extra deductions cut regular tax to $39,214 instead of $39,858.42, while its $163 NIIT matched the $4,300 base." +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there is no substantive answer to evaluate." +us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 with no derivation. That ignores $196,918 of taxable income, which produces $39,858.42 of regular tax at 2026 single brackets plus $163.40 of NIIT, with no nonrefundable credit to offset any of it." +us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $3,744 of mortgage interest without prorating for the $750,000 acquisition-debt limit on $1,000,000 of debt (only $2,808 is allowed). It added the $1,750 state refund to AGI and omitted the $163.40 NIIT on $4,300 of interest and rental income. These errors offset each other to land $20 high." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took the $18,525 standard deduction instead of itemizing $35,717 ($19,001 of charitable gifts after the 0.5% floor, ~$13,908 of SALT, and $2,808 of mortgage interest), overstating taxable income by about $17,600. It also skipped the $617 401(k) exclusion, used 2025 bracket thresholds, and omitted the $163.40 NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used married-filing-separately status for an unmarried single filer, added the full $20,164 of charitable gifts on top of the standard deduction instead of itemizing, and counted the $6,147 of tax-exempt pension as taxable. It also invented a $750 retirement income credit, and its reported $48,557 does not match its own $45,287 − $750 = $44,537." us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. @@ -8700,89 +9542,100 @@ us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model returned 0 without applying Medicare's age-based eligibility rule. At age 76, the head exceeds the age-65 threshold, so the eligibility value is 1 regardless of the household's income, assets, or employer-sponsored coverage." us,scenario_120,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived every component correctly in its own reasoning — SS $10,267.01, Medicare $2,401.16, CT paid leave 0.5% x $165,597 = $827.99 — and even wrote the correct total of $13,496.16, then submitted $13,736.63 instead. The submitted number is $240.47 above its own stated sum and corresponds to no step in its derivation, so the failure is the transcription of the final value rather than the tax logic." -us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model computed only the two federal components (6.2% x $165,597 = $10,267.01 plus 1.45% x $165,597 = $2,401.16) and stopped, omitting Connecticut's mandatory employee Paid Family and Medical Leave contribution of 0.5% on covered earnings, which the question explicitly asks for as a ""mandatory employee state payroll tax."" That omission is exactly the $827.98 shortfall between its $12,668.17 and the reference $13,496.16." -us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model recognized that CT imposes a mandatory employee state payroll tax but then discarded its own 0.64% estimate and substituted a fabricated ""CT payroll tax adjustment"" of negative $48.79, subtracting money from a tax that only adds. CT's employee paid-leave contribution is 0.5% of covered wages, $827.98 on $165,597, so replacing a positive $827.98 with a negative $48.79 produced the $876.75 gap below the reference." -us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model dismissed the state component outright (""CT has no mandatory employee payroll tax included here besides standard"") and totaled only federal FICA at 7.65% of $165,597. Connecticut's employee-paid Family and Medical Leave contribution of 0.5% on covered earnings is a mandatory employee state payroll tax of $827.98, and its absence is the entire error." -us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model identified all three correct components in its reasoning — $10,267 Social Security, $2,401 Medicare, and CT's 0.5% paid-leave contribution of $828, summing to the reference $13,496 — then submitted $10,680 ""after accounting for PolicyEngine's employee payroll tax computation,"" an adjustment it never derived and that corresponds to no rate or base in the problem. The tax reasoning was complete; the model overwrote its own correct total with an invented figure $2,816 below it." -us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model asserted ""Connecticut has no mandatory employee-side state payroll tax (no state disability/UI employee withholding),"" checking only for an SDI/UI-style levy and thereby missing CT's Paid Family and Medical Leave program, which is funded entirely by a 0.5% employee wage contribution. Adding that $827.98 to its correct federal $12,668.17 yields the reference." -us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model used a FICA base of $157,551, i.e. $165,597 less the $7,429 employer-sponsored insurance premium and the $617 traditional 401(k) deferral; elective 401(k) deferrals are wages for Social Security and Medicare purposes and PolicyEngine's taxable_earnings_for_social_security is the full $165,597. It compounded this by declaring ""Connecticut has no employee payroll tax,"" dropping the mandatory 0.5% paid-leave contribution of $827.98." -us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model correctly included CT's 0.5% Paid Family Leave contribution but applied all three rates to a reduced base of $158,168, subtracting the $7,429 employer-sponsored insurance premium from wages. PolicyEngine taxes the full $165,597 of earnings, so the $7,429 exclusion understated the 8.15% combined employee levy by $605.46." -us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied the 0.9% Additional Medicare Tax to earnings of $165,597 despite stating the $200,000 threshold in the same sentence; that tax applies only to wages above $200,000 for a single filer, so it is zero here. Layering a phantom 0.9% surtax on top of federal FICA drove the answer $701 above the reference, while its stated derivation never accounts for CT's 0.5% paid-leave contribution." -us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model got the component structure exactly right — 6.2% Social Security, 1.45% Medicare, and CT's 0.5% PFML — but ran all three on ""FICA-subject wages ($158,168),"" having netted the $7,429 employer-sponsored insurance premium out of the $165,597 of gross wages. PolicyEngine applies these rates to the full $165,597, and the $7,429 base reduction is the entire $605 shortfall." -us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 ESI premium from gross wages to get a $158,168 payroll base before applying 6.2%, 1.45%, and CT's 0.5% PFML rate. The payroll-tax base here is the full $165,597 of wages and salaries, so every one of its three otherwise-correct components is understated by 4.5%." -us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"The model gave only the label ""Social Security tax and Medicare tax on wages"" as its derivation, yet 6.2% plus 1.45% of $165,597 is $12,668.17 — its submitted $14,352.19 exceeds that by $1,684.02 and the correct total by $856.03, matching no rate or base in the household. The number is an unanchored estimate rather than the sum of employee Social Security ($10,267.01), Medicare ($2,401.16), and CT's 0.5% paid-leave contribution ($827.98)." -us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model named all three correct components and rates but computed them on ""FICA wages of $158,168,"" excluding the $7,429 employer-sponsored insurance premium from the wage base. Applying 6.2%, 1.45%, and CT's 0.5% to the full $165,597 gives the reference $13,496.16." -us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model included Social Security, Medicare, and CT paid family leave, but its $12,891 total is exactly 8.15% of $158,168 — gross wages less the $7,429 ESI premium — showing it reduced ""covered wages"" by the health premium. The covered-earnings base for all three levies is the full $165,597." -us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model's component list is correct (6.2% SS, 1.45% Medicare, 0.5% CT PFML), but its $12,891.10 equals the 8.15% combined rate applied to $158,168, i.e. wages net of the $7,429 employer-sponsored insurance premium. PolicyEngine's taxable earnings for both FICA and the CT contribution are the full $165,597 of wages." -us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model stripped three amounts out of the FICA base — $7,429 of ESI premiums, the $617 traditional 401(k) deferral, and the $109 Roth 401(k) contribution — reaching $157,442; Roth contributions are after-tax and reduce no wage base, and traditional 401(k) elective deferrals remain fully subject to Social Security and Medicare tax. It also concluded with no state component, dropping CT's mandatory 0.5% paid-leave contribution of $827.98." -us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model stated flatly that ""CT has no employee payroll tax"" and submitted federal FICA alone. Connecticut's Paid Family and Medical Leave program is financed by a mandatory 0.5% employee contribution on covered wages, $827.98 on $165,597, which is exactly the amount by which its $12,668 falls short." -us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model asserted ""Connecticut has no mandatory employee state payroll tax,"" dropping the $827.98 paid-leave contribution, and then submitted $10,087 — a figure below even the $10,267.01 Social Security component alone, so its Medicare tax of $2,401.16 never entered the total either. Its stated method (SS plus Medicare plus additional Medicare) applied to $165,597 yields $12,668.17, which its own answer contradicts." -us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model described the correct federal method — 6.2% plus 1.45% on $165,597 with no Additional Medicare Tax — which produces $12,668.17, then submitted $6,180, roughly 3.7% of wages and less than half its own stated computation. It never performed the arithmetic it narrated and omitted CT's 0.5% employee paid-leave contribution of $827.98." -us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model had the two federal components exact ($10,267.01 and $2,401.16) and correctly identified CT paid-family-leave withholding, but charged $922.50 — 0.5% of the $184,500 Social Security wage cap — instead of 0.5% of the head's actual $165,597 of covered wages. The CT contribution is levied on wages up to that cap, not on the cap itself, so the correct figure is $827.98." -us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory 0.5% employee paid-leave contribution entirely, and additionally reduced the FICA base to $164,980 by deducting the $617 traditional 401(k) deferral, which stays fully subject to Social Security and Medicare tax. The $828 state omission plus the $47 base error account for the full gap to $13,496.16." -us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model assembled the right three levies — 6.2% Social Security, 1.45% Medicare, 0.5% CT paid leave — but applied them to ""estimated payroll-taxable wages of $158,168,"" removing the $7,429 employer-sponsored insurance premium from the $165,597 of wages. The base for all three is the full wage figure, making its answer 4.5% low across the board." -us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model reported only ""employee FICA on gross wages at standard SS and Medicare rates,"" i.e. 7.65% of $165,597 = $12,668. It never applied Connecticut's mandatory employee Paid Family and Medical Leave contribution of 0.5% on covered earnings, $827.98, which the requested output explicitly includes as a mandatory employee state payroll tax." -us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model stated there is ""no employee state payroll tax in CT"" and submitted 7.65% of $165,597. Connecticut funds its Paid Family and Medical Leave program through a 0.5% employee-only wage contribution, adding $827.98 here and accounting for the entire difference from the reference." -us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model computed the federal components correctly and then concluded ""no other employee payroll taxes,"" overlooking Connecticut's mandatory 0.5% employee paid-leave contribution on covered wages. That contribution is $827.98 on $165,597 and is the sole missing term in its $12,668 total." -us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"The model claimed ""CT has no mandatory employee state payroll taxes"" and submitted federal FICA alone at 7.65% of $165,597. Connecticut's Paid Family and Medical Leave contribution is a mandatory 0.5% employee withholding on covered wages, $827.98 here." -us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for payroll_tax, so the required key was absent from its submitted outputs rather than answered incorrectly. No substantive tax reasoning exists to evaluate." -us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model produced no payroll_tax value and no accompanying explanation, failing the output contract that requires every requested key with a numeric value. There is no computation to diagnose." -us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"The model submitted $0 with no derivation for a household with $165,597 of covered wages, a figure that only follows from treating the head's earnings as exempt from FICA — a 76-year-old drawing Social Security retirement benefits still pays employee Social Security and Medicare tax on wages. The correct computation is 6.2% ($10,267.01) plus 1.45% ($2,401.16) plus CT's 0.5% paid-leave contribution ($827.98)." -us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model asserted ""CT has no mandatory employee payroll tax"" and stopped at federal FICA of $12,668. Connecticut's employee-funded Paid Family and Medical Leave contribution of 0.5% on covered wages adds $827.98, which is precisely the amount missing from its total." -us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model built the federal side exactly right on the full $165,597 base — $10,267.01 Social Security and $2,401.16 Medicare with no Additional Medicare Tax — then declared ""CT has no mandatory employee state payroll tax."" Connecticut's 0.5% employee Paid Family and Medical Leave contribution of $827.98 is the one omitted component." -us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a 2026 employee Social Security tax cap of $9,655 and submitted it as the entire payroll tax; wages of $165,597 are below the Social Security wage base, so the tax is 6.2% of actual wages, $10,267.01, and no cap binds. It also omitted the $2,401.16 Medicare tax and CT's $827.98 employee paid-leave contribution entirely." +us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right: $10,267.01 Social Security, $2,401.16 Medicare, $827.99 CT Paid Leave, total $13,496.16. It then submitted $13,736.63, a number its own work does not support and $240.47 above the total it had just computed." +us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It got federal FICA right at $12,668.17 but left out Connecticut's mandatory 0.5% Paid Leave employee contribution on wages up to the Social Security wage base. That contribution is $827.98 and brings the total to $13,496.16." +us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It used the old $168,600 wage base and got the Social Security ($10,267.14) and Medicare ($2,401.06) products slightly wrong. For the state piece it used a made-up −$48.79 'net from itemizations' adjustment. The actual CT Paid Leave contribution is 0.5% × $165,597 = $827.98, and adding it gives $13,496.16." +us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said CT has no mandatory employee payroll tax and stopped at federal FICA of $12,668.17. It left out the CT Paid Leave employee contribution of 0.5% × $165,597 = $827.98." +us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"It found all three pieces: $10,267 Social Security, $2,401 Medicare, and the 0.5% CT paid-leave tax of $828. It then threw away the correct sum of $13,496 and submitted $10,680, citing an 'employee payroll tax computation' adjustment that does not exist." +us,scenario_120,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,It deliberately left out the CT paid-leave contribution on the false assumption that PolicyEngine does not model it. PolicyEngine includes ct_employee_paid_leave_contribution of $827.98 (0.5% of wages) in employee state payroll tax. +us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It said Connecticut has no mandatory employee-side payroll tax. CT's Paid Leave program requires a 0.5% employee contribution on wages up to the Social Security wage base, which is $827.98 here, and the model left it out of the $12,668.17 FICA total." +us,scenario_120,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare ($12,668.17) and explicitly added no state payroll tax. It missed Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98." +us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It cut FICA wages to $157,551 by subtracting the $7,429 in employer insurance premiums and the $617 traditional 401(k) deferral. 401(k) elective deferrals are FICA wages, and PolicyEngine applies payroll tax to the full $165,597 of listed gross wages. It also said CT has no employee payroll tax and left out the $827.98 Paid Leave contribution." +us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $7,429 in employer insurance premiums from wages and computed every piece, including the 0.5% CT Paid Leave contribution, on $158,168. PolicyEngine applies Social Security, Medicare, and the CT contribution to the full $165,597 of listed gross wages, which gives $13,496.16." +us,scenario_120,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at federal FICA of $12,668.17 and did not add Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98." +us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It brought in an Additional Medicare Tax adjustment that does not apply, because $165,597 of wages is under the $200,000 threshold. That pushed its answer to $14,197, well above the correct $10,267.01 Social Security + $2,401.16 Medicare + $827.98 CT Paid Leave = $13,496.16." +us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It used $158,168 of FICA wages by subtracting the $7,429 in employer insurance premiums. PolicyEngine applies the 6.2%, 1.45%, and 0.5% CT Paid Leave rates to the full $165,597 of listed gross wages." +us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It reduced FICA-taxable wages to $158,168 by subtracting the $7,429 in employer insurance premiums, which pulled every component down. The correct base for Social Security, Medicare, and the CT PFML contribution is the full $165,597 of listed gross wages." +us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,False,"It said it used only Social Security and Medicare on wages, but $14,352.19 is well above what those rates give on $165,597 ($12,668.17). It therefore applied them to an inflated base or wrong rates, and it also left out the $827.98 CT Paid Leave contribution." +us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It computed Social Security, Medicare, and the 0.5% CT Paid Leave contribution on $158,168, having subtracted the $7,429 in employer insurance premiums. PolicyEngine applies all three to the full $165,597 of gross wages." +us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 is the three-part total on $158,168: gross wages minus the $7,429 in employer insurance premiums. PolicyEngine does not reduce the $165,597 wage input by those premiums for payroll tax." +us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It applied the right rates, but its $12,891.10 matches computing them on $158,168, which is gross wages minus the $7,429 in employer insurance premiums. The correct base is the full $165,597, which gives $13,496.16." +us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the traditional 401(k) deferral and the employer insurance premiums, then also subtracted the $109 Roth 401(k) contribution, which is after-tax, ending at $157,442. FICA applies to the full $165,597 of wages. It also left out the $827.98 CT Paid Leave contribution." +us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It said CT has no employee payroll tax and reported only federal FICA of $12,668. It missed the mandatory 0.5% CT Paid Leave employee contribution of $827.98." +us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It said CT has no mandatory employee payroll tax, which drops the $827.98 Paid Leave contribution. Its $10,087 is also below the Social Security component alone (6.2% × $165,597 = $10,267.01), so it dropped the $2,401.16 of Medicare tax and misapplied the Social Security rate." +us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It stated the right rates, 6.2% plus 1.45% on wages under the base, but submitted $6,180, less than half of 7.65% × $165,597 = $12,668.17. It also left out the $827.98 CT Paid Leave contribution." +us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It computed the CT paid-leave contribution as 0.5% of the $184,500 Social Security wage cap ($922.50) rather than 0.5% of actual wages. The cap only limits covered earnings, so the contribution is 0.5% × $165,597 = $827.98." +us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $617 traditional 401(k) deferral to get $164,980 of FICA wages. Elective deferrals stay in Social Security and Medicare wages under IRC §3121(v)(1)(A). It also left out the $827.98 CT Paid Leave contribution." +us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used $158,168 of payroll-taxable wages by subtracting the $7,429 in employer insurance premiums. PolicyEngine applies Social Security, Medicare, and the 0.5% CT paid-leave contribution to the full $165,597 of gross wages." +us,scenario_120,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $7,429 in employer insurance premiums as pretax and cut the payroll-tax base to $158,168. PolicyEngine applies the 6.2%, 1.45%, and 0.5% CT paid-leave rates to the full $165,597 of listed gross wages, which gives $13,496.16." +us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only the standard Social Security and Medicare rates to gross wages ($12,668). It left out Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98." +us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It said CT has no employee state payroll tax and stopped at 7.65% × $165,597 = $12,668. It missed the CT Paid Leave employee contribution of 0.5% of wages, $827.98." +us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It said there are 'no other employee payroll taxes' beyond federal FICA. Connecticut requires a 0.5% Paid Leave employee contribution, $827.98 here, which the model left out." +us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"It said CT has no mandatory employee state payroll tax. The CT Paid Leave program requires a 0.5% employee contribution on wages up to the Social Security base, $827.98, which is missing from its $12,668." +us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no payroll_tax value and no explanation, so there was no answer to score." +us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"It submitted $0 despite $165,597 of wages, applying no payroll tax at all. The correct total is $10,267.01 Social Security + $2,401.16 Medicare + $827.98 CT Paid Leave = $13,496.16." +us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It said CT has no mandatory employee payroll tax and reported only federal FICA of $12,668. It missed the 0.5% CT Paid Leave employee contribution of $827.98." +us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It got Social Security and Medicare right ($12,668.17) but said CT has no mandatory employee state payroll tax. That left out the $827.98 Paid Leave contribution, which is 0.5% of covered wages." +us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It treated employee Social Security tax as capped at $9,655. The 2026 wage base of $184,500 is above the $165,597 of wages, so the tax is the full 6.2% × $165,597 = $10,267.01. It also dropped the $2,401.16 of Medicare tax and the $827.98 CT Paid Leave contribution." us,scenario_120,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It added $2,852 of tax-exempt interest into the Connecticut base and left the $1,750 state tax refund in federal AGI, then applied a vague ""limited"" Social Security subtraction instead of CT's statutory adjustment of federally taxable benefits less 25% of benefits received ($35,615 - $10,475 = $25,140). Its stated $205,000 CT taxable income sits $2,495 below the correct $207,495.12, yet the $12,130 it reported exceeds the $11,917.18 that CT's 2%/4.5%/5.5%/6%/6.5% schedule plus the $250 phase-out add-back and $430 recapture produce, so its number tracks neither the statute nor its own base." -us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied the correct $25,140 Social Security subtraction and the correct rate schedule but counted only $430 of high-income adjustment; Connecticut stacks a $250 bottom-bracket phase-out add-back, a $250 low-tax-rate recapture, and a $180 middle-rate recapture for $680. Its CT AGI of $209,274 also overstates the correct $207,495.12 because it kept the $1,750 state tax refund in federal AGI and omitted the traditional IRA deduction." -us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented two deductions Connecticut does not have — a deduction for federal income tax paid and an age-65 deduction — and pulled federal itemized deductions (mortgage interest, real estate taxes, charitable gifts) into the state base, which starts from federal AGI and permits no itemized deductions. Connecticut taxable income here is $207,495.12, producing $11,237.18 of bracket tax plus $680 of phase-out add-back and recapture, not $0." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It set the Social Security subtraction to zero because AGI exceeds $75,000; Connecticut instead caps the Social Security retained in CT AGI at 25% of benefits received ($10,475), yielding a $25,140 subtraction at any income. It compounded that by adding $2,852 of tax-exempt interest to the base and pricing the bottom bracket at the repealed 3% rate, then discounted its own $13,247 bracket result down to $11,890 with no stated basis." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used Connecticut's pre-2024 schedule (3% and 5% bottom brackets rather than the enacted 2% and 4.5%) and then reported $9,970, roughly $1,080 below the ~$11,050 its own stated $200,000 base produces under that schedule. The correct base is $207,495.12 — federal AGI of $232,635.12 less the $25,140 Social Security adjustment — taxed to $11,237.18 before $680 of phase-out add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It truncated Connecticut's rate schedule at 5.5%, omitting the 6% bracket covering $100,000-$200,000 and the 6.5% bracket above $200,000, and its $198,800 base is federal AGI with all $35,615 of taxable Social Security stripped out rather than the statutory $25,140. It also omitted the $250 phase-out add-back and the $430 benefit recapture that CT imposes at $207,495.12 of CT AGI." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It concluded that CT AGI above $100,000 eliminates the Social Security exemption and set the subtraction to zero; Connecticut's Social Security Benefit Adjustment includes only 25% of benefits received ($10,475) in CT AGI, so $25,140 is subtracted regardless of income. Taxing the full $234,414 instead of $207,495.12 overstated bracket tax by $1,749.82, which its omission of the $680 add-back and recapture only partly offset." -us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own stated CT AGI of about $198,000 yields roughly $10,650 under Connecticut's graduated schedule, yet it submitted $7,200 with no arithmetic bridging the two, and it reduced income by the after-tax Roth 401(k) and Roth IRA contributions ($109 and $45), which are not deductible anywhere. The correct base is $207,495.12 after the $25,140 Social Security adjustment, giving $11,237.18 of bracket tax plus $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It stripped the entire $35,615 of federally taxable Social Security from the base and added back $2,852 of tax-exempt interest, then subtracted a $15,000 personal exemption that Connecticut reduces $1,000 per $1,000 of CT AGI over $30,000 and eliminates completely by $45,000. The statutory adjustment removes only $25,140, leaving $207,495.12 taxable with no exemption and $680 of phase-out add-back and recapture on top." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It applied a $1,549.11 Connecticut personal tax credit; that credit is a declining percentage of tax that reaches zero for single filers once CT AGI passes roughly $105,500, so none is available at $207,495.12. It also exempted all Social Security instead of subtracting $25,140, which is why its pre-credit figure of $10,532.20 already fell short of the correct $11,917.18." -us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It removed the $1,750 state tax refund but never applied Connecticut's Social Security Benefit Adjustment, taxing $232,664 instead of $207,495.12; the $25,140 subtraction is the federally taxable $35,615 less the 25% of benefits ($10,475) that CT retains. Its $300 of recapture also undershoots the $250 phase-out add-back plus $250 low-rate and $180 middle-rate recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It computed the subtraction as 25% of the federally taxable Social Security ($8,904); Connecticut's rule is the mirror image — the federally taxable amount less 25% of benefits received, $35,615 - $10,475 = $25,140. Under-subtracting by $16,236 inflated CT AGI to $223,731 against the correct $207,495.12, and it also invoked the superseded $181,700 recapture threshold rather than the $200,000-plus low- and middle-rate recapture totaling $430." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It submitted $7,800 behind a contentless explanation naming no subtraction, bracket, or threshold. That figure matches Connecticut's schedule applied to roughly $151,000 — the base left after exempting all $41,900 of Social Security and the $30,216 pension and taking a $15,000 personal exemption; CT subtracts only $25,140 of Social Security, denies the pension exemption to single filers at or above $75,000 of AGI, and zeroes the personal exemption above $45,000, leaving $207,495.12 taxable." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It stated outright that no Social Security exemption applies at this income and taxed the full $234,414 of federal AGI; Connecticut's benefit adjustment subtracts $25,140 — the federally taxable $35,615 less 25% of the $41,900 received — at every income level. That omission drives the entire $1,354.82 overstatement, only partly offset by its $285 of recapture against the correct $680." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Its federal AGI of $225,235 falls $7,400 short of the correct $232,635.12, dropping income that belongs in the base, and its $26,711 Social Security subtraction exceeds the statutory $25,140 (federally taxable $35,615 less 25% of benefits received). It then credited only $150 of high-income adjustment against the $250 phase-out add-back plus $250 low-rate and $180 middle-rate recapture that CT imposes at $207,495.12 of CT AGI." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It submitted $11,770 with an explanation that restates the variable definition and shows no computation. Netting out the $680 of phase-out add-back and recapture, that figure corresponds to CT AGI of about $205,231 — $2,264 below the correct $207,495.12, i.e. a Social Security subtraction taken $2,264 larger than the statutory $25,140." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It put CT taxable income at $212,320, $4,825 above the correct $207,495.12, which corresponds to a Social Security subtraction of only about $20,315 instead of the statutory $25,140 (federally taxable $35,615 less 25% of the $41,900 received). Its $660 of high-income adjustment also misses the exact $250 phase-out add-back plus $250 low-rate and $180 middle-rate recapture totaling $680." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no derivation at all; $13,125 corresponds to Connecticut's schedule on about $232,000 with roughly $300 of recapture — that is, full federal AGI with no Social Security Benefit Adjustment applied. CT subtracts $25,140 because only 25% of benefits received stays in CT AGI, leaving $207,495.12 taxable, $11,237.18 of bracket tax, and $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It gave no derivation; $9,706.75 matches Connecticut rates on roughly $182,000 — federal AGI with all $35,615 of taxable Social Security removed and a $15,000 personal exemption subtracted. CT removes only $25,140 of Social Security and phases the single personal exemption to zero above $45,000 of CT AGI, so the base is $207,495.12 and the tax $11,917.18 including $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted a $15,000 ""standard deduction"" that Connecticut does not provide — CT's $15,000 single personal exemption phases out entirely above $45,000 of CT AGI — and claimed a $200 property tax credit that is fully phased out for single filers far below $207,495. It also exempted Social Security in full rather than subtracting the statutory $25,140, and omitted the $250 phase-out add-back and $430 recapture." -us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It multiplied its entire $197,049 base by the single 4.5% bracket rate, ignoring that Connecticut's schedule steps to 5.5% above $50,000, 6% above $100,000, and 6.5% above $200,000. It also removed all $35,615 of federally taxable Social Security rather than the statutory $25,140, so the correct base is $207,495.12 taxed to $11,237.18 plus $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It submitted $4,414 with an explanation that restates the variable definition and performs no computation. That figure equals Connecticut's graduated schedule applied to roughly $94,400 of taxable income, less than half the correct $207,495.12 CT AGI, which produces $11,237.18 of bracket tax plus the $250 phase-out add-back and $430 recapture." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It submitted $7,929 with no derivation beyond a restatement of the prompt and a reference to itemized deductions, which Connecticut does not allow. The figure matches CT rates on about $153,000 — the base left after dropping both the $41,900 of Social Security and the $30,216 taxable pension — while CT subtracts only $25,140 of Social Security and fully taxes the pension for a single filer above $75,000 of AGI, leaving $207,495.12." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It subtracted 75% of the federally taxable Social Security ($26,711) instead of the taxable amount less 25% of benefits received ($25,140), and added back $2,852 of tax-exempt interest that Connecticut leaves out of CT AGI, putting its base at $208,805 against the correct $207,495.12. Its $680 of add-back plus recapture is correct in total (split $500/$180 rather than $250/$250/$180), so the whole $84.82 error traces to that CT AGI overstatement." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It taxed roughly $235,031 with no Social Security Benefit Adjustment, ignoring the $25,140 subtraction Connecticut allows because only 25% of the $41,900 received ($10,475) remains in CT AGI. It then layered on about $1,240 of recapture against the correct $250 phase-out add-back plus $430 recapture, compounding the overstatement of the true $11,917.18 on a $207,495.12 base." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It applied the graduated schedule and high-income recapture to about $234,000 of Connecticut AGI, which is federal AGI with no Social Security Benefit Adjustment; CT subtracts $25,140 (the federally taxable $35,615 less 25% of the $41,900 received), bringing the base to $207,495.12. That single omission accounts for essentially the whole $1,700 overstatement, since its ~$657 of recapture nearly matches the correct $680." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It asserted that income is too high for any Connecticut Social Security relief and fixed CT AGI at $234,414; the benefit adjustment subtracts $25,140 at every income level because CT includes only 25% of benefits received in CT AGI. Its $10,895 is also $2,772 below the roughly $13,667 that its own stated base yields under CT's schedule with recapture, so the submitted number follows neither the statute nor its own reasoning." -us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Its $200,095 CT taxable income falls $7,400 below the correct $207,495.12: it removed the $1,750 state tax refund that is not in the $232,635.12 federal AGI to begin with and took a Social Security subtraction roughly $5,650 larger than the statutory $25,140. It also applied about $590 of high-income adjustment rather than the $250 phase-out add-back plus $250 low-rate and $180 middle-rate recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It submitted $7,850 behind a one-line explanation naming no adjustment, deduction, bracket, or threshold. That figure equals Connecticut rates on about $151,700, consistent with exempting both the $41,900 of Social Security and the $30,216 pension; CT subtracts only $25,140 of Social Security, allows no pension exemption to a single filer at this AGI, and taxes $207,495.12 to $11,917.18 including $680 of add-back and recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It removed all $35,615 of federally taxable Social Security to reach $197,049, where Connecticut subtracts only $25,140 because 25% of the $41,900 received ($10,475) stays in CT AGI. Its bracket arithmetic on that base is exact at $10,572.94, so the entire $1,344 shortfall is the $10,475 wrongly excluded plus the $250 phase-out add-back and $430 recapture it never applied." -us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It applied a 100% pension and annuity exemption, which Connecticut denies to single filers with AGI at or above $75,000, and exempted all $35,615 of taxable Social Security rather than the statutory $25,140, dropping CT AGI to $166,833 instead of $207,495.12. It also priced the bottom brackets at the repealed 3% and 5% rates and used $200 of recapture in place of the $250 phase-out add-back plus $430 recapture." -us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It excluded all $35,615 of federally taxable Social Security and added back $2,852 of tax-exempt interest to reach $201,651; Connecticut subtracts $25,140 and leaves tax-exempt interest out entirely, giving $207,495.12. It also used the repealed 3% and 5% bottom bracket rates instead of 2% and 4.5% and omitted the $250 phase-out add-back and $430 recapture." -us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It taxed the full $234,385 of federal AGI with no Social Security Benefit Adjustment — CT subtracts $25,140 because only 25% of the $41,900 received remains in the base — and then subtracted a $300 property tax credit that phases out completely for single filers with CT AGI above roughly $115,500. The correct computation is $11,237.18 of bracket tax on $207,495.12 plus $680 of add-back and recapture, with no nonrefundable credit." -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no substantive computation exists to evaluate. The contract required a numeric value with a supporting explanation for this key and neither was submitted." -us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so no substantive computation exists to evaluate. The contract required a numeric value with a supporting explanation for this key and neither was submitted." -us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with ""value = 0"" as its entire explanation, asserting no Connecticut liability for a single filer with $207,495.12 of CT AGI. Connecticut's graduated schedule alone produces $11,237.18 on that base, and the $250 phase-out add-back plus $250 low-rate and $180 middle-rate recapture bring it to $11,917.18, with no nonrefundable CT credit available at this income." -us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted charitable contributions above 2% of AGI ($20,164 - $4,688 = $15,476) from the state base; Connecticut allows no itemized deductions of any kind and computes tax directly from CT AGI after statutory modifications. It simultaneously denied the $25,140 Social Security subtraction, and the two errors partly cancel to leave $218,938 taxable instead of $207,495.12, with the $680 of add-back and recapture also omitted." -us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It exempted the full $41,900 of Social Security, subtracted $6,147 of tax-exempt pension income that was never in federal AGI, added back $2,852 of tax-exempt interest, and applied a $15,000 personal exemption Connecticut phases out entirely above $45,000 of CT AGI. The only statutory modification here is the $25,140 Social Security subtraction, leaving $207,495.12 taxed at 2%/4.5%/5.5%/6%/6.5% rather than the repealed 3%/5% schedule it used." -us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It subtracted a $500 Connecticut property tax credit; that credit is capped at $300 and phases out completely for single filers with CT AGI above roughly $115,500, so none applies at $207,495.12. Its $202,564 base also runs $4,931 low, and its $11,943 pre-credit figure omits the $250 bottom-bracket phase-out add-back plus the $250 low-rate and $180 middle-rate recapture that carry the correct answer to $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It never applied Connecticut's high-income Social Security rule (subtract taxable benefits minus 25% of total benefits = $25,140) and guessed a 'limited' subtraction instead. It also added back $2,852 of tax-exempt interest that the computation does not add. Its $12,130 is also more than the 2026 schedule plus $680 of add-back and recapture produce on its own ~$205,000 base (about $11,755), so its bracket arithmetic is wrong too." +us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied the correct $25,140 Social Security subtraction. However, it started from a federal AGI of $234,414 that includes the $1,750 state refund and skips the $29 IRA deduction, overstating CT AGI by $1,779 (+$116 of tax). It added the $250 2% phase-out add-back and the $180 middle recapture but left out the separate $250 low-rate benefit recapture, which leaves it $134 short net." +us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It invented Connecticut deductions that do not exist: federal income tax paid, federal itemized deductions and an over-65 deduction. Connecticut taxes CT AGI of $207,495 (federal AGI less the $25,140 Social Security subtraction) with the personal exemption fully phased out, which gives $11,917, not $0." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It said no Social Security subtraction applies above $75,000, missing that Connecticut still subtracts taxable benefits above 25% of total benefits ($25,140). It also added back $2,852 of tax-exempt interest, which inflated CT AGI to $236,883 instead of $207,495. It then used a 3% first-bracket rate instead of 2% and arbitrarily cut its own $13,247 to $11,890 rather than adding the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It never computed the $25,140 Social Security subtraction and called it only 'partial'. It used the pre-2024 3%/5% bottom rates and left out the 6.5% bracket above $200,000. Its $9,970 is below even the graduated tax on its own ~$200,000 base ($11,050 at those rates), and it included neither the $250 phase-out add-back nor the $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It claimed the Social Security subtraction phases out to zero but then used a ~$198,800 base. It described the schedule as 2%–5.5%, dropping the 6% ($100k–$200k) and 6.5% brackets. The correct computation taxes $207,495 at 2/4.5/5.5/6/6.5% ($11,237.18) and adds $680 of phase-out add-back and recapture, not the $9,450 it reported." +us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It got CT AGI ($207,495) and the graduated tax ($11,237) right and added the $250 2% phase-out add-back and $180 middle recapture. It left out the separate $250 low-rate benefit recapture for single filers above $200,000, so it is exactly $250 short." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It concluded there is no Social Security subtraction above $100,000. That misses Connecticut's rule that high-income filers still subtract taxable benefits above 25% of total benefits ($35,615 − $10,475 = $25,140). It also kept the $1,750 refund in AGI, so it taxed $234,414 instead of $207,495. It treated the graduated tax as final and left out the $250 2% phase-out add-back and the $430 benefit recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted non-deductible Roth contributions and never quantified the $25,140 Social Security subtraction. Its $7,200 is far below what its own 3%–6.99% schedule gives on its ~$198,000 base (about $10,900). The correct result taxes $207,495 at the 2026 rates ($11,237.18) plus $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It used a CT AGI of $209,891, about $2,400 above the correct $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction). It added only a $180 recapture and left out both the $250 2% phase-out add-back and the $250 low-rate benefit recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security. Above the single threshold, Connecticut subtracts only the $25,140 above 25% of benefits. It also added back $2,852 of tax-exempt interest and applied the $15,000 personal exemption, which is fully phased out at this income. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It exempted Social Security in full instead of limiting the subtraction to taxable benefits above 25% of total benefits ($25,140), and it used the pre-2024 3%/5% bottom rates. It subtracted a $1,549 personal tax credit that is fully phased out for a single filer at CT AGI of $207,495. It also left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It removed the refund ($232,664) but made no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. It used the pre-2024 3%/5% bottom rates ($11,050 through $200,000 instead of $10,750) and left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of only the $25,140 above 25% of benefits, which put CT AGI at $197,049 instead of $207,495. It then used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It inverted Connecticut's Social Security rule, subtracting only 25% of taxable benefits ($8,904). The rule subtracts taxable benefits in excess of 25% of total benefits ($35,615 − $10,475 = $25,140), so its CT AGI of $223,731 is too high. It also started the recapture at a nonexistent $181,700 threshold and added about $1,020, instead of the $250 add-back plus $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. The correct path taxes CT AGI of $207,495 (federal AGI less the $25,140 Social Security subtraction) at 2/4.5/5.5/6/6.5% ($11,237.18) plus $680 of add-back and recapture. Its $7,800 matches a ~$152,000 base on the 2026 schedule, consistent with exempting all Social Security and pension income (which Connecticut taxes at this AGI) and skipping the add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It said high income eliminates the Social Security subtraction and taxed roughly the full federal AGI. Connecticut still subtracts taxable benefits above 25% of total benefits ($25,140) for filers over the threshold, which makes CT AGI $207,495, not about $232,000–$234,000." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It understated federal AGI at $225,235 (should be $232,635) and used a $26,711 Social Security subtraction instead of $25,140, which put CT AGI at $198,524 instead of $207,495. It then used the pre-2024 3%/5% bottom rates ($10,961 base) and added only $150 of recapture instead of the $250 phase-out add-back plus $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. The correct path taxes CT AGI of $207,495 at the 2026 schedule ($11,237.18) and adds the $250 2% phase-out add-back plus $430 benefit recapture ($250 low + $180 middle). Its $11,770 is $147 short, consistent with a roughly correct base but an under-count of the $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used CT taxable income of $212,320, about $4,825 above the correct $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction). That extra income taxed at 6.5% (+$314) accounts for its $294 overstatement." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $13,125 matches the 2026 graduated tax on roughly the full $232,635 federal AGI, consistent with skipping Connecticut's $25,140 Social Security subtraction (taxable benefits above 25% of total benefits). The correct base is $207,495, taxed at $11,237.18 plus $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $9,706.75 matches a ~$182,600 base on the 2026 schedule, consistent with exempting all Social Security and also taking a $15,000 exemption that is fully phased out at this income. Connecticut subtracts only the $25,140 of taxable benefits above 25% of total benefits, so the base is $207,495 and the tax is $11,237.18 plus $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It fully exempted Social Security. Above the $75,000 single threshold, Connecticut subtracts only taxable benefits above 25% of total benefits ($25,140), leaving $10,475 taxable. It also took a $15,000 'standard deduction' that Connecticut does not have and a $200 property tax credit that is fully phased out at this income, and it left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 allowed above the income threshold. It then applied a flat 4.5% rate to $197,049, ignoring Connecticut's graduated 2/4.5/5.5/6/6.5% schedule and the $680 of phase-out add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $4,414 is less than the graduated tax on the first $100,000 alone ($4,750), consistent with taxing under half of the correct $207,495 CT AGI. Connecticut subtracts only the $25,140 of Social Security above 25% of benefits and allows no exemption or credit at this income, which gives $11,237.18 plus $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It relied on itemized deductions, which Connecticut does not allow. CT AGI is federal AGI ($232,635) less the $25,140 Social Security subtraction ($207,495), with the personal exemption fully phased out. Its $7,929 matches a ~$153,000 base on the 2026 schedule and leaves out the $680 of phase-out add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It computed the Social Security subtraction as 75% of taxable benefits ($26,711) instead of taxable benefits minus 25% of total benefits ($25,140). It also added back $2,852 of tax-exempt interest and skipped the $29 IRA deduction, putting CT AGI at $208,805 instead of $207,495. Its $680 of phase-out add-back and recapture is correct, so the $1,310 base overstatement taxed at 6.5% accounts for the whole $85 overshoot." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $235,031 with no Social Security subtraction, missing that Connecticut still removes taxable benefits above 25% of total benefits ($25,140) for filers over the threshold. It also applied an oversized high-income recapture instead of the $250 phase-out add-back plus $430 recapture that apply at CT AGI of $207,495." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed about $234,000 of CT AGI with no Social Security subtraction. Connecticut subtracts the $25,140 of taxable benefits above 25% of total benefits, which puts CT AGI at $207,495, and at that level the add-back plus recapture total $680." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It said Social Security relief is unavailable and taxed $234,414, missing the $25,140 subtraction of taxable benefits above 25% of total benefits. Its $10,895 is also below the graduated tax on its own base ($12,987), so its bracket arithmetic is wrong too." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It over-subtracted Social Security, reaching CT taxable income of $200,095 instead of $207,495. The subtraction is federally taxable benefits ($35,615) minus 25% of total benefits ($10,475), which is $25,140. The understated base cost $481 of tax at 6.5% and cut the middle recapture to $90 instead of $180." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It taxed $234,414 with no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. It then added about $2,250 of recapture on top of the $12,987 graduated tax, far more than the $250 phase-out add-back plus $430 recapture that apply at CT AGI of $207,495." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It removed only the state refund and made no Social Security subtraction, missing Connecticut's $25,140 subtraction of taxable benefits above 25% of total benefits. Its $13,286.91 equals the pre-2024 3%/5% schedule applied to $234,414, not even to its stated $232,664 base." +us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It over-subtracted Social Security, reaching CT taxable income of $200,095 instead of $207,495. The subtraction is taxable benefits ($35,615) minus 25% of total benefits ($10,475), which is $25,140. Its $500 add-back matched the combined $250 phase-out add-back and $250 low recapture, but the understated base lost $481 at 6.5% and cut the middle recapture from $180 to $90." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no computation. Its $7,850 matches a ~$152,000 base on the 2026 schedule, consistent with exempting all Social Security and pension income. At this income Connecticut exempts only the $25,140 of taxable Social Security above 25% of benefits and no pension income, so CT AGI is $207,495 and the tax is $11,237.18 plus $680 of add-back and recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of the $25,140 allowed above the threshold (taxable benefits minus 25% of total benefits), understating CT AGI by about $10,450 at $197,049. It also left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all taxable Social Security plus a 100% pension exemption. The pension/annuity subtraction is fully phased out for a single filer at this income, and the Social Security subtraction is limited to $25,140, so CT AGI is $207,495, not $166,833. It also used the pre-2024 3%/5% bottom rates and left out the $430 benefit recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It taxed federal AGI plus $2,852 of tax-exempt interest ($235,516) with no Social Security subtraction, missing the $25,140 subtraction. Its $11,050 base at $200,000 reflects the pre-2024 3%/5% bottom rates instead of $10,750, and it left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140 and added back $2,852 of tax-exempt interest. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It made no Social Security subtraction and taxed $234,385 instead of $207,495. It subtracted a $300 property tax credit that is fully phased out for a single filer at this income, and it left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so this output is missing. The correct derivation taxes CT AGI of $207,495 for $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so this output is missing. The correct derivation taxes CT AGI of $207,495 for $11,917.18." +us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It returned $0 with no reasoning. Connecticut taxes CT AGI of $207,495 (federal AGI $232,635 less the $25,140 Social Security subtraction) with no exemption or credit available at this income. A $0 answer is consistent with treating an elderly, disabled filer as exempt from Connecticut income tax, and no Connecticut rule provides that." +us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It denied the Social Security subtraction; Connecticut still subtracts the $25,140 of taxable benefits above 25% of total benefits. It also invented a $15,476 charitable itemized deduction, which Connecticut does not allow, and it left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It left Social Security out of federal AGI and then subtracted the full $41,900 again. It also subtracted $6,147 of tax-exempt pension that was never in AGI and applied a $15,000 personal exemption that is fully phased out, cutting taxable income to $136,825 instead of $207,495. It used the pre-2024 3%/5% bottom rates and left out the $250 phase-out add-back and $430 recapture." +us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Its pre-credit tax of $11,943 is within $26 of the correct $11,917. It then subtracted a $500 property tax credit, but that credit is capped at $300 and fully phased out for a single filer at CT AGI of $207,495, so no nonrefundable credit applies." us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied the §86 provisional-income test to the $13,044 disability benefits and added 50% of them ($6,522) to AGI, but `disability_benefits` is a fully non-taxable income source that never enters AGI, which is wages alone at $25,665. That phantom inclusion both inflated income and raised the 7.5% floor: at the correct AGI the medical deduction is $32,600 − $1,925 = $30,675, which exceeds AGI and zeroes taxable income, whereas the model's $2,001 residue times the 10% bracket produced its $200." -us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the disability benefits and child support from AGI but then took the standard deduction without ever testing itemized deductions, ignoring the $32,600 of medical costs ($2,200 premiums + $30,000 other medical + $400 OTC). Those exceed the 7.5%-of-AGI floor of $1,925 by $30,675, an itemized deduction larger than the $25,665 AGI itself, so taxable income is $0 rather than the $9,915 the model taxed at 10% to reach $992." -us,scenario_121,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model made two compounding errors: it treated the $13,044 disability benefits as fully taxable when they are a non-taxable income source excluded from AGI (AGI is $25,665 of wages), and it applied a $15,000 standard deduction while ignoring the $32,600 of medical expenses that yield a $30,675 itemized medical deduction after the 7.5% floor. Correcting either alone still leaves tax at $0 because the medical deduction exceeds AGI; instead the model ran $23,709 through the 10% and 12% brackets for $2,607." -us,scenario_121,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model got AGI right at $25,665 and the 2026 single standard deduction right at $16,100, but never compared it against itemized deductions and so dropped the $32,600 of medical costs entirely. The medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675 beats the standard deduction and exceeds AGI, driving taxable income to $0 instead of the $9,565 the model taxed at 10% for $956.50." +us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model applied the §86 provisional-income test to the $13,044 disability benefits and added 50% of them ($6,522) to AGI, but `disability_benefits` is a fully non-taxable income source that never enters AGI, which is wages alone at $25,665. That phantom inclusion both inflated income and raised the 7.5% floor: at the correct AGI the medical deduction is $32,600 − $1,925 = $30,675, which exceeds AGI and zeroes taxable income, whereas the model's $2,001 residue times the 10% bracket produced its $200." +us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"The model correctly excluded the disability benefits and child support from AGI but then took the standard deduction without ever testing itemized deductions, ignoring the $32,600 of medical costs ($2,200 premiums + $30,000 other medical + $400 OTC). Those exceed the 7.5%-of-AGI floor of $1,925 by $30,675, an itemized deduction larger than the $25,665 AGI itself, so taxable income is $0 rather than the $9,915 the model taxed at 10% to reach $992." +us,scenario_121,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model made two compounding errors: it treated the $13,044 disability benefits as fully taxable when they are a non-taxable income source excluded from AGI (AGI is $25,665 of wages), and it applied a $15,000 standard deduction while ignoring the $32,600 of medical expenses that yield a $30,675 itemized medical deduction after the 7.5% floor. Correcting either alone still leaves tax at $0 because the medical deduction exceeds AGI; instead the model ran $23,709 through the 10% and 12% brackets for $2,607." +us,scenario_121,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model got AGI right at $25,665 and the 2026 single standard deduction right at $16,100, but never compared it against itemized deductions and so dropped the $32,600 of medical costs entirely. The medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675 beats the standard deduction and exceeds AGI, driving taxable income to $0 instead of the $9,565 the model taxed at 10% for $956.50." us,scenario_121,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model set AGI equal to $17,276 by subtracting employer-sponsored insurance premiums from wages and omitted the $13,044 of disability benefits from the EITC phaseout income calculation. Including those benefits puts AGI above the childless EITC limit, reducing the credit to $0 rather than $139." us,scenario_121,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model incorrectly used $17,276 as both earned income and AGI and therefore calculated a partially phased-out childless EITC. The disability benefits raise AGI beyond the applicable income ceiling, fully phasing the EITC out instead of leaving $127." us,scenario_121,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model asserted childless EITC eligibility from $17,276 of earned income without applying the phaseout using AGI that includes the $13,044 of disability benefits. That higher AGI exceeds the childless EITC limit, so no $168 credit remains." @@ -8797,19 +9650,22 @@ us,scenario_121,head_medicaid_eligible,grok-4.6,llm_error,categorical_eligibilit us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied a medically needy spend-down plus an asset limit to reach eligibility; neither is a Medicaid pathway in the engine, where medical expenses and the $630 bank balance feed no category determination. With SSI at $0 and MAGI at 1.61 x FPL, medicaid_category resolves to NONE." us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for head_medicaid_eligible, so the key was absent from the submitted outputs rather than answered incorrectly. This is a failure to produce the required output object, not a substantive error about SC Medicaid pathways." us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model equated the `is_disabled` flag and $13,044 of disability benefits with SSI-related Medicaid, but that pathway keys on actual SSI receipt, which is $0 because $25,665 of wages plus $13,044 of unearned disability benefits blow past the SSI federal benefit rate after the standard disregards. Meeting disability criteria establishes no Medicaid category on its own in a non-expansion state, and MAGI at 1.61 x FPL clears every SC MAGI limit." -us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"It correctly noted the head is 53 and under the age-65 threshold, then asserted an SSDI-based Medicare pathway ""under PolicyEngine rules"" that does not exist — is_medicare_eligible is an age-65 test and takes no input from is_disabled or disability_benefits. It also converted the generic disability_benefits input ($13,044) into Social Security disability income, which PolicyEngine tracks separately as social_security_disability and which is $0 here." -us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"It equated ""receives disability benefits"" with SSDI entitlement and stopped there, never applying the age-65 test that is the only condition is_medicare_eligible evaluates. The listed input is the generic disability_benefits variable, not social_security_disability, so nothing in the household record establishes the SSDI status its one-sentence chain depends on." -us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"It named the correct real-world rule — 24 months of SSDI entitlement — and then supplied the missing fact itself, reading ""status constant throughout the tax-benefit year"" as evidence that the 24-month clock had already run, when constancy within the year says nothing about prior-year entitlement and the prompt forbids inferring unlisted facts. PolicyEngine models no SSDI-duration input and computes is_medicare_eligible from age alone, so the head's age of 53 returns 0." -us,scenario_121,head_medicare_eligible,deepseek-v4-flash-0731,llm_error,age_disability,False,"It relabeled the household's generic disability_benefits amount as ""SSDI disability benefits"" and then assumed away the qualifying period with the phrase ""after the qualifying period,"" asserting a condition no listed fact supports. is_medicare_eligible is decided by the age-65 threshold, which a 53-year-old fails." -us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"It explicitly stated the 24-month waiting period ""is assumed to have passed,"" inventing an entitlement history the household record does not contain and that the instruction to treat unlisted facts as false rules out. PolicyEngine's is_medicare_eligible has no SSDI or waiting-period input; it compares age to 65, and 53 < 65 yields 0." -us,scenario_121,head_medicare_eligible,deepseek-v4-pro-0813,llm_error,age_disability,False,"It treated the combination of the is_disabled flag and any disability benefit receipt as itself satisfying Medicare disability eligibility, dropping even the 24-month entitlement requirement its peers cited. Medicare eligibility in PolicyEngine is the age-65 test, and is_disabled drives SSI, the SNAP disabled-member rules, and the elderly/disabled deductions — not is_medicare_eligible." -us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"It read the ""disability benefits"" line item as SSDI and applied the real-world two-year Title II waiting rule, then answered as though the two years had elapsed. The head is 53, below the 65-year threshold that is_medicare_eligible actually tests, and PolicyEngine encodes no disability or duration route into that variable." -us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"It flatly asserted that the head ""has met the waiting period requirements for SSDI/Medicare eligibility"" — a fact absent from the household record, which lists only the is_disabled flag and a generic disability_benefits amount. The governing computation is age >= 65, and the head's age of 53 makes is_medicare_eligible 0." -us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"It applied a real-world generalization (""disabled and receives Social Security disability benefits ... generally qualifies"") in place of the age-65 test that is_medicare_eligible performs, and mislabeled the generic disability_benefits input as Social Security disability income, which is $0 in this household. At age 53 the age test fails and the value is 0." -us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"It asserted that disability status plus benefit receipt qualifies for Medicare without ever evaluating the head's age against the 65 threshold that governs is_medicare_eligible. The $13,044 is PolicyEngine's generic disability_benefits variable, not SSDI, and it feeds no Medicare pathway." -us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"It rested the entire answer on the is_disabled boolean, invoking ""the applicable disability eligibility rule"" without SSDI entitlement, duration, or an age check. is_disabled in PolicyEngine gates SSI, the SNAP medical-expense and shelter deductions, and elderly/disabled provisions; is_medicare_eligible is computed from age >= 65, so a 53-year-old returns 0." -us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"It explicitly overrode the age test — ""eligible ... under the disability pathway even though the head is under age 65"" — asserting a pre-65 route that PolicyEngine does not implement for is_medicare_eligible. The disability_benefits input is not SSDI and carries no entitlement-duration information, leaving the age-65 comparison as the only operative condition, which the head fails at 53." -us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"It attributed the SSDI statutory waiting-period pathway to ""PolicyEngine rules,"" but PolicyEngine has no SSDI-entitlement or waiting-period input and decides is_medicare_eligible on the age-65 threshold alone. It also treated the generic disability_benefits amount as merely ""consistent with SSDI"" and then reasoned from that inference, which the prompt's rule against inferring unlisted facts forecloses; at 53 the answer is 0." +us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"Relabeled the generic $13,044 disability benefits as Social Security disability benefits and said they alone qualify someone for Medicare before 65. It skipped the 24-month SSDI entitlement requirement. SSDI months are unlisted and therefore 0, and ESRD is false, so a 53-year-old fails every Medicare pathway." +us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"Treated the head as an SSDI recipient even though the input is only generic disability benefits. It also treated SSDI receipt itself as Medicare eligibility and never applied the 24-month entitlement requirement, which is unmet because no SSDI duration is given." +us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Named the 24-month SSDI waiting period but assumed it was met because disability status is constant within the year. Constant status during the year does not show 24 prior months of SSDI entitlement. SSDI months are unlisted and therefore 0, and the $13,044 is generic disability benefits, not SSDI." +us,scenario_121,head_medicare_eligible,deepseek-v4-flash-0731,llm_error,age_disability,False,"Called the generic disability benefits SSDI and asserted eligibility 'after the qualifying period'. Nothing in the facts supports the 24-month SSDI entitlement; it is 0 by the unlisted-input rule, so the head, at 53, fails the Medicare age and disability tests." +us,scenario_121,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"Explicitly assumed the 24-month SSDI waiting period had passed, which the prompt forbids by making unlisted inputs 0 and barring inferred benefit receipt. It also treated generic disability benefits as SSDI, when SSDI entitlement is the only non-age pathway that applies here." +us,scenario_121,head_medicare_eligible,deepseek-v4-pro-0813,llm_error,age_disability,False,"Treated being disabled and receiving generic disability benefits as meeting Medicare's disability test. That test requires 24 months of SSDI entitlement (or ESRD or ALS), and none of these is present. The head is 53, below the age-65 threshold." +us,scenario_121,head_medicare_eligible,deepseek-v4.1-flash,llm_error,age_disability,False,"Asserted that the head 'receives SSDI' when the input is only generic disability benefits. It then equated SSDI receipt with Medicare eligibility, skipping the 24-month entitlement requirement, which is 0 months here." +us,scenario_121,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"Labeled the generic disability benefits as SSDI and applied the two-year rule as if it were already satisfied. No SSDI duration is given, so it is 0, the disability pathway fails, and the head is under 65." +us,scenario_121,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"Invented the fact that the head 'has met the waiting period requirements'. The household facts contain no SSDI entitlement or duration, so the 24-month SSDI requirement is unmet, and at 53 the head fails the age test." +us,scenario_121,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"Recast the $13,044 generic disability benefits as Social Security disability benefits and treated them as generally conferring Medicare. It ignored the 24-month SSDI entitlement requirement, which is unmet with 0 listed months." +us,scenario_121,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"Equated the generic disability benefits with SSDI and treated disability status as enough for Medicare. It never applied the 24-month SSDI entitlement requirement, so a 53-year-old with no ESRD is ineligible." +us,scenario_121,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"Treated the is-disabled flag alone as qualifying for Medicare. The under-65 disability pathway requires 24 months of SSDI entitlement (or ESRD or ALS), not a disability status, and none of these is present." +us,scenario_121,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"Applied a 'disability pathway' that it keyed to being disabled and receiving generic disability benefits. The actual pathway requires 24 months of SSDI entitlement, and that is 0 because SSDI receipt and duration are unlisted." +us,scenario_121,head_medicare_eligible,gpt-6-luna,llm_error,age_disability,False,"Treated disability status plus generic disability benefits as satisfying Medicare's disability test. It omitted the required 24 months of SSDI entitlement, which is 0 here, so the 53-year-old head fails the age-65 threshold with no alternative pathway." +us,scenario_121,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"Invoked a 'disability-based Medicare eligibility rule' as if disability status were enough. That rule requires 24 months of SSDI entitlement (or ESRD or ALS), none of which is in the household facts." +us,scenario_121,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"Inferred both SSDI receipt ('consistent with SSDI') and completion of the statutory waiting period from generic disability benefits. The prompt forbids inferring unlisted benefit receipt and sets unlisted inputs to 0, so the 24-month SSDI requirement is unmet and the head, at 53, is ineligible." us,scenario_121,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_121,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $8,389 employer-sponsored insurance premium as a pre-tax payroll deduction even though no pre-tax treatment was listed. FICA applies to the full $25,665 of wages, not $17,276." us,scenario_121,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from FICA wages. With no listed pre-tax deduction, Social Security and Medicare taxes apply to the full $25,665." @@ -8819,170 +9675,197 @@ us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"T us,scenario_121,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model wrongly treated employer-sponsored insurance premiums as automatically reducing the FICA wage base. The full $25,665 is subject to employee Social Security and Medicare tax." us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct full-wage FICA calculation but submitted $1,911, which does not follow from its own formula. Applying 6.2% and 1.45% to $25,665 produces $1,591.23 and $372.14, totaling $1,963.37; South Carolina adds no employee payroll tax here." us,scenario_121,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,snap,claude-fable-5,llm_error,age_disability,False,"It waived the gross income test off the bare ""is disabled"" flag, but SNAP's disabled-member definition (7 CFR 271.2) requires age 60+ or receipt of SSI, Title II disability, disability-based Medicaid, a government disability retirement, or VA total-disability compensation, none of which a 53-year-old wage earner with an unspecified $13,044 of disability benefits meets. With the gross test applied, its own $3,325.75/month figure is roughly double the one-person limit of about $1,696/month (130% FPL) and still exceeds the 200% FPL BBCE ceiling of $2,608, so the household screens out at $0. The same status it lacked is also the sole gateway to the $2,681.67 excess medical deduction it used to zero out net income." -us,scenario_121,snap,claude-fable-5.1,llm_error,age_disability,False,"It stated flatly ""no gross income test"" because the head is disabled, but at age 53 with wages and employer-sponsored insurance the head is not an elderly or disabled member under 7 CFR 271.2, so the $3,325.75/month gross income is tested against the one-person limit of about $1,696/month and fails outright. Its ~$3,381 excess medical deduction depends on that same missing elderly/disabled status under 7 CFR 273.9(d)(3), so both the waiver and the deduction that produced the $298 maximum allotment fall away." -us,scenario_121,snap,claude-opus-4.7,llm_error,age_disability,False,"It identified the binding constraint — South Carolina's BBCE gross ceiling of about $2,510/month for one person against gross income of $3,326/month — and then discarded it by asserting that an elderly/disabled household faces only the net income test, a waiver this 53-year-old does not qualify for because SNAP requires receipt of SSI, Title II disability, or comparable qualifying disability benefits. Its $31,780 medical deduction is gated by the identical status, so net income never becomes the operative test and the answer is $0 rather than a maximum-allotment figure trimmed by an unexplained adjustment from $3,504 to $3,168." -us,scenario_121,snap,claude-opus-4.8,llm_error,age_disability,False,"It dropped the $1,200 of child support from countable SNAP unearned income and then declared the household ""disabled so subject to net income test only,"" a waiver reserved for members age 60+ or receiving SSI/Title II disability, which excludes this 53-year-old wage earner. Applying the gross income test to the full $3,325.75/month against the one-person limit of roughly $1,696/month yields $0, and its submitted $4,860 also exceeds twelve months of the FY2026 one-person maximum allotment ($298 × 12 = $3,576) that its own ""near-maximum benefit"" reasoning implies." -us,scenario_121,snap,claude-sonnet-4.6,llm_error,age_disability,False,"It correctly computed that $39,909 of gross income fails the one-person gross income test, then overrode that result with the claim that ""disabled households are exempt from the gross income test"" — an exemption that requires age 60+ or a qualifying disability benefit (SSI, Title II disability, disability-based Medicaid, VA total disability) under 7 CFR 271.2, which a 53-year-old with wages, employer coverage, and generic disability benefits does not satisfy. Because the same elderly/disabled status gates the $32,180 excess medical deduction, net income never falls to $18.33/month and the benefit is $0, not $286/month." -us,scenario_121,snap,claude-sonnet-5,llm_error,age_disability,False,"It asserted that ""as an elderly/disabled household only the net income test applies,"" but the head is 53 and holds no SSI, Title II disability, or other qualifying disability benefit, so the gross income test binds and $3,325.75/month against a one-person limit of about $1,696/month ends eligibility at $0. It compounded this by counting the $8,389 of pre-tax employer-sponsored premiums as deductible medical expense and by inventing an uncapped shelter deduction from a $310,000 mortgage balance with no listed housing payment, then applying an unexplained cut from $3,504 to $3,200 that none of its own arithmetic supports." -us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,age_disability,False,"It jumped straight to the ""single-person disabled household"" net-income path with an uncapped medical deduction, skipping the gross income screen entirely; SNAP's elderly-or-disabled category requires age 60+ or receipt of SSI/Title II disability, so this 53-year-old's $3,325.75/month gross is measured against the one-person limit of about $1,696/month and fails. Its $299/month maximum allotment also overstates the FY2026 one-person figure of $298, but the household receives $0 regardless." -us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,age_disability,False,"Its single premise — ""disabled household is exempt from the gross income test"" — misapplies 7 CFR 273.9(a): the exemption attaches only to households containing a member age 60+ or receiving SSI, Title II disability, disability-based Medicaid, or VA total-disability compensation, not to a 53-year-old flagged as disabled with $25,665 of wages. The gross test therefore applies and $3,325.75/month exceeds the one-person limit of roughly $1,696/month, so the excess medical deduction never comes into play and the benefit is $0 rather than the $292/month maximum allotment." -us,scenario_121,snap,gemini-3-flash-preview,llm_error,age_disability,False,"It waived the gross income test ""for a household with a disabled member,"" but the head is 53 and receives no SSI or Title II disability payment, so the household is not elderly-or-disabled for SNAP and its $3,325 monthly gross fails the one-person limit of about $1,696/month. It also stacked a shelter deduction onto the calculation despite no rent or mortgage payment being listed, and its $292/month maximum allotment is below the FY2026 one-person figure of $298 — neither matters once the gross screen returns $0." -us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,age_disability,False,"It exempted the household from the gross income test ""because the Head is disabled,"" an exemption limited to members age 60+ or receiving a qualifying federal disability benefit, which this 53-year-old wage earner does not receive. With the gross test applied, $3,325.75/month against the one-person limit of roughly $1,696/month ends eligibility, so the medical deduction that it used to drive net income to $0 and reach a $291/month allotment is never reached and the benefit is $0." -us,scenario_121,snap,gemini-3.5-flash,llm_error,age_disability,False,"It concluded the ""gross income limit is waived"" because the head is disabled, but SNAP's disabled-member test requires age 60+ or receipt of SSI, Title II disability, disability-based Medicaid, or VA total-disability compensation; a 53-year-old with $25,665 of wages and employer-sponsored coverage meets none. The gross income test therefore binds at $3,325.75/month against about $1,696/month for one person, so the high medical expenses cannot create eligibility and the answer is $0, not the $3,504 maximum-allotment estimate." -us,scenario_121,snap,gemini-3.6-flash,llm_error,age_disability,False,"It declared the head ""exempt from the gross income test"" as a disabled individual and then netted out income with a $2,865 excess medical deduction; both the exemption and that deduction require elderly-or-disabled status under 7 CFR 271.2, which requires age 60+ or a qualifying disability benefit rather than a bare disability flag. Applying the gross test to $3,325.75/month against the one-person limit of about $1,696/month terminates eligibility at $0 before the standard and 20% earned income deductions matter." -us,scenario_121,snap,gemini-3.7-flash,llm_error,age_disability,False,"It claimed the head is ""exempt from the SNAP gross income test and qualifies for the excess medical expense deduction"" purely on the disability flag; both privileges attach only to members age 60+ or receiving SSI, Title II disability, or comparable qualifying benefits, and this 53-year-old qualifies for neither. The gross income test therefore applies to $3,325.75/month against roughly $1,696/month for one person and produces $0, not $3,336." -us,scenario_121,snap,glm-5.3,llm_error,age_disability,False,"It excluded the $1,200 of child support from countable unearned income and then exempted the household from the gross income test as ""a one-person household with a disabled member,"" a status requiring age 60+ or receipt of SSI/Title II disability that this 53-year-old wage earner does not hold. Counting all $39,909, the $3,325.75 monthly gross far exceeds the one-person limit of about $1,696/month, so the $30,000-plus medical deduction it relied on is both unavailable and irrelevant, and the benefit is $0." -us,scenario_121,snap,gpt-5.5,llm_error,age_disability,False,"It treated the household as ""a one-person disabled SNAP household"" whose eligibility rests only on net income after the earned-income, standard, and medical deductions, skipping the gross income screen that applies because the head is 53 and receives no SSI, Title II disability, or other qualifying disability benefit. At $3,325.75/month against a one-person gross limit of roughly $1,696/month the household screens out entirely, so the FY2026 $298 maximum allotment is never awarded and the correct value is $0." -us,scenario_121,snap,gpt-5.6-sol,llm_error,age_disability,False,"Its whole answer rests on the disabled one-person household deducting qualifying medical costs to zero net income, which presumes elderly-or-disabled status under 7 CFR 271.2 — available at age 60+ or with SSI/Title II disability receipt, neither of which applies to this 53-year-old. Without that status the gross income test applies and $3,325.75/month exceeds the one-person limit of about $1,696/month, yielding $0 instead of $298/month." -us,scenario_121,snap,gpt-5.6-terra,llm_error,age_disability,False,"It applied the excess medical deduction and the net-income-only path reserved for elderly-or-disabled SNAP households, but that category requires age 60+ or receipt of SSI, Title II disability, disability-based Medicaid, or VA total-disability compensation, and this 53-year-old with wages and employer coverage qualifies under none. The gross income test therefore governs: $3,325.75/month against roughly $1,696/month for one person gives $0, not the $298 monthly maximum contiguous-state allotment." -us,scenario_121,snap,gpt-6-astra,llm_error,age_disability,False,"It computed net income of $88 by subtracting a $32,180 ""disability-related medical deduction"" and then took the maximum allotment less 30% of that net figure, a calculation that presupposes an elderly-or-disabled household — a status requiring age 60+ or receipt of a qualifying disability benefit, which a 53-year-old with $25,665 of wages does not have. Without it the gross income test applies to $39,909/year ($3,325.75/month) against a one-person limit near $1,696/month, so the household is ineligible and the benefit is $0 rather than $3,549.60." -us,scenario_121,snap,grok-4.6,llm_error,age_disability,False,"It stated that the ""one-person disabled SNAP unit skips the gross-income test,"" but that skip belongs only to households with a member age 60+ or receiving SSI, Title II disability, or comparable qualifying benefits, which excludes this 53-year-old with a generic disability flag. Gross income of $3,325.75/month against the one-person limit of about $1,696/month ends eligibility, so the excess medical deduction on more than $30,000 of costs never applies and the benefit is $0, not $292/month." -us,scenario_121,snap,grok-build-0.1,llm_error,age_disability,False,"It applied the net-income-only test on the basis of a ""disabled head,"" a treatment reserved for members age 60+ or receiving SSI/Title II disability, and this 53-year-old wage earner qualifies for neither. The gross income test therefore applies to $3,325.75/month against roughly $1,696/month for one person and yields $0; its $300/month maximum allotment estimate also overstates the FY2026 one-person figure of $298." -us,scenario_121,snap,inkling,llm_error,age_disability,False,"It ran the ""disabled 1-person household"" path — 20% earned-income, standard, and large out-of-pocket medical deductions to $0 net income — without applying the gross income screen, which binds because SNAP's disabled-member definition requires age 60+ or receipt of SSI, Title II disability, disability-based Medicaid, or VA total-disability compensation. Gross income of $3,325.75/month exceeds the one-person limit of about $1,696/month, so the FY2026 $298 maximum allotment is never reached and the benefit is $0." -us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for snap, so nothing was submitted against the required key rather than a substantive computation being wrong. The correct derivation applies the SNAP gross income test — $25,665 wages plus $13,044 disability benefits plus $1,200 child support equals $39,909, or $3,325.75/month, against a one-person limit of roughly $1,696/month — and returns $0." -us,scenario_121,snap,kimi-k3,llm_error,age_disability,False,"It counted the full $39,909 of gross income correctly and then bypassed the gross income test by granting the ""uncapped medical deduction"" and net-income-only treatment to a 53-year-old whose disability is a bare status flag rather than receipt of SSI, Title II disability, or another qualifying benefit under 7 CFR 271.2. With the gross test applied, $3,325.75/month exceeds the one-person limit of about $1,696/month and the benefit is $0; its asset analysis is also beside the point, since South Carolina's broad-based categorical eligibility removes the $3,000 resource limit it applied." +us,scenario_121,snap,claude-fable-5,llm_error,age_disability,False,"The model waived the gross income test because the head is disabled, then subtracted $2,681.67 a month in excess medical costs to get about $11 of net income and a $3,540 benefit. SNAP counts a member as disabled only if they receive SSI, Social Security disability or similar government disability payments (7 CFR 271.2); generic disability benefits do not qualify. The household therefore faces the gross test, and $3,325.75 a month exceeds South Carolina's 130% FPL limit (about $1,696), so SNAP is $0." +us,scenario_121,snap,claude-fable-5.1,llm_error,age_disability,False,"The model declared 'no gross income test' for a disabled one-person household and used about $3,381 of excess medical costs to zero out net income, giving the $3,576 maximum. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and neither the gross-test exemption nor the medical deduction applies. Gross income of about $3,326 a month fails South Carolina's 130% FPL test, so the benefit is $0." +us,scenario_121,snap,claude-opus-4.7,llm_error,age_disability,False,"The model saw that $3,326 a month exceeds the broad-based categorical eligibility limit, which it took as 200% FPL (South Carolina's is actually 130%). It then exempted the household anyway as disabled, and trimmed its $3,504 maximum to $3,168 with no stated basis. The head does not meet SNAP's receipt-based disability definition (no SSI or Social Security disability), so failing the gross test gives $0." +us,scenario_121,snap,claude-opus-4.8,llm_error,age_disability,False,"The model applied only the net income test because the head is disabled, left $1,200 of child support out of gross income, and answered $4,860, which is more than the one-person maximum of about $3,576 a year. The generic disability benefits are not SSI or Social Security disability, so the head is not SNAP-disabled. The gross income test applies and gross income above 130% FPL makes the household ineligible, so SNAP is $0." +us,scenario_121,snap,claude-opus-5.5,llm_error,age_disability,False,"The model put the household on the disabled track: net income test only, the $4,500 asset limit, and a $2,682 monthly medical deduction, giving about $7 of net income and $3,552 a year. SNAP's disabled status depends on receiving SSI, Social Security disability or similar payments, and this head receives none of those. The household therefore fails the 130% FPL gross test with about $3,326 a month and gets $0." +us,scenario_121,snap,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly found that $39,909 exceeds the 130% FPL gross limit, then overrode that result by exempting the household as disabled and deducting $32,180 of medical costs to reach $3,432. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and the failed gross test is final, making SNAP $0." +us,scenario_121,snap,claude-sonnet-5,llm_error,age_disability,False,"The model admitted gross income exceeds the one-person limit but applied the elderly/disabled exemption. It then built deductions from employer premiums, the $2,200 premium counted twice, and shelter costs invented from a mortgage balance with no stated payment, and cut the $3,504 maximum to an arbitrary $3,200. The head does not meet SNAP's receipt-based disability definition, so the gross test controls and SNAP is $0." +us,scenario_121,snap,claude-sonnet-5.5,llm_error,age_disability,False,"The model treated the head as a SNAP disabled member and used about $32,600 of medical costs to bring net income to zero and claim the $298-a-month maximum ($3,576 a year). The head receives no SSI or Social Security disability, so the household gets no gross-test exemption and no medical deduction. Gross income of about $3,326 a month exceeds South Carolina's 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,age_disability,False,"The model called the head's household 'disabled' and applied the uncapped medical deduction to reach near-zero net income and a $299-a-month maximum ($3,588). Under 7 CFR 271.2, SNAP disability requires receiving SSI, Social Security disability or similar payments, which this head does not. The household therefore fails the 130% FPL gross test and receives $0." +us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,age_disability,False,"The model exempted the household from the gross income test as disabled and used the excess medical deduction to reach zero net income and a $292-a-month maximum ($3,504). The head's generic disability benefits are not SSI or Social Security disability, so the household is non-elderly and non-disabled for SNAP. Its gross income of about $3,326 a month is over the 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,deepseek-v4.1-flash,llm_error,age_disability,False,"The model called the household 'exempt from gross income limit' because it is disabled and computed about $294 a month ($3,520.20 a year) after a medical deduction. SNAP's disabled status is receipt-based (SSI, Social Security disability or similar), and this head qualifies under none of those. The gross test applies, gross income exceeds 130% FPL, and SNAP is $0." +us,scenario_121,snap,gemini-3-flash-preview,llm_error,age_disability,False,"The model waived the gross test because the head is disabled, took a medical deduction of more than $2,800, added a shelter deduction even though no housing costs are listed, and awarded the $292 maximum ($3,504). The head receives no SSI or Social Security disability, so the household must pass the gross test. It fails with about $3,326 a month against South Carolina's roughly $1,696 limit, so SNAP is $0." +us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model exempted the household from the gross income test because the head is disabled and used medical deductions to reach zero net income and a $291-a-month maximum ($3,492). SNAP's disabled-member definition requires receiving SSI, Social Security disability or similar government payments, which the head does not. The household fails the 130% FPL gross test and receives $0." +us,scenario_121,snap,gemini-3.5-flash,llm_error,age_disability,False,"The model said the gross income limit is waived because the head is disabled and let medical costs reduce net income to zero, giving an estimated $3,504 maximum. The head's generic disability benefits are not SSI or Social Security disability, so the head is not a SNAP disabled member. Gross income of about $3,326 a month fails South Carolina's 130% FPL test, so SNAP is $0." +us,scenario_121,snap,gemini-3.6-flash,llm_error,age_disability,False,"The model exempted the head from the gross test as disabled and subtracted a $2,865 excess medical deduction to reach zero net income and $3,504 a year. The head receives no SSI or Social Security disability, so neither the gross-test exemption nor the medical deduction applies. Monthly gross income of about $3,326 is above the roughly $1,696 limit, so SNAP is $0." +us,scenario_121,snap,gemini-3.7-flash,llm_error,age_disability,False,"The model exempted the head from the gross income test as disabled and let the excess medical deduction cancel almost all net income, giving $3,336 a year. SNAP counts a member as disabled only if they receive SSI, Social Security disability or similar payments, and this head does not. The household fails the 130% FPL gross test and receives $0." +us,scenario_121,snap,glm-5.3,llm_error,age_disability,False,"The model treated the household as disabled, skipped the gross test, left out the $1,200 of child support (using $38,709), and used more than $30,000 of medical costs to reach zero net income and $3,504. The head does not meet SNAP's receipt-based disability definition, so the gross test applies. The full $39,909 of gross income exceeds South Carolina's 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,gpt-5.5,llm_error,age_disability,False,"The model treated this as a disabled one-person SNAP household, so it allowed the medical-expense deduction and awarded the $298-a-month maximum ($3,576). The head receives no SSI or Social Security disability, so they are not SNAP-disabled and the household must pass the gross test. It fails with about $3,326 a month against a limit of about $1,696, so SNAP is $0." +us,scenario_121,snap,gpt-5.6-sol,llm_error,age_disability,False,"The model let the disabled one-person household deduct its large medical costs, reaching zero net income and the $298-a-month maximum ($3,576). SNAP disability status requires receiving SSI, Social Security disability or similar government payments, which this head lacks. The household therefore fails the 130% FPL gross income test and receives $0." +us,scenario_121,snap,gpt-5.6-terra,llm_error,age_disability,False,"The model treated the household as SNAP-disabled and let medical costs reduce net income to zero, giving the $298-a-month maximum ($3,576). The head's generic disability benefits are not SSI or Social Security disability, so the household gets no gross-test exemption or medical deduction. Gross income of about $3,326 a month exceeds South Carolina's 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,gpt-6-astra,llm_error,age_disability,False,"The model subtracted a $32,180 'disability-related medical deduction' to reach $88 of annual net income and a $3,549.60 benefit, which assumes the head counts as disabled for SNAP. Under 7 CFR 271.2 that status requires receiving SSI, Social Security disability or similar payments, which this head does not. The gross test therefore applies and $39,909 of gross income makes the household ineligible, so SNAP is $0." +us,scenario_121,snap,gpt-6-luna,llm_error,age_disability,False,"The model applied the elderly/disabled medical deduction to bring net income to zero and awarded the $298-a-month maximum ($3,576). The head receives no SSI or Social Security disability, so the household is not elderly/disabled for SNAP and must pass the gross test. Its gross income of about $3,326 a month is far above the roughly $1,696 limit, so SNAP is $0." +us,scenario_121,snap,gpt-6-sol,llm_error,age_disability,False,"The model treated the head as a disabled SNAP member, deducted the medical costs to reach zero net income, and awarded $298 a month for 12 months ($3,576). SNAP's disability definition is based on receiving SSI, Social Security disability or similar payments, and the head's generic disability benefits do not qualify. The household fails the 130% FPL gross test and receives $0." +us,scenario_121,snap,gpt-6.1-sol,llm_error,age_disability,False,"The model stated that 'disability permits the net-income eligibility test', skipped the gross test, and took a $32,180 medical deduction to reach $88 of net income and $3,549.60. The head receives no SSI or Social Security disability, so the household is not SNAP-disabled and must pass the gross test. It fails with $39,909 against South Carolina's 130% FPL limit of about $20,345 a year, so SNAP is $0." +us,scenario_121,snap,grok-4.6,llm_error,age_disability,False,"The model called this a 'disabled SNAP unit' that skips the gross income test and used the excess medical deduction to reach zero net income and the $292 maximum ($3,504). The head does not meet SNAP's receipt-based disability definition (no SSI or Social Security disability). Gross income of about $3,326 a month therefore fails the 130% FPL test, so SNAP is $0." +us,scenario_121,snap,grok-4.7,llm_error,age_disability,False,"The model waived the gross test for a 'disabled one-person SNAP unit' and subtracted a $31,780 medical deduction to reach $488 of net income and $3,429.60. The head's generic disability benefits are not SSI or Social Security disability, so the household is not SNAP-disabled. The $39,909 of gross income exceeds the 130% FPL limit and makes the household ineligible, so SNAP is $0." +us,scenario_121,snap,grok-build-0.1,llm_error,age_disability,False,"The model applied only the net income test because the head is disabled, let medical costs reduce net income to zero, and used an estimated maximum of $3,600 a year. The head receives no SSI or Social Security disability, so they are not a SNAP disabled member and the gross test applies. Gross income of about $3,326 a month fails South Carolina's 130% FPL limit, so SNAP is $0." +us,scenario_121,snap,inkling,llm_error,age_disability,False,"The model treated this as a disabled one-person household, deducted large out-of-pocket medical costs to reach zero net income, and awarded the $298 maximum ($3,576). SNAP's disabled status requires receiving SSI, Social Security disability or similar government payments, which this head lacks. The household must pass the gross test, fails it at about $3,326 a month, and receives $0." +us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value and no explanation, so there was no usable answer to compare. The correct result is $0: the head receives no SSI or Social Security disability, so the household must pass the 130% FPL gross income test, and about $3,326 a month of gross income fails it." +us,scenario_121,snap,kimi-k3,llm_error,age_disability,False,"The model gave the head the disabled-member earned-income and uncapped medical deductions, counting both $2,200 premium entries, to reach zero net income and the $298 maximum ($3,576). It never applied the gross test, even though the head receives no SSI or Social Security disability and so is not SNAP-disabled. Gross income of $39,909 (about $3,326 a month) exceeds South Carolina's 130% FPL limit, so SNAP is $0." us,scenario_121,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model raised the itemized-deduction question and then dismissed it — ""the standard deduction likely exceeds itemized here"" — when the $32,600 of medical outlays less the 7.5%-of-AGI floor leaves about $30,000 of deduction against a $16,100 standard deduction, zeroing federal taxable income and therefore SC's starting point. It then built its $400 on a $14,600 standard deduction (the 2024 federal figure, not 2026's $16,100), a fabricated $4,610 SC personal exemption (SC grants no exemption for the filer, only for dependents), and a flat 6.2% rate in place of SC's 0%/3%/top-rate schedule." -us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, but $686 is exactly 6.2% × $11,065, i.e. wages of $25,665 reduced by a $14,600 standard deduction and taxed at a single flat rate. That path never claims the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor exceeds the standard deduction by roughly $14,000 and drives SC taxable income to zero, and it also ignores SC's zero-rate bracket on the first ~$3,700 and the 3% bracket above it." -us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model supplied no derivation; $321 equals 3% of $10,700, consistent with an SC taxable income near $14,260 sitting above the zero-rate bracket. Producing any positive SC tax requires positive SC taxable income, and the $32,600 of medical expenses netted against the 7.5%-of-AGI floor eliminates it entirely, so the model's error is taking a standard deduction instead of the far larger medical itemized deduction." -us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the disability benefits and correctly applied SC's 0% bracket followed by the 3% bracket, but it subtracted a $15,400 standard deduction from wages instead of the $32,600 of medical outlays less the 7.5%-of-AGI floor. Claiming that itemized deduction drives taxable income below the top of SC's zero-rate bracket, so the $10,265 base it taxed at 3% does not exist and the correct result is $0." -us,scenario_121,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model built federal taxable income of $23,709 by including the full $13,044 of disability benefits and subtracting only a $15,000 standard deduction, missing both the $32,600 medical expense itemized deduction (about $30,000 after the 7.5% floor) and SC's subtraction of disability income for a permanently disabled filer. With either applied, SC taxable income falls to zero, so its three-tier 0%/3%/6.2% computation had no base to run on." -us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model taxed $38,709 of gross income less a $15,100 standard deduction, never applying the $32,600 medical expense itemized deduction that after the 7.5%-of-AGI floor exceeds that standard deduction by roughly $14,000 and zeroes the SC base. It compounded this with a fabricated bracket schedule containing 6.4% and 6.5% tiers (SC has only a zero-rate bracket, a 3% bracket, and one top rate) and then cut its own $1,395 result to $574 with an unquantified ""exemptions/credits"" adjustment it never identified." -us,scenario_121,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used the correct 2026 federal standard deduction of $16,100 and correctly kept disability benefits out of the base, but stated that ""no SC subtractions"" apply and never itemized: the $32,600 of medical expenses less the 7.5% floor exceeds its own $25,665 AGI, so federal taxable income is $0, not $9,565. With a zero starting point, SC's 0%/3% schedule produces $0 rather than $180." -us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model reduced wages by a $14,600 standard deduction while ignoring the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor eliminates taxable income entirely. It then applied an invented four-tier SC schedule (0%/3%/4%/5.5%) that does not exist — SC has a zero-rate bracket, a 3% bracket, and a single top rate — and its $94.26 does not even follow from that schedule, which would yield about $314 on its own $11,065 base." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its worksheet correctly built AGI of $70,646, a $20,200 standard deduction plus the $6,000 senior deduction, and computed $4,866.64 of tax, but it then submitted $6,552 — a figure its own arithmetic never produces. The derivation itself ran $120 high because it omitted the $1,000 non-itemizer charitable deduction that raises the deduction stack to $27,200." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included only 50% of Social Security ($16,000) instead of running the provisional-income formula, which caps taxable benefits at 85% of $32,000 = $27,200 here, understating AGI by $11,200. It also invented a third additional standard deduction for disability — only age 65+ and blindness generate the $2,050 add-ons — and then submitted $1,065 while its own text computed $3,668." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"AGI of $70,646 and the $20,200 age/blind standard deduction were right, but it stopped there, omitting the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income came out $50,446 instead of $43,446.18. It compounded that by pushing $996 of qualified dividends into the 15% bracket, when correct taxable income leaves the full $1,824 below the ~$49,450 zero-rate ceiling." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used 2025-vintage standard deduction figures (~$15,000 base with $2,000 age and blind add-ons) instead of 2026's $16,100 + $2,050 + $2,050 = $20,200, then abandoned its own $5,011 computation and asserted without arithmetic that additional deductions drop taxable income to ~$40,000 and tax to $1,971. The statutory stack is $27,200 against $70,646.18 of AGI, giving $43,446.18 taxable." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It reached taxable income of ~$43,546, within $100 of the reference, then misapplied the rate schedule: it claimed 10%/12%/22% on ~$41,722 of ordinary income yields $2,336, when the 2026 single brackets (10% to $12,400, 12% to $50,400) yield $4,750 on that base. It also charged $273 of 15% tax on the $1,824 of qualified dividends, which fall entirely in the 0% band at this taxable income." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed TCJA sunsets for 2026 and rebuilt the return under pre-TCJA rules — an $8,300 standard deduction, a $5,300 personal exemption, 10%/15%/25% brackets, and 2%-floor miscellaneous itemized deductions for the $3,510 of employee expenses. OBBBA made the TCJA structure permanent: personal exemptions remain $0, miscellaneous itemized deductions remain suspended, the second bracket is 12%, and the standard deduction is $20,200 plus the $6,000 senior and $1,000 non-itemizer charitable deductions." -us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It dropped the $1,824 of qualified dividends out of AGI entirely, using $41,622 of non-Social-Security income instead of $43,446, and it applied only the $20,200 standard deduction, omitting the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Taxable income is $43,446.18, not the $48,622 it taxed." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"AGI of $70,646 was correct, but it guessed a $19,200 standard deduction instead of $20,200 and omitted both the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, leaving taxable income roughly $8,000 too high. It also taxed the $1,824 of qualified dividends in the 15% bracket when the correct $43,446.18 of taxable income places them entirely in the 0% band." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a TCJA-sunset 2026: a $4,840 personal exemption, $2,097 of 2%-floor miscellaneous employee expenses, a state income tax itemized deduction, and a 15% second bracket. Under permanent OBBBA law exemptions are $0, miscellaneous deductions are suspended, the second bracket is 12%, and the $27,200 standard-deduction stack beats the $14,926 of itemized deductions." -us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used post-sunset parameters — a $5,300 personal exemption, $2,097 of 2%-floor miscellaneous employee expenses, an uncapped SALT itemized deduction including Minnesota income tax, and 10%/15% brackets on a $12,150 first-bracket width. The 2026 schedule is 10% to $12,400 then 12%, with no personal exemption, no miscellaneous deduction, and a $27,200 standard-deduction stack that dominates itemizing." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It itemized $18,126, subtracted a $5,200 personal exemption, and taxed part of the income at 25% — all pre-TCJA parameters that OBBBA eliminated for 2026. Correct 2026 treatment takes the $27,200 standard-deduction stack (including the $6,000 senior and $1,000 charitable deductions), leaves $43,446.18 taxable, and tops out at the 12% bracket with qualified dividends at 0%." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its narrative is arithmetically self-contradictory: it reports AGI of $71,468 less a $29,600 standard deduction yet states taxable income of $69,455, then subtracts $10,808 of itemized deductions on top of the standard deduction. The standard and itemized deductions are alternatives, and the correct chain is $70,646.18 of AGI less a $27,200 stack, giving $43,446.18 taxable and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It offered no computation, only an assertion of 'around $6,300.' The correct build is $70,646.18 of AGI less $20,200 standard, $6,000 senior, and $1,000 non-itemizer charitable deductions, giving $43,446.18 taxable and $4,746.66 of tax; $6,300 is consistent with taxing roughly $48,000 of income under pre-TCJA-sunset 15%/25% rates with only a base deduction and a personal exemption." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It itemized $17,023 — including $2,097 of 2%-floor unreimbursed employee expenses — and subtracted a $5,150 personal exemption, both pre-TCJA features that do not exist in 2026 under permanent OBBBA law. It also capped the capital loss at $1,500 instead of allowing the full $1,518.82 net loss, and it never applied the $6,000 senior or $1,000 non-itemizer charitable deductions that bring the standard stack to $27,200." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It computed $4,917 of tax in its own explanation and then submitted $0, attributing the write-off to 'nonrefundable credits or standard deduction rules.' No nonrefundable credit applies — the elderly/disabled credit zeroes out because half of AGI over $7,500 exceeds the $5,000 base — and the $27,200 deduction stack still leaves $43,446.18 of taxable income and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It built AGI of $70,646 and the age/blind standard deduction correctly but omitted the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, which together cut taxable income from $50,446 to $43,446.18. Its $5,994 also exceeds the $5,736 that a $20,200-only deduction with partial 15% dividend treatment produces, so it additionally taxed qualified dividends outside the 0% band." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It supplied no derivation, and the amount stated in its explanation ($6,979) contradicts the $6,712 it submitted. Both sit in the range produced by pre-TCJA-sunset brackets and a personal exemption; 2026 law gives $70,646.18 of AGI less a $27,200 deduction stack, $43,446.18 taxable, and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 standard deduction schedule ($15,000 + $2,000 age + $2,000 blind = $19,000) instead of 2026's $16,100 + $2,050 + $2,050 = $20,200, and omitted the $6,000 senior deduction and $1,000 non-itemizer charitable deduction, leaving $51,646 of taxable income instead of $43,446.18. It also charged $273.60 of 15% tax on qualified dividends that fall in the 0% band." -us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of provisional income, getting $59,635 instead of $61,459, which made the second-tier formula ($4,500 + 85% of the excess over $34,000) bind at $26,289.75 rather than the 85%-of-benefits cap of $27,200. It then used a guessed $19,570 standard deduction and omitted both the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction." -us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as a flat 85% of the excess over $34,000 ($23,340), dropping the $4,500 second-tier addend, so the correct binding figure — the $27,200 cap at 85% of benefits — never applied and AGI came in $3,860 low. It also used $3,300 for the age and blindness additions instead of $4,100 (two $2,050 amounts), leaving taxable income of ~$40,386 rather than $43,446.18." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction with age and blindness additions fully offsets taxable income. Those amounts total $20,200, and even adding the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction the stack is $27,200 against $70,646.18 of AGI, leaving $43,446.18 taxable and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed 'large tax-exempt income and itemized deductions implied by the medical and real-estate taxes' drive liability non-positive. The $2,013 of tax-exempt interest and $60 of tax-exempt pension never enter AGI (they only feed the Social Security provisional-income test), the $2,300 of medical costs falls entirely under the 7.5% AGI floor of $5,298, and total itemized deductions of $14,926 lose to the $27,200 standard stack, leaving $43,446.18 taxable." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It explicitly applied 'post-TCJA-sunset rules,' itemizing $17,023 including 2%-floor employee expenses and subtracting a $5,300 personal exemption. For 2026 the TCJA structure is permanent: personal exemptions are $0, miscellaneous itemized deductions are suspended, and the $20,200 standard deduction plus $6,000 senior and $1,000 non-itemizer charitable deductions leaves $43,446.18 taxable and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It identified every component correctly — 85% taxable Social Security, the enhanced age/blind standard deduction, the $6,000 senior deduction, the $1,000 non-itemizer charitable deduction, and qualified dividends in the 0% band — but its deduction stack totaled $27,100 instead of $27,200 ($16,100 + $2,050 + $2,050 + $6,000 + $1,000), leaving taxable income of $43,546 rather than $43,446.18. That $100 of excess taxable income at the 12% rate is the entire $12 gap between its $4,759 and $4,746.66." -us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $4,957 is what running the full $43,446.18 of taxable income through the ordinary 2026 single brackets produces ($4,965.54); it never carved out the $1,824 of qualified dividends for the preferential 0% rate. That carve-out is worth $218.88 of tax at the 12% margin and brings the liability to $4,746.66." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed income after deductions falls below the 2026 filing threshold, treating the absence of earned income as decisive. Gross income is $72,165 ($25,000 pension + $14,400 IRA + $5,552 dividends + $13 interest + $27,200 of taxable Social Security), AGI is $70,646.18 after the $1,518.82 capital loss, and $27,200 of deductions still leaves $43,446.18 taxable." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-sunset parameters throughout: a $5,300 personal exemption, $2,097 of 2%-floor employee expenses, a Minnesota income tax itemized deduction, a 15% second bracket, and a standard deduction of only ~$12,400 for a 65+/blind single. The 2026 figures are a $20,200 standard deduction plus $6,000 senior and $1,000 non-itemizer charitable deductions, no personal exemption, and a 12% second bracket." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It itemized charity, real estate taxes, and 2%-floor miscellaneous employee expenses, subtracted a personal exemption, and landed the ordinary income in a 15% bracket — all pre-TCJA features repealed or suspended permanently by OBBBA. For 2026 the $27,200 standard-deduction stack beats $14,926 of itemized deductions, and the top rate reached here is 12%." -us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It indexed the 2017 pre-TCJA brackets forward by 1.353 to get a 15% rate above $12,612, itemized $20,210 including $2,097 of 2%-floor employee expenses and a $3,187 state income tax deduction, and used no senior or non-itemizer charitable deduction. The 2026 brackets are 10% to $12,400 then 12% to $50,400, and the correct $27,200 standard-deduction stack leaves $43,446.18 taxable." -us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It itemized ~$20,100 with an uncapped SALT deduction (~$7,054 including estimated Minnesota tax) and $2,097 of 2%-floor miscellaneous deductions, then subtracted a ~$5,050 personal exemption and taxed at 10%/15% — the pre-TCJA regime. Under permanent 2026 law the $20,200 standard deduction plus the $6,000 senior and $1,000 non-itemizer charitable deductions gives $43,446.18 taxable at 10%/12%, or $4,746.66." -us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for federal_income_tax_before_refundable_credits, so nothing was submitted against the required key. This is an output-contract failure rather than a substantive tax error." -us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It acknowledged taxable income is positive and then asserted the senior standard deduction plus 'personal exemptions/credits' fully offset it. Personal exemptions are $0 under permanent OBBBA law and no nonrefundable credit applies here, so $70,646.18 of AGI less the $27,200 stack leaves $43,446.18 taxable and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"Its worksheet was correct through the $20,200 standard deduction and $6,000 senior deduction but omitted the $1,000 non-itemizer charitable deduction, deriving $4,866.64 rather than $4,746.66. It then submitted $3,491.46, a number that follows from none of its stated arithmetic." -us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its text derived $5,544 — from a guessed $20,050 standard deduction with no senior or non-itemizer charitable deduction, plus a spurious 22% bracket starting at $50,126 — and it then submitted $9,505, unconnected to that arithmetic. The correct chain is a $27,200 deduction stack, $43,446.18 taxable, $1,824 of qualified dividends at 0%, and $4,746.66 of tax." -us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It stated Social Security benefits are not included in AGI and used $52,092, when provisional income of $61,459 far exceeds the $34,000 second threshold and makes 85% of the $32,000 in benefits — $27,200 — taxable, giving AGI of $70,646.18. It also claimed a $1,875 elderly/disabled credit that is fully phased out because half of AGI over $7,500 exceeds the $5,000 base amount, and submitted $5,065 against its own $3,933.60 tentative tax." +us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model raised the itemized-deduction question and then dismissed it — ""the standard deduction likely exceeds itemized here"" — when the $32,600 of medical outlays less the 7.5%-of-AGI floor leaves about $30,000 of deduction against a $16,100 standard deduction, zeroing federal taxable income and therefore SC's starting point. It then built its $400 on a $14,600 standard deduction (the 2024 federal figure, not 2026's $16,100), a fabricated $4,610 SC personal exemption (SC grants no exemption for the filer, only for dependents), and a flat 6.2% rate in place of SC's 0%/3%/top-rate schedule." +us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model gave no derivation, but $686 is exactly 6.2% × $11,065, i.e. wages of $25,665 reduced by a $14,600 standard deduction and taxed at a single flat rate. That path never claims the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor exceeds the standard deduction by roughly $14,000 and drives SC taxable income to zero, and it also ignores SC's zero-rate bracket on the first ~$3,700 and the 3% bracket above it." +us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"The model supplied no derivation; $321 equals 3% of $10,700, consistent with an SC taxable income near $14,260 sitting above the zero-rate bracket. Producing any positive SC tax requires positive SC taxable income, and the $32,600 of medical expenses netted against the 7.5%-of-AGI floor eliminates it entirely, so the model's error is taking a standard deduction instead of the far larger medical itemized deduction." +us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"The model correctly excluded the disability benefits and correctly applied SC's 0% bracket followed by the 3% bracket, but it subtracted a $15,400 standard deduction from wages instead of the $32,600 of medical outlays less the 7.5%-of-AGI floor. Claiming that itemized deduction drives taxable income below the top of SC's zero-rate bracket, so the $10,265 base it taxed at 3% does not exist and the correct result is $0." +us,scenario_121,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"The model built federal taxable income of $23,709 by including the full $13,044 of disability benefits and subtracting only a $15,000 standard deduction, missing both the $32,600 medical expense itemized deduction (about $30,000 after the 7.5% floor) and SC's subtraction of disability income for a permanently disabled filer. With either applied, SC taxable income falls to zero, so its three-tier 0%/3%/6.2% computation had no base to run on." +us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"The model taxed $38,709 of gross income less a $15,100 standard deduction, never applying the $32,600 medical expense itemized deduction that after the 7.5%-of-AGI floor exceeds that standard deduction by roughly $14,000 and zeroes the SC base. It compounded this with a fabricated bracket schedule containing 6.4% and 6.5% tiers (SC has only a zero-rate bracket, a 3% bracket, and one top rate) and then cut its own $1,395 result to $574 with an unquantified ""exemptions/credits"" adjustment it never identified." +us,scenario_121,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"The model used the correct 2026 federal standard deduction of $16,100 and correctly kept disability benefits out of the base, but stated that ""no SC subtractions"" apply and never itemized: the $32,600 of medical expenses less the 7.5% floor exceeds its own $25,665 AGI, so federal taxable income is $0, not $9,565. With a zero starting point, SC's 0%/3% schedule produces $0 rather than $180." +us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model reduced wages by a $14,600 standard deduction while ignoring the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor eliminates taxable income entirely. It then applied an invented four-tier SC schedule (0%/3%/4%/5.5%) that does not exist — SC has a zero-rate bracket, a 3% bracket, and a single top rate — and its $94.26 does not even follow from that schedule, which would yield about $314 on its own $11,065 base." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning stacked the $20,200 standard deduction and the $6,000 senior deduction but left out the 2026 $1,000 charitable deduction for non-itemizers. That gave $44,446 of taxable income and $4,867 of tax, $120 above the correct figure. It then submitted $6,552, a number its own computation does not support." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It taxed only 50% of Social Security ($16,000) when provisional income of $61,459 makes the full 85% ($27,200) taxable. It also invented a $21,700 base and a 'disabled' additional standard deduction, and it omitted the $6,000 senior and $1,000 non-itemizer charitable deductions. Its final $1,065 does not even follow from its own $3,668 tentative tax." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted only the $20,200 standard deduction (basic plus age and blind amounts). It omitted the $6,000 OBBBA senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income came out at $50,446 instead of $43,446.18. That inflated ordinary-rate tax and pushed part of the qualified dividends into the 15% band." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used 2025-level standard deduction amounts ($15,000 base plus $2,000 each for age and blindness) and a rounded 'about $25,000' total, and it omitted the $1,000 non-itemizer charitable deduction. It then cut taxable income to about $40,000 with no basis and reported $1,971, which is far below the 10%/12% bracket tax on any taxable income in that range (about $4,500)." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of $43,546 was close to the correct figure despite an overstated $17,000 base standard deduction and the missing $1,000 charitable deduction. It then misapplied the rate schedule: $2,336 of tax on $41,722 of ordinary income is impossible under 2026 brackets (10% to $12,400, 12% to $50,400), which give about $4,759. It also charged $273 on qualified dividends that sit entirely in the 0% band." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026, which OBBBA prevented by making the TCJA structure permanent. Under that wrong assumption it used an $8,300 pre-TCJA standard deduction, itemized with a 2%-floor miscellaneous deduction for employee business expenses, subtracted a $5,300 personal exemption, and applied 15%/25% brackets. The correct law gives $27,200 of standard, senior and non-itemizer charitable deductions and 10%/12% brackets." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of gross income, putting AGI at $68,822 instead of $70,646. It then deducted only the $20,200 standard deduction, omitting the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, which overstated taxable income at $48,622." +us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the $20,200 standard deduction and the $6,000 senior deduction right. It missed the $1,000 charitable deduction that OBBBA gives single non-itemizers starting in 2026, so its taxable income was $44,446 instead of $43,446.18 and its tax was $120 (12% of $1,000) too high." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used an understated $19,200 standard deduction and omitted both the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. That produced $51,446 of taxable income instead of $43,446.18 and pushed qualified dividends into the 15% band." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules that OBBBA prevented: it itemized with a 2%-floor miscellaneous deduction and state income tax, took a $4,840 personal exemption, and taxed income at 15%. It never applied the $20,200 standard deduction, the $6,000 senior deduction, the $1,000 non-itemizer charitable deduction or the 12% bracket." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It treated 2026 as a post-TCJA-sunset year. It itemized miscellaneous employee expenses and uncapped SALT, subtracted a $5,300 personal exemption, and taxed ordinary income at 15% above $12,150. Under OBBBA the correct result comes from the $27,200 combined standard, senior and non-itemizer charitable deductions and 10%/12% brackets." +us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It stopped at the $20,200 standard deduction and omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $50,446 is $7,000 too high, which also pushed $996 of qualified dividends into the 15% band." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied projected post-TCJA-expiration rules: $18,126 of itemized deductions, a $5,200 personal exemption and a 25% bracket. OBBBA kept the TCJA brackets and added the $6,000 senior and $1,000 non-itemizer charitable deductions on top of the $20,200 standard deduction, for $43,446.18 of taxable income." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It overstated AGI at $71,468 and reported $69,455 of taxable income after a $29,600 deduction, an arithmetic impossibility. It then also subtracted $10,808 of itemized deductions on top of the standard deduction, which is double-counting. It never built the correct $27,200 deduction stack (standard, senior and non-itemizer charitable), so its $4,658 does not come from a valid taxable income." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $6,300 implies about $54,500 of ordinary taxable income at 10%/12%, which fits deducting only a basic standard deduction and omitting the age/blind additions, the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Those deductions reduce taxable income to $43,446.18." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used TCJA-sunset rules: $17,023 of itemized deductions including $2,097 of 2%-floor employee business expenses, a $5,150 personal exemption, and pre-TCJA brackets. It also rounded the capital loss to $1,500. OBBBA made the standard deduction, the 10%/12% brackets and the suspension of miscellaneous deductions permanent, and it added the senior and non-itemizer charitable deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It computed about $4,917 of tax before credits, then cut it to $0 by citing nonrefundable credits. None applies: there are no dependents, and the elderly/disabled credit is fully phased out at $70,646 of AGI with $4,800 of nontaxable Social Security, so tax before refundable credits stays positive." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It deducted only the basic plus age/blind standard deduction and omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction. Its $5,994 implies about $7,000 more taxable income than the correct $43,446.18." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It computed tax from itemized deductions instead of the larger $20,200 standard deduction plus the $6,000 senior and $1,000 non-itemizer charitable deductions, which overstated taxable income. It submitted $6,712 while its explanation ends with 6979, so the answer is inconsistent with itself as well." +us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used 2025 amounts ($15,000 base plus $2,000 each for age and blindness, $19,000 in total) instead of the 2026 $20,200. It also omitted the $6,000 senior deduction and the $1,000 non-itemizer charitable deduction, so taxable income was $51,646 and it taxed qualified dividends at 15% instead of 0%." +us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $1,824 of qualified dividends out of provisional income and AGI, which understated taxable Social Security at $26,289.75 when the correct figure is the $27,200 cap. It also used a $19,570 standard deduction and omitted the $6,000 senior and $1,000 non-itemizer charitable deductions, which overstated taxable income at $48,341.75." +us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It computed taxable Social Security as 0.85 times the excess over $34,000 ($23,340), dropping the $4,500 base amount. The statutory formula gives $27,840, which the $27,200 cap limits to $27,200. It also used $1,650 per age/blind addition instead of the single-filer $2,050. Both errors understated taxable income at $40,386, even though it correctly included the $6,000 senior and $1,000 charitable deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age/blind deductions fully offset taxable income. The full $27,200 of deductions (standard, senior and non-itemizer charitable) set against $70,646.18 of AGI still leaves $43,446.18 taxable, which produces $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It concluded that tax-exempt income and itemized deductions push tax to zero. Tax-exempt interest is not taxed and does not reduce tax, and $27,200 of deductions against $70,646.18 of AGI leaves $43,446.18 of taxable income taxed at 10%/12%." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'post-TCJA-sunset' rules: about $17,023 of itemized deductions including 2%-floor miscellaneous expenses, a $5,300 personal exemption and pre-TCJA brackets. OBBBA made the TCJA standard deduction and 10%/12% brackets permanent and added the $6,000 senior and $1,000 non-itemizer charitable deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly stacked the enhanced standard deduction, the senior deduction and the $1,000 non-itemizer charitable deduction. Its taxable income of $43,546 is $100 above $43,446.18 because its basic-plus-age/blind standard deduction fell $100 short of $20,200 (16,100 + 2 × 2,050). That $100 difference produced the $12 overstatement." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its explanation names the standard, age/blind and senior deductions but not the $1,000 non-itemizer charitable deduction. Its $4,957 implies about $43,375 of ordinary taxable income, roughly $1,750 above the correct $41,622.18 after the full $27,200 deduction stack." +us,scenario_122,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but omitted the $1,000 charitable deduction for single non-itemizers that starts in 2026. That left taxable income at about $44,465 instead of $43,446.18 and tax $122 too high." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed that taxable income after deductions falls below the filing threshold. With $70,646.18 of AGI and $27,200 of deductions, $43,446.18 remains taxable and produces $4,746.66 of tax, and the lack of earned income is irrelevant to that result." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It projected TCJA-sunset rules: itemized deductions including 2%-floor employee expenses and MN income tax, a $12,400 pre-TCJA standard deduction comparison, a $5,300 personal exemption and a 15% bracket. OBBBA made the $16,100-base standard deduction and 10%/12% brackets permanent and added the $6,000 senior and $1,000 non-itemizer charitable deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It itemized with miscellaneous employee expenses above the 2% floor, added a personal exemption and taxed income in a 15% bracket, all of which assume the TCJA expired. Under OBBBA the $27,200 standard, senior and non-itemizer charitable deductions and the 10%/12% brackets apply." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied the standard, age/blind and $6,000 senior deductions but omitted the $1,000 non-itemizer charitable deduction that starts in 2026. That left taxable income at $44,396 instead of $43,446.18. Small errors in the base ($16,150 vs $16,100) and the 10% bracket top ($12,250 vs $12,400) did not offset the $120 overstatement." +us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used 'post-TCJA reversion' rules: itemized deductions including 2%-floor miscellaneous expenses and state income tax, and 15% brackets indexed from 2017. OBBBA kept the TCJA 10%/12% brackets and gives $27,200 of standard, senior and non-itemizer charitable deductions." +us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It itemized uncapped SALT with MN income tax and 2%-floor miscellaneous deductions, subtracted a $5,050 personal exemption and taxed at 15%, all of which assume the TCJA expired. OBBBA made the TCJA structure permanent, and the $27,200 of standard, senior and non-itemizer charitable deductions with 10%/12% brackets yields $4,746.66." +us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score." +us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It admitted taxable income is positive, then zeroed the tax with 'personal exemptions/credits' that do not exist in 2026. The personal exemption is permanently $0 under OBBBA, and no nonrefundable credit applies at $70,646 of AGI, so $43,446.18 of taxable income produces $4,746.66 of tax." +us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its reasoning stacked the $20,200 standard deduction and the $6,000 senior deduction but omitted the $1,000 non-itemizer charitable deduction, which gave $44,446 of taxable income and $4,866.64 of tax. It then submitted $3,491.46, which contradicts its own computation." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted only a $20,050 standard deduction and omitted the $6,000 senior and $1,000 non-itemizer charitable deductions. It also treated the $1,447 long-term capital loss as preferentially taxed gain. Its reasoning reached $5,544, but it submitted $9,505, which no step supports." +us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It left all Social Security out of AGI, although $27,200 is taxable, and applied an elderly/disabled credit that is fully phased out at this AGI and nontaxable Social Security. It also added a charitable 'deduction' to the tax instead of subtracting $1,000 from taxable income, so its $5,065 exceeds its own $3,933.60 tentative tax." us,scenario_122,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 73, blind, and disabled as automatic categorical eligibility and conflated Medicare Savings Programs such as QMB and SLMB with full Medicaid eligibility. It also dismissed resource limits despite the disclosed $44,000 in bank assets and $177,000 in stock assets; the head qualified through none of Minnesota’s Medicaid pathways." us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found income far above the 138% FPL MAGI limit, then reversed that result solely because the head was elderly, blind, and disabled. Those characteristics do not automatically confer Medicaid eligibility, and the head qualified through no alternative category." us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base." us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract." us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Asserted Minnesota's Social Security subtraction is ""phased out at this income level, so little to none""; the phaseout begins well above this filer's $70,646 AGI, so the entire $27,200 of federally taxable Social Security is subtractable. It also itemized instead of taking the $19,250 aged-and-blind standard deduction plus the $4,977.18 non-itemizer charitable subtraction, leaving ~$52,000 of taxable income against the actual $19,219 and reaching into the 6.8% bracket that never applies." -us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Built the correct structure — full $27,200 Social Security subtraction, non-itemizer charitable subtraction, aged/blind standard deduction, 5.35% bracket — but overstated both deductions. It used a $19,500 standard deduction instead of $19,250 and computed the charitable subtraction as 50% x ($10,808 - $500) = $5,154 without first applying the 0.5%-of-AGI floor to the contributions, which yields $4,977.18; the $427 of excess deductions understated tax by $22.85." -us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started its Minnesota base at $43,446, a figure that already excludes all Social Security, then claimed a further Social Security exclusion, and used a $13,175 standard deduction rather than the $19,250 available to a single filer who is both 65+ and blind. Its own base less its own deduction leaves over $30,000 taxable, so the submitted $0 follows from no arithmetic at all." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Derived ~$1,040 — within $12 of the reference, including the full Social Security subtraction and the non-itemizer charitable subtraction — then discarded it by invoking a Minnesota ""senior subtraction"" and a ""credit for long-term care"" to drive the result to zero. Minnesota's age-related relief here is the Social Security subtraction it had already taken, and the long-term care credit requires qualifying LTC insurance premiums, which this household reports none of." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Proposed subtracting the full $32,000 of gross Social Security rather than the $27,200 federally taxable amount, itemized instead of taking the $19,250 aged/blind standard deduction with the $4,977.18 non-itemizer charitable subtraction, and then declared the result ""approximately 0"" without carrying the arithmetic. Its own ~$72,000 AGI less those amounts still leaves roughly $20,000 taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Claimed the Minnesota Social Security subtraction is ""limited given income"" when the full $27,200 is subtractable at $70,646 of AGI, and omitted the $4,977.18 non-itemizer charitable subtraction entirely. Its stated ~$44,000 of taxable income (and its submitted $1,650, which implies ~$30,800) sits far above the actual $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Invented a $15,000 Minnesota pension/IRA subtraction for filers age 65+; Minnesota's retirement subtractions are the Social Security subtraction and a public-pension subtraction, neither of which reaches this filer's $25,000 private pension or $14,400 IRA distributions. It also itemized $18,436 (including unreimbursed employee business expenses) rather than taking the $19,250 aged/blind standard deduction with the $4,977.18 non-itemizer charitable subtraction, so its $10,010 base misses $19,219 in both directions of error." -us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Set federal AGI at $43,446, a figure that already omits the $27,200 of federally taxable Social Security, then applied the Minnesota Social Security subtraction a second time on top of it and concluded taxable income falls ""near zero or below."" That base less even the full $19,250 standard deduction and $4,977.18 charitable subtraction leaves over $19,000 taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Never applied Minnesota's $27,200 Social Security subtraction or the $4,977.18 non-itemizer charitable subtraction, and added a ~$5,300 personal exemption that Minnesota grants only per dependent, of which this household has none. The resulting $50,146 of taxable income is 2.6x the actual $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Omitted the $27,200 Minnesota Social Security subtraction and the $4,977.18 non-itemizer charitable subtraction, and claimed a $5,094 personal exemption Minnesota allows only per dependent. Its $48,529 base pushed the calculation into the 6.8% bracket, while the actual $19,219 falls entirely within the 5.35% bracket." -us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Took the full $27,200 Social Security subtraction and the 5.35% bottom rate correctly, but omitted Minnesota's non-itemizer charitable subtraction of $4,977.18 and used an $18,500 standard deduction instead of $19,250. Its $24,946 of taxable income exceeds the actual $19,219 by $5,727, the exact sum of those two omissions." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Taxed the entire $70,646 federal AGI less a standard deduction with no Minnesota Social Security subtraction and no charitable subtraction, leaving $51,946 taxable, and applied a 7.05% second-bracket rate that does not exist in Minnesota (the second rate is 6.80%). The correct $19,219 base sits wholly inside the 5.35% bracket." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Gave no derivation; $2,420 corresponds to roughly $42,000-$45,000 of Minnesota taxable income, which is federal AGI less ordinary deductions with the $27,200 Social Security subtraction never applied. The $4,977.18 non-itemizer charitable subtraction is absent as well, against an actual base of $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied only a ""partial Social Security subtraction"" when this filer's $70,646 AGI sits below the phaseout threshold and the full $27,200 is subtractable, and omitted the $4,977.18 non-itemizer charitable subtraction. Its implied taxable income near $51,000 drove the calculation into the 6.8% bracket, which never applies to the actual $19,219 base." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Used a ""partial Social Security subtraction"" instead of the full $27,200 available at this AGI, and took no non-itemizer charitable subtraction, leaving $51,665 of estimated taxable income against the actual $19,219. Its $3,132 is roughly triple the correct $1,028.22 as a direct result." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Gave no derivation; $1,111 at Minnesota's 5.35% bottom rate implies $20,766 of taxable income, $1,547 above the actual $19,219. Its deductions therefore fell short of the $19,250 aged-and-blind standard deduction plus the $4,977.18 non-itemizer charitable subtraction that apply here." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Correctly subtracted the full $27,200 of federally taxable Social Security and used an $18,270 standard deduction, but omitted Minnesota's non-itemizer charitable subtraction of $4,977.18 (50% of gifts over $500 after the 0.5%-of-AGI floor). That single omission plus the $980 shortfall in the standard deduction left $25,176 taxable instead of $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Submitted $2,389 while its own explanation states $1,308, so the reported number is not the one it derived. Both are wrong: $1,308 corresponds to taking the $27,200 Social Security subtraction but omitting the $4,977.18 non-itemizer charitable subtraction, and $2,389 corresponds to never removing the Social Security subtraction at all." -us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Correctly subtracted the full $27,200 of federally taxable Social Security and used a $19,000 standard deduction with age-65 and blind additions, but omitted Minnesota's non-itemizer charitable subtraction of $4,977.18. That left $24,446 of taxable income instead of $19,219, overstating tax by $280." -us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied 2.85% and 4.80% bracket rates that do not exist in Minnesota, whose bottom rate is 5.35% and covers the whole $19,219 base. It also itemized $14,926 rather than taking the $19,250 aged-and-blind standard deduction and omitted the $4,977.18 non-itemizer charitable subtraction, producing $28,520 of taxable income; the two errors partly cancelled to land near the reference by accident." -us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Arrived at the right Minnesota AGI of $43,446 after the Social Security subtraction, but omitted the $4,977.18 non-itemizer charitable subtraction and took only an ~$18,560 standard deduction instead of $19,250. Its $24,886 of taxable income exceeds the actual $19,219, overstating tax by $303." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Claimed the standard deduction and ""senior/disabled-related subtractions"" fully eliminate Minnesota taxable income at this income level. Minnesota's senior relief here is the $27,200 Social Security subtraction, and after that, the $4,977.18 charitable subtraction and the $19,250 aged/blind standard deduction, $19,219 still remains taxable at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted the Minnesota result is non-positive and floored at zero with no derivation at all. The actual chain — $70,646 AGI less the $27,200 Social Security subtraction, less the $4,977.18 charitable subtraction, less the $19,250 aged/blind standard deduction — leaves $19,219 of taxable income." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Named every correct component but added the $2,013 of tax-exempt interest to the Minnesota base; the only adjustments to the $70,646 AGI here are the $27,200 Social Security and $4,977.18 charitable subtractions against a $19,250 standard deduction. Its $1,094 implies $20,449 of taxable income, $1,230 above the actual $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its ~$49,900 taxable-income estimate is federal AGI less about $20,700 of deductions, meaning the $27,200 Social Security subtraction it names was never actually removed from the base. The $4,977.18 non-itemizer charitable subtraction is missing as well, and the real base of $19,219 sits entirely in the 5.35% bracket rather than the higher rates its $2,918 requires." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $21,055 base reconstructs exactly as $43,446 + $2,013 tax-exempt interest - $5,154 charitable - $19,250 standard deduction: it added the tax-exempt interest to Minnesota income, which is not an addition here, and computed the charitable subtraction as 50% x ($10,808 - $500) without applying the 0.5%-of-AGI floor to contributions, which yields $4,977.18. Those two errors put taxable income $1,836 above the actual $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $2,640 implies roughly $49,000 of Minnesota taxable income — federal AGI less a standard deduction, with the $27,200 Social Security subtraction never actually applied despite being named and the $4,977.18 non-itemizer charitable subtraction omitted. The correct base is $19,219 taxed at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Reproduced the correct structure but overstated two amounts: a $19,400 standard deduction instead of $19,250, and a $5,154 charitable subtraction from 50% x ($10,808 - $500) instead of $4,977.18, because it never applied the 0.5%-of-AGI floor to the contributions. The $327 of excess deductions dropped taxable income to $18,892 and understated tax by $17.50." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Asserted Minnesota taxable income is zero ""after subtractions and credits"" with no computation shown. After the $27,200 Social Security subtraction, the $4,977.18 non-itemizer charitable subtraction, and the $19,250 aged-and-blind standard deduction, $19,219 remains taxable at the 5.35% rate." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Took the full $27,200 Social Security subtraction correctly but used a ~$19,600 standard deduction and omitted Minnesota's $4,977.18 non-itemizer charitable subtraction, which it explicitly ruled out by comparing itemized deductions to the standard deduction instead. That left $23,844 taxable rather than $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted the federally taxable Social Security correctly but omitted the $4,977.18 non-itemizer charitable subtraction, landing on ~$22,000 of taxable income instead of $19,219. Its roughly $21,400 of deductions exceeds the $19,250 standard deduction yet falls short of the $24,227 total allowed once the charitable subtraction is added." -us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Never applied Minnesota's $27,200 Social Security subtraction, and itemized $17,023 including miscellaneous employee expenses net of a 2% floor rather than taking the $19,250 aged/blind standard deduction plus the $4,977.18 charitable subtraction. Its $53,623 of taxable income triggered a 6.8%-bracket computation that never applies to the actual $19,219 base." -us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Put Minnesota taxable income ""near $50,000"" by starting from federal AGI and applying only generic deductions and exemptions: the $27,200 Social Security subtraction and the $4,977.18 non-itemizer charitable subtraction are both missing, and Minnesota grants no personal exemption to the filer. The actual base is $19,219, entirely at 5.35%." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing was submitted against the $1,028.22 reference. The failure is a missing output rather than a substantive computation error." -us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Correctly took the full $27,200 Social Security subtraction and a $19,425 aged/blind standard deduction, but then added a fabricated ~$5,330 personal exemption — Minnesota's exemption is per dependent and this filer has none — while omitting the $4,977.18 non-itemizer charitable subtraction. The two errors nearly cancelled, leaving $18,690 taxable instead of $19,219." -us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Invoked a Minnesota ""senior age-65 subtraction"" that zeroes out taxable income; the state's age-related relief here is the Social Security subtraction, and it also treated Social Security as only ""partially taxable"" at the federal level while never carrying the $27,200 into the base. Subtracting $27,200 and $4,977.18 from $70,646 of AGI and taking the $19,250 standard deduction leaves $19,219 taxable." -us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Itemized $17,367 — including $2,441 of unreimbursed employee business expenses after a 2% floor — instead of taking the $19,250 aged-and-blind standard deduction together with Minnesota's $4,977.18 non-itemizer charitable subtraction. That left $26,079 of taxable income against the actual $19,219, overstating tax by $367." -us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Added back $2,013 of tax-exempt interest and $60 of tax-exempt pension, never applied the $27,200 Minnesota Social Security subtraction, and used a $14,600 standard deduction with no age-65 or blind additions, reaching $58,119 of taxable income and $3,492.59 of tax. It then cut that to $2,151.08 with a property-tax credit, but Minnesota's property tax refund is a separate refundable program and does not reduce income tax before refundable credits." -us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Used a fabricated $52,092 federal AGI and a $30,100 Minnesota standard deduction when the figures are $70,646 and $19,250, and its stated $21,992 of taxable income reconciles with neither the $27,200 Social Security subtraction nor the $4,977.18 charitable subtraction. Its own base at Minnesota's 5.35% rate yields $1,176.57, not the $1,450 it submitted." -us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed Minnesota household income correctly at about $77,500 and even noted this was ""near the income limit,"" then failed to apply the limit itself: $77,459 exceeds the top household-income bracket of the 2026 renter's credit schedule under Minn. Stat. 290.0693, which terminates the credit rather than merely shrinking it. It also invented a ""50% of the excess"" refund rate and then raised its own $1,470 estimate to $2,265 with no rule behind the increase, landing near the schedule's low-income maximum refund instead of the few-hundred-dollar maximum that applies in the highest brackets." -us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It asserted qualification for the Minnesota renter's credit and jumped straight to ""the maximum allowable refund,"" never computing household income under Minn. Stat. 290A.03 subd. 3 ($70,646 AGI + $4,800 nontaxable Social Security + $2,013 tax-exempt interest = $77,459), which is above the top bracket of the 2026 credit schedule. The maximum refund it applied is the figure for the lowest income brackets; the schedule's state share declines to a few hundred dollars before the credit terminates, so $2,000 is unreachable at any income this household could have." -us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It applied a ""senior/disabled household-income schedule"" to the renter's credit; the credit has one schedule for all claimants, and 290A.03 subd. 3 grants a household-income subtraction only for dependents, none for age 65+ or disability. Working from correct rent-constituting-property-tax of $4,800 (17% of $28,235), it never tested the resulting household income of $77,459 against the 2026 schedule's maximum household income, which the household exceeds, so the credit is $0 rather than $1,070." -us,scenario_122,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It built its $2,228 from a ""senior/disabled household-income adjustment"" that does not exist in the renter's credit computation — household income under 290A.03 subd. 3 adds back nontaxable Social Security ($4,800) and tax-exempt interest ($2,013) to AGI of $70,646 with no age or disability subtraction, giving $77,459. That figure is past the top household-income bracket of the 2026 schedule, so the credit is fully phased out; the amount it reported is close to the schedule's maximum refund for the lowest-income claimants." -us,scenario_122,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It derived federal AGI of $70,646 correctly but then applied an ""elderly/disabled income subtraction"" to reach a qualifying income figure, when Minn. Stat. 290A.03 subd. 3 instead requires adding back $4,800 of nontaxable Social Security and $2,013 of tax-exempt interest, yielding household income of $77,459. That income is above the top bracket of the 2026 renter's credit schedule, which terminates the credit, so the correct amount is $0, not the $1,050 it produced from an income-based cap inside the table." -us,scenario_122,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for state_refundable_credits, so the submission failed the output contract rather than the substantive computation. The correct derivation yields $0: Minnesota household income of $77,459 exceeds the 2026 renter's credit income ceiling, and the working family, child, K-12 education, and child care credits all require earned income or children this household does not have." -us,scenario_123,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable child1_chip_eligible value or explanation, violating the required output contract." -us,scenario_123,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated age 16 as satisfying Pennsylvania CHIP’s age rule and assumed paid CHIP extends eligibility regardless of income. Under the traced Pennsylvania program structure, the child fails the enrollment age cutoff, and household income of approximately $145,002 also exceeds the CHIP threshold." -us,scenario_123,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model declared eligibility solely from the child’s age without applying Pennsylvania’s specific CHIP enrollment cutoff or the income test. The child fails the program’s age criterion, and the approximately $145,002 household income independently exceeds the CHIP threshold." +us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It treated Minnesota's Social Security subtraction as phased out at this income, but $70,646 AGI is below the single-filer threshold, so the full $27,200 is subtracted. It also claimed itemized deductions beat the standard deduction, though charitable plus property tax ($14,926) is less than the $19,300 aged/blind standard deduction. That kept about $52,000 taxable and pushed income into the 6.8% bracket instead of $19,169 taxed at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The structure was right, but two deduction amounts were too high. It used a $19,500 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $18,792 instead of $19,169, so tax was $1,005.37 instead of $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Its own numbers leave about $30,000 of Minnesota taxable income: $43,446 of non-Social Security income minus a $13,175 standard deduction. It still reported $0 with no deduction or credit to support it. It also understated the aged/blind standard deduction ($19,300) and missed the $4,977.18 nonitemizer charitable subtraction, which together leave $19,169 taxed at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,age_disability,False,"It correctly reached about $19,442 of taxable income and about $1,040 of tax. It then zeroed the tax by applying a Minnesota senior/elderly-disabled subtraction and a long-term-care credit. The elderly/disabled subtraction phases out entirely at $70,646 AGI, and the facts support no long-term-care credit. The tax is $19,169 × 5.35% = $1,025.54." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It never computed taxable income. It asserted that the Social Security subtraction plus deductions bring income near zero, and it wrongly treated the standard deduction as a nonrefundable credit. Subtracting $27,200 of Social Security, the $19,300 standard deduction and the $4,977.18 charitable subtraction from $70,646.18 AGI still leaves $19,169 taxed at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated Minnesota's Social Security subtraction as only partial because of income limits. At $70,646 AGI the full $27,200 is subtracted. It also skipped the $4,977.18 nonitemizer charitable subtraction and understated the aged/blind standard deduction. Its $1,650 implies about $30,800 of taxable income instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $19,300 standard deduction and the full $27,200 Social Security subtraction. But it computed the charitable subtraction as 50% × ($10,808 − $500) = $5,154, while PolicyEngine's subtraction is $4,977.18. Taxable income came out at $18,992 instead of $19,169, which accounts for the $9 shortfall." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It invented a $15,000 Minnesota subtraction for private pension and IRA income at age 65+. Minnesota has no such subtraction for this income, and the error removed $15,000 that is actually taxable. It also used a $14,575 standard deduction without the age and blind additions, so it chose itemizing at $18,436. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It left taxable Social Security out of federal AGI and then described the Social Security subtraction as shrinking income further, so it double-counted the exclusion. It also invoked a Minnesota disability/elderly credit to reach $0. The elderly/disabled subtraction is zero at this AGI, and $43,446 minus the $19,300 standard deduction and the $4,977.18 charitable subtraction leaves $19,169 taxed at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It correctly subtracted all $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,400 standard deduction instead of $19,300. That left $24,046 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It skipped the full $27,200 Minnesota Social Security subtraction. It also subtracted a personal exemption of about $5,300, which Minnesota does not give filers, and used a standard deduction without the age and blind additions. That produced $50,146 of taxable income reaching the 6.8% bracket instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction and invented a $5,094 personal exemption. It also used $17,023 of itemized deductions, which is less than the $19,300 aged/blind standard deduction. Taxable income came out at $48,529 in the 6.8% bracket instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It correctly subtracted $27,200 of Social Security. It then used an $18,500 standard deduction instead of $19,300 and left out the $4,977.18 nonitemizer charitable subtraction. That left $24,946 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted an invented $5,000 retirement subtraction, which does not exist for private pension and IRA income. It also used an $18,500 standard deduction instead of $19,300 and left out the $4,977.18 charitable subtraction. Its $19,946 of taxable income is close only because these errors partly offset." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied only the standard deduction to the full $70,646 AGI. It skipped the $27,200 Social Security subtraction and the $4,977.18 charitable subtraction. It also used 7.05% as the second-bracket rate, but Minnesota's second rate is 6.8%. The result was $51,946 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $2,420 implies about $42,500 of taxable income reaching the 6.8% bracket. That means it did not combine the full $27,200 Social Security subtraction, the $19,300 aged/blind standard deduction and the $4,977.18 charitable subtraction, which together leave $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied only a partial Social Security subtraction. With $70,646 AGI, below the single-filer phaseout, Minnesota subtracts the full $27,200. Keeping that income taxable pushed it into the 6.8% bracket, and it also left out the $4,977.18 charitable subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It called its Social Security subtraction partial, but its $51,665 of taxable income reflects essentially no subtraction from the roughly $70,665 AGI. The full $27,200 should have been subtracted. It also left out the $4,977.18 charitable subtraction, taxing income at 6.8% instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,111 implies about $20,770 of taxable income, roughly $1,600 above the correct $19,169. That means it under-applied the combined $19,300 standard deduction and $4,977.18 charitable subtraction after the $27,200 Social Security subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly subtracted $27,200 of Social Security. It then used an $18,270 standard deduction instead of $19,300 and left out the $4,977.18 nonitemizer charitable subtraction. That left $25,176 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It submitted $2,389, but its explanation ends with value = 1308, so the answer contradicts itself. The $1,308 figure matches $24,446 × 5.35%: it subtracted Social Security but used a $19,000 standard deduction and left out the $4,977.18 charitable subtraction. Neither number reflects the correct $19,169 × 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It correctly subtracted the full $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,000 standard deduction instead of $19,300. That left $24,446 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied rates of 2.85% and 4.80%, which are not Minnesota rates; Minnesota's first bracket is 5.35%. It used a $14,600 standard deduction without the age and blind additions, so it chose $14,926 of itemized deductions over the correct $19,300 standard deduction. It also left out the $4,977.18 charitable subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used an $18,560 standard deduction instead of $19,300. It also left out the $4,977.18 nonitemizer charitable subtraction. That left $24,886 taxable instead of $19,169. Its understated taxable Social Security ($23,340) canceled out through the subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It assumed senior/disabled subtractions wipe out Minnesota taxable income. The elderly/disabled subtraction phases out to zero at $70,646 AGI. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction, $19,169 remains taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It asserted that tax after nonrefundable credits is zero without computing taxable income. Minnesota taxable income is $70,646.18 − $27,200 − $19,300 − $4,977.18 = $19,169, which produces $1,025.54 at 5.35%, and no nonrefundable credit applies." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine's calculation does not include. That raised taxable income to about $20,450 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It said it applied the Social Security subtraction, but its $49,900 of taxable income equals AGI minus only a standard deduction, so the $27,200 subtraction was never taken. It also left out the $4,977.18 charitable subtraction and taxed income in the 6.8% bracket." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine does not include. It also used a $5,154 charitable subtraction instead of $4,977.18. That raised taxable income to $21,055 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $2,640 implies about $45,500 of taxable income reaching the 6.8% bracket. That matches AGI minus the standard and charitable deductions with no Social Security subtraction. The full $27,200 subtraction applies at this AGI." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The structure was right, but it used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $18,892 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It applied the Social Security subtraction and an aged/blind standard deduction, but it left out the $4,977.18 nonitemizer charitable subtraction. That left $23,265 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 of taxable income means it overstated its deductions by about $827. It computed the charitable subtraction as $5,154 instead of $4,977.18, and its standard deduction was roughly $650 above $19,300." +us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted $27,200 of Social Security and a $19,400 standard deduction, but it left out the $4,977.18 nonitemizer charitable subtraction. It also used $19,400 instead of the $19,300 standard deduction, leaving $24,046 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,age_disability,False,"It claimed age-based exemptions zero out Minnesota taxable income. Minnesota gives filers no personal exemption, and the elderly/disabled subtraction is zero at $70,646 AGI. $19,169 remains taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It correctly subtracted all $27,200 of Social Security but left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,600 standard deduction instead of $19,300. That left $23,844 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Its $22,000 of taxable income means it did not combine the $19,300 aged/blind standard deduction with the separate $4,977.18 nonitemizer charitable subtraction after subtracting Social Security. Those together leave $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which PolicyEngine does not include. It also used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $20,905 instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction. It also itemized $17,023, which is below the $19,300 aged/blind standard deduction, and left out the charitable subtraction. That taxed $53,623 across the 5.35% and 6.8% brackets instead of $19,169 at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It started near the full $70,600 AGI and reached about $50,000 of taxable income. So it never took the $27,200 Social Security subtraction or the $4,977.18 charitable subtraction and wrongly taxed income at 6.8%." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score." +us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It subtracted a $5,330 personal exemption, which Minnesota does not give filers, and left out the $4,977.18 nonitemizer charitable subtraction. It also used a $19,425 standard deduction. Its $18,690 of taxable income lands near $19,169 only because the invented exemption offsets the missing charitable subtraction." +us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It applied a senior age-65 subtraction to reach zero taxable income. That subtraction phases out to nothing at $70,646 AGI. The Social Security subtraction, the $19,300 standard deduction and the $4,977.18 charitable subtraction leave $19,169 taxable at 5.35%." +us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a $15,300 standard deduction without the $4,000 of age and blind additions, so it chose $17,367 of itemized deductions. The correct choice is the $19,300 standard deduction plus the $4,977.18 charitable subtraction. That left $26,079 taxable instead of $19,169." +us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction and added back tax-exempt interest and tax-exempt pension income. It also used a $14,600 standard deduction without the age and blind additions. It then cut the tax with the property tax refund, which is a separate refund program and not a nonrefundable credit against income tax." +us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It misstated federal AGI as $52,092 instead of $70,646 and used a made-up $30,100 standard deduction instead of $19,300. It also left out the Social Security and charitable subtractions. Its reported $1,450 does not even equal 5.35% of its own $21,992 taxable income." +us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It computed household income correctly at about $77,500 and even said that was near the income limit. It still applied a 2.4% threshold with a 50% state copayment to $4,800 of rent-as-tax and got about $1,470. It then raised that to $2,265 while saying the phaseout reduces the refund. At this household income the 2026 MN renter's credit is $0, so no refundable state credit applies." +us,scenario_122,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It found the right household income ($77,459) and rent-as-tax ($4,800). It then applied a mid-schedule bracket: a 2.5% income threshold with a 50% state share, and no maximum-refund cap. The MN renter's credit schedule is fully phased out at this income, so the credit is $0, not $1,431.74." +us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It assumed the head qualifies for the Minnesota renter's refund and simply entered a $2,000 'maximum allowable refund'. It never checked the household income of about $77,460 against the renter's credit income schedule. That schedule pays nothing at this income, so state refundable credits are $0." +us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It treated 17% of rent (about $4,800) as property tax and applied a senior/disabled income schedule with a high-income copayment, leaving $1,070. The MN renter's credit gives $0 at about $77,460 of household income, so the copayment does not leave any refundable credit." +us,scenario_122,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It estimated a $2,228 renter's credit using a senior/disabled household-income adjustment and a paying bracket of the schedule. With nontaxable Social Security and tax-exempt interest included, household income is about $77,460. At that income the Minnesota renter's credit is fully phased out and the state refundable credit total is $0." +us,scenario_122,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It measured income with federal AGI ($70,646) less an elderly/disabled subtraction. That leaves out the nontaxable Social Security and tax-exempt interest that count toward Minnesota household income, which is about $77,460. The lower income put the household in a paying bracket worth $1,050. At the correct household income the renter's credit is $0." +us,scenario_122,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_refundable_credits, so there is no answer to score. The correct derivation gives $0 because the Minnesota renter's credit is fully phased out at about $77,460 of household income, and no other refundable state credit applies." +us,scenario_123,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no child1_chip_eligible value and no explanation, so its output is missing rather than a wrong eligibility call. The correct derivation is 0: household MAGI of about $145,002, which includes the child's $45,000 of wages, is over 500% FPL and far above Pennsylvania's 314% FPL CHIP ceiling." +us,scenario_123,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model counted Pennsylvania's paid CHIP for higher-income families as eligibility and never applied the CHIP income ceiling. PolicyEngine's CHIP eligibility requires household MAGI at or below Pennsylvania's 314% FPL limit. This household is at about $145,002, over 500% FPL for three people, because the child's $45,000 of wages counts in MAGI. The child is therefore ineligible (0), not 1." +us,scenario_123,child1_chip_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model based eligibility on Pennsylvania's full-cost CHIP option having 'no upper family-income limit'. PolicyEngine's is_chip_eligible does not count that unsubsidized buy-in. It applies the 314% FPL CHIP income ceiling, and household MAGI of about $145,002 (over 500% FPL, including the child's $45,000 of wages) fails it. The answer is 0, not 1." +us,scenario_123,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model decided CHIP eligibility from age 16 alone and never ran the income test. Household MAGI is about $145,002: $100,000 head wages, the child's $45,000 wages (the child must file), and $2 of interest. That is over 500% FPL for three people and far above Pennsylvania's 314% FPL CHIP limit, so the child is ineligible (0)." us,scenario_123,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took the $32,200 joint standard deduction but omitted the 2026 §170(p) $2,000 non-itemizer cash charitable deduction, leaving taxable income at $67,802 instead of $65,802 and tax at $7,640 instead of $7,400.24. It then computed the dependent child's separate $3,220 liability and declared a household total of $8,660, while submitting $5,860 — a figure that matches neither that total nor its own parents-only $5,440." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It placed the 16-year-old dependent's $45,000 of wages into the filers' joint AGI, producing $145,002 where PolicyEngine's tax-unit AGI counts head and spouse income only, for $100,002. Every other step — the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction, the 2026 joint brackets, and the $2,200 nonrefundable CTC — is exact, so that single $45,000 overstatement produces the entire $6,400.20 gap." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied the 2024 $29,200 joint standard deduction, invented $25,200 of mortgage interest by pricing the $420,000 balance at an assumed rate when no interest amount is listed (so it is 0), and counted two qualifying children at $2,000 each when the household has one child entitled to the 2026 $2,200 credit. Its submitted $8,316 follows from none of the three chains it wrote, which ended at $4,300, $2,600, and $2,600." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It structured the return correctly — dependent's wages excluded, standard deduction taken over $18,557 of itemized — but used wrong 2026 parameters: a $31,500 standard deduction instead of $32,200, no §170(p) $2,000 non-itemizer charitable deduction, and a $2,000 CTC instead of $2,200. Those three gaps account for exactly the $524 by which its own $5,724 exceeds $5,200.24; it then discarded that figure and submitted an unexplained $5,316." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It manufactured $29,400 of mortgage interest by applying a 7% rate to the $420,000 balance — no interest amount is listed, so the input is 0 — which flipped the return to itemizing $47,957 instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction. It then denied the $2,200 CTC on a support-test theory that does not disqualify the child here, and submitted $12,053 after its own arithmetic produced $5,765." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It correctly excluded the dependent's wages and took the $32,200 standard deduction, but omitted the 2026 §170(p) $2,000 non-itemizer cash charitable deduction that a filer taking the standard deduction receives, leaving taxable income at $67,802 rather than $65,802. It also estimated the bracket tax at '≈7,610' instead of computing $24,800×10% + $41,002×12% = $7,400.24, so its post-CTC $5,410 misses on both counts." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After correctly abandoning its TCJA-sunset path, it rebuilt 2026 from indexed guesses rather than enacted figures: a ~$30,750 standard deduction instead of $32,200, $24,450/$99,400 bracket thresholds instead of $24,800/$100,800, a $2,000 CTC instead of $2,200, and no §170(p) $2,000 non-itemizer charitable deduction. Each error pushes liability the same direction, turning $5,200.24 into $5,821." -us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It computed itemized deductions of $18,557, stated that the ~$32,200 standard deduction exceeds them, and then subtracted the $18,557 anyway, giving taxable income of $81,445 instead of $65,802 and skipping the $2,000 non-itemizer charitable deduction. It then reported a $7,300 conclusion in its explanation while submitting $15,300, a number no step of its work produces." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the joint AGI for $145,002, when PolicyEngine's tax-unit AGI uses filer income only and equals $100,002. It compounded that with a $30,800 standard deduction instead of $32,200, no $2,000 non-itemizer charitable deduction, and a $2,000 CTC instead of $2,200; the child-wage inclusion alone accounts for about $6,400 of the $7,579 error." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It summed two returns — parents' $7,778.25 plus the child's own $4,979.25 — when this output is the single tax unit's liability and the dependent's $45,000 of wages never enter it. It also ran 2026 under repealed pre-TCJA law with three $5,055 personal exemptions, 10%/15% brackets, and a $1,000 CTC, instead of the $32,200 standard deduction, $2,000 non-itemizer charitable deduction, 10/12/22 brackets, and $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It added the child's separate $4,896.25 return to a parents' figure of $7,154.99, while the requested value covers only the head-and-spouse tax unit, from which the dependent's $45,000 of wages are excluded. Its parents' return also used three $5,300 pre-TCJA personal exemptions and a $1,000 CTC, and itemized $21,627 — a total that both omits OBBBA's 0.5%-of-AGI charitable floor and loses to the $32,200 standard deduction plus $2,000 charitable add-on." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It filed the child separately for $4,955 of tax and added that to the parents' $4,826.25, but the output is the filers' tax unit alone at $5,200.24. Its parents' return also assumed TCJA sunset ($15,000 of personal exemptions, a $1,000 CTC) and invented $16,800 of mortgage interest from the $420,000 balance plus $5,527 of SALT, when interest is unlisted and therefore 0 and the deductible PA income tax on filer wages is $3,070.06." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It built the joint return on $145,002 of AGI, folding in the dependent 16-year-old's $45,000 of wages that PolicyEngine keeps out of filer AGI, and applied no child tax credit at all. Its $17,290 corresponds to roughly $126,700 of taxable income at 2026 joint rates with zero credit offset, against the correct $65,802 of taxable income, $7,400.24 of tax, and a fully used $2,200 nonrefundable CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It combined a parents' return of $7,211 with the child's own $4,925 single return, but the requested output is one tax unit's liability and excludes the dependent's $45,000 of wages and the tax on them. Its parents' figure also rests on pre-TCJA personal exemptions and a $1,000 CTC rather than 2026's $32,200 standard deduction, $2,000 non-itemizer charitable deduction, and $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It added the child's separate $5,023 return to the parents' $7,784 to reach $12,807, when the output is the filers' tax unit alone. Its parents' computation applied post-TCJA-expiration rules — $15,150 of personal exemptions, itemized $18,557, a $1,000 CTC — instead of the 2026 $32,200 standard deduction plus $2,000 non-itemizer charitable deduction and $2,200 CTC that yield $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted a bare assertion with no derivation. Its $5,304 is exactly the 2026 joint-bracket tax on $65,000 of taxable income ($7,304) less a $2,000 credit, so it used the pre-OBBBA $2,000 CTC instead of $2,200 and a $35,002 deduction stack instead of the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction that leaves $65,802 taxable." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It used $145,002 of AGI, pulling the dependent's $45,000 of wages into the filers' return that PolicyEngine computes from head and spouse income only ($100,002). It then itemized $23,008.56 with personal exemptions that 2026 law does not provide and zeroed the child credit as 'phased out due to income', when the 2026 joint CTC phaseout starts at $400,000 and the full $2,200 offsets the $7,400.24 liability." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It set AGI at $145,002, including the dependent child's $45,000 of wages that belong to the child rather than the filers' tax unit, and layered on personal exemptions repealed through 2026. It also itemized $18,557 in place of the larger $32,200 standard deduction plus $2,000 non-itemizer charitable deduction and applied no $2,200 CTC against the resulting liability." -us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It put the child's $45,000 into household AGI for $145,002 and used a $30,000 standard deduction with 2025 bracket thresholds ($23,850/$96,950) rather than 2026's $32,200 deduction, $2,000 non-itemizer charitable deduction, and $24,800/$100,800 thresholds, producing $15,128.44 of regular tax against $7,400.24. Only its $2,200 CTC step is right, so the full $7,928 error is income and deduction misstatement." -us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It split the household into a parents' return and a child's return and summed them, when the requested output is the single tax unit's liability in which the dependent's $45,000 of wages never appear. Its parents' figure of $5,824.38 also overstates by using a $30,750 standard deduction with no $2,000 non-itemizer charitable deduction and a $2,000 CTC instead of $2,200, and its stated total is $1,000 below its own components ($5,824.38 + $3,312.08 = $9,136.46, not $8,136.46)." -us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It merged the dependent's $45,000 of wages into the joint AGI ($145,002) and then claimed both the $32,200 MFJ standard deduction and a separate $16,100 dependent standard deduction against that one return. It also declared the $2,200 CTC 'fully refundable' and applied no nonrefundable offset, when the credit is nonrefundable up to liability and the $7,400.24 of pre-credit tax absorbs the entire $2,200." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It treated the joint return as having $145,001 of wages, folding the dependent child's earnings into the filers' AGI where PolicyEngine counts $100,002, and reached $116,751 of taxable income against the correct $65,802. Its submitted $7,313 does not follow even from its own taxable income, which yields about $15,109 of tax and $12,909 after the CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It gave no derivation; its $14,677 is the 2026 joint-bracket tax on roughly $114,800 of taxable income with no child credit — that is, $145,002 of AGI including the dependent's $45,000 of wages, less about a $30,000 standard deduction. Filer AGI is $100,002, the deduction stack is $34,200 ($32,200 standard plus $2,000 non-itemizer charitable), and the $2,200 CTC is fully used." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used one joint tax unit with $145,002 of AGI, adding the 16-year-old's $45,000 of wages that PolicyEngine excludes from filer AGI, and also omitted the §170(p) $2,000 non-itemizer cash charitable deduction that stacks on top of the $32,200 standard deduction. Its $32,200 deduction, bracket math, and $2,200 CTC steps are otherwise correct, so taxable income of $112,802 replaces the correct $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"Every parameter it used is right — the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction, the 2026 joint brackets, and the $2,200 nonrefundable CTC — but it computed AGI as $145,002, including the dependent child's $45,000 of wages. PolicyEngine's tax-unit AGI is filer-only at $100,002, making taxable income $65,802 rather than $110,802." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It included the child's $45,000 in joint income and used a $30,000 standard deduction with 2025 brackets rather than 2026's $32,200 deduction plus $2,000 non-itemizer charitable deduction and $24,800/$100,800 thresholds, producing $15,128.44 of regular tax against $7,400.24. Only the $2,200 CTC subtraction is correct." -us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It computed the filers' tax unit exactly right — $65,802 of taxable income after the $32,200 standard deduction and $2,000 non-itemizer charitable deduction, $7,400.24 of tax, $5,200.24 after the $2,200 CTC — and then added the dependent child's separate $3,220 return. This output is the tax unit's own liability; the dependent's $45,000 of wages and the tax on them never enter it." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It rounded to '$14k' from $145,002 of income that folds in the dependent child's $45,000 of wages, when filer AGI is $100,002, and claimed mortgage interest deductions from the $420,000 balance although no interest amount is listed and unlisted numeric inputs are 0. The $32,200 standard deduction plus $2,000 non-itemizer charitable deduction beats the $21,127.05 of itemized deductions, giving $7,400.24 of tax and $5,200.24 after the $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It added the child's separate ~$4,851 return to a parents' $7,068, while the requested value covers only the filers' tax unit and excludes the dependent's $45,000 of wages. Its parents' return applied post-sunset 2026 law — indexed personal exemptions and a $1,000 CTC — and itemized $21,627 (omitting the 0.5%-of-AGI charitable floor), instead of the $32,200 standard deduction, $2,000 non-itemizer charitable deduction, and $2,200 CTC." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It built a single MFJ unit with $145,002 of AGI that includes the dependent's wages, when PolicyEngine's filer-only AGI is $100,002, and applied restored pre-TCJA 10/15/25 brackets with three $5,300 personal exemptions. It then zeroed the CTC on the unindexed $110,000 phaseout threshold, while 2026 law sets the joint phaseout start at $400,000 so the full $2,200 credit offsets the $7,400.24 liability." -us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It added the child's own $4,846 return to the parents' $7,533, splitting one requested tax-unit value into two filings. Its parents' computation also used pre-TCJA 2026 rules — $16,302 of personal exemptions, a 10%/15% schedule, a $1,000 CTC, and a 10%-of-AGI medical floor — and itemized $18,557 rather than taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction." -us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"Its parents' computation is essentially exact — $65,802 of taxable income after the $32,200 standard deduction and $2,000 non-itemizer charitable deduction, roughly $7,410 of tax, $5,210 after the $2,200 CTC — and the sole error is adding the dependent child's separate $3,225 return. The requested output is the filers' tax-unit liability, which excludes the child's $45,000 of wages and any tax computed on them." -us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for this variable, so no substantive computation reached the grader. The required figure is $7,400.24 of tax on $65,802 of taxable income less the $2,200 nonrefundable CTC, or $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It included the dependent's $45,000 of wages in joint income for $145,002 of AGI and invented a $12,000 'qualified overtime premium deduction' from the head's 55-hour schedule; the OBBBA deduction covers only the FLSA-required premium above the regular rate, no such premium is listed, and unlisted numeric inputs are 0. It also omitted the $2,000 non-itemizer charitable deduction, so its $100,802 of taxable income replaces the correct $65,802." -us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 on the theory that liability is 'fully offset by refundable credits (CTC)', but this output is defined as tax after nonrefundable credits and before refundable ones, and the $2,200 CTC here is entirely nonrefundable because the $7,400.24 pre-credit liability exceeds it. The correct value is $7,400.24 − $2,200 = $5,200.24." -us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It set AGI at $145,002, pulling the dependent's $45,000 of wages into the filers' return where PolicyEngine counts $100,002, and imputed a $12,500 overtime deduction from 15 hours a week above 40 at a $17.50 premium — the stated gross wages already include any overtime, no overtime premium input is listed, so it is 0. It also skipped the $2,000 non-itemizer charitable deduction; its $32,200 standard deduction and $2,200 CTC steps are correct." -us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It added the child's $45,000 to joint wages for $145,002 of AGI and fabricated $27,300 of mortgage interest at an assumed 6.5% on the $420,000 balance, flipping the return to $45,857 of itemized deductions when actual itemized deductions total $21,127.05 and lose to the $34,200 standard-plus-charitable stack. It further used 2025 bracket thresholds and a $2,000 CTC in place of the 2026 $24,800/$100,800 thresholds and $2,200 credit." -us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It stacked deductions that do not exist — a $2,250 QBI deduction on the child's W-2 wages, $16,800 of imputed mortgage interest from a balance with no listed interest, and $125 of medical expenses above a 7.5% floor that $2,100 of expenses cannot clear — and simultaneously claimed both the $32,200 standard deduction and itemized items. It then erased its own $7,687 of tax with a Child and Dependent Care Credit despite zero care expenses and an irrelevant foreign earned income exclusion, submitting $0 where the answer is $7,400.24 less the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the 2026 OBBBA charitable deduction for non-itemizers ($2,000 on a joint return). It therefore taxed $67,802 instead of $65,802 and got $5,440 for the parents instead of $5,200.24. It then added the child's separate tax and submitted $5,860, which matches neither its $5,440 parent-only result nor its $8,660 combined total." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It put the dependent child's $45,000 of wages into joint AGI ($145,002), but the tax unit's AGI counts only the head's and spouse's income ($100,002). Its deductions ($32,200 plus $2,000), 2026 brackets and $2,200 CTC were right. The extra $45,000 of taxable income ($110,802 vs $65,802) explains the whole $6,400.20 overstatement." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the outdated $29,200 standard deduction and invented $25,200 of mortgage interest from the $420,000 balance, although no interest paid is listed, so mortgage interest is $0. It also claimed a $4,000 CTC for a second child who does not exist. The correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, $65,802 of taxable income and one $2,200 CTC; its $8,316 does not follow from any of its own steps." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $31,500 instead of the 2026 amount of $32,200, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of the 2026 amount of $2,200, reaching $5,724. It then submitted $5,316 after an unexplained 'adjustment' that its computation does not support." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It invented $29,400 of mortgage interest from the balance alone, although no interest paid is listed. It also dropped the $2,200 CTC by wrongly deciding the 16-year-old fails the support test, when the child is the couple's qualifying child. Its $12,053 matches neither its own ~$5,765 figure nor the correct path: $32,200 standard plus $2,000 charitable deduction, $7,400.24 of tax, minus the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $2,000 charitable deduction that non-itemizing joint filers get in 2026, so it taxed $67,802 instead of $65,802. It also mis-added the bracket tax ($7,610 instead of $7,640.24 on its own base) and ended at $5,410 instead of $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002), but the parents' tax unit AGI is only $100,002. With the right $32,200 standard deduction, $2,000 charitable deduction and $2,200 CTC, that $45,000 of extra taxable income is the entire gap between its $11,600.44 and $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated 2026 figures by inflating 2025 values: a $30,750 standard deduction, a 10% bracket up to $24,450 and a $2,000 CTC. The enacted OBBBA figures are $32,200, $24,800 and $2,200. It also left out the $2,000 charitable deduction, producing $5,821 instead of $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It stated that $18,557 of itemized deductions beat the ~$32,200 standard deduction it had just cited and itemized. The correct deduction is the $32,200 standard deduction plus the $2,000 charitable deduction ($34,200 total). It then submitted $15,300, which contradicts its own estimate of $7,300." +us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It gave the head a $13,650 overtime deduction, but only Child 1 is flagged as paid hourly, so the head has no qualifying overtime pay and gets no overtime deduction. It also added the child's $3,220 separate tax, even though the output is the joint unit's liability alone ($7,400.24 minus the $2,200 CTC = $5,200.24)." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002 instead of $100,002). It also used a $30,800 standard deduction instead of $32,200, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the expired pre-TCJA rules: personal exemptions, 10%/15% brackets and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 standard deduction plus a $2,000 charitable deduction, no exemptions, 10%/12% brackets and a $2,200 CTC. It also added $4,979.25 of separate tax for the child, although the output covers only the joint unit." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three $5,300 personal exemptions, a 15% bracket and a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's $4,896.25 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules: itemized deductions plus $15,900 of personal exemptions, 10%/15% brackets and a $1,000 CTC. Under OBBBA, 2026 has the $32,200 standard deduction, the $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's $4,855 separate tax, which is not part of the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (personal exemptions, a $1,000 CTC) and invented $16,800 of mortgage interest from a balance with no interest paid listed, so it itemized. Correct 2026 law gives the $32,200 standard deduction plus the $2,000 charitable deduction and a $2,200 CTC. It also added the child's $4,955 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It taxed the joint unit on $145,002 of AGI, counting the child's $45,000 of wages that are not the parents' income. Its $17,290 is higher even than the $13,800.44 of tax before credits that 2026 brackets give on that inflated base, so it also understated the $34,200 of deductions (standard plus charitable) and never subtracted the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (itemizing plus personal exemptions, restored brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's ~$4,925 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three personal exemptions totaling $15,150, 10%/15% brackets, a $1,000 CTC). OBBBA kept the TCJA structure for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's $5,023 separate tax, although the output is the joint unit's liability alone." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. The correct path is $100,002 of AGI minus the $32,200 standard deduction and $2,000 charitable deduction, $7,400.24 of tax on $65,802, minus the $2,200 CTC. Its $5,304 matches that same path with 2025 figures (a $31,500 standard deduction and a 10% bracket ending at $23,850 give $5,303.24), so it used last year's amounts instead of 2026's $32,200 and $24,800." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI, used expired pre-TCJA rules (personal exemptions, restored brackets) and treated the CTC as phased out. Under 2026 OBBBA law the parents' AGI is $100,002, they get the $32,200 standard deduction, and their $2,200 CTC phases out only above $400,000 for joint filers." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI and used expired pre-TCJA rules (itemizing $18,557 plus personal exemptions, restored brackets). The parents' AGI is $100,002, the OBBBA $32,200 standard deduction plus $2,000 charitable deduction applies, and the $2,200 CTC is fully used." +us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002). It also used a $30,000 standard deduction and 2025 bracket thresholds instead of 2026's $32,200 and $24,800/$100,800, left out the $2,000 charitable deduction, and used a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It added the child's $3,312.08 separate tax to the parents' liability, although the output is the joint unit's figure alone. For the parents it also used an inflated $30,750 standard deduction and estimated brackets instead of 2026's $32,200 and a 10% bracket up to $24,800. It left out the $2,000 charitable deduction and used a $2,000 CTC instead of $2,200." +us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It put the dependent's $45,000 of wages into joint AGI and subtracted a second $16,100 'dependent standard deduction' that does not exist on a joint return. It then withheld the nonrefundable CTC by treating a $2,400 credit as fully refundable. In fact the $2,200 CTC first offsets liability, so all of it is used against the $7,400.24 of tax on the parents' $65,802 of taxable income." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It combined the child's $45,000 with the head's wages into a $145,002 joint AGI instead of the parents' $100,002. Its $7,313 also does not follow from its own stated taxable income of ~$116,751, on which 2026 brackets give over $15,000 of tax before credits." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It gave no computation. The correct figure is $7,400.24 of tax on $65,802 of taxable income ($100,002 AGI minus the $32,200 standard and $2,000 charitable deductions), minus the $2,200 CTC. Its $14,677 is about the size of the tax before credits on a $145,002 AGI that includes the child's $45,000 of wages, so it counted the dependent's wages and did not subtract the CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002) and left out the $2,000 charitable deduction, taxing $112,802 instead of $65,802. Its standard deduction, brackets and $2,200 CTC were correct." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in joint AGI ($145,002), but the parents' tax unit AGI is $100,002. Its $32,200 standard deduction, $2,000 charitable deduction, 2026 brackets and $2,200 CTC were all correct, so that $45,000 alone raises the answer from $5,200.24 to $11,600.44." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Its $15,128.44 of tax before credits equals $145,002 of AGI minus a $30,000 standard deduction under 2025 brackets. So it counted the child's $45,000 of wages, used a pre-OBBBA standard deduction and last year's thresholds instead of $32,200 and 2026 brackets, and left out the $2,000 charitable deduction." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($5,200.24) and then added $3,220 from a separate return for the dependent child. The output is the joint unit's liability, which does not include the child's own return, so the answer stays at $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,household_unit_or_filing_status,False,"It added the child's $3,220 separate tax to the joint unit's liability, which is not part of this output. It also left out the $2,000 charitable deduction, taxing $67,802 instead of $65,802 and getting $5,440.24 instead of $5,200.24 for the parents." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It took a $13,650 overtime deduction for the head, but only Child 1 is flagged as paid hourly, so the head gets no overtime deduction. It also left out the $2,000 charitable deduction and added the child's $3,220 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It computed the joint return exactly ($5,200.24) and then added $3,220 from a separate return for the dependent child. The requested output is the joint unit's liability alone, so the child's own return tax does not belong in it." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002) and deducted mortgage interest, although only the loan balance is given and interest paid is $0. The correct path is the $32,200 standard deduction plus the $2,000 charitable deduction, $7,400.24 of tax and a $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (itemizing $21,627 plus three personal exemptions, restored brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It also added the child's ~$4,851 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into a $145,002 joint AGI, used expired pre-TCJA rules (exemptions, 15%/25% brackets) and phased out the CTC above an old $110,000 threshold. Under 2026 OBBBA law the parents' AGI is $100,002 and they get the $32,200 standard deduction and 10%/12% brackets. Their $2,200 CTC is safe because it phases out only above $400,000 for joint filers." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a $145,002 joint AGI, used expired pre-TCJA rules (three $5,400 exemptions, 15%/25% brackets) and phased out the CTC at an old $110,000 threshold. The parents' $100,002 AGI gets the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 10%/12% brackets and a full $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used expired pre-TCJA rules (three $5,434 exemptions, 10%/15% brackets, a $1,000 CTC) instead of the permanent OBBBA rules for 2026: $32,200 standard deduction, $2,000 charitable deduction, 10%/12% brackets and a $2,200 CTC. It then added the child's $4,846 separate tax to the joint unit's liability." +us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It added the child's separate $3,225 tax to the parents' ~$5,210, although the output is the joint unit's liability alone. Its parent figure was also mis-added: the tax on $65,802 is $7,400.24, not $7,410, so the correct result is $5,200.24." +us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for this output, so there was nothing to compare with the $5,200.24 that comes from $65,802 of taxable income minus the $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and made up a $12,000 overtime deduction (12% of wages) for a head who is not flagged as paid hourly and so gets no overtime deduction. It also left out the $2,000 charitable deduction. The correct base is $100,002 of AGI minus $34,200 of deductions." +us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It claimed credits wipe out the whole liability, but tax before credits is $7,400.24 on $65,802 of taxable income, far more than the single $2,200 CTC. Refundable credits are excluded from this output by definition. The correct result is $5,200.24, not $0." +us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and took a $12,500 overtime deduction for a head who is not flagged as paid hourly, so the head gets no overtime deduction. It also left out the $2,000 charitable deduction. The correct base is the parents' $100,002 AGI minus $34,200 of deductions." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI ($145,002) and invented $27,300 of mortgage interest from a balance with no interest paid listed, so it itemized. It also used 2025 brackets and a $2,000 CTC. The correct path is the parents' $100,002 AGI, the $32,200 standard deduction plus the $2,000 charitable deduction, 2026 brackets and a $2,200 CTC." +us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It invented items the facts do not support: a QBI deduction on W-2 wages, $16,800 of mortgage interest from a balance with no interest listed, a CDCC with no care expenses, and a foreign earned income exclusion. It then zeroed out a liability that its own arithmetic put at $5,687. With only the $32,200 standard deduction, the $2,000 charitable deduction and the $2,200 CTC, the tax is $5,200.24." us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model paid out a flat $1,600 ""refundable CTC"" without running the ACTC limitation: the Child Tax Credit is first applied as a nonrefundable credit, and with MFJ taxable income of roughly $112,800 (AGI $145,002 less the 2026 MFJ standard deduction) generating about $14,000 of federal income tax, the entire CTC is absorbed nonrefundably and no additional child tax credit arises. It compounded this by inventing $205 of EITC at $145,002 of AGI, when the one-child EITC for joint filers is fully phased out in the upper $50,000s." us,scenario_123,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model asserted that satisfying the 15%-of-earnings phase-in makes ""the full credit refundable,"" skipping the step that governs here: the CTC offsets the household's roughly $14,000 federal income tax liability as a nonrefundable credit first, and only the unused remainder — zero — flows to the refundable additional child tax credit, which is separately capped at $1,700 per child in 2026 rather than the full $2,400 the model reported." us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model claimed ""no limitation from the nonrefundable portion,"" which is exactly backwards: the nonrefundable CTC is applied first against the household's roughly $14,000 of federal income tax before refundable credits, exhausting the credit and leaving $0 of additional child tax credit. Its $1,500 figure is also a stale pre-2022 ACTC cap rather than the 2026 $1,700 ceiling." @@ -8998,80 +9881,92 @@ us,scenario_123,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou us,scenario_123,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model inferred Philadelphia residence solely from the household's Pennsylvania state and applied Philadelphia's wage tax despite the instruction to treat unlisted locality facts as false. Its unexplained reduction from the computed $5,613.24 to $3,545 also applies no identified exemption or valid computation step." us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Philadelphia as the default Pennsylvania locality and taxed both workers' wages even though Philadelphia residence or employment was never listed. It also submitted $3,830.90 after its own calculations produced $5,437.50 or $5,495.50, so the final number does not follow from its stated rate or tax base." -us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning reconstructs the reference exactly — $6,200 + $1,450 (head) + $2,790 + $652.50 (child) plus PA employee UC at 0.07% of $70 + $31.50 — and states the sum as $11,194, but the submitted value is $11,663.35, a figure produced by no step in its own work. The substantive rule set was right; the reported number was not the one it derived." -us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It applied only the 7.65% federal employee FICA rate to $145,000 and stopped, omitting Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% withheld from all gross wages. That contribution is $145,000 × 0.0007 = $101.50, which is exactly the $11,194 − $11,092.50 gap." -us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"After correctly concluding that no Additional Medicare Tax applies because the head's $100,000 is below the $200,000/$250,000 thresholds, it added $648.89 of Additional Medicare Tax anyway, inventing a 0.9% liability on wages with zero excess over the threshold. It also declared that Pennsylvania has no mandatory state payroll tax, dropping the $101.50 employee unemployment compensation contribution levied at 0.07% on all $145,000 of wages." -us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It computed the correct Pennsylvania component in its own reasoning — 'PA SUI employee tax 0.07% × $145,000 = $101.50' — then reversed itself, asserting PA has no employee-side mandatory payroll tax and submitting the federal-FICA-only total of $11,092.50. Discarding the $101.50 it had already calculated is the entire error." -us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"It identified every component correctly, including PA's 0.07% employee UC at $70 on the head and $31.50 on the child, and its reasoning explicitly totals $7,720 + $3,474 = $11,194. It then submitted $11,097.50, a value inconsistent with the sum it had just verified." -us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It computed only federal employee FICA at 7.65% on the head's $100,000 and the child's $45,000, never reaching state payroll taxes. Pennsylvania withholds an employee unemployment compensation contribution of 0.07% on all gross wages with no taxable wage base, adding $101.50 to the $11,092.50 it reported." -us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It correctly identified PA's 0.07% employee unemployment compensation contribution but applied it to only the first $10,000 of each worker's wages, yielding $14. The Pennsylvania employee UC withholding has no taxable wage base — it applies to every dollar of gross covered wages — so the correct amount is $145,000 × 0.0007 = $101.50, giving $11,194 rather than $11,106.50." -us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It asserted that Pennsylvania has no mandatory employee-side state payroll withholding, which drops the state's 0.07% employee unemployment compensation contribution. That contribution is withheld from all $145,000 of wages with no wage-base cap and equals $101.50, the difference between its $11,092.50 and the $11,194 reference." -us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It stopped at federal employee FICA — $8,990 Social Security and $2,102.50 Medicare on $145,000 — and never added a state component. Pennsylvania's employee unemployment compensation contribution of 0.07% on all gross wages adds $101.50 to reach $11,194." -us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It computed per-worker Social Security and Medicare correctly and concluded at $11,092.50 without any state payroll component. The missing item is Pennsylvania's mandatory 0.07% employee unemployment compensation withholding on the full $145,000 of wages, worth $101.50." -us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It explicitly stated that no mandatory state payroll tax applies, which is wrong for Pennsylvania: employees pay a 0.07% unemployment compensation contribution withheld on all gross wages with no wage base. On $145,000 that is $101.50, so the total is $11,194, not $11,092.50." -us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its stated method — combined 7.65% FICA on $145,000 — yields $11,092.50, yet it submitted $11,108, a figure its own arithmetic does not produce. Either way it omitted Pennsylvania's 0.07% employee unemployment compensation contribution of $101.50 on all wages, the component that brings the total to $11,194." -us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It named only Social Security and Medicare as components; those total $8,990 + $2,102.50 = $11,092.50 on $145,000 of wages, and its submitted $11,138 matches neither that figure nor the reference. It never applied Pennsylvania's 0.07% employee unemployment compensation withholding, which adds the $101.50 needed to reach $11,194." -us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It limited the household total to federal employee FICA on the head's $100,000 and the child's $45,000. Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% on the full $145,000 adds $101.50, which it omitted entirely." -us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It stated that Pennsylvania has no mandatory employee-side state payroll tax, dropping the state's 0.07% employee unemployment compensation contribution. That withholding applies to all gross wages with no taxable wage base, so $145,000 × 0.0007 = $101.50 must be added to its $11,092.50." -us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It asserted PA levies no mandatory employee payroll tax and reported federal FICA alone. Pennsylvania withholds a 0.07% employee unemployment compensation contribution from every dollar of covered wages, adding $101.50 on $145,000 and producing the $11,194 total." -us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its described method — 6.2% OASDI and 1.45% Medicare on the head's $100,000 and the child's $45,000, no Additional Medicare Tax, no state component — produces $11,092.50, but it submitted $12,366, which corresponds to none of the rates it named. It also assumed away Pennsylvania's 0.07% employee unemployment compensation contribution of $101.50 on all wages." -us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The rates it recited (6.2% plus 1.45% on $145,000) give $11,092.50, but it reported $15,399 — a magnitude consistent with applying the combined 15.3% employer-plus-employee rate to a $100,000 wage rather than the 7.65% employee share the question requires. It also omitted Pennsylvania's 0.07% employee unemployment compensation withholding of $101.50." -us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 7.65% employee FICA to each earner and stopped, checking only the Social Security wage base. The uncounted component is Pennsylvania's mandatory employee unemployment compensation contribution at 0.07% of all $145,000 in wages, or $101.50." -us,scenario_123,payroll_tax,gpt-6-astra,llm_error,state_local_rule,False,"It structured the answer correctly — $8,990 Social Security, $2,102.50 Medicare, plus a Pennsylvania employee unemployment contribution — but priced the PA piece at $87, which is 0.06% of $145,000, a superseded rate. The current Pennsylvania employee UC withholding rate is 0.07%, giving $101.50 and a total of $11,194." -us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its stated computation, 7.65% on $100,000 plus 7.65% on $45,000, equals $11,092.50 rather than the $11,107.50 it submitted. Neither figure includes Pennsylvania's 0.07% employee unemployment compensation contribution on all gross wages, worth $101.50." -us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It stated that no mandatory PA employee state payroll tax exists and reported federal FICA only. Pennsylvania withholds 0.07% for employee unemployment compensation on every dollar of covered wages, adding $101.50 to the $11,092.50 it computed." -us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It verified the OASDI wage base and the Additional Medicare Tax thresholds correctly, then declared PA has no mandatory employee-side state payroll tax. The state's 0.07% employee unemployment compensation contribution on all $145,000 of wages is $101.50, the missing piece of the $11,194 total." -us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It computed the federal components exactly right and then asserted no mandatory state employee payroll tax applies. Pennsylvania's employee unemployment compensation withholding of 0.07% on all gross wages adds $101.50, bringing the household total to $11,194." -us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,"It set state mandatory employee payroll taxes to zero, when Pennsylvania withholds an employee unemployment compensation contribution of 0.07% on all gross wages with no taxable wage base. That is $101.50 on the household's $145,000 of wages, the amount separating its $11,092.50 from the $11,194 reference." -us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No value and no explanation were returned for payroll_tax, so the required key was absent from the submitted outputs. The failure is a missing submission rather than a substantive computation error." -us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that the head's $100,000 and the child's $45,000 in wages generate employee Social Security and Medicare tax, then submitted $0 on the fabricated claim that 'this benchmark output reports 0 for this scenario.' The household's wages produce $8,990 Social Security, $2,102.50 Medicare, and $101.50 of PA employee unemployment compensation contribution, totaling $11,194." -us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that Pennsylvania levies no mandatory employee payroll tax and stopped at $11,092.50 of federal FICA. PA withholds 0.07% from all employee gross wages for unemployment compensation, adding $101.50 on $145,000." -us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It claimed Pennsylvania has no state payroll taxes apart from the separate income tax, which omits the state's mandatory 0.07% employee unemployment compensation contribution. Applied to all $145,000 of wages with no wage-base cap, that is $101.50, making the total $11,194." -us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It invented $270 of Additional Medicare Tax by claiming $30,000 of wages above the $250,000 MFJ threshold when household wages are $145,000 — no excess exists — and labeled $7,650 as 6.2% Social Security on $100,000 when 6.2% gives $6,200. It then adjusted its own $10,812.50 subtotal to $10,851.95 for a nonexistent rounding rule, while omitting the actual PA employee unemployment contribution of $101.50." +us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right, including PA employee UC at 0.07% ($70 + $31.50), and summed them to exactly $11,194. It then submitted an unrelated $11,663.35, so the error is purely in transcribing the final answer." +us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It applied 7.65% FICA to both earners ($11,092.50) but left out Pennsylvania's mandatory employee unemployment compensation contribution. That contribution is 0.07% of all $145,000 in wages and adds $101.50." +us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It said outright that the head's $100,000 is below the Additional Medicare Tax threshold, then added a fabricated $648.89 of that tax anyway. It also claimed PA has no mandatory employee payroll tax, which drops the 0.07% UC contribution ($101.50) on $145,000 of wages." +us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly worked out PA's employee UC contribution as 0.07% × $145,000 = $101.50, then put $0 into the sum and called it 'minimal.' The result was $11,092.50 instead of $11,194." +us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning included Social Security, Medicare and PA employee UC at 0.07% and summed to exactly $11,194 twice. It then submitted $11,097.50, which matches none of its own figures, so this is a final-answer transcription error." +us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It counted only federal FICA at 7.65% on the head's and child's wages ($11,092.50). It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all $145,000 of wages ($101.50)." +us,scenario_123,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied 7.65% to the $145,000 of combined wages and stopped there. It missed Pennsylvania's mandatory 0.07% employee UC contribution, which adds $101.50." +us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It recognized PA's 0.07% employee UC contribution but wrongly limited it to the first $10,000 of each worker's wages ($14 in total). Pennsylvania's employee UC withholding applies to all gross wages: $70 on the head and $31.50 on the child, $101.50 in all." +us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It stated that PA has 'no state disability/UI employee withholding.' Pennsylvania in fact withholds 0.07% of all wages for employee unemployment compensation, which is $101.50 on $145,000, so the model stopped at $11,092.50." +us,scenario_123,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It computed federal Social Security and Medicare correctly but declared that no employee state payroll tax applies. That leaves out Pennsylvania's mandatory 0.07% employee UC contribution on $145,000 of wages ($101.50)." +us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only employee FICA on $145,000 ($8,990 + $2,102.50). It never applied Pennsylvania's 0.07% employee unemployment compensation contribution, which adds $101.50." +us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It totaled Social Security and Medicare for the head and child ($11,092.50) and never considered state payroll tax. That misses Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It stated that no mandatory state payroll tax applies. Pennsylvania levies a 0.07% employee unemployment compensation contribution on all wages, which is $101.50 on $145,000." +us,scenario_123,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only federal Social Security and Medicare on $145,000 ($11,092.50). It left out Pennsylvania's 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $145,000, which is $11,092.50, and then misreported the total as $11,108. It also left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It included only Social Security and Medicare, which are $11,092.50 on $145,000, then submitted $11,138, a figure its stated components do not produce. It never applied Pennsylvania's 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It summed federal FICA for the head ($7,650) and child ($3,442.50) and stopped. It missed Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on both workers' wages ($101.50)." +us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It stated that 'PA has no mandatory employee-side state payroll tax.' Pennsylvania withholds 0.07% of all employee wages for unemployment compensation, which is $101.50 here." +us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It claimed PA has no mandatory employee payroll tax, which leaves out the 0.07% employee UC contribution on $145,000 of wages ($101.50). Its total stopped at $11,092.50." +us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It described only Social Security and Medicare on $145,000, which are $8,990 + $2,102.50 = $11,092.50, yet submitted $12,366, an overstatement of $1,273.50 that its own components do not produce. It also assumed no PA employee payroll tax, which drops the 0.07% UC contribution ($101.50)." +us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It named only 6.2% Social Security and 1.45% Medicare on $145,000, which yield $11,092.50, but reported $15,399, an effective rate of 10.6% that no stated rule supports. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It applied 7.65% FICA to each earner and ignored state payroll tax. It missed Pennsylvania's mandatory 0.07% employee unemployment compensation contribution ($101.50). +us,scenario_123,payroll_tax,gpt-6-astra,llm_error,thresholds_rates,False,"It correctly included the PA employee unemployment contribution on all $145,000 of wages but used a 0.06% rate ($87). The employee rate is 0.07%, which gives $101.50." +us,scenario_123,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It asserted that no Pennsylvania employee payroll tax applies. PA withholds 0.07% of all wages for employee unemployment compensation, which adds $101.50 to the $11,092.50 of FICA." +us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $145,000, which is $11,092.50, and misreported the sum as $11,107.50. It also left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)." +us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It stated there is 'no mandatory PA employee state payroll tax.' That leaves out Pennsylvania's 0.07% employee unemployment compensation withholding on $145,000 of wages ($101.50)." +us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,It claimed PA has no mandatory employee-side state payroll tax and stopped at 7.65% FICA. It missed the 0.07% employee UC contribution on all wages ($101.50). +us,scenario_123,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It declared that Pennsylvania has no mandatory employee state payroll tax, so it left out the 0.07% employee unemployment compensation contribution on the head's and child's wages ($70 + $31.50)." +us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It ruled out any mandatory state employee payroll tax. Pennsylvania's 0.07% employee UC contribution applies to all $145,000 of wages and adds $101.50." +us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,"It set state mandatory employee payroll taxes to zero. Pennsylvania withholds a 0.07% employee unemployment compensation contribution on all wages, which is $101.50 here." +us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there is no substantive answer to evaluate. The correct figure is $8,990 Social Security + $2,102.50 Medicare + $101.50 PA employee UC = $11,194." +us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that the head's $100,000 and the child's $45,000 wages generate employee Social Security and Medicare tax, then reported $0 anyway, claiming the benchmark 'reports 0.' That drops all $11,194 of employee payroll tax, including the $101.50 PA UC contribution." +us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that 'PA levies no mandatory employee payroll tax.' Pennsylvania's 0.07% employee unemployment compensation contribution on $145,000 of wages adds $101.50." +us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It asserted that PA has no state payroll taxes. That leaves out the mandatory 0.07% employee UC contribution on the head's $100,000 and the child's $45,000 wages ($101.50)." +us,scenario_123,payroll_tax,qwen3.8-max,llm_error,other,False,"It labeled the head's full 7.65% FICA ($7,650) as Social Security, then added Medicare ($1,450) again, which double-counts it. It invented a $270 Additional Medicare Tax even though the couple's $100,000 of wages is far below the $250,000 threshold. It also left out PA's 0.07% UC contribution and replaced the resulting sum with an unexplained $10,851.95." us,scenario_123,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible output, violating the required submission contract and omitting the engine-derived result of 0." us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category." us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. -us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It added Child 1's $45,000 of wages to the parents' $100,002 base to build a $145,002 PA base, but the dependent's earnings sit outside the tax unit's filer income (head and spouse only), so the PA base is $100,002 and the tax is $3,070.06. It even computed the parents' $3,070.06 piece correctly before summing in the child's $1,381.50, and then submitted $4,453.06 despite writing $4,451.56 twice." -us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It applied the 3.07% flat rate to a household-wide $145,002 base that folds in the 16-year-old dependent's $45,000 of wages. The dependent's compensation is not part of the parents' tax unit's filer income for this output, so the base is $100,000 wages plus $2 interest and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented a $15,000 Pennsylvania standard deduction and an $8,500 dependent standard deduction; Pennsylvania grants no standard deduction, personal exemption, or itemized deductions, and its filing-status labels (head of household) do not exist in PA law either. It compounded that by including the dependent child's $45,000 in the base, arriving at $121,501 × 0.0307 instead of the correct $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It correctly ruled out deductions but built the PA base as head $100,000 + child $45,000 + $2 interest = $145,002, pulling the dependent 16-year-old's wages into the parents' tax unit. Only head and spouse income enters this tax unit's PA taxable income, giving $100,002 × 0.0307 = $3,070.06; it also submitted $4,452.08 against its own stated $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It summed Head, Spouse, and Child 1 income into one $145,002 PA base, but the dependent child's $45,000 of wages is excluded from the tax unit's filer income, leaving $100,002 and $3,070.06 of tax. It then submitted $4,452.61 while its own explanation computed $4,451.56." -us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It derived the correct head/spouse figure — $100,002 × 0.0307 ≈ $3,070 — and then added $1,381.50 for the child's separately taxed $45,000. The benchmark output is the tax unit's PA liability, which excludes a dependent's own wages, so the answer stops at $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It explicitly computed the joint return's $100,002 × 0.0307 = $3,070.06 and the child's separate $1,381.50, then chose the wrong branch by summing them because 'all are listed in one tax unit.' A dependent's own wages are taxed on the dependent's own PA return and never enter the parents' tax unit income, so the correct output is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It got the unit scope right — excluding the child's wages and computing $100,002 × 0.0307 = $3,070.06 — then discarded that result for $3,060 on an invented 'compensation-only interest treatment nuance,' a haircut with no basis in PA law, which taxes compensation and interest identically at the flat rate. Its submitted value also contradicts the $3,070.06 stated in its own explanation." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It taxed $145,002 at 3.07%, a base that includes the 16-year-old dependent's $45,000 of wages. Those wages belong to the child's own return, not the parents' tax unit, so the PA base is $100,002 and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It computed the head's $3,070 correctly but then added $1,381.50 for Child 1's $45,000, treating the dependent's compensation as part of the household's state tax liability. It also dropped the $2 of taxable interest from the base; the tax unit's PA tax is $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It applied the 3.07% rate to $145,002, folding the dependent child's $45,000 of wages into the parents' PA taxable income. The tax unit's PA base is the head's and spouse's $100,000 of wages plus $2 of interest, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It itemized the base as $100,001 for Head, $1 for Spouse, and $45,000 for Child 1, summing a dependent's earnings into the parents' tax unit. Dropping the child's wages leaves $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a 3% rate rather than Pennsylvania's statutory 3.07% and applied it to $145,000 of household-wide wages, producing $4,350 from two compounding errors. The correct rate on the tax unit's $100,002 base — which excludes the dependent child's $45,000 — gives $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It defined 'total household taxable income' as $145,002 and taxed it at 3.07%, treating the dependent 16-year-old's $45,000 of wages as part of the parents' PA return. The tax unit's base is $100,002, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It separated the two returns correctly — $3,070 for the head/spouse joint return and $1,382 for Child 1 — and then added them into a single household figure. The requested output is the tax unit's own liability, which contains only the $100,002 filer base, so the answer is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation at all; $3,385 corresponds to a PA base of roughly $110,260 at the 3.07% flat rate, matching neither the $100,002 filer base that yields $3,070.06 nor the $145,002 household-wide base. The number is an unanchored guess rather than an application of PA's flat tax to any stated income." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It taxed a 'total household taxable income of $145,002' at 3.07%, which includes the dependent child's $45,000 of wages. The parents' tax unit reports only $100,000 of wages plus $2 of interest, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to a $145,002 base built by summing every household member's income, including the 16-year-old dependent's $45,000 of wages. Excluding those wages from the tax unit leaves $100,002 and $3,070.06 of PA tax." -us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It set PA taxable compensation at $145,000 by combining the head's $100,000 with the dependent child's $45,000, then added $2 of interest. Only head and spouse compensation enters the tax unit's PA base, so the correct computation is $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It correctly stated that the head and spouse owe $100,002 × 0.0307 = $3,070.06 and that Child 1 files separately, then added the child's $1,381.50 back into the household total. A separately filing dependent's tax is not part of this tax unit's state income tax, so the answer is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It summed $100,000 + $45,000 + $2 into a single $145,002 PA base, pulling the dependent 16-year-old's wages onto the parents' return. The tax unit's base is $100,002, yielding $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It taxed 'the household's wages and interest' at 3.07%, importing the dependent child's $45,000 into the parents' PA base, and its $4,807 does not even equal that base times the rate ($145,002 × 0.0307 = $4,451.56) — it implies about $156,580 of income, more than the household earns. The tax unit's base is $100,002, so the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It applied 'itemized deductions using the provided medical and real estate tax amounts,' but Pennsylvania allows no itemized deductions, standard deduction, or personal exemption against the flat 3.07% rate. Its $9,055 implies a base near $295,000, exceeding total household income; the tax unit's base is $100,002 and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It built the PA base as $100,000 + $45,000 + $2 = $145,002, treating the dependent 16-year-old's wages as taxable to the parents' unit. Excluding them gives $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It taxed the household's combined $145,002 of wages and interest, which includes the dependent child's $45,000, and its $4,455.06 does not match even that base times 3.07% ($4,451.56). The tax unit's PA base excludes the dependent's wages, giving $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied the 3.07% flat rate to $145,002 of wages and interest aggregated across all three people. Child 1's $45,000 is taxed on the child's own PA return, leaving the parents' tax unit with $100,002 and $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It taxed 'the household's $145,002 of reported wages and taxable interest' at 3.07%, aggregating the dependent child's earnings with the parents'. The tax unit's PA taxable income is $100,002, so the tax before refundable credits is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of 'household wages and interest,' folding in the 16-year-old dependent's $45,000. The parents' tax unit reports $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It taxed a combined household base of about $145,000 at 3.07% and truncated to $4,451, and it referred to 'deductions' that Pennsylvania's flat tax does not provide. The dependent child's $45,000 is excluded from the tax unit, so the base is $100,002 and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It applied the flat 3.07% rate to a combined $145,002 of compensation and interest spanning all three household members. Child 1's $45,000 belongs on the dependent's own return; the tax unit's base is $100,002, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It correctly ruled out deductions, exemptions, and tax forgiveness but set taxable income at $145,002, including the dependent 16-year-old's $45,000 of wages. Removing them leaves $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It enumerated the base as head $100,001 + spouse $1 + child $45,000 = $145,002, explicitly adding the dependent's wages to the parents' tax unit. Only the $100,002 of head and spouse income is taxed here, giving $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It identified the two separate filing units — $100,002 for the parents and $45,000 for Child 1 — and then deliberately taxed both together at 3.07%. The requested output covers only the parents' tax unit, so the correct value is $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It supplied no reasoning, and $2,939.86 implies a PA base of about $95,761 — roughly $4,241 shaved off the correct $100,002 of wages plus interest. Pennsylvania grants no standard deduction, personal exemption, or itemized deductions against the 3.07% flat rate, so the base is the full $100,002 and the tax is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It correctly stated that PA allows no deductions but set taxable income at $145,002, which includes the dependent child's $45,000 of wages. The parents' tax unit reports $100,002, so the tax before credits is $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It asserted a Pennsylvania standard deduction that does not exist, then wrote that PA liability is positive while submitting 0 — a value that contradicts its own explanation. The tax unit's $100,002 of wages and interest at the 3.07% flat rate produces $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It correctly excluded charity, medical, and property-tax deductions and correctly ruled out Tax Forgiveness, but taxed $145,002 — the household total including Child 1's $45,000. The dependent's wages are outside the parents' tax unit, leaving $100,002 × 0.0307 = $3,070.06." -us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It built the base as $145,000 of wages ($100,000 head plus $45,000 child) plus $2 of interest, importing the dependent's earnings into the parents' PA return. The tax unit's base is $100,002, giving $3,070.06 rather than $4,451.57." -us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income of $67,525 — a figure net of the federal standard deduction — but Pennsylvania does not use federal taxable income or allow any deduction; its base is gross compensation plus interest, here $100,002. Its own arithmetic on that wrong base produced $2,073.02 while it submitted $2,565.30; the correct computation is $100,002 × 0.0307 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It correctly computed the parents' joint PA tax as $3,070.06, then added $1,381.50 for a separate return for the dependent child, which falls outside this tax unit. The number it submitted ($4,453.06) also does not match its own sum of $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which counts the dependent child's $45,000 of wages. The tax unit's PA base is only the spouses' $100,002, so the correct tax is $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It invented a $15,000 PA standard deduction and an $8,500 dependent deduction. Pennsylvania has neither: it taxes compensation and interest with no standard deduction. It also added the child's $45,000 of wages to the base, when the correct base is the spouses' $100,002 × 3.07%." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the parents' taxable compensation, making the base $145,002 instead of $100,002. It also submitted $4,452.08, which does not match the $4,451.56 it computed." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It included Child 1's $45,000 of wages in the PA base ($145,002) even though the joint return covers only the spouses' $100,002. It also submitted $4,452.61, which does not match its own computed $4,451.56." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It computed the head and spouse's tax as about $3,070, then added $1,381.50 for the child's own return. That separate return is outside this tax unit's liability, which is $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the dependent child's wages with the parents' income. The joint return's base is the spouses' $100,002, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It correctly found that the PA joint return covers only the spouses' $100,002 ($3,070.06) and that the child files separately. It then overrode this by reading 'one tax unit' as pooling everyone's income. A dependent's wages are never part of the parents' PA joint return." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It correctly derived 100,002 × 0.0307 = $3,070.06 and correctly excluded the child's wages. It then submitted $3,060, citing 'compensation-only interest treatment nuances' that do not exist in PA law." +us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It computed the parents' unit correctly at $3,070.06, then added the child's separate $1,381.50. The requested output is the single tax unit's liability, not the sum of two filing units." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It taxed $145,002 at 3.07%, which includes the dependent child's $45,000 of wages. The tax unit's PA base is only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It added a separate $1,381.50 tax on the dependent child's wages to the head's tax. It also dropped the spouses' $2 of taxable interest, which PA taxes as a separate income class. The correct figure is $100,002 × 3.07% = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the child's $45,000 of wages into the parents' base. The joint return's base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,100 charitable donation from the head's income, but PA allows no charitable deduction against compensation. It also added the child's $45,000 of wages. The correct base is the full $100,002 of the spouses' income." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It summed the income of all three members ($145,002), including the dependent child's $45,000 of wages. PA's joint return taxes only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a 3% rate instead of PA's 3.07% flat rate. It applied that rate to $145,000, which includes the child's wages and omits interest. The correct figure is 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to 'total household taxable income' of $145,002, which includes the dependent child's wages. The tax unit's base is the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It correctly computed the parents' joint tax as about $3,070, then added $1,382 for the child's separate return. That return is not part of this tax unit's liability." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,385 equals 3.07% of about $110,260, which matches no PA base in this household. The spouses' compensation plus interest is $100,002, which gives $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of 'total household' income, pooling the dependent child's wages. The correct base is only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes Child 1's $45,000 of wages. The PA joint return for this tax unit includes only the head and spouse's $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of compensation, which includes the dependent child's wages. The joint return's compensation is only the head's $100,000, which gives $3,070.06 once the $2 of interest is added." +us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It correctly computed the joint return at $3,070.06, then added the child's separately filed $1,381.50. The output covers only the one tax unit's return." +us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It pooled the child's $45,000 of wages with the parents' income to get a $145,002 base. The tax unit's PA base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It pooled all household wages, including the dependent child's, into the PA base. Its $4,807 implies a base of about $156,580, which is more than all listed income combined. The correct figure is 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It applied federal-style itemized deductions, which PA does not allow, and an effective rate of about 6.2% on all household wages. PA taxes the spouses' $100,002 at a flat 3.07%, giving $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the PA base ($145,002). The joint return taxes only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's wages. Its submitted $4,455.06 also does not equal 3.07% of that base ($4,451.56). The correct figure is 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling Child 1's wages into the parents' base. The tax unit's base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It taxed all $145,002 of household wages and interest. The dependent child's $45,000 is not on the parents' PA joint return, so the base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It taxed $145,002 of 'household' income, which includes the dependent child's wages. The correct base is only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,household_unit_or_filing_status,False,"It correctly taxed the parents' $100,002, then added the child's separate $45,000 return. The requested output is the single tax unit's liability of $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of wages, which includes the dependent child's $45,000. The joint return's base is the head's $100,000 plus $2 of interest." +us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It counted $145,000 of wages, which includes the dependent child's $45,000. The tax unit's PA base is only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It taxed about $145,000 of combined household income, pooling the dependent child's wages. The correct base is $100,002, which gives $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of combined compensation, which includes the dependent child's wages. The joint return's base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It correctly ruled out PA deductions, but it put the child's $45,000 of wages in the tax unit's base ($145,002) instead of using only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's wages. The parents' joint return, the only return in this tax unit, has a base of $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It explicitly added the child's $45,000 of wages to the head and spouse's income. The dependent's wages are not on the parents' PA joint return, so the base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It correctly identified two filing units, then summed tax across both. The output is the tax unit's liability, which is the parents' return alone: $100,002 × 3.07% = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It gave no reasoning. $2,939.86 is 3.07% of about $95,761, so it took roughly $4,241 off the spouses' $100,002. PA allows no deduction against compensation for any of the listed expenses." +us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, pooling the dependent child's $45,000 of wages. The joint return's base is $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It stated that PA tax liability is positive, then submitted 0, contradicting its own reasoning. It also cited a PA standard deduction that does not exist. The correct figure is the full 3.07% × $100,002 = $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It correctly disallowed the charitable, medical and property-tax deductions, but it included the child's $45,000 of wages. The tax unit's base is only the spouses' $100,002." +us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It added Child 1's $45,000 of wages to the head's $100,000 of compensation. The PA joint return covers only the spouses' $100,002, which gives $3,070.06." +us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($67,525), but PA taxes gross compensation and interest with none of the federal standard or itemized deductions. It then submitted $2,565.30, which matches neither its own $2,073.02 nor the correct 3.07% × $100,002." us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions. diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv index 76c0e46f..e8afbc5b 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv @@ -1,19 +1,19 @@ country,scenario_id,variable,wrong_model_count,case_failure_sources,case_failure_subtypes,reference_suspect,reference_bug_hypothesis,case_annotation -us,scenario_000,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"This case turns on three linked steps. First, a 77-year-old widow with no dependent child files as single, so Social Security taxability uses the $25,000/$34,000 thresholds: provisional income of $50,520 gives $18,542.70 of taxable benefits (not the $22,182 85% cap, and not the $11,542 that joint thresholds produce), for AGI of $50,437.11. Second, under OBBBA the 2026 SALT cap is $40,400, so the full $16,470 of Texas real estate tax plus the optional state sales-tax deduction yields $18,341.33 of itemized deductions, which narrowly beats the $18,150 aged single standard deduction; the $6,000 senior deduction stacks on top for $24,341.33 total. Third, the permanent TCJA schedule applies — no personal exemption, 10% to $12,400 then 12% — so $26,095.78 of taxable income produces $2,883.49." +us,scenario_000,federal_income_tax_before_refundable_credits,42,llm_error,household_unit_or_filing_status,False,,"A 77-year-old surviving spouse with no dependent child cannot use Qualifying Surviving Spouse status, so the return is filed as Single. Taxable Social Security uses the $25,000/$34,000 thresholds: $4,500 + 0.85 × ($50,520 − $34,000) ≈ $18,542. That puts AGI at about $50,437. Under OBBBA-extended law, deductions are the $16,100 single standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, totaling $24,150. That leaves $26,287 of taxable income, taxed at 10% up to $12,400 and 12% above, for $2,906.45. Most wrong answers made one of three mistakes: they used joint (QSS) thresholds, deductions and brackets; they assumed the TCJA sunset brought back personal exemptions and the 15% bracket; or they left out the $6,000 senior deduction." us,scenario_000,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_000,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Texas Medicaid eligibility at age 77 requires qualification through an aged Medicaid pathway; age alone does not establish a qualifying category. This person has no Medicaid eligibility category, receives no SSI, and has MAGI income of 3.98 times FPL, so the immigration-status requirement does not create eligibility by itself." us,scenario_000,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wage and salary compensation, not to Social Security retirement benefits, pensions, IRA distributions, interest, or veterans benefits. Because the household reports no wages or other employee compensation, the employee Social Security and Medicare tax base is zero and payroll tax is $0." -us,scenario_001,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The whole case turns on the 2026 standard-deduction regime: OBBBA (P.L. 119-21) made the TCJA doubled standard deduction and the zero personal exemption permanent, so a 2026 joint return gets a $32,200 basic standard deduction plus $1,650 per spouse age 65+ ($3,300 here) plus the new $6,000-per-senior deduction — roughly $47,500 of deductions — while personal exemptions remain $0. AGI is $31,402 (wages $29,064 plus $2,338 of Social Security, since combined income of $36,677 sits in the 32k–44k tier where only 50% of the excess over $32,000 is included), which is below the basic standard deduction by itself, so taxable income and tax are $0. All four wrong models priced in a TCJA sunset: they cut the standard deduction to roughly $15,000–$20,000 and revived about $10,000 of personal exemptions, manufacturing $900–$20,000 of taxable income taxed at 10%." +us,scenario_001,federal_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"Every model assumed the TCJA expired after 2025. They restored personal exemptions and a pre-TCJA-sized standard deduction of roughly $12,700 to $16,600. In fact, P.L. 119-21 (the One Big Beautiful Bill Act, July 2025) made the TCJA structure permanent: personal exemptions stay at $0, and the 2026 joint standard deduction is $32,200. That basic deduction alone is larger than the couple's AGI of $31,402.25. The AGI is $29,064 in wages plus $2,338.25 of taxable Social Security, under the 50% tier. The deduction grows further with the $1,650-per-spouse aged addition and the $6,000-per-senior deduction for both spouses, so taxable income is $0 and tax is $0. Four models got AGI right and left only a small positive taxable income because of the wrong deduction regime. grok-build-0.1 also overstated taxable Social Security by applying the 85% tier." us,scenario_001,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"The trap is which Medicaid pathway a 71-year-old can use: Virginia's MAGI categories (expansion adults, parents, pregnant individuals) are closed to the head both by age and by income at 2.05x FPL, leaving only the SSI-linked aged pathway, which requires actually passing SSI's income and asset tests. Applying SSI counting rules to $15,225 of Social Security and $29,064 of wages leaves roughly $29,700 of annual countable income, so SSI is 0 and the aged category does not attach, yielding medicaid_category = NONE. All four wrong models skipped that income test — treating age 65+, Medicare Savings Program status, or unquantified disregards as sufficient on their own." us,scenario_001,payroll_tax,4,llm_error,payroll_tax_base,False,,"Employee FICA applies to the head's full $29,064 of wages despite the head's age. Social Security tax is $1,801.97 and Medicare tax is $421.43, which sum to $2,223.39; there is no Additional Medicare Tax and the spouse has no taxable wages." us,scenario_001,snap,1,llm_error,thresholds_rates,False,,"SNAP first applies the gross-income eligibility limit to this two-person household. Its $44,289 annual gross income exceeds the applicable 2026 limit, so the calculation ends with no eligibility and deductions cannot produce a benefit." us,scenario_001,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The spouse is 69, which excludes them from Virginia's ACA expansion adult group (ages 19-64) and from every other MAGI category, and the household's Medicaid MAGI of $44,289 ($29,064 wages plus $15,225 Social Security, counted in full for MAGI) sits at 2.05x the 2-person FPL, far above the 138% FPL expansion ceiling. The only route left for a 65+ adult is the non-MAGI aged/blind/disabled pathway, which is not conferred by age: it requires countable income at or below Virginia's SSI-related ABD standard (roughly 80% FPL, about $17,300 for a couple) with a $3,000 couple resource limit, and $44,289 clears that standard by more than 2.5x even after the $20 general and $65-plus-half earned disregards (countable about $2,427/month). The spouse also received $0 SSI, so the SSI-recipient pathway never triggers, leaving medicaid_category = NONE. Every wrong model treated age 65+ as itself creating an aged pathway, or assumed the spouse's own $0 earnings rather than the couple's combined countable income governs that pathway's income test." us,scenario_001,spouse_medicare_eligible,1,llm_error,categorical_eligibility,False,,"PolicyEngine's Medicare eligibility variable applies the standard age criterion, which both spouses satisfy because they are 71 and 69. The spouse's eligibility therefore equals Yes without requiring a separately listed work-credit history." us,scenario_001,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Virginia gives each taxpayer age 65 or older a $12,000 age deduction under Va. Code § 58.1-322.02(10)-(11), reduced dollar-for-dollar only when adjusted federal AGI — a measure that excludes taxable Social Security — exceeds $75,000 on a joint return. Virginia also subtracts the federally taxable Social Security ($2,338.25 of the $15,225), so Virginia AGI is $29,064 and AFYA is $29,064, far below the $75,000 joint threshold; both spouses therefore take the full $12,000, and $24,000 of age deductions alone wipes out Virginia AGI before the joint standard deduction and the $930 personal plus $800 age exemptions are even applied. Taxable income is $0 and the tax is $0 at any 2026 standard-deduction level. Every wrong answer either replaced the $12,000-per-person age deduction with Virginia's small $800 additional 65+ exemption, granted it to only one spouse, or left taxable Social Security inside Virginia AGI." -us,scenario_002,federal_income_tax_before_refundable_credits,10,llm_error,taxable_income_or_deductions,False,,"The correct 2026 AGI is $23,130: pension $18,340 + interest $400 + $4,390 of taxable Social Security. The $24,000 disability benefits and $48,000 veterans benefits are excluded from gross income, so provisional income is $18,740 + half of $44,080 = $40,780, which sits between the MFJ $32,000 and $44,000 thresholds and therefore includes only 50% × ($40,780 − $32,000) = $4,390 of benefits. That AGI is below the 2026 MFJ standard deduction alone ($32,200 base + two $1,650 age-65 additions = $35,500), and further below it once the $6,000-per-senior deduction ($12,000) is applied, so taxable income and tax before refundable credits are $0, and NIIT is $0 because AGI is far under $250,000. Nine models manufactured taxable income by pulling the non-taxable disability benefits into gross/provisional income (which also flips Social Security from the 50% tier to the 85% tier), by taxing 100% of Social Security, or by filing the couple as a single individual." +us,scenario_002,federal_income_tax_before_refundable_credits,10,prompt_ambiguity,taxable_income_or_deductions,False,,"The correct 2026 AGI is $23,130: pension $18,340 + interest $400 + $4,390 of taxable Social Security. The $24,000 disability benefits and $48,000 veterans benefits are excluded from gross income, so provisional income is $18,740 + half of $44,080 = $40,780, which sits between the MFJ $32,000 and $44,000 thresholds and therefore includes only 50% × ($40,780 − $32,000) = $4,390 of benefits. That AGI is below the 2026 MFJ standard deduction alone ($32,200 base + two $1,650 age-65 additions = $35,500), and further below it once the $6,000-per-senior deduction ($12,000) is applied, so taxable income and tax before refundable credits are $0, and NIIT is $0 because AGI is far under $250,000. Nine models manufactured taxable income by pulling the non-taxable disability benefits into gross/provisional income (which also flips Social Security from the 50% tier to the 85% tier), by taxing 100% of Social Security, or by filing the couple as a single individual. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." us,scenario_002,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits, pensions, disability benefits, veterans benefits, or taxable interest. Because the household has no wages or other payroll-taxable earnings, every employee-side payroll-tax component is zero." us,scenario_002,spouse_medicaid_eligible,1,llm_error,household_unit_or_filing_status,False,,"Medicaid eligibility for the spouse cannot be established by treating the spouse's lack of individually reported income as zero countable income. The spouse qualifies through no Washington Medicaid pathway: the engine assigns category NONE, records no SSI, and calculates household MAGI at 2.90 times FPL, above the applicable MAGI limits." -us,scenario_003,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on 2026 post-OBBBA parameters — a $32,200 MFJ standard deduction with no age additions (head 64, spouse 61, so neither the $1,650 aged addition nor the $6,000 senior deduction applies) and 10/12/22% brackets breaking at $24,800 / $100,800 / $211,100 — applied to AGI of $182,172.27, which is $184,872.25 of gross income (including $50,583 of taxable Social Security at the 85% cap) less $2,699.98 of above-the-line deductions: the $1,582 net capital loss and the $1,118 traditional IRA contribution. That leaves $149,972.27 taxable, of which $146,212.27 is ordinary income taxed at $21,590.70 and $3,760 of qualified dividends taxed at 15% for $564. Nine models assumed a TCJA sunset that OBBBA repealed (personal exemptions, a ~$16,000 standard deduction, 10/15/25% rates); a large second group had the framework right but dropped the $1,118 IRA above-the-line deduction, double-counted the capital-loss items, or used 2025 bracket edges; a third group submitted figures their own arithmetic never produced or answered $0 against six figures of retirement income." +us,scenario_003,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine subtracts the $1,118 traditional IRA contribution above the line without applying the IRC §219(b)(1)(B) limit, which caps the deduction at compensation includible in gross income, or the §219(c) spousal limit. Both spouses have $0 wages and $0 self-employment income, so the correct deduction is $0; allowing it understates AGI by $1,118 and tax by about $246.","The reference builds AGI of $182,172.27 by subtracting both the $1,582 net capital loss and the $1,118 traditional IRA contribution from gross income. That gross income includes 85% of Social Security, the IRA distributions, the pension, the interest and the dividends. It then subtracts the 2026 MFJ standard deduction of $32,200, taxes the $146,212.27 ordinary portion at 10/12/22% with thresholds of $24,800 and $100,800 ($21,590.70), and adds 15% on the $3,760 of qualified dividends. The strongest cluster of models matched every step except the IRA deduction, which they refused because there is no compensation, and landed at $22,400.71. The remaining models made their own errors: TCJA-sunset personal exemptions and 15%/25% brackets, 2025 or guessed brackets and standard deductions, a 401(k) deduction with no wages, dropped or mis-taxed Social Security, double-counted capital items, or submitted values that did not match their own work. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 22,154.70; the corrected value is 22,400.65. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)." us,scenario_003,head_medicare_eligible,3,llm_error,age_disability,False,,"Standard Medicare eligibility requires the person to be at least age 65 unless a separate disability or disease-based pathway applies. The head is 64 throughout 2026, and all unlisted disability and qualifying disease statuses are false, so neither current nor projected eligibility applies." us,scenario_003,payroll_tax,2,llm_error,payroll_tax_base,False,,"Usual weekly hours and employer-sponsored insurance establish work and coverage facts but do not supply wage income or an hourly rate. Because unlisted numeric inputs are zero, both people have zero wages subject to employee Social Security and Medicare taxes, so employee payroll tax is zero." us,scenario_003,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no state individual income tax, and exclusive Texas residency makes every other state's income-tax component zero. The aggregate state income tax before refundable credits is therefore the sum of 51 zero components, yielding $0." @@ -23,19 +23,20 @@ us,scenario_004,head_medicare_eligible,4,llm_error,age_disability,False,,"Standa us,scenario_004,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other covered earned compensation, not Social Security retirement benefits, taxable interest, or partnership/S-corporation losses. Because unlisted wages and hourly rates are zero, the household has no employee payroll-tax base and employee payroll tax is $0." us,scenario_004,spouse_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"New York's ACA expansion pathway requires the spouse's MAGI income level to be at or below 138% of FPL. The household's applicable MAGI is 2.39 times FPL, and the 51-year-old spouse has no disability, blindness, SSI, age-based, dependent, or other categorical pathway, so the Medicaid category is NONE." us,scenario_004,state_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"Both spouses show 40-hour work weeks but no wage amount, and the prompt directs that unlisted numeric inputs be treated as 0, so the household's only non-Social-Security income is $1,200 of taxable interest less a $135 partnership loss = $1,065. Provisional income is $1,065 + half of $51,475 = $26,803, below the $32,000 married-filing-jointly base, so none of the Social Security benefit is federally taxable, and New York subtracts any taxable Social Security from federal AGI regardless. New York AGI of $1,065 is entirely absorbed by the ~$16,050 MFJ standard deduction, leaving $0 of NY taxable income and $0 of tax before refundable credits. Every wrong answer either taxed the $51,475 Social Security benefit, imputed phantom wages from the hours-worked fact, or both." -us,scenario_005,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The reference is a three-layer sum on a 2026 return governed by OBBBA's permanent TCJA structure: $89,095.92 of bracket tax on $425,574.75 of ordinary taxable income, $11,688 on $77,920 of adjusted net capital gain (qualified dividends only, since the $51,920 LTCG is fully absorbed by the $62,831 short-term loss) at 15%, and $5,721.98 of NIIT on $150,578.34. Getting taxable income of $503,494.75 requires three specific steps: the combined traditional+Roth elective deferrals of $27,240 per worker exceed the 2026 $24,500 401(k) limit, so the traditional deduction prorates to $20,825 each ($41,650, not $46,308); above-the-line deductions are the $3,000 capital-loss allowance plus the $2,164 traditional IRA contributions; and with no mortgage interest listed (unlisted numerics are 0), itemizing yields only the OBBBA-capped SALT of about $31,000, so the $32,200 standard deduction plus the $1,069.77 non-itemizer cash charitable deduction governs. The wrong answers split into models that assumed a TCJA sunset for 2026 (pre-TCJA brackets, Pease, personal exemptions, restored 2%-floor miscellaneous deductions, phantom AMT), models that imputed mortgage interest or re-subtracted employer health premiums from wages, and models that dropped the 3.8% NIIT or the qualified-dividend layer from the final sum." +us,scenario_004,state_refundable_credits,1,llm_error,state_local_rule,False,,"This couple has no wages or rent, so its federal AGI is only about $1,065: $1,200 of interest minus a $135 partnership loss. None of the $51,475 in Social Security is taxable, because provisional income is below the $32,000 joint base. The only NY credit this low income could trigger is the household credit under Tax Law §606(b), and that credit is nonrefundable. It can only reduce NY tax, which is already zero, and it never counts toward state_refundable_credits. None of New York's refundable credits apply either: there is no federal EITC, no qualifying child, and no property tax or rent for the real property tax credit, so the refundable total is $0." +us,scenario_005,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference uses 2026 law after OBBBA, which made the TCJA rules permanent. AGI is $536,765. Each spouse's traditional 401(k) deferral is scaled pro rata to $20,825 so that traditional plus Roth stays within the $24,500 combined limit. The frozen reference allows the $2,164 IRA deduction, excludes the $4,148 state refund and caps the net capital loss at $3,000. It allows the IRA deduction because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion: under 26 U.S.C. 219(g) both spouses' traditional 401(k) deferrals make them active participants, their modified AGI for 219(g), $543,076, is far above $149,000, the top of the 2026 joint phase-out range, and the exclusion's corrected value, $107,198.34, allows no IRA deduction. The deduction is the $32,200 MFJ standard deduction plus the new $1,070 cash-charity deduction for non-itemizers, giving taxable income of $503,495. Tax is $89,096 at 2026 brackets, plus 15% on $77,920 of qualified dividends and 3.8% NIIT on $150,578, which this variable includes; there is no AMT and no mortgage interest deduction. Wrong answers mostly (a) assumed TCJA sunset rules such as personal exemptions, Pease, miscellaneous deductions, pre-TCJA brackets and AMT, (b) imputed mortgage interest from the balance even though no interest was listed, (c) left out NIIT, or (d) came within a few hundred dollars but missed the $1,070 non-itemizer deduction, the 401(k) proration or the refund exclusion; several also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the corrected value is 107,198.34. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_005,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_005,payroll_tax,39,llm_error,state_local_rule,False,,"The $36,044 is built from four legs on $430,000 of gross wages: employee Social Security of $22,599 (6.2% on the head's full $180,000 plus the spouse's wages capped at the 2026 taxable maximum of $184,500), Medicare of $6,235 (1.45% × $430,000), Additional Medicare Tax of $1,620 (0.9% × the $180,000 of combined wages above the $250,000 joint threshold), and California SDI of $5,590 — 1.3% of the entire $430,000, because California repealed the SDI taxable wage ceiling effective 2024, making it a mandatory uncapped employee contribution. The federal legs sum to $30,454, and nearly every model reproduced that structure; the case turns almost entirely on the state leg, where models dropped SDI as ""not a mandatory employee tax,"" priced it at 1.0%–1.2%, or re-imposed the repealed per-worker wage cap. The remaining misses come from stale Social Security wage bases ($168,600, $176,100, $181,800, $183,600) and from netting the $13,369 of pre-tax employer-sponsored insurance premiums out of the FICA base, which the reference does not do — it taxes gross employment income of $430,000." +us,scenario_005,payroll_tax,42,llm_error,state_local_rule,False,,"The reference total is $36,044. It adds employee Social Security of $22,599 (6.2% × ($180,000 + the $184,500 2026 wage base)), Medicare of $6,235 (1.45% × $430,000), and Additional Medicare Tax of $1,620 (0.9% × ($430,000 − $250,000 MFJ threshold)), for $30,454. It then adds California SDI of $5,590, which is the 2026 rate of 1.3% on all $430,000 of wages with no wage ceiling. The main mistake was California SDI: many models left it out, used last year's 1.1% or 1.2% rate, or applied the wage cap that ended in 2024. Other errors were using a pre-2026 Social Security wage base (capping the head's $180,000) and subtracting employer health premiums from FICA/SDI wages, even though the facts give gross wages as the payroll tax base." us,scenario_005,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_005,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"California taxable income is $521,425.72 = CA AGI $536,764.50 less $15,338.79 of itemized deductions. The AGI is higher than the naive ""wages less the full $46,308 of traditional 401(k)"" figure of $534,270 because combined traditional ($23,154) plus Roth ($4,086) deferrals of $27,240 per filer exceed the 2026 $24,500 elective-deferral limit — only $20,825 each is deductible — while the full $2,164 traditional IRA deduction is allowed since the workplace-plan-coverage flag is unlisted and therefore false. The deduction side is just $17,280 of uncapped real estate tax plus $2,574 of charity, cut by California's 6%-of-excess-AGI high-income itemized limitation to $15,338.79: no mortgage interest is deductible (a $2,850,000 balance is listed but no interest amount, and unlisted numeric inputs are zero) and no unreimbursed-employee-expense deduction enters the base. Nearly every model fabricated $22,000–$65,000 of mortgage interest, added the employee-expense deduction, skipped the 6% cutback, or used an un-phased exemption credit instead of the $156.93 that survives phaseout." +us,scenario_005,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's 2026 CA itemized base is exactly the $17,280 of real estate taxes ($17,280 − 6% × ($536,764.50 − $504,411) = $15,338.79 to the cent), so it omits the $2,574 of charitable gifts — consistent with passing CA the federal charitable deduction after OBBBA's 0.5%-of-AGI floor, which California has not adopted under its January 1, 2015 IRC conformity date (R&TC §17024.5) — and omits unreimbursed employee expenses above 2% of AGI, which stay deductible on Schedule CA (540) because California never conformed to IRC §67(g); including both gives ~$21,025 of itemized deductions and ~$40,523 of tax.","California taxable income is $521,425.72. CA AGI is $536,764.50: federal AGI minus the $4,148 state refund. Each spouse's combined traditional+Roth 401(k) request of $27,240 is capped at the $24,500 limit, the $2,164 of IRA deductions are taken, and employer insurance premiums, tax-exempt interest, the tax-exempt pension and mortgage interest (none is listed) are all left out. From that AGI come $15,338.79 of itemized deductions: the $17,280 of real estate taxes cut by 6% of AGI above $504,411, which beats the $11,412 standard deduction. The frozen reference's projected 2026 MFJ schedule taxes this income at a top rate of 9.3%, giving $41,208.45, and its projected MFJ exemption credits, reduced by $6 per $2,500 of excess AGI, leave $156.93 to subtract. The release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $41,369.87 of tax and $150 of credits on the same income. The wrong answers come mostly from imputing mortgage interest on the $2.85M balance, subtracting employer health premiums or adding tax-exempt income, excluding the full 401(k) request, putting charity and employee expenses into the itemized base or skipping the limitation, zeroing or over-sizing the exemption credits, and misapplying the brackets. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 41,051.51; the corrected value is 40,920.40. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)." us,scenario_005,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"The case turns on two 2026 parameters and one formula. Taxable Social Security here is the §86 second-tier lesser-of amount — 0.85×($44,922.50 − $34,000) + $4,500 = $13,784.12, not the 85%-of-benefits ceiling of $16,392.25 — and provisional income excludes the $284 tax-exempt private pension, since only tax-exempt interest is added back. Applying the 2026 single standard deduction of $16,100 (the TCJA structure is permanent, so there is no personal exemption and no sunset-year schedule) to the $49,064.12 AGI leaves $32,964.12 taxable, taxed at 10% through $12,400 and 12% above, for $3,707.70. Wrong answers cluster into models that took the 85% cap, models that excluded SSDI altogether, models that used stale or pre-TCJA deduction and bracket figures, and models that derived the right number and then submitted a different one." +us,scenario_007,federal_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"Getting the reference takes two steps. First, apply the IRC §86 two-tier formula to the SSDI. Provisional income is $35,280 + $9,642.50 = $44,922.50, and it excludes the $284 tax-exempt pension, which is not tax-exempt interest. Taxable SS is the lesser of 85% of benefits or 0.85×(PI − $34,000) + min($4,500, 50% of benefits), which gives $13,784.12 and AGI of $49,064.12. Second, use the 2026 parameters that OBBBA made permanent: a $16,100 single standard deduction and a 10% bracket through $12,400. That gives taxable income of $32,964.12 and tax of $1,240 + 12% × $20,564.12 = $3,707.70, with no nonrefundable credit available. The wrong answers split into two groups. Some broke the §86 step: they taxed a flat 85% of the SSDI, used only the 50% tier, added the tax-exempt pension to provisional income, or left the SSDI out. The others used out-of-date deduction and bracket figures: 2024 or 2025 amounts, rough inflation guesses, or pre-TCJA law with a personal exemption and a 15% bracket. A few derived a figure and then submitted a different, unsupported number. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." us,scenario_007,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,head_medicare_eligible,21,prompt_ambiguity,age_disability,False,,"PolicyEngine's `is_medicare_eligible` is a pure age test against the Medicare age threshold: a person is eligible when age ≥ 65, and the variable carries no SSDI, disability, or ESRD pathway. The head is 56, so the variable resolves to False regardless of the $19,285 of Social Security disability income. Every wrong model converted an income line item into a categorical status, then imported the real-world SSDI 24-month Medicare entitlement rule — a rule that also requires a disability-onset and entitlement history the fact sheet never supplies, and that the prompt's ""treat any unlisted household fact, boolean, or status input as false"" instruction forecloses. The trap is that receipt of a disability-linked income stream is not the same as the modeled disability status, and even a true disability flag would not open a Medicare pathway in this variable. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_007,head_medicare_eligible,22,prompt_ambiguity,age_disability,False,,"The head is 56, so Medicare's age-65 test fails, and the only other route is disability entitlement. Under 42 U.S.C. 426(b), that route requires 24 months of SSDI entitlement, not just SSDI income in the current year. The household lists no disability status, no months of SSDI receipt, no ESRD and no ALS. The prompt says unlisted status inputs are false and unlisted numbers are 0, and a constant status across one tax year shows at most 12 months. All 22 models treated the $19,285 of SSDI income as proof of Medicare eligibility, either by assuming the 24-month wait was already met or by claiming PolicyEngine grants eligibility whenever SSDI is positive, so they answered Yes instead of No. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_007,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other FICA-covered earned compensation, not to Social Security disability benefits or private pension income. The prompt specifies no wage or self-employment income and requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and payroll tax is $0." us,scenario_007,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_007,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Idaho's 2026 tax has two features almost every model missed: the flat rate is 5.3% (not the superseded 5.8%/5.695%), and it applies only to Idaho taxable income in excess of an inflation-indexed zero-rate threshold of $4,920 for a single filer. With federal AGI of $49,064.12, Idaho subtracting the $13,784.12 of federally taxable Social Security, and the $16,100 standard deduction, taxable income is $19,180 and the tax is 0.053 × ($19,180 − $4,920) = $755.78. The cluster at $1,016.54 comes from models that nailed the $19,180 base and the 5.3% rate but taxed all of it; the cluster above $1,100 comes from stale rates, understated standard deductions, or failure to apply the Idaho Social Security subtraction; the zeros come from fabricated retirement-income exclusions or the claim that Idaho has no income tax." -us,scenario_007,state_refundable_credits,33,llm_error,state_local_rule,False,,"The entire answer is the Idaho Grocery Credit (Idaho Code § 63-3024A), a refundable per-person credit that requires only full-year Idaho residency without being claimed as a dependent — no income test, no phaseout, no children, and no dependence on Idaho tax liability. Idaho HB 231 (2025) raised the base amount from $120 to $155 per person effective for tax year 2025 and later, so a 56-year-old full-year resident with 12 qualifying months receives exactly $155 in 2026. The wrong answers split into two camps: those that found the right credit but priced it at a superseded statutory amount ($100 pre-2023, $120 for 2023-2024, or $140, the old age-65-plus figure), and those that zeroed the output by denying the credit exists, calling it repealed, calling it nonrefundable or liability-limited, or inventing an income phaseout. One model additionally denied that Idaho levies an income tax at all, and one returned no value." +us,scenario_007,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"Getting to Idaho taxable income is simple. Federal AGI is $49,064.12. Idaho subtracts all $13,784.12 of federally taxable Social Security, leaving $35,280, and the federal-conforming 2026 standard deduction of $16,100 brings the frozen reference's taxable income to $19,180. The frozen reference leaves out Idaho's subtraction of the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes, for taxable income of $17,100. The step that separates the reference from most wrong answers is Idaho's 2026 zero-rate band. The frozen reference taxes the first $4,920 of single taxable income, the engine's projected band, at 0% and only the remaining $14,260 at 5.3%, giving $755.78. The exclusion's corrected value uses the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026, for 5.3% × ($17,100 − $4,811) = $651.32, which gpt-6-astra matches. Most models reached $19,180 but taxed it from the first dollar, at 5.3% ($1,016.54) or at the old 5.695%/5.8% rates. Others made earlier mistakes: they skipped the Social Security subtraction, used a pre-OBBBA or TCJA-sunset standard deduction, or invented exclusions. Near misses differ from the frozen reference only in the zero-rate band: a guessed one ($4,990 or $5,000), or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 755.78; the corrected value is 651.32. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)." +us,scenario_007,state_refundable_credits,36,llm_error,thresholds_rates,False,,"Idaho's only refundable state credit for this household is the Grocery Credit. For 2026 it is $155 for a resident under 65. It has no income phaseout, and it counts as refundable whether or not the filer owes Idaho tax. The head received no SNAP, so all 12 months qualify and the full $155 is refundable. Eighteen models knew the credit exists but used an outdated amount: $100 (pre-2023), $120 (2023) or $140 (the old 65+ amount, or an invented inflation adjustment). Seventeen models answered $0. They said the credit is nonrefundable, used up against tax liability, phased out by income, repealed, or that Idaho has no income tax. One model returned no answer." us,scenario_008,child1_chip_eligible,9,llm_error,health_coverage,False,,"CHIP is a residual program: it covers children who do NOT qualify for Medicaid, so its income band has a floor at the state's children's Medicaid limit, not just a ~350% FPL ceiling. With household income of about $30,915 for a family of eight — roughly 54% of FPL — the 17-year-old clears New Jersey's Medicaid test in the OLDER_CHILD category, and that Medicaid eligibility forecloses CHIP. All nine models tested only the upper CHIP ceiling and read very low income as evidence for CHIP, when low income is exactly what routes the child to Medicaid instead." us,scenario_008,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A 17-year-old tax-unit dependent falls within New Jersey's OLDER_CHILD Medicaid category. The household's MAGI is 0.55 times FPL, below the category's income threshold, so the child is eligible without any additional eligibility trigger." us,scenario_008,child1_wic_eligible,2,llm_error,categorical_eligibility,False,,WIC child eligibility is restricted to infants and children younger than age five; a 17-year-old is categorically ineligible regardless of household income. Both models incorrectly treated age 17 as within the WIC child age range. @@ -51,7 +52,7 @@ us,scenario_008,child4_medicaid_eligible,1,llm_error,categorical_eligibility,Fal us,scenario_008,child4_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old is categorically ineligible. The household’s approximately $30,915 of income also exceeds the applicable WIC income ceiling in PolicyEngine’s computation, providing an independent income-based disqualification." us,scenario_008,child5_chip_eligible,11,llm_error,health_coverage,False,,"CHIP is residual coverage: a child who qualifies for Medicaid is barred from CHIP, so the binding step is the Medicaid screen, not the CHIP ceiling. Child5 is 5 years old, and household income of about $30,915 for an 8-person unit (~54-59% FPL) sits far below New Jersey's Medicaid threshold for the under-6 (YOUNG_CHILD) age band, so the child is Medicaid-eligible and therefore CHIP-ineligible. Every wrong model ran a single test — is income below NJ FamilyCare's ~355% FPL child limit — and answered yes, inverting the logic: the further below that ceiling the household sits, the more certainly Medicaid captures the child and forecloses CHIP. None applied the Medicaid-first crowd-out or the separate, lower under-6 Medicaid income band that the child's age triggers." us,scenario_008,child5_early_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Early Head Start serves infants and toddlers under age 3, while Head Start is the distinct program for preschool-age children. Child 5 is age 5 and therefore fails the Early Head Start age requirement regardless of household income or housing-assistance receipt." -us,scenario_008,child5_head_start_eligible,4,llm_error,age_disability,False,,"The Head Start age rule used by this benchmark includes a 5-year-old; age five does not automatically disqualify the child. Child 5 also satisfies Head Start’s SNAP categorical pathway and direct income test, so applying the inclusive age rule yields eligibility." +us,scenario_008,child5_head_start_eligible,5,llm_error,age_disability,False,,"Every model rejected Child 5 on age alone, treating preschool Head Start as serving only 3- and 4-year-olds. Under 45 CFR 1302.12(b)(1) the range runs from age 3 up to the age of compulsory school attendance, which is 6 in New Jersey, and PolicyEngine's Head Start age window includes 5-year-olds. The age test is met. The child is also income-eligible, both categorically through the household's SNAP receipt and directly because household income of about $30.6K is well under the poverty-guideline threshold for a family of 8. So the answer is eligible. No model reached the income or categorical test, because each stopped at an age cutoff that was set too low." us,scenario_008,child5_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A five-year-old in New Jersey falls within the YOUNG_CHILD Medicaid category. At MAGI equal to 0.55 times FPL, the child satisfies that category's income test and is Medicaid eligible." us,scenario_008,child5_wic_eligible,12,llm_error,age_disability,False,,"WIC's categorical definition of a child covers a person who has reached the first birthday but not the fifth, so a child listed as age 5 has aged out of the child category entirely and never reaches the 185%-of-poverty income screen. Income is not the discriminator in this household: on the identical ~$30,915 of income, the age-1 child is WIC-eligible while the age-5 child is not. Every wrong model treated ""age 5"" as inside the range — several stating the correct ""under 5"" rule and then reversing it, others stretching the boundary to ""through age 5"" — and then reinforced the wrong answer with an income or adjunctive-eligibility finding that cannot cure a categorical failure. Adjunctive eligibility through SNAP, Medicaid, TANF, or housing assistance substitutes only for the income test, never for the age category." us,scenario_008,child6_chip_eligible,12,llm_error,health_coverage,False,,"CHIP is residual coverage: PolicyEngine returns is_chip_eligible False for any child who already qualifies for Medicaid. The 1-year-old in this eight-person New Jersey household, with income near $30,915 (roughly 54% FPL), falls squarely inside the YOUNG_CHILD Medicaid category, so the Medicaid flag is 1 and the CHIP flag is 0. All twelve models tested only the upper NJ FamilyCare CHIP ceiling (300–355% FPL), found the household far beneath it, and treated that as proof of CHIP eligibility — when clearing that ceiling by such a margin is exactly what places the child under the lower young-child Medicaid threshold. Several went further and merged Medicaid and CHIP into a single ""NJ FamilyCare"" eligibility, erasing the mutual exclusivity that decides this output." @@ -59,23 +60,23 @@ us,scenario_008,child6_early_head_start_eligible,1,llm_error,categorical_eligibi us,scenario_008,child6_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Head Start requires both preschool-age categorical eligibility and satisfaction of the program's income or another qualifying eligibility pathway. Child 6 is age 1 and therefore belongs in the Early Head Start age category, not the preschool-age Head Start category; the household also exceeds the applicable income cutoff." us,scenario_008,child6_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid applies a YOUNG_CHILD eligibility pathway to the 1-year-old dependent. At a MAGI income level of 0.55 times FPL, the child satisfies that category's income test and is eligible without any additional benchmark-triggering fact." us,scenario_008,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"The reference is the $8,231 maximum 2026 EITC for three or more qualifying children plus a refundable CTC of exactly 15% x ($30,517.41 - $2,500) = $4,202.61. Two steps separate it from the wrong answers: at $30,517 of joint earned income the 3+-child EITC is still on its plateau, so no phase-out reduction applies, and the CTC phase-in base counts self-employment income net of the deductible half of SE tax ($4,000 - $282.59 = $3,717.41), not gross. Most models either subtracted a phase-out the household never reaches, imported a stale-year EITC maximum ($8,046) or the repealed $3,000 CTC phase-in floor, or ran the phase-in on $30,800 of gross earnings and landed at $4,245. A smaller group added a refundable AOTC despite zero listed tuition expenses, or reported $0 by denying qualifying children or treating refundable credits as unavailable at zero liability." +us,scenario_008,federal_refundable_credits,41,llm_error,thresholds_rates,False,,"The reference adds two pieces. The first is the 2026 EITC maximum for three or more children, $8,231, with no reduction because AGI of about $30,632 is below the joint-filer phase-out start. The second is the refundable CTC: 15% x ($30,517.41 - $2,500) = $4,202.61 for the five children under 17. Earned income is wages plus self-employment income minus the half-SE-tax deduction ($4,000 - $282.59). Models went wrong in three recurring ways: a stale, inflated or phased-out EITC; earned income taken as gross $30,800 or as 0.9235 x SE income; and the $3,000 ACTC threshold instead of $2,500. A few also added a refundable AOTC even though qualified education expenses are $0." us,scenario_008,free_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Free school meals do not require an explicitly listed direct-certification indicator in the household inputs. The household qualifies through both routes established by the computation: its school-meal income is 55% of the federal poverty guideline, below the 130% free-tier threshold, and all household members are SNAP-eligible, which supplies categorical eligibility." us,scenario_008,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA Medicaid expansion, so any non-elderly adult (19-64) with MAGI at or below 138% FPL for their household size qualifies through the MAGI-based ADULT category — no disability, pregnancy, or caretaker status required. This household of eight has total countable MAGI income of $30,915 ($4,000 self-employment + $15 interest + $26,800 wages + $100 interest), which is 0.55 x the eight-person FPL (roughly $55,000 in 2026), far under the 138% line of roughly $76,000. Every wrong model either compared that income to a threshold sized for a much smaller household, demanded a pre-ACA categorical hook, refused to compute eligibility absent an explicit input flag, or invented a housing-assistance disqualification that does not exist in MAGI Medicaid." us,scenario_008,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not conferred on every household member by the presence of an eligible infant or young child. The 42-year-old Head is not pregnant, postpartum, or breastfeeding and is not an infant or child under age 5, so satisfying the household income limit does not make the Head eligible." us,scenario_008,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,payroll_tax,39,llm_error,state_local_rule,False,,"Federal employee FICA on the spouse's $26,800 of wages is uncontested at $1,661.60 Social Security plus $388.60 Medicare = $2,050.20, so the entire spread comes from New Jersey's mandatory employee-side contributions. This measure adds exactly two NJ items on the full $26,800 (well under the taxable wage base): temporary disability insurance at 0.19% = $50.92 and family leave insurance at 0.23% = $61.64, for $112.56 and a total of $2,162.76. It excludes the employee UI/workforce-development contribution (0.3825%/0.0425%), which is the item most models reached for first. The wrong answers split into three modes: dropping the NJ portion entirely to land on $2,050.20, asserting the NJ employee TDI rate is 0% and substituting UI/WF, or bolting on a UI-inclusive NJ rate of 0.5%-1.0% that overshoots the in-scope 0.42% combined TDI+FLI rate." +us,scenario_008,payroll_tax,46,reference_engine_defect,payroll_tax_base,False,,"Federal employee FICA on the $26,800 of wages is $2,050.20, and nearly every model got that part right. What separates the answers is New Jersey's mandatory employee payroll tax. The reference adds NJ temporary disability insurance at 0.19% ($50.92) and family leave insurance at 0.23% ($61.64), for $112.56. It counts no NJ unemployment, workforce or supplemental workforce fund employee contribution, although New Jersey workers pay 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds; the exclusion's corrected value counts them ($113.90 more), for $2,276.66. Most models either stopped at FICA ($2,050.20) or added the 0.3825–0.425% UI/WF/SWF contribution, often with outdated or inflated TDI/FLI rates (0% TDI, 0.09% FLI, or 0.33% FLI). gpt-6-astra and gpt-6-sol match the exclusion's corrected value, $2,276.66. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (payroll_tax_base). The frozen reference is 2,162.76; the corrected value is 2,276.66. PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax. Reference is an engine defect; output excluded from scoring (N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates)." us,scenario_008,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a generic indicator that income is below the 185% FPL ceiling. This household is at 55% of the federal poverty guideline and also meets categorical eligibility, so it receives the superior free-meals tier and reduced_price_school_meals_eligible equals 0." us,scenario_008,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax is 15.3% of 92.35% of the $4,000 net self-employment income: $4,000 × 0.9235 × 0.153 = $565.18. The deduction for one-half of self-employment tax is an income-tax deduction and does not reduce the self-employment tax liability itself." -us,scenario_008,snap,39,llm_error,period_annualization,False,,"The benefit is the maximum allotment for eight minus 30% of net income, and net income here is gross $2,576.25 less exactly two deductions totaling $812.33: the 20% earned-income deduction ($513.33) and the large-household standard deduction ($299). Excess shelter is zero because $579.08 monthly rent falls below half of adjusted income ($881.96) and no utility expense is listed, and the disabled spouse's $300 of annual OTC expenses is $25/month, under the $35/month excess-medical threshold. The two traps that separate the reference from the wrong answers are the maximum allotment schedule — $1,789/month through September 2026 and about $1,829/month from the October 2026 adjustment, so calendar 2026 blends nine months at ~$1,260.83 with three at ~$1,300.83 — and TANF non-cash categorical eligibility, which waives the resource test despite $69,500 in bank assets. The tight near-miss cluster at $15,117.90 is exactly $1,259.83 × 12: the whole derivation right, the October step-up dropped." +us,scenario_008,snap,45,llm_error,thresholds_rates,False,,"The correct benefit takes $2,576.25 in monthly gross income and subtracts the $513.33 earned income deduction and the FY2026 standard deduction of $299 for households of six or more. That leaves net income of $1,763.92, which rounds to $1,764. The 30% contribution is rounded up to $530, and the FY2026 maximum allotment for eight people is $1,789, so the benefit is $1,259 a month, or $15,108 a year. No shelter deduction applies: rent of $579 a month is below half of adjusted income, and no utility costs are listed. No medical deduction applies either: the spouse's over-the-counter costs are $25 a month, under the $35 threshold. NJ broad-based categorical eligibility waives the asset test. The wrong answers fall into four groups: stale or invented parameters (the FY2025 $1,756 maximum, standard deductions of $198 to $291); skipping the round-up of net income and the 30% contribution ($1,259.82 or $1,260 instead of $1,259); subtracting a monthly standard deduction once from annual income; and zeroing SNAP through an asset test, housing assistance or a wrong gross-income limit." us,scenario_008,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA adult expansion, so a non-elderly adult 19-64 who is not a tax-unit dependent qualifies on the MAGI pathway at or below 138% FPL, with no categorical hook, no asset test, and no benefit-receipt disqualifier. The test compares household MAGI to the FPL for the full household size: roughly $30,900 of wages, self-employment income and interest against the eight-person FPL is 0.55 x FPL, less than half the 1.38 limit. Housing assistance is excluded from MAGI and the $69,500 in bank accounts is irrelevant because MAGI categories carry no resource test. Each wrong answer either skipped this income test entirely or overrode it with a disqualifier that does not exist." us,scenario_008,spouse_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility here turns on a single age test: the person must be 65 or older, and the Spouse is 41. The trap is the `is_disabled` flag: every wrong model converted it into a disability-based Medicare pathway, but real-world under-65 Medicare entitlement runs through 24 months of SSDI cash benefits (or ESRD/ALS), none of which is present — the household reports no Social Security disability income, and the prompt directs that unlisted income and status inputs be treated as 0/false. A disability flag alone establishes neither SSDI entitlement nor the 24-month qualifying period, so it cannot produce Medicare eligibility for a 41-year-old." us,scenario_008,spouse_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific: an adult must be pregnant, breastfeeding, or within the applicable postpartum period, while infants and children qualify only through their own age category. Low household income and the presence of a young child do not make the child's mother or caretaker independently eligible." us,scenario_008,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_008,state_income_tax_before_refundable_credits,37,llm_error,state_local_rule,False,,"New Jersey gross income of $30,915 (wages $26,800 + self-employment $4,000 + interest $115) reduces by exactly $12,000 of exemptions: $1,000 for each joint filer, $1,500 for each of the six dependents, and one $1,000 addition for the spouse who is blind or disabled. The resulting $18,915 sits entirely inside the married-joint 1.4% first bracket, giving $264.81. Four traps separate the reference from the wrong answers: the blind/disabled addition is $1,000 per qualifying person rather than $1,000 for blindness plus $1,000 for disability; the regular taxpayer exemption is $1,000, not the $1,500 dependent rate; NJ has no standard deduction, so low income does not zero the liability; and neither the $800 of over-the-counter health spending nor the $6,949 of pre-subsidy rent produces a medical deduction, an 18%-of-rent tenant deduction, or a $50 tenant credit against this output." -us,scenario_008,state_refundable_credits,39,llm_error,credit_phaseout,False,,"The household's ~$30,800 of earned income sits on the plateau of the 2026 federal EITC schedule for married-joint filers with three or more qualifying children: the joint phase-out does not begin until roughly $31,000, so the full $8,231 credit applies and the NJ EITC is a flat 40% of it, $3,292.40, with no separate state phase-out. New Jersey's refundable CTC is keyed to NJ taxable income, which is $18,915 here, placing the household in the top bracket at $1,000 per child under age 6; both the 5-year-old and the 1-year-old are under 6, giving $2,000. The third component is the $50 refundable NJ property tax credit, granted to this rent-paying household on the age/tenant pathway. Wrong answers came from phasing out an EITC that is not in phase-out range, applying a lower NJ CTC bracket keyed to gross rather than NJ taxable income (or counting one child under 6 instead of two), and dropping the $50 tenant credit." +us,scenario_008,state_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"NJ gross income is $30,915. That is wages plus Schedule C profit plus interest, with no federal half-SE-tax adjustment and no standard deduction. Exemptions total exactly $12,000: $1,000 for each spouse, one $1,000 blind-or-disabled exemption for the spouse (a person who is both blind and disabled still gets only one), and 6 × $1,500 for the dependents. That leaves $18,915 taxable, and 1.4% of that is $264.81. The models went wrong in four recurring ways. Some counted blind and disabled as two exemptions or valued the personal and dependent exemptions wrong. Some deducted over-the-counter health expenses, which don't qualify for NJ's medical deduction. Some took the 18%-of-rent property tax deduction. Some subtracted the $50 property tax credit. That credit is refundable and is worth more than the deduction's $17.51 tax saving, so it is claimed below this line and the rent deduction isn't taken. A final group invented a standard deduction, credits or a low-income phase-out that drove the tax to zero." +us,scenario_008,state_refundable_credits,46,llm_error,state_local_rule,False,,"The reference has three parts. The NJ EITC is 40% of the full $8,231 federal 3+-child EITC, or $3,292.40, because this married-filing-jointly household is below the 2026 federal phase-out start. The NJ Child Tax Credit is $2,500 for the two children aged 5 and 1, under the schedule New Jersey enacted June 30, 2026 for tax years 2026–2028. That schedule's tiers use NJ taxable income, which is about $18,900 after $12,000 of exemptions, not gross income of about $30,900. The last part is the $50 refundable property tax credit for tenants, since $6,949 of rent is listed. Most models used the old $1,000-per-child schedule, or wrongly put the family in the $800 or $400 tier based on gross income. Many also left out the $50 property tax credit, phased out the federal EITC by mistake, or said NJ has no refundable credits. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors." us,scenario_008,tanf,1,llm_error,thresholds_rates,False,,"New Jersey's WorkFirst NJ TANF is not a residual top-up: it is gated on countable income falling below the family-size eligibility standard tied to the maximum grant (roughly $770/month for a household of eight) and on countable resources at or below the $2,000 limit. This household's $26,800 in wages plus $4,000 of self-employment income is about $2,576/month, which clears the standard by a wide margin even after New Jersey's 50% earned-income disregard, and the $69,500 bank balance independently fails the resource test. Six children plus a blind and disabled spouse make the household read as categorically needy, which is the trap; family size raises the grant schedule but nowhere near enough to admit income at this level. The grant is therefore $0." -us,scenario_009,federal_income_tax_before_refundable_credits,31,llm_error,thresholds_rates,False,,"2026 is governed by OBBBA's permanent extension of the TCJA schedule: the MFJ standard deduction is $32,200, personal exemptions remain repealed, and the brackets run 10% to $24,800 then 12% to $100,800, so $73,700 of taxable income yields $2,480 + 12% × $48,900 = $8,348. Nearly every model reached the correct $105,900 AGI — full $60,000 wages, $20,400 taxable pension, and the $25,500 85% cap on the $30,000 Social Security benefit — and then lost the case on the deduction and bracket parameters. One group applied a TCJA sunset (roughly $16,000–$17,000 standard deduction plus personal exemptions and a 15% second bracket), a second carried 2025's $30,000 deduction and $23,850 bracket ceiling forward, and a third subtracted the spouse's $8,139 employer-sponsored insurance premium from wages, which is not an AGI reduction for this household. A fourth group derived $8,3xx correctly in its own reasoning and then emitted an unrelated number in the value field." +us,scenario_009,federal_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"The correct path is: AGI of $105,900 ($60,000 gross wages, $20,400 taxable pension, and $25,500 of Social Security, the 85% cap), minus the 2026 MFJ standard deduction of $32,200, gives taxable income of $73,700. Under the 2026 brackets (10% to $24,800, 12% to $100,800), with no nonrefundable credits, tax is $2,480 + $5,868 = $8,348. The wrong answers fall into four groups. Some models assumed TCJA expired in 2026 and applied personal exemptions, a roughly $16,000 standard deduction and a 15% bracket, landing near $10,500. Others inflated the pre-OBBBA $30,000 standard deduction and the 2025 brackets instead of using $32,200 and $24,800, landing near $8,500. A third group subtracted the spouse's $8,139 employer-sponsored insurance premium from gross wages. Finally, several Claude models derived about $8,300-$8,349 and then submitted an unrelated number." us,scenario_009,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_009,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable under-65 pathway such as qualifying disability, ESRD, or ALS. Both adults are 57, and the prompt makes every unlisted disability or health status false, so the listed Social Security retirement income cannot be reclassified as evidence of disability-based Medicare eligibility." us,scenario_009,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -83,42 +84,43 @@ us,scenario_009,payroll_tax,7,llm_error,payroll_tax_base,False,,"The $8,139 empl us,scenario_009,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_009,spouse_medicare_eligible,1,llm_error,other,False,,"Medicare eligibility is absent because the spouse is age 57, below the standard eligibility age of 65, and no alternative eligibility status was listed. The required binary encoding assigns 0 to “No” and 1 to “Yes,” so the spouse’s result is 0." us,scenario_009,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_009,state_income_tax_before_refundable_credits,25,llm_error,taxable_income_or_deductions,False,,"The reference chain is short and fully determined: federal AGI of $105,900 (the stated $60,000 gross wages with no offset for employer-sponsored insurance premiums, $20,400 taxable pension, and $25,500 of Social Security taxable under the 85% cap), less NC's subtraction of that same $25,500 of federally taxable Social Security and the $25,500 married-filing-jointly standard deduction, leaving $54,900 taxed at North Carolina's 3.99% flat rate for tax year 2026. Three parameters decide the answer — the 3.99% rate, the $25,500 NC standard deduction, and the fact that NC subtracts the $25,500 of taxable Social Security actually included in AGI rather than the $30,000 gross benefit — and the AGI input must keep wages at gross. The wrong answers split into three groups: models that netted the $8,139 ESI premium (or a guessed mortgage-interest itemized deduction) out of the income base, models that used a stale 4.25% or 4.5% rate or a non-NC standard deduction ($21,000, $26,000, $26,500, $29,250, $30,000), and models that invented NC credits or exclusions to zero the liability. A fourth group derived $2,190.51 correctly in their reasoning and then submitted a different number." +us,scenario_009,state_income_tax_before_refundable_credits,26,llm_error,taxable_income_or_deductions,False,,"The reference result comes from four steps. Federal AGI is $105,900: the spouse's full $60,000 gross wages, plus the $20,400 taxable pension, plus $25,500 taxable Social Security. North Carolina then subtracts the $25,500 of Social Security and the $25,500 married-filing-jointly standard deduction, which leaves $54,900 of taxable income. The 2026 flat rate of 3.99% on $54,900 gives $2,190.51, and no nonrefundable credits apply. The wrong answers split into two main groups. The first group cut the spouse's wages by the $8,139 employer-sponsored insurance premium, giving an AGI of $97,761 and taxable income of $46,761; some of these also imputed itemized deductions. The second group used a stale flat rate (4.25%, 4.5%) or a wrong standard deduction ($21,000 to $30,000). A few models computed $2,190.51 correctly and then submitted a different number." us,scenario_009,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_009,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_012,child1_chip_eligible,9,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying CHIP age and income limits alone is not sufficient. The 10-year-old qualifies for Medicaid under Mississippi's OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility." us,scenario_012,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi evaluates a 10-year-old dependent through the Medicaid older-child MAGI category. Household MAGI is 0.81 times FPL, below the applicable eligibility threshold, so the category and income tests produce eligibility." us,scenario_012,child1_wic_eligible,3,llm_error,categorical_eligibility,False,,WIC child eligibility is limited to children under age five; low household income does not extend eligibility to older children. Child 1 is age 10 and therefore fails the categorical age requirement before the income test can establish eligibility. us,scenario_012,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_012,federal_refundable_credits,34,llm_error,thresholds_rates,False,,"The case turns on two 2026 parameters: the inflation-indexed one-child EITC maximum of $4,427, which applies in full because $22,000 of earnings sits past the phase-in end and below the joint phaseout start above $31,000, and the $1,700 per-child refundable CTC cap, which binds because 15% × ($22,000 − $2,500) = $2,925 exceeds it while tax liability is zero under the $32,200 joint standard deduction. Nearly every wrong answer decomposes as EITC + ACTC with one or both legs wrong: stale 2025 EITC ceilings ($4,328), invented inflation projections, the repealed pre-TCJA $1,000/$3,000-floor ACTC, an $1,800 or $2,000 or fully refundable $2,200 child credit, or a phantom phaseout at $22,000. A second cluster counted the pregnant 18-year-old spouse or the unborn child as a qualifying child, moving to two- or three-child schedules that federal law does not allow." -us,scenario_012,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The household receives SNAP, which categorically qualifies the K-12 child for free school meals. Independently, the household's school-meal income ratio is 0.81, below the free-meal threshold of 1.30, so both qualifying routes yield the FREE tier." +us,scenario_012,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household files jointly with one qualifying child (age 10). $22,000 of earnings sits on the one-child EITC plateau, which is past the phase-in end of about $13,000 and below the MFJ phase-out start, so the full 2026 maximum of $4,427 applies. Tax liability is zero because $22,000 is below the MFJ standard deduction, so the OBBBA-permanent $2,200 CTC is refundable only up to the 2026 ACTC cap of $1,700. The 15% × ($22,000 − $2,500) = $2,925 earnings test does not bind. Most wrong answers used the wrong parameter year or regime. They used the 2025 EITC maximum ($4,328), a hand-inflated EITC, the TCJA-sunset ACTC ($1,000 cap above $3,000), a $1,800 or $2,000+ refundable cap, or counted the unborn child or the pregnant spouse as a qualifying child." +us,scenario_012,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"This three-person Mississippi household has $22,000 in wages, which is 81% of the federal poverty guideline. That is far below the 130% free-meal cutoff. The household also receives about $407.90 a month in SNAP, which makes the 10-year-old categorically eligible for free meals, and PolicyEngine treats a 10-year-old as enrolled in K-12. So the engine puts the household in the FREE tier. The wrong answers came from three different errors: one model never ran the income or SNAP test, one treated the child's school enrollment as an unlisted input set to false, and one wrongly claimed income was above 130% of the poverty guideline." us,scenario_012,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Mississippi Medicaid eligibility for a nondisabled 40-year-old adult requires qualification through a defined categorical pathway; income below 138% FPL does not independently establish eligibility. The head’s MAGI is 0.81 FPL, but the Medicaid category is NONE because the head satisfies none of Mississippi’s available pathways." us,scenario_012,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not extended to every member of a household containing a pregnant participant. The pregnant spouse satisfies the categorical requirement, but the 40-year-old head is neither pregnant, postpartum, breastfeeding, an infant, nor a child under age five, so the head is not WIC-eligible despite household income below the limit." us,scenario_012,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_012,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_012,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. At a school-meal FPG ratio of 0.81 and with categorical eligibility, the household is classified as FREE, so it receives no reduced-price support." -us,scenario_012,snap,39,llm_error,taxable_income_or_deductions,False,,"The trap is the $411 SPM unit energy subsidy: most models read it as LIHEAP receipt that confers a Standard Utility Allowance, added a $360–$540 phantom utility expense to the $450 monthly rent, and thereby manufactured an excess-shelter deduction. With no utility expense input, deductible shelter is rent alone, and $450 is below half of the $1,257.67 income remaining after the 20% earned-income deduction ($366.67) and the FY2026 $209 standard deduction, so the excess shelter deduction is $0 and the monthly allotment is $785 − 30% × $1,257.67 = $407.90. A second trap sized the unit at four by counting the spouse's unborn child, which SNAP does not do, pulling in the four-person $973–$994 maximum allotment. A third separated the near-misses: calendar year 2026 straddles two federal fiscal years, so October–December 2026 use FY2027 standard deductions and poverty guidelines, lifting the annual total from 12 × $407.90 to $4,952.09." +us,scenario_012,snap,45,llm_error,taxable_income_or_deductions,False,,"This is a three-person SNAP unit. The unborn child does not count as a member. Gross income is $1,833.33 a month. Subtracting the $209 standard deduction and the $366.67 earned income deduction (20% of earnings) gives net income of $1,258 after rounding. Under the 2026 rule (P.L. 119-21), receiving an energy (LIHEAP) subsidy gives a Standard Utility Allowance only to households with an elderly or disabled member, so this household gets no utility allowance. Its $450 rent is below half of net income, so there is no excess shelter deduction. Thirty percent of net income, rounded up, is $378, and $785 − $378 = $407 a month, or $4,884 a year. Most wrong answers either granted a LIHEAP-triggered utility allowance and shelter deduction, counted the unborn child to make a household of four, used outdated standard-deduction or maximum-allotment figures, or skipped SNAP's whole-dollar rounding (30% of net income rounded up, allotment rounded down) and got $4,892–$4,896." us,scenario_012,spouse_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person not already qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under Mississippi's OLDER_CHILD category, so that Medicaid eligibility precludes CHIP regardless of pregnancy status or how far household income falls below a CHIP-related threshold." us,scenario_012,spouse_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi Medicaid applies an OLDER_CHILD eligibility pathway to the 18-year-old spouse. The spouse's MAGI of 0.67 times FPL is below the applicable income threshold, so this category produces eligibility despite the spouse not being a tax-unit dependent." us,scenario_012,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC covers pregnant individuals who satisfy the program's income test. The spouse is pregnant, and the household's approximately $22,000 annual income is below the applicable 185% federal-poverty income limit, so the spouse meets both categorical and financial eligibility requirements." us,scenario_012,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_012,state_income_tax_before_refundable_credits,8,llm_error,thresholds_rates,False,,"Mississippi's flat individual income tax reaches only taxable income above the $10,000 zero-rate amount retained through the HB 531 phase-down, and for 2026 the rate on the excess is 4.0%. This household's base is $22,000 in wages less the $4,600 married-filing-jointly standard deduction, the $12,000 joint personal exemption, and the $1,500 dependent exemption for the 10-year-old, leaving $3,900 — entirely inside the untaxed first $10,000, so state income tax before refundable credits is $0. Every wrong model multiplied a rate against its whole taxable-income figure instead of only the portion above $10,000, and most also mis-stated Mississippi's exemption amounts or its 2026 rate." +us,scenario_012,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"Mississippi taxable income for this married-joint household is $22,000 minus the $4,600 MFJ standard deduction, the $12,000 MFJ personal exemption, and the $1,500 exemption for the one dependent child, which leaves $3,900. Mississippi taxes the first $10,000 of taxable income at 0% and applies its flat rate (4.0% in 2026) only above $10,000, so the joint liability is $0. PolicyEngine's per-spouse allocation gives the head $118 (4% of $12,950 minus $10,000), but the unit-level result is $0. Four models computed the $3,900 base correctly and then taxed it from the first dollar. The rest also got the deductions or exemptions wrong, and one of them used another state's credit structure." us,scenario_013,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The trap is the Social Security provisional-income test: half of the $23,736 benefit ($11,868) plus the $6,720 pension, $16 of dividends and $72 of capital gain gives provisional income of $18,676, below the $25,000 first base amount for a single filer, so none of the Social Security is includible. AGI is therefore $6,808, which is under the single standard deduction before any age-65 additional amount or the OBBBA senior deduction is even reached, leaving taxable income and tax of $0. A second, independent guardrail is that federal income tax before refundable credits is floored at zero — nonrefundable credits can only zero out liability, never drive it negative." -us,scenario_013,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Age and disability do not themselves establish Arizona Medicaid eligibility; the person must qualify under a specific MAGI or non-MAGI category. At 1.91 times FPL, the head fails the applicable MAGI income pathways, receives no SSI, and qualifies through none of the aged, disabled, institutional-care, or state-specific pathways, producing Medicaid category NONE." +us,scenario_013,head_medicaid_eligible,3,llm_error,age_disability,False,,"The head is 80, so the 138% FPL MAGI adult expansion group (ages 19-64) is closed to them. The only route left is the aged/blind/disabled pathway, which counts Social Security in full. Social Security of $23,736 plus the $6,720 pension and $88 of investment income comes to about $30,544, or 1.91x FPL. That is far above the SSI benefit rate and the 100% FPL aged/disabled standard, and with SSI at 0 there is no SSI-linked eligibility either, so the engine returns category NONE. All three models answered Yes. One asserted eligibility without testing income, one excluded most of the Social Security, and one used AGI, which leaves out the nontaxable Social Security." +us,scenario_013,snap,46,llm_error,categorical_eligibility,False,,"Countable gross income is $2,539.33 a month. That is Social Security, pension and dividends; the $72 of capital gains is not counted. From March 2026, Arizona's expanded categorical eligibility limit is 200% FPL: $2,608.33 a month, rising to $2,660 from October. Because income is under that limit, the household is categorically eligible, and the asset test and net income test are treated as met. A categorically eligible one-person household gets the $24 minimum allotment even when $298 minus 30% of net income is below zero, so benefits are $24 × 10 months (March–December) = $240. Every model either applied the regular asset or net income tests, used the old 185% limit, or stopped when the benefit formula came out at zero, and never applied the minimum benefit to a categorically eligible household." us,scenario_013,state_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Arizona’s standard deduction eliminates the remaining taxable income after excluding Social Security, so applying the 2.5% rate directly to gross dividends and capital gains is incorrect. The resulting Arizona income tax before refundable credits is zero." -us,scenario_013,state_refundable_credits,32,llm_error,state_local_rule,False,,"Arizona's increased excise tax credit (A.R.S. § 43-1072) is a refundable credit equal to $25 per resident person on the return — taxpayer, spouse, and dependents — capped at $100, allowed whenever federal AGI is at or below $12,500 for single/MFS ($25,000 for MFJ/HoH). This household's federal AGI is $6,736, and the one-person unit therefore earns exactly one $25 unit. The trap is that the credit requires no dependents, no earned income, no rent or property tax, and no state tax liability to offset — the conditions the models kept importing from Arizona's nonrefundable family income tax credit, the QCO ""working poor"" charitable credit, and the Form 140PTC property tax credit. Every wrong answer either denied that any refundable Arizona credit exists, grafted one of those other programs' prerequisites onto it, or multiplied the $25 by exemptions the statute does not count." -us,scenario_014,federal_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"The case turns on the 2026 parameters actually in force for a joint return with a single $88,927.65 wage earner: a $32,200 MFJ standard deduction, no personal exemptions, and brackets of 10% to $24,800 then 12%, giving $88,927.65 − $32,200 = $56,727.65 taxable and $2,480 + 12% × $31,927.65 = $6,311.32. Two shared traps drove the misses: assuming the TCJA individual provisions lapse in 2026 (restoring personal exemptions and a 15% second bracket) or carrying 2025 amounts ($30,000 / $23,850) forward, and subtracting the spouse's $14,717 employer-sponsored-insurance premium from wages, when that input never reduces PolicyEngine's employment income. Veterans benefits of $76,176 and $18,480 are excluded from gross income and the $3,000 of child support paid is nondeductible, so neither changes the base. A separate cluster of models derived a figure at or near $6,311 and then submitted a different number." +us,scenario_013,state_refundable_credits,36,llm_error,state_local_rule,False,,"The entire $25 is Arizona's refundable Credit for Increased Excise Taxes (A.R.S. 43-1072.01). It pays $25 per person in the tax unit (taxpayer, spouse and dependents, up to $100) when AGI is at or below $12,500 for a single filer. Because it is refundable, it does not depend on tax liability, earned income, dependents, rent or property tax. This single filer's AGI is $6,736, since Social Security is excluded, so the tax unit of one gets $25. Almost every model left this credit out and assumed Arizona's refundable credits require dependents, earnings or property-tax/rent payments. Of the rest, two tested the wrong income, two misread the credit's conditions, and one multiplied the $25 by age and disability exemptions." +us,scenario_014,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The only taxable income is the spouse's $88,927.65 of gross wages. Veterans benefits are excluded, and the listed ESI premium does not reduce the stated gross wages. Under 2026 law as amended by OBBBA, the TCJA structure is permanent: the MFJ standard deduction is $32,200, personal exemptions are zero, and the brackets are 10% to $24,800 and 12% to $100,800. That gives taxable income of $56,727.65 and tax of $2,480 + 12% x $31,927.65 = $6,311.32, with no nonrefundable credits. Most wrong answers came from assuming the TCJA sunset (pre-TCJA deduction, personal exemptions, 15% bracket), using 2025 or guessed deduction and bracket figures, subtracting the $14,717 premium from wages, or replacing a correct calculation with a different number." us,scenario_014,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"Federal refundable credits are zero because this childless joint tax unit has income far above the EITC phase-out range and has no qualifying child for refundable CTC. No education, rebate, or refundable payroll-tax-credit facts create another refundable federal credit." us,scenario_014,head_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"Medicaid eligibility for the head uses household tax-unit MAGI rather than only the head's earned income. The household's MAGI is approximately $183,584, or 4.11 times FPL, exceeding every applicable West Virginia Medicaid income limit, and the head has no alternative categorical pathway." -us,scenario_014,payroll_tax,15,llm_error,payroll_tax_base,False,,"The only wage earner is the spouse, and the employee-side payroll tax is 6.2% Social Security plus 1.45% Medicare on the full $88,928 of gross employment income — below the 2026 Social Security wage base, below the $250,000 MFJ Additional Medicare Tax threshold, and with no West Virginia employee-side payroll tax — giving $5,513.51 + $1,289.45 = $6,802.96. The decisive trap is the spouse's $14,717 employer-sponsored insurance premium: seven models subtracted it to build a $74,211 FICA base, but PolicyEngine taxes gross employment income and that input drives coverage-affordability and medical-expense logic, not the payroll tax base; the listed figure is the total policy cost, predominantly employer-paid and therefore already outside wages, not an employee §125 salary reduction that could be netted out again. The remaining misses are a fabricated West Virginia employee payroll tax, application of the combined 15.3% employer-plus-employee rate when employer taxes are explicitly excluded, and numeric fields that contradict the model's own correct derivation." -us,scenario_014,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Two steps separate the reference from every wrong answer. First, the West Virginia base: PolicyEngine's AGI is the full $88,927.65 of gross wages (veterans benefits excluded, employer-sponsored premiums not netted out) less the $2,000-per-person exemption for a two-person unit, giving $84,927.65 — West Virginia grants no standard deduction and no federal standard deduction flows through. Second, the rate vintage: the 2026 West Virginia MFJ schedule is 2.11% / 2.81% / 3.16% / 4.22% / 4.58% across the $10k/$25k/$40k/$60k bracket boundaries, a 5% trigger reduction from the 2025 rates of 2.22/2.96/3.33/4.44/4.82%, and it reproduces $3,092.19 to the cent. Models split into those that subtracted the $14,717 insurance premium or invented a standard deduction (bases of $70,211–$74,211 or lower) and those that got the base right but priced it at 2023 or 2025 rates ($3,255 or $3,459.81); several then applied unexplained credit haircuts to their own arithmetic." -us,scenario_015,federal_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"The case turns entirely on the 2026 parameter set: AGI of $42,909.38 ($45,000 wages less the $1,041.93 traditional 401(k) deferral, plus $2,000 interest, less the $3,000 §1211(b) capital loss deduction and the $48.69 traditional IRA deduction), minus the $16,100 single standard deduction, leaves $26,809.38 taxed at 10% to $12,400 and 12% above, for $2,969.13. Most models reproduced the AGI and then missed the deduction and bracket parameters — substituting 2024/2025 values ($14,600/$15,000/$15,750 standard deductions; $11,600/$11,925/$12,150 bracket tops) or pricing a TCJA sunset with a personal exemption and a 15% second bracket. A second cluster inflated deductions with mortgage interest invented from the $230,000 balance or by excluding the $2,080 premium from wages, and a third cluster derived the reference figure step by step and then submitted a different number." +us,scenario_014,payroll_tax,16,llm_error,payroll_tax_base,False,,"Only the spouse has wages, so the correct answer is the 7.65% employee FICA rate applied to her full $88,928 gross wages: $5,513.51 Social Security plus $1,289.45 Medicare, or $6,802.96. The facts do not say she pays premiums through a salary-reduction (Section 125) plan, so the $14,717 in employer-sponsored insurance premiums does not come out of the FICA wage base. Veterans benefits are not wages, and West Virginia has no employee state payroll tax. The largest group of wrong answers (7 models) assumed a cafeteria plan that the facts never mention and taxed only $74,211. Most of the rest applied 7.65% to $88,928 and then invented an extra tax, made an arithmetic slip, or submitted a number different from their own total." +us,scenario_014,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"West Virginia AGI is the spouse's full $88,927.65 of wages; veterans benefits are exempt, and the listed ESI premium does not reduce the listed gross wages. Subtracting two $2,000 exemptions leaves $84,927.65 of taxable income, with no standard deduction. For 2026, WV applies MFJ rates 5% below the 2025 schedule: 2.11% to $10k, 2.81% to $25k, 3.16% to $40k, 4.22% to $60k and 4.58% above. That gives 211 + 421.50 + 474 + 844 + 1,141.69 = $3,092.19. Most models used the correct base but either the 2025 rates (2.22–4.82%, giving $3,255.03) or the 2023–24 rates (2.36–5.12%, giving $3,459.81). The rest invented standard deductions, subtracted the ESI premium, or added credits that do not exist." +us,scenario_015,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The correct path is AGI of $42,909 ($45,000 wages minus the $1,042 traditional 401(k) deferral, plus $2,000 interest, minus the $3,000 capital-loss cap and the $49 IRA deduction; workers' comp excluded). Subtract the 2026 single standard deduction of $16,100 to get $26,809 taxable income, then apply the 2026 brackets: 10% to $12,400 and 12% above, for $2,969.13. No nonrefundable credit applies, because the Saver's Credit and the elderly/disabled credit are both phased out at this AGI. Most models got AGI right and then missed on the 2026 parameters. Some used 2024/2025 standard deductions and bracket tops ($14,600, $15,000, $15,750; $11,600, $11,925). Others assumed the TCJA sunset (personal exemption plus a 15% bracket), even though the 2025 law made the TCJA rate and standard-deduction structure permanent. The rest fabricated itemized deductions (mortgage interest from the loan balance, double-counted premiums), excluded health premiums from wages, invented credits, or submitted a number that contradicts their own arithmetic." us,scenario_015,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A single adult with no dependent children or qualifying education expenses receives neither the refundable Child Tax Credit nor the refundable American Opportunity Credit. At approximately $43,450 of income, the childless EITC is also zero, and all other evaluated refundable-credit components are zero, yielding total federal refundable credits of $0." us,scenario_015,head_chip_eligible,1,parse_contract_failure,other,False,,"CHIP eligibility runs through categorical gates first: coverage is for children under 19 and, at state option, pregnant women, so a 36-year-old head never reaches the income test — PolicyEngine assigns chip_category = 3 (categorically ineligible) and is_chip_eligible = False. The ~$43,450 household income and Indiana's Hoosier Healthwise/CHIP income thresholds are irrelevant to a person who fails the age/pregnancy gate, and the separate Medicaid pathways also return NONE for a non-aged, childless adult at that income in a state whose adult expansion he clears on neither disability-SSI nor MAGI grounds. The single wrong answer stated this categorical rule correctly in prose and then emitted the opposite binary." us,scenario_015,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish Medicaid eligibility; the person must satisfy a specific Indiana eligibility pathway and its applicable requirements. The head falls into Medicaid category NONE, receives no SSI, is not a dependent, and has MAGI at 2.69 times FPL, so no MAGI, disability-related, or state-specific pathway confers eligibility." -us,scenario_015,head_medicare_eligible,3,llm_error,age_disability,False,,"The trap is the `is disabled` demographic flag: all three models treated it as a Medicare qualifying pathway for an under-65 adult. Medicare entitlement below age 65 runs only through 24 months of Social Security disability insurance entitlement (42 U.S.C. §426(b)), ESRD, or ALS — none of which is a bare disability status. PolicyEngine's Medicare eligibility test compares age to the 65 age threshold, and this head is 36 with $45,000 of wages, 28 usual weekly hours, no Social Security disability income (every unlisted amount is 0, and the $960 workers' compensation is not SSDI), so no pathway opens and the value is False." -us,scenario_015,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $45,000 of wages because neither the stated health premiums nor traditional 401(k) contributions were identified as reductions to the FICA wage base. Applying 6.2% and 1.45% produces $2,790 and $652.50, respectively, totaling $3,442.50; several models instead reduced the payroll-tax base or failed to add their own components correctly." -us,scenario_015,state_income_tax_before_refundable_credits,30,llm_error,thresholds_rates,False,,"Indiana's individual income tax for 2026 is a flat 2.95% under the HEA 1001-2022 rate schedule, applied to federal AGI reduced only by the $1,000 base exemption. Federal AGI here is $42,909.38 — $45,000 wages less the $1,042 traditional 401(k) deferral and $49 IRA deduction, plus $2,000 interest, less the $3,000 capital-loss limitation, with the $960 of workers' compensation excluded — giving Indiana taxable income of $41,909.38 and tax of $1,236.33. Almost every wrong answer failed on one of two points: it used a superseded Indiana rate (3.15%, 3.05%, 3.0%) or a fabricated one (2.79%, 2.85%, 2.90%, 2.97%, 3.025%), or it invented an Indiana subtraction that does not exist — a disability exemption (Indiana's additional $1,000 exemption requires age 65+ or blindness, and IC 6-3-2-9's disability retirement deduction requires disability retirement income this wage earner does not have), a health-insurance-premium deduction, a child-support deduction, or a federal standard deduction. Several models derived the base correctly and then overrode their own arithmetic with an unjustified hand adjustment." +us,scenario_015,head_medicare_eligible,4,llm_error,age_disability,False,,"A person under 65 becomes eligible for Medicare on disability grounds only after 24 months of entitlement to Social Security Disability Insurance, or through the ESRD or ALS pathways. Being disabled is not enough on its own. The head is 36, has no Social Security disability benefits, and has no ESRD or ALS status. That means they don't meet the age-65 test or any disability-entitlement pathway, so they are not Medicare eligible. All four models treated the bare 'is disabled' flag as automatic Medicare eligibility and skipped the SSDI-receipt requirement." +us,scenario_015,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $45,000 of gross wages: 6.2% Social Security ($2,790, well under the 2026 wage base) plus 1.45% Medicare ($652.50) gives $3,442.50. Indiana has no mandatory employee state payroll tax, and wages are far below the $200,000 Additional Medicare Tax threshold. The most common wrong move was shrinking the FICA base. Four models treated the listed $2,080 health insurance premiums as a Section 125 pre-tax salary reduction, and one subtracted the traditional 401(k) deferral, even though elective 401(k) deferrals stay subject to FICA. The rest reached or approached $3,442.50 and then broke the total with arithmetic slips or made-up add-ons." +us,scenario_015,state_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"Indiana starts from federal AGI of $42,909.38. That is wages less the $1,042 traditional 401(k) and the $49 IRA deduction, plus $2,000 of interest, less the capital loss capped at $3,000. Workers' compensation is excluded. Indiana then subtracts only the $1,000 personal exemption and applies the scheduled 2026 flat rate of 2.95%: $41,909.38 x 0.0295 = $1,236.33. Most models built this base correctly but used the wrong rate: 3.05% (2024), 3.0% (2025), 2.9%, 2.85% or a made-up rate. Others added deductions Indiana does not allow. These were an extra exemption for being disabled (Indiana's additional $1,000 exemptions cover age 65+ and blindness only), a deduction for the $2,080 in health premiums, a child support deduction, or a standard deduction." us,scenario_015,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_016,federal_income_tax_before_refundable_credits,8,llm_error,taxable_income_or_deductions,False,,"The household reports $47,000 of deductible medical outlays — $26,000 of health insurance premiums excluding Medicare Part B, $20,000 of other medical expenses, and $1,000 of over-the-counter health expenses — against an AGI of $32,137.94 (wages of $34,638 less the $2,500 statutory cap on the student loan interest deduction). Net of the 7.5%-of-AGI floor of $2,410, the itemized medical deduction of roughly $44,590 exceeds AGI outright, so itemizing dominates the ~$16,100 standard deduction, taxable income is zero, and federal income tax before refundable credits is $0. Seven of the eight models ran only the standard-deduction path and clustered in the $1,271–$1,873 band; the eighth recognized the medical deduction but reported the unused excess as a large negative liability instead of flooring the tax at zero." us,scenario_016,federal_refundable_credits,3,llm_error,credit_phaseout,False,,"This is a single childless 49-year-old with $34,638 in wages, so the only refundable credit in play is the childless EITC, and its complete phase-out point for a single filer is roughly $19,100 of earned income/AGI in 2026 — barely more than half this filer's wages. With no qualifying children, no education expenses, no recovery rebate, and no excess payroll tax withholding, every component PolicyEngine evaluated (EITC, refundable AOTC, refundable CTC, Recovery Rebate Credit, refundable payroll tax credit) is $0. Both models that produced a number stopped at ""the childless EITC exists"" without applying the terminating phase-out threshold that zeroes the credit far below this income." @@ -136,13 +138,13 @@ us,scenario_016,ssi,1,parse_contract_failure,missing_output,False,,All wrong res us,scenario_016,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_016,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_016,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_018,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The correct 2026 computation is pinned by three post-OBBBA parameters: AGI of $61,592.09 (wages less the $6,174.40 401(k) deferral and $288.51 deductible IRA, plus $50 of interest), the $16,100 single standard deduction, and brackets of 10% to $12,400 and 12% to $50,400, giving $1,240 + 0.12 × $33,092.09 = $5,211.05. Almost every model rebuilt the AGI correctly and then missed on the parameters or invented deductions: one group carried 2024–2025 standard deductions and bracket edges into 2026, a second assumed the TCJA sunset and restored a personal exemption with 15%/25% rates, and a third subtracted OBBBA auto-loan-interest or qualified-overtime amounts whose qualifying inputs are unlisted and therefore zero. A fourth group applied a Saver's Credit that reaches zero above roughly $40,500 of AGI, or a child credit for dependents this single filer does not have; two models derived $5,211 exactly and then submitted a different number." +us,scenario_018,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"The correct path is simple: AGI of $61,592 (wages, less the traditional 401(k), plus $50 interest, less the $289 IRA deduction), minus the 2026 OBBBA standard deduction of $16,100, gives $45,492 of taxable income. The 2026 single brackets (10% to $12,400, then 12%) put the tax at $5,211.05, with no nonrefundable credits. Most wrong models used stale parameters: the 2024 or 2025 standard deduction ($14,600, $15,000, $15,400 or $15,750) and 10% bracket tops ($11,925 or $12,150), or they assumed TCJA had expired, bringing back personal exemptions and the 15% and 25% brackets. A second group took deductions that the listed facts do not support: an inferred $6,760 qualified overtime premium, a $1,625 car-loan interest deduction, or itemized medical deductions that counted the $9,000 premium twice. A few made arithmetic errors, submitted numbers that contradict their own work, or invented dependents or a saver's credit." us,scenario_018,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_018,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $68,005 of wages here: $4,216.31 of Social Security tax plus $986.07 of Medicare tax equals $5,202.38. The listed health-insurance premiums were not identified as employee pre-tax salary-reduction contributions, and Arizona's lack of a separate state payroll tax does not remove federal employee FICA." us,scenario_018,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_018,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Arizona conforms to federal AGI and taxes it at a single flat 2.5% rate, so this case reduces to two numbers: the AGI ($68,005 wages − $6,174 traditional 401(k) + $50 interest − $289 traditional IRA = $61,592.09) and the Arizona standard deduction, which for 2026 is $15,750 and beats the $10,786.71 of itemized deductions ($9,500 medical + $1,286.71 SALT). Nearly every model recovered the AGI and the 2.5% rate and then plugged in a different standard deduction — $16,100 (the federal figure), $14,600, $15,000, $15,400, $15,650 — landing $9 to $40 off the $1,146.05 reference. A second cluster computed a near-correct figure and then overrode it with an unexplained number, invented Arizona health-premium adjustments, resurrected the pre-2023 graduated brackets, or double-counted the $9,000 employer-plan premiums that are already excluded pre-tax from the wage figure." +us,scenario_018,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The calculation has three steps: federal AGI of $61,592.09 (wages minus the traditional 401(k), plus interest, minus the traditional IRA deduction), then Arizona's 2026 single standard deduction of $15,750, then the flat 2.5% rate on $45,842.09, which gives $1,146.05. Itemizing does not win: Arizona itemized deductions are $10,786.71 ($9,500 medical plus $1,286.71 in state and local taxes), well below $15,750. Most models got AGI and the 2.5% rate right but used the wrong standard deduction. The largest group used the federal 2026 figure of $16,100 and got $1,137.30. Others used stale or guessed amounts ($13,790–$15,700), added an Arizona personal exemption that does not exist, applied the repealed graduated rates, or made arithmetic or transcription errors." us,scenario_018,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_020,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The trap is the 2026 deduction map for a high-earning Texas single filer: alimony expense is deducted above the line ($5,708.82, giving AGI $300,150.19), the $138,375 of unreimbursed employee business expenses is a permanently repealed miscellaneous itemized deduction, and SALT of $17,090.34 — the $13,834 of real estate taxes plus $3,256.34 of state and local sales tax deductible in lieu of a nonexistent Texas income tax, all well under the $40,400 2026 cap — narrowly beats the $16,100 standard deduction, so the filer itemizes to $283,059.84 of taxable income. Applying the 2026 single schedule with $30.60 of qualified dividends at 15% yields $67,834.07 of regular tax, and the $222.64 net investment income tax on $5,859 of investment income brings the total to $68,056.71. Wrong answers split into two clusters: models that assumed TCJA sunset and deducted the employee business expenses plus a personal exemption (landing near $31,000-$35,000, sometimes rescued upward by an AMT that does not arise), and models that got the 2026 framework right but took the standard deduction because they counted only real estate taxes as SALT, denied the alimony deduction, subtracted the $8,389 ESI premium, or double-counted the $42,857 overtime premium already inside the $300,000 of wages." +us,scenario_020,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,other,False,,"Texas has no income tax, so the SALT itemized deduction is real estate tax ($13,834) plus the general sales tax deduction from the IRS optional sales tax table. The frozen reference projects that table amount from the IRS's 2023 edition, for $17,090.34 in all, which is under the $40,400 SALT cap and above the $16,100 standard deduction, so it itemizes and the $196 gift falls below the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) uses the IRS 2025 tables instead, whose $1,595 for a single Texas filer at this income leaves SALT below the standard deduction, so the household takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction. The correct path is: deduct the $5,709 alimony above the line to reach AGI of $300,150; deduct the $16,296 of standard and nonitemizer charitable deductions for taxable income of $283,854.19; tax that at 2026 rates, with qualified dividends at 15%; then add $222.64 of NIIT, for $68,334.73, or $68,112.09 without it, the exclusion's corrected value. The frozen reference instead itemizes $17,090.34, for taxable income of $283,059.84 and a total of $68,056.71. The best answers ($68,111.95–$68,404) took the standard deduction, as the convention does; those that also took the $196 nonitemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or of $68,112.09 without it. Many others dropped the alimony deduction, left out NIIT, subtracted ESI premiums, or applied TCJA-sunset rules. Those rules are wrong because OBBBA made the suspension of miscellaneous itemized deductions and personal exemptions permanent. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)." us,scenario_020,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_020,payroll_tax,29,llm_error,payroll_tax_base,False,,"The reference is the plain three-component FICA stack on the head's $300,000 of gross wages: 6.2% on the 2026 Social Security wage base of $184,500 ($11,439), 1.45% on all $300,000 ($4,350), and the 0.9% Additional Medicare Tax on the $100,000 above the $200,000 single-filer threshold ($900), with Texas imposing no employee state payroll tax. Two parameter traps separate the reference from the wrong answers: the 2026 wage base is $184,500, not the stale $176,100 (2025), $168,600 (2024), or invented $180,000-$184,200 figures most models substituted; and the payroll base is the full $300,000 of gross wages, so the $8,389 employer-sponsored insurance premium is not shaved off Medicare wages (a $197 understatement) and the $42,857 FLSA overtime premium is already inside the $300,000 rather than added to it. A third cluster derived $16,689 or a near-miss correctly and then submitted a completely different number, discarding their own arithmetic at the reporting step." us,scenario_020,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident's state income tax before refundable credits is zero. The nationwide aggregation does not create liability in jurisdictions where the household does not reside." @@ -154,43 +156,43 @@ us,scenario_021,self_employment_tax,1,parse_contract_failure,missing_output,Fals us,scenario_021,snap,1,llm_error,thresholds_rates,False,,"An elderly or disabled SNAP household is exempt from the gross-income test but must still satisfy the one-person net-income limit. With monthly net income of approximately $2,063 even under the model’s own deductions, the household fails that test, so the calculation stops at ineligibility and produces $0 rather than an allotment." us,scenario_021,state_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"Missouri tax is computed from Missouri taxable income after applicable deductions and the 2026 taxable-income threshold, not by applying a flat rate directly to taxable pension income. After those steps, this household has no Missouri taxable income and therefore owes $0 before refundable credits." us,scenario_021,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on the 2026 OBBBA deduction stack for an itemizing 77-year-old single filer: SALT is nowhere near the $40,400 cap, so it carries CA income tax and SDI on top of the $8,829 property tax for $14,807.14; charitable gifts are cut by the new 0.5%-of-AGI floor to $14,847.67; the $6,000 senior deduction phases down by 6% of MAGI above $75,000 to $3,951.05; and $910 of §199A QBI comes off the rental income, giving $34,515.86 of deductions and $74,633.30 of taxable income. The single 2026 schedule (10% to $12,400, 12% to $50,400, 22% above) reproduces $11,131.33 exactly. Most models instead capped SALT at $10,000 with property tax only, deducted gross charity, and skipped or failed to phase the senior deduction. Two other clusters missed by more: one assumed TCJA sunset in 2026 and restored personal exemptions, 2%-floor miscellaneous itemized deductions and 15%/25% rates — all permanently repealed — and the other filed on the qualifying-surviving-spouse joint schedule with a joint standard deduction." +us,scenario_022,federal_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"The head is widowed but has no dependent child, so qualifying surviving spouse status is unavailable and the return is filed as single, with the 22% rate starting above $50,400. Under 2026 OBBBA law the return itemizes $29,735.52. That figure is $14,887.85 of SALT, meaning $8,829 of property tax plus about $6,059 of estimated CA income tax under the $40,400 cap, and $14,847.67 of charity after the new 0.5%-of-AGI floor. On top of that come the $910 QBI deduction on rental income and the senior deduction phased down to $3,951.05, which leaves $74,552.59 of taxable income. The wrong answers split three ways: 16 used joint/QSS brackets, 8 applied the expired pre-TCJA law (personal exemptions, 2% miscellaneous deductions, 15%/25% brackets), and the rest filed single but left out or understated the CA income-tax SALT component, the senior deduction, or the QBI deduction." us,scenario_022,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_022,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Age 77 establishes an aged status but does not itself confer Medicaid eligibility, and Medicare eligibility or assumed enrollment is not a Medicaid eligibility pathway. The head receives no SSI and qualifies under none of California's Medicaid categories, so the aged status cannot overcome the absence of a qualifying pathway." us,scenario_022,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,payroll_tax,36,llm_error,state_local_rule,False,,"The federal half is trivial and every model got it: Social Security 6.2% x $52,635 = $3,263.37 and Medicare 1.45% x $52,635 = $763.21, with no Additional Medicare Tax. The case turns entirely on the third component the prompt explicitly requests — mandatory employee state payroll tax — which for a California wage earner is the State Disability Insurance employee contribution, withheld at 1.3% of all wages with no taxable wage ceiling in 2026, giving $684.26 and a total of $4,710.83. The models split into two failure classes: those that dropped the state component altogether (landing at $4,026.58 or a rounded variant) and those that included SDI but at a stale rate — 1.1% (the 2024-25 uncapped rate), 1.2% (the 2025 rate), or 0.9% (the pre-2024 capped rate) — landing at $4,605.57, $4,658.20, or $4,500.30. A handful compounded the error by submitting a number that contradicts their own stated arithmetic." +us,scenario_022,payroll_tax,39,llm_error,state_local_rule,False,,"Every model computed the federal parts correctly: 6.2% Social Security ($3,263.37) plus 1.45% Medicare ($763.21), with no Additional Medicare Tax. The whole $684.26 gap is California's mandatory employee State Disability Insurance contribution, which for 2026 is 1.3% of all $52,635 in wages with no wage cap. The models split into two groups. Sixteen left CA SDI out entirely and landed near $4,026.58. The other twenty-two included SDI but used an old or guessed rate: 1.2% ($631.62), 1.1% ($578.99, the 2024 rate) or 0.9% ($473.72, the 2023 rate). One model returned no answer." us,scenario_022,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_022,state_income_tax_before_refundable_credits,39,llm_error,household_unit_or_filing_status,False,,"Two steps separate the reference from nearly every wrong answer. First, the ""is a surviving spouse"" fact does not buy joint-width brackets: qualifying-surviving-spouse status under IRC §2(a)(1)(B) requires a dependent child in the household and none is listed, so California applies the single rate schedule, the single $5,706 standard deduction, and two exemption credits (personal plus age-65) worth $312.93 — thirteen models instead ran the doubled MFJ schedule and cut the bracket tax roughly in half. Second, California itemized deductions here total only $23,676.67 — charitable $14,847.67 (contributions of $15,393 reduced by the 0.5%-of-AGI floor) plus real estate taxes $8,829 — with no unreimbursed employee business expenses, no medical (below the 7.5%-of-AGI floor), and no mortgage interest (unlisted inputs are 0); eleven models added ~$6,200–$6,700 of employee expenses and understated taxable income by about $7,000. The correct chain is federal AGI $109,149.16 less the $17,624.75 Social Security subtraction = CA AGI $91,524.41, less $23,676.67 = taxable income $67,847.75, bracket tax $2,752.58, less $312.93 of exemption credits = $2,439.65." +us,scenario_022,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"A surviving spouse can use qualifying-surviving-spouse status only with a dependent child in the home, and this person has no dependents. PolicyEngine therefore treats the filer as single: $5,706 standard deduction, single rate brackets, and one personal credit plus one senior exemption credit, totaling $312.93. California AGI is $91,524.41 after subtracting taxable Social Security. Itemized deductions are only charitable gifts ($14,847.67, less than the full $15,393) plus $8,829 of real estate taxes, for $23,676.67; unreimbursed employee business expenses are not deducted. That leaves taxable income of $67,847.75 and gross tax of $2,752.58. The largest group of models used joint/surviving-spouse brackets and double credits, which gives about $650–$1,140. A second group used single brackets but deducted about $6,500 of employee business expenses under a 2% floor, which gives about $1,870–$2,180. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,439.65; the corrected value is 1,968.81. PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (Cal. R&TC 17024.5 (conformity date), 17076)." us,scenario_022,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,federal_income_tax_before_refundable_credits,34,llm_error,credit_phaseout,False,,"The case turns on two parameters that no shortcut recovers: the 2026 single standard deduction is $16,100, and no nonrefundable credit reduces this return. With AGI of $22,534.34 (wages of $17,442.65 less the $2,778.48 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, less the $129.83 IRA deduction), taxable income is $6,434.34 and the entire amount sits in the 10% bracket for $643.43. The AOTC is $0 because no qualified tuition amount is listed and unlisted numeric inputs are 0 — the 1098-T, EIN, half-time and credential flags are eligibility switches, not expenses — and the Saver's Credit is $0 because IRC 25B(d)(2) nets the $8,000 testing-period 403(b) distribution against the $3,599 of total retirement contributions. Twenty models reproduced the pre-credit tax almost exactly and then erased it with one of those two phantom credits, while most of the rest used a lapsed-TCJA or stale standard deduction, added a repealed personal exemption, or subtracted the $8,089 employer health premium from an already-net wage figure." -us,scenario_023,federal_refundable_credits,19,llm_error,credit_phaseout,False,,"The refundable American Opportunity Tax Credit requires qualified education expenses; enrollment and documentation flags alone create no credit because unlisted expenses are zero. The childless EITC is also zero because the taxable $8,000 403(b) distribution raises AGI beyond the applicable phaseout endpoint, even though the distribution is not earned income. With no qualifying child or other refundable-credit basis, all five refundable components equal zero." +us,scenario_023,federal_income_tax_before_refundable_credits,41,llm_error,other,False,,"AGI is $22,534 ($14,664 of wages after the traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction). Subtracting the $16,100 OBBBA standard deduction leaves $6,434 of taxable income, which is taxed at 10% for $643.43. No nonrefundable credit applies. For the Saver's Credit, Form 8880 subtracts retirement distributions received during the testing period from contributions, and the $8,000 403(b) distribution wipes out the $3,599 of contributions. The AOTC is zero because no qualified tuition expenses are listed. Most wrong answers of $0 came from applying one of these two credits anyway. The rest used a stale standard deduction (such as $15,000, $15,350, $15,750, or TCJA-sunset amounts with a personal exemption), wrongly excluded employer health premiums from wages, or left out the 403(b) distribution." +us,scenario_023,federal_refundable_credits,20,llm_error,credit_phaseout,False,,"This case has two traps, and a model had to avoid both to reach $0. First, the AOTC eligibility flags do not create a credit. The credit is 100% of the first $2,000 of qualified tuition expenses plus 25% of the next $2,000, and no expenses are listed, so the base is $0 and the refundable 40% portion is also $0. Second, the childless EITC phaseout is measured on the greater of earned income or AGI. AGI is about $22,535: wages of $17,443, minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. That is above the 2026 single childless completion point of about $19,540, so the EITC is fully phased out." us,scenario_023,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"The household qualifies for the FREE meal tier through California’s universal program and categorical eligibility, despite its 1.59 federal-poverty-guideline ratio exceeding the federal income cutoff. The requested output is narrower: it equals 1 only when PolicyEngine returns positive annual free-school-meal support, and this adult-only household has no student receiving meals, so the annual support is zero." us,scenario_023,head_chip_eligible,2,llm_error,categorical_eligibility,False,,"CHIP eligibility requires being in a covered CHIP category and not already eligible for Medicaid; it is not a general adult coverage pathway based solely on falling below an extended income ceiling. The 28-year-old head qualifies for Medicaid through the WORKING_DISABLED_BUY_IN category, which places them in CHIP category 3 and categorically excludes them from CHIP." -us,scenario_023,head_medicaid_eligible,19,llm_error,categorical_eligibility,False,,"The head is 28, disabled, and employed (24 hrs/week at $14/hr, $17,443 in wages), which places them in Medicaid's buy-in for working disabled category — a non-MAGI group, implemented in California as the 250% Working Disabled Program, whose income ceiling of roughly 250% FPL (~$39,900 for one person) sits far above the adult expansion limit. MAGI here is $22,535 (wages less the $2,778 traditional 401(k), plus the $8,000 taxable 403(b) distribution, less the $130 traditional IRA), equal to 141% FPL against a 138% FPL expansion cutoff of about $22,055 — a near-miss that every wrong model treated as dispositive. SSI is $0, so the SSI-linked route genuinely fails, and models that checked it were right about that but wrong to treat SSI receipt as the only disability-based door. The buy-in category conditions on disability plus earnings, so the very wages that pushed MAGI past 138% FPL are what open it." -us,scenario_023,head_medicare_eligible,3,llm_error,categorical_eligibility,False,,"Medicare eligibility before age 65 requires a qualifying pathway such as sufficient duration of Social Security disability-benefit entitlement or end-stage renal disease; a bare disability flag does not establish Medicare eligibility. The 28-year-old head has no listed Medicare-qualifying entitlement or medical criterion, so the eligibility output is 0." +us,scenario_023,head_medicaid_eligible,24,prompt_ambiguity,categorical_eligibility,False,,"The head is 28, disabled, and working 24 hours a week at $14 an hour. That puts them in California's working-disabled Medicaid buy-in category (medicaid_category = WORKING_DISABLED_BUY_IN), whose income limit is far above the 138% FPL adult-expansion cap. At 1.41x FPL, the head is ineligible under the MAGI adult group but eligible through the buy-in. Every model that gave reasoning tested only the 138% FPL expansion limit and, at most, the SSI-linked aged/blind/disabled pathway. None of them considered the employed-disabled buy-in, even though the facts list both the disability and the employment. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1 (eligible); the alternative is 0. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability, which the prompt does not state; policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in the program. With the program's test reading the Social Security definition, the head qualifies only by meeting it. The same unlisted input excludes this household's SNAP. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Flagged in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster excl_snap_ssi_disability) and excluded on review of the release. Reference depends on an unlisted input; output excluded from scoring (meets_ssi_disability_criteria, the Social Security definition of disability that 42 CFR 435.540(a) and California's 250% Working Disabled Program require)." +us,scenario_023,head_medicare_eligible,4,llm_error,age_disability,False,,"The head is 28, so age-based Medicare eligibility at 65 does not apply. For someone under 65, disability-based Medicare requires 24 months of Social Security Disability Insurance (SSDI) entitlement or an end-stage renal disease diagnosis, not just a disability status. This household has no SSDI income and no ESRD, so the head is not eligible. The wrong models treated the `is_disabled` flag by itself as enough for Medicare." us,scenario_023,head_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_023,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,payroll_tax,36,llm_error,state_local_rule,False,,"PolicyEngine's $1,561.12 is federal FICA on the full $17,442.65 of employment income (Social Security $1,081.44 at 6.2%, Medicare $252.92 at 1.45%) plus California's mandatory employee State Disability Insurance contribution of $226.75, which is 1.3% of gross wages with no taxable wage ceiling in 2026. The separating step is that CA SDI is a mandatory employee-side state payroll tax at the current 1.3% rate: wrong answers either dropped it entirely, applied a stale rate (0.9%, the 2024 rate of 1.1%, or the 2025 rate of 1.2%), or shrank the wage base. The base is the full $17,442.65 — the $2,778 traditional 401(k) elective deferral remains FICA and SDI wages under IRC 3121(a)(5)(D), the $8,089 employer-sponsored insurance premium figure is not an employee Section 125 payroll reduction of stated gross wages, and the $8,000 taxable 403(b) distribution is not wages at all." +us,scenario_023,payroll_tax,41,llm_error,thresholds_rates,False,,"The reference has three parts, all applied to the full $17,442.65 of wages: employee Social Security at 6.2% ($1,081.44), Medicare at 1.45% ($252.92), and California SDI at the 2026 employee rate of 1.3% with no wage cap ($226.75). The derivation text says 1%, but $226.75 is exactly 1.3% of the wage base. Most models got FICA right and then missed the SDI piece in one of two ways: they left SDI out, or they used an older rate (0.9% from 2023, 1.1% from 2024, or 1.2% from 2025). A second group shrank the wage base by subtracting the $8,089 employer-sponsored insurance premiums or the $2,778 traditional 401(k) deferral. Neither amount reduces the stated gross wages that serve as the FICA and SDI base." us,scenario_023,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_023,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_023,snap,39,prompt_ambiguity,age_disability,False,,"The case turns on whether the head's `is_disabled` flag confers USDA elderly-or-disabled status for SNAP. It does not - that status requires age 60+ or receipt of a qualifying disability benefit - so no excess medical deduction applies and the excess shelter deduction stays bounded by the FY2026 $744/month cap rather than absorbing the full $2,000 rent. With the 20% earned income deduction ($290.72) and the $209 standard deduction, monthly net income is $876.53, 67% of the $1,304.17 poverty guideline, and the gross test is cleared not by disability but by California's TANF-funded broad-based categorical eligibility at 200% FPL. The benefit is therefore $298 - $262.80 = $35.20/month, rising to $304.68 - $256.50 = $48.18 for October-December 2026, for $461.34 annually. Twenty-five models unlocked the uncapped shelter deduction and landed at or near the one-person maximum allotment; ten applied the 130% FPL gross test without BBCE and returned zero. Developer adjudication (2026-09-05): the judge (gpt-5.6-sol) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules." +us,scenario_023,snap,46,prompt_ambiguity,age_disability,False,,"A ""disabled"" flag alone does not make this a SNAP elderly/disabled household. Under the 7 CFR 271.2 definition, the member must receive a disability-based benefit such as SSI or SSDI, and the head receives neither. Because of that, the household gets no excess medical deduction, and its excess shelter deduction is capped at $744. Monthly gross income is $2,120.25 ($1,453.58 wages plus $666.67 from the 403(b)). Subtracting the $290.72 earned-income deduction, the $209 standard deduction and the $744 capped shelter deduction leaves net income of about $876.51. That gives the frozen reference's $298 − $262.80 = $35.20/month for January–September and $304.68 − $256.50 = $48.18/month for October–December under the engine's projected FY2027 schedule, or $461.34 for the year. The release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $34/month ($408). Most models wrongly treated the head as SNAP-disabled, used an uncapped shelter deduction and a medical deduction, and got large benefits or the maximum allotment. A smaller group applied the 130% gross income test instead of California's 200% FPL broad-based categorical eligibility limit, or left out the shelter deduction, and got $0. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)." us,scenario_023,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to satisfy the aged, blind, or disabled categorical gate before income exclusions or benefit-rate calculations apply. The head's unlisted SSI disability status is false under the prompt instructions, so `is_ssi_aged_blind_disabled=False` and SSI is $0 without an income calculation." -us,scenario_023,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"California AGI is $22,534.34 (wages $17,443 less the $2,778 elective 401(k) deferral, plus the $8,000 taxable 403(b) distribution, less the $130 traditional IRA deduction), the 2026 single standard deduction of $5,706 leaves $16,828.34 of taxable income, and the 1%/2% brackets (1% through roughly $11,330) produce $223.27 of tax. The nonrefundable credits available to a 28-year-old single renter are exactly two: one $156.47 personal exemption credit and the $60 nonrefundable renter's credit, totaling $216.47 — $6.80 short of the tax. The case turns on precision: the residual survives only if the indexed 2026 standard deduction and bracket ceiling are carried exactly and no second exemption credit is invented, since California's extra exemption credit is for blindness or age 65+, not disability generally, and CalEITC is refundable rather than nonrefundable. Most models either dropped the $8,000 403(b) distribution from state income, used stale 2023–2025 parameters, or reached a positive residual by their own arithmetic and then rounded it to $0." -us,scenario_023,state_refundable_credits,38,llm_error,credit_phaseout,False,,"CalEITC is a stand-alone California credit, not a percentage of the federal EITC and not limited by federal eligibility: for a childless filer it phases in on California earned income, plateaus below $300, and declines linearly to zero only at the indexed version of the $30,000 statutory cap, near $33,000. PolicyEngine computes it on adjusted earnings of $17,442.65 — the full wages, with no subtraction for the $2,778 elective 401(k) deferral and no addition of the $8,000 taxable 403(b) distribution — which lands roughly halfway down that phase-out at $148.31, and the renter's credit stays out because it is nonrefundable. The wrong answers split into two clusters: models that zeroed the credit by importing the federal childless phase-out end near $19,000, the repealed percentage-of-federal rule, or a bogus disqualified-income test on the 403(b) distribution; and models that returned the plateau maximum or more by never applying the phase-out at all." +us,scenario_023,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"California AGI is $22,534 ($17,443 wages − $2,778 traditional 401(k) + $8,000 taxable 403(b) − $130 IRA). Subtracting the $5,706 single standard deduction gives $16,828 taxable. Tax at 1% up to $11,079 and 2% above is $225.78. The $153 personal exemption credit and the $60 nonrefundable renter's credit bring it down to exactly $12.78. The answer is a small residual, so it only comes out right with the exact 2025-published parameters and both credits. Models went wrong in four main ways: they declared the tax fully offset even though $225.78 is more than $213 in credits, they dropped the $8,000 403(b) distribution, they left out one of the two credits, or they used 2026-inflated or outdated parameters that moved the residual by a few dollars." +us,scenario_023,state_refundable_credits,45,llm_error,credit_phaseout,False,,"The filer's only state refundable credit is the childless CalEITC, which comes to $95.15. A 28-year-old filer without children qualifies. The $8,000 in taxable 403(b) distributions is neither earned income nor disqualifying investment income, but it raises AGI to $22,534 (wages $17,443 + $8,000 − $2,778 traditional 401(k) − $130 IRA). In the phase-out range, CalEITC uses whichever is greater, earned income or AGI, so the credit comes from CalEITC's two-segment phase-out schedule at AGI $22,534, not at wages of $17,443. Models went wrong in four ways: they denied eligibility outright, tied CalEITC to the federal EITC, phased the credit out using only wages, or guessed amounts near the maximum credit or above it." us,scenario_023,tanf,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_025,federal_income_tax_before_refundable_credits,29,llm_error,thresholds_rates,False,,"The case turns entirely on the 2026 parameter set: OBBBA made the TCJA structure permanent and inflation-indexed, so the MFJ standard deduction is $32,200, there are no personal exemptions, and the brackets are 10% to $24,800 then 12%. AGI is uncontested at $94,925.30 ($62,725.29 employment income + $32,200 taxable pension), itemizing yields only $2,354.43, and no nonrefundable credit applies to a childless couple with no care or education expenses, so taxable income is $62,725.30 and tax is $2,480 + 0.12 × $37,925.30 = $7,031.04. The wrong answers split into three families: models that applied a TCJA-sunset baseline (~$16,600 deduction plus ~$10,600 of exemptions and a 15% second bracket, landing near $8,900–$9,150), models that anchored on 2025 or invented deduction/bracket figures ($29,900–$31,450 deductions, $23,850–$24,600 bracket tops, landing $7,130–$7,314), and models that derived the right number or a near-right number and then submitted a different one." +us,scenario_025,federal_income_tax_before_refundable_credits,30,llm_error,thresholds_rates,False,,"The engine uses the OBBBA-permanent TCJA structure for 2026 MFJ: a $32,200 standard deduction and 10%/12% brackets breaking at $24,800. That gives taxable income of $94,925.30 − $32,200 = $62,725.30 and tax of $2,480 + 12% × $37,925.30 = $7,031.04, with no nonrefundable credits for this childless couple under 65. The wrong answers fall into four groups. Some assumed the TCJA sunset and used pre-TCJA deductions, personal exemptions and a 15% bracket, landing near $8,900–$9,150. Some used the pre-OBBBA 2025 figures, a $30,000 deduction and a $23,850 bracket, and got $7,314. Some used near-miss 2026 parameters. Several computed $7,031 or close to it and then submitted an unrelated number, including $0." us,scenario_025,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_025,head_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 in the absence of a qualifying disability, ESRD, or other explicitly listed pathway. The 61-year-old head has no such condition, so the age test yields 0 (No)." us,scenario_025,payroll_tax,7,llm_error,payroll_tax_base,False,,"The employee-side payroll tax base here is exactly one number: the spouse's $62,725 of gross wages. The head reports no wages at all (only ESI premiums, hours, and medical expenses), the spouse's $32,200 taxable private pension is not FICA wages, employer-sponsored insurance premiums are not netted out of the FICA base in PolicyEngine's employee_payroll_tax, and Ohio imposes no mandatory employee-side state payroll tax (no employee SDI/UI, and state income tax is a separate output). So the answer is 6.2% + 1.45% on $62,725 = $3,888.97 Social Security + $909.52 Medicare = $4,798.48, with no Additional Medicare Tax below the $250,000 MFJ threshold. Every wrong answer either enlarged or shrank that $62,725 base, bolted on a nonexistent state levy or the employer share, or submitted a number that contradicted its own correct arithmetic." -us,scenario_025,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Ohio's 2026 bracket for taxable income between $26,050 and $100,000 is not a bare 2.75% of the excess: it is a $332.00 base plus 2.75% of the excess over $26,050, so $91,125.30 of taxable income produces exactly $2,121.57 of tax. The base itself is federal AGI of $94,925.30 with no standard deduction and no medical deduction (the head's premiums are employer-sponsored and the remaining $850 of unreimbursed expenses falls under the $7,119 7.5%-of-FAGI floor), less two $1,900 exemptions — Ohio's tier for OAGI above $80,000 — and the only credit is the $200 nonrefundable retirement income credit, since the joint filing credit requires each spouse to have $500 of qualifying Ohio income and the senior credit requires age 65. Every wrong answer breaks on that chain: the models that got the exemption tier and the retirement credit right still dropped the $332 base and landed at $1,589.56, while the rest substituted $2,400–$2,800 exemptions, federal standard deductions, phantom medical deductions, invented credits, or repealed pre-2023 graduated rate schedules." -us,scenario_026,child1_chip_eligible,4,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; it is not determined merely by comparing household income with the CHIP ceiling. Child 1 qualifies for Medicaid under North Carolina's OLDER_CHILD category, so the Medicaid determination precludes CHIP eligibility." -us,scenario_026,child1_medicaid_eligible,38,llm_error,state_local_rule,False,,"The trap is North Carolina's April 2023 elimination of NC Health Choice: the state's separate CHIP program was folded into Medicaid, raising the limit for children ages 6–18 from 133% FPL to roughly 216% FPL (211% plus the five-percentage-point disregard). The household's MAGI is $85,209 of wages less $2,908 of traditional 401(k)/IRA contributions = $82,301, which is 213% of the five-person poverty guideline — above every pre-2023 threshold the models cited and below the current one, so the 11-year-old qualifies under the OLDER_CHILD category. Secondary errors compounded this: double-counting the $7,746 FLSA overtime premium already inside the $85,209 wage total, deducting the head's $21,208 employer-sponsored insurance premium from MAGI, and using a four- or three-person poverty guideline for a five-person household. A separate cluster imported CHIP's uninsured-child requirement into Medicaid, treating the ESI flag as an automatic disqualifier when the Medicaid MAGI test has no other-coverage condition." -us,scenario_026,child2_chip_eligible,4,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child not to qualify for Medicaid; it is not established solely by comparing household MAGI with a CHIP income ceiling. Child 2 qualifies for Medicaid under the OLDER_CHILD category, so Medicaid precedence excludes the child from CHIP regardless of the CHIP income threshold." -us,scenario_026,child2_medicaid_eligible,38,llm_error,thresholds_rates,False,,"The trap is North Carolina's older-child Medicaid pathway. Nearly every model applied the federal ACA floor of 133–138% FPL for ages 6–18 and missed that NC moved its NC Health Choice population into Medicaid in April 2023, so children 6–18 are covered by Medicaid at 211% FPL, which the 5-percentage-point MAGI disregard lifts above 213%. The household's MAGI is $85,209 of wages less the $2,778 traditional 401(k) and $130 traditional IRA, or $82,301 — 2.13x the 2026 five-person poverty guideline — which sits inside that limit, so the 11-year-old qualifies under the OLDER_CHILD category. The rest inflated the ratio by skipping the pre-tax deductions or dividing by a stale or smaller-household poverty guideline, or invented an employer-coverage bar that PolicyEngine's MAGI test does not contain." -us,scenario_026,child3_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, so satisfying a CHIP age or income condition is not sufficient. Child 3 qualifies for Medicaid under North Carolina's OLDER_CHILD category, and that Medicaid eligibility makes the child ineligible for CHIP regardless of the cited CHIP income thresholds." -us,scenario_026,child3_medicaid_eligible,38,llm_error,thresholds_rates,False,,"North Carolina folded NC Health Choice into Medicaid effective April 2023, so the state's Medicaid income limit for children ages 6-18 is 211% FPL, not the 133%/138% ACA minimum, and the 5-percentage-point MAGI disregard puts the effective cutoff at 216% FPL. Household MAGI is $85,209 of wages less the $2,778 traditional 401(k) and $130 traditional IRA contributions — the $7,746 FLSA overtime premium is a below-the-line qualified-overtime deduction and the $21,208 ESI premium is not a MAGI deduction — giving $82,301, or 2.13 times the roughly $38,600 five-person 2026 poverty guideline. At 213% FPL the 9-year-old clears the OLDER_CHILD limit. Employer-sponsored insurance bars CHIP, not Medicaid, so the child's ESI does nothing here; nearly every wrong answer either substituted the 133-138% federal floor for NC's 211% level, divided by a smaller household's guideline, or compared gross wages instead of MAGI." +us,scenario_025,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Ohio AGI is $94,925. Subtract two $1,900 exemptions (the amount for Ohio MAGI above $80,000) to get $91,125 of taxable income. In the 2026 schedule used here, the bracket from $26,050 to $100,000 is taxed at $332 plus 2.75% of the excess over $26,050, which gives $2,121.57. The $200 nonrefundable retirement income credit for the $32,200 pension then brings this to $1,921.57. The largest group of wrong answers (about $1,589.56) left out the $332 base amount. Other models used the wrong exemption amount, subtracted a federal standard deduction or health premiums, used outdated multi-bracket schedules, or claimed a joint filing credit or senior credit the household does not qualify for." +us,scenario_026,child1_chip_eligible,5,llm_error,categorical_eligibility,False,,"NC Health Choice merged into NC Medicaid in 2023. North Carolina now covers school-age children up to 211% FPL, plus a 5% disregard, as Medicaid (Medicaid-expansion CHIP) under the OLDER_CHILD category, not as a separate CHIP program. PolicyEngine finds the 11-year-old Medicaid-eligible at this household income. Because CHIP covers only children who fail Medicaid, that Medicaid eligibility makes the child CHIP-ineligible. Every model that answered treated the 211% band as a separate CHIP program sitting above a lower Medicaid limit. Each one called the child CHIP-eligible instead of routing them to Medicaid first and excluding CHIP." +us,scenario_026,child1_medicaid_eligible,45,llm_error,thresholds_rates,False,,"In April 2023, North Carolina folded its separate CHIP program (NC Health Choice) into Medicaid. Since then, Medicaid for children ages 6–18 covers incomes up to 211% FPL, or 216% with the 5% MAGI disregard, not the old 133% FPL limit. Household MAGI is the spouse's $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA contributions, which gives $82,301, or 2.13x FPL. That is inside the OLDER_CHILD limit, so Child 1 is eligible. Most models used the outdated 133%/138% limit. Others overstated MAGI by double-counting overtime, skipping the pre-tax deductions, or using the 2025 poverty guideline. Others wrongly treated employer-sponsored insurance as a bar to Medicaid, but only CHIP requires a child to be uninsured." +us,scenario_026,child2_chip_eligible,5,llm_error,categorical_eligibility,False,,"Child 2 is 11. In the engine, North Carolina's Medicaid OLDER_CHILD category covers this child at the household's income, so the child is Medicaid-eligible. CHIP only covers children who do not qualify for Medicaid, which makes child2_chip_eligible = No. NC folded NC Health Choice into Medicaid, so the income band that the models called ""CHIP, above Medicaid and up to 211-216% FPL"" is Medicaid coverage for a school-age child. All four models that answered found income below the NC children's limit but assigned the child to CHIP instead of Medicaid, and never applied the rule that Medicaid eligibility excludes CHIP." +us,scenario_026,child2_medicaid_eligible,45,llm_error,thresholds_rates,False,,"In April 2023, North Carolina folded NC Health Choice (its separate CHIP) into Medicaid. Since then, children ages 6-18 get Medicaid up to 211% FPL, which becomes about 216% after the 5-point MAGI disregard. PolicyEngine puts child2 in the OLDER_CHILD category with household MAGI (spouse wages minus the traditional 401(k) and IRA deductions, about $82,300) at 2.13× the 2026 FPL for a family of five. That is under the older-child limit, so child2 is Medicaid-eligible. Almost every model used the old 133%/138% Medicaid limit (with Health Choice CHIP above it), applied a bare 210-211% cap with no disregard, or misstated MAGI or the FPL. A few also wrongly treated employer coverage as a bar to Medicaid." +us,scenario_026,child3_chip_eligible,7,llm_error,categorical_eligibility,False,,"Every model placed 9-year-old Child 3 in a CHIP band that sits above Medicaid, using the old 133%-to-211% FPL NC Health Choice structure. North Carolina moved NC Health Choice into Medicaid in 2023, and PolicyEngine finds Child 3 Medicaid-eligible under the OLDER_CHILD category. A Medicaid-eligible child is excluded from CHIP by construction, so the answer is No however each model computed MAGI. The decisive step is the Medicaid-first ordering with North Carolina's older-child Medicaid limit, not any CHIP income ceiling." +us,scenario_026,child3_medicaid_eligible,45,llm_error,thresholds_rates,False,,"NC Health Choice, the state's separate CHIP, was folded into Medicaid on April 1, 2023. Since then, NC covers children ages 6-18 in its Medicaid older-child category up to 211% FPL, or about 216% with the 5% MAGI disregard, instead of the old 133%/138% federal minimum. Household MAGI is $85,209 in wages minus $2,778 in traditional 401(k) and $130 in traditional IRA contributions, which comes to $82,301, or 2.13x the 2026 FPL for five people. That is under the older-child limit, so Child 3 is Medicaid-eligible. Almost every model either used the pre-merger 133%/138% limit, pushed the 138-211% band into CHIP, compared gross wages against 210/211% with no disregard, or wrongly treated employer coverage as blocking Medicaid." us,scenario_026,federal_income_tax_before_refundable_credits,22,llm_error,taxable_income_or_deductions,False,,"2026 law keeps the TCJA structure — personal exemptions repealed, a $32,200 MFJ standard deduction — and adds the deduction for qualified FLSA overtime premiums, so the $7,746.26 premium comes out of taxable income rather than being stacked onto wages that already include it, leaving $42,354.26 taxable and $4,586.51 of tentative tax. The child tax credit is $2,200 per qualifying child in 2026, and its nonrefundable portion offsets the entire liability first, with the $1,700-per-child ACTC as the residual, so $6,600 of credit for three children drives the answer to exactly $0. The wrong answers split into two clusters: models that applied the expired pre-TCJA regime (a reverted ~$16,600 standard deduction, ~$5,200 personal exemptions, a 15% bracket, a $1,000-per-child credit), and models that got the deduction stack roughly right but capped the nonrefundable CTC at $500–$1,100 per child by treating the refundable ACTC as consumed first. Every wrong model ignored the qualified-overtime deduction, and three double-counted the $7,746 premium on top of the $85,209 gross wage figure." -us,scenario_026,federal_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"The case turns on two 2026 OBBBA provisions acting together: the Child Tax Credit is $2,200 per child, so three children generate $6,600, and the $7,746 FLSA overtime premium is a below-the-line qualified-overtime deduction stacking on the $32,200 MFJ standard deduction. AGI is $82,301 (wages $85,209 less the $2,778 traditional 401(k) and the $130 deductible traditional IRA), taxable income is $42,355, and pre-credit tax in the 10%/12% brackets is $4,586.51. Because the $6,600 credit exceeds that liability, the $2,013.49 remainder is refundable — far under the $5,100 refundable cap (3 x $1,700) and the 15%-of-earnings-over-$2,500 limit of $12,406 — while EITC is $0 at this income. Models that assumed the CTC was fully absorbed, used $2,000 or $1,000 per child, or skipped the overtime deduction all landed at or near zero; the one model that overshot deducted the $378 of auto loan interest, which requires qualifying-vehicle facts that are unlisted and therefore false." +us,scenario_026,federal_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Under 2026 OBBBA law the CTC is $2,200 per child ($6,600 for three), refundable up to $1,700 per child. AGI is $82,301: $85,209 in wages minus the $2,778 traditional 401(k) and the $130 traditional IRA. Subtracting the $32,200 MFJ standard deduction and the $7,746 qualified overtime (FLSA premium) deduction leaves taxable income of about $42,355. At the 10% and 12% rates, pre-credit tax is about $4,586.51, so $2,013.49 of the CTC goes unused. That amount is under both the $5,100 per-child cap and the 15%-of-earnings limit, so all of it is refundable ACTC, and EITC is zero at this income. The wrong answers came from four errors: leaving out the overtime deduction (which pushes tax toward $5,500), using pre-OBBBA CTC amounts ($1,000 or $2,000 per child), saying the tax absorbs the whole credit when tax is below the credit, and adding deductions or credits that do not apply (auto-loan interest, ESI premiums, Saver's Credit)." us,scenario_026,free_school_meals_eligible,1,llm_error,thresholds_rates,False,,"Free school meals require income within the free-meal limit or categorical eligibility through a qualifying program. This household's countable income of $85,208.82 equals 220% of the $38,680 federal poverty guideline, exceeds even the 185% reduced-price limit, and has no categorical eligibility, so its school-meal tier is paid." us,scenario_026,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -199,7 +201,7 @@ us,scenario_026,reduced_price_school_meals_eligible,1,llm_error,categorical_elig us,scenario_026,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_026,spouse_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_026,state_income_tax_before_refundable_credits,30,llm_error,taxable_income_or_deductions,False,,"North Carolina's 2026 tax equals federal AGI — $82,300.52 here, after the $2,778 traditional 401(k) and $130 traditional IRA contributions — minus the $25,500 married-filing-jointly standard deduction and the AGI-graduated child deduction, taxed at the 3.99% flat rate. The child deduction is the central trap: the schedule grants $1,500 per qualifying child for joint filers with AGI between $80,000 and $100,000, so three children yield $4,500, not the $3,000-per-child top tier, not $500, and not zero. The second trap is the $7,746 FLSA overtime premium, which is already contained in the $85,209 of gross wages and, as a below-the-line federal deduction, never reduces the federal AGI North Carolina starts from — models both added it a second time and subtracted it. Nearly every wrong answer is 3.99% applied to a base that misstates one of these two items, or the right base run through a superseded rate (4.25%, 4.5%, 4.75%)." +us,scenario_026,state_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI. Here that is $82,300.52: the spouse's wages, which already include the overtime premium, minus the traditional 401(k) and IRA contributions. From that, subtract the $25,500 joint standard deduction and the NC child deduction. For joint filers, the child deduction is tiered by AGI, and the $80k–$100k tier gives $1,500 per child, or $4,500 for three children. The resulting $52,300.52 is taxed at the flat 3.99% rate for 2026. Most models got this wrong in one of four ways: they left out the child deduction, used the wrong AGI tier ($500, $1,000, $2,000, $2,500, $3,000, or $4,000 per child), added the FLSA overtime premium on top of wages or subtracted it from them, or used an outdated rate or standard deduction." us,scenario_026,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The household has no positive federal income tax before refundable credits because taxable income and every additional component of that output are zero. A missing submission is a parse-contract failure, while a negative submission contradicts both the model's own zero-tax derivation and the output's nonnegative liability construction." us,scenario_027,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -212,55 +214,55 @@ us,scenario_027,local_income_tax,1,parse_contract_failure,missing_output,False,, us,scenario_027,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_027,snap,38,llm_error,categorical_eligibility,False,,"Connecticut confers broad-based categorical eligibility through TANF non-cash assistance, which deletes the SNAP resource test outright — the $48,000 bank balance is not countable against any limit — and sets the gross screen at 200% FPG ($3,525/month against $2,536 gross, the 1.44 ratio in the trace), so the household is eligible and the separate 100%-FPG net income test does not disqualify it. The remaining step is arithmetic: the two-person maximum allotment ($546, then $558.24) minus 30% of net income ($635.10, then $633.60) is negative. An eligible one- or two-person household whose computed benefit is zero or negative is paid the statutory minimum allotment, $23.84/month under the FY2026 parameters and $24.37/month from October 2026, and nine months at $23.84 plus three at $24.37 sums to $287.68. Every wrong answer either killed eligibility with a resource or net income test that categorical eligibility waives, or ran the formula to a negative result and reported $0 instead of the minimum allotment floor." +us,scenario_027,snap,45,llm_error,categorical_eligibility,False,,"Connecticut's broad-based categorical eligibility (TANF non-cash pathway) makes this household categorically eligible. Gross income is 1.44× the poverty guideline, below CT's 200% limit, so the asset test and the 100% FPL net income test do not block eligibility. Under 7 CFR 273.10(e)(2)(ii)(C), categorically eligible households of one or two people receive the minimum allotment even when the formula gives zero. Here 30% of the $2,118 net income is $636, which exceeds the $546 maximum, so the FY2026 minimum of $24/month applies, or $288 a year. Most models ended at $0: they either applied the $4,250–$4,500 elderly/disabled asset limit or the net income test, or they stopped when the formula benefit came out to zero. A few applied the minimum but used the older $23 amount." us,scenario_027,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_027,spouse_medicaid_eligible,12,llm_error,categorical_eligibility,False,,"Blindness or disability alone does not establish Medicaid eligibility; the spouse qualifies through none of Connecticut's modeled categorical pathways, so the Medicaid category is NONE. Her Medicaid MAGI is 1.41 times FPL, above the applicable MAGI-category limit, and she receives no SSI to establish an SSI-linked pathway." -us,scenario_027,spouse_medicare_eligible,10,llm_error,age_disability,False,,"Medicare eligibility attaches to the individual, and PolicyEngine's is_medicare_eligible turns on that person's own age being 65 or older — the head at 68 returns True, the spouse at 39 returns False, and an `is_disabled` or `is_blind` flag opens no separate route in the engine. The statutory under-65 pathway (42 U.S.C. §426(b)) requires 24 months of the person's own entitlement to Social Security disability insurance benefits, which requires insured status from that person's own work credits; the spouse here has $0 of disability income and $0 of earnings under the ""treat any unlisted numeric input as 0"" instruction, so the $30,160 SSDI belongs solely to the 68-year-old head. Nine models converted the bare ""is disabled"" demographic flag into an assumed 24-month SSDI entitlement for the spouse, and one invented a derivative pathway in which a Social Security beneficiary's disabled spouse gets Medicare — no such pathway exists. Being disabled is not the same as being entitled to disability benefits, and Medicare entitlement never passes from one spouse to another." +us,scenario_027,spouse_medicaid_eligible,13,llm_error,categorical_eligibility,False,,"The spouse, 39, qualifies through no Medicaid category in the engine. For the adult expansion group, Medicaid MAGI adds back all non-taxable Social Security benefits, so the head's $30,160 SSDI plus $275 interest gives household MAGI of $30,435. That is 1.41x FPL, above the 138% limit. Being blind and disabled confers no separate category either: the spouse receives $0 SSI because the couple's $30,435 income and $48,000 in resources far exceed the SSI couple limits. Models said yes by either dropping Social Security from MAGI or treating blindness and disability as enough on their own under an aged, blind and disabled (ABD, HUSKY C) pathway." +us,scenario_027,spouse_medicare_eligible,11,llm_error,age_disability,False,,"Under 65, the only Medicare pathways are 24 months of Social Security disability benefit receipt, ESRD, or ALS. A disability or blindness flag does not qualify a person on its own. The 39-year-old spouse has no Social Security disability income of their own; the $30,160 SSDI belongs to the head. The spouse also cannot borrow the head's SSDI entitlement, because Medicare disability coverage requires the person's own entitlement to disability benefits. PolicyEngine therefore marks only the 68-year-old head as Medicare eligible. Every model that answered treated the spouse's 'is disabled' flag as enough for Medicare." us,scenario_027,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_027,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_028,child1_chip_eligible,16,llm_error,health_coverage,False,,"PolicyEngine runs two independent gates for CHIP: the child must fall outside every Medicaid category, and must separately clear CHIP's own age and income tests. Child1's Medicaid category is NONE, and the CHIP income test is what fails — counted income for the unit is $61,277 (the head's $60,000 in wages, $1,267 in child support received, and $10 of taxable interest), which exceeds the Pennsylvania CHIP income limit the engine applies to a 10-year-old in a four-person unit for 2026. Every wrong model instead recalled the 314–319% FPL headline figure that tops PA's low-cost/full-cost CHIP buy-in tier, checked only that roughly 185–190% FPL sits beneath it, and returned eligible. Most also dropped the $1,267 of child support from the income they compared, and several explicitly waved off the three children's employer-sponsored coverage as ""enrollment, not eligibility,"" which removed the last check that would have made them re-examine the CHIP-specific test." +us,scenario_028,child1_chip_eligible,20,llm_error,health_coverage,False,,"Every model that answered stopped at CHIP's age and income screens. Child 1 is 10, and household MAGI is $60,010 ($60,000 wages plus $10 interest; child support is excluded). For a family of four that is about 187% FPL, which is above PA's child Medicaid limit and below the 314% FPL CHIP ceiling. The step they skipped is CHIP's coverage bar: under 42 U.S.C. 1397jj(b)(1)(C) and 42 CFR 457.310(c)(1)(ii), a targeted low-income child must not be covered under a group health plan. Child 1 has employer-sponsored insurance, so Child 1 fails CHIP's own criteria even though Child 1 is not Medicaid-eligible. claude-opus-4.7, kimi-k3 and ox-alpha went further and said outright that existing coverage has no bearing on CHIP eligibility." us,scenario_028,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The decisive step is assigning the child to a Pennsylvania Medicaid eligibility category rather than treating the broader Medicaid/CHIP coverage continuum as Medicaid. At 1.82 times FPL, the child satisfies none of the Medicaid pathways evaluated by the engine, has no disability pathway or SSI receipt, and therefore has medicaid_category NONE and is not Medicaid eligible." us,scenario_028,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child1_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC categorical eligibility for children ends at the fifth birthday; it does not extend to school-age children. Child 1 is age 10, so the child fails the categorical age requirement regardless of household income." -us,scenario_028,child2_chip_eligible,17,llm_error,health_coverage,False,,"PolicyEngine's is_chip_eligible is not a pure FPL-band test: a child must be a targeted low-income child under 42 U.S.C. 1397jj(b)(1) — not Medicaid-eligible and not covered under a group health plan or other health insurance — before the state's CHIP age and income criteria can produce eligibility. Child2 carries employer-sponsored insurance, and the engine returns is_chip_eligible = False alongside is_medicaid_eligible = False (category NONE). Every wrong model correctly computed the family-of-four FPL ratio (~185-192%) and correctly recalled Pennsylvania's ~314% FPL separate-CHIP ceiling, then treated that band plus ""under 19"" as the complete test. Two models went further and explicitly asserted that the household's employer coverage does not affect PolicyEngine CHIP eligibility, which inverts the controlling requirement." +us,scenario_028,child2_chip_eligible,21,llm_error,thresholds_rates,False,,"Every answering model reasoned that child2 is a 10-year-old in a family of four at about 185-192% FPL, above PA's child Medicaid limit and under PA's roughly 314% FPL CHIP ceiling, so it answered Yes. PolicyEngine agrees that child2 is not Medicaid-eligible (category NONE), but it also finds that child2 fails CHIP's own age-or-income criteria. Age 10 meets CHIP's under-19 age rule, so the criterion child2 fails is the CHIP income test as PolicyEngine applies it to this PA household for 2026. The shared wrong step is the models' conclusion that household income falls within the CHIP income limit, which turned a CHIP No into a Yes. glm-5.2 gave no answer at all." us,scenario_028,child2_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_medicaid_eligible,6,llm_error,thresholds_rates,False,,"A 10-year-old Pennsylvania dependent at 1.82 times FPL does not satisfy any Medicaid eligibility pathway evaluated by PolicyEngine, so the Medicaid category is NONE. CHIP’s higher income ceiling is a separate program and cannot establish Medicaid eligibility." us,scenario_028,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child2_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five and also requires household income at or below 185% of the federal poverty guideline. Child 2 is age 10, and the household's approximately $61,277 of income exceeds the applicable four-person limit, so both the categorical age test and the income test fail." -us,scenario_028,child3_chip_eligible,16,llm_error,thresholds_rates,False,,"PolicyEngine counts $61,277 of income for this four-person household — $60,000 of wages plus $1,267 of child support and $10 of taxable interest — and that counted income exceeds the Pennsylvania separate-CHIP income limit the engine applies to a 10-year-old, so child3 fails CHIP's own income test while also having no Medicaid pathway (engine category NONE). Every wrong model anchored instead on Pennsylvania's headline ~314% FPL figure, which is the outer ceiling of the tiered CHIP structure including the unsubsidized full-cost buy-in, and against that number no household at ~185–190% FPL can fail. Most compounded it by counting only $60,010 of wages and interest and dropping the $1,267 of child support the engine includes. Several also affirmatively waved away the children's employer-sponsored coverage, which runs against CHIP's targeted-low-income-child definition at 42 U.S.C. 1397jj(b)(1)(C) excluding children covered under a group health plan." +us,scenario_028,child3_chip_eligible,20,llm_error,health_coverage,False,,"Child 3 passes CHIP's age and income tests: household MAGI of $60,010 is 182% of the $33,000 guideline for four, above Pennsylvania's 138% Medicaid limit for children aged 6 to 18 and below the 319% CHIP limit. What decides the case is the coverage condition. Child 3 has employer-sponsored insurance, and a child covered by a group health plan is not a targeted low-income child (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii)), so the engine's disqualifying-coverage test fails. Every model that answered stopped at the age and income screen and then ignored the listed ESI or, in several cases, stated outright that ESI does not affect CHIP eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (health_coverage). Child 3 has employer-sponsored group coverage, which bars CHIP; the household's income passes Pennsylvania's CHIP limit, so the judge's income-limit hypothesis fails. Reference affirmed (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7)." us,scenario_028,child3_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid and CHIP are separate eligibility outputs, so CHIP coverage at higher income levels does not establish Medicaid eligibility. At 1.82 times FPL, child3 exceeds every applicable Medicaid income threshold and qualifies under no Medicaid category; employer-sponsored insurance and zero SSI provide no alternative Medicaid pathway." us,scenario_028,child3_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,child3_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old fails the categorical age requirement. The household also has approximately $61,277 of income, exceeding the applicable 185%-of-poverty WIC income limit; Child 3 therefore fails both the categorical and income tests." us,scenario_028,federal_income_tax_before_refundable_credits,19,llm_error,credit_phaseout,False,,"The decisive step is the ordering and size of the 2026 child tax credit: AGI of $60,010 (wages $60,000 plus $10 of interest; child support received is excluded from gross income) less the $24,150 head-of-household standard deduction leaves $35,860 of taxable income, taxed at 10% on the first $17,700 and 12% on the remaining $18,160 for $3,949.20. Three qualifying children generate $6,600 of CTC at the 2026 rate of $2,200 each, and the nonrefundable CTC is applied first against the full liability, zeroing it out; the refundable ACTC is the $2,650.80 residual, bounded by the $1,700-per-child cap and 15% of earnings above $2,500 ($8,625), neither of which binds. Every wrong answer either priced 2026 with lapsed pre-TCJA parameters (personal exemptions, a ~$12,000 standard deduction, a $1,000-per-child CTC) or reversed the ordering by carving a fixed refundable ACTC out of the credit first and applying only the stub nonrefundably." -us,scenario_028,federal_refundable_credits,37,llm_error,credit_phaseout,False,,"Two 2026-specific parameters separate the reference from nearly every wrong answer. First, OBBBA sets the CTC at $2,200 per child for 2026 ($6,600 here) with a $1,700-per-child refundable ceiling; with HoH standard deduction $24,500, taxable income of $35,510 produces $3,949.20 of tax before credits, so the refundable CTC is the unused remainder $6,600 − $3,949.20 = $2,650.80 — the $5,100 ceiling and the 15%-of-earnings-over-$2,500 limit ($8,625) are both slack and neither one is the answer. Second, the 2026 three-child EITC has a maximum of $8,231 phasing out at 21.06% from a HoH start near $23,890, so it does not zero out until roughly $62,970; at $60,010 of AGI it is still worth $624.13. Models that used $2,000 (or $1,000) per child, that substituted a cap for the computed remainder, or that carried 2024–2025 EITC phase-out endpoints (~$57k–$59.9k) into 2026 all missed one or both legs of $3,274.93." +us,scenario_028,federal_refundable_credits,41,llm_error,credit_phaseout,False,,"The reference has two parts. First, OBBBA sets the 2026 CTC at $2,200 per child, so the credit is $6,600. Head-of-household tax on $60,010 AGI is $3,949.20 ($24,150 standard deduction, $35,860 taxable income, 10% bracket to $17,700, then 12%). The refundable ACTC is the unused $2,650.80, which is below both the $5,100 cap ($1,700 per child) and the $8,625 earned-income limit. Second, the 3-child EITC phases down from $8,231 at 21.06% above $23,890 and leaves $624.13 at $60,010, because it does not fully phase out until about $62,974. The wrong answers split into three groups: models that zeroed the EITC by placing the phase-out end below $60,010; models that used the pre-OBBBA $2,000 per child (or $1,000) CTC, or misstated the pre-credit tax; and models that paid the full $5,100 refundable cap without limiting the ACTC to the CTC left after offsetting tax." us,scenario_028,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_028,payroll_tax,31,llm_error,state_local_rule,False,,"The whole case turns on one component beyond federal FICA: Pennsylvania's employee unemployment compensation contribution, withheld at 0.07% of all covered wages with no wage base cap, which adds $42 on $60,000 of wages. PA is one of the few states (with Alaska and New Jersey) that imposes an employee-side UI contribution, and the question explicitly asked for ""mandatory employee state payroll taxes."" Federal employee Social Security (6.2% × $60,000 = $3,720) and Medicare (1.45% × $60,000 = $870) give $4,590, with no Additional Medicare Tax below $200,000; adding the $42 PA employee UC contribution yields $4,632. Every wrong answer clusters at $4,590 or a near-variant, so the separator is purely the state component." -us,scenario_028,reduced_price_school_meals_eligible,32,llm_error,thresholds_rates,False,,"The case turns entirely on the level of the 2026 federal poverty guideline for a four-person unit: PA has no universal free meals and the household has no SNAP/TANF categorical pathway, so the school-meal tier is set by the income test alone, and countable income (wages $60,000 + child support $1,267 + interest $10) lands at a school_meal_fpg_ratio of 1.82 — inside the 185% reduced-price band, above the 130% free band, hence tier REDUCED and $3,009.82 of subsidy for three school-age children. Nearly every model assembled the income correctly and then compared it against a stale guideline: 185% of the 2024 four-person guideline is $57,720 and 185% of the 2025 guideline is $59,478, both of which $61,277 clears, while the 2026 guideline leaves roughly a thousand dollars of headroom. The handful that tried to project 2026 undershot the uprating by two to five percent, which is exactly the margin between 1.82 and their claimed 1.86–1.91. Two models failed differently: one never ran the income test at all, and one returned no answer." +us,scenario_028,payroll_tax,36,llm_error,state_local_rule,False,,"Employee FICA on $60,000 of wages is $3,720 in Social Security plus $870 in Medicare, or $4,590, with no Additional Medicare Tax. Pennsylvania is one of the few states that withholds an employee unemployment compensation contribution: 0.07% of total gross wages with no wage cap, which is $42 here. That brings the total to $4,632. Almost every model stopped at FICA and either ignored the PA employee UC withholding or said outright that PA has no employee payroll tax. One model applied the UC rate to the wrong base, one made an arithmetic slip, one derived $4,632 but submitted a different number, and two returned no answer." +us,scenario_028,reduced_price_school_meals_eligible,8,llm_error,thresholds_rates,False,,"School-meal countable income is $61,277 because child support received counts alongside wages and interest. Against the $33,000 poverty guideline for a household of four, that is 1.86 times the guideline, $227 above the $61,050 reduced-price cutoff (185%), so the household is in the paid tier. Models that dropped the $1,267 of child support used $60,010, which is 1.82 times $33,000 (the same figure as the trace's school_meal_fpg_ratio), and landed under the line. Models that did count $61,277 overstated the 185% threshold, putting it somewhere between about $61,300 and $61,975 instead of $61,050." us,scenario_028,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_028,state_income_tax_before_refundable_credits,10,llm_error,taxable_income_or_deductions,False,,"Pennsylvania's personal income tax is a flat 3.07% on the eight enumerated classes of income with no standard deduction, no personal exemptions, and no filing-status allowances; the base here is $60,000 of compensation plus $10 of taxable interest, and the $1,267 of child support is not a taxable class, giving 3.07% × $60,010 = $1,842.31. The only income-based relief, the Schedule SP tax forgiveness credit, is unavailable because eligibility income of $61,277 far exceeds the $35,000 limit for an unmarried claimant with three dependents ($6,500 + 3 × $9,500). Most wrong models imported federal deduction or exemption concepts into PA's no-deduction structure or fabricated a base or rate; the remainder either failed to execute the multiplication or subtracted a credit that is refundable and zero-valued here." -us,scenario_028,state_refundable_credits,38,llm_error,state_local_rule,False,,"The trap is that Pennsylvania's refundable personal income tax credit here is a straight percentage match of the federal EITC — 10% × $624.13 = $62.41 — so the state answer is fully determined by the federal EITC rather than by any PA-specific income test. This household is a head of household with three qualifying children and $60,010 of AGI ($60,000 wages + $10 interest), which sits just inside the 2026 three-child phase-out range (21.06% phase-out running to roughly $63,000), leaving a small residual federal EITC of $624.13 instead of zero. Every wrong model asserted that Pennsylvania has no refundable individual income tax credit and then reasoned only about two dead ends — the nonrefundable Tax Forgiveness (Schedule SP) credit, which is zero at this income, and the Child and Dependent Care Enhancement Credit, which is zero because no child care expenses are listed — and so never carried the federal EITC through the state match. Nobody computed the residual federal EITC and multiplied it by the state match rate; that single missing step is the entire $62.41." +us,scenario_028,state_refundable_credits,43,llm_error,state_local_rule,False,,"Starting in tax year 2025, Pennsylvania has a refundable state EITC, the Working Pennsylvanians Tax Credit, equal to 10% of the federal EITC. This head of household has three qualifying children, $60,000 of earned income and $60,010 of AGI (child support is excluded). That puts the household near the end of the federal EITC phase-out, so the federal credit is $624.13 and the PA refundable credit is $62.41. Almost every model relied on the pre-2025 rule that PA has no refundable income tax credit. They looked only at Tax Forgiveness, the CDCC or the Property Tax/Rent Rebate and answered $0. The one model that used the 10% PA EITC overestimated the federal EITC." us,scenario_028,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_029,federal_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only AGI item in this household is $312 of taxable interest: ""financial assistance"" is a non-taxable assistance receipt in PolicyEngine's income taxonomy and feeds no AGI component, so the $50,000 never reaches the 1040. With AGI of $312 against a 2026 single standard deduction of roughly $16,100 under the permanent OBBBA rules, taxable income is $0 and federal income tax before refundable credits is $0. All three models converted the $50,000 assistance figure into taxable income (wages or other ordinary income) and then ran a full bracket computation on roughly $50,312 of AGI, which is the single step that separates every wrong answer from the reference." us,scenario_029,head_medicaid_eligible,19,llm_error,taxable_income_or_deductions,False,,"The case turns on what enters Medicaid MAGI: the $50,000 of financial assistance is a non-taxable receipt excluded from MAGI, so the head's countable income is the $312 of taxable interest alone — 0.02 x FPL — and Oklahoma, which adopted ACA adult expansion under State Question 802 effective July 1, 2021, covers childless adults 19-64 up to 138% FPL. Fifteen of the nineteen models folded the $50,000 into countable income to manufacture ~$50,312 and compared that against roughly $21,000-$21,600. Two more (gpt-5.4-nano, qwen3.8-max) applied pre-ACA categorical framing, demanding pregnancy, disability, or a dependent child when the expansion group itself is the pathway. Secondary traps drew several models in as well: asset tests on the $54,985 of bank and stock holdings, which 42 CFR 435.603(g) bars for MAGI groups, and employer-sponsored insurance, which disqualifies from the premium tax credit rather than from Medicaid." us,scenario_029,payroll_tax,1,llm_error,payroll_tax_base,False,,"The household lists no wages at all: its only income items are $312 of taxable interest and $50,000 of financial assistance, alongside assets and expenses. Employee Social Security and Medicare taxes attach exclusively to wages and salaries under FICA (IRC §3101, on ""wages"" as defined in §3121(a)); interest is unearned investment income and financial assistance is a non-employment transfer, and neither enters the FICA wage base. With a zero wage base, employee OASDI, employee HI, and Additional Medicare Tax are each $0, giving a household employee-side payroll tax of $0." us,scenario_029,snap,1,llm_error,asset_resource,False,,"SNAP eligibility requires satisfying Oklahoma’s applicable resource test before the benefit formula is applied. This one-person household has $4,250 in bank assets and $50,735 in stock assets, so it fails the resource limit and receives no SNAP benefit regardless of its low interest income or the maximum allotment." us,scenario_029,state_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"The only taxable income in this household is $312 of taxable interest; the $50,000 of ""financial assistance"" is a non-taxable transfer that never enters federal AGI, and Oklahoma's income tax starts from federal AGI, while the $4,250 bank balance and $50,735 of stock are assets that generate no income under the ""treat any unlisted numeric input as 0"" instruction. Against Oklahoma's single standard deduction plus the $1,000 personal exemption, $312 of interest leaves zero Oklahoma taxable income, so ok_income_tax_before_refundable_credits — and therefore the 51-jurisdiction sum — is $0. Every wrong answer is produced by the same step: adding the $50,000 financial assistance into Oklahoma AGI and then applying the graduated OK rate schedule to roughly $37,000–$43,000 of invented taxable income." -us,scenario_029,state_refundable_credits,38,llm_error,state_local_rule,False,,"Oklahoma's sales tax relief credit (68 O.S. § 5011, modeled as ok_stc) is refundable and pays $40 per qualified exemption to any Oklahoma resident whose gross household income falls under the applicable ceiling — $20,000 for a filer who is under 65, not disabled, and claiming no dependents. It has no earned-income requirement, no dependent requirement, no asset test, and no tax-liability limit, and the $50,000 financial assistance is not Oklahoma gross household income: countable income here is the $312 of taxable interest, with the head's age of 21 satisfying the credit's age condition. One exemption at $40 therefore produces ok_stc = $40, the only nonzero contributor to the 51-jurisdiction aggregate. Every wrong model failed at one of three points: not knowing the credit exists (or denying Oklahoma has refundable credits or an income tax at all), folding the $50,000 financial assistance and investment assets into the $20,000 income test, or grafting an earned-income / dependent / age-65 / positive-liability precondition onto a credit that has none." +us,scenario_029,state_refundable_credits,44,llm_error,state_local_rule,False,,"The $40 is Oklahoma's refundable Sales Tax Relief Credit (ok_stc): $40 per personal exemption for a household with gross income under the $20,000 limit, filed by an adult head. The household's gross income for the credit is only the $312 of taxable interest. The $50,000 of financial assistance is not part of that income measure, and the credit has no earned-income test, no asset test, and no requirement to have dependents or to be 65+ or disabled. Every model answered $0. Some ignored the Sales Tax Relief Credit and looked only at the EITC. Others counted the $50,000 of financial assistance (or the assets) toward the $20,000 limit. A few invented an age, disability, or dependent requirement." us,scenario_030,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Only the $13,000 of wages enters adjusted gross income in this household. The listed educational assistance and financial assistance do not add $17,000 of taxable income, so AGI remains below the 2026 single-filer standard deduction and taxable income and pre-refundable-credit federal income tax are both zero." us,scenario_030,federal_refundable_credits,12,llm_error,categorical_eligibility,False,,"A childless EITC claimant in 2026 must satisfy the minimum-age requirement, and this 23-year-old does not. Pregnancy does not create a qualifying child for EITC or CTC purposes, so every evaluated refundable credit component is zero." us,scenario_030,head_chip_eligible,5,llm_error,categorical_eligibility,False,,The head qualifies for Medicaid under the pregnant-person category. Medicaid eligibility precludes CHIP eligibility under PolicyEngine because CHIP covers qualifying children who do not qualify for Medicaid; the Texas CHIP perinatal income threshold therefore never determines this output. @@ -269,12 +271,12 @@ us,scenario_030,head_wic_eligible,2,llm_error,household_unit_or_filing_status,Fa us,scenario_030,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_030,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on the $13,000 wage base. Those components are $806.00 and $188.50, totaling $994.50 without whole-dollar rounding." us,scenario_030,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_030,snap,39,llm_error,categorical_eligibility,False,,"Countable gross income is $2,083.33/month — $13,000 of wages plus the $12,000 financial assistance as unearned income, with the $5,000 educational assistance excluded — which is 160% of the one-person federal poverty guideline: over the 130% federal gross test but under Texas's 165% BBCE limit, conferred through a TANF-funded non-cash service, which also waives the 100% FPG net test that the $1,657/month net income would fail. Once eligible, 30% of net income ($497.10) exceeds the $298 maximum allotment, so the household is paid the one- and two-person minimum allotment of 8% of the maximum: $23.84/month, rising to $24.37 when the October COLA lifts the maximum to $304.68, for $287.68 annually. Most models stopped at the 130% gross test or mis-valued the 165% limit, and every model that reached the benefit formula either dropped the $12,000 of unearned assistance from countable income or returned $0 instead of the minimum allotment. The $8,500 of medical expenses is not deductible because no member is elderly or disabled, and pregnancy does not add a household member." +us,scenario_030,snap,46,llm_error,categorical_eligibility,False,,"The SNAP unit has one person. The $12,000 of financial assistance counts as unearned income, so gross income is $2,083.33 a month, about 160% of the $1,304.17 poverty guideline. Texas has broad-based categorical eligibility through TANF non-cash benefits, with a 165% FPL gross limit (about $2,152 a month), so this household is categorically eligible and does not face the 130% gross test or the 100% net test. Net income is $1,658, so the expected contribution ($498) is more than the $298 maximum allotment. Because the household is an eligible 1-person unit, it still gets the $24 minimum allotment ($288 a year). Most models either used the 130% test or net test and got $0, or left out the financial assistance and got a formula benefit of about $100 a month. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))." us,scenario_030,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_030,tanf,2,llm_error,categorical_eligibility,False,,Texas TANF requires a family with a dependent child; pregnancy alone does not turn the unborn child into a qualifying dependent child or create a two-person assistance unit for this annual TANF calculation. The single adult therefore has no qualifying child and receives zero TANF. us,scenario_031,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The $1,165 alimony expense reduces $14,408 of IRA and pension income to $13,243 of adjusted gross income, while Social Security remains nontaxable. The applicable single-filer standard deduction, including the age-65 addition, exceeds that income, producing zero taxable income and therefore zero federal income tax before refundable credits." -us,scenario_031,head_medicaid_eligible,37,llm_error,health_coverage,False,,"Every model stopped at a MAGI-style screen: gross income of $23,853 against a 138% FPL line for a single Californian. That gross is 142% FPL — only four points over — and the deciding step is the non-MAGI senior-or-disabled income test, which deducts health insurance premiums including the Medicare Part B premium the head pays at 67, taking countable income to roughly 127% FPL and inside California's Aged & Disabled limit. A second cluster never reached the income test at all, either denying that age 67 by itself satisfies the aged/blind/disabled criterion, asserting Medicare eligibility bars Medicaid, or applying a $2,000 resource limit California eliminated on January 1, 2024, which the $4,200 bank balance would have failed." -us,scenario_031,snap,5,llm_error,categorical_eligibility,False,,"The trap is that California's broad-based (Modified) categorical eligibility waives only the gross income and resource tests — the 100%-FPL net income test still decides eligibility, and the 1–2 person minimum allotment is payable only to a household that is eligible. Countable unearned income here is $23,853/year = $1,987.75/month (gross Social Security $9,445 + private pension $13,608 + taxable IRA distributions $800, all counted in full for SNAP), and the only deduction that applies is the ~$207 one-person standard deduction: there are no earnings, no reported rent, mortgage payment, or utility costs (a $77,000 mortgage balance is not a housing expense), and $50/year of over-the-counter expenses is far below the $35/month excess-medical threshold for an elderly member; the $1,165 alimony expense is not a SNAP deduction. Net income of roughly $1,780/month exceeds the ~$1,305 one-person net income limit, so the household is ineligible. Independently, 30% of net income (~$534) exceeds the one-person maximum allotment (~$298), so the computed allotment is $0 and there is no eligible household for the minimum benefit to floor." +us,scenario_031,head_medicaid_eligible,43,llm_error,taxable_income_or_deductions,False,,"At 67, the head qualifies as aged, so the engine tests Medi-Cal through the optional senior-or-disabled pathway (California's Aged & Disabled FPL program). This is a non-MAGI pathway with a 138% FPL income limit. Countable income is gross income minus health insurance premiums, including the Medicare Part B standard premium (about $202.90/month, or about $2,435/year in 2026). That brings the $23,853 gross down to about $21,400, below 138% FPL (about $21,600). The $4,200 in bank assets easily passes California's asset test. Almost every model compared gross or near-gross income with the limit and never subtracted the Part B premium. Others used the wrong threshold (100% FPL or the SSI rate), applied the $2,000 SSI resource limit, or denied that an aged person has a Medicaid category." +us,scenario_031,snap,6,llm_error,categorical_eligibility,False,,"The head is 67, so this is an elderly household. PolicyEngine does not let California's 200% FPL broad-based categorical eligibility skip the net income test for this household. Monthly gross income is $1,987.75 (Social Security, pension and IRA). The only deduction is the $209 standard deduction. There are no shelter costs, and the $50/yr OTC medical expense is below the $35/mo threshold, so net income is about $1,779/mo. That is well above the 1-person net limit of about $1,305/mo (100% FPL), so the household is not SNAP-eligible. The one/two-person minimum benefit only applies to eligible households, so SNAP is $0. Five models treated BBCE as automatic eligibility and paid the minimum benefit. The sixth undercounted income." us,scenario_031,state_income_tax_before_refundable_credits,1,llm_error,credit_phaseout,False,,"California first computes $75.37 of tax on $7,537 of taxable income, then applies $312.93 of nonrefundable credits, including the aged-or-blind exemption credit. Those credits fully eliminate the liability, so state income tax after nonrefundable credits and before refundable credits is $0." us,scenario_032,child1_chip_eligible,13,llm_error,health_coverage,False,,"CHIP is residual by statute: 42 U.S.C. 1397jj(b)(1)(A) and 42 CFR 457.310 exclude from ""targeted low-income child"" any child who is eligible for Medicaid, so the CHIP question is settled by first running the Medicaid test. Minnesota's Medical Assistance covers children ages 6-18 — PolicyEngine's OLDER_CHILD category — far above this household's $42,664 for three people (~160% FPL), so the child is Medicaid-eligible and thereby barred from CHIP. Every wrong model instead applied a standalone CHIP income ceiling, most of them quoting Minnesota's 275%/283% FPL child limits, which are the Medical Assistance and MinnesotaCare bands rather than CHIP bands. They read ""income below the child-coverage limit"" as qualifying when it is precisely what disqualifies: several stated the Medicaid fact outright and then used it backwards." us,scenario_032,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -283,37 +285,37 @@ us,scenario_032,child1_medicaid_eligible,4,llm_error,categorical_eligibility,Fal us,scenario_032,child1_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility is determined separately for each person and does not pass from a Social Security beneficiary to a dependent. The six-year-old child is under age 65, and the stated facts include no end-stage renal disease, ALS, disability-based Medicare entitlement, or other qualifying pathway, so the child's eligibility is No." us,scenario_032,child1_wic_eligible,6,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday; a six-year-old is outside the federally eligible categorical group. Income eligibility cannot override this age restriction, so the child is ineligible regardless of whether household income falls below 185% of the federal poverty guidelines." us,scenario_032,federal_income_tax_before_refundable_credits,3,llm_error,thresholds_rates,False,,"The decisive rule is the 2026 federal parameter regime: the household receives a $2,200 CTC, split into $500 nonrefundable and $1,700 refundable. The $500 nonrefundable portion fully eliminates pre-credit income tax, while the refundable CTC and $4,427 EITC are excluded from federal_income_tax_before_refundable_credits." -us,scenario_032,federal_refundable_credits,34,llm_error,thresholds_rates,False,,"The case turns on two 2026 parameters that most models replaced with stale or lapsed values: the one-child EITC maximum of $4,427, which applies in full because a joint filer with $29,000 of wages and $30,916 of AGI (wages plus $1,916 of taxable Social Security, from 0.5 × ($35,832 − $32,000)) sits on the plateau below the joint phase-out start, and the refundable child tax credit cap of $1,700 against a $2,200 per-child credit with a $2,500 earnings floor. A large bloc assumed the TCJA child credit lapsed for 2026 and paid the pre-TCJA $1,000 additional child tax credit under the $3,000 floor rule, losing exactly $700; a second bloc invented an EITC phase-out at this income by importing single-filer thresholds. Because AGI of $30,916 falls below the 2026 married-joint standard deduction of $32,200, taxable income and pre-credit tax are zero, so nothing is absorbed nonrefundably and the answer is simply $4,427 + $1,700 = $6,127." -us,scenario_032,free_school_meals_eligible,19,llm_error,state_local_rule,False,,"The decisive rule is Minnesota's universal free school meals statute (Minn. Stat. § 124D.111, enacted March 2023), which PolicyEngine encodes as `state_has_universal_free_school_meals = True` and which forces `school_meal_tier = FREE` for every K-12 student in the state regardless of household income or direct certification. The income test is therefore irrelevant here: the engine itself records a school_meal_fpg_ratio of 1.56 (above the 1.30 federal free cutoff) and categorical eligibility of False, yet still returns FREE and $1,130.96 of annual free-meal support for the single 6-year-old in K-12. Every wrong model ran only the federal NSLP means test — 130% FPG plus SNAP/TANF direct certification — and treated failing it as dispositive, so none reached the state override that the MN state code triggers. Several also mis-stated the FPG ratio (claiming 160-170% against the engine's 156%), but that arithmetic slack is incidental; the single shared trap is the missing state universal-meals pathway." +us,scenario_032,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The household files MFJ with one qualifying child. Earned income is $29,000. AGI is $30,916: wages plus $1,916 of taxable Social Security, because provisional income of $35,832 exceeds the $32,000 base but stays below $44,000. Both figures sit below the 2026 MFJ one-child EITC phase-out start (above $31,000), so the EITC is the full 2026 maximum of $4,427. Tax before credits is $0 because AGI is below the $32,200 standard deduction, and 15% of ($29,000 - $2,500) = $3,975 exceeds the cap. The refundable CTC is therefore the full OBBBA refundable cap of $1,700 out of the $2,200 credit, for a total of $6,127. The wrong answers split into three groups: models that used the expired post-TCJA-sunset $1,000 ACTC, models that phased out the EITC using a single-filer or stale threshold, and models that misstated the EITC maximum or the $1,700 refundable cap." +us,scenario_032,free_school_meals_eligible,24,llm_error,state_local_rule,False,,"The household's income ($42,664, 1.56 times the poverty guideline) does exceed the federal NSLP 130% FPL free-meal limit, and it has no SNAP/TANF categorical eligibility. Minnesota, however, runs a universal free school meals program (in place since the 2023-24 school year), so PolicyEngine sets school_meal_tier to FREE for every K-12 child in the state regardless of income. Every model that answered stopped at the federal income test or the categorical-eligibility check and never applied Minnesota's universal free-meals override." us,scenario_032,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_032,head_medicaid_eligible,14,llm_error,taxable_income_or_deductions,False,,"The controlling computation places the head's Medicaid MAGI at 1.56 times FPL, above Minnesota's income limit for the applicable adult MAGI pathways. Being age 50 or living with a child does not supply a separate qualifying category at that income, so the engine assigns category NONE and returns no Medicaid eligibility. The wrong answers instead understated MAGI or the applicable FPL ratio, used an incorrect household size, or invented a higher parent/caretaker threshold." +us,scenario_032,head_medicaid_eligible,15,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds back the non-taxable part of Social Security benefits. The Head's $13,664 in Social Security dependent benefits therefore counts in full alongside the $29,000 in wages. That puts household MAGI at $42,664, or 1.56 times the FPL for a 3-person household, above Minnesota's 138% expansion-adult limit. Minnesota's Medical Assistance limit for parents and caretakers is the same 133% FPL plus the 5% disregard, so the 50-year-old Head qualifies through no pathway. Most models counted only wages, or wages plus the taxable part of Social Security (about $30,916). The rest correctly found $42,664 above 138% but then invented a higher Medicaid limit for parents, or a larger household." us,scenario_032,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific and requires the applicant to be pregnant, postpartum, breastfeeding, an infant, or a child under age five. A 50-year-old head with all relevant unlisted statuses set to false fails categorical eligibility regardless of household income or another household member's circumstances." -us,scenario_032,payroll_tax,35,llm_error,state_local_rule,False,,"The reference total decomposes as employee Social Security $1,798 (6.2% × $29,000) + employee Medicare $420.50 (1.45% × $29,000) + Minnesota Paid Leave employee contribution $127.60, and the last component is the trap. Minnesota Paid Leave begins collecting premiums on wages paid from January 1, 2026 at a 0.88% total rate on covered wages, of which the employer may deduct up to half from the employee, so 0.44% × $29,000 = $127.60 is a mandatory employee state payroll tax within the requested output's scope. Almost every wrong model computed the two federal components exactly right and then affirmatively declared that Minnesota imposes no employee-side payroll tax, landing on $2,218.50 (or its roundings, $2,219). The remaining wrong answers either used the wrong state rate (0.35% instead of 0.44%), doubled the federal share into the excluded employer side, invented a composite rate, or returned nothing." -us,scenario_032,reduced_price_school_meals_eligible,21,llm_error,state_local_rule,False,,"Reduced-price and free are mutually exclusive tiers: a student placed in the FREE tier generates no reduced-price support, so the flag is 0. Minnesota's Free School Meals for Kids program (Minn. Stat. § 124D.111, effective 2023-24) gives every enrolled student free breakfast and lunch regardless of income, so the engine assigns school_meal_tier = FREE at an FPG ratio of 1.56 and returns the $1,130.96 subsidy through the free tier. Every wrong model applied only the federal 130%–185% FPG band from the National School Lunch Act, landed the household in the reduced-price window, and stopped there. None consulted the state field (state: MN) to check for a universal free meals statute that overrides the federal income tiering." +us,scenario_032,payroll_tax,40,llm_error,state_local_rule,False,,"The reference adds three pieces: federal FICA on the head's $29,000 in wages ($1,798 Social Security plus $420.50 Medicare), and Minnesota's Paid Leave employee contribution of $127.60 (0.44% of $29,000 in MN paid-leave taxable wages). Minnesota Paid Leave premiums began in 2026. Almost every model stopped at 7.65% federal FICA ($2,218.50), and many said outright that Minnesota has no mandatory employee payroll tax or that the paid-leave deduction is optional. The one model that did include the paid-leave premium used 0.35% instead of the 0.44% employee share. The few remaining errors come from made-up rates, doubling toward the employer-plus-employee total, or missing output." +us,scenario_032,reduced_price_school_meals_eligible,26,llm_error,state_local_rule,False,,"Minnesota has universal free school meals, so PolicyEngine gives every student the FREE tier whatever the family's income. Reduced-price eligibility is false only because the better free tier replaces it. Income is not the reason: the household sits at 1.56 of the poverty guideline, inside the 130-185% reduced-price band. Every model that answered placed income in the federal 130-185% band and stopped there. None applied Minnesota's universal free-meals rule, which moves the student out of the reduced-price tier." us,scenario_032,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,snap,8,llm_error,categorical_eligibility,False,,"SNAP eligibility ends at the gross-income screen for this three-person Minnesota household: $29,000 of wages plus $13,664 of Social Security dependent benefits produces $42,664 of annual gross income, above the applicable limit. Take-up assumptions and benefit-allotment calculations do not create eligibility after that screen fails, so the annual benefit is $0." us,scenario_032,spouse_chip_eligible,4,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The 18-year-old spouse qualifies for Minnesota Medicaid under the OLDER_CHILD category, so that Medicaid eligibility precludes CHIP eligibility without regard to whether household income is below the CHIP threshold." -us,scenario_032,spouse_medicaid_eligible,15,llm_error,categorical_eligibility,False,,"The trap is category assignment, not arithmetic: PolicyEngine classifies anyone aged 6–18 as OLDER_CHILD for Medicaid regardless of their role in the household, so the 18-year-old spouse is tested against Minnesota's Medical Assistance limit for children under 19, not the 138% FPL adult expansion limit. Minnesota's children's MA ceiling sits far above 138% FPL, so a MAGI of 1.56 × FPL clears it comfortably and the engine returns eligible. Nearly every wrong model computed household MAGI correctly (~$42,664, ≈1.56–1.60 × FPL for a family of three) and then measured it against the adult 133/138% FPL expansion threshold because the person was labeled ""Spouse."" Marital status does not remove a person under 19 from the child category; the head, at age 50 and above 138% FPL, is the one the adult test correctly excludes." +us,scenario_032,spouse_medicaid_eligible,17,llm_error,age_disability,False,,"The spouse is 18, and under Medicaid rules anyone under 19 falls in the child category, whether or not they are married or claimed as a dependent. PolicyEngine puts the spouse in the OLDER_CHILD category (ages 6-18). Minnesota's income limit for that category is well above the spouse's MAGI of 1.56x FPL, so the spouse is eligible. Nearly every wrong model treated the spouse as an adult and tested household MAGI against the 138% FPL expansion limit, a parent/caretaker limit or a disability/SSI pathway. None of those is the category that decides this case." us,scenario_032,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_032,spouse_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC does not cover every low-income woman of childbearing age or every adult living with a child. An adult woman must be pregnant, breastfeeding, or within the applicable postpartum period; the spouse has none of these statuses because all unlisted statuses are false, so she fails categorical eligibility regardless of the income test." us,scenario_032,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Minnesota starts from federal AGI ($29,000 wages plus the $1,916 of Social Security made taxable by the §86 provisional-income formula = $30,916), then applies two Minnesota-specific subtractions the models ignored or undersized: the Minnesota Social Security benefit subtraction, which removes the entire $1,916 for a joint filer at this income (far below the phase-out range), and the Minnesota dependent exemption for Child 1 (roughly $5,300, indexed). That leaves $29,000 of Minnesota income against a 2026 joint standard deduction of roughly $30,700 plus the dependent exemption, so Minnesota taxable income is zero and tax before refundable credits is $0. Every wrong answer pairs a too-small joint standard deduction with a missing Social Security subtraction and a missing dependent exemption, manufacturing a few hundred to a few thousand dollars of phantom taxable income taxed at the 5.35% first-bracket rate." -us,scenario_032,state_refundable_credits,39,llm_error,state_local_rule,False,,"Minnesota's 2023 tax act repealed the old percentage-of-federal-EITC Working Family Credit and replaced it with a flat credit — 4% of the first ~$8,750 of earned income, plus a per-child amount that applies only to qualifying *older* children (ages 18–23) — paired with a separate fully refundable Child Tax Credit ($1,750 base, inflation-indexed) for each child under 18; PolicyEngine computes the pair jointly as the Child and Working Families Credit. The single child here is age 6, so the child generates the CTC and contributes nothing to the WFC's older-child amount, leaving the WFC as the 4%-of-earnings piece on the head's $29,000 of wages. AGI of $30,916 — wages plus $1,916 of taxable Social Security (50% of provisional income $35,832 above the $32,000 joint base) — sits below the joint-filer phaseout threshold, so the 12% combined reduction never begins and the full 2026 inflation-indexed amounts stand at $2,187.60. Nearly every wrong answer resurrected the repealed EITC-percentage WFC (overshooting by $1,000–$1,400), dropped one of the two components entirely, denied Minnesota has refundable credits at all, or used the unindexed 2023 statutory dollars." -us,scenario_033,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The correct 2026 computation is fully determined: gross income $96,095 ($79,000 wages + $16,320 taxable Social Security at the 85% cap + $775 interest), less $12,886.76 of above-the-line items (the $11,400 SSTB self-employment loss and the $1,487 HSA deduction), gives AGI $83,208.23; the deduction stack is $32,200 MFJ base + $1,650 per aged spouse + the OBBBA $6,000-per-spouse senior deduction (fully allowed, MAGI far below the $150,000 joint phaseout) = $47,500, leaving taxable income $35,708.23 and tax of $2,480 + 12% × $10,908.23 = $3,788.99. Two exclusions matter as much as the deductions: the $243 state/local tax refund produces no gross income under the tax benefit rule for a standard-deduction filer in a state with no income tax, and the $1,225 of auto loan interest, the $220,000 mortgage balance, the $400 of premiums and $50 of OTC expenses generate no deduction at all. The wrong answers cluster into four traps: dropping the $12,000 senior deduction (the largest single miss), assuming a TCJA sunset with personal exemptions and a 15% second bracket, adding back the $243 refund, and inventing auto-loan-interest or overtime-premium deductions. A fifth group derived the right taxable income and then submitted a different number than its own arithmetic produced." +us,scenario_032,state_refundable_credits,46,llm_error,state_local_rule,False,,"Starting in 2023, Minnesota's Child and Working Families Credit replaced the old Working Family Credit, which was a share of the federal EITC. The new credit has two parts: a per-child credit, originally $1,750 and now inflation-indexed, and a working-family part equal to 4% of earnings, capped at roughly $365 per return. Both parts phase out together at 12% of income above a joint-filer threshold. This household's AGI is $30,916 ($29,000 in wages plus $1,916 of taxable Social Security), which is well below the 2026 joint-filer threshold. So the full indexed child credit plus the capped working-family part comes to $2,187.60. Most models went wrong in one of four ways: they used the repealed EITC-percentage schedule for the working-family part, left out one of the two parts, used unindexed 2023 amounts, or applied a phaseout that does not reach this AGI." +us,scenario_033,federal_income_tax_before_refundable_credits,37,prompt_ambiguity,taxable_income_or_deductions,False,,"The correct 2026 computation includes the $243 state tax refund in gross income, which gives AGI of $83,451.23. It then subtracts $47,500 of deductions: the $32,200 MFJ standard deduction, two $1,650 aged additions, and the OBBBA $6,000 senior deduction for each spouse. That leaves taxable income of $35,951.23, taxed as $2,480 plus 12% of $11,151.23, or $3,818.15. The largest group of wrong answers either left out the $12,000 senior deduction or assumed the TCJA had expired, bringing back personal exemptions and the 15% bracket. Answers close to the reference went wrong in smaller ways. Some dropped the $243 refund. Others took deductions this household does not qualify for: auto-loan interest and a qualified-overtime deduction, even though the spouse's usual hours are 40 a week. Several models computed a figure and then submitted a different, unsupported number. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." us,scenario_033,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_033,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on each spouse's wages, producing $4,898 and $1,145.50 respectively and a total of $6,043.50. Both workers are below the Social Security wage base, their combined wages are below the married-filing-jointly Additional Medicare Tax threshold, and the self-employment loss belongs outside employee payroll tax. Most wrong answers correctly identified this base and these rates but then replaced their own exact result with an unsupported rounded or adjusted number." us,scenario_033,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Self-employment tax is a nonnegative liability assessed only on positive net earnings from self-employment. The listed negative SSTB amount does not create a negative self-employment tax or refundable tax offset, so the self-employment tax base and liability are both zero." -us,scenario_036,federal_income_tax_before_refundable_credits,29,llm_error,taxable_income_or_deductions,False,,"The case turns on one parameter set: for tax year 2026 a single filer's standard deduction is $16,100 with no personal exemption, and the rate schedule is 10% to $12,400 then 12% to $50,400, so $59,220 of taxable pension income leaves $43,120 taxable and $1,240 + 12% x $30,720 = $4,926.40, with no nonrefundable credits available to a 56-year-old with no dependents. Eight models assumed the TCJA individual provisions lapse and rebuilt the return on a pre-TCJA base of roughly an $8,300 standard deduction plus a $5,300 personal exemption and a 15% second bracket, landing near $6,250; another cluster carried stale 2023-2025 deductions and bracket edges into 2026 and landed near $5,070-$5,120. A third cluster reached $4,926 exactly and then overrode it with an unexplained figure. Secondary traps — inferring mortgage interest from a loan balance, itemizing medical costs below the 7.5% AGI floor, head-of-household filing, and an elderly credit that requires age 65 — account for the remaining misses." +us,scenario_036,federal_income_tax_before_refundable_credits,31,llm_error,thresholds_rates,False,,"The correct computation is short. Taxable income is $59,220 of pension income minus the 2026 single standard deduction of $16,100, which gives $43,120. The tax is 10% of $12,400 plus 12% of $30,720, which gives $4,926.40, and no nonrefundable credit applies. The shared trap is the 2026 parameter set under OBBBA (P.L. 119-21), which made the TCJA rate structure permanent and set the personal exemption to zero. Models went wrong in three ways. Many applied a TCJA sunset instead: an $8,300 standard deduction plus a personal exemption of about $5,300, with a 15% bracket. Others used a stale 2023, 2024 or 2025 standard deduction or bracket threshold. Several computed exactly $4,926 and then submitted a different number." us,scenario_036,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New Jersey Medicaid requires both membership in a qualifying eligibility category and satisfaction of that category's financial rules. This 56-year-old has medicaid_category NONE, and MAGI income of 3.71 times FPL exceeds the limits for all applicable MAGI pathways; the listed expenses and assets do not create another pathway." us,scenario_036,payroll_tax,1,llm_error,payroll_tax_base,False,,"Taxable private pension income is not wages and is excluded from the employee Social Security, Medicare, and mandatory state payroll-tax bases. With no wages or other earned income listed, every employee-side payroll-tax component is zero." us,scenario_036,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_036,state_income_tax_before_refundable_credits,27,llm_error,taxable_income_or_deductions,False,,"The correct New Jersey computation is short: the $59,220 private pension is fully taxable because NJ's retirement-income exclusion requires age 62+ or disability and the head is 56, so nj_agi is $59,220, less the $1,000 single personal exemption gives $58,220, taxed on the single schedule (1.4% to $20,000, 1.75% to $35,000, 3.5% to $40,000, 5.525% above) for $1,724.16. Two traps separate that from the wrong answers. Ten models subtracted an NJ medical-expense deduction for costs above 2% of gross income — most of them counting the $7,389 employer-sponsored premium, which is withheld pre-tax and already outside NJ income — and landed near $1,260 or $1,670, while PolicyEngine's nj_taxable_income is nj_agi less the exemption alone. Five more applied the age-62 pension exclusion to a 56-year-old and answered $0; the remainder dropped the $1,000 exemption, invented an NJ standard deduction or property-tax credit that does not exist, mis-stated the bracket rates, or abandoned a correct bracket computation for an unexplained round number." +us,scenario_036,state_income_tax_before_refundable_credits,30,llm_error,taxable_income_or_deductions,False,,"The reference starts from NJ gross income of $59,220 (all taxable pension), subtracts only the $1,000 single personal exemption to reach $58,220, and applies the NJ single schedule: $280 + $262.50 + $175 + 5.525% x $18,220 = $1,724.16. The head is 56 and not disabled, so NJ's pension/retirement exclusion (age 62+ or disabled) does not apply, and the reference allows no medical expense deduction. Twelve models subtracted a medical deduction above the 2% floor, several of them counting the $7,389 employer-sponsored premium. Six models wrongly excluded the whole pension or part of it despite the age test. The rest invented standard deductions or credits, used wrong bracket rates, or left out the $1,000 exemption." us,scenario_036,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_037,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"The case turns on one 2026 provision: the OBBBA below-the-line deduction for qualified FLSA overtime premium, which lets a non-itemizer stack $9,629.03 on top of the $16,100 single standard deduction, cutting taxable income from $25,439 to $15,810.43 and tax from $2,804.68 to $1,649.25 under the 2026 schedule (10% to $12,400, then 12%). A second trap is the $5,789 employer-sponsored insurance premium, which does not reduce the stated $41,943.38 of employment income — only the $385.90 traditional 401(k) deferral and the $18.03 traditional IRA deduction do. A third cluster assumed TCJA sunset parameters (an ~$8,300 standard deduction, a ~$5,300 personal exemption and a 15% second bracket), all of which OBBBA superseded, and a fourth applied a saver's credit whose rate is 0% at AGI $41,539. Every wrong answer traces to one or more of these four." +us,scenario_037,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"The step that separates the reference from most wrong answers is the OBBBA qualified-overtime deduction. The full $9,629 FLSA overtime premium is deductible below the line, and it stacks on the $16,100 2026 single standard deduction. From AGI of $41,539 ($41,943 minus the $386 traditional 401(k) and the $18 IRA deduction), taxable income is $15,810, and tax is $1,240 + 12% × $3,410 = $1,649.25. Most models left out the overtime deduction, and many made other mistakes: they assumed the TCJA had sunset (personal exemptions and a 15% bracket), subtracted the ESI premiums from wages that were already stated, used 2024 or 2025 standard deductions and brackets, or claimed a Saver's Credit even though AGI of $41,539 is above the 2026 single limit." us,scenario_037,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A 44-year-old single filer with no dependent children or qualifying education expenses and $41,943 of wages receives neither EITC nor refundable CTC. All other refundable federal credit components evaluated to zero, so federal refundable credits total $0." us,scenario_037,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"North Carolina’s Medicaid expansion does not make every adult aged 19–64 eligible; the expansion pathway applies an income limit. At 2.60 times FPL, the head exceeds the applicable MAGI threshold and qualifies through no other Medicaid category, so the eligibility category is NONE." -us,scenario_037,payroll_tax,13,llm_error,payroll_tax_base,False,,"Employee-side payroll tax here is a flat application of 6.2% OASDI and 1.45% HI to the full $41,943 of gross wages: $2,600.49 + $608.18 = $3,208.67, with no wage-base cap in play, no Additional Medicare Tax, and no mandatory employee payroll tax in North Carolina. The trap is the payroll-tax base: the listed $5,789 employer-sponsored insurance premium is an employer-paid premium input, not a section 125 salary-reduction election, so it does not shrink FICA wages; likewise the $404 of traditional 401(k)/IRA contributions and the $9,629 FLSA overtime premium leave the FICA base untouched (the overtime deduction is income-tax-only). Seven models subtracted the $5,789 premium and landed on 7.65% × $36,154 = $2,765.78, understating the tax by ~$443. The remaining wrong models identified the correct base and rates but submitted figures inconsistent with their own arithmetic, or submitted nothing." -us,scenario_037,state_income_tax_before_refundable_credits,32,llm_error,taxable_income_or_deductions,False,,"North Carolina's 2026 individual income tax is a single flat 3.99% rate on federal AGI less the $12,750 single standard deduction, so the whole case turns on building the right base: AGI = $41,943 wages less only the traditional 401(k) ($386) and deductible traditional IRA ($18), i.e. $41,539.45, giving $28,789.45 taxable and $1,148.70 of tax. Two subtractions are traps: the $5,789 employer-sponsored insurance premium is already excluded from the reported wage figure and is not a further above-the-line adjustment, and the $9,629 FLSA overtime premium feeds a below-the-line federal deduction that never touches AGI and that North Carolina does not conform to. Nine models subtracted the ESI premium (AGI $35,750 → $23,000 taxable → $917.70), three subtracted the overtime premium (AGI $31,910 → $19,160 taxable → ~$764), and most of the remainder substituted a stale rate (4.75%, 4.5%, 4.25%, 4.0%) or an invented standard deduction ($10,750, $13,750, $15,400, $15,500, $21,500) for the 2026 parameters. Several models reached $1,148.68 in their own work and then submitted a different number." +us,scenario_037,payroll_tax,14,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 7.65% of the full $41,943 of gross wages, which already include the overtime: 6.2% Social Security ($2,600.49) plus 1.45% Medicare ($608.18), for $3,208.67. The facts list no cafeteria-plan salary reduction, so the $5,789 employer-sponsored insurance premium stays in the FICA wage base. The main trap was subtracting that premium as a pre-tax deduction, which eight models did to reach $36,154 and $2,765.78, about $442.89 too low. The other wrong models used the right base but then made arithmetic slips, added unexplained adjustments, rounded coarsely, or gave no answer." +us,scenario_037,state_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"NC tax starts from federal AGI of $41,539.45, which is gross wages less only the traditional 401(k) ($386) and traditional IRA ($18) contributions. Subtracting the $12,750 single standard deduction leaves $28,789.45 of taxable income, and the 2026 flat 3.99% rate gives $1,148.70. The largest group of models also subtracted the $5,789 employer-sponsored insurance premiums from the stated gross wages, which gives an AGI of $35,750 and a tax of $917.70. A second group subtracted the $9,629 overtime premium, which is a federal below-the-line deduction that never reduces the AGI NC starts from. That gives $31,910 and about $764. The remaining models used outdated or invented standard deductions or rates, made up credits, or changed a correct $1,148.68 figure into a different final number." us,scenario_038,child1_chip_eligible,8,llm_error,health_coverage,False,,"CHIP is residual coverage: it reaches only children who fail Medicaid, so a CHIP determination requires a Medicaid floor test before any CHIP ceiling test. Household income of ~$22,992 for a family of four is roughly 71% FPL, far under Louisiana's Medicaid limit for children ages 6–18 (142% FPL), so 7-year-old child1 is Medicaid-eligible under the OLDER_CHILD category and CHIP eligibility is therefore 0. Every wrong model screened only against an upper LaCHIP ceiling (200–250% FPL), read ""income well below the ceiling"" as qualifying, and never applied the lower bound — the very fact that made the answer No." us,scenario_038,child1_head_start_eligible,3,llm_error,age_disability,False,,"Head Start for preschool-age children requires the child to fall within the program’s preschool age range; a seven-year-old does not satisfy that categorical requirement. Income eligibility cannot override the age restriction, so Child 1 is ineligible even under an income calculation that falls below a poverty guideline." us,scenario_038,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Louisiana evaluates a 7-year-old under the Medicaid older-child category. The household's MAGI is 0.66 times FPL, below that category's 2026 income limit, so the child qualifies; the listed assets do not alter this MAGI eligibility determination." @@ -323,7 +325,7 @@ us,scenario_038,child2_head_start_eligible,4,llm_error,categorical_eligibility,F us,scenario_038,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Louisiana evaluates the 6-year-old under the older-child Medicaid category using household MAGI relative to the applicable child income limit. The household's MAGI is 0.66 times FPL, so the child satisfies that category's income test; the listed assets do not disqualify a MAGI-category child." us,scenario_038,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday, so a six-year-old is categorically ineligible regardless of household income. All four models incorrectly treated age six as falling within the WIC child category; the models that also evaluated income therefore never reached the correct result because they first misapplied the age rule." us,scenario_038,federal_income_tax_before_refundable_credits,2,llm_error,other,False,,"The requested amount cannot fall below zero: taxable income and income tax before credits are zero, and nonrefundable credits used are capped at the existing tax liability. Refundable credits are excluded from this output, so the resulting federal income tax before refundable credits is $0." -us,scenario_038,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The case turns on two 2026 parameters plus one definition. The indexed EITC maximum for two qualifying children is $7,316, and earned income of $21,896 — wages $7,468 plus self-employment income $15,525 less the $1,097 deduction for half of self-employment tax — sits on the plateau, past the end of the 40% phase-in and far below the married-joint phase-out start, so the full maximum applies. The refundable CTC is then the 15% phase-in on earnings above $2,500, or $2,909.35, which binds well below the $1,700-per-child refundable cap. Models failed by using stale or invented EITC maxima (including the $8,231 three-child figure), by reverting the refundable CTC to a pre-TCJA $1,000-per-child cap with a $3,000 floor, or by running the 15% phase-in on gross earnings of $22,993 instead of earned income net of the SE-tax deduction." +us,scenario_038,federal_refundable_credits,40,llm_error,thresholds_rates,False,,"The reference is the sum of two credits. The first is the 2026 two-child EITC plateau maximum of $7,316: earned income of about $21,896 (wages plus SE income minus the deductible half of SE tax) sits above the phase-in end and below the MFJ phase-out start. The second is the refundable ACTC, equal to 15% × (earned income − $2,500) = $2,909.35. That amount binds below the $1,700-per-child ($3,400) refundable cap under the permanent post-OBBBA $2,200 CTC. The wrong answers fall into four groups: (1) assuming the TCJA lapsed, so the CTC reverted to $1,000 per child and the ACTC capped at $2,000; (2) granting the full $3,400/$3,600 refundable cap without the 15% earned-income phase-in; (3) projecting the wrong 2026 EITC maximum; (4) measuring earned income as gross SE income or as 92.35% of SE income instead of SE income minus half the SE tax." us,scenario_038,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The household qualifies for the free tier through two independent routes: SNAP receipt confers categorical eligibility, and school-meal income is 70% of the federal poverty guideline, below the 130% cutoff. The two K–12 children therefore generate positive annual free-school-meal support, so the binary output is Yes." us,scenario_038,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_038,head_medicaid_eligible,3,llm_error,health_coverage,False,,"The trap is Louisiana's ACA adult expansion: LA adopted expansion effective July 1, 2016, so a 25-year-old non-disabled, non-pregnant adult qualifies through the MAGI ADULT category at up to 138% FPL, with no resource test. The household's countable MAGI of about $22,992 (wages $7,468 plus self-employment $15,525) measured against the four-person poverty line is 0.66 FPL — less than half the 138% ceiling — so the head clears the only test that applies. Every wrong model priced the head against a pre-expansion parent/caretaker standard (or an asset screen) instead of running the single MAGI-versus-138%-FPL comparison, and none of them actually computed the FPL ratio the record supplies." @@ -331,19 +333,19 @@ us,scenario_038,head_wic_eligible,3,llm_error,categorical_eligibility,False,,"WI us,scenario_038,payroll_tax,5,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes only employee-side FICA on the head’s wage earnings; self-employment income belongs in the separate self-employment-tax output. PolicyEngine applies the underlying wage amount and component-level calculations to produce $462.98 of Social Security tax plus $108.28 of Medicare tax, totaling $571.26." us,scenario_038,reduced_price_school_meals_eligible,2,llm_error,thresholds_rates,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a general test of whether income is below 185% of the federal poverty guideline. At 70% of the guideline, and with categorical eligibility, this household receives the superior free-meals tier, so reduced-price support is zero." us,scenario_038,self_employment_tax,8,llm_error,payroll_tax_base,False,,"The SECA computation here is a single closed-form line with no branch points: net earnings from self-employment are $15,525 × 0.9235 = $14,337.3375, and because $7,468 of wages plus that base sit far below the Social Security wage base, the full 15.3% applies, giving exactly $2,193.61. The separating step is executing that arithmetic to the cent and stopping there — the 92.35% factor already embodies the employer-share equivalent, and the one-half-of-SE-tax deduction under §164(f) reduces AGI for income tax only, never the SECA base itself. Seven of the eight wrong answers name the right formula and then miss on execution: rounded or miscomputed multiplications ($2,191.73, $2,192, $2,194.96, $2,196.28, $2,197, $1,360.68) or a second, invalid reduction of the base by the half-SE-tax deduction ($2,054.08)." -us,scenario_038,snap,39,llm_error,thresholds_rates,False,,"The correct chain is fully mechanical: $22,993 of earnings ($7,468 wages plus $15,525 self-employment, counted in full because no cost of producing the income is listed) is $1,916.04 monthly; the 20% earned-income deduction removes $383.21 and the FY2026 four-person standard deduction removes $223, leaving net income of $1,309.83, with no shelter deduction (a $75,000 mortgage balance is not a housing payment) and no medical deduction (nobody is elderly or disabled). The benefit is the $994 FY2026 four-person maximum allotment less 30% of net income, $601.30 a month, and because SNAP is evaluated month by month the October 1, 2026 start of FY2027 uprates the allotment and standard deduction for the final quarter of calendar 2026, producing $7,286.94 rather than a flat $7,215.60. Nearly every wrong answer breaks at one of three points: a maximum allotment or standard deduction taken from the wrong fiscal year or invented outright ($879, $973, $975, $1,020, $1,044, $1,050); a phantom shelter or dependent-care deduction conjured from the mortgage balance; or an outright zero from models that never ran the eligibility tests the household clears at 72% of the poverty guideline. A distinct cluster reproduced the reference month exactly and then flat-multiplied by 12, losing only the October 2026 step-up." +us,scenario_038,snap,38,llm_error,thresholds_rates,False,,"The right answer follows from the standard SNAP formula with FY2026 parameters. Monthly earnings of $1,916.04, from wages plus self-employment, get the 20% earned income deduction ($383.21) and the $223 standard deduction for four people. That leaves $1,310 net, with no shelter deduction because no rent, mortgage payment or utility costs are listed. Thirty percent of net is $393, and the $994 maximum allotment minus $393 gives $601 a month, or $7,212 a year. Most models set up this formula correctly but plugged in the wrong year's parameters: FY2025's $975/$204, guessed maximums of $1,000-$1,050, or a $879 maximum. Others made up shelter, dependent or medical deductions, applied 20% to annual income and then subtracted a monthly deduction, or wrongly declared the household ineligible." us,scenario_038,spouse_chip_eligible,4,llm_error,health_coverage,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The spouse qualifies for Medicaid under Louisiana's OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility without regard to whether other coverage was listed." us,scenario_038,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The trap is that the 18-year-old is labeled \""spouse,\"" and all three models let that label override the age-based pathway: PolicyEngine assigns her the OLDER_CHILD medicaid_category, which covers ages 6-18 in Louisiana and does not require being a tax dependent or an unmarried child. The income test is not close either — household MAGI is 0.66 x FPL, below Louisiana's ~142% FPL older-child limit and below the 138% FPL adult expansion limit, so both the child pathway and the adult pathway the models invoked return eligible. Every wrong answer therefore combines a category error (adult instead of older child) with an unperformed or inverted FPL comparison, and one model additionally imported a nonexistent asset test into a MAGI category." us,scenario_038,spouse_wic_eligible,5,llm_error,categorical_eligibility,False,,"WIC income eligibility does not independently confer eligibility: a person must also be pregnant, breastfeeding, postpartum, an infant, or a child under age five. The spouse is 18, all unlisted statuses are false, and the only children are ages six and seven, so neither parenthood, childbearing age, nor low household income places the spouse in a WIC categorical group." -us,scenario_038,state_income_tax_before_refundable_credits,9,llm_error,taxable_income_or_deductions,False,,"The decisive step is Louisiana taxable-income calculation for a joint filer with two dependents: the household’s approximately $22,992 of income falls below the applicable taxable-income threshold, leaving zero taxable income and therefore zero tax before refundable credits. The wrong substantive answers created positive taxable income by using inapplicable deduction, exemption, and rate regimes; one model derived zero correctly but submitted a contradictory nonzero value." -us,scenario_038,state_refundable_credits,32,llm_error,state_local_rule,False,,"The case turns on one state rule plus one federal parameter: Louisiana's earned income credit is refundable and equals 5% of the federal EITC (raised from 3.5% for tax years beginning on or after January 1, 2019), and it is the only Louisiana refundable credit that fires for this household. The federal base is at its ceiling — $7,468 of wages plus 92.35% of $15,525 in self-employment income gives roughly $21,805 of earned income, which lands above the two-child phase-in completion point and below the joint phase-out threshold, so the 2026 two-child maximum of $7,316 applies in full. Five percent of $7,316 is $365.80. Seventeen models answered $0 by denying the credit exists, calling it nonrefundable, or conditioning it on positive Louisiana tax liability — which refundability specifically overrides, since this household's AGI falls under the joint standard deduction; most of the rest had the 5% structure right and missed only on the federal maximum-credit parameter or the phase-out position." -us,scenario_039,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on the composition of AGI: PolicyEngine's AGI is $32,699.07 — $10,129.10 of taxable Social Security plus $26,800 of IRA distributions plus $2,030 of pension, less the $6,260.03 self-employment loss — with the $25,950 of estate income outside AGI. Holding estate income out leaves $22,569.97 of non-Social-Security income and provisional income of $40,622.47, so the tiered worksheet taxes only $10,129.10 of the $36,105 in benefits ($4,500 + 85% of the $6,622.47 above the $34,000 single threshold) rather than the 85% ceiling of $30,689.25. The single $16,100 standard deduction then leaves $16,599.07, against which the QBI deduction is limited to 20% = $3,319.81, giving taxable income of $13,279.25 and tax of $1,240 (10% to $12,400) + $105.51 (12% on $879.25) = $1,345.51. Every wrong model pulled estate income into AGI, defaulted Social Security to the 85% cap, and took no QBI deduction; many compounded that with joint surviving-spouse brackets and standard deduction or with a sunset-TCJA parameter set that OBBBA superseded." +us,scenario_038,state_income_tax_before_refundable_credits,10,llm_error,state_local_rule,False,,"Louisiana's 2024 tax reform took effect in 2025. It replaced the graduated brackets and the old combined personal exemption–standard deduction ($9,000 MFJ, plus $1,000 per dependent) with a flat 3% rate and a $25,000 standard deduction for joint filers. Federal AGI is $21,896: $22,993 of wages and SE income minus the half-SE-tax deduction of about $1,097. That is below the $25,000 MFJ deduction, so Louisiana taxable income is zero and so is the tax. Most models used pre-reform exemptions and brackets (1.85%, 2%/4%) or applied only the $12,500 single-filer deduction to a joint return, which left positive taxable income. One model reached $0 in its reasoning but submitted a different number." +us,scenario_038,state_refundable_credits,36,llm_error,state_local_rule,False,,"Louisiana has a refundable state EITC equal to 5% of the federal EITC. This couple files jointly with two children, and their earned income is about $21,900 (wages plus self-employment income, minus half of the SE tax). That puts them on the federal EITC plateau at the 2026 two-child maximum of $7,316, so the LA credit is 0.05 x $7,316 = $365.80. Half of the models (18 of 36) said Louisiana has no refundable EITC, or that it is nonrefundable, and answered $0. Most of the others used the 5% rate but misstated the federal EITC maximum. A few used the wrong state rate (3.5% or 50%), and two badly misplaced the household on the federal EITC schedule." +us,scenario_039,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's gross-income source list leaves out estate_income, while its qualified-business-income sources include it. As a result, the $25,950 is excluded from AGI and from the §86 provisional income that sets taxable Social Security (AGI $32,699.07 = $10,129.10 + $26,800 + $2,030 − $6,260.03), yet it still generates a $3,319.81 §199A deduction. That conflicts with IRC §61(a)(15)/§662(a), which tax estate distributions to the beneficiary, and with §199A(c)(3)(A)(ii), which counts an item as QBI only if it is included in taxable income.","The reference files the head as single, because no dependent child is listed. It leaves the $25,950 estate income out of AGI and out of Social Security provisional income, so taxable SS is $10,129.10 and AGI is $32,699.07. It then subtracts the $16,100 standard deduction and a $3,319.81 QBI deduction and taxes $13,279.25 at 10%/12%, giving $1,345.51. Every model that answered except glm-5.3 put the estate income into AGI, which pushed SS to the 85% cap and AGI to about $79,209; many also used joint qualifying-surviving-spouse status, TCJA-sunset parameters, or stale brackets. The models that used single status and correct 2026 parameters all got $8,596.04, and that entire gap comes from how the estate income and QBI deduction are handled. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 1,345.51; the corrected value is 8,596.03. PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))." us,scenario_039,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_039,head_medicaid_eligible,3,llm_error,health_coverage,False,,"Every Virginia Medicaid pathway open to a 61-year-old is income-tested: the ACA expansion adult group caps MAGI at 138% FPL, and the SSI-linked aged/blind/disabled group sits far below that and keys off SSI receipt, which is $0 here. This head's Medicaid MAGI counts the full $36,105 Social Security benefit (untaxed portion included) plus $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, offset only by a $6,260 self-employment loss, landing at 368% FPL — 2.7x the expansion limit — so the engine assigns medicaid_category NONE. All three models either asserted without arithmetic that MAGI fell under 138% FPL or treated the ""is disabled"" flag as a categorical pass that skips the income test entirely. Disability and the expansion group's waived asset test change which income test applies, never whether one applies." -us,scenario_039,head_medicare_eligible,15,llm_error,age_disability,False,,"Medicare's only under-65 entitlement pathways are 24 months of entitlement to Social Security disability insurance benefits (42 U.S.C. §426(b)), ESRD, or ALS — none of which is a bare ""is disabled"" demographic flag. This household's Social Security is coded as retirement income, SSDI/disability-benefit receipt and ESRD are unlisted and therefore zero under the prompt's ""treat any unlisted input as 0/false"" rule, so the only operative test is age ≥ 65, and the head is 61. All 15 models substituted the generic disability flag for the DIB-entitlement trigger, converting a demographic characteristic used elsewhere in the model (SSI, the SNAP elderly/disabled deduction, state disability exemptions) into a Medicare pathway that does not exist." +us,scenario_039,head_medicare_eligible,19,llm_error,age_disability,False,,"A person under 65 qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or with ESRD or ALS. A disability flag on its own does not qualify anyone. The head is 61, and their only Social Security income is retirement income. No SSDI income or SSDI months are listed, so both count as 0, and ESRD and ALS default to false. None of the qualifying pathways applies, so the head is not eligible. All 19 models treated ""is disabled"" as automatic Medicare eligibility, and several also mistook the Social Security retirement income for SSDI." us,scenario_039,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary income, not estate income, Social Security retirement benefits, IRA distributions, private pensions, or self-employment income. No wages were listed, and the prompt requires unlisted numeric inputs to be zero, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax bases are all zero." us,scenario_039,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_039,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Virginia subtracts the federally taxable Social Security in full, so Virginia AGI here equals the non-Social-Security part of federal AGI: $26,800 of IRA distributions plus $2,030 of pension less the $6,260 self-employment loss, or $22,569.97 — the $25,950 of estate income is not in the federal AGI base the state starts from. Nearly every model anchored on the ""$84,625 household income"" figure, kept the estate income in the state base, and arrived at $48,520 of Virginia AGI, roughly quadrupling the taxable amount. Secondary errors compounded it: oversized surviving-spouse or joint standard deductions of $12,000–$17,500 in place of Virginia's $8,750 single amount, $8,000–$9,000 substitutes for that $8,750, and fabricated age-61 or disability subtractions. Correctly, $22,569.97 less $8,750 and the $930 exemption leaves $12,889.97, which stays below the $17,000 start of the 5.75 percent bracket: $60 + $60 + 5 percent of $7,889.97 = $514.50." +us,scenario_039,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine's federal gross-income sources omit the estate_income variable. That drops the $25,950 of estate/trust income from AGI, even though IRC §61(a)(15) and §662(a) include it: $32,699.07 = 26,800 + 2,030 − 6,260 + 10,129.10 taxable Social Security. With it included, taxable Social Security is $30,689.25, Virginia AGI is $48,520, Virginia taxable income is $38,840, and tax is $1,975.80.","The reference's federal AGI of $32,699.07 is IRA $26,800 + pension $2,030 − self-employment loss $6,260 + taxable Social Security $10,129.10. The $25,950 of estate income never enters AGI, which also cuts provisional income enough to reduce taxable Social Security from $30,689 to $10,129. Virginia then subtracts all taxable Social Security, applies the single $8,750 standard deduction (a surviving spouse files as single in Virginia) and one $930 exemption, and taxes $12,889.97 for $514.50. Nineteen models included estate income and landed between $1,969 and $2,029, or exactly $1,975.80 with 2026 parameters. The rest also used the married-filing-jointly deduction, skipped the Social Security subtraction, or invented age or disability relief. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 514.50; the corrected value is 1,975.80. PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))." us,scenario_039,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_040,federal_income_tax_before_refundable_credits,9,llm_error,taxable_income_or_deductions,False,,"Every model got the same AGI the engine used ($28,000 wages + $4,300 taxable Social Security = $32,300, since workers' compensation is excluded and provisional income of $40,600 falls in the 50% tier between the $32,000 and $44,000 MFJ thresholds) or missed it, and then lost the case on the deduction side. For tax year 2026 the TCJA structure is permanent: personal exemptions remain repealed, the MFJ standard deduction is $32,200, the aged additional standard deduction is $1,650 per 65+ spouse ($3,300), and the OBBBA senior deduction adds $6,000 per spouse aged 65+ ($12,000) for a total of $47,500 — well above the $32,300 AGI, so taxable income and tax are $0. Six of the nine models instead applied a TCJA-sunset 2026 with a ~$16,500–$20,000 standard deduction plus revived personal exemptions of ~$10,200–$10,600, a combination that leaves a small positive taxable income taxed at 10%. The two that used a post-OBBBA standard deduction still failed by inflating includible Social Security or by treating the senior deduction as a tax offset rather than a $6,000-per-spouse deduction." us,scenario_040,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -355,11 +357,11 @@ us,scenario_040,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,Fal us,scenario_040,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility applies at age 65, so the 69-year-old spouse qualifies through the age pathway without needing disability-based eligibility or 24 months of SSDI receipt. The spouse’s workers’ compensation and disability facts do not alter that age-based result." us,scenario_040,state_income_tax_before_refundable_credits,6,llm_error,taxable_income_or_deductions,False,,"The case turns on two Arizona-specific rules: Arizona subtracts from federal AGI all Social Security benefits included there, and Arizona's standard deduction conforms to the base federal married-filing-jointly amount (about $32,200 for 2026) with no additional age-65 amount. Removing the $4,300 of taxable Social Security leaves $28,000 of wages as Arizona AGI, which is more than $4,000 below the standard deduction, so taxable income is zero and the 2.5% flat tax yields nothing before refundable credits; itemizing is worse at $16,324 of mortgage interest plus real estate taxes. The wrong answers all manufacture a positive base either by shrinking the deduction — inventing $20,000, $22,000, or an unstated ~$22,700 figure, in two cases by grafting a federal-style age-65 add-on Arizona does not have — or by leaving Social Security in the Arizona base. One model produced the zero result and then overrode it, and one returned nothing at all." us,scenario_040,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"Arizona’s refundable credits are not awarded merely because a married filer is elderly and has income below a general threshold. The household satisfies no applicable Arizona refundable-credit eligibility pathway in 2026, so az_refundable_credits and the aggregate state_refundable_credits are both zero; the other failure is a missing required output." -us,scenario_042,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on three linked steps: this widowed 79-year-old with no dependent files Single, so deductions are $16,100 base + $2,050 aged + the $6,000 OBBBA senior deduction = $24,150; the $3,753 non-Schedule-D capital gains and $71 BDC income stay outside IRS gross income, leaving provisional income of $44,771.71 ($31,109 of dividends/interest/IRA/pension less the $1,127.29 farm loss, plus half of $29,580 in benefits), so the §86 tier-2 formula ($4,500 + 85% × $10,771.71) makes exactly $13,655.95 of Social Security taxable; AGI of $43,637.65 less $24,150 leaves $19,487.65 of taxable income, of which $928 of qualified dividends falls in the 0% band. Taxing the remaining $18,559.65 at 10% up to $12,400 and 12% above yields $1,979.16. Nearly every wrong answer promoted the filer to surviving-spouse/married deduction amounts and $44,000 Social Security thresholds, inflated taxable benefits by folding the capital gains into provisional income or jumping straight to the 85% cap, or dropped the $6,000 senior deduction; a further cluster applied expired pre-TCJA 2026 parameters (personal exemptions and a 15% bracket)." +us,scenario_042,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"A widow with no dependent child files as Single, not Qualifying Surviving Spouse. Under OBBBA's permanent TCJA parameters for 2026, the Single deduction is $16,100 plus the $2,050 aged add-on, plus the $6,000 senior deduction, for $24,150 in total. Taxable Social Security is $16,846.25 under the 25k/34k provisional-income formula, which makes AGI $50,580.71. Of the $26,430.71 in taxable income, the $4,681 of qualified dividends and capital gain falls in the 0% band, and the $21,749.71 of ordinary income is taxed at 10%/12% (1,240 + 12% × 9,349.71 = $2,361.96). The models split into three main error groups: using QSS/joint parameters, which makes the answer too low; leaving out the $6,000 senior deduction or applying expired pre-TCJA law, which makes it too high; and getting the Social Security taxable amount wrong. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)." us,scenario_042,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_042,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax uses employee wages as its base, and the prompt lists no wage or salary income. The pension, IRA distributions, investment income, Social Security benefits, and farm loss do not generate employee-side Social Security or Medicare tax, so employee payroll tax is $0." us,scenario_042,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_042,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Wisconsin gives this 79-year-old single filer no general retirement-income exclusion — its $5,000 retirement subtraction is barred once federal AGI exceeds $15,000 — so the $19,200 of taxable IRA distributions and $4,886 of pension stay in Wisconsin AGI of $29,981.71 after the $13,655.95 of subtractions led by taxable Social Security. The sliding-scale single standard deduction of $12,671.36 plus exemptions of $700 + $250 (age 65+) leave $16,360.34 of taxable income, producing $584.74 at the 3.50%/4.40% brackets. The only nonrefundable offset is the school property tax credit — 12% of the first $2,500 of the $5,011 of real estate taxes, a flat $300 cap — leaving $284.74. Wrong answers split into a zero cluster (fabricated $5,000–$24,000 retirement exclusions, joint-filer standard deductions inferred from the ""surviving spouse"" label, or an uncapped property-tax credit) and a $693–$1,973 cluster (skipping the standard deduction and exemptions, or dropping the $300 credit)." +us,scenario_042,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,"PolicyEngine's Wisconsin module does not apply the retirement income exclusion created by 2025 Wisconsin Act 15 (up to $24,000 of retirement-plan/IRA/pension income for filers age 67+, effective tax year 2025): the trace subtracts exactly the $13,655.95 of taxable Social Security and nothing more, so the 79-year-old head's $24,086 of taxable IRA and pension income stays in WI AGI. Separately, the engine's non-Social-Security AGI ($29,981.71) is about $3,824 below the listed income items, which suggests the $3,753 of non-Schedule-D capital gain distributions was left out of federal AGI.","PolicyEngine subtracts exactly the $13,655.95 of taxable Social Security from $43,637.65 of federal AGI. That leaves $29,981.71 of WI AGI, with all $24,086 of IRA and pension income still taxed. It then applies the single-filer sliding-scale standard deduction of $12,671.36 and $950 of exemptions, taxes $16,360.34 at 3.5%/4.4% for $584.74, and subtracts the $300 school property tax credit to reach $284.74. One group of models zeroed the tax with a $24,000 age-67+ retirement income exclusion or a blanket pension exclusion, which the engine does not apply. The other group kept retirement income taxable but made one or more other errors: they inflated WI AGI with the $3,753 of capital gain distributions, mis-sized or skipped the standard deduction and exemptions, treated the surviving spouse as a joint filer, used wrong bracket rates, or left out or over-sized the $300 school property tax credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 284.74; the corrected value is 0.00. PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)." us,scenario_042,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"The Wisconsin homestead credit requires qualifying household income and property-tax circumstances to be computed from the stated annual inputs under the 2026 rules. The household receives substantial Social Security, IRA, pension, dividend, interest, and capital-gain income, so reducing household income to roughly $14,420 and applying a maximum-credit shortcut creates a refundable credit that the full eligibility computation does not allow." us,scenario_043,federal_refundable_credits,4,llm_error,age_disability,False,,"The single trap is the upper age limit on the childless EITC: IRC §32(c)(1)(A)(ii)(II) allows the no-qualifying-children credit only to filers at least 25 and under 65 at the close of the tax year, and the American Rescue Plan's one-year suspension of that ceiling applied to 2021 only. The head is 66 in tax year 2026, so the EITC is $0 regardless of the $4,088 in earnings sitting squarely in the phase-in range, and with no children, no education expenses, and no Recovery Rebate or refundable payroll tax credit, total federal refundable credits are $0. All four models jumped straight to the phase-in arithmetic on earned income and never applied the age test, three of them landing on 7.65% × ~$4,088 ≈ $312–313 and one on a fabricated 15.3% rate." us,scenario_043,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -368,18 +370,18 @@ us,scenario_043,head_medicaid_eligible,9,llm_error,categorical_eligibility,False us,scenario_043,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_043,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_043,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_043,payroll_tax,29,llm_error,state_local_rule,False,,"The separating step is Colorado's Family and Medical Leave Insurance (FAMLI) premium, a mandatory employee-side state payroll tax that belongs in this output. For 2026 the total FAMLI premium is 0.88% of wages split evenly between employer and employee, so the employee pays 0.44% — $17.99 on the $4,088.15 wage base — on top of employee Social Security at 6.2% ($253.47) and employee Medicare at 1.45% ($59.28), for $330.73. Twenty-five of the 29 models stopped at the 7.65% federal FICA sum of $312.73 (or rounded it to $313/$312), asserting Colorado imposes no employee payroll tax; the remaining four either applied a wrong FAMLI rate, submitted a number matching neither of their own computations, or zeroed the FICA wage base outright." +us,scenario_043,payroll_tax,30,llm_error,state_local_rule,False,,"The requested output covers mandatory employee state payroll taxes. Colorado's FAMLI program charges employees half of a 0.88% premium in 2026, or 0.44% of wages, which adds $17.99 on $4,088.15 of wages to the $312.75 of FICA. Almost every model stopped at 7.65% FICA ($312.73) and either said Colorado has no employee payroll tax or dropped FAMLI on purpose. The outliers made other mistakes: one subtracted ESI premiums from the FICA base, one used a 0.6% FAMLI rate, one submitted a value that did not match its own arithmetic, and one returned zero with no calculation." us,scenario_043,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_043,snap,39,llm_error,thresholds_rates,False,,"Every model correctly saw that the 20% earned-income deduction, the standard deduction, and the elderly excess-medical deduction (medical costs above $35/month) drive net income to zero, so the benefit is the full one-person maximum allotment — the trap is the allotment parameter itself and its mid-year reset. Calendar year 2026 spans two SNAP fiscal years: $298/month for January–September 2026 (FY2026) and the uprated $304.68/month for October–December 2026 (FY2027), giving 9 × $298 + 3 × $304.68 = $3,596.04. The largest cluster used the stale FY2025 rate of $292 ($3,504) or the FY2024 rate of $291 ($3,492); the next-largest got $298 right but multiplied it flat across all 12 months ($3,576), missing the October 1 uprating worth exactly $20.04. A minority answered $0 by applying the non-elderly $2,250–$3,000 resource limit or asserting ineligibility outright, when the head is 66, the elderly/disabled resource limit applies, and Colorado's broad-based categorical eligibility through TANF non-cash assistance waives the asset test entirely." +us,scenario_043,snap,32,llm_error,thresholds_rates,False,,"This one-person Colorado household is eligible and gets the maximum benefit. Its 66-year-old head counts as elderly, so the excess medical deduction, the standard deduction and the 20% earned-income deduction bring net income to $0. The $2,800 in assets is under the elderly limit, and the household is also categorically eligible through Colorado's TANF non-cash eligibility test. The benefit is the 2026 one-person maximum allotment for the contiguous states, $298 a month, or $3,576 a year. Most models got the eligibility right and then used a stale or guessed maximum ($291 for FY2024, $292 for FY2025, or $299-$328). A smaller group wrongly said the household fails the asset or income tests and answered $0." us,scenario_043,ssi,4,llm_error,period_annualization,False,,"SSI eligibility is determined for the individual, not by subtracting annualized countable income from twelve times a monthly federal benefit rate. The head’s approximately $4,642 of individual income exceeds the applicable SSI income limit, so the income-eligibility gate fails and no SSI is payable despite aged categorical eligibility and assumed take-up." -us,scenario_043,state_refundable_credits,39,llm_error,state_local_rule,False,,"The entire $19 comes from Colorado's TABOR state sales tax refund (C.R.S. 39-22-2003), a refundable credit claimed on the Colorado individual return by every full-year resident age 18+ who files, with a six-tier amount schedule keyed only to AGI and filing status — no earned-income floor, no qualifying child, and no dependence on the federal EITC. This single filer's $4,625.40 MAGI lands in the bottom tier, which pays $19 under the 2026 schedule. Nearly every model enumerated only federally-derived Colorado credits (CO EITC as a percentage of federal EITC, CO CTC, child care) plus the property-tax/rent/heat PTC rebate, correctly zeroed those, and stopped — never reaching the sales tax refund. The handful that did reach it either recast it as a senior-specific credit or substituted surplus-year Colorado Cash Back amounts ($800–$1,200+) for the 2026 bottom-tier $19." +us,scenario_043,state_refundable_credits,46,llm_error,state_local_rule,False,,"The Colorado sales tax refund (the TABOR refund) is the only credit that pays anything here. PolicyEngine counts it as a refundable state income tax credit, and this 66-year-old full-year resident filer qualifies. At modified AGI of $4,625.40 it pays $19 for 2026. Every other Colorado refundable credit is zero: the head has no children, and at 66 is too old for the federal childless EITC, so the Colorado EITC is also zero. Most models checked only the EITC and child credits and never included the sales tax refund. A few invented senior or EITC amounts, or took the TABOR amount from the wrong tier." us,scenario_043,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_044,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Social Security benefits are not treated as fully taxable income: the taxable portion is determined from provisional income, which includes only half of Social Security when there is no other income. The resulting taxable Social Security is entirely absorbed by the 2026 married-filing-jointly standard deduction and applicable age-based deductions, leaving zero taxable income and zero federal income tax before refundable credits." +us,scenario_044,federal_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"The couple's only income is $56,806 of Social Security. Their provisional income is 50% of benefits plus other income: 0.5 × $56,806 + $0 = $28,403. That is below the $32,000 married-filing-jointly base amount under IRC §86, so none of the benefits are taxable. AGI and taxable income are $0, and so is regular income tax. Every wrong model skipped this test. They treated the benefits as taxable in full, assumed the 85% maximum inclusion applied automatically, or wrongly said the threshold was exceeded. Each then put a positive tax on income that is excluded by statute." us,scenario_044,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered earned compensation, not Social Security retirement benefits. Because the household has no reported wages or self-employment income, every employee payroll-tax component has a zero base and payroll tax is $0." us,scenario_044,spouse_medicare_eligible,1,parse_contract_failure,other,False,,"The spouse is age 61 and has no listed disability or other Medicare-qualifying status, so the spouse is not Medicare eligible in 2026. The model’s explanation correctly derives 0, but its submitted numeric value is 1, creating an internally inconsistent contract output." us,scenario_044,state_income_tax_before_refundable_credits,3,llm_error,taxable_income_or_deductions,False,,"Kansas subtracts federally taxable Social Security benefits from Kansas adjusted gross income, so the household’s only reported income does not enter the Kansas income-tax base. With no remaining Kansas taxable income, state income tax before refundable credits is $0; applying Kansas deductions, exemptions, brackets, or credits to the $56,806 benefit is the wrong computation path." -us,scenario_044,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"The Kansas food sales tax credit is nonrefundable, so it is not included in the requested state_refundable_credits output. Eligibility for that credit therefore does not produce a refundable state credit, and the refundable-credit total remains $0." -us,scenario_045,federal_income_tax_before_refundable_credits,32,llm_error,taxable_income_or_deductions,False,,"This case turns on three 2026-specific facts: PolicyEngine's employment income of $36,275.59 is the taxable wage figure, and the listed $21,208 employer-sponsored premium is a health-coverage input that does not reduce it; OBBBA carries the TCJA structure into 2026 with a $16,100 single standard deduction and no personal exemption; and the 2026 single 10% bracket runs to $12,400, with 12% above it. That yields $20,175.59 of taxable income and $1,240 + 0.12 × $7,775.59 = $2,173.07. Eleven models netted the ESI premium out of wages, collapsing AGI to $15,068 and producing zeros or two-digit answers; ten more used either post-sunset pre-TCJA parameters (an ~$8,300 standard deduction plus a ~$5,200 personal exemption and a 15% second bracket) or the 2025 $15,000/$15,750 deduction and $11,925 bracket top. A third cluster derived $2,173 correctly in its own reasoning and then submitted an unrelated number." +us,scenario_044,state_refundable_credits,3,llm_error,state_local_rule,False,,"All three models awarded the Kansas food sales tax credit of $125 per exemption (2 x $125 = $250). Kansas cut its state sales tax on food to 0% on January 1, 2025, and ended that refundable credit after tax year 2024, so it does not exist in tax year 2026. Kansas's other refundable credit, the state EITC (17% of the federal EITC), is also $0 because the household has no earned income: its only income is Social Security. That leaves Kansas refundable credits, and state_refundable_credits, at $0." +us,scenario_045,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"The reference is a plain single-filer computation. AGI equals the stated gross wages of $36,275.59. The $21,208 of employer-sponsored insurance premiums is not subtracted from the listed gross wages and is not an itemized medical expense, so itemized deductions are only $1,541.71. The 2026 standard deduction under the permanent TCJA/OBBBA schedule is $16,100, which leaves $20,175.59 of taxable income. At 10% up to $12,400 and 12% above, that gives $1,240 + $933.07 = $2,173.07, and no nonrefundable credits apply. The wrong answers fall into three groups. Some subtracted the ESI premiums from wages, which drives taxable income to near zero. Some used a stale or sunset-era 2026 schedule: 2025 amounts ($15,000–$15,750 and $11,925), or pre-TCJA personal exemptions with a 15% bracket. The rest derived $2,173 or something close and then submitted an unrelated number." us,scenario_045,federal_refundable_credits,10,llm_error,credit_phaseout,False,,"The full $36,276 of annual gross wages enters the childless EITC income test; the separately listed employer-sponsored insurance premiums do not reduce earned income or AGI because no pre-tax payroll treatment is stated. At that income, the childless EITC is fully phased out, and the household has no qualifying child or other basis for a refundable federal credit." us,scenario_045,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_045,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -387,25 +389,27 @@ us,scenario_045,head_medicaid_eligible,8,llm_error,taxable_income_or_deductions, us,scenario_045,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_045,payroll_tax,10,llm_error,payroll_tax_base,False,,"The trap is the FICA wage base: seven of the ten models treated the $21,208 employer-sponsored insurance premium as a Section 125 pre-tax salary reduction and shrank the base from $36,276 to $15,068, collapsing on $1,152.70 (7.65% of the reduced figure). PolicyEngine computes employee payroll tax on employment income of $36,276 with no cafeteria-plan exclusion, and the listed ESI amount is the total premium including the employer share, which was never employee wages in the first place. The correct components are 6.2% Social Security ($2,249.09, far below the 2026 wage base cap) and 1.45% Medicare ($526.00), with no Additional Medicare Tax and no Michigan employee payroll tax, totaling $2,775.08. The three remaining misses used the right base and rates but failed at arithmetic, at answer transcription, or at emitting the key at all." us,scenario_045,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_045,snap,39,llm_error,categorical_eligibility,False,,"This household reaches SNAP not through a gross income screen but through categorical eligibility from TANF non-cash assistance, which removes the gross test and leaves the net income test controlling: the 20% earned income deduction, the standard deduction, the $433.33/month legally obligated child support deduction, and an excess shelter deduction built on $880 rent plus Michigan's standard utility allowance bring monthly net income to $1,102.06, under the $1,304.17 (100% FPG) net limit, with $800 in assets under the resource limit. Because 30% of net income ($330.60) exceeds the $298 one-person maximum allotment, the max-minus-contribution formula returns a negative number, and the statutory minimum allotment for one- and two-person households (8% of the one-person max) governs: $23.84/month under FY2026 parameters for January–September and $24.37/month under FY2027 parameters for October–December, summing to $287.68 for calendar 2026. Thirty-plus models stopped at a 130% or 200% FPL gross income screen and declared ineligibility; the few that reached the benefit formula computed a negative allotment and reported $0 instead of applying the minimum benefit floor." +us,scenario_045,snap,4,llm_error,categorical_eligibility,False,,"This one-person household has no elderly or disabled member, so it must either pass the 130% FPL gross income test or qualify through Michigan's broad-based categorical eligibility (BBCE), which has a 200% FPL gross limit. Michigan counts child support paid in gross income and deducts it only when computing net income (7 CFR 273.9(d)(5)). That leaves gross income at the full $3,022.97 a month, about 232% of the guideline, which fails both limits, so SNAP is $0 for the year. The wrong answers came from three errors: taking child support or health premiums out of gross income, still paying a benefit after finding the household over the limit, and a submitted value that contradicts the model's own finding that the household is ineligible." us,scenario_045,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_045,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Michigan's 2026 tax on this filer is mechanical: state AGI equals the full $36,275.59 of wages, one personal exemption of $5,950 comes off, and the flat 4.25% rate applies to the remaining $30,325.59, giving $1,288.84 with no nonrefundable credit available. Three moves separate the reference from the wrong answers. Fourteen models got the structure exactly right but plugged in an exemption of $5,000–$6,000 instead of $5,950; seven subtracted the $21,208 employer-sponsored insurance premium from wages to reach a $15,068 AGI, double-counting an exclusion already embedded in the wage figure. The rest either imported a federal standard deduction or a child-support deduction into Michigan's base, or applied the refundable homestead property tax credit — which is excluded from a before-refundable-credits output — to drive the liability toward zero." -us,scenario_045,state_refundable_credits,36,llm_error,state_local_rule,False,,"The entire $760.79 is the Michigan Homestead Property Tax Credit claimed by a renter on MI-1040CR: 23% of $10,560 gross rent = $2,428.80 of deemed property tax, less 3.2% of total household resources ($36,275.59 × 0.032 = $1,160.82), leaving $1,267.98, of which a non-senior, non-disabled claimant receives 60% = $760.79. The Michigan EITC contributes nothing because the childless federal EITC is fully phased out at $36,276 of wages. Two traps separate the reference from the wrong answers: total household resources are the full wages — neither the $5,200 of child support paid nor the $21,208 employer-sponsored insurance premium reduces THR — and the current parameters are 23% of rent with a 3.2% offset and a 60% rate, not the superseded 20%/3.5%/17%/80% figures. Models split between omitting the renter credit entirely and answering $0, and computing it off a wrong rent conversion, a wrong resource offset, or a shrunken THR base." +us,scenario_045,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Michigan tax for this filer is the flat 4.25% rate applied to AGI equal to full wages ($36,275.59) less the 2026 personal exemption of $5,900, which gives $1,290.96. Michigan has no standard deduction for a 44-year-old, and no nonrefundable credit applies here. Most models used the right structure but the wrong exemption: $5,000, $5,200, $5,600, $5,800, $6,000 or $7,350 in place of $5,900. A second group subtracted the $21,208 employer-sponsored insurance premium input from wages, which cut AGI to $15,068. The rest invented a standard deduction, used a 4.05% rate, treated the refundable Homestead Property Tax Credit as nonrefundable, or submitted a number different from their own computation." +us,scenario_045,state_refundable_credits,42,llm_error,state_local_rule,False,,"The whole $760.79 is Michigan's refundable Homestead Property Tax Credit for a renter. The steps are: take 23% of the $10,560 rent as property tax ($2,428.80), subtract 3.2% of total household resources of about $36,276 ($1,160.82), and multiply the $1,267.98 excess by 60%. Federal EITC, and so Michigan EITC, is $0 at this income. About half of the models either left out the homestead credit (they checked only the EITC, or treated renters or the credit as ineligible or nonrefundable) or computed it incorrectly. The errors were using 20%, 17% or no rent share instead of 23%, using a 3.5% threshold, using 80% or no 60% multiplier, subtracting the $21,208 ESI premiums or the $5,200 child support from household resources, or adding a Michigan EITC that does not exist here." +us,scenario_046,child1_chip_eligible,1,llm_error,state_local_rule,False,,"Oklahoma covers school-age children through Medicaid (SoonerCare), not through a separate CHIP. The engine therefore gives a 10-year-old in Oklahoma no CHIP pathway on age grounds. The household's MAGI of about $92,003 (wages plus interest) is also far above any Oklahoma children's income limit for a family of four. A wrong Yes comes from applying a generic, high CHIP income limit and understating MAGI, instead of Oklahoma's own CHIP rules for this child's age and income." us,scenario_046,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A dependent child qualifies for Oklahoma Medicaid only through an applicable eligibility category. At age 10 and MAGI income of 2.79 times FPL, child1 exceeds the income limits for the regular children's group, CHIP, and every other applicable pathway, so the Medicaid category is NONE and eligibility is No." +us,scenario_046,child2_chip_eligible,1,llm_error,taxable_income_or_deductions,False,,"Child2's CHIP eligibility depends on the family's Medicaid/CHIP MAGI measured against the federal poverty line for a family of four. That MAGI is the full $92,003 of wages plus interest, which is about 280% FPL. The $14,839 FLSA overtime premium is already inside the $92,000 of wages. The 2025-2028 overtime deduction is taken below the line, in computing taxable income, so it never lowers AGI or MAGI. Once the overtime premium is subtracted from MAGI, income falls into the CHIP band. Kept in, income is above Oklahoma's CHIP ceiling for a 9-year-old, and child2 is ineligible because Medicaid is also unavailable." us,scenario_046,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Oklahoma Medicaid eligibility requires the child to qualify through an available categorical pathway and its applicable income limit. Child 2 has no qualifying Medicaid category, receives no SSI, and the household's MAGI income of 2.79 times FPL exceeds every applicable child Medicaid threshold, so eligibility is No." -us,scenario_046,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"The case turns on two 2026 OBBBA provisions. The $14,838.71 FLSA overtime premium generates a qualified-overtime deduction that stacks on top of the $32,200 MFJ standard deduction (the $25,000 joint cap leaves the whole premium deductible), cutting taxable income from $59,803 to $44,964.29 and tax to $4,899.71; the CTC is $2,200 per child, and the full $4,400 applies as a nonrefundable credit against that liability, leaving $499.71. Most models omitted the overtime deduction entirely, and several layered on a TCJA sunset that does not occur — personal exemptions and a 15% bracket — or capped the CTC at $2,000 per child. A second cluster inflated itemized deductions with health-insurance premiums and over-the-counter expenses past the $32,200 standard deduction, or carved the $1,700 refundable ceiling out of the nonrefundable CTC." -us,scenario_046,federal_refundable_credits,13,llm_error,categorical_eligibility,False,,"Two gates close every refundable channel here. The American Opportunity Credit is computed on qualified tuition and related expenses, and the household lists none (the prompt directs that any unlisted numeric input be treated as 0), so the student-status flags alone produce a $0 credit and a $0 refundable 40% portion. The Child Tax Credit ($2,200 × 2 = $4,400 in 2026) is entirely absorbed as a nonrefundable credit because federal tax before credits exceeds $4,400 — $92,003 AGI less the ~$32,200 MFJ standard deduction and the $14,839 FLSA overtime deduction leaves roughly $45,000 of taxable income and about $4,900 of tax — and §24(d) reduces the refundable ACTC by the nonrefundable CTC actually allowed, leaving $0. EITC is zero at $92,000 of earnings with two children, so the total is $0." +us,scenario_046,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"To get the right answer, a model had to apply four 2026 rules together. The OBBBA qualified overtime deduction ($14,838.71 here, well under the $25,000 joint cap) is subtracted on top of the $32,200 MFJ standard deduction, giving taxable income of $44,964.29. That income is taxed under the permanent post-TCJA brackets (10% up to $24,800, 12% above), for $4,899.71. The full $2,200-per-child CTC, $4,400 in total, is then taken as a nonrefundable credit against that tax, leaving $499.71. Most wrong answers left out the overtime deduction, assumed the TCJA had sunset (personal exemptions, a 15% bracket, a $1,000 CTC), used the old $2,000 CTC, itemized by counting health insurance premiums as deductible medical expenses, or applied an AOTC even though no qualified tuition was listed." +us,scenario_046,federal_refundable_credits,14,llm_error,categorical_eligibility,False,,"None of the refundable credits pays out, so federal refundable credits are $0. The EITC is fully phased out at $92,003 AGI for a two-child joint return. The American Opportunity Credit needs qualified tuition and related expenses, and none are listed, so both its nonrefundable and refundable parts are $0. The $4,400 CTC (2 x $2,200) is fully used against tax owed before credits. Even after the $32,200 standard deduction and the $14,839 overtime deduction, that tax is about $4,900, and the refundable CTC only pays back CTC left over after tax, so it is $0. The wrong answers fall into three groups: some made up tuition to claim a $1,000 refundable AOTC, some paid the $1,700-per-child refundable cap without first applying the CTC to tax, and some put tax below $4,400 so a small slice of CTC looked refundable." us,scenario_046,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_046,payroll_tax,16,llm_error,payroll_tax_base,False,,"Employee Social Security tax is 6.2% of the head's $92,000 wages, yielding $5,704, and employee Medicare tax is 1.45%, yielding $1,334. The listed employer-sponsored insurance premium and FLSA overtime premium do not alter the $92,000 wage base here: gross wages already include overtime, and the facts do not establish a pre-tax payroll exclusion for the insurance premium. With no Additional Medicare Tax or mandatory Oklahoma employee payroll tax, total payroll tax is $7,038." us,scenario_046,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_046,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Oklahoma starts from federal AGI of $92,003, removes four $1,000 personal exemptions and its own $12,700 MFJ standard deduction — not the ~$32,000 federal standard deduction, and not an uncapped federal itemized total, since Oklahoma caps the non-medical portion of itemized deductions at $17,000 (holding the state itemized figure to $22,429.78, which includes $5,429.78 of medical expense above the 7.5%-of-AGI floor) — for taxable income of $75,303. The 2026 Oklahoma graduated schedule on that base produces $2,959.14, materially below the $3,221.89 the pre-2026 six-bracket schedule topping at 4.75% yields. The final step is Oklahoma's nonrefundable child care/child tax credit, the greater of 20% of the federal CDCC or 5% of the federal CTC: 5% of $4,400 (two children at the $2,200 per-child amount) is $220, allowed because federal AGI is under $100,000, leaving $2,739.14. Every wrong answer breaks at least one of these three links: Oklahoma's own deduction and exemption amounts, the 2026 rate schedule, or the $220 state credit." +us,scenario_046,state_income_tax_before_refundable_credits,45,llm_error,thresholds_rates,False,,"Oklahoma lets a household itemize only if it also itemizes on its federal return. Here the $32,200 federal standard deduction is larger than the roughly $26,309 of federal itemized deductions, so Oklahoma's $12,700 joint standard deduction applies. With four $1,000 exemptions, Oklahoma taxable income is $75,303. The 2026 Oklahoma joint rate schedule is 0% up to $7,500, 2.5% up to $9,800, 3.5% up to $14,400, and 4.5% above that, which gives $2,959.14 of tax. The nonrefundable Oklahoma child credit is 5% of the $4,400 federal CTC ($2,200 per child), or $220, leaving $2,739.14. Most models failed in one of four ways: they used the pre-2026 brackets (0.25% to 4.75%, or kept the old lower brackets with a 4.5% top rate), itemized on the Oklahoma return even though the household takes the federal standard deduction, used the federal standard deduction or the federal overtime deduction to shrink Oklahoma income, or left out or understated the 5%-of-CTC credit." us,scenario_046,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_048,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Age 85 establishes membership in an aged population but does not itself confer Medicaid eligibility. The head receives no SSI, is not a dependent, and qualifies through none of the Medicaid pathways; the 2.17-times-FPL MAGI level also does not establish eligibility in a MAGI category. Low assets cannot substitute for satisfying a pathway's income and categorical requirements." us,scenario_048,head_medicare_eligible,1,llm_error,other,False,,"Medicare eligibility is satisfied because the head is age 85, well above the age-65 threshold. The model’s prose reached that result but its submitted binary value reversed it: eligible must be encoded as 1, not 0." us,scenario_048,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies only to the $7,656 of wages: 6.2% employee Social Security tax is $474.67 and 1.45% employee Medicare tax is $111.01, totaling $585.68. Ohio adds no mandatory employee state payroll tax to this output, and the annual result is not rounded to a whole dollar." us,scenario_048,snap,5,llm_error,thresholds_rates,False,,"The household’s $34,626 annual income exceeds the applicable $17,420 SNAP eligibility threshold for a one-person Ohio household in 2026, so SNAP eligibility ends before any minimum-allotment calculation. A minimum benefit applies only to an otherwise eligible household; it cannot override a failed income test." us,scenario_048,state_income_tax_before_refundable_credits,2,llm_error,thresholds_rates,False,,"The trap is Ohio's zero-tax bracket. Ohio's individual income tax schedule imposes no tax on Ohio taxable nonbusiness income at or below roughly $26,050, so this filer's $7,656 in wages — the only Ohio-taxable income, since the $26,970 in Social Security survivor benefits stays out of federal AGI at a provisional income of $21,141, below the $25,000 base amount — produces $0 of liability before any credit. Both models correctly excluded the Social Security benefits and then applied the 2.75% rate starting at the first dollar of taxable income, converting a below-threshold filer into a taxpayer. Their differing answers ($106 vs. $94.54) come only from downstream inventions — a fictitious Ohio standard deduction in one case, a recharacterized exemption credit plus the $50 senior citizen credit in the other — layered on the same missing zero bracket." -us,scenario_049,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on the 2026 OBBBA deduction stack: a $32,200 MFJ standard deduction, the $25,000 cap on the qualified-overtime deduction against the $44,647 FLSA overtime premium (no phase-out below $300,000 of joint MAGI), and the $579.77 non-itemizer cash charitable deduction — $57,779.77 in all against AGI of $246,663.72, leaving taxable income of $188,883.95. Most wrong models omitted the overtime deduction entirely, and a large cluster priced 2026 as a TCJA-sunset year with personal exemptions and pre-TCJA 10/15/25/28% brackets, when the permanent structure is 10/12/22% with no exemptions. A second cluster stripped the $16,408 of employer-sponsored insurance premiums out of the $253,000 of wages, which the reference's $237,564 of employment income does not do. The two near-misses got the whole OBBBA stack right and failed only by denying the $721 deductible traditional IRA contribution, worth $158.63 at the 22% rate." +us,scenario_049,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 OBBBA law. It starts from AGI of $246,663.72: wages net of the traditional 401(k), $3,501 interest and $6,320 dividends, minus the $721 traditional IRA contribution taken above the line. It then subtracts the $32,200 joint standard deduction, the $25,000-capped qualified overtime deduction (from the spouse's $44,647 FLSA premium) and the roughly $580 non-itemizer cash charitable deduction, giving taxable income of $188,883.95. Ordinary income is taxed through the 10/12/22% brackets and the $6,208 of qualified dividends at 15%. Most models failed for one of three reasons: they left out the $25,000 overtime deduction, they applied TCJA-sunset law (personal exemptions and 15/25/28% brackets), or they subtracted the $16,408 of employer-paid premiums from wages that never included them. The closest group disallowed the $721 IRA deduction under the active-participant phaseout, as the exclusion's corrected value ($30,702.59) does under 26 U.S.C. 219(g) and the frozen reference, which has no active-participant phase-out, does not; some also dropped the $580 non-itemizer charitable deduction or claimed auto-loan interest, which the reference does not allow. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 30,543.91; the corrected value is 30,702.59. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_049,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -413,7 +417,7 @@ us,scenario_049,head_medicaid_eligible,1,parse_contract_failure,missing_output,F us,scenario_049,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or a qualifying early-eligibility condition such as disability, end-stage renal disease, or ALS. The 53-year-old head has none of those conditions, so the eligibility result is No." us,scenario_049,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_049,payroll_tax,30,llm_error,payroll_tax_base,False,,"The case turns on two facts: the 2026 Social Security wage base is $184,500, so the employee share is 6.2% × $184,500 = $11,439; and Medicare tax applies to the full $253,000 of gross wages, which keeps wages $3,000 above the $250,000 joint Additional Medicare Tax threshold for the $27 surtax. Nearly every wrong answer fails on one of two traps: substituting a prior-year or guessed cap ($168,600, $174,900, $176,100, $180,900, $181,200, $183,000, $183,600, $183,900, $184,200), or netting the $16,408 employer-sponsored insurance premium out of FICA wages to $236,592, which both drops $237.92 of Medicare tax and erases the $27 surtax by pushing wages under $250,000. A second cluster ignored the wage cap entirely (7.65% on all $253,000), and several models — claude-fable-5, claude-opus-4.8, qwen-3.7-max, qwen3.8-max, gemini-3.1-flash-lite-preview — submitted values that contradict their own arithmetic, two of them after deriving $15,134.50 exactly." +us,scenario_049,payroll_tax,31,llm_error,payroll_tax_base,False,,"The reference total of $15,134.50 has three parts. Social Security tax is 6.2% of the 2026 wage base of $184,500, which is $11,439. Medicare tax is 1.45% of the spouse's full $253,000 gross wages, which is $3,668.50. Additional Medicare Tax is 0.9% of the $3,000 above the $250,000 married-filing-jointly threshold, which is $27. Most models fell into one of two traps. About half used a stale or guessed wage base, such as the 2025 figure of $176,100 or estimates from $174,900 to $183,900. About as many subtracted the $16,408 employer-sponsored insurance premiums from the listed gross wages when computing FICA wages. That cut Medicare tax to $3,430.58 and pushed wages under the $250,000 threshold, so the Additional Medicare Tax disappeared. The prompt gives gross wages and does not say the premiums are paid pre-tax." us,scenario_049,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -422,27 +426,27 @@ us,scenario_049,spouse_medicaid_eligible,1,parse_contract_failure,missing_output us,scenario_049,spouse_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility requires a qualifying pathway such as reaching age 65 or having a qualifying disability, ESRD, or ALS status. The spouse is 52 and every unlisted status is false, so no Medicare eligibility pathway applies and the binary output is 0." us,scenario_049,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_049,state_income_tax_before_refundable_credits,8,llm_error,state_local_rule,False,,"New Hampshire levies no tax on wages, so the entire question turns on its Interest & Dividends tax, and that tax carries no liability for this household in tax year 2026 — it was repealed for taxable periods beginning after December 31, 2024, superseding the earlier phase-down calendar that had penciled in 1% for 2026. Seven of the eight wrong models reconstructed the reference's own arithmetic — $3,501 interest plus $6,320 dividends equals $9,821, less the $4,800 joint exemption equals $5,021 — and then multiplied that base by a rate that no longer applies (1% for five of them, the retired 2024 rate of 3% for one, and an unanchored broad-based wage tax for gpt-5.4-nano). The separating step is purely the 2026 rate/assessment, not the base: every model that got $5,021 was one factor away from the reference's $0. One model returned nothing at all." +us,scenario_049,state_income_tax_before_refundable_credits,9,llm_error,thresholds_rates,False,,"New Hampshire does not tax wages. Its only personal income levy, the Interest and Dividends Tax, produces no liability on this household in 2026. The trap is the rate schedule. The 2021 law cut the rate one point a year and would have left it at 1% in 2026. The 2023 law (HB 2) replaced that schedule and ended the tax for taxable periods beginning after December 31, 2024. PolicyEngine computes the same $5,021 base the models used ($9,821 of interest and dividends minus the $4,800 joint exemption) and assesses $0. Almost every wrong model found that base correctly and then applied a superseded nonzero rate: 1%, or in one case 3%." us,scenario_049,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_049,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_051,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"The 2026 parameters are the OBBBA-permanent ones: a $16,100 single standard deduction, a 10% bracket running to $12,400, and 12% above it. $40,000 of wages less $16,100 leaves $23,900 of ordinary taxable income, and $1,240 + 12% × $11,500 = $2,620, with the $180 of non-Schedule-D capital gain distributions adding nothing to the liability. The wrong answers fall into four groups: models applying a TCJA-expired schedule (an ~$8,300 standard deduction plus a personal exemption and a 15% second bracket), models carrying stale 2025 parameters (a $15,000–$15,750 standard deduction and an $11,925 bracket edge), models that reached ~$2,642 correctly and then submitted an unrelated number, and models that zeroed the liability outright. A secondary trap adds $21.60 for models that push the $180 of capital gain distributions into the 12% ordinary bracket." +us,scenario_051,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The reference applies the 2026 post-OBBBA single parameters: a $16,100 standard deduction against $40,000 of wages gives $23,900 of taxable income. The tax is 10% of the first $12,400 ($1,240) plus 12% of the remaining $11,500 ($1,380), for $2,620. The $180 of non-Schedule D capital gains adds no tax, because capital gain distributions are long-term gains that fall in the 0% bracket at this income. The biggest cluster (about eight models, at or near $2,641.60) taxed that $180 at 12%. Most of the rest used stale 2024/2025 parameters, assumed the TCJA had expired (bringing back personal exemptions and a 15% bracket), subtracted the $5,389 employer premium from wages, or submitted numbers that do not follow from their own arithmetic." us,scenario_051,federal_refundable_credits,4,llm_error,thresholds_rates,False,,"A childless single filer with approximately $40,000 of earnings is above the 2026 EITC phaseout endpoint, so the EITC is $0. With no qualifying children and no facts supporting another refundable federal credit, all five refundable-credit components are $0." us,scenario_051,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Louisiana Medicaid eligibility requires qualification through a specific categorical pathway, not merely acceptable immigration status or the absence of an expressly listed disqualification. This 25-year-old, nondependent adult has MAGI income of 2.51 times FPL, receives no SSI, and falls into Medicaid category NONE, so no MAGI or alternative state-program pathway confers eligibility." us,scenario_051,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $40,000 of wages here, producing $2,480 and $580 respectively, for a $3,060 total. The dominant error was subtracting the listed employer-sponsored insurance premium without any fact establishing a pre-tax salary-reduction arrangement; several other models derived $3,060 correctly but then submitted a different number." us,scenario_051,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_051,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The trap is Louisiana's post-reform flat tax as it stands in 2026: a single 3% rate applied to the $40,000 wage base less a standard deduction that is indexed annually and equals $12,835 for 2026, giving $27,165 x 3% = $814.95 with no nonrefundable credits. Most models recalled the 2024 reform correctly but froze the standard deduction at its un-indexed statutory level of $12,500 and pulled the $180 of non-Schedule-D capital gains into the state base, inflating taxable income by exactly $515 and landing on $830.40. A second cluster reconstructed the repealed pre-2025 regime — 1.85%/3.5% graduated brackets, a $4,500 combined standard deduction/personal exemption, and the deduction for federal income taxes paid — often compounding it by subtracting the $5,389 employer insurance premium from wages already reported net of Section 125 amounts. A third cluster derived one number in its explanation and submitted a different, unsupported one." +us,scenario_051,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Louisiana's post-2024 reform (Act 11, 2024 Third Extraordinary Session) applies a flat 3% rate to federal AGI minus a single standard deduction. That deduction was $12,500 for 2025 and is inflation-indexed starting in 2026, which gives $12,835 for the 2026 tax year. On $40,180 of AGI, that leaves $27,345 taxable and $820.35 of tax. The largest group of models knew the flat 3% structure but used the unindexed $12,500 deduction and got $830.40. The other models used the repealed pre-2025 law: 1.85%/3.5% brackets, a $4,500 combined exemption, or a deduction for federal income tax. Some of those also subtracted ESI premiums from AGI or claimed Louisiana has no income tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)." us,scenario_051,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_052,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The reference is a plain married-filing-jointly return whose precision rests on four steps a model had to get all of: wages reduced only by the $926.16 pre-tax 401(k) and not by the $16,208 employer-sponsored insurance premium; above-the-line deductions of $3,043.28 combining the $3,000 net-capital-loss limit with the deductible $43.28 traditional IRA contribution; the $32,200 MFJ standard deduction for 2026 with no personal exemptions; and then $127.20 of 15% qualified-dividend tax plus $218.42 of NIIT on $5,748 of net investment income (interest and dividends net of the $3,000 loss), with the $2,250 Additional Medicare Tax excluded as payroll rather than income tax. The wrong answers split into two camps. One camp priced 2026 as a TCJA-sunset year — a $15,000-$17,000 standard deduction, revived personal exemptions, and the pre-TCJA 28/33/35% rate schedule — and landed between $117,000 and $135,000; the other got the structure right but shaved the IRA deduction, dropped the NIIT, mispriced the dividend at 0% or 20%, or stripped the ESI premium out of wages, landing within a few hundred to five thousand dollars. A tail of models submitted values that contradict their own bracket walks outright." +us,scenario_052,federal_income_tax_before_refundable_credits,45,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine takes the head's $43.28 traditional IRA contribution as an above-the-line deduction (AGI $504,778.56 instead of $504,821.84) without applying the IRC §219(g) active-participant phase-out (2026 MFJ range $129,000–$149,000). The head's elective 401(k) deferrals make them an active participant, and MAGI is about $504,800, so the deduction should be $0 and the value about $104,225.25.","The reference builds AGI of $504,778.56 from wages net of the 401(k) deferral ($499,073.84), $7,900 interest, $848 qualified dividends, the $3,000 capital-loss limit and a $43.28 traditional IRA deduction. It then subtracts the OBBBA 2026 MFJ standard deduction of $32,200 and taxes the $471,730.56 of ordinary income on the 2026 brackets (24% to $403,550, then 32%) for $103,865.78. To that it adds $127.20 on the qualified dividends at 15% and $218.42 of NIIT on $5,748 of net investment income. The closest models (104,225.30) match every step except the IRA deduction. Most other models went wrong for their own reasons: they assumed the TCJA sunset (pre-TCJA rates, exemptions), subtracted ESI premiums from wages, used outdated standard deductions or brackets, folded the Additional Medicare Tax into income tax, left out NIIT, or computed NIIT without netting the $3,000 capital loss. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_052,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_052,head_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility requires age 65 unless a separate qualifying pathway, such as disability or end-stage renal disease, applies. The head is 59 and all unlisted disability and health-status facts are false, so no Medicare eligibility pathway applies." -us,scenario_052,payroll_tax,30,llm_error,payroll_tax_base,False,,"The correct payroll tax is three components on the head's wages: 6.2% Social Security capped at the 2026 wage base of $184,500 ($11,439), 1.45% Medicare on the full $500,000 of gross wages ($7,250), and 0.9% Additional Medicare Tax on the $250,000 of wages above the $250,000 MFJ threshold ($2,250), with Texas imposing no employee state payroll tax. Two traps separate the reference from nearly every wrong answer: the 2026 Social Security wage base is $184,500, not the 2025 ($176,100), 2024 ($168,600), or various guessed values models substituted; and the FICA base is full gross wages — neither the $16,208 employer-sponsored insurance premiums nor the $926 traditional 401(k) deferral reduces Social Security or Medicare wages. A third trap caught several models: the Additional Medicare Tax filing threshold for joint filers is $250,000, not the $200,000 single-filer withholding trigger. A further cluster derived a defensible subtotal and then submitted an entirely different number." +us,scenario_052,payroll_tax,31,llm_error,thresholds_rates,False,,"The correct payroll tax uses the Head's full $500,000 gross wages as the FICA base: 6.2% × the 2026 Social Security wage base of $184,500 ($11,439), plus 1.45% × $500,000 ($7,250), plus 0.9% Additional Medicare Tax on wages over the $250,000 MFJ threshold ($2,250), for $20,939. Models went wrong in three main ways. Many used a stale or guessed wage base ($168,600, $176,100, $180,000–$183,144). Several shrank the FICA base by treating the $16,208 ESI premiums or the $926 traditional 401(k) deferral as excluded from wages. Others applied the $200,000 single threshold instead of the $250,000 MFJ threshold, or submitted totals that don't match their own component calculations." us,scenario_052,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable disability or other qualifying pathway. The 55-year-old spouse has no listed disability or other Medicare-qualifying status, so the spouse is not eligible." -us,scenario_053,federal_income_tax_before_refundable_credits,32,llm_error,thresholds_rates,False,,"The trap is the 2026 parameter set: the TCJA rate structure is permanent, so a single filer gets a $16,100 standard deduction, no personal exemption, and bracket edges at $12,400 (10%→12%) and $50,400 (12%→22%), giving $66,968.68 − $16,100 = $50,868.68 taxable and $1,240 + $4,560 + 22% × $468.68 = $5,903.11. A second trap is the wage base: the listed $3,389 employer-sponsored insurance premium and the $950 auto loan interest do not reduce AGI here, so gross wages are the starting point. The wrong answers split into a TCJA-sunset camp (an $8,300-ish standard deduction plus a $5,300 personal exemption and 15%/25% rates, landing at $7,200–$9,073), a stale-parameter camp ($13,850/$14,600/$15,000/$15,700 deductions with 2023–2025 bracket edges), and an ESI-exclusion camp (taxable income $47,480, tax $5,449.60). A third group derived $5,903 correctly and then submitted a different number." +us,scenario_053,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The reference takes the listed wages ($66,968.68) as AGI without subtracting anything, then subtracts the 2026 OBBBA single standard deduction of $16,100 to get $50,868.68 of taxable income. It then applies the permanent TCJA-structure 2026 brackets: 10% up to $12,400 ($1,240), 12% up to $50,400 ($4,560) and 22% on the remaining $468.68 ($103.11), for $5,903.11 with no nonrefundable credits. The models went wrong in five main ways: they assumed TCJA sunsets in 2026 (it does not, because OBBBA made the TCJA structure permanent); they used 2023–2025 standard deductions or brackets; they subtracted the $3,389 ESI premium from wages that were given as the taxable wage figure; they took the new-vehicle auto loan interest deduction on a $7,410 vehicle; or they submitted numbers that didn't match their own arithmetic, including invented child credits for a filer with no children." us,scenario_053,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"A single 25-year-old filer with no children and $66,969 of wages receives neither a refundable Child Tax Credit nor an Earned Income Tax Credit in 2026. With the other enumerated refundable-credit components also equal to zero, federal refundable credits total $0." -us,scenario_053,payroll_tax,9,llm_error,payroll_tax_base,False,,"The employer-sponsored insurance premium is a separate household input and is not identified as a pre-tax payroll deduction, so it does not reduce FICA wages under the prompt’s rule that unlisted statuses are false. Payroll tax therefore uses the full $66,969 wage base, applying 6.2% Social Security and 1.45% Medicare, with no Additional Medicare Tax or Idaho employee payroll tax." +us,scenario_053,payroll_tax,10,llm_error,payroll_tax_base,False,,"PolicyEngine applies the employee FICA rates to the full $66,969 of gross wages: 6.2% Social Security ($4,152.06) plus 1.45% Medicare ($971.05) equals $5,123.10. The listed employer-sponsored insurance premium ($3,389) does not reduce the payroll-tax wage base. The household facts never say the premium is paid through a Section 125 cafeteria plan, and the prompt forbids inferring unlisted facts. Eight models subtracted the premium anyway and taxed $63,580, which gives $4,863.87. The two GPT-5.4 models used the correct base and rates but multiplied them out wrong." us,scenario_053,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_053,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Idaho's 2026 individual income tax is a flat 5.3% (the rate cut enacted in 2025, replacing 5.695%) applied only to taxable income above the indexed zero-tax bracket of $4,920 for single filers. Idaho taxable income here is federal AGI of $66,968.68 — the full stated wages, with no reduction for the $3,389 employer-plan premium — less the greater of the conformed federal standard deduction ($16,100) or itemized deductions ($377.35), giving $50,868.68; ($50,868.68 − $4,920) × 5.3% = $2,435.28. The trap that separates the reference from nearly every wrong answer is the $4,920 zero bracket: models that got the $16,100 deduction and the 5.3% rate right still landed on $2,696.06 by taxing the first $4,920. The remaining spread comes from keeping the repealed 5.695%/5.8% rate, using un-indexed or fabricated standard deductions ($8,300, $8,522, $15,000, $15,750, $7,200), or subtracting the employer-plan premium from wages." -us,scenario_053,state_refundable_credits,36,llm_error,thresholds_rates,False,,"Idaho's grocery credit (Idaho Code 63-3024A) is a flat, fully refundable per-person credit with no income test and no phase-out: a full-year resident earns one-twelfth of the annual amount for each qualifying month, and the amount is paid out regardless of tax liability, which is why PolicyEngine routes id_grocery_credit into state_refundable_credits at $155 for this single 25-year-old with 12 qualifying months. Two distinct errors separate the reference from the wrong answers. Twenty models zeroed the output by declaring the credit nonrefundable, absorbed by tax liability, or phased out at $66,969 of wages — none of which the statute provides. Fifteen more correctly treated the credit as refundable but priced it at a superseded per-person amount ($120 from the earlier increase, or the older $100 non-elderly tier) instead of the $155 in force for 2026." +us,scenario_053,state_income_tax_before_refundable_credits,46,reference_engine_defect,state_local_rule,False,,"The frozen reference's Idaho taxable income is federal AGI of $66,968.68 (full gross wages; the ESI premiums do not reduce it) minus the $16,100 standard deduction, which gives $50,868.68; it leaves out Idaho's subtraction of the head's $5,000 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes on the assumption that they are not paid through a pre-tax salary reduction. Idaho's 2026 flat 5.3% rate (cut from 5.695% by 2025 HB 40) applies only above the single zero-rate bracket, which the frozen reference sets at the engine's projected $4,920: 0.053 × $45,948.68 = $2,435.28. The corrected value uses the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026: 0.053 × ($45,868.68 − $4,811) = $2,176.06; gpt-6-astra's $2,176.07 is $0.013 above it. The most common error was applying 5.3% to the whole $50,869 base, which gives $2,696.06 and ignores the zero bracket. Other models used the superseded 5.695% or 5.8% rates, subtracted the $3,389 ESI premiums from AGI, used stale or invented standard deductions ($15,000, $15,750, $8,300), or used another zero-bracket threshold: $4,489, or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (state_local_rule). The frozen reference is 2,435.28; the corrected value is 2,176.06. PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays. Reference is an engine defect; output excluded from scoring (Idaho Code 63-3022P; Idaho Form 39R)." +us,scenario_053,state_refundable_credits,39,llm_error,state_local_rule,False,,"The whole $155 is Idaho's grocery credit (id_grocery_credit). It is a refundable credit with no income limit. This full-year resident qualifies for all 12 months with no SNAP or incarceration reduction, and the 2026 per-person base for someone under 65 is $155. The wrong answers split into two groups. The first group dropped the credit: some called it nonrefundable, some said it only offsets tax liability, some invented an income phase-out, and some never considered it. The second group used an outdated per-person amount: $100 (the pre-2023 figure), $120 (the 2023-2025 figure) or $150, instead of $155." us,scenario_054,child1_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying a CHIP income ceiling alone is not sufficient. At this very low household income, the 10-year-old qualifies for Medicaid under North Carolina’s OLDER_CHILD category, and that Medicaid eligibility excludes the child from CHIP." us,scenario_054,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Child 1 qualifies through the ordinary children's MAGI pathway: at age 10 he falls in the OLDER_CHILD category (ages 6–18), whose North Carolina income limit is 133% FPL plus the 5-percentage-point disregard, and NC folded its separate NC Health Choice tier into Medicaid so school-age children are covered under Medicaid rather than being pushed to CHIP. The household's only income is $17,100 of self-employment earnings, which after the deductible half of SE tax leaves MAGI at 0.58 × FPL for a family of three — a fraction of the applicable limit. Both models returned 0 without ever performing that categorical-plus-income test: one asserted an income failure that the arithmetic contradicts, the other treated unlisted household facts as missing prerequisites even though the prompt instructs that unlisted booleans are false and take-up is assumed." us,scenario_054,child1_wic_eligible,2,llm_error,age_disability,False,,"WIC categorical eligibility covers pregnant, postpartum, and breastfeeding women, infants, and children under age five; it does not extend to nutritionally at-risk school-age children. Child 1 is age 10, so income eligibility and another household member’s WIC receipt cannot make Child 1 eligible." @@ -452,16 +456,20 @@ us,scenario_054,child2_head_start_eligible,3,llm_error,categorical_eligibility,F us,scenario_054,child2_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"A 2-year-old qualifies for Medicaid through the age-based YOUNG_CHILD MAGI category, whose North Carolina income standard for ages 1-5 exceeds 200% FPL; this household sits at 0.58 x FPL, so the income test is satisfied outright. The trap is that Medicaid, unlike CHIP, imposes no bar on children who already hold employer-sponsored insurance, and it requires no disability, SSI, or other special categorical status beyond age. All three wrong answers refused the category: one swapped in CHIP's other-coverage exclusion, one measured the child against an adult-level FPL cutoff, and one defaulted to 0 for want of an extra eligibility flag that the children's pathway never requires." us,scenario_054,child2_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC eligibility for a child turns on two things only: falling in the child category (age 1 through the fifth birthday) and household income at or below 185% of the federal poverty guideline, with adjunctive eligibility through Medicaid, SNAP, or TANF as an independent qualifying path. This three-person North Carolina household has at most $17,100 of self-employment income against a 185% FPL limit near $49,000 for 2026, so the income screen is cleared with enormous margin, and the head's listed WIC receipt reflects a household already certified under that same screen. Both wrong answers denied a child who satisfies the age category and the income test — one by inventing an income ceiling the household never approaches, the other by treating an unlisted enrollment flag as dispositive of eligibility." us,scenario_054,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_054,federal_refundable_credits,33,llm_error,credit_phaseout,False,,"Both components turn on a single earned-income figure: self-employment income less the deductible half of SE tax, $17,100 − $1,208.07 = $15,891.93 — not gross $17,100 and not the 92.35% net-earnings figure of $15,791.85. With two qualifying children that income sits inside the EITC phase-in range, so the credit is 40% × $15,891.93 = $6,356.77, and the refundable CTC is 15% × ($15,891.93 − $2,500) = $2,008.79, well under the $1,700-per-child refundable cap. The two traps that separate the wrong answers are the earned-income base and the ACTC earned-income floor: models applying the repealed $3,000 floor land on $8,290.56, and models using gross $17,100 land on $9,030 or $8,840. A third cluster either zeroes out the credits entirely or abandons its own correct arithmetic for an unexplained round number." +us,scenario_054,federal_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"For a self-employed filer, earned income for both the EITC and the refundable CTC is net SE profit minus the deductible half of SE tax. Here that is $17,100 - $1,208.07 = $15,891.93. The EITC is still in its 40% two-child phase-in, giving $6,356.77. The 2026 refundable CTC uses the permanent OBBBA $2,500 earned-income threshold, giving 15% x $13,391.93 = $2,008.79, which is under the $1,700-per-child refundable cap. The wrong answers split into four groups. Some used the wrong earned-income base: gross $17,100, the 92.35% figure of $15,791.85, or a double-reduced $14,584. Some used the pre-TCJA $3,000 threshold or the $1,000-per-child CTC. Some dropped a component or misplaced the EITC on its schedule. The rest denied eligibility outright." us,scenario_054,free_school_meals_eligible,2,llm_error,thresholds_rates,False,,"The decisive step is the free-meal income test: $17,100 of self-employment income for a household of three is 63% of the federal poverty guideline, less than half the 130% ceiling for free meals, and the household is independently categorically eligible through SNAP ($524.80/month), which by itself confers free-meal status for the 10-year-old K-12 student. NC has no universal free school meal program, so the household qualifies through the standard free tier rather than a statewide waiver, and PolicyEngine values the benefit at $7.15/day for one enrolled child ($1,130.96/year). Both wrong models denied eligibility for a household at less than one-third of the free-meal income limit — one by asserting the income exceeded the threshold, the other by treating the absence of an explicit eligibility flag in the prompt as a negative." us,scenario_054,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The head is a 40-year-old living with two of their own children, which places them in the parent/caretaker-relative Medicaid category (medicaid_category = PARENT), a mandatory MAGI pathway that exists in every state independent of expansion status. Their only income is $17,100 of self-employment earnings, which after the deductible half of SE tax gives MAGI at 0.58 x FPL for a household of three — far below both the parent/caretaker limit and the 138% FPL adult limit North Carolina adopted when it implemented expansion on December 1, 2023 (S.L. 2023-7), in force for 2026. Every wrong model either declared the head a childless/non-categorical adult despite two listed children, or asserted NC is still a non-expansion state, and so answered 0 on a household that clears the income test by a wide margin on two independent pathways." -us,scenario_054,head_wic_eligible,15,llm_error,categorical_eligibility,False,,"WIC eligibility is determined per person and requires the individual to occupy a WIC category — infant under 1, child aged 1 through 4, pregnant, postpartum, or breastfeeding — before the 185% FPL income test is even reached. The head is 40 with no pregnancy, postpartum, or breastfeeding status listed, and the prompt directs that unlisted booleans be treated as false, so her category is NONE and she fails at the categorical prong regardless of the household's near-zero income; only the 2-year-old holds a category, which is exactly the [False, False, True] pattern the engine returned. The listed ""receives wic"" fact is a receipt/takeup input, and the question asks for eligibility under PolicyEngine rules explicitly rather than current enrollment, so it carries no eligibility content. Every wrong model either read the receipt flag as a self-证ifying eligibility determination or invented a household-level ""has a child under 5"" / caretaker pathway that WIC does not contain." +us,scenario_054,head_wic_eligible,16,llm_error,categorical_eligibility,False,,"WIC eligibility is decided person by person, and each person must first fall into a categorical group. An adult qualifies only if pregnant, postpartum or breastfeeding, and a child qualifies only if under age 5. The 40-year-old head has none of the pregnancy, postpartum or breastfeeding flags, and unlisted status inputs count as false, so her WIC category is none and income never enters the test. The trap is the ""receives wic"" flag listed under the head. It reports that the household takes up WIC, which here happens through the 2-year-old, and it gives no adult categorical status. Every model treated either that receipt or being the caretaker of a child under 5 as making the head eligible herself." us,scenario_054,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages, not SSTB self-employment income; that income instead enters the separate self-employment-tax calculation. With no reported wages and all unlisted numeric inputs set to zero, every component of the requested employee-side payroll tax is zero." us,scenario_054,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. The household is in the FREE tier because it meets categorical eligibility through SNAP or TANF and its school-meal income is 63% of the federal poverty guideline, below the 130% free-meals threshold; therefore reduced-price support is zero." us,scenario_054,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax equals 15.3% of the statutory net-earnings base: $17,100 × 92.35% × 15.3% = $2,416.15305, which rounds to $2,416.15. The deduction for one-half of self-employment tax affects adjusted gross income, not the self-employment tax liability itself, and the requested amount requires cent-level arithmetic rather than whole-dollar estimation." -us,scenario_054,snap,39,llm_error,thresholds_rates,False,,"The SNAP arithmetic here is fully determined: $17,100 of self-employment income with no listed business expenses gives $1,425 countable gross per month, the 20% earned-income deduction removes $285, the three-person standard deduction removes $209, and net income is $931 — with no shelter or dependent-care deduction because no rent or child care is listed. North Carolina's broad-based categorical eligibility waives the resource test on the $14,000 bank balance, and TANF non-cash assistance supplies an independent categorical pathway, so the household is eligible. The discriminating step is the maximum allotment: calendar 2026 spans two federal fiscal years, so the benefit is $785 − $279.30 = $505.70 for January–September and about $525 for October–December once the FY2027 maximum of $803 takes effect, totaling $6,125.69. The wrong answers cluster into three failures — applying an asset or income limit and zeroing the benefit, inventing self-employment deductions SNAP does not grant (a 40% cost-of-business allowance or half of SE tax), and running the formula on one fiscal year's allotment multiplied by twelve." +us,scenario_054,snap,44,llm_error,thresholds_rates,False,,"The correct monthly allotment is the FY2026 three-person maximum of $785 minus an expected contribution of 30% of $931 net income ($1,425 gross minus the $285 earned income deduction and the $209 standard deduction). That contribution rounds up to a whole $280 under 7 CFR 273.10(e)(2)(ii)(A), which gives $505 a month and $6,060 a year. North Carolina's broad-based categorical eligibility means the $14,000 bank balance does not disqualify the household. The wrong answers fall into four groups: +- Right inputs, no round-up: nine models used $279.30 instead of $280 and landed at $6,068–$6,072. +- Wrong parameters: most used stale or misremembered ones, such as the FY2024 $766/$198 or FY2025 $768/$204 pairs, or maximum allotments of $800 or more. +- Asset test: six zeroed the benefit with a resource test that categorical eligibility waives. +- Income or arithmetic: the rest miscounted self-employment income or made arithmetic errors." us,scenario_054,state_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Two independent paths lead to $0 of North Carolina tax. First, the only income fact listed is the SSTB *component* input; total self-employment income is unlisted and therefore 0 under the prompt's ""treat any unlisted numeric input as 0"" rule, so the engine's trace shows essentially no income entering nc_income_tax. Second, even crediting the full $17,100 as net self-employment earnings, federal AGI is $17,100 − $1,208 (half of SE tax) = $15,892, and North Carolina's 2026 head-of-household standard deduction of $19,125 — the head has two qualifying children — exceeds that, driving NC taxable income to $0 and the 3.99% 2026 flat rate onto a zero base. Every wrong model produced a positive liability; two of them derived exactly $0 in their own reasoning and then submitted a different number anyway." -us,scenario_055,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"Every model that reached the right AGI ($83,635.15 wages + $67,760 pension + $4,880.71 of Social Security at the 85% cap = $156,275.86) then failed on the 2026 deduction stack, which under OBBBA is $20,273.45: the $16,100 single standard deduction plus the $2,050 age-65 additional ($18,150), plus the new $1,000 non-itemizer charitable deduction, plus the $6,000 senior deduction reduced 6% of MAGI over $75,000 ($81,275.86 × 6% = $4,876.55, leaving $1,123.45). That yields taxable income of $136,002.41, taxed on the permanent TCJA single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, 24% above) for exactly $25,238.58. Two shared traps split the field: one group assumed the TCJA sunset and applied pre-2018 10/15/25/28 rates with a personal exemption, and the other used 2026 rates but dropped one or more of the three stacked senior/charitable deduction pieces, or abandoned its own arithmetic for an unsupported final number." +us,scenario_055,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The reference subtracts three amounts from AGI of $156,275.86, totaling $20,273.45: the 2026 standard deduction of $18,150 ($16,100 plus the $2,050 aged add-on), the OBBBA senior deduction of $6,000 phased down by 6% of MAGI above $75,000 to $1,123.45, and the new $1,000 non-itemizer cash charitable deduction (IRC 170(p), in effect from 2026), which applies because the filer takes the standard deduction and gave $15,647 in cash. That leaves taxable income of $136,002.41. The 2026 brackets, made permanent under OBBBA, tax it at $17,966 through $105,700 plus 24% of the rest. The closest models (claude-fable-5, claude-sonnet-5.5, gpt-6-luna) got every step except the $1,000 non-itemizer charitable deduction and came out exactly $240 high. Eleven models applied the pre-TCJA sunset rules, with personal exemptions and 15/25/28% brackets. Most of the rest dropped or mis-phased the senior deduction or used guessed standard deductions and brackets." us,scenario_055,payroll_tax,5,llm_error,thresholds_rates,False,,"The only FICA-taxable income here is the $83,635 of wages: the $67,760 taxable pension and $5,742 Social Security retirement benefit are outside the payroll tax base, the wage is far below the 2026 Social Security wage base, age 70 does not exempt wages from FICA, and Florida levies no mandatory employee state payroll tax. The answer is therefore exactly 6.2% × $83,635 = $5,185.37 plus 1.45% × $83,635 = $1,212.71, or $6,398.08. What separates the reference from the wrong answers is execution of that single multiplication at the employee-side rate: two models stated the right rule and then mis-multiplied or rounded, one submitted a number contradicting its own correct explanation, one used the combined 12.4%/2.9% employer-plus-employee rates before guessing, and one submitted a placeholder zero." us,scenario_056,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The HSA deduction and other applicable adjustments reduce adjusted gross income to -$5,882.05, and taxable income remains zero after the standard deduction. With zero income tax before credits and no other included tax components, federal income tax before refundable credits is $0." us,scenario_056,head_medicaid_eligible,12,llm_error,health_coverage,False,,"New Jersey’s ACA expansion covers this 20-year-old as a MAGI-based adult because the person satisfies the immigration rule and the engine’s MAGI income level is 0.00 times FPL. Employer-sponsored insurance does not disqualify an otherwise eligible adult from Medicaid; it affects other health-coverage calculations, including ACA premium assistance. The wrong answers either imposed that nonexistent ESI exclusion or replaced the Medicaid MAGI calculation with a sum of inputs that the engine does not count as MAGI in that manner." @@ -469,51 +477,51 @@ us,scenario_056,head_wic_eligible,1,parse_contract_failure,missing_output,False, us,scenario_056,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_056,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage earnings, and the household has no reported wages or salary. The listed SSTB self-employment income belongs in the separately requested self-employment-tax calculation, while alimony and financial assistance are not employee FICA wages." us,scenario_056,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax applies because $4,340 exceeds the $400 filing threshold. Multiplying the 92.35% net-earnings base by the combined 15.3% Social Security and Medicare rate yields $613.22; the threshold does not reduce the taxable base or eliminate liability." -us,scenario_056,snap,39,llm_error,state_local_rule,False,,"The household clears both SNAP income tests (gross $1,486.54/mo = 1.12x the $1,330 poverty guideline; net $805.03/mo = 0.61x after $674.49 of deductions — $213.68 standard, $72.33 earned-income, and an uncapped $395.49 excess shelter deduction built on the $995.75/mo real-estate-tax shelter charge), so the formula allotment is $304.68 - $241.50 = $63.18/mo. The decisive step is that New Jersey guarantees a state-funded $95/month minimum SNAP benefit, which replaces the $63.18 formula result and yields $95 x 12 = $1,140. Every wrong answer failed on one of two things: declaring the household income-ineligible or the allotment zero (which the NJ floor forecloses even when the formula runs negative), or inflating the excess shelter deduction by adding the $564.08/mo mortgage interest and applying the ~$712-$744 cap, which pushed net income down to ~$460-$500 and produced a formula benefit of $140-$160/mo that sits above the floor. Only gpt-5.5 reached the correct structure, and it substituted the federal $23/mo minimum for New Jersey's $95." +us,scenario_056,snap,45,prompt_ambiguity,categorical_eligibility,False,,"Countable SNAP gross income is about $1,486.54 a month: $361.67 of self-employment income, $124.92 of alimony and $1,000 of financial assistance. That is 112% of FPL, well under NJ's broad-based categorical eligibility (BBCE) gross limit, and the asset test is waived. After the $213.68 standard deduction and the $72.33 earned-income deduction, adjusted income is $1,200.53. Shelter costs are only the $995.75 in monthly real-estate taxes; the tax-side home-mortgage-interest input is not a SNAP shelter cost. That gives an uncapped excess shelter deduction of $395.49 and net income of $805.03. The formula benefit is $304.68 − $241.50 = $63.18, and New Jersey's $95 state minimum benefit raises it to $95 a month ($1,140 a year). Models either added mortgage interest and hit the shelter cap (about $1,900), left out the shelter deduction or the NJ minimum (and gave $0), or dropped the financial assistance (and gave the maximum allotment). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)." us,scenario_056,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_056,state_income_tax_before_refundable_credits,5,llm_error,thresholds_rates,False,,"New Jersey's gross income tax exempts a single filer entirely when New Jersey gross income falls at or below the $10,000 minimum filing threshold, and this household's NJ gross income is $5,838.53 (alimony $1,498.53 plus self-employment $4,340.00; the $12,000 of financial assistance is not NJ gross income). PolicyEngine therefore computes the bracket tax as an intermediate figure — $4,838.53 of taxable income after the $1,000 regular exemption, times the 1.4% first-bracket rate, equals $67.74 — and then zeroes it out under the below-threshold exemption, leaving $0.00. Every wrong model stopped at the 1.4% bracket arithmetic and never applied the threshold exemption; three of them also mis-specified the NJ income and deduction base along the way." -us,scenario_056,state_refundable_credits,39,llm_error,state_local_rule,False,,"New Jersey decouples its EITC from the federal childless age band: since TY2021 the NJEITC reaches filers aged 18-24 (and 65+) with no qualifying children, computing the credit from the federal formula the filer would have qualified for but for age. This 20-year-old's EITC earned income is $4,033.39 (self-employment income of $4,340 less half of self-employment tax), which phases in at 7.65% to $308.55, and with AGI of -$5,882.05 nothing phases out; New Jersey's minimum NJEITC benefit for filers without dependents ($215 statutory, $265.60 at 2026 levels) then sets the credit, exceeding the 40% federal match of $123.42. Thirty-one models imported the federal 25-64 age floor and returned $0, six applied the age expansion but stopped at the 40% match ($123-$133), and one substituted NJ's $50 property tax credit." +us,scenario_056,state_refundable_credits,46,llm_error,credit_phaseout,False,"The $265.60 reference is exactly 40% × $664, the full 2026 childless EITC maximum. The derivation itself says the credit phased in to only $308.55 (7.65% × $4,033.39), and no phase-out applies with negative AGI and $4,033 of earnings. Under N.J.S.A. 54A:4-7 and IRC §32(a), the credit should be 40% × $308.55 = $123.42, so PolicyEngine's NJ path for young childless filers appears to use eitc_maximum (minus the reduction) instead of the phased-in amount.","The key rule is that New Jersey pays its 40% EITC match to childless filers aged 18 and over who fail the federal 25–64 age test. Most models stopped at 'federal EITC = $0, so NJ EITC = $0' or used the outdated 21+ age floor. The models that found the age-18 pathway computed 40% of the phased-in federal-formula credit (7.65% × $4,033.39 = $308.55), giving $123.42. The reference value of $265.60 equals 40% of the full $664 childless maximum, so it skips the earned-income phase-in. Two models also used gross self-employment income without subtracting half of the self-employment tax, and two others claimed the $50 property tax credit, which is refundable only for filers who are 65 or older, blind, or disabled. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)." us,scenario_056,tanf,2,llm_error,categorical_eligibility,False,,"TANF is categorically limited to assistance units containing a dependent child: New Jersey's WorkFirst NJ/TANF requires a needy child under 18 (or 18 and a full-time secondary student) in the unit, and childless adults are served instead by WFNJ General Assistance, which is not TANF. This household is a single 20-year-old head with no children, so the benefit is $0 regardless of the income test. The second trap is that `financial_assistance` is an input income line in the household facts, not a computed program output; echoing it back as the TANF benefit both invents categorical eligibility and confuses an input with the requested computed value." -us,scenario_057,federal_refundable_credits,33,llm_error,thresholds_rates,False,,"The case turns on the 2026 indexed zero-qualifying-child EITC maximum, $664 (the $8,690 earned-income amount times the 7.65% phase-in rate), and on the joint-filer age test being satisfied by either spouse. The head's $10,400 of self-employment income yields EITC earned income of $9,665 ($10,400 less the $735 deductible half of SE tax), which clears the $8,690 phase-in ceiling and sits roughly $8,500 below the joint zero-child phaseout start (~$18,180), so the credit is the flat statutory maximum, and no other federal refundable credit reaches a childless couple. The wrong answers split into two families: those that located the plateau correctly but priced it with a stale 2025 maximum ($649) or an invented one ($611/$620/$630/$648/$650/$660/$666/$667/$1,231), and those that zeroed the credit by demanding both spouses be 25 or older, demanding W-2 wages, or capping refundable credits at tax liability." +us,scenario_057,federal_refundable_credits,35,llm_error,thresholds_rates,False,,"The household's only refundable credit is the no-qualifying-child EITC. On a joint return, the age test is met when either spouse is 25-64, and the head is 43. EITC earned income is about $9,665: $10,400 of SE profit minus the deductible half of SE tax. That is past the 2026 phase-in end (about $8,700) and well below the joint phase-out start (about $18,000), so the credit sits on the plateau at the 2026 statutory maximum of $664. The wrong answers fall into three groups. Some used the wrong maximum: the 2025 figure of $649, or guessed inflation-indexed values from $620 to $667. Some misplaced the household in the phase-in or phase-out range. The rest zeroed the credit through invented disqualifiers: requiring both spouses to be 25 or older, requiring wages, requiring positive tax liability, or claiming an income or disability-benefit bar." us,scenario_057,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The head is blind and disabled with only $10,400 of self-employment income, which after the SSI $20 general, $65 earned-income, and one-half-of-remainder exclusions leaves countable income far below the federal benefit rate, producing $7,231 of SSI for the year. Louisiana is a Section 1634 state, so SSI receipt is itself a Medicaid eligibility category (medicaid_category = SSI_RECIPIENT) conferring automatic coverage with no income test applied. The trap is that all three models tested income or hunted for a MAGI category instead of following the SSI-receipt pathway, and each also ignored that Louisiana's ACA adult expansion covers 19-64-year-olds to 138% FPL while this household's MAGI is 0.45 x FPL — two independent routes to Yes." -us,scenario_057,head_medicare_eligible,8,llm_error,age_disability,False,,"Medicare's under-65 pathway is not a disability-status test: it requires 24 months of entitlement to Social Security Title II disability insurance benefits (42 U.S.C. §426(b)), with only ESRD and ALS bypassing the waiting period, and SSI or any other non-Title II disability payment confers nothing. PolicyEngine's is_medicare_eligible implements the age test — age >= 65 — and the head is 43 with a spouse of 20, so the household returns [False, False]. The household facts list a disabled/blind flag and $14 of generic disability benefits, neither of which establishes Title II entitlement, let alone 24 months of it, and the prompt's instruction to treat unlisted facts as false forecloses inferring an SSDI entitlement history. All eight models converted ""is disabled"" plus a token disability-benefit amount directly into Medicare entitlement, skipping the entitlement-type and waiting-period conditions that separate the two." +us,scenario_057,head_medicare_eligible,10,llm_error,age_disability,False,,"Before 65, a person qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or through the ESRD/ALS pathways. PolicyEngine checks this with months of Social Security disability receipt, and it does not use the is_disabled or is_blind flags. The head is 43. Their only disability income is $14 of generic ""disability benefits"", which is not Social Security disability. No months of SSDI receipt are listed, so that input is 0, and neither ESRD nor ALS is listed either. Every model treated disability status plus a nominal disability-benefit amount as enough for Medicare under 65. That skips both the SSDI-entitlement requirement and the 24-month waiting period." us,scenario_057,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_057,self_employment_tax,7,llm_error,thresholds_rates,False,,"The whole case reduces to one canonical computation: net earnings from self-employment are 92.35% of the $10,400 SSTB profit ($9,604.40), and the combined OASDI/HI rate of 15.3% applies in full because $9,604.40 is far under the 2026 Social Security wage base, giving 0.153 × 9,604.40 = $1,469.47. The $400 figure in §1402(b) is the floor at which SE tax begins, not a ceiling below which it is waived, and no other floor exists; the one-half-of-SE-tax figure is an above-the-line deduction under §164(f), not the liability. The prompt further defines this output as SE tax excluding Additional Medicare Tax, and the 0.9% AMT threshold for joint filers is $250,000 of combined earnings, so nothing is added on top. Every wrong answer comes from inverting the $400 floor, inventing a different floor, truncating the 0.9235 base factor, reporting the deduction instead of the tax, or bolting on a nonexistent AMT." -us,scenario_057,snap,39,prompt_ambiguity,age_disability,False,,"The case turns on PolicyEngine imputing SSI to the blind and disabled head — $602.58/month, i.e. the $995 individual federal benefit rate less $392.42 of countable income (half of $866.67 in monthly self-employment earnings after the $20 general and $65 earned-income exclusions) — and on SNAP counting that SSI dollar-for-dollar as unearned income. Monthly gross income is therefore $866.67 + $3.17 + $602.58 = $1,472.42; subtracting the 20% earned-income deduction ($173.33) and the $209 FY2026 standard deduction leaves net income of $1,090.08, a 30% contribution of $327, and an allotment of $546 − $327 = $219/month for nine months plus $558.24 − $325.50 = $232.74 for the three FY2027 months, totaling $2,669.22. No medical deduction applies ($40 of annual OTC expenses is below the $35-per-month excess-medical threshold) and no shelter costs are listed. The wrong answers split three ways: omitting the imputed SSI entirely and landing near $4,000–$6,200, counting SSI at the ~$1,100 couple-rate figure and landing near $700–$900, and declaring the household ineligible at $0. Developer adjudication (2026-09-05): the judge (gpt-5.6-sol) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income." +us,scenario_057,snap,46,prompt_ambiguity,age_disability,False,,"The trap is SNAP countable income. The household's SNAP gross income has to include the head's computed SSI payment of $602.58/month. That is the individual rate: the $994 FBR minus $391.42 of countable income from the head's own $866.67 of self-employment earnings and $1.17 of benefits. Adding it to $866.67 of earnings and $3.17 of disability benefits gives gross income of $1,472.42. After the 20% earned-income deduction and the $209 standard deduction, net income is $1,090.08. The benefit is $546 − $327 = $219/month for January–September and $232.74/month for October–December under FY2027 parameters, totaling $2,669.22. About 18 models left SSI out entirely and got net income near $488 and roughly $4,700/year. About 17 counted SSI at the couple rate of about $1,099/month and got net income near $1,586 and roughly $840/year. Most of the rest wrongly zeroed out the benefit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." us,scenario_057,spouse_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires the person to be outside Medicaid eligibility; a low income alone does not establish CHIP eligibility. The spouse is Medicaid-eligible in Louisiana's adult category, so the Medicaid exclusion makes `spouse_chip_eligible` false without applying a CHIP income threshold." us,scenario_057,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Louisiana adopted the ACA adult expansion effective July 1, 2016, so any non-elderly adult aged 19–64 who is not otherwise eligible qualifies for Medicaid on income alone at or below 138% FPL under MAGI methodology — no pregnancy, parenthood, or disability category is required. The spouse is 20 with household MAGI at 0.45 x FPL (the only income is the head's $10,400 SSTB self-employment plus $38 of disability benefits, against a two-person 2026 FPL near $21,150 and a 138% cutoff near $29,200), placing the household roughly a third of the way to the expansion limit. Every wrong model either invented an income excess that does not exist or applied pre-ACA categorical gatekeeping that Louisiana abolished, and each therefore returned 0 where the ADULT expansion pathway returns eligible." -us,scenario_057,spouse_medicare_eligible,6,llm_error,age_disability,False,,"PolicyEngine's `is_medicare_eligible` is a pure age test — true only at age 65 and older — with no disability, ESRD, or ALS branch, so a 20-year-old spouse returns False no matter what the disability flags say. Every wrong model substituted the real-world Title II pathway (Medicare after 24 months of SSDI entitlement, or immediately for ALS/ESRD) for the modeled rule, and treated the household's `is_disabled` flag plus $24 of annual disability benefits as if it established that entitlement. That $24 figure is not an SSDI benefit level, and the prompt's instruction to hold statuses constant across the tax-benefit year says nothing about a 24-month prior entitlement history, so even the borrowed real-world rule was not satisfied on these facts." +us,scenario_057,spouse_medicare_eligible,8,llm_error,age_disability,False,,"To be eligible for Medicare before age 65, a person must be entitled to Social Security Disability Insurance (SSDI) and have received it for 24 months, or have ESRD or ALS. Being disabled is not enough on its own. The spouse is 20 and has only $24 of generic `disability_benefits`. That is private or other disability income, not SSDI. Months of SSDI receipt and ESRD/ALS status are not listed, so they default to 0/false. The spouse fails both the age-65 test and the disability pathway. Every model read ""is disabled"" plus a small disability-benefit amount as satisfying the disability pathway." us,scenario_057,spouse_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC does not cover women merely because they are young, disabled, or income-eligible. A woman must be pregnant, breastfeeding, or postpartum; the spouse has none of these statuses, and the household has no infants or children under age five." -us,scenario_057,ssi,39,prompt_ambiguity,age_disability,False,,"SSI is an individual benefit, and the 20-year-old spouse is not SSI aged/blind/disabled, so the head is an eligible individual with an ineligible spouse whose $24 of income falls below the couple/individual FBR differential — nothing is deemed and the eligible-individual FBR of $994/month ($11,928/year) governs, not the ~$1,490/month eligible-couple rate. Countable income is the head's alone: $14 of unearned income is fully absorbed by the $240 general exclusion, leaving $226 to offset earnings, so ($10,400 − $226 − $780)/2 = $4,697, or $391.42/month. The benefit is $994 − $391.42 = $602.58/month, or $7,231 annually. The wrong answers split into two clusters: ~$12,400–$13,600 from models that applied the couple FBR (often with the spouse's $24 pooled into unearned income), and $0 from models that dropped the 50% earned-income disregard, misread the $912 resource balance, or refused to compute. Developer adjudication (2026-09-05): the judge (gpt-5.6-sol) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading." +us,scenario_057,ssi,46,prompt_ambiguity,age_disability,False,,"The decisive step is that the 20-year-old spouse does not meet the SSI aged/blind/disabled test. That makes the head an eligible individual with an ineligible spouse, not half of an eligible couple. The head is therefore paid from the $994/month individual FBR, not the $1,491/month couple rate. None of the spouse's $2/month disability income is deemed because it is below the $497 couple-minus-individual threshold. The head's $1.17/month of unearned income uses part of the $20 general exclusion, and the remaining $18.83 applies to earnings: ($866.67 − $18.83 − $65) / 2 = $391.42 countable income. SSI is $994 − $391.42 = $602.58/month, or $7,231/year. Almost every nonzero answer used the eligible-couple rate and pooled income, producing about $12,600–$13,600. The $0 answers wrongly concluded that income or assets exceeded the SSI limits, or never calculated the benefit. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." us,scenario_057,state_income_tax_before_refundable_credits,3,llm_error,state_local_rule,False,,"The trap is Louisiana's flat-tax reform, effective tax year 2025: Act 11 of the 2024 Third Extraordinary Session replaced the graduated 1.85/3.5/4.25% schedule and the $4,500-per-filer personal exemptions with a single 3% rate and a $25,000 standard deduction for married filing jointly (indexed from 2026), and Louisiana taxable income starts from federal AGI. This household's only taxable income is $10,400 of SSTB self-employment income, yielding federal AGI of about $9,665 after the deductible half of self-employment tax — far below the joint standard deduction — so Louisiana taxable income and tax are both zero. Both wrong answers subtracted $9,000, the repealed exemption figure, manufacturing a $665 taxable base that the current deduction fully absorbs." -us,scenario_057,state_refundable_credits,33,llm_error,state_local_rule,False,,"The single decisive step is Louisiana's refundable EITC under La. R.S. 47:297.8, equal to 5% of the federal EITC and paid out in full despite zero state liability. Reaching it requires recognizing that this childless married couple qualifies for the federal EITC: the 43-year-old head satisfies the 25–64 age test on a joint return, and earned income of $9,665 ($10,400 of self-employment income less the $735 deductible half of self-employment tax) sits on the plateau — above the ~$8,700 phase-in ceiling and far below the ~$18,000 joint phase-out start — for the full 2026 childless maximum of $664. Most wrong models denied the credit outright, either claiming Louisiana has no income tax or no refundable credits, or gating the state EITC on qualifying children; the rest got the 5%-of-federal structure right and missed on a parameter, using a stale federal maximum ($630/$632/$649), a stale or invented state match rate (3.5% or 10%), or running the 7.65% phase-in past the credit cap." -us,scenario_059,federal_income_tax_before_refundable_credits,11,llm_error,taxable_income_or_deductions,False,,"AGI here is $31,729.20: $25,600 of IRA distributions plus $6,129.20 of taxable Social Security, because provisional income of $44,152 exceeds the MFJ second threshold by only $152, so the tier-2 worksheet gives $6,000 + 0.85 × $152 rather than the 85% maximum of $31,538.40. The 2026 MFJ standard deduction of $32,200 plus two $1,650 age-65 additions is $35,500, which by itself exceeds that AGI, and the OBBBA $6,000-per-senior deduction adds $12,000 more for this 79/78 couple. Taxable income is therefore zero and federal income tax before refundable credits is $0. Every wrong answer either applied the flat 85% cap to Social Security instead of the tier-2 formula, or replaced current law with a TCJA-sunset stack of a small standard deduction plus personal exemptions, or used a single-filer standard deduction on a joint return." +us,scenario_057,state_refundable_credits,34,llm_error,state_local_rule,False,,"This is a married couple with no children. The head is 43, and his net self-employment earnings after the half-SE-tax deduction are about $9,665 ($10,400 minus about $735). That is above the roughly $8,680 point where the childless phase-in ends and well below the MFJ phase-out start, so the federal childless EITC is at its 2026 maximum of $664. Louisiana's refundable EITC is 5% of the federal EITC, which gives $33.20. Most models answered $0 for one of three reasons: they did not know Louisiana has a refundable EITC, they assumed the EITC requires children or that the 20-year-old spouse disqualifies the couple, or they put federal EITC at zero. The models that did apply the Louisiana EITC used a stale federal maximum ($632, $649 or $630), the wrong Louisiana match rate (3.5% or 10%), or a federal EITC that was never capped at the maximum." +us,scenario_059,federal_income_tax_before_refundable_credits,12,llm_error,taxable_income_or_deductions,False,,"Provisional income is $25,600 + $18,552 = $44,152, only $152 above the $44,000 MFJ adjusted base. Taxable Social Security is therefore $6,000 + 85% x $152 = $6,129.20, and AGI is $31,729.20. Under 2026 law (TCJA made permanent by OBBBA), deductions total $47,500: the $32,200 MFJ standard deduction, $1,650 x 2 additional aged amounts, and the $6,000-per-person senior deduction (no phase-out at this MAGI). That is far above AGI, so taxable income and tax are $0. Most models got AGI right but used TCJA-sunset deductions and personal exemptions of about $28-30K and left out the senior deduction. The rest over-taxed Social Security with a flat 85% or a mangled worksheet and also left out the $12,000 senior deduction." us,scenario_059,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered earnings, not Social Security retirement benefits or taxable IRA distributions. With no wages or self-employment income listed, the employee payroll-tax base is zero and the resulting payroll tax is $0." us,scenario_060,federal_refundable_credits,1,llm_error,age_disability,False,,"For 2026, the temporary 2021 removal of the upper age limit for the childless EITC has expired. A taxpayer without a qualifying child must be under age 65, so this 70-year-old receives no EITC and all other refundable credit components are also zero." us,scenario_060,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Texas Medicaid does not grant eligibility merely because a 70-year-old has MAGI of 1.08 times FPL or minimal bank assets. The head must qualify through a specific Medicaid category, and the aged pathway is not satisfied; the engine therefore assigns Medicaid category NONE." us,scenario_060,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee-side FICA on $3,000 of wages is $186.00 of Social Security tax at 6.2% plus $43.50 of Medicare tax at 1.45%, totaling $229.50. The 2.9% Medicare rate combines the employee and employer shares, while the requested payroll-tax output excludes employer taxes; no additional Texas employee payroll tax applies." us,scenario_060,snap,4,llm_error,taxable_income_or_deductions,False,,"The trap is the SNAP treatment of the $45,600 in veterans benefits: 7 CFR 273.9(b)(2)(ii) counts veterans' benefits as unearned income, making countable gross income $62,775 rather than the $17,175 three models used. The elderly household skips the gross income test but must still pass the net income test, and even after the 20% earned income deduction, the standard deduction, the uncapped excess medical deduction over $35/month, and the uncapped shelter deduction available because the head is 70, net income lands near $45,000/year (~$3,760/month) against a one-person net limit around $1,300/month. Failing the net income test zeroes the allotment entirely — there is no minimum-benefit floor for an income-ineligible household — which is how the reference reaches $0. Three models excluded the VA income and therefore hit $0 net income and the one-person maximum allotment; the fourth counted it, derived ineligibility correctly, and then submitted a positive number anyway." -us,scenario_062,federal_income_tax_before_refundable_credits,21,llm_error,taxable_income_or_deductions,False,,"The trap is which income is countable and how large the 2026 deduction stack is for a single filer age 85. PolicyEngine counts $4,793 farm rent + $257 interest + $11,280 pension + $21,312 Social Security survivor benefits = $37,642, and the standalone $28,800 survivor-benefits input never enters AGI; with $16,330 of non-Social-Security income, provisional income is $26,986, so only 50% of the excess over the $25,000 single base — $993 — is taxable, giving AGI of $17,323. That AGI is below the single 65+ deduction stack (about $16,100 standard deduction made permanent by OBBBA, a $2,050 age addition, and the $6,000 senior deduction), so taxable income, income tax before credits, and NIIT (MAGI far under $200,000) are all $0. Sixteen models added the $28,800 to gross income, which pushed provisional income past the $34,000 breakpoint and forced an $18,115 85% Social Security inclusion; a second cluster kept AGI at $17,323 but assumed TCJA sunset, deducting a ~$8,000 standard deduction plus a repealed personal exemption, and glm-5.3 drove a correct $0 liability negative with a nonrefundable §22 credit that is both capped at liability and zeroed by the $21,312 of nontaxable Social Security." +us,scenario_062,federal_income_tax_before_refundable_credits,25,prompt_ambiguity,taxable_income_or_deductions,False,,"The separate $28,800 survivor-benefits input is not part of federal gross income. AGI is farm rent $4,793 + interest $257 + pension $11,280 + taxable Social Security of $993 = $17,323. The $993 comes from provisional income of $16,330 + $10,656 = $26,986, which exceeds the $25,000 single base by only $1,986, and half of that is taxable. The head files as single because surviving-spouse (QSS) status requires a dependent child. The 2026 OBBBA-era $16,100 standard deduction, the $2,050 aged addition and the $6,000 senior deduction exceed AGI, so taxable income and tax are $0. Most models counted the $28,800 as taxable, which inflated provisional income enough to make 85% of Social Security taxable. The few models that got AGI right instead applied TCJA-sunset rules (a small standard deduction plus a personal exemption), and those rules left a small positive tax. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 4,328.39. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72. Reference depends on an unlisted input; output excluded from scoring (taxability of survivor_benefits)." us,scenario_062,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_062,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The single trap is that an 85-year-old disabled Floridian has exactly one non-MAGI route to Medicaid in the engine — SSI receipt — and SSI is $0 here because unearned income of about $37,600 (Social Security survivor benefits $21,312, private pension $11,280, farm rent $4,793, taxable interest $257) is more than three times the ~$12,000 SSI federal benefit rate, and $29,100 in bank assets exceeds the $2,000 SSI resource limit. The MAGI route is closed twice over: Florida did not adopt the ACA adult expansion, and MAGI income sits at 2.36 x FPL, above every MAGI threshold. That leaves medicaid_category = NONE. All six models manufactured a yes by importing a pathway outside the eligibility determination — medically needy share-of-cost spend-down, Medicare Savings Programs, or spousal/institutional protections — or by excluding Social Security survivor benefits from SSI-related countable income, where they count in full past the $20 general disregard." us,scenario_062,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_062,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Farm rent income is rental income, not net earnings from self-employment. With no reported self-employment income, the self-employment tax base is zero and the 92.35% adjustment and 15.3% rate do not apply." us,scenario_062,ssi,1,llm_error,asset_resource,False,,"SSI pays only after two means tests that this household fails twice over: Title II Social Security survivor benefits, private pension income, and farm rental income are all countable unearned income under 20 CFR 416.1121, offset only by the $20/month general income exclusion, and an individual with countable resources above $2,000 is ineligible regardless of income. Countable unearned income here is about $37,400 for the year ($21,312 + $11,280 + $4,793 + $257, less the $240 annual general exclusion) against a federal benefit rate under $12,000, and the $29,100 bank account is over fourteen times the resource limit, so the benefit is $0. The trap is that the head is categorically eligible as aged and disabled, which invites treating the task as computing a payment rather than running the income and resource screens; Florida also pays no optional state supplement to a recipient living independently." -us,scenario_064,child1_chip_eligible,8,llm_error,health_coverage,False,,"Wisconsin's BadgerCare Plus children's ceiling (306% FPL, the top of the CHIP-funded band above the 156% FPL Medicaid limit for ages 6–18) is applied to the tax unit's MAGI, not to the child's own income and not to income net of losses that MAGI still counts. Here MAGI is $97,295 of wages plus $20,000 of taxable 401(k) distributions plus $33,350 of pass-through income ($19,350 partnership/S-corp and $14,000 self-employment partnership), offset only by the $16,800 farm loss, the $3,000 annual capital-loss cap, and ~$2,827 of traditional 401(k)/IRA contributions — about $130,700, roughly 340% of the ~$38,400 2026 guideline for the five-person unit. Every wrong model landed in the 253%–300% range by dropping the retirement distribution and pass-through income, deducting the full $18,235 capital loss, deducting the single farm loss twice under its two variable names, or treating Roth contributions and ESI premiums as income reductions. The 12-year-old therefore clears neither the Medicaid limit nor the CHIP band above it, which is why both is_medicaid_eligible and is_chip_eligible are false." +us,scenario_064,child1_chip_eligible,10,llm_error,thresholds_rates,False,,"Child 1 is 12, so CHIP's age test (under 19) passes. The CHIP decision comes down to PolicyEngine's income test: it takes the tax unit's Medicaid MAGI, divides it by the federal poverty guideline for a 5-person unit, and compares that ratio with its Wisconsin CHIP child income limit. This household's income level is above that limit. The child is also above Wisconsin's child Medicaid limit (engine category NONE), so no coverage pathway remains. Every model that answered assumed the household sat inside a 300-306% FPL BadgerCare Plus/CHIP band and returned eligible, which is wrong under PolicyEngine's Wisconsin threshold." us,scenario_064,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,child1_medicaid_eligible,7,llm_error,thresholds_rates,False,,"The trap is Wisconsin's headline BadgerCare Plus number: the 300%/306% FPL figure that every wrong model cited is the combined Medicaid-plus-CHIP ceiling, while the Medicaid MAGI limit for a child aged 6-18 is 156% FPL, with CHIP covering the band above it. This household's MAGI is 3.22x FPL, which clears 156% by more than double and also clears the 306% ceiling, so no MAGI pathway opens and medicaid_category is NONE. The secondary trap is income arithmetic: the -$18,235 long-term capital loss is deductible only to $3,000 of net capital loss, so the losses cannot pull MAGI under any children's limit, and the tax household is five people, not six. The child's own $0 income is irrelevant — MAGI is measured on the tax household in which the child is a dependent." +us,scenario_064,child1_medicaid_eligible,9,llm_error,thresholds_rates,False,,"The engine puts this 5-person tax unit's MAGI at 3.22 × FPL. That is above Wisconsin's Medicaid-funded limit for children aged 6–18 (about 151% FPL, or 156% with the 5% disregard). It is also above the ~301%/306% ceiling of the CHIP-funded BadgerCare Plus band, so child1 qualifies through no Medicaid category (medicaid_category = NONE). Every model that answered Yes used the ~300% BadgerCare Plus figure as if it were the Medicaid limit. Most also put household MAGI below that figure, either by deducting the full capital loss (only $3,000 is allowed), by inflating the household size, or by skipping the income calculation entirely." us,scenario_064,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,child1_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC eligibility requires both a categorical status and an income test, and this household fails both. Categorically, the ""child"" category covers children from their first birthday only until their fifth birthday; Child 1 is 12 and falls outside every WIC category (pregnant, postpartum, breastfeeding, infant, child under 5). Independently, the income test caps countable household income at 185% of the federal poverty guideline, and this household's roughly $98,810 of income for a 5-person unit is far above that ceiling. Both gates independently force is_wic_eligible = False." us,scenario_064,dependent1_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent1_medicaid_eligible,28,llm_error,household_unit_or_filing_status,False,,"The trap is MAGI household composition for an adult tax dependent. Under 42 CFR 435.603(f)(2)(i), an individual other than a spouse or a child under 19 who is claimed as a tax dependent is exempted from the tax-filer household rule and has their household constructed under non-filer rules, so the 27-year-old's Medicaid household contains only himself and only his own income counts — $0, i.e. 0% FPL — which is why PolicyEngine returns True in the MAGI-based ADULT category. Two further guardrails matter: MAGI categories carry no asset or resource test (42 CFR 435.603(g)), and holding employer-sponsored insurance bars CHIP, not Medicaid. Every wrong model instead charged the head's ~$97k in wages plus 401(k) distributions and partnership income to the dependent, and several stacked on an SSI/ABD-style resource test using the household's $56,000 bank and $58,676 stock balances." -us,scenario_064,dependent1_medicare_eligible,5,llm_error,age_disability,False,,"Medicare's under-65 pathway is entitlement-based, not status-based: it requires 24 months of actual SSDI (or RRB disability) entitlement, or an ESRD/ALS diagnosis, and PolicyEngine's is_medicare_eligible turns on the age-65 threshold rather than the is_disabled flag. Dependent 1 is 27, works 20 hours a week, carries employer-sponsored insurance, and the household lists no Social Security or disability benefit income — with all unlisted inputs set to 0, there is no SSDI entitlement to start a 24-month clock and no ESRD/ALS fact. Every wrong model read the bare ""is disabled"" input as conferring Medicare entitlement, collapsing a disability characteristic into a benefit-receipt precondition it never satisfies." +us,scenario_064,dependent1_medicaid_eligible,30,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent1 is a 27-year-old with no income of their own: 20 usual weekly hours but $0 of wages. The engine puts dependent1's MAGI income level at 0.00 x FPL and places them in the MAGI-based ACA adult category, which returns eligible. That category has no asset test (42 U.S.C. 1396a(e)(14)(C)), and dependent1's employer-sponsored coverage and disability status do not block it. Every answering model instead charged dependent1 with the parents' income (the head's ~$97k wages plus 401(k) distributions and partnership income) or ran an ABD/SSI-style resource test on the parents' ~$115k of bank and stock assets, which turned a zero-income adult into a household that looked over the limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." +us,scenario_064,dependent1_medicare_eligible,6,llm_error,age_disability,False,,"A person under 65 qualifies for Medicare only after 24 months of entitlement to Social Security Disability Insurance (SSDI), or through end-stage renal disease (ESRD) or ALS. Having a disability flag alone does not qualify anyone. Dependent 1 is 27. The household lists no SSDI or other Social Security benefits and no months of SSDI receipt, and all unlisted numeric inputs are 0, so Dependent 1 has no path to Medicare. The five models that answered treated the ""is disabled"" flag as Medicare eligibility and skipped the SSDI-entitlement step. The sixth model gave no usable answer." us,scenario_064,dependent1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,dependent2_chip_eligible,8,llm_error,age_disability,False,,"The trap is the age boundary PolicyEngine applies to the CHIP child category: dependent2 is 18, and the derivation is explicit that this age is above the CHIP age ceiling, so no CHIP category attaches while the same person is assigned Medicaid category NONE, forcing is_chip_eligible = False. Every wrong model instead recited the generic statutory framing that CHIP covers children ""under 19,"" then treated the only remaining question as an income screen against Wisconsin BadgerCare Plus's 300-306% FPL children's ceiling. Once the age gate is applied as PolicyEngine applies it, the income comparison the models spent their reasoning on never decides the output, and the two missing answers produced no determination at all." -us,scenario_064,dependent2_medicaid_eligible,4,llm_error,thresholds_rates,False,,"Wisconsin never adopted the ACA adult expansion, so an 18-year-old's only MAGI route is the BadgerCare Plus children-under-19 category, capped at 306% FPL (301% plus the 5-percentage-point disregard), and this household's MAGI sits at 3.22 x FPL — above that cap. With SSI of 0 and no disability, pregnancy, or caretaker status, no non-MAGI category applies either, so the engine's category check returns NONE. Both substantive models named the right pathway but placed the household's MAGI below the children's limit: one asserted it without computing the ratio, the other computed a MAGI that dropped a $14,000 income component." +us,scenario_064,dependent2_chip_eligible,10,llm_error,health_coverage,False,,"Dependent 2 fails CHIP on two independent grounds. First, Dependent 2 is enrolled in employer-sponsored insurance, and CHIP excludes a child already covered by a group health plan. Second, household income is 321.5% of the federal poverty guideline, above Wisconsin's 306% CHIP limit. Every model that answered checked only the age-under-19 test and an income test it had underestimated (273-300% FPL), and none applied the rule that existing ESI coverage disqualifies the child. Dependent 2 is also not Medicaid-eligible, so no Medicaid pathway rescues the answer." +us,scenario_064,dependent2_medicaid_eligible,6,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent 2 is 18, so the only pathway open is the older-child MAGI category. It is income-tested against the five-person tax household's MAGI, which the engine puts at 3.22× FPL (322%). That is above Wisconsin's Medicaid limit for ages 6–18, above the roughly 306% BadgerCare/CHIP ceiling the models quoted, and above that ceiling plus the 5-point disregard (about 311%). No other pathway applies: there is no disability, pregnancy, caretaker status or SSI. Every model that answered Yes assumed or computed household MAGI below about 300–306% FPL and treated the CHIP ceiling as the Medicaid child limit. Its income measure fell short of the engine's 322%. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 0.00; the alternative is 1.00. The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household. Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." us,scenario_064,dependent2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,dependent2_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on the 2026 post-OBBBA parameter set applied to a mixed wage/pass-through household: the $13,899.29 deduction for the FLSA overtime premium, the $32,200 MFJ standard deduction with no personal exemptions, a QBI deduction of 20% of the $2,550 of partnership income remaining after the $16,800 farm loss ($510), the $3,000 capital-loss cap folded into the $19,800 loss deduction, and $3,200 of nonrefundable credits ($2,200 child credit plus two $500 other-dependent credits). AGI of $114,403.37 less $46,609.29 of deductions gives taxable income of $67,794.08 and $7,639.29 of tax, leaving $4,439.29. The wrong answers cluster into four families: omitting the overtime deduction (worth $1,667.91 at the 12% bracket), reverting to pre-TCJA law (personal exemptions, a 15% bracket, a $1,000 credit phased out at $110,000), double-counting the $14,000 self-employment partnership share already inside the $19,350 and computing QBI on the un-netted business income, and stripping employer-sponsored insurance premiums out of wages. A separate group produced a correct derivation and then submitted a number inconsistent with it." +us,scenario_064,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"The traditional-IRA above-the-line deduction adds up traditional_ira_contributions for every tax-unit member. That puts Dependent 1's $18 contribution, made with $0 compensation, into the joint return's $126.22 IRA adjustment, which conflicts with IRC §219(a)/(c) (only the contributor, and a spouse on a joint return, can deduct) and §219(b)(1)(B) (the deduction is capped at includible compensation); dropping it raises the value by about $2.19 to about $4,441.48.","The answer depends on 2026 law after the One Big Beautiful Bill Act (OBBBA). AGI is $114,403.37: wages after the 401(k) deferral, plus $19,350 of partnership income (which already includes the $14,000 of self-employment partnership income), plus the $20,000 distribution, minus the $16,800 farm loss and a capital loss capped at $3,000. From that come the $32,200 MFJ standard deduction, the new qualified overtime deduction for the $13,899 FLSA premium, and a $510 QBI deduction (20% × $2,550), giving $67,794.08 taxable and $7,639.29 of tax. Then $2,200 CTC plus $500 ODC for each of the two older dependents is subtracted, leaving $4,439.29. There is no SE tax because the farm loss exceeds the SE partnership income, and the DPAD is repealed. Most misses came from leaving out the overtime deduction, counting the $14,000 of SE income twice, taking the repealed DPAD, or assuming TCJA sunset rules. The three $4,441.56 answers differ only because they left Dependent 1's $18 IRA contribution out of the tax unit's $126.22 IRA adjustment. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,439.29; the corrected value is 4,441.46. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)." us,scenario_064,federal_refundable_credits,6,llm_error,thresholds_rates,False,,"The refundable child credit is a residual, not an automatic payment: the $2,000-per-child CTC first offsets federal income tax, and only the portion left unused is refundable, subject to the $1,700-per-child ceiling and 15% of earned income above $2,500. This MFJ household's ~$98,810 of income (wages $97,295, $20,000 of 401(k) distributions, $19,350 partnership income, against a farm loss and a $3,000-capped capital loss) produces federal income tax before refundable credits far above $2,000, so the single under-17 child's entire CTC is absorbed nonrefundably, and the 18- and 27-year-old dependents generate only the $500 nonrefundable other-dependent credit. EITC is zero because income is multiples of the MFJ one-child phaseout ceiling, and no refundable AOTC (no students), recovery rebate, or refundable payroll tax credit applies, leaving $0. Every wrong model reported either the full CTC or the $1,700 ACTC ceiling as if refundability were automatic rather than conditional on unused credit." us,scenario_064,free_school_meals_eligible,2,llm_error,taxable_income_or_deductions,False,,"The National School Lunch Program income test counts gross income by source — wages plus taxable retirement distributions — with no netting of farm losses, capital losses, or other tax-code offsets, and no deduction for premiums or medical expenses. PolicyEngine's countable income is $97,295 in wages plus $20,000 in taxable 401(k) distributions = $117,295 against an SPM-unit FPG of $38,680, a ratio of 3.03, far above the 130% free line ($50,284) and the 185% reduced-price line ($71,558). Wisconsin has no universal free meals and no SNAP/TANF categorical eligibility exists here, so the household lands in the PAID tier. The trap is importing tax-return netting (the -$16,800 farm loss and -$18,235 long-term capital loss) into a gross-income benefit test." us,scenario_064,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -529,92 +537,93 @@ us,scenario_064,spouse_chip_eligible,2,parse_contract_failure,missing_output,Fal us,scenario_064,spouse_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_064,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_064,ssi,14,prompt_ambiguity,age_disability,False,,"The case turns entirely on SSI's categorical gate, not on income: the engine records `is_ssi_aged_blind_disabled=False` for all five members, including the 27-year-old whose facts include a general ""is disabled"" flag, because SSI's disability determination is the separate `is_ssi_disabled` input, which is unlisted and therefore false under the prompt's unlisted-equals-false rule. Two readings of the fact list are defensible: (1) the stated ""is disabled"" fact establishes disability for every disability-conditioned program, including SSI, which yields roughly a full 2026 FBR of $994/month for Dependent 1; (2) SSI keys on its own disability input, distinct from the general disability flag that governs other provisions, so an unlisted `is_ssi_disabled` leaves no categorically eligible member and SSI is $0. Twelve of the fourteen models took the first reading and went straight to FBR arithmetic, deeming, and in some cases a value-of-the-one-third reduction or a Wisconsin supplement — all downstream of a gate that never opened. Two models returned nothing at all. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support." -us,scenario_064,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Wisconsin taxes this return on $102,144.25 — federal AGI of $114,403.37 less the sliding-scale standard deduction, which at that AGI has fallen to $8,759.12, less five $700 personal exemptions ($3,500). Two Wisconsin-specific rules decide the case: that sliding-scale deduction rather than a flat or federal-size amount, and the 4.40% second bracket that Wisconsin's 2025 budget act (Act 15) widened to roughly $69,400 of married-joint taxable income for 2026, so only about $32,700 reaches 5.30% and the schedule yields exactly $4,606.01. Nearly every model either used a flat or federal standard deduction or the superseded schedule that starts 5.30% near $38,000 — the bracket error alone adds about $292 — and several inflated AGI by stacking the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp figure that already contains it, or deducted the full $18,235 capital loss instead of the $3,000 federal limit. No Wisconsin nonrefundable credit applies: deductible medical expenses of $8,340 fall under the 7.5%-of-AGI floor of $8,580, no mortgage interest or property tax is listed, and the married-couple credit is zero because the spouse has no earnings." +us,scenario_064,ssi,19,prompt_ambiguity,age_disability,False,,"Every substantive wrong answer assumes that Dependent 1's generic ""is disabled"" flag satisfies SSI's aged/blind/disabled requirement. In the engine, is_ssi_aged_blind_disabled is False for all five members: Head is 48, Spouse 47, Dependent 1 27, Dependent 2 18 and Child 1 12. Because no member passes that test, SSI is $0 before any income, resource, deeming or living-arrangement test applies. The models' later steps (the full federal benefit rate, the one-third or PMV reduction, the Wisconsin supplement, the parental-deeming analysis) all compute a benefit for a person the engine does not count as SSI-disabled. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of the $11,928 federal benefit (five exactly); deepseek-v4-pro, deepseek-v4-pro-0813 and inkling also applied a one-third in-kind reduction the facts do not support." +us,scenario_064,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The engine starts from Wisconsin AGI of $114,403 and subtracts the MFJ sliding-scale standard deduction, which is phased down to $8,759 at this income, plus 5 personal exemptions of $700 each ($3,500). That leaves $102,144 of taxable income, taxed on the 2026 MFJ schedule. In that schedule the 4.4% bracket was widened by the 2025 budget and runs to about $50,500, so 5.3% applies only above that. No nonrefundable credit applies: the spouse has no earnings, so there is no married-couple credit, and itemized deductions fall below the standard deduction, so there is no itemized-deduction credit. Models went wrong in five ways: they overstated or understated WI AGI (double-counting partnership income, taking off the overtime premium, or adding back capital losses), mis-sized the phased standard deduction, dropped exemptions or treated them as credits, used the old narrower 4.4% bracket or stale rates, or invented nonrefundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,606.00; the corrected value is 4,599.62. PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67)." us,scenario_064,state_refundable_credits,4,llm_error,state_local_rule,False,,"Wisconsin's refundable individual income tax credits are the Homestead Credit, the Wisconsin EITC, the Farmland Preservation Credit, and the Veterans and Surviving Spouses Property Tax Credit; this household qualifies for none, so wi_refundable_credits and therefore state_refundable_credits are $0. The WI EITC is a flat percentage of the federal EITC (4%/11%/34% by number of qualifying children), and the federal EITC is $0 at $97,295 of wages plus partnership and 401(k) distribution income, so the WI EITC is $0. The Homestead Credit is barred both by its household-income ceiling near $24,680 and by the absence of any rent or property tax paid (only a mortgage balance is listed), and the Farmland Preservation Credit requires a farmland preservation agreement or certified zoning, which is unlisted and therefore false. The trap is arriving at an exact $0 rather than assuming that a household with a farm loss and Wisconsin residency picks up some small residual refundable credit." us,scenario_064,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_066,federal_refundable_credits,9,llm_error,credit_phaseout,False,,"The childless EITC phases in from the first dollar of earned income at 7.65% with no minimum-earnings floor and no earned-income ""kick-in"" threshold, so $520 of wages for a 40-year-old single filer (inside the 25–64 age window, no disqualifying investment income) yields exactly 0.0765 × $520 = $39.78 — the reference value. Every wrong answer turns on inventing a floor that does not exist in IRC §32 or denying the childless-worker category outright; the phase-in is linear and unrounded in PolicyEngine, so the correct answer is a one-line multiplication. The $14,000 bank balance is irrelevant because no interest is listed and the §32(i) investment-income limit ($12,200-scale in 2026) applies to income, not assets. The only non-zero refundable component is EITC: no children rules out refundable CTC, and no education expenses rule out the refundable AOTC." us,scenario_066,head_medicaid_eligible,9,llm_error,health_coverage,False,,"Virginia's ACA adult expansion covers this 40-year-old nondependent because annual MAGI is 0.03 times FPL, below the MAGI limit. Other means-tested health coverage does not disqualify the person, MAGI-based expansion eligibility has no asset test, and the stated $520 annual wages control instead of annualizing the separate hourly-rate and weekly-hours facts." -us,scenario_066,snap,39,llm_error,thresholds_rates,False,,"This one-person Virginia household is categorically eligible through TANF non-cash assistance, so the $14,000 bank balance never faces a resource test, and $43.33 of monthly earnings vanishes under the 20% earned-income deduction and the standard deduction, leaving $0 net income and entitlement to the full one-person maximum allotment. The remaining trap is that a calendar-year total is not twelve times one monthly figure: the reference blends $298 per month for nine months of 2026 with $304.68 for the final three, giving $3,596.04. The wrong answers fall into three buckets — outright $0 denials on a resource test that categorical eligibility waives (or, for claude-sonnet-5, reading the $520 annual wage as weekly), stale or invented maximum allotments ($292, $291, $294, $297, $300, $283, $275, $213), and the near-miss $3,576 from correctly using $298 but annualizing it flat. Three models added distinct errors on top: an ABAWD time limit despite 40 weekly work hours, an unexplained subtraction from a correctly derived $3,576, and a 30% benefit reduction applied against income that deductions had already zeroed." -us,scenario_066,state_refundable_credits,32,llm_error,state_local_rule,False,,"The trap is Virginia's refundable EITC election, which for tax year 2026 pays 20% of the federal EITC and is claimed in lieu of the nonrefundable 20% credit and the low-income credit. The federal childless EITC phases in at 7.65% from the first dollar of earnings with no minimum-earnings floor, so $520 of wages yields $39.78; the head is 40, satisfying the 25-64 childless age test, and a $14,000 bank balance is not investment income. Twenty percent of $39.78 is $7.96, paid in cash despite zero Virginia liability. The wrong answers split into two groups: those that denied Virginia has a refundable credit at all, tied refundability to tax liability, or invented an earned-income floor that zeroed the federal EITC; and those that reached the right structure but priced the credit at the superseded 15% rate, landing on $5.97 or $6." -us,scenario_067,dependent1_medicaid_eligible,18,llm_error,household_unit_or_filing_status,False,,"The trap is whose income counts for a 23-year-old tax dependent's Medicaid MAGI determination. PolicyEngine evaluates this person's own MAGI income level at 0.00 x FPL — his $10,800 in disability benefits is outside MAGI and he has no earnings — so Indiana's ACA adult expansion group (HIP: non-elderly adults, ages 19-64, up to 138% FPL, no resource test) covers him. All 18 wrong models instead charged the parents' $82,360 in wages and $4,942 pension to the dependent, or counted his non-MAGI disability benefits, or diverted to the SSI-linked disability pathway with its income and asset limits. The disability facts are a distraction: the ADULT expansion category resolves the case on age and MAGI alone." -us,scenario_067,dependent1_medicare_eligible,8,llm_error,age_disability,False,,"The trap is the gap between statutory Medicare and modeled Medicare. Under 42 U.S.C. 426(b) a disabled worker reaches Medicare only after 24 months of SSDI entitlement, and PolicyEngine implements no disability pathway at all: is_medicare_eligible turns solely on age relative to the 65 threshold, so a 23-year-old returns False. Dependent 1's inputs, is_disabled = true and disability_benefits = $10,800, drive SSI, the EITC disabled-dependent rules and medical-expense logic; neither input is read by the Medicare eligibility variable, and neither establishes SSDI entitlement, since disability_benefits also carries SSI, workers' compensation, state disability and private-policy income. Every wrong model substituted the real-world SSDI pathway, and waived its 24-month waiting period, for the age test that decides this output." -us,scenario_067,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"The 2026 computation is short: $82,360 wages plus $4,942 taxable pension = $87,302, less the 2026 MFJ standard deduction of $32,200 (OBBBA made the TCJA structure permanent, so there are no personal exemptions and no reversion to a 15% bracket), leaving $55,102 taxed at 10% to $24,800 and 12% above, for $6,116.24, less the $500 nonrefundable credit for other dependents that the disabled 23-year-old generates, for $5,616.24. Three traps separate this from the wrong answers: the 2026 parameter values themselves ($32,200 and the $24,800 breakpoint, not $30,000/$23,850 or sunset-era figures), the dependent's $10,800 disability benefit, which is that person's own income and never enters the filers' AGI, and the $500 other-dependent credit, which applies precisely because the dependent fails the CTC age test rather than being disallowed by it. The auto loan interest, mortgage balance, and medical expenses are inert here: the $16,621 of medical expenses above the 7.5% floor falls far short of the standard deduction, no mortgage interest amount is listed, and no auto loan interest deduction is taken. Every wrong answer traces to one or more of these four points." +us,scenario_066,snap,31,llm_error,thresholds_rates,False,,"Two steps separate the reference from the wrong answers. First, Virginia's broad-based categorical eligibility (TANF non-cash) makes the household categorically eligible despite its $14,000 in bank assets, so the standard $2,750–$3,000 SNAP resource limit does not apply. Second, $43.33 of monthly earnings is wiped out by the 20% earned income deduction and the standard deduction, so net income is $0 and the benefit is the full FY2026 one-person contiguous-US maximum of $298 a month, or $3,576 a year. Seven models denied eligibility by applying the plain asset limit. Most of the rest got eligibility and zero net income right but used a stale or guessed maximum allotment ($275–$294, or $300) instead of $298, and a few subtracted a spurious 30% income contribution or applied a nonexistent time limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 3,596.04 is regenerated as 3,576.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14))." +us,scenario_066,state_refundable_credits,34,llm_error,state_local_rule,False,,"The $7.96 is Virginia's refundable EITC at 20% of the federal EITC. The federal childless EITC phases in at 7.65% from the first dollar, so $520 × 7.65% = $39.78, and $39.78 × 20% = $7.96. The credit is paid whether or not there is any state tax liability. Models fell into two traps. Most answered $0 because they denied that a Virginia refundable EITC exists, called it nonrefundable, said it had expired, or tied it to having tax liability. The models that did find the credit used the older 15% rate, which gives $5.97 or $6." +us,scenario_067,dependent1_medicaid_eligible,20,prompt_ambiguity,household_unit_or_filing_status,False,,"PolicyEngine puts dependent1, age 23, in Indiana's ACA adult expansion group with a MAGI income level of 0.00×FPL. The parents' $82,360 in wages and $4,942 in pension never enter dependent1's Medicaid income measure, and dependent1's own $10,800 in disability benefits are not MAGI income. So dependent1 is far below the 138% FPL limit and eligible. All 20 models fell into the same trap in some form. Most added the parents' roughly $87K–$98K to a 3-person ""household MAGI"" of about 317% FPL. A few applied SSI-style or parental income and asset tests that the ACA adult category does not use, or counted the disability benefits as income that fails the limit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)." +us,scenario_067,dependent1_medicare_eligible,9,llm_error,age_disability,False,,"Dependent 1 is 23, so Medicare eligibility needs the disability pathway. Under 42 U.S.C. 426(b), that pathway requires 24 months of entitlement to Social Security disability benefits (or end-stage renal disease). The household lists only a generic 'disability benefits' amount and an 'is disabled' flag. It records no Social Security Disability Insurance entitlement, no months of SSDI receipt and no ESRD, and the prompt says every unlisted status is false. Every model treated 'disabled plus disability benefits' as enough for Medicare, so all nine answered Yes where PolicyEngine returns No." +us,scenario_067,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The reference comes from a simple chain under 2026 law, where OBBBA made the TCJA structure permanent. AGI is $87,302 (wages plus pension). The dependent's $10,800 disability benefits stay off the joint return. The $32,200 MFJ standard deduction applies, with no auto-loan interest deduction because the facts establish no qualified vehicle loan. That leaves taxable income of $55,102, taxed at 10% up to $24,800 and 12% above for $6,116.24, less the $500 nonrefundable credit for the 23-year-old disabled dependent. The models split into groups: some applied a TCJA sunset (personal exemptions, 15% bracket, lower standard deduction); others used stale 2024/2025 standard deductions and bracket edges, took the auto-loan deduction, added the dependent's benefits to AGI, or mishandled the $500 dependent credit." us,scenario_067,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The 23-year-old disabled household member does not generate a refundable Child Tax Credit because the CTC requires a qualifying child under age 17. With no CTC-qualifying child and no eligibility for the other refundable credit components, federal refundable credits equal $0." us,scenario_067,head_medicare_eligible,1,llm_error,age_disability,False,,"Ordinary age-based Medicare eligibility begins at age 65, and the 60-year-old head is below that threshold. No qualifying disability-based Medicare pathway or other exception is supplied, so the head is not Medicare eligible." us,scenario_067,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee-side payroll tax here is a single, closed computation: 6.2% Social Security plus 1.45% Medicare on the head's $82,360 of wages — $5,106.32 + $1,194.22 = $6,300.54 — with no Additional Medicare Tax (wages far below $200,000/$250,000) and no Indiana mandatory employee payroll tax. The trap has two halves: the wage base is exactly the one listed wage figure (the spouse's 50 weekly hours carry no listed wage and are 0 under the ""treat any unlisted numeric input as 0"" instruction; the head's $67 hourly rate imputes nothing extra), and non-wage receipts — the head's $4,942 taxable pension and the dependent's $10,800 disability benefits — are not FICA wages under IRC §3121(a). The second half is precision: the answer is a cents-exact product, so truncating components or ""rounding"" the total is itself the error. Every wrong model either inflated the base with unlisted or non-wage income, or named the correct rule and then submitted a number its own arithmetic does not produce." us,scenario_067,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_067,ssi,12,prompt_ambiguity,age_disability,False,,"SSI is an individual benefit gated by a categorical test that runs before any income arithmetic: the person must be 65+, blind, or disabled for SSI purposes. The head is 60 and the spouse 58, and the SSI disability determination is a distinct input (is_ssi_disabled) from the generic is_disabled flag listed for the 23-year-old — unlisted inputs are false under the prompt's own rule — so is_ssi_aged_blind_disabled is False for all three members and SSI is $0.00. Every wrong model skipped that gate entirely and jumped to the income test, subtracting the dependent's $10,800 in disability benefits (less a general income exclusion) from an assumed 2026 federal benefit rate. The spread in their answers ($824 to $12,156) is purely disagreement about the FBR and the exclusion mechanics, not about eligibility, which none of them tested. Developer adjudication (2026-09-05): the judge (gpt-5.6-sol) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading nine of the twelve models that paid SSI land within 10% of $1,368 (six exactly); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion." -us,scenario_067,state_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"Indiana's flat rate steps down by statute — 3.05% (2024), 3.00% (2025), 2.95% (2026), 2.90% (2027) — so the 2026 rate is 2.95%, and the only subtraction from federal AGI is $1,000 per exemption for the head, spouse, and the single dependent, because Indiana grants no standard or itemized deduction and its additional $1,500 exemption is confined to a dependent child under 19, or under 24 and a full-time student, which a 23-year-old non-student fails. The chain is $82,360 wages + $4,942 taxable pension = $87,302, less $3,000 of exemptions = $84,302, times 2.95% = $2,486.91; the dependent's $10,800 of disability benefits never enter the filers' AGI. Every wrong answer either carried a neighboring year's rate (3.05%, 3.00%, or 2.90%) or padded the exemption stack with a $1,500 dependent-child or disability add-on Indiana does not allow, and several fabricated Indiana deductions and credits outright." +us,scenario_067,ssi,17,prompt_ambiguity,age_disability,False,"The question lists Dependent 1 (age 23) as ""is disabled,"" and that person has no earnings, so there is no substantial gainful activity to disqualify them; under 42 U.S.C. §1382c(a)(1)(A) and (a)(3) and 20 CFR 416.202(a), that person is categorically eligible for SSI, yet the engine computes is_ssi_aged_blind_disabled=False and the derivation says no member is flagged disabled in the engine's inputs, so the displayed disability flag is not the input that feeds is_ssi_disabled (an input-mapping mismatch).","The deciding step is SSI's aged/blind/disabled gate. The engine evaluates is_ssi_aged_blind_disabled=False for all three members: the head (60), the spouse (58), and Dependent 1 (23), who has no SSI disability status in the engine's inputs. That makes SSI $0 before any income, resource or deeming test runs. All 17 models treated Dependent 1's listed disability as meeting the SSI disability criterion and subtracted the $900/month disability benefit, less the $20 general exclusion, from the 2026 federal benefit rate (FBR), which lands near $1,368. The outliers come from separate mistakes: an old or invented FBR, or applying the $20 exclusion once a year instead of every month. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_data_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)." +us,scenario_067,state_income_tax_before_refundable_credits,38,llm_error,state_local_rule,False,,"Indiana starts from federal AGI of $87,302 (wages plus taxable pension; the dependent's disability benefits are not in the filers' AGI) and has no standard deduction. It subtracts $1,000 personal exemptions for head, spouse, and dependent, for $3,000 total, and taxes the resulting $84,302 at the 2026 flat rate of 2.95%, giving $2,486.91. The $1,500 additional dependent-child exemption applies only to a child under 19 or a full-time student under 24, so it does not cover a 23-year-old disabled non-student. The wrong answers split into two groups. Some models used a stale or guessed rate (3.0%, 3.05%, 2.9%, 3.12%). Others added a $1,500 disabled-dependent exemption or a $1,500 dependent exemption, which takes exemptions to $4,500 or $5,000. A smaller group invented a standard deduction, pension deduction, or credits, or included the dependent's disability benefits in AGI." us,scenario_067,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_068,federal_income_tax_before_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"The case turns on two parameters the models had to get right simultaneously: PolicyEngine's educational_assistance is student financial aid that is excluded from AGI, so AGI equals the $34,083.53 of employment income with no addition for the $73,000 and no subtraction for the $8,389 ESI premium; and 2026 law gives a single standard deduction of $16,100 with no personal exemption and brackets of 10% to $12,400 then 12% to $50,400. That yields taxable income of $17,983.53 and tax of $1,240 + 12% × $5,583.53 = $1,910.02, with no nonrefundable credit available to a childless 31-year-old. Twelve models grossed up income by applying the IRC §127 $5,250 employer-tuition cap to the $73,000, pushing $67,750–$73,000 into the 22% bracket; twelve more excluded it correctly but priced the year with stale 2024/2025 parameters or a TCJA sunset that did not occur; the rest either zeroed the liability by netting the EITC or a phantom CTC against a before-refundable-credits measure, or submitted a number their own derivation contradicted." +us,scenario_068,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"PolicyEngine taxes only the $34,083.53 of listed wages. The separately listed $73,000 of educational assistance is not income, and the ESI premiums are not subtracted from the stated gross wages. Tax year 2026 runs under the permanent post-OBBBA TCJA structure: a $16,100 single standard deduction, 10% up to $12,400 and 12% up to $50,400. That gives $1,240 + 12% x $5,583.53 = $1,910.02, with no nonrefundable credits. The wrong answers fall into five groups: turning the educational assistance into $67,750-$73,000 of extra wages, assuming a TCJA sunset (personal exemption and 15%/25% brackets), subtracting the $8,389 ESI premium from wages, using stale 2024/2025 deductions and bracket edges, or wiping out the tax with an EITC or CTC this childless filer does not get." us,scenario_068,federal_refundable_credits,4,llm_error,credit_phaseout,False,,"The decisive rule is the 2026 EITC income limit for a single filer with no qualifying children. At $34,084 of wages, the head is above the childless EITC phaseout endpoint, and the household has no qualifying child or other facts generating any refundable federal credit, so every component is zero." us,scenario_068,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Maryland Medicaid eligibility for this 31-year-old requires qualification through a specific MAGI or non-MAGI category. The head's MAGI income is 2.14 times FPL, above the applicable adult MAGI limit, and the head qualifies through no non-MAGI pathway, so the eligibility output is 0." us,scenario_068,payroll_tax,14,llm_error,payroll_tax_base,False,,"PolicyEngine's payroll_tax here is simply the employee FICA pair on the head's $34,084 of employment income: 6.2% Social Security ($2,113.18) plus 1.45% Medicare ($494.21), with the small cent-level offset from the engine's float32 wage representation. The trap has three parts: the FICA wage base is gross wages with no reduction for the $8,389 employer-sponsored insurance premium (PolicyEngine applies payroll tax to employment_income, not to a section 125 net-of-premium wage), the $73,000 educational assistance is student financial assistance rather than employment compensation and never enters the FICA base, and Maryland imposes no mandatory employee payroll contribution in 2026 — the FAMLI employee contribution is not in effect and is not part of this output. Every wrong answer comes from adding one of those three phantom adjustments, or from arithmetic that contradicts the model's own stated components." -us,scenario_068,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference computes Maryland tax from an AGI of $34,083.53 that contains wages only: the $73,000 of educational assistance is outside AGI, and the $8,389 employer-sponsored insurance premium is not a further AGI subtraction. From that AGI it takes Maryland's 2026 standard deduction of $3,400 — the flat, indexed deduction that replaced the old 15%-of-Maryland-AGI formula capped near $2,550-$2,700 — plus the $3,200 personal exemption, leaving $27,483.53 taxed at 2%/3%/4% on the first $3,000 ($90) and 4.75% on the remaining $24,483.53 ($1,162.97), for $1,252.97. Three traps separate the wrong answers: eleven models grossed up AGI to roughly $101,834 or $107,084 by adding educational assistance above the IRC §127 $5,250 cap, five subtracted the ESI premium from AGI, and the largest group kept the right structure but used a stale $2,000-$2,900 standard deduction instead of $3,400." +us,scenario_068,state_income_tax_before_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"Maryland's 2025 Budget Reconciliation and Financing Act replaced the 15%-of-income standard deduction and its roughly $2,350–$2,900 single cap with a flat $3,350 single standard deduction. Most models that used wages only kept the repealed capped formula, which overstated taxable income by about $450–$1,000. A second group counted the $73,000 of educational assistance as income (all of it, or the part above $5,250), or subtracted the $8,389 employer insurance premium from the listed gross wages. Maryland AGI here is only the $34,083.53 of wages. The correct computation is $34,083.53 − $3,350 − $3,200 = $27,533.53 of taxable income, and $90 + 4.75% × $24,533.53 = $1,255.34." us,scenario_068,state_refundable_credits,2,llm_error,credit_phaseout,False,,"Maryland’s refundable EITC is tied to federal EITC eligibility, and a single 31-year-old filer with no qualifying children and $34,084 of wages is above the childless EITC phaseout ceiling. The federal EITC is therefore zero, producing no Maryland refundable EITC or other applicable refundable Maryland credit." -us,scenario_070,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The case turns on two 2026 parameters and one credit. First, 2026 keeps the permanent post-OBBBA structure: a $16,100 single standard deduction, no personal exemption, and a 12% bracket above $12,400 — so AGI of $36,269.67 leaves taxable income of $20,169.67 and tax of $2,172.36; the models that assumed a TCJA sunset (standard deduction near $8,000 plus a $5,000+ exemption and a 15% bracket) or a stale/projected deduction between $14,600 and $15,750 landed high. Second, the nonrefundable Saver's Credit applies at 10% of $1,924 of qualified contributions — the traditional 401(k) and IRA plus the Roth 401(k) and Roth IRA amounts, which count in the base — for $192.40, because AGI sits in the 10% rate band rather than the 50% band. Models that skipped the credit stopped at $2,172, models that took it at 50% or on traditional contributions only landed low, and several zeroed the liability with credits (CDCC, Credit for Other Dependents) that a childless single filer cannot claim." +us,scenario_070,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"Getting the reference takes two steps. First, apply the 2026 current-law parameters: TCJA rates were made permanent, the single standard deduction is $16,100 and the 10% bracket ends at $12,400. That gives $2,172.36 of tax on $20,169.67 of taxable income. Second, subtract the nonrefundable Saver's Credit at the 10% AGI tier on all $1,924 of retirement contributions, counting Roth 401(k) and Roth IRA with the traditional ones, which gives $1,979.96. The wrong answers fall into four groups: models that used a TCJA-sunset personal exemption and 15% bracket, models that used stale 2025-style standard deduction or bracket figures, models that left out the Saver's Credit entirely, and models that applied the wrong credit rate or dropped the Roth contributions from the credit base." us,scenario_070,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"The refundable Child Tax Credit requires at least one qualifying child; this household contains only a single 57-year-old adult. With no qualifying child and income above the childless EITC phaseout range, every refundable federal credit component is zero." us,scenario_070,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Illinois’s ACA adult Medicaid pathway requires income no greater than 138% FPL, while the head’s MAGI income is 2.27 times FPL. The head also qualifies through no non-MAGI pathway, so the Medicaid category is NONE and eligibility is No." us,scenario_070,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $38,000 of wages because the prompt lists $300 of health insurance premiums but never identifies them as a pre-tax cafeteria-plan deduction from FICA wages. The correct computation is $38,000 × 6.2% = $2,356 plus $38,000 × 1.45% = $551, totaling $2,907; the Additional Medicare Tax and Illinois employee payroll taxes are zero." -us,scenario_070,state_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"Illinois's calculation here has exactly three moving parts: federal AGI of $36,269.67 (wages net of the $1,485 traditional 401(k) deferral, plus $100 interest, less the $69 traditional IRA and $276 student loan interest above-the-line deductions), a single personal exemption, and the 4.95% flat rate, with no nonrefundable credit available to a renter with no dependents or education expenses. The separating parameter is the exemption: 35 ILCS 5/204 indexes it annually, and the 2026 amount is $2,925, while nearly every wrong answer used a prior-year vintage — $2,850 (2025) giving $1,654.29, $2,775 (2024) giving $1,658.00, or $2,425 (2023) — or invented a value. A second cluster misapplied the exemption's form or Illinois conformity: taxing full AGI with no exemption, stacking a nonexistent Illinois standard deduction on top of it, adding the student loan interest and IRA deductions back into the state base, converting the exemption into a nonrefundable credit, or double-counting it as both deduction and credit to force a $0 liability. Several models computed the right intermediate figure and then discarded it for an unexplained upward ""rounding"" adjustment." +us,scenario_070,state_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"Illinois tax here is 4.95% of federal AGI ($36,269.67, which already excludes the traditional 401(k) and deducts the IRA and student loan interest) less one personal exemption. For 2026 that exemption is inflation-indexed to $2,925, giving $33,344.67 of taxable income and $1,650.56 of tax. Most models got the AGI right but used a stale exemption: $2,850 (2025), $2,775 (2024), or $2,425 (pre-indexation). Each $75 of exemption shifts the tax by about $3.71. The rest broke the base itself: they left student loan interest out of AGI, subtracted health premiums, invented a standard deduction or renter's credit, applied the exemption twice, or dropped it entirely." us,scenario_070,state_refundable_credits,1,llm_error,credit_phaseout,False,,"Illinois's Earned Income Credit is calculated as a percentage of the federal EITC, so expanding the eligible age range does not create a credit when the federal EITC amount is zero. At this childless filer's income, the federal EITC has fully phased out, making the Illinois refundable credit zero." us,scenario_071,head_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Age 76 does not itself establish Medicaid eligibility, and this person qualifies through none of the Medicaid pathways represented by the engine. Their MAGI income is 2.28 times FPL, they receive no SSI, and the engine assigns medicaid_category = NONE; Medicare eligibility, housing assistance, and the mere availability of aged or spend-down programs do not create Medicaid eligibility." us,scenario_071,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to listed wages and other covered earnings, not to work hours by themselves. The household has no listed wage or self-employment income, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll tax bases are all zero, yielding payroll_tax of $0." -us,scenario_071,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"New York’s household credit is nonrefundable and therefore cannot enter the state_refundable_credits output. With no qualifying New York refundable credit, the aggregate of state refundable credits is $0." +us,scenario_071,state_refundable_credits,2,llm_error,state_local_rule,False,,"Both models counted the New York household credit as a refundable credit. Under Tax Law §606(b) that credit is nonrefundable: it can only reduce NY tax. Here NY tax is already zero, because the $20,000 pension/annuity exclusion for filers 59½ and older and the $8,000 single standard deduction bring NY taxable income to zero. That leaves only New York's refundable credits. There is no NY EITC or Empire State child credit, since the head has no earned income and no children. The refundable real property tax credit (IT-214) is also out, because household gross income counts all Social Security, and $31,320 + $5,222 = $36,542 is well over the $18,000 cap. Total NY refundable credits are $0." us,scenario_072,head_medicare_eligible,1,llm_error,age_disability,False,,"Standard age-based Medicare eligibility applies at age 65 or older and does not require an explicit enrollment or eligibility-status input. The 80-year-old head therefore qualifies based directly on the listed age, regardless of household income." us,scenario_072,payroll_tax,3,llm_error,payroll_tax_base,False,,"Employee Medicare tax applies at 1.45% from the first dollar of covered wages; the high-income threshold applies only to the 0.9% Additional Medicare Tax. On $20,000 of wages, employee Social Security tax is $1,240 and regular Medicare tax is $290, totaling $1,530 with no rounding adjustment." +us,scenario_072,snap,1,llm_error,categorical_eligibility,False,,"The household's SNAP gross income is $73,974: $20,000 in wages, $34,608 in Social Security, $19,200 in veterans benefits and $166 in dividends and interest. That is far above the gross income limit for a two-person household in Michigan in 2026, so PolicyEngine finds the household ineligible and the benefit is $0. The model applied no gross income test at all. It went straight to a net-income calculation, used a medical deduction that wiped out almost all of the income, and so produced a positive benefit." us,scenario_072,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility applies at age 65. The spouse is age 77 in 2026, so the age-based pathway yields Medicare eligibility without any separately listed enrollment or eligibility flag." -us,scenario_072,state_income_tax_before_refundable_credits,15,llm_error,state_local_rule,False,,"Michigan zeroes this household through the senior ""Tier 3"" standard deduction — $20,000 single / $40,000 joint applied against ALL income (wages included) for filers who have reached age 67, fully available in 2026 after the Lowering MI Costs Act (PA 4 of 2023) phase-in — which exceeds the household's Michigan AGI of $20,166 (wages $20,000 + qualified dividends $147 + interest $19, with Social Security subtracted and VA benefits never in federal AGI). Both filers are 67+, so Michigan taxable income is $0 and tax before refundable credits is $0. Every wrong model stopped at Michigan's personal exemptions (~$5,000–$6,500 each), left $4,200–$20,000 of taxable income, and multiplied by 4.25% (or by the 4.05% rate that applied only to tax year 2023), producing $71–$1,638. The single model that named the $40,000 deduction canceled it by subtracting gross Social Security benefits from it — a reduction Michigan applies only to military pay and railroad/military retirement benefits." -us,scenario_073,head_medicaid_eligible,7,llm_error,categorical_eligibility,False,,"Michigan’s expansion-adult pathway requires MAGI at or below the applicable limit, but the head’s MAGI is 1.60 times FPL and therefore exceeds it. SSDI income alone does not establish a disabled Medicaid category, and this head qualifies through no other pathway, yielding Medicaid category NONE." -us,scenario_073,head_medicare_eligible,17,prompt_ambiguity,age_disability,False,,"PolicyEngine's `is_medicare_eligible` is an age-based flag: it turns on when age reaches the 65 threshold (or when an explicit disability/ESRD determination flag is set), and receipt of Social Security disability income is not one of its triggers. The head is 57 and the prompt directs that any unlisted status input — including a disability determination — is false, so nothing sets the flag. Every wrong model substituted the real-world SSDI→Medicare pathway for the modeled rule and, on top of that, supplied the 24-month qualifying period from imagination: the facts give one benefit-year of SSDI with no entitlement onset date, and the ""constant throughout the tax-benefit year"" stipulation fixes status inside the year only, never establishing two prior years of SSDI receipt. The correct derivation is a single age comparison, 57 < 65, yielding 0. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_072,state_income_tax_before_refundable_credits,17,llm_error,state_local_rule,False,,"The head is 80 and the spouse is 77 in 2026, so they were born in 1946 and 1949. Both fall in Michigan's Tier 2 birth cohort (1946-1952) and are past age 67. That qualifies the couple for the Michigan standard deduction of $40,000 on a joint return, which applies against all income, wages included. Their only Michigan-taxable income is $20,000 of wages plus $166 of interest and dividends; Social Security is subtracted and veterans benefits are exempt. The $40,000 deduction wipes out that $20,166 entirely, so tax before refundable credits is $0. Every wrong model left a positive base: most subtracted only the ordinary personal exemptions, one used the wrong birth cohort, and one found the $40,000 deduction but shrank it by Social Security benefits." +us,scenario_073,head_medicaid_eligible,8,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI (42 CFR 435.603(e), which follows IRC 36B(d)(2)(B)) adds the untaxed portion of Social Security benefits back into income. That means the full $22,646 of SSDI counts, on top of the $3,108 pension. The head has no wages, so the traditional 401(k) and IRA contributions reduce nothing. MAGI is about $25,754, or 1.60 times FPL, which is above Michigan's 138% FPL limit for the adult expansion group. No other category applies: the head is 57, gets no SSI, is not pregnant, has no dependent children and has no disability flag, so the engine assigns medicaid_category NONE. Six models wrongly left untaxed SSDI out of MAGI or simply called the income 'low'. One subtracted 401(k) deferrals the head cannot make without wages. One treated SSDI receipt on its own as a disability route into Medicaid." +us,scenario_073,head_medicare_eligible,18,prompt_ambiguity,age_disability,False,,"The head is 57, so the age-65 route is out. The only other route is disability-based Medicare, which starts only after 24 months of Social Security disability benefits (42 U.S.C. 426(b)). The household lists this year's SSDI amount but not how long the head has received it. The prompt sets every unlisted number to 0 and every unlisted status to false, so the months of SSDI receipt count as 0 and the waiting period is not met. That leaves the head not Medicare eligible. All 18 models made the same mistake: they treated this year's SSDI as proof of disability-based Medicare, either by skipping the 24-month rule or by assuming it was already met. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_073,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earnings, not Social Security disability benefits or taxable private pension income. With no wage, salary, or self-employment income listed, the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components are all zero." -us,scenario_073,snap,37,llm_error,thresholds_rates,False,,"Michigan's broad-based categorical eligibility makes this TANF non-cash household exempt from the net income and asset tests, and its gross income sits at 165% of the poverty guideline, inside the 200% FPL BBCE screen. The computed allotment is genuinely zero — net income of $1,755.47/mo after the $200.70 standard deduction and the $190/mo excess medical deduction (($2,600 + $100)/12 − $35, with the $5,789 employer-sponsored premium excluded) produces a $526.50 expected contribution against the $298 one-person maximum — but eligible one- and two-person households are floored at the minimum allotment of 8% of the maximum, $23.84/mo, stepping to $24.37 with the October fiscal-year uprating, for $287.68 across calendar 2026. Nearly every model stopped either at a failed gross/net income test or at the zero computed allotment and never applied that minimum-benefit floor. The three that reached the floor valued it at a rounded $23–$25 per month for all twelve months instead of the uprated 8%-of-maximum figures." +us,scenario_073,snap,43,llm_error,categorical_eligibility,False,,"This household is categorically eligible for SNAP through Michigan's TANF non-cash (BBCE) rules: gross income is 165% of FPL, under the 200% limit, and categorical eligibility waives both the asset test and the 100% FPL net income test. Its net income of $1,755 gives a 30% contribution of $527, more than the $298 one-person maximum. Because the household is eligible, the one-to-two-person minimum allotment of $24 a month still applies, for $288 a year. Most models either applied the 100% FPL net income test, or a 130% gross test, and declared the household ineligible, or they stopped at a computed benefit of zero without applying the minimum. A few applied the minimum but used the FY2025 $23 or a rounded $25, and two built large medical deductions or a wrong maximum allotment that produced a sizeable benefit." us,scenario_074,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_074,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_074,head_medicaid_eligible,6,llm_error,health_coverage,False,,"Medicaid MAGI is not taxable income: 26 U.S.C. §36B(d)(2)(B) adds back to AGI all Social Security benefits excluded from gross income, so the head's $33,640 SSDI counts in full alongside the $384 taxable IRA distribution, producing 2.13 x FPL — roughly 213% of poverty against Louisiana's 138% FPL expansion-adult ceiling. Every wrong model shrank that base: four dropped SSDI entirely as ""non-taxable,"" one halved it using the §86 provisional-income 50% factor, and one asserted a disability pathway. The non-MAGI routes are also closed: the SSI-related aged/blind/disabled categories require SSI receipt (SSI = $0) or age 65+ (the head is 54), and the head is not a tax-unit dependent, so medicaid_category resolves to NONE." -us,scenario_074,head_medicare_eligible,24,prompt_ambiguity,age_disability,False,,"PolicyEngine's Medicare eligibility test keys on the age threshold of 65, and the head is 54, so the value is False. Every wrong model converted the input line ""Social Security disability income: $33,640"" — a dollar amount for the social_security_disability income variable — into a disability status flag plus a completed 24-month SSDI entitlement period, neither of which is stated; the prompt sets unlisted status inputs to false and forbids inferring unlisted health coverage, and a single tax-benefit year of income cannot establish a 24-month prior entitlement history. Reading that income line as an age-independent eligibility switch is the one step that separates the reference No from every Yes here. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_074,head_medicare_eligible,27,prompt_ambiguity,age_disability,False,,"Under-65 Medicare eligibility through disability requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)). PolicyEngine tracks the months of SSDI receipt as a separate input. The household facts list only a positive SSDI amount for the year, with no receipt duration, and the prompt says to treat unlisted numeric inputs as 0 and unlisted statuses as false. It also says facts are constant only across the one tax-benefit year, which covers at most 12 months of receipt. So the disability pathway fails, and eligibility falls back to the age-65 test, which the 54-year-old head does not meet. All 27 models treated current SSDI income as proof that the 24-month waiting period was already met and answered Yes. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_074,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not to Social Security disability benefits, pension income, IRA distributions, or tax refunds. The household has no listed wages, and the prompt requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and the resulting payroll tax is $0." -us,scenario_075,federal_income_tax_before_refundable_credits,26,llm_error,thresholds_rates,False,,"The case turns on 2026 federal parameters as they actually stand after OBBBA: the TCJA rate schedule and deduction structure persist, so the single standard deduction is $16,100, there are no personal exemptions, and the brackets run 10% to $12,400, 12% to $50,400, 22% to $105,700. The head is 62, which rules out both the age-65 additional standard deduction and the $6,000 senior deduction, and disability adds no deduction while the Credit for the Elderly or Disabled is fully phased out at this income. Taxable income is $109,286.13 - $16,100 = $93,186.13, of which $6,225.76 of long-term gain plus qualified dividends stacks above $86,960.37 of ordinary income and falls entirely in the 15% band, giving $13,843.28 + $933.86 = $14,777.15. The dominant failure was assuming a TCJA sunset with reinstated personal exemptions and 15%/25% rates; the rest used stale 2025 deduction or bracket values, invented age/disability deductions and credits, or submitted numbers their own worksheets never produced." +us,scenario_075,federal_income_tax_before_refundable_credits,27,llm_error,thresholds_rates,False,,"The correct 2026 computation uses the TCJA structure that OBBBA made permanent. That means a $16,100 single standard deduction, no personal exemption, and single brackets of 10% to $12,400, 12% to $50,400 and 22% to $105,700. This gives $13,843.28 of ordinary tax on $86,960, plus 15% on $6,226 of LTCG and qualified dividends ($933.86), for $14,777.15. No nonrefundable credit applies: at age 62 there is no senior deduction or age-65 add-on, and the Credit for the Elderly or Disabled phases out completely at $109,286 AGI. The wrong answers fall into four groups. Models that assumed the TCJA sunset (personal exemption and 10/15/25 brackets) landed near $17.5k-$18.9k. Models using the 2025 or guessed standard deductions and brackets landed near $14.9k-$15.2k. Two models computed $14,777 and then submitted a different number. The rest invented age or disability deductions or credits." us,scenario_075,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_075,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not establish Florida Medicaid eligibility. The head qualifies through none of Florida's Medicaid pathways, so the engine assigns medicaid_category NONE; the 6.85× FPL income level also rules out any applicable MAGI category." -us,scenario_075,head_medicare_eligible,13,llm_error,age_disability,False,,"Medicare eligibility turns on the age-65 threshold; the under-65 disability route (42 U.S.C. § 426(b)) is not conferred by a disability status flag but by 24 months of actual entitlement to Social Security disability insurance benefits (or an ESRD/ALS diagnosis), and PolicyEngine's Medicare eligibility variable accordingly keys on age >= 65. This household head is 62, receives $0 Social Security disability income (every unlisted input is 0), and lives on $12,656 of wages plus $90,020 of taxable private pension — no SSDI entitlement exists to start the 24-month clock, and no ESRD or ALS fact is listed. Every wrong model read the bare ""is disabled"" line — a flag the benchmark uses for SSI, SNAP and EITC disability rules — as a self-executing Medicare pathway, and several compounded it by inventing an age-62 Medicare threshold out of the Social Security early-retirement age. The correct derivation is a single age comparison, 62 < 65, yielding not eligible." +us,scenario_075,head_medicare_eligible,15,llm_error,age_disability,False,,"Under age 65, Medicare disability eligibility depends on 24 months of Social Security Disability Insurance entitlement, or on end-stage renal disease. A general ""is disabled"" flag does not qualify anyone. This head is 62 and has no Social Security disability benefits: there are no months of SSDI receipt and no ESRD. Their income is wages, pensions, dividends and capital gains. So the age test (65+) fails and neither non-age pathway applies. Every model instead treated the disability flag, or age 62, as enough for Medicare." us,scenario_075,payroll_tax,4,llm_error,payroll_tax_base,False,,"Employee payroll tax applies only to the $12,656 of wages; pension, dividends, and capital gains are outside the employee FICA base. PolicyEngine separately computes $784.69 of employee Social Security tax and $183.52 of employee Medicare tax, totaling $968.21, with no Additional Medicare Tax or Florida employee payroll tax." us,scenario_076,child2_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC serves children only through their fifth birthday and generally requires household income at or below 185% of the federal poverty guideline. The model also inverted the income comparison: $173,820 is far above 185% of the poverty guideline for a three-person household, not 184% of the guideline." -us,scenario_076,federal_income_tax_before_refundable_credits,36,llm_error,taxable_income_or_deductions,False,,"2026 runs on the OBBBA-permanent parameters: a $24,150 head-of-household standard deduction, no personal exemptions, a $2,200-per-child CTC that is unreduced below the $200,000 HoH phaseout, and an $800 QBI deduction on the $4,000 of rental income. AGI is the unadjusted $164,220 — the $6,480 employee ESI premium is not an AGI adjustment, the $9,600 of child support is excluded from gross income, and the $420,000 mortgage balance carries no listed interest, so the mortgage-interest deduction is zero and itemized deductions fall below the standard deduction. Tax on $139,270 of taxable income is $24,211.80, and because liability far exceeds the credits, the entire $4,400 CTC plus the $200 CDCC is absorbed as nonrefundable credit, leaving $19,611.80. The wrong answers cluster on three substitutes: sunset-era personal exemptions with a pre-TCJA standard deduction, an ESI or imputed-mortgage-interest reduction to income, and a CTC that is zeroed by phaseout or diverted to the refundable ACTC." +us,scenario_076,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"The correct 2026 chain under post-OBBBA law runs as follows. AGI is $164,220: full $160,000 wages plus $4,000 rent plus $220 interest, with child support excluded and no reduction for the ESI premium. Subtract the $24,150 HoH standard deduction and the $800 QBI deduction on rental income to get taxable income of $139,270. The 2026 HoH brackets (10% to $17,700, 12% to $67,450, 22% to $105,700, 24% above) give $24,211.80 of tax. Subtracting the full $4,400 CTC ($2,200 x 2) and the $200 CDCC leaves $19,611.80. Models went wrong in several ways. Many assumed TCJA had expired, so they applied personal exemptions, 10/15/25 brackets and a phased-out CTC. Others used 2025 bracket thresholds or $2,000-per-child CTC. Some cut wages by the ESI premium or omitted the rental QBI deduction. Some treated most of the CTC as refundable even though tax liability far exceeds the credit, and a few imputed mortgage interest from a loan balance." us,scenario_076,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is not an entitlement of $1,700 per child; it is only the portion of the $2,200-per-child CTC that survives after the credit offsets tax liability, and it is further capped at 15% of earned income over $2,500 and at $1,700 per child. This head-of-household filer has AGI of $164,220 (wages $160,000 + rents $4,000 + interest $220; child support is not taxable), below the $200,000 HoH phase-out threshold, so the full $4,400 CTC is available — but taxable income near $140,000 after the 2026 HoH standard deduction and the QBI deduction produces federal tax well above $25,000, so the entire CTC is absorbed nonrefundably and nothing spills into refundable_ctc. EITC is zero on this income, and there is no AOTC, recovery rebate, or refundable payroll tax credit here, so federal refundable credits are $0. Every wrong model computed the ACTC cap and then reported the cap itself as a payment, skipping the tax-liability offset that defines the refundable portion." us,scenario_076,payroll_tax,12,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are calculated on the full $160,000 of wages in the computation trace; the listed $6,480 employer-sponsored insurance premium is not subtracted from the payroll-tax base. This yields $9,920 of Social Security tax plus $2,320 of Medicare tax, for $12,240 total, with no Additional Medicare Tax or mandatory Idaho employee payroll tax." us,scenario_076,self_employment_tax,2,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to net earnings from a trade or business, and the prompt lists no self-employment income. The $4,000 of rental income remains rental income and does not create a self-employment tax base, so the computation yields $0." -us,scenario_076,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Idaho's 2026 individual income tax is a 5.3% flat rate (HB 40 of 2025 cut it from 5.695%, itself down from 5.8%) applied only to taxable income above an indexed zero-rate bracket of $9,840 for head-of-household filers. Idaho taxable income here is $138,270: federal AGI of $164,220 (child support excluded) less the $24,150 head-of-household standard deduction, Idaho's $1,000 household and dependent care expense deduction, and the $800 §199A deduction on $4,000 of rental income that flows through Idaho's conformity to federal taxable income. So 0.053 × ($138,270 − $9,840) = $6,806.79, with no nonrefundable credit netted out of this output. Every wrong answer drops at least one of these four pieces — most keep the repealed 5.695% or 5.8% rate, tax the first $9,840, skip the Idaho household and dependent care deduction, or subtract an Idaho child tax credit this output does not net out." -us,scenario_076,state_refundable_credits,36,llm_error,state_local_rule,False,,"Idaho's grocery credit (Idaho Code §63-3024A) is a refundable, universal per-person credit claimed by every full-year resident and their dependents with no income test and no phaseout, and House Bill 231 (2025) raised the base amount to $155 per person — with the separate senior increment eliminated — effective for tax years beginning on or after January 1, 2025. Three full-year residents (head plus two children, all under 65) therefore generate 36 qualifying months and 3 × $155 = $465, the household's entire state refundable credit total. Eighteen models zeroed the output by asserting either that Idaho has no refundable credit or that the grocery credit is nonrefundable or income-phased at $160,000 of wages — none of which exists in Idaho law. Most of the remainder carried a stale per-person amount: $120 (the 2022–2024 rate under HB 509) or $100 (the pre-2022 rate); two substituted Idaho's $205-per-child credit under §63-3029L, which is nonrefundable." -us,scenario_077,federal_income_tax_before_refundable_credits,28,llm_error,thresholds_rates,False,,"The case turns on the 2026 single standard deduction of $16,100 applied to AGI of $23,006 — wages of $26,006 less the $3,000 IRC §1211(b) capital-loss limitation on net losses of $7,968 — leaving taxable income of $6,906 entirely inside the 10% bracket for $690.62. Two traps separate the reference from the wrong answers: models that assumed a TCJA sunset for 2026 (an ~$8,300 standard deduction plus a revived personal exemption) or reused stale/guessed deduction amounts ($14,600, $15,000, $15,350, $15,400, $15,450, $15,750, $8,150), and models that subtracted the $8,389 employer-sponsored insurance premium from wages PolicyEngine already treats as post-premium W-2 employment income. A third group ran the correct chain to $691 and then submitted a different number." +us,scenario_076,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Idaho's 2026 tax is a flat 5.3%, but it applies only to taxable income above a zero-rate amount. For a head-of-household filer that amount is $9,622, the joint-level figure. Idaho taxable income also comes out lower than most models assumed: federal AGI of $164,220, minus Idaho's $1,000 household and dependent care subtraction, minus the $24,150 standard deduction, minus the $800 QBI deduction on rental income, gives $138,270. The result is ($138,270 − $9,622) × 5.3% = $6,818.34, with no further nonrefundable credit subtracted. Most models made at least one of these mistakes: they taxed all taxable income without the zero-rate amount, used the single-level threshold (about $4,900), used the old 5.695% or 5.8% rate, missed the $1,000 or $800 deductions, or subtracted a $410 Idaho child tax credit that the reference computation does not include. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). Idaho's $205 child tax credit applies only to taxable years beginning in 2018 through 2025, so it does not reduce 2026 tax; the judge's missing-credit hypothesis fails. The reference applies the Idaho zero-rate threshold held at its published 2025 amount. Reference affirmed (Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601)." +us,scenario_076,state_refundable_credits,39,llm_error,thresholds_rates,False,,"The whole $465 is Idaho's refundable grocery credit at the 2026 amount of $155 per person, times 3 household members (head plus two children), with 12 qualifying months each. The credit has no income test, so the household's $160,000 in wages does not reduce it. The wrong answers split three ways. Most models applied a stale per-person amount: $120 (the 2023-2025 level, giving $360), $100 (the pre-2023 level, giving $300) or an invented $170. Others declared the grocery credit nonrefundable or income-limited and answered $0. Two models swapped in Idaho's nonrefundable $205 child tax credit." +us,scenario_077,federal_income_tax_before_refundable_credits,29,llm_error,thresholds_rates,False,,"The correct path is AGI = $26,006 wages minus the $3,000 capital-loss limit = $23,006. Under current law after the OBBBA, the 2026 single standard deduction is $16,100 and there is no personal exemption, so taxable income is $6,906. All of it falls in the 10% bracket, which gives about $691, and no nonrefundable credits apply. The models went wrong in four ways. Most used an outdated standard deduction: $15,000, $15,350, $15,400, $15,450 or $15,750. Others applied the TCJA-sunset rules ($8,300 standard deduction plus a $5,300 exemption). Several also subtracted the $8,389 ESI premium from the stated wages, which the reference does not do. A group of models asserted zero tax, as if the standard deduction exceeded AGI or unnamed credits wiped out the liability." us,scenario_077,federal_refundable_credits,13,llm_error,credit_phaseout,False,,"The only potentially applicable refundable credit is the childless EITC, and its phaseout uses the greater of earned income or adjusted gross income. The $26,006 of wages is earned income; employer-sponsored insurance premiums do not reduce that amount, and at this earnings level the childless EITC is fully phased out. With no qualifying child or student and no applicable rebate or payroll credit, every refundable component is zero." us,scenario_077,head_medicaid_eligible,10,llm_error,taxable_income_or_deductions,False,,"Louisiana's expansion pathway requires MAGI at or below the applicable adult limit, but the engine computes the head's MAGI income level as 1.44 times FPL, so that pathway fails. The listed employer-sponsored insurance premium is not established as a pre-tax payroll deduction and cannot simply be subtracted from wages; after the applicable MAGI computation, the head qualifies through no Medicaid category." us,scenario_077,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_077,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are computed from the applicable payroll-tax wage base, and the reference calculation applies the 6.2% and 1.45% employee rates to the full $26,006 of wages. The employer-sponsored insurance premium input does not reduce that wage base in this calculation, producing $1,612.38 of Social Security tax plus $377.09 of Medicare tax, or $1,989.47." us,scenario_077,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_077,snap,5,llm_error,categorical_eligibility,False,,"Louisiana’s broad-based categorical eligibility gross-income screen does not itself guarantee a SNAP allotment; the household must still produce an eligible benefit under the net-income and allotment calculations. SNAP countable income does not subtract capital losses, a mortgage balance is not a shelter expense, and a non-elderly, nondisabled household cannot deduct medical expenses. With no listed deductible shelter payment and countable wages of $26,006, this one-person household receives $0." -us,scenario_077,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Louisiana's flat-tax reform taxes federal AGI at a single 3% rate after one combined standard deduction/exemption of $12,500 for single filers, and that deduction is inflation-indexed beginning in tax year 2026, where it is $12,835. Federal AGI here is $26,006 of wages less the $3,000 net capital-loss limitation (the −$7,964 long-term and −$4 short-term losses), or $23,006.18, so Louisiana taxable income is $10,171.18 and the tax is $305.14. The modal wrong answer of $315.18 is exactly the result of freezing the deduction at the unindexed $12,500; the remaining wrong answers either resurrect Louisiana's repealed pre-flat-tax structure (graduated 1.85%/3.5% brackets, a $4,500 personal exemption, the federal income tax deduction), subtract the $8,389 ESI premium from already-taxable wages, or assert that Louisiana levies no income tax at all." +us,scenario_077,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"Louisiana's 2025 reform (Act 11 of the 2024 Third Extraordinary Session) replaced the graduated brackets, the $4,500 combined personal exemption and the federal-income-tax deduction with a flat 3% rate and a $12,500 single standard deduction. That deduction is indexed for inflation from 2026, so it is $12,835 in 2026. The reference taxes $23,006.18 AGI − $12,835 = $10,171.18 at 3%, which gives $305.14. Most models used the unindexed $12,500 deduction and got $315.18. The others applied the repealed pre-2025 exemptions and brackets, wrongly took the ESI premiums out of AGI, stacked extra deductions on top, or claimed Louisiana has no income tax." us,scenario_077,state_refundable_credits,4,llm_error,state_local_rule,False,,"The models applied Louisiana’s former refundable earned income tax credit, equal to 5% of the federal EITC, to tax year 2026. Louisiana no longer provides that refundable credit in 2026, so the Louisiana refundable-credits computation returns zero regardless of an estimated federal EITC amount." us,scenario_077,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_078,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The trap is the 2026 SALT deduction. Under OBBBA the cap is $40,400 for 2026 and phases down only above $505,000 of MAGI, so this filer's entire $36,233.77 of state and local taxes — the $26,460 of real estate taxes plus the household's own $9,773.77 Maryland state-and-county income tax liability — is deductible alongside $28,210 of mortgage interest, for $64,443.77 of itemized deductions against a $16,100 standard deduction. AGI is the $198,505 of wages alone; the $1,632 state tax refund is not added. Taxable income of $134,061.22 through the 2026 single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, 24% to $201,775) gives exactly $24,772.69. The wrong answers cluster into three shortcuts: the repealed $10,000 SALT cap (~$31,4xx), a SALT deduction limited to listed property taxes with the refund added to AGI ($27,510.08), and a TCJA-sunset schedule with a resurrected personal exemption and 10/15/25/28 rates (~$27,000–$28,500)." +us,scenario_078,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The key step is the SALT deduction. It combines Maryland state income tax, Maryland county (local) income tax and the $26,460 real estate tax, and together they exceed the 2026 OBBBA cap of $40,400, so SALT is exactly $40,400. Add $28,210 of mortgage interest for $68,610 in itemized deductions. AGI of $200,136.91 minus $68,610 leaves taxable income of $131,526.91, and the 2026 TCJA-permanent single brackets (10/12/22/24%) give $24,164.46. The wrong answers fall into four groups: (1) the expired $10,000 SALT cap (~$31.4–31.5k); (2) treating TCJA as sunset, which restores personal exemptions and the 15/25/28% brackets (~$27–28.5k); (3) counting only real estate tax in SALT ($27,510.08); and (4) adding state income tax but leaving out county income tax, which keeps SALT below the cap (~$25.85k). Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." us,scenario_078,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_078,local_income_tax,1,llm_error,state_local_rule,False,,"The requested local_income_tax output is limited to the separately modeled local taxes listed in the prompt: NYC, Philadelphia, Kansas City, and St. Louis. Maryland county income tax is not included in this output, so a Maryland household outside those modeled jurisdictions has a value of zero." us,scenario_078,payroll_tax,26,llm_error,payroll_tax_base,False,,"The whole case reduces to two federal components: 6.2% on the 2026 Social Security taxable wage base of $184,500 ($11,439) plus uncapped 1.45% Medicare on the full $198,505 ($2,878.32), for $14,317.32. Wages fall $1,495 short of the $200,000 single-filer Additional Medicare Tax threshold, so that piece is zero, and Maryland imposes no mandatory employee-side payroll tax for 2026 — FAMLI employee contributions do not begin in that year. The wrong answers fall into three groups: stale or invented Social Security wage bases ($137,700, $168,600, $176,100, $177,600, ~$181,800), fabricated Maryland FAMLI employee contributions of roughly 0.45% of wages, and submissions whose numeric value contradicts the model's own correct arithmetic." -us,scenario_078,state_income_tax_before_refundable_credits,31,llm_error,taxable_income_or_deductions,False,,"The case turns on Maryland's itemized deduction: MD itemized deductions equal federal Schedule A less state and local income taxes only, so the $28,210 of mortgage interest and the full $26,460 of real estate taxes survive (2026 federal SALT cap is $40,000, above the $33,396 of SALT here), giving $54,670 against a maximum single standard deduction of $3,400. MD AGI also subtracts the $1,632 state tax refund from federal AGI of $200,137, and the personal exemption is $0 for a single filer with FAGI above $150,000, leaving taxable income of $198,505 − $54,670 = $143,835. The bracket schedule then gives $20 + $30 + $40 + 4.75%×$97,000 + 5%×$25,000 + 5.25%×$18,835 = $6,936.34 — the top applicable rate is 5.25%, not 5.5%. Most wrong models took the standard-deduction path (taxable ≈ $195,000, tax ≈ $9,700–$9,900); the near-misses failed on one sub-rule apiece — a phantom SALT or Pease-style cap on the deduction, a non-existent exemption, the un-subtracted refund, or added county tax." +us,scenario_078,state_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"Maryland AGI subtracts the $1,632 state tax refund, leaving $198,505. The filer itemizes $28,210 of mortgage interest plus the full $26,460 of property tax, which is under the 2026 $40,400 SALT cap. Maryland's phase-out cuts itemized deductions by 7.5% of federal AGI over $200,000, a $10.27 reduction that leaves $54,659.73. The personal exemption is $0 for single filers above $150,000, so taxable income is $143,845.27. The 2%/3%/4%/4.75%/5%/5.25% brackets give $6,936.88 with no county tax. The wrong answers split into five errors: taking the $2,400-$3,350 standard deduction instead of itemizing, capping property tax at $10,000, skipping the refund subtraction, keeping a partial exemption, or adding county piggyback tax." us,scenario_078,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_079,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The operative rule is categorical, not income-based: the head's blindness, disability, and $7,560 of SSDI leave countable income below the SSI federal benefit rate, producing $5,286 of SSI, and Arizona is a 1634 state where SSI receipt automatically confers Medicaid with no MAGI test. Both models ran an income/coverage screen instead of the SSI-recipient category, and neither recognized that SSI here is a computed output of the listed facts rather than an unlisted input. The screen they ran also points to eligibility rather than away from it — 0.35 x FPL is far under Arizona's 138% FPL expansion threshold — so 'No' is unreachable by either the categorical or the MAGI route." -us,scenario_079,head_medicare_eligible,21,prompt_ambiguity,age_disability,False,,"PolicyEngine's Medicare eligibility variable resolves on a single condition — age at least 65 — so Head at 58 returns False no matter what disability status or SSDI income the household reports. Every wrong model reached for the real-world under-65 pathway, which is not a disability flag but 24 months of SSDI entitlement; the household lists an annual SSDI amount and disability booleans, never a duration of prior receipt, and the prompt directs that unlisted numeric inputs be treated as 0. The models that named the 24-month rule then supplied the elapsed months themselves, several by misreading ""treat status as constant throughout the tax-benefit year"" as covering the prior two years, and the models that skipped the rule collapsed SSDI receipt straight into Medicare entitlement. The separating step is recognizing that the age-65 test governs and that the disability pathway's qualifying period is an unlisted fact worth zero months. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_079,head_medicare_eligible,24,prompt_ambiguity,age_disability,False,,"Medicare eligibility here has two routes: reaching age 65, or completing 24 months of Social Security disability entitlement (42 U.S.C. 426(b)). PolicyEngine checks the second route against a months-of-SSDI-receipt input. The household lists SSDI income but no months of receipt, and the prompt says to treat unlisted numeric inputs as 0 and not to infer unlisted facts, so the waiting period is not met. Even if Head received SSDI for all of 2026, that adds up to only 12 months. With Head aged 58, neither route applies, but all 24 models treated SSDI receipt or the disability flag as enough for Medicare, and several simply assumed the 24-month wait had already passed. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_079,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_079,snap,39,llm_error,taxable_income_or_deductions,False,,"The case turns on countable income: both adults are blind and disabled with only $630/month of SSDI, so PolicyEngine pays $881/month of SSI (the 2026 couple FBR less $610 of countable SSDI after the $20 general income exclusion) and counts that SSI as SNAP unearned income, making gross income $1,511/month rather than $630. Deductions total $354/month (standard deduction plus the elderly/disabled medical deduction on the $732 of annual medical and over-the-counter expenses), leaving net income of $1,157; no shelter deduction applies because only a mortgage balance is listed, not a housing payment. The benefit is the maximum allotment less 30% of net income: $546 − $347.10 = $198.90 for January–September 2026 and $558.24 − $345.60 = $212.64 after the October 2026 uprating, so the tax-year total is 9 × $198.90 + 3 × $212.64 = $2,428.02. Nearly every wrong model either dropped the SSI from countable income (landing near $5,000), counted it but allowed only ~$230 of deductions with a flat 12-month allotment (landing near $1,950), or wrongly zeroed the household on vehicle equity, absent shelter costs, or SSI receipt." +us,scenario_079,snap,46,llm_error,state_local_rule,False,,"The reference makes two moves. First, it counts the couple's $881/month of SSI (the couple benefit reduced by $610 of countable SSDI) together with the $630 of SSDI, for $1,511 of gross unearned income. Second, it applies Arizona's $145 standard medical deduction for elderly/disabled households whose medical costs exceed $35/month, rather than the $26 actual excess ($61 − $35). No model applied the $145 Arizona deduction. Models that counted SSI came out about $36/month too high ($1,932–$2,136 a year). Models that dropped SSI inflated the benefit to $3,400–$6,600, and the zero answers invented asset, shelter, or SSI cash-out barriers that don't apply in Arizona." us,scenario_079,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The spouse is blind and disabled with zero countable income and $1,045 in resources, so she receives SSI ($5,286 in the household trace); Arizona is a 1634 state, where SSI receipt confers automatic Medicaid through the SSI-recipient category with no separate income test. Both models evaluated only an income/MAGI-style adult pathway and never checked the SSI-recipient categorical route that the engine actually used (medicaid_category = SSI_RECIPIENT). The error is compounded because even the MAGI route qualifies here: household MAGI is 0.35 × FPL, far under Arizona's 138% FPL expansion-adult ceiling, and Indian Health Service coverage is not minimum essential coverage that bars Medicaid. Zero-income, SSI-receiving disabled adults in an expansion state are Medicaid-eligible on two independent pathways, so ""No"" is unreachable from the stated facts." -us,scenario_079,spouse_medicare_eligible,3,llm_error,age_disability,False,,"Medicare's disability pathway is not triggered by disability status itself — it requires 24 months of entitlement to Social Security disability benefits (SSDI/DIB), and PolicyEngine implements Medicare eligibility as an age test (65+) with no disability branch. The spouse is 55 and the fact sheet lists no Social Security disability income for the spouse at all; the $7,560 SSDI belongs to the head, so the spouse has no SSDI entitlement to start a 24-month clock. Blindness likewise confers no Medicare entitlement — statutory blindness matters for SSI/SSDI earnings tests, not for Medicare. All three models collapsed ""is disabled"" into ""is Medicare eligible,"" skipping the benefit-receipt and duration conditions that make the disability pathway operative." -us,scenario_079,ssi,32,llm_error,thresholds_rates,False,,"The case turns on one parameter: the 2026 SSI couple federal benefit rate of $1,491/month, the 2025 couple rate of $1,450 indexed by the 2.8% COLA. Countable income is straightforward — $7,560 SSDI is $630/month, the $20 general exclusion leaves $610/month ($305 per spouse after the deeming split), and the couple clears the $3,000 resource limit with $1,045.29 — so the benefit is ($1,491 − $610) × 12 = $10,572, split $5,286 per spouse. Nearly every wrong answer got the $610 countable income right and missed only on the rate: models that used the 2025 rate of $1,450 landed on $10,080, those using the 2024 rate of $1,415 landed on $9,660, and the cluster at $10,464–$10,560 used projections $1 to $9 per month low. The remaining failures either treated the FBR as a hard income cliff or asserted zero on an incorrect resource or take-up premise." -us,scenario_079,state_refundable_credits,34,llm_error,state_local_rule,False,,"Arizona's increased excise tax credit (A.R.S. § 43-1072.01) is a refundable, per-person sales-tax offset: $25 for each member of the tax unit, capped at $100, available to Arizona residents whose federal AGI is at or below $25,000 for married filing jointly. It carries no earned-income test, no dependent requirement, and no tax-liability requirement, so a two-adult married-filing-jointly unit with $7,560 of Social Security disability income collects the full $25 × 2 = $50. Nearly every model gated the credit on something it does not use — positive taxable income, wages, qualifying children, or a nonzero liability — or asserted Arizona has no refundable individual credit at all, and answered $0; one model derived $50 correctly in its reasoning and then submitted $200, exceeding the statutory $100 household cap." +us,scenario_079,spouse_medicare_eligible,4,llm_error,age_disability,False,,"The spouse is 55, so the only route to Medicare is the disability pathway. That route needs 24 months of Social Security disability (SSDI) entitlement, or ESRD/ALS. Being disabled or blind does not qualify anyone for Medicare by itself. The spouse has no Social Security disability income at all (the $7,560 belongs to the head), and no months of SSDI receipt or ESRD are listed, so all of these inputs default to zero or false. Every model treated ""is disabled"" as enough for Medicare and skipped the SSDI entitlement and 24-month waiting-period requirement." +us,scenario_079,ssi,33,llm_error,thresholds_rates,False,,"Both spouses are blind and disabled, so they form an SSI eligible couple. Countable resources are the $1,045 bank account, because the one vehicle and the home are excluded. Countable income is the head's $630/mo SSDI less the $20 general exclusion, or $610/mo. The 2026 couple federal benefit rate is $1,491/mo ($745.50 per spouse), so SSI is ($1,491 - $610) x 12 = $10,572. Most models got the $610 of countable income right but used the wrong couple rate: the 2024 rate ($1,415), the 2025 rate ($1,450), or projections between $1,482 and $1,490. The other misses zeroed the benefit through a false income or resource test, or gave an answer that contradicts their own arithmetic." +us,scenario_079,state_refundable_credits,39,llm_error,state_local_rule,False,,"Arizona's increased excise tax credit is a refundable credit of $25 per filer, spouse and dependent, capped at $100, for tax units whose federal AGI is at or below $12,500 (MFJ/HOH) or $6,250 (single/MFS). It needs no earnings, children, tax liability or taxable income. This couple's only income is $7,560 of SSDI, which is below the federal Social Security taxation base, so federal AGI is $0 and head plus spouse qualify for 2 x $25 = $50. Every model that answered $0 tied Arizona's refundable credits to earnings, children, liability or property-tax/age tests, or wrongly called the excise credit nonrefundable, and so never counted this credit. The one model that did find the $50 credit then submitted $200 anyway." us,scenario_080,federal_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_080,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. +us,scenario_080,head_chip_eligible,2,parse_contract_failure,other,False,,"The policy step is simple. The 41-year-old head has about $940 of income, so she qualifies for Pennsylvania Medicaid in the ACA expansion adult category, and Medicaid eligibility rules out CHIP. The correct answer is 0. Neither miss is a substantive policy error. Sonnet's explanation reached 0, but its scored value field records Yes. GLM returned no usable answer for this key at all." us,scenario_080,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"The trap is that a disabled 41-year-old holding $19,828 in bank assets invites the aged/blind/disabled Medicaid pathway with its SSI-linked $2,000 resource limit, when Pennsylvania's ACA adult expansion group is what governs: it covers any adult 19-64 at or below 138% FPL, is MAGI-based, applies no asset or resource test, and requires no disability or SSI status. Countable MAGI is $940 ($924 long-term capital gains plus $16 qualified dividends; the $3,600 financial assistance is not MAGI income), which is 0.06 x FPL against a 138% FPL limit, and the prompt's default status facts satisfy the age and immigration/5-year tests the engine applied. Five of the six models either imported a resource or categorical screen that the MAGI adult group forbids, or inverted the income test by reading the near-total absence of income as evidence against eligibility." us,scenario_080,head_medicare_eligible,7,llm_error,categorical_eligibility,False,,"Medicare eligibility below age 65 requires a qualifying pathway such as Medicare entitlement after the required period of SSDI receipt, ALS, or end-stage renal disease; a generic disability flag alone does not establish any of those conditions. The head is 41, and SSDI receipt, the waiting period, ALS, and end-stage renal disease are all unlisted and therefore false, so the correct eligibility output is 0." us,scenario_080,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -622,95 +631,95 @@ us,scenario_080,local_income_tax,1,parse_contract_failure,missing_output,False,, us,scenario_080,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_080,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_080,snap,39,llm_error,asset_resource,False,,"The $19,828 bank balance is a decoy: Pennsylvania confers broad-based categorical eligibility through TANF non-cash services, which waives the SNAP resource test entirely, so the only live question is the allotment computation. Countable monthly income is $301.33 ($3,600 financial assistance = $300/mo plus $16/yr dividends = $1.33/mo; long-term capital gains are not SNAP countable income), the standard deduction absorbs most of it, and the $2,000 SPM-unit energy subsidy is a LIHEAP-type payment that entitles the household to the heating standard utility allowance — whose excess shelter cost is deductible without cap because the head is disabled — driving net income to $0 and the benefit to the full one-person maximum. Calendar 2026 straddles two allotment schedules: $298/month for January–September (FY2026) and $304.68/month for October–December after the October 1, 2026 uprating, so 9 × $298 + 3 × $304.68 = $3,596.04. Thirteen models disqualified the household on a resource test that PA does not apply, nine kept a 30%-of-net-income reduction by dropping the SUA shelter deduction, and most of the remainder priced the maximum allotment at the stale FY2025 $291–$293 or flat-multiplied $298 by twelve." +us,scenario_080,snap,46,reference_engine_defect,categorical_eligibility,False,,"The reference depends on three steps, and each wrong model departed from at least one. First, Pennsylvania's broad-based categorical eligibility through its TANF non-cash program waives the SNAP asset test, so the $19,828 bank balance does not matter; 14 models answered $0 on assets. Second, the $2,000 energy subsidy triggers Pennsylvania's $857 heating standard utility allowance (SUA) under heat-and-eat. For a disabled household the excess shelter deduction is uncapped, so it wipes out the $92.33 left after the $209 standard deduction ($301.33 gross, from $300 assistance plus $1.33 dividends, with capital gains excluded); 13 models stopped at the standard deduction. Third, the frozen reference pays FY2026's $298/month for January–September and the engine's projected FY2027 $304.68/month for October–December, whereas the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 for all 12 months (USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze); of the 16 models that answered 12 times a monthly maximum, 11 used a stale maximum ($291–$293) and 5 used $298 for all 12 months, which matches the convention. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (categorical_eligibility). The frozen reference is 3,596.04; the corrected value is 3,240.00. PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. Reference is an engine defect; output excluded from scoring (7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04))." us,scenario_080,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to be aged 65 or older, blind, or disabled under the program's categorical determination. The engine input sets the sole person's SSI aged/blind/disabled status to false, so benefit-rate and countable-income calculations never produce an award." us,scenario_080,state_income_tax_before_refundable_credits,8,llm_error,credit_phaseout,False,,"Pennsylvania first applies its 3.07% rate to the $940 of taxable dividends and gains, then applies the nonrefundable Tax Forgiveness credit in computing income tax before refundable credits. The household's eligibility income remains within the range for 100% forgiveness, so the credit eliminates the preliminary tax and yields $0." us,scenario_080,state_refundable_credits,2,llm_error,state_local_rule,False,,"Pennsylvania Schedule SP tax forgiveness is a nonrefundable credit that reduces Pennsylvania income-tax liability; it is not included in state refundable credits. With no qualifying Pennsylvania refundable credit, the requested aggregate is $0." us,scenario_080,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The 2026 computation turns on two facts: the TCJA rate schedule and standard deduction are permanent (single brackets 10/12/22/24 breaking at $12,400/$50,400/$105,700; standard deduction $16,100, no personal exemption), and the SALT cap for 2026 is $40,400, not $10,000. That makes the entire $14,127.41 of state and local tax deductible — $5,643 of real estate tax plus $8,484.41 of Massachusetts income tax computed on this same household — alongside $16,900 of mortgage interest, giving $31,028.65 of itemized deductions against AGI of $174,088.12 and taxable income of $143,059.47, which yields $1,240 + $4,560 + $12,166 + 24% × $37,359.47 = $26,932.27. Wrong answers fall into three families: itemizing only mortgage interest plus property tax (or capping SALT at the repealed $10,000) and so dropping the $8,484.41 state income tax; assuming a TCJA sunset with a restored personal exemption and 10/15/25/28 rates; and subtracting the $7,389 employer-sponsored insurance premium from wages that already exclude it. A fourth cluster landed within ~$60 of the reference and missed only on the size of the imputed Massachusetts income tax inside SALT." +us,scenario_081,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"For 2026, OBBBA makes the TCJA 10/12/22/24% brackets and the $0 personal exemption permanent and raises the SALT cap to $40,400. That makes $14,127.41 of SALT fully deductible: $5,643 of real estate tax plus $8,484.41 of Massachusetts income tax. Adding $16,900 of mortgage interest gives $31,027.41 of itemized deductions, well above the $16,100 standard deduction. Taxable income is $143,059.47, and the tax is $17,966 + 24% × $37,359.47 = $26,932.27. The wrong answers fall into five traps: applying the expired $10,000 SALT cap (about $27,923), leaving Massachusetts income tax out of SALT (about $28,969), assuming a TCJA sunset with personal exemptions and 15/25/28% brackets (about $29,100–$30,000), subtracting the $7,389 ESI premium from the $175,002 wages, and near misses that put Massachusetts income tax a few hundred dollars low." us,scenario_081,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,payroll_tax,33,llm_error,state_local_rule,False,,"The separator in this case is the Massachusetts Paid Family and Medical Leave employee contribution: 0.46% × $175,002 = $805.01, a mandatory employee-side state payroll tax that PolicyEngine adds to federal FICA. Wages of $175,002 sit below the 2026 Social Security wage base, so full-wage FICA is $10,850.12 + $2,537.53 = $13,387.65, and no Additional Medicare Tax applies below the $200,000 single-filer threshold. The $7,389 employer-sponsored insurance premium does not reduce the Social Security and Medicare wage base here, which is the second trap several Gemini models fell into. Almost every wrong answer either dropped the PFML component entirely (landing on $13,387.65 or a rounding of it) or substituted a homemade employee rate — 0.24%, 0.256%, 0.318%, 0.375%, 0.38%, 0.532%, 0.55%, 0.604% — for the 0.46% employee rate." +us,scenario_081,payroll_tax,37,llm_error,state_local_rule,False,,"The reference has three parts: employee Social Security of 6.2% x $175,002 = $10,850.12 (the wages are below the 2026 wage base), employee Medicare of 1.45% x $175,002 = $2,537.53 with no Additional Medicare Tax, and the mandatory Massachusetts Paid Family and Medical Leave (PFML) employee contribution of 0.46% x $175,002 = $805.01. Most wrong models stopped at SS plus Medicare ($13,387.65) because they left out MA PFML or treated it as employer-only. Others included PFML at a wrong employee rate (0.256%, 0.318%, 0.375%, 0.44%, 0.532%, 0.55% or 0.604%). A third group shrank the FICA base to $167,613 by subtracting the $7,389 employer-sponsored insurance premiums, but the payroll tax base here is the full $175,002." us,scenario_081,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_081,state_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The trap is Massachusetts's Part B deduction stack, which sits on top of the $4,400 single personal exemption: a Social Security/Medicare deduction capped at $2,000 and a rental deduction of 50% of rent capped at $4,000 for 2026 (the $44,400 of pre-subsidy rent maxes it out even though the household also carries a mortgage). Part B gross income is wages $175,002 plus the $56 of taxable interest = $175,058, less $2,000 and $4,000 = $169,058, less the $4,400 exemption = $164,658, taxed at 5% = $8,232.90; the $110.12 of non-qualified dividends is Part A income taxed at 5% = $5.51, with the $1,080 short-term capital loss producing no offset, for $8,238.41. Nearly every wrong model applied a bare 5% to income less only the exemption, used the superseded $3,000 rental cap, deducted actual FICA instead of the $2,000 cap, or refused the rental deduction outright. A second cluster subtracted the $7,389 employer-sponsored insurance premium from wages that already exclude it, or imported federal itemized deductions that Massachusetts disallows." +us,scenario_081,state_income_tax_before_refundable_credits,43,reference_engine_defect,taxable_income_or_deductions,False,,"The reference has two parts. Part B income is $175,058: $175,002 of wages plus the $56 of taxable interest. Three items come off it: the $2,000 cap on the Social Security/Medicare deduction, the $4,000 rent deduction (50% of $44,400, capped at $4,000), and the $4,400 single exemption. That leaves $164,658, which is taxed at 5% for $8,232.90. Part A adds 5% on the $110 of dividends ($5.51), because the short-term capital loss is not netted against it, for a total of $8,238.41. Most models left out the $2,000 payroll-tax deduction or the $4,000 rent deduction, or used the old $3,000 rent cap. Others netted the $1,080 short-term loss against ordinary or Part A income, subtracted federal itemized deductions or ESI premiums that Massachusetts does not allow, or added arbitrary final adjustments. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 8,238.41; the corrected value is 8,232.90. PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes. Reference is an engine defect; output excluded from scoring (M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21)." us,scenario_081,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_chip_eligible,3,llm_error,age_disability,False,,"The decisive rule in the engine trace is CHIP's age criterion: child1 is age 1 and falls outside the applicable CHIP age range. The alternative Medicaid pathway also returns ineligible, so neither route produces CHIP eligibility." us,scenario_082,child1_early_head_start_eligible,6,llm_error,categorical_eligibility,False,,"Early Head Start eligibility is a conjunction: the child must be under 3 (or the household must contain a pregnant person) AND the household must pass the means test — income at or below the federal poverty guideline, with limited over-income slots up to 130% FPL, or a categorical pathway such as TANF/SSI receipt, foster-care placement, or homelessness. Every wrong model treated the birth-to-3 age band as sufficient and either skipped or explicitly discarded the income screen. This household's countable income of ~$90,132 is more than four times the two-person poverty guideline (~$21k, ~$28k at 130%), no listed fact establishes TANF, SSI, foster care, or homelessness, and there is no pregnancy, so both the categorical and income conditions evaluate False and eligibility is No." us,scenario_082,child1_medicaid_eligible,6,llm_error,health_coverage,False,,"The trap is that New York's generous children's coverage above 154% FPL runs through Child Health Plus — the state's separate CHIP program — not through Medicaid: NY Medicaid covers a child age 1–5 only up to 154% FPL, and the 400–405% FPL figures the models cited are the CHIP ceiling. The second half of the trap is income: Medicaid MAGI here is about $117.8k (543% FPL for a two-person household) because the $33,000 long-term capital loss is limited to a $3,000 deduction against ordinary income and because $20,580 of qualified and non-qualified dividends, taxable interest, short-term gain, and the state tax refund all count, while the $21,208 ESI premium is not subtracted from MAGI. Every model either imported a CHIP-scale threshold into Medicaid, understated MAGI to roughly $90k–$99k, or asserted child eligibility with no income test at all. At 543% FPL the child clears no Medicaid category (medicaid_category = NONE), so is_medicaid_eligible is False." us,scenario_082,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,child1_wic_eligible,2,llm_error,thresholds_rates,False,,"WIC requires both categorical eligibility and financial eligibility; being age 1 satisfies only the categorical test. The household's approximately $90,132 annual income exceeds the 2026 WIC limit of 185% of the federal poverty guideline for a two-person household, so the child is not eligible." -us,scenario_082,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The reference chain is: gross income $120,593.62 (wages $100,013.62 after the $181 traditional 401(k), $20,480 dividends, $100 interest), less $3,008.48 above the line (the $3,000 §1211(b) cap on the $30,710.82 net capital loss plus the $8 traditional IRA), less the $24,150 HOH standard deduction and a separate $700 deduction for the cash gift under the 2026 non-itemizer charitable deduction, giving $92,735.15 taxable — $79,311.15 ordinary taxed on the 2026 HOH schedule (10% to $17,700, 12% to $67,450, 22% above) for $10,349.45, plus $13,424 of qualified dividends at 15% for $2,013.60, less $600 CDCC (20% of the $3,000 one-child cap) and the $2,200 CTC. Two 2026-specific items separate the reference from nearly every wrong answer: the $700 charitable deduction available to a standard-deduction filer, which only two models took, and the exclusion of the $68 state and local tax refund under the tax-benefit rule for a filer who does not itemize. The wrong answers cluster into three groups: models that ran the right structure but dropped the $700 deduction, added the $68 refund, or used pre-2026 bracket thresholds and a $2,000 CTC (landing $95–$700 high); models that assumed TCJA expired and applied personal exemptions, 10/15/25 rates, and a $1,000 CTC phased out at $75,000; and models that mishandled the capital loss or the qualified-dividend stack, or submitted a number their own arithmetic never produced." +us,scenario_082,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference starts from AGI of $117,585.14. That is wages net of the $181 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capped capital loss and the $8 IRA deduction. The $68 state refund is excluded because the filer did not itemize. It then subtracts the $24,150 2026 head-of-household standard deduction and the new $700 non-itemizer charitable deduction, giving taxable income of $92,735.15. It taxes $79,311.15 of ordinary income on the 2026 HoH schedule (10% to $17,700, 12% to $67,450, 22% above) for $10,349.45, and the $13,424 of qualified dividends at 15% for $2,013.60. Finally it subtracts the $600 CDCC and the $2,200 CTC. The closest answers ($9,578–$9,734) differ only by skipping the $700 non-itemizer charitable deduction and/or adding the $68 refund. The rest used stale brackets or standard deductions, a $2,000 CTC, TCJA-sunset rules (personal exemptions, 10/15/25 brackets, $1,000 CTC), an uncapped capital loss, or deducted the $21,208 employer-sponsored premiums. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 9,563.05; the corrected value is 9,564.92. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_082,federal_refundable_credits,8,llm_error,credit_phaseout,False,,"The separating step is that the Additional Child Tax Credit is a residual, not a per-child entitlement: the 2026 CTC of $2,200 is first applied nonrefundably against tax liability, and only the unused remainder — capped at $1,700 and further limited to 15% of earned income over $2,500 — becomes refundable. This head-of-household filer has roughly $117,650 of AGI (wages net of the $181 traditional 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capital-loss limitation on a $30,711 net loss and the $8 IRA deduction), which produces federal tax before credits many times larger than $2,200, so the entire credit is consumed nonrefundably and nothing spills into the refundable bucket. EITC is zero on income grounds and independently on the investment-income limit, and no American Opportunity Credit, Recovery Rebate Credit, or refundable payroll tax credit applies. Every wrong model quoted a statutory refundable ceiling as the answer instead of computing the leftover after the nonrefundable CTC." us,scenario_082,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_082,local_income_tax,5,llm_error,state_local_rule,False,,"New York State residence does not establish New York City residence, and the prompt supplies no city or locality. Because unlisted location facts are false, no covered local income, wage, or earnings tax applies, so the separate local-income-tax output is zero." -us,scenario_082,payroll_tax,38,llm_error,state_local_rule,False,,"The output definition counts ""mandatory employee state payroll taxes,"" and New York imposes two of them on this wage earner: Paid Family Leave contributions of $411.91 (0.411% of the full $100,195, with no wage cap binding at this income) and statutory disability (DBL) contributions of $31.20 ($0.60/week × 52), totaling $443.11 on top of federal FICA. The federal side is uncontroversial and every arithmetically competent model got it: 6.2% × $100,195 = $6,212.09 plus 1.45% × $100,195 = $1,452.83 = $7,664.92, with no Additional Medicare Tax below $200,000. The payroll tax base is the stated gross wage of $100,195 — neither the $21,208 employer-sponsored insurance premium nor the $181 traditional 401(k) deferral reduces it. Three failure clusters follow: models that stopped at $7,664.92 by denying NY has any employee payroll tax, models that shaved ESI and/or 401(k) off the wage base and landed near $6,000–$6,400, and models that included NY PFL but with an invented rate or a cap that does not bind." +us,scenario_082,payroll_tax,42,llm_error,state_local_rule,False,,"The requested payroll_tax output includes mandatory employee state payroll taxes. In New York those are Paid Family Leave (PFL) and disability benefits (DBL) contributions. The federal part is simple: 7.65% of the full $100,195 wage base, or $7,664.92. On top of that, 2026 NY PFL is 0.432% of wages capped at the statewide average weekly wage base of $95,350.72, which caps the contribution at $411.91, and DBL is $0.60/week, or $31.20 a year. The largest group of models dropped the $443.11 of NY contributions entirely. Others used stale 2024/2025 PFL rates or caps, applied the 0.432% rate without the cap, or wrongly cut the FICA base by subtracting the employer-sponsored insurance premium or the 401(k) deferral." us,scenario_082,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_082,state_income_tax_before_refundable_credits,38,llm_error,state_local_rule,False,,"New York's 2026 head-of-household computation has four parts the field collapsed: NYAGI of $117,585 (federal AGI with the $3,000 capital-loss limit and the $181 401(k)/$8 IRA deductions, less the $68 state-refund subtraction), a $11,200 HOH standard deduction that beats the $11,169 of NY itemized deductions ($11,057 SALT plus $112.07 of charity surviving the 0.5%-of-AGI floor), a $1,000 dependent exemption, and the 2026 reduced rate schedule (3.9%/4.4%/5.15%/5.4% through $107,650) applied to $105,385.15, giving $5,442.17. Because NYAGI exceeds $107,650, NY's supplemental tax adds $156.38: the $786.88 recapture base — the benefit of the sub-5.9% brackets measured at $107,650 — times the 19.87% phase-in across the $50,000 range. No nonrefundable NY credit applies: the household credit is fully phased out above $32,000 of NYAGI, and NY's child and dependent care credit and Empire State Child Credit are refundable and belong in state_refundable_credits. Nearly every wrong answer dropped the supplemental tax, substituted the $8,000 single standard deduction or the single/MFJ bracket schedule for the HOH ones, kept the 2025 5.5% rate, deducted the employer-insurance premium or the full capital loss a second time, or subtracted a refundable NY credit as though it were nonrefundable." -us,scenario_082,state_refundable_credits,39,llm_error,state_local_rule,False,,"The reference total of $650.50 is exactly two New York credits: the Empire State Child Credit (ny_ctc) of $290.50 and the refundable NY child and dependent care credit (ny_cdcc) of $360.00. The ESCC trap is the enhanced credit in effect for 2026, which pays a flat $1,000 for a child under age 4 and reduces it by $16.50 per $1,000 of AGI above the $75,000 head-of-household threshold — $42,585 of excess AGI on $117,585.15 wipes out $709.50 and leaves $290.50, rather than zero and rather than the superseded ""greater of $100 or 33% of the federal CTC"" formula. The CDCC trap is New York's expanded applicable-percentage schedule: the $3,000 one-child expense cap times the 20% federal rate gives $600, and NY pays 60% of that at NY AGI $117,585 — not the pre-expansion 20%-of-federal rate that applies only under the old $65,000 cliff — and the credit is fully refundable regardless of NY liability. Every wrong answer traces to using the repealed ESCC formula, zeroing the ESCC through a fabricated phase-out or age gate, or applying the wrong NY CDCC percentage (or omitting the CDCC entirely)." +us,scenario_082,state_income_tax_before_refundable_credits,45,llm_error,state_local_rule,False,,"The filer is a New York head of household. NY AGI is $117,585. That is federal AGI less the $68 refund subtraction, with the capital loss capped at $3,000 and the listed wages not reduced by the ESI premiums. Subtracting the $11,200 HOH standard deduction (which beats the $11,169 of NY itemized deductions) and a $1,000 exemption for the child only leaves $105,385.15. The 2026 reduced HOH rates (3.9%/4.4%/5.15%/5.4%) give $5,442.17 on that amount, and a $156.38 supplemental recapture tax applies because NY AGI exceeds $107,650. No nonrefundable credit applies: the NY child and dependent care credit and the Empire State Child Credit are refundable, and the household credit is phased out. The models went wrong in several ways: filing as single or using the $8,000 deduction, adding an exemption for the head, using pre-2026 rates, leaving out the supplemental tax, subtracting refundable credits, over-itemizing, or removing the ESI premiums or the uncapped capital loss from income." +us,scenario_082,state_refundable_credits,46,llm_error,state_local_rule,False,,"This household gets two refundable New York credits. The first is the enhanced 2026 Empire State Child Credit: $1,000 for a child under 4, reduced by $16.50 for each whole $1,000 of federal AGI ($117,652.65) above the $75,000 head-of-household threshold. That is 42 steps, or $693, which leaves $307. The second is the refundable NY child and dependent care credit at 60% of the $600 federal credit, or $360, for a total of $667. The wrong answers fall into four patterns. Some applied the pre-2025 ESCC (33% of the federal CTC, a $100 minimum, or no credit for children under 4). Some treated the ESCC as fully phased out. Some used the 20% NY CDCC percentage instead of 60% or called the CDCC nonrefundable. Others ignored one of the two credits entirely." us,scenario_083,federal_income_tax_before_refundable_credits,1,llm_error,thresholds_rates,False,,"The 2026 single-filer standard deduction exceeds the household’s $11,010 AGI, leaving zero taxable income. With no other federal tax components or nonrefundable credits affecting the result, federal income tax before refundable credits is $0." us,scenario_083,federal_refundable_credits,9,llm_error,categorical_eligibility,False,,"A childless filer must satisfy the EITC age requirement, and the 20-year-old head does not. With no qualifying children and no applicable refundable education, recovery rebate, or payroll tax credit, every component of federal refundable credits is zero." us,scenario_083,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Texas did not adopt ACA Medicaid expansion, so income below 138% of FPL does not create Medicaid eligibility for a nondependent adult. The 20-year-old head has no qualifying Texas category—such as pregnancy, parent/caretaker status, disability, or SSI eligibility—so `medicaid_category` is NONE regardless of the 0.69 FPL MAGI level." us,scenario_083,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 20-year-old adult passes the income and nutritional-risk tests but belongs to category 5, so categorical ineligibility makes head_wic_eligible equal to No." us,scenario_083,payroll_tax,5,llm_error,payroll_tax_base,False,,"Payroll tax applies the 6.2% Social Security rate and 1.45% Medicare rate to the $7,050 wage base, with no Additional Medicare Tax or Texas employee payroll tax. The components are $437.10 and $102.22, producing $539.33 after rounding; rental income and the separately listed overtime premium do not enter this wage base." us,scenario_083,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Ordinary rental income is generally excluded from net earnings from self-employment and therefore does not enter the self-employment tax base. With no reported self-employment earnings, the $400 filing threshold is not met and self-employment tax is zero." -us,scenario_083,snap,14,llm_error,asset_resource,False,,"The binding constraint in this case is the SNAP resource test, not the benefit formula. Texas is a broad-based categorical eligibility state that retains a liquid-resource limit — $5,000 in countable assets for a household with no elderly or disabled member — and the head's $6,620 bank account balance exceeds it (it also exceeds the $3,000 federal non-BBCE limit), so the unit fails the asset test and receives $0. The $6,620 bank-account figure is the one household fact that does nothing in the allotment arithmetic, and every wrong model either ignored it or asserted Texas waives asset testing under BBCE; all 14 then ran the standard net-income formula (20% earned-income deduction, ~$200 standard deduction, max allotment minus 30% of net income) and landed in the $1,300–$2,900 band that the formula produces when the resource screen is skipped." +us,scenario_083,snap,18,llm_error,asset_resource,False,,"This one-person Texas household passes every SNAP income test: gross income is $917.50 a month, well under 130% and 165% of FPL, and net income is about $591, under 100%. What makes it ineligible is its resources. Texas's broad-based categorical eligibility keeps a $5,000 asset limit, and the federal standard for a non-elderly household is about $3,000. The head's $6,620 bank balance exceeds both, so SNAP is $0. Every model ran the income-based benefit formula, getting about $110-$245 a month. None applied the resource test, and several said outright that Texas BBCE waives the asset test." us,scenario_084,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Disability, blindness, SSDI receipt, and qualifying immigration status do not independently establish Medicaid eligibility; the person must satisfy a specific North Carolina coverage pathway and its financial rules. The head qualifies through no Medicaid category, and Medicaid MAGI is 1.56 times FPL, above the expansion threshold." -us,scenario_084,head_medicare_eligible,28,prompt_ambiguity,age_disability,False,,"PolicyEngine's is_medicare_eligible resolves on the age-65 threshold, with the under-65 route requiring an explicit qualifying-disability or ESRD entitlement input; the head is 62 and no such input is listed, so the prompt's treat-unlisted-facts-as-false rule zeroes out any prior SSDI entitlement history, ESRD, or existing Medicare enrollment. The is_disabled and is_blind flags and the $24,828 of Social Security disability income drive SSI, the blind/elderly-disabled deductions, and taxable Social Security in the engine, not Medicare entitlement, and the household's own coverage facts run the other way: employer-sponsored insurance with $4,789 of premiums and a health-premium line stated as excluding Medicare Part B with no Part B premium listed. Every wrong model imported the real-world SSDI-to-Medicare pathway and then supplied the missing element itself, either dropping the 24-month qualifying period entirely or manufacturing it from the head's age, the annual benefit amounts, or the prompt's within-year constancy clause. That clause fixes status inside the 2026 tax-benefit year and creates no pre-2026 entitlement history. Developer adjudication (2026-09-05): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." +us,scenario_084,head_medicare_eligible,33,prompt_ambiguity,age_disability,False,,"The Head is 62, which is under the Medicare age threshold of 65, and has no end-stage renal disease. That leaves the disability pathway, which requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)). PolicyEngine tests this with a count of months receiving SSDI. That count is not listed, so under the prompt's rule it is 0. The prompt's instruction to hold status constant covers only the 12 months of the tax year, not the earlier history needed for 24 months. Every model treated being disabled plus current SSDI income as enough for Medicare. Many said the 24-month wait was met even though no listed fact supports that. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it." us,scenario_084,snap,2,llm_error,thresholds_rates,False,,"The single-person household has $33,233 of gross annual income ($2,769/month), which exceeds North Carolina's broad-based categorical eligibility gross cap of 200% FPL (about $2,680/month for one person in 2026), and after the ~$204 standard deduction and the elderly/disabled excess-medical deduction its net income of roughly $1,738/month still exceeds the 100%-FPL net income limit of about $1,340/month; the $56,500 bank balance also exceeds the elderly/disabled resource limit of roughly $4,650. Ineligibility ends the calculation at $0. Both models stopped at the observation that the gross income test is waived when an elderly or disabled member is present and never tested net income against the 100%-FPL ceiling, then manufactured a benefit — one by overriding its own $0 formula result, the other by applying the one-to-two-person minimum allotment, which is a floor for households already found eligible, not a substitute for eligibility." -us,scenario_085,federal_income_tax_before_refundable_credits,32,llm_error,taxable_income_or_deductions,False,,"The case turns on two steps: the Tier-2 Social Security worksheet, which makes only $14,050.60 of the $27,426 in survivor benefits taxable (0.85 × ($45,236 − $34,000) + $4,500, capped at 85% of benefits), and the 2026 deduction stack of a $16,100 basic standard deduction + $2,050 age-65 addition + the $6,000 OBBBA §70103 senior deduction, taken in full because MAGI $45,574 sits below the $75,000 phase-out start. AGI $45,573.60 less $24,150 leaves taxable income of $21,423, taxed at 10% to $12,400 and 12% above, for $2,322.76. Nearly every wrong answer either pushed taxable benefits toward the 85% ceiling, omitted the $6,000 senior deduction, or modeled 2026 as a TCJA-sunset year with a personal exemption and a 15% second bracket; a handful derived the reference number and then submitted a different one." +us,scenario_085,federal_income_tax_before_refundable_credits,34,llm_error,taxable_income_or_deductions,False,,"AGI is $45,573.60: $31,523 of other income plus taxable Social Security of $14,050.60, which is 85% of the $11,236 by which provisional income exceeds $34,000, plus $4,500. The step that separates the reference from most wrong answers is the 2026 deduction stack under OBBBA. It combines the $16,100 standard deduction, the $2,050 additional amount for age 65+, and the $6,000 senior deduction (2025-2028, no phase-out below $75,000 MAGI), for $24,150 in total. That leaves taxable income of $21,423, taxed at 10% up to $12,400 and 12% above, for $2,322.76. Models that left out the $6,000 senior deduction landed near $3,040-$3,190. Models that assumed the TCJA would sunset, bringing back personal exemptions and a 15% bracket, landed near $3,800-$4,200. The rest overstated taxable Social Security, invented nonrefundable credits, or submitted a value their own derivation did not support." us,scenario_085,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_085,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_085,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered employment earnings, not Social Security survivor benefits, retirement distributions, pension income, or taxable interest. Because the household has no listed wages or other payroll-taxable earnings, the employee-side payroll-tax base is zero and payroll tax is $0." us,scenario_085,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_085,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_085,state_income_tax_before_refundable_credits,28,llm_error,credit_phaseout,False,,"The trap is Pennsylvania's Tax Forgiveness credit (PA-40 Schedule SP), a nonrefundable credit that this output is measured after. PA taxable income here is only the $1,675 of taxable interest — Social Security survivor benefits and post-retirement-age distributions from eligible retirement plans are outside PA's eight taxable classes — producing $51.42 of gross tax at the 3.07% flat rate. Schedule SP eligibility income excludes Social Security and Railroad Retirement benefits and old-age/retirement-plan distributions received after retirement age, so this 67-year-old single claimant's eligibility income is the same $1,675, far under the $6,500 unmarried-with-no-dependents poverty limit, earning 100% forgiveness. The 100% credit eliminates the entire $51.42, leaving $0 of state income tax before refundable credits; every wrong answer either skipped Schedule SP entirely or computed eligibility income from gross household income instead of the statutory exclusions." +us,scenario_085,state_income_tax_before_refundable_credits,34,llm_error,state_local_rule,False,,"This 67-year-old's Social Security and retirement distributions are exempt from PA tax, so PA taxable income is just the $1,675 of interest. At 3.07% that is $51.42 of tax. Pennsylvania's Special Tax Forgiveness (Schedule SP) then removes it entirely. Eligibility income for forgiveness leaves out Social Security and commonly recognized retirement benefits, so it is only $1,675, well under the $6,500 single-filer limit for 100% forgiveness. Most models stopped at $51.42, either never applying forgiveness or rejecting it because they counted the $27,426 of Social Security and $29,848 of retirement income as eligibility income. A few taxed the retirement distributions outright." us,scenario_086,child1_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires Medicaid ineligibility and satisfaction of CHIP’s independent age and income criteria. Child 1 is age 12, so age is not the failing criterion; the household income exceeds Georgia’s applicable CHIP income threshold for 2026." us,scenario_086,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The models conflated Georgia’s separate PeachCare/CHIP pathway with Medicaid or assumed that being a child creates Medicaid eligibility at higher income levels. At 3.44 times FPL, child1 exceeds Georgia’s child Medicaid thresholds and qualifies for no Medicaid category, so employer-sponsored coverage and non-MAGI expense deductions do not create eligibility." -us,scenario_086,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The case turns on two things models had to get right together: the $12,824 of Social Security dependent benefits are the head's Social Security income, and with provisional income of $68,074.50 (wages less the educator deduction plus half the benefits) far above the $34,000 upper threshold for a non-joint filer, the 85% cap binds and $10,900.40 enters AGI. The second trap is the 2026 parameter set — a $24,150 head-of-household standard deduction, no personal exemptions, a 10% bracket ending at $17,700 with 12% above it, and a $2,200 per-child CTC applied nonrefundably up to liability. Most wrong answers either reassigned the benefits to the child and dropped $10,900.40 of income, or built 2026 on pre-TCJA-sunset parameters (personal exemptions, a 15% bracket, a $1,000 CTC), or substituted stale standard-deduction and CTC amounts. The near misses came from small parameter slips: a $300 educator cap instead of $337.50, a $17,000 bracket break instead of $17,700, or an inapplicable $275 auto-loan interest deduction." +us,scenario_086,federal_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The Social Security dependent benefits are the Head's input. Provisional income ($61,662.50 + $6,412 = $68,074.50) exceeds the $34,000 single/HoH second base, so the 85% cap applies and $10,900.40 is taxable, which puts AGI at $72,562.90 after the full $337.50 educator deduction. Under OBBBA's permanent TCJA structure, the 2026 HoH standard deduction is $24,150, which leaves taxable income of $48,412.90. Tax is $1,770 + 12% of $30,712.90 = $5,455.55, and the full $2,200 CTC is absorbed as a nonrefundable credit, giving $3,255.55. The wrong answers split into four groups. Some dropped the taxable SS as the child's income. Some applied TCJA-sunset rules (personal exemptions, 15% bracket, $1,000 CTC). Some used stale 2025 or pre-OBBBA parameters ($22,500/$23,100 SD, $17,000 bracket, $2,000 CTC). The rest used the wrong filing status or took an unsupported auto-loan-interest deduction or $300 educator cap. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)." us,scenario_086,federal_refundable_credits,11,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit is a residual, not an entitlement: IRC 24(d)(1) makes it the lesser of the per-child refundable cap ($1,700 for 2026) and the amount of CTC that the §26(a) tax-liability limitation actually blocked, which Schedule 8812 computes as line 12 minus line 14 (""If zero, stop here; you cannot take the additional child tax credit""). This head-of-household filer has AGI of $72,562.90 ($62,000 wages plus $10,900.40 of the $12,824 Social Security dependent benefit that is taxable once combined income exceeds the $34,000 tier-2 threshold, less the $337.50 educator deduction), taxable income of $48,412.90 after the 2026 HoH standard deduction of $24,150, and pre-credit tax of $5,455.55 — so the full $2,200 CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is $0 because earned income of $62,000 is far past the 2026 one-child HoH phase-out endpoint of roughly $51,600, computed at the statutory 15.98% phase-out rate, and no education expenses, rebate, or refundable payroll credit exist here. Ten of eleven models treated the $1,700 refundable cap (or a stale $1,000/$1,500 figure) as a floor paid out per qualifying child and never performed the unused-CTC subtraction against liability." us,scenario_086,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_086,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_086,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee payroll tax is $3,844 of Social Security tax plus $899 of Medicare tax on the head’s full $62,000 wage amount, totaling $4,743. The listed insurance premiums do not reduce the FICA wage base in this computation, Georgia unemployment insurance is employer-paid rather than a mandatory employee payroll tax, and the dependent child has no wages." -us,scenario_086,state_income_tax_before_refundable_credits,38,llm_error,state_local_rule,False,,"Georgia's 2026 parameters decide this case: Georgia AGI of $61,662.50 (federal AGI $72,562.90 less the full $10,900.40 taxable Social Security subtraction), a $15,000 head-of-household standard deduction, a $5,000 per-dependent exemption, and a 4.99% flat rate — $41,662.50 × 0.0499 = $2,078.96. Nearly every wrong answer stalls the rate at an earlier rung of Georgia's phase-down (5.49/5.39/5.29/5.19/5.12/5.09%) and pairs it with the wrong deduction stack: pre-flat-tax $5,400 standard deductions with $2,700 personal exemptions, or $12,000/$18,000/$18,500/$24,000 standard deductions, or $3,000/$4,000 dependent exemptions. A second cluster breaks the Social Security wash in one direction or the other — leaving the $10,900.40 taxable amount in the Georgia base, or removing the full $12,824 benefit a second time from an AGI that already excluded it. Four models computed a Georgia figure and then submitted a different number that no stated calculation produces." +us,scenario_086,state_income_tax_before_refundable_credits,45,llm_error,thresholds_rates,False,,"Georgia 2026 tax starts from federal AGI ($72,562.90), subtracts the $10,900.40 of taxable Social Security to reach GA AGI of $61,662.50, then subtracts the $15,000 head-of-household standard deduction and a $5,000 dependent exemption. There is no taxpayer personal exemption. The resulting $41,662.50 is taxed at the flat 2026 rate of 4.99%, giving $2,078.96. Almost every model used out-of-date parameters: a $12,000 (or $18,500 or $24,000) standard deduction, a $3,000–$4,000 dependent exemption, a flat rate between 5.09% and 5.49%, or the repealed pre-2024 structure of a $5,400 standard deduction plus $2,700 personal exemptions. A smaller group also left Social Security in Georgia income or took deductions that don't exist (employer insurance premiums, auto-loan interest, or the educator expense counted twice)." us,scenario_086,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"Georgia does not provide the claimed $260 refundable credit for each dependent child under age 17. With no applicable Georgia refundable credit, the state-level refundable-credit aggregate is $0." us,scenario_088,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The decisive computation step is the 2026 deduction for qualified tip income: the $9,600 of tips included in wages reduces taxable income in addition to the applicable standard deduction. The state income-tax refund is also excluded because no prior-year itemized-deduction tax benefit is established. These deductions reduce taxable income to zero, so federal income tax before refundable credits is $0 and cannot be negative." us,scenario_088,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The only refundable credit in play is the childless EITC, and IRC §32(a)(2)(B) phases it out against the GREATER of earned income or AGI. Earned income is $18,577, but AGI is $24,035 once the $2,733 unemployment compensation, $1,862 taxable pension, $675 state tax refund, $140 interest and $48 dividends are added; the tip deduction is below-the-line and does not reduce AGI. The 2026 childless EITC completes its phase-out below $20,000, so an AGI of $24,035 zeroes the credit, and with no qualifying children, no education expenses and no rebate, every other refundable component is $0. Every wrong model ran the phase-out on earned income alone, or on a max-credit shortcut, and never applied the AGI leg of the test." us,scenario_088,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Blindness and disability establish a potential aged, blind, or disabled Medicaid category; they do not confer Medicaid automatically. The head receives no SSI, is not a dependent, has MAGI income of 1.46 times FPL, and satisfies none of PolicyEngine’s Medicaid pathways, so the eligibility category is NONE." us,scenario_088,head_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility for a person under 65 is not conferred by disability status itself: it requires 24 months of entitlement to Social Security Disability Insurance (or RRB disability benefits), or an ESRD/ALS diagnosis. The head is 56 with $0 Social Security disability income — every unlisted input is 0, so there is no SSDI entitlement and no 24-month qualifying period — and the only qualifying pathway left, age 65, is unmet, so the answer is No. All six models converted the household's `is_disabled` (and `is_blind`) boolean directly into Medicare entitlement, skipping the SSDI-receipt precondition that actually gates the under-65 pathway; blindness in particular is an SSI/SSDI disability criterion, not a Medicare one." us,scenario_088,payroll_tax,5,llm_error,payroll_tax_base,False,,"The employee payroll tax base here is the full $18,577 of W-2 wages, tips included: IRC 3121(a) and 3121(q) make reported tips wages for both Social Security and Medicare, and the tip deduction is an income-tax deduction that leaves the FICA base untouched. Everything else in the household — the $1,862 taxable pension, $2,733 unemployment compensation, $140 interest, $48 dividends — sits outside the payroll tax base, wages fall far below both the Social Security wage base and the $200,000 Additional Medicare Tax threshold, and Texas imposes no mandatory employee payroll tax. The entire computation is 0.062 × 18,577 = $1,151.78 plus 0.0145 × 18,577 = $269.37, totaling $1,421.14. The wrong answers come from zeroing the wage base entirely, carving tips out of Social Security wages while inventing an employee tip credit, or misexecuting the 7.65% multiplication." -us,scenario_088,snap,4,llm_error,asset_resource,False,,"All four models evaluated SNAP income eligibility but omitted the household’s $31,776 in bank-account assets. A Texas SNAP household containing a disabled member remains subject to the applicable countable-resource limit, and these countable liquid assets exceed that limit, making the household ineligible before any net-income, shelter-deduction, or allotment calculation can produce a benefit." -us,scenario_089,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"2026 is governed by the OBBBA regime: a $32,200 MFJ standard deduction, the 10%/12% schedule with the 12% bracket opening at $24,800, and a new below-the-line deduction for qualified overtime equal to the spouse's $3,467.43 FLSA overtime premium. AGI is $106,200.13 — wages net only of the spouse's $6,946 traditional 401(k) (the head has no wages, so the head's $5,403 deferral buys nothing), plus $30,335.29 self-employment and $19,650 partnership income, less the $2,143.12 half-SE-tax deduction and $577 of deductible traditional IRAs — and the $32,200 standard deduction, $9,568.44 §199A deduction (20% of $47,842.18, business income net of the half-SE-tax deduction), and $3,467.43 overtime deduction leave $60,964.27 taxable and $6,819.71 of tax. Nearly every model omitted the overtime deduction, and most compounded it by understating the standard deduction, computing §199A on gross rather than half-SE-tax-reduced business income, excluding the $7,789 employer health premium from income, allowing the head's wageless 401(k) deferral, or applying a pre-TCJA sunset schedule with personal exemptions; self-employment tax is excluded from this output." +us,scenario_088,snap,5,llm_error,asset_resource,False,,"The head holds $31,776 in bank account assets, and that fails the SNAP resource test outright. The balance is far above the federal $4,500 limit for households with an elderly or disabled member and also above Texas's $5,000 resource limit under broad-based categorical eligibility. Because the household fails the asset test, SNAP is $0 whatever its net income or shelter deduction works out to. Every wrong model took the disability-based waiver of the gross income test and went straight to the net-income and shelter-deduction calculation. None of them applied the resource test to the listed bank balance." +us,scenario_089,federal_income_tax_before_refundable_credits,41,llm_error,taxable_income_or_deductions,False,,"AGI is $106,200.13. That is the spouse's wages net of only the spouse's own $6,946 traditional 401(k), plus $30,335 of self-employment income and $19,650 of partnership/S-corp income, minus half of SE tax ($2,143) and both traditional IRAs ($577). The employer-sponsored insurance premiums are not excluded from wages, and the head's 401(k) has no wages to reduce. Taxable income then subtracts three below-the-line deductions under 2026 OBBBA law: the $32,200 MFJ standard deduction, a QBI deduction of 20% of $47,842.18 (QBI net of the half-SE-tax deduction), and the new qualified overtime deduction for the spouse's $3,467 FLSA overtime premium. That leaves $60,964.27 taxed at 10% up to $24,800 and 12% above. Most models missed the overtime deduction. Others assumed the TCJA sunset (personal exemptions, 15% bracket), subtracted the ESI premiums from wages, deducted the head's 401(k), or skipped the QBI deduction." us,scenario_089,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Refundable CTC requires a qualifying child, and this two-adult household has none. Excess Additional Medicare Tax withholding is not a refundable tax credit, and the model also invented wages for the head despite the instruction that unlisted numeric inputs are zero; therefore all enumerated refundable-credit components equal zero." -us,scenario_089,payroll_tax,20,llm_error,payroll_tax_base,False,,"The whole case turns on what enters the employee FICA base: the spouse's $65,881 of gross wages and nothing else. The head has no wage entry, so unlisted income is 0; the $3,467 FLSA overtime premium is a component of the stated gross wage total, not an addition; traditional 401(k) deferrals stay in the Social Security and Medicare base; the $19,650 partnership and $30,335 self-employment earnings belong to the separate self-employment tax output; and North Carolina levies no mandatory employee payroll tax. The dominant wrong answer, $4,444.04, comes from subtracting the $7,789 employer-sponsored insurance premium as a section 125 salary reduction — that input is a listed household premium fact, not a withholding from the $65,881 the prompt fixes as the annual gross wage total. Applying 6.2% and 1.45% to $65,881 gives $4,084.63 + $955.28 = $5,039.91." +us,scenario_089,payroll_tax,21,llm_error,payroll_tax_base,False,,"Only the spouse's $65,881 of gross wages counts toward employee payroll tax. The prompt says gross wages already include overtime, so the $3,467 FLSA overtime premium is not added again. The facts list no pre-tax salary-reduction election, so the $7,789 employer-sponsored insurance premium, the traditional 401(k) deferrals, and the individually paid health premiums do not reduce the wage base. The result is 6.2% × $65,881 = $4,084.63 plus 1.45% × $65,881 = $955.28, for $5,039.91. The head has no wages, self-employment and partnership income are outside employee FICA, and North Carolina has no mandatory employee state payroll tax. The most common error, made by 9 models, was subtracting the $7,789 ESI premium to get $58,092 × 7.65% = $4,444. The other models double-counted overtime, invented wages for the head, subtracted 401(k) or other premiums, added self-employment tax, or made arithmetic slips." us,scenario_089,self_employment_tax,12,llm_error,payroll_tax_base,False,,"The self-employment tax base here is the spouse's $30,335 self-employment income and nothing else: the $19,650 of partnership/S-corp income flows to taxable income but stays outside net earnings from self-employment, and the household's $69,348 of wages are far below the Social Security wage cap, so no wage-base coordination reduces the SECA Social Security component. Taxing 92.35% of $30,335 at the full 15.3% (12.4% Social Security + 2.9% Medicare) produces the $4,286.24 reference. Five models inflated the base to $49,985 by adding the partnership/S-corp income, landing near $7,063; the rest chose the right base but broke on execution — mis-multiplying the 92.35% factor, dropping the factor entirely, or submitting a final number their own stated arithmetic never produced." -us,scenario_089,state_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"The answer is a two-step computation: build federal AGI of $106,200.13, subtract North Carolina's $25,500 joint standard deduction, and apply the 2026 flat rate of 3.99% to $80,700.13. AGI is where nearly every model broke: only the spouse's $6,946 traditional 401(k) is deductible (the head has no wages to defer from, so the head's $5,403 is not), both traditional IRA contributions ($252 + $325) are deductible, the full $30,335 of self-employment income enters AGI with only half of the $4,286 SE tax ($2,143) removed, and the $19,650 partnership/S-corp income carries no SE tax at all. The $7,789 employer-sponsored insurance premium and the $3,467 FLSA overtime premium do not reduce AGI — the largest wrong-answer cluster ($2,909) comes from excluding the ESI premium as a cafeteria-plan pre-tax amount. The second cluster used stale rates (4.25%, 4.5%, 4.0%) or inflated standard deductions ($27,500–$32,200) instead of 3.99% and $25,500." +us,scenario_089,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"North Carolina taxes federal AGI minus the $25,500 married-filing-jointly standard deduction at the 2026 flat rate of 3.99%. Federal AGI is $106,200.13. It starts with the spouse's $65,881 in wages, which already include overtime, minus only the spouse's own $6,946 traditional 401(k) deferral. It adds the full $30,335 of self-employment income and the $19,650 partnership/S-corp income, then subtracts the roughly $2,143 half-of-self-employment-tax deduction and both spouses' $577 in IRA deductions. The largest group of wrong answers ($2,909) removed the spouse's $7,789 employer-sponsored insurance premium from wages. Other models deducted Head's 401(k) even though Head has no wages, disallowed Head's spousal IRA deduction, used the 0.9235 self-employment tax factor in AGI, subtracted the overtime deduction (which is taken after AGI), or used old 4.25%/4.5% rates or wrong standard deductions." us,scenario_089,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_090,federal_refundable_credits,10,llm_error,age_disability,False,,"For tax year 2026, a filer with no qualifying children must be at least age 25 to claim the childless EITC. The head is 22, so the EITC is $0 regardless of the 7.65% phase-in calculation; with every other refundable-credit component also equal to $0, federal refundable credits total $0." us,scenario_090,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Kansas has no Medicaid expansion pathway for a nondisabled, nonpregnant, childless 22-year-old adult. A low MAGI of 0.44 times FPL does not independently establish eligibility: the person must first belong to a covered Medicaid category, and this head belongs to none." us,scenario_090,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires categorical eligibility in addition to income eligibility and nutritional risk. A 22-year-old adult with no pregnancy, postpartum, or breastfeeding status is outside every eligible WIC category, so passing the other two tests does not confer eligibility." us,scenario_090,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security tax plus 1.45% Medicare tax on the full $7,000 of wages, with no Additional Medicare Tax or mandatory Kansas employee payroll tax. The components are $434.00 and $101.50, which sum to $535.50." -us,scenario_090,snap,7,llm_error,asset_resource,False,,"Kansas is a non-BBCE state — it repealed broad-based categorical eligibility under the 2015 HOPE Act — so the federal SNAP countable-resource test applies in full, with a limit of roughly $3,000 for a household containing no elderly or disabled member. The household's stated $20,300 in bank account assets is countable liquid resource that exceeds that limit by nearly sevenfold, so the household is resource-ineligible and SNAP is $0 regardless of how the income side comes out. Every wrong model ran the income-and-deduction benefit formula through to a positive allotment; the $20,300 asset figure is the single fact that decides the case, and no model let it bind." -us,scenario_090,state_income_tax_before_refundable_credits,17,llm_error,state_local_rule,False,,"Kansas SB 1 (2024 special session) rewrote the state's individual income tax for 2024 forward: it consolidated the brackets to 5.2% / 5.58% and, decisively here, raised the personal exemption to $9,160 for a single filer ($18,320 married filing jointly, plus $2,320 per dependent), alongside a $3,605 single standard deduction, both inflation-indexed after 2024. A single Kansas filer with no dependents therefore shelters at least $12,765 of Kansas AGI, so this household's $7,020 of Kansas AGI ($7,000 wages + $20 taxable interest; the $4,800 in financial assistance is not taxable) yields zero Kansas taxable income and $0 tax before refundable credits. All 17 wrong models applied either the repealed pre-2024 $2,250 personal exemption, the $2,320 per-dependent amount, or no exemption at all, manufacturing $1,100–$3,520 of taxable income that does not exist. Several compounded this with the repealed 3.1% bottom bracket or invented flat rates of 5.15% and 5.7%." +us,scenario_090,snap,8,llm_error,asset_resource,False,,"Kansas does not use broad-based categorical eligibility (BBCE) for SNAP, so the federal resource test applies. For a household with no elderly or disabled member, the limit is $3,000. The head's $20,300 bank balance exceeds that limit several times over, so SNAP is $0 however the income side works out. Every model produced a positive benefit, either by ignoring the $20,300 in assets or by wrongly claiming that Kansas BBCE waives the asset test." +us,scenario_090,state_income_tax_before_refundable_credits,20,llm_error,taxable_income_or_deductions,False,,"Kansas's 2024 tax law (SB 1) replaced the old $2,250 per-person exemption with a $9,160 personal exemption for a single filer, on top of the $3,605 single standard deduction. Together they shelter $12,765, which is more than this filer's $7,020 Kansas AGI, so Kansas taxable income is $0 and so is the tax. Every model used the pre-2024 exemption ($2,250), the dependent-sized $2,320 figure, a $110 credit, or no exemption at all, which left $900 to $3,520 of phantom taxable income. Several models then made it worse with retired rates (3.1%, 5.15%, 5.5%, 5.7%), but the exemption error alone produces the nonzero answer." us,scenario_090,state_refundable_credits,5,llm_error,categorical_eligibility,False,,"Kansas's refundable individual credit is its EITC, set at 17% of the federal EITC, and the head is 22 — below the age-25 floor for the childless federal EITC — so the federal credit is $0 and the state match on it is $0. The other candidate, the Kansas food sales tax credit, is nonrefundable for tax years after 2012 and in any case requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 in the home, none of which describes a 22-year-old single filer with no dependents. Every wrong answer cleared a household income test ($7,020 is well under both credits' ceilings) and then stopped, never applying the categorical age/dependent gate that actually decides both credits. The correct derivation is 17% × $0 federal EITC plus $0 qualifying food sales tax credit = $0." us,scenario_090,tanf,1,llm_error,categorical_eligibility,False,,"TANF is a benefit for families with dependent children; a single 22-year-old with no children fails Kansas's categorical requirement outright, and the $20,300 in bank assets independently exceeds the state's TANF resource limit, so the correct answer is $0. The trap is the ""financial assistance"" line item in the household facts: it is an unearned-income input, not a TANF payment, and treating it as the answer skips the eligibility determination entirely. Answering this variable requires running the categorical test — dependent child present, resources under the limit — rather than pattern-matching a dollar figure labeled ""assistance"" in the inputs." -us,scenario_091,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The 2026 chain turns on three items: the $16,100 single standard deduction, the qualified-overtime deduction of $627.91 generated by the $628 FLSA overtime premium, and the $120 saver's credit (10% of $1,199 of traditional 401(k), Roth 401(k), traditional IRA and Roth IRA contributions at a $35,012.56 AGI). AGI is $35,012.56 — wages net of the $926.16 pre-tax deferral, plus $4,806 of interest and $4,176 of dividends, less the $43.28 IRA deduction — leaving taxable income of $18,284.66, of which the $3,968 of qualified dividends is taxed at 0%, so $14,316.66 of ordinary income produces $1,470 at 10%/12% and $1,350 after the saver's credit. The wrong answers cluster in five ways: applying TCJA-sunset parameters (personal exemptions, 15% second brackets, ~$8,300 standard deductions), omitting the overtime deduction, deducting the $6,589 of employer-sponsored insurance premiums a second time, fabricating an American Opportunity Credit where no qualified tuition expense is listed, and skipping the saver's credit. Several models also pulled the $1,170 of non-Sch D capital gains into the ordinary bracket base, which the reference leaves out of the income total entirely." +us,scenario_091,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"The reference comes from AGI of $35,012.56 ($27,000 wages less only the $926 traditional 401(k), plus interest and dividends, less the $43 IRA deduction). From that it subtracts the $16,100 2026 OBBBA standard deduction (no personal exemption) and the $627.91 qualified overtime deduction. The $3,968 of qualified dividends falls in the 0% band, leaving $14,316.66 of ordinary income, taxed at 10% up to $12,400 and 12% above for $1,470. A $120 saver's credit (10% of all $1,199 in traditional and Roth 401(k)/IRA contributions) then brings it to $1,350. The AOTC is $0 because no qualified education expenses are listed. The wrong models fell into five traps: dropping the overtime deduction; using pre-OBBBA or TCJA-sunset parameters; treating the $6,589 ESI premiums as a pre-tax wage reduction; claiming an AOTC with zero expenses; and leaving out or understating the saver's credit." us,scenario_091,federal_refundable_credits,10,llm_error,other,False,,"The decisive step is the American Opportunity Credit's base: the credit is 100% of the first $2,000 plus 25% of the next $2,000 of qualified tuition and related expenses, and this household lists no tuition — an unlisted numeric input the prompt explicitly instructs be treated as $0 — so the credit is $0 and its 40% refundable portion is $0. The listed AOTC items (eligible institution, 1098-T/exception, institution EIN, at least half-time, pursuing a credential) are administrative gating conditions, not expenditures, and nine of ten models converted those flags into the statutory maximum $2,500 credit and booked $1,000 (or a mis-scaled $400/$1,200) of refundability. The EITC is separately $0 three times over: the head is 22, below the age-25 floor for a childless filer, and both the ~$36,000 AGI and the childless phase-out ceiling foreclose it; with no children there is no refundable CTC, no recovery rebate credit and no refundable payroll tax credit, leaving $0." us,scenario_091,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Wisconsin Medicaid eligibility requires qualification through an applicable categorical pathway and its income rules. This 22-year-old nondependent falls into no Medicaid category, and MAGI of 2.19 times FPL does not satisfy an adult eligibility pathway, so immigration eligibility alone does not confer coverage." -us,scenario_091,payroll_tax,16,llm_error,payroll_tax_base,False,,"The reference is straight statutory FICA on the reported wage: 6.2% × $27,000 = $1,674.00 plus 1.45% × $27,000 = $391.50 = $2,065.50, with no Additional Medicare Tax (wages far below $200,000), no Social Security wage-base bite, and no Wisconsin employee-side payroll tax since the state's unemployment insurance is employer-financed and it levies no employee disability or paid-leave contribution. The trap that separated the reference from most wrong answers was the payroll tax base: models converted the household's premium and contribution inputs into pre-tax exclusions, subtracting the $6,589 employer-sponsored insurance premium (or the $1,800 other premium) from Social Security and Medicare wages, or adding the $628 FLSA overtime premium that the prompt already includes inside gross wages. The remaining failures were rate and arithmetic errors on the correct base — a 7.5% combined rate, a doubled 2.9% Medicare rate, a botched sum, and a fabricated rounding step — plus one invented Wisconsin state payroll tax that was really state income-tax withholding." +us,scenario_091,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 6.2% Social Security plus 1.45% Medicare on the full $27,000 of gross wages, which comes to $1,674.00 + $391.50 = $2,065.50. The listed $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums do not lower the FICA wage base. The $628 FLSA overtime premium is already inside the $27,000 gross, and Wisconsin has no mandatory employee state payroll tax. Most of the wrong models treated the ESI premiums as a pre-tax Section 125 deduction and taxed only $20,411. The rest added the overtime twice, invented a state payroll tax, used the wrong combined rate, or made an arithmetic or reporting slip." us,scenario_091,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_091,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Wisconsin's answer hinges on three state-specific mechanics: the sliding-scale standard deduction, which at the reference AGI of $35,012.56 equals $12,067.66 (a base near $13,900 reduced by 12% of AGI above roughly $20,076, and not exhausted until about $132,000 of AGI); the separate $700 personal exemption deduction; and the 2026-indexed rate schedule whose 3.5% band runs to about $15,013 before 4.4% applies to the remainder of the $22,244.90 taxable income. The reference base is wages net of the $926 traditional 401(k) and $43 IRA deductions plus $4,806 interest and $4,176 of dividends, with the $1,170 non-Schedule-D capital-gain distribution outside it; nearly every wrong model instead built $36,183 and then guessed the deduction anywhere from $0 to $14,050. The high answers came from declaring the standard deduction fully phased out at $36,000 of AGI; the low ones from stripping the $6,589 employer-sponsored premium or the $1,800 health premiums out of the wage base; and the zeros from crediting the tax away with the federal American Opportunity Credit or the $300-capped school property tax credit, neither of which offsets Wisconsin tax on these facts." +us,scenario_091,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference starts with wages net of the $926.16 401(k) deferral ($26,073.84), adds $4,806 interest and $4,176 dividends, leaves the $1,170 non-Schedule-D capital gain distributions out of gross income, and subtracts the $43.28 IRA deduction. That gives WI income of $35,012.56. Wisconsin's sliding-scale standard deduction at that income is $12,067.66, and the $700 personal exemption is also a deduction, not a credit. That leaves $22,244.90 of taxable income, taxed at 3.5% on the first ≈$15,010 and 4.4% above that, for $843.66. Most models put the $1,170 into income (often net of a 30% exclusion) and got the 2026 sliding standard deduction wrong: some treated it as fully phased out, some applied no phase-down, and others phased it down too steeply. Many also dropped the $700 exemption or used old bracket widths or pre-2023 rates. The rest zeroed out the tax with credits that don't apply here (AOTC, School Property Tax Credit) or subtracted ESI premiums, overtime, or health premiums from income. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 843.66; the corrected value is 878.52. PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%. Reference is an engine defect; output excluded from scoring (Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5)." us,scenario_091,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_092,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Only the $14,980 taxable pension enters adjusted gross income; the Social Security benefit contributes no taxable amount under the federal provisional-income calculation. The age-77 single filer’s applicable standard deduction exceeds that AGI, leaving zero taxable income and therefore zero federal income tax before refundable credits." us,scenario_092,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and disability do not independently establish Alabama Medicaid eligibility; the person must qualify through a specific Medicaid category. This person receives no SSI, has MAGI income of 2.03 times FPL, and qualifies through none of the available pathways, so the Medicaid category is NONE and eligibility is No." us,scenario_092,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered employment earnings, not Social Security retirement benefits or taxable private pension income. With no wages or other payroll-taxable earnings listed, the employee Social Security and Medicare tax base is zero, yielding payroll tax of $0." us,scenario_092,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_092,state_income_tax_before_refundable_credits,20,llm_error,state_local_rule,False,,"Alabama's individual income tax does not reach either of this household's income sources: Social Security is excluded, and payments from a private defined-benefit pension plan (IRC 414(j)) are exempt from Alabama gross income under Ala. Code § 40-18-19, so the $14,980 pension never enters Alabama AGI. With Alabama AGI at $0, taxable income is $0 and the 2%/4%/5% brackets never engage, producing $0 of tax before refundable credits regardless of which deduction and exemption stack is layered on top. Every wrong model applied only the Social Security exclusion, carried the full $14,980 pension into Alabama taxable income, and then differed only in which standard/itemized deduction, personal exemption, and phantom age-65 exemption it subtracted before running the brackets. Several compounded that with Alabama parameters that do not exist: a $1,000 or $1,500 age-65 exemption, a 1.5% bottom bracket, a 4.95% top rate, a $2,500 or $4,200 standard deduction, and a $4,000 single personal exemption." +us,scenario_092,state_income_tax_before_refundable_credits,23,llm_error,taxable_income_or_deductions,False,,"Alabama excludes Social Security and also lets a taxpayer aged 65 or older exclude the first $6,000 of taxable retirement income, including private pension income. That leaves Alabama AGI of $14,980 - $6,000 = $8,980. Itemized deductions of $9,028 ($2,824 mortgage interest, $1,094 real estate tax and $5,110 in charitable gifts) exceed that AGI, so taxable income is zero even before the $1,500 personal exemption, and the tax is $0. Every model taxed the full $14,980 pension. The models that itemized correctly still landed at $182.60 ($4,452 taxable). The models that used the standard deduction, or invented an age-65 exemption, landed higher." us,scenario_092,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_093,dependent1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not confer Missouri Medicaid eligibility. The person must satisfy a specific non-MAGI pathway's categorical and financial requirements; this 27-year-old dependent receives no SSI, qualifies for no non-MAGI category, and at 2.82 times FPL exceeds the applicable MAGI limits." us,scenario_093,dependent1_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's is_medicare_eligible turns on a single age test — age 65 or older — with the narrow ESRD/ALS entitlements not modeled and no disability branch at all; Dependent 1 is 27, so the variable is False. Every wrong model substituted the household's bare ""is disabled"" flag for that age test, asserting a disability-based Medicare pathway either as a PolicyEngine rule or as a real-world shortcut. The statutory disability pathway requires 24 months of SSDI entitlement, and Dependent 1 has $45,000 in annual wages (~$3,750/month), far above substantial gainful activity, plus no listed SSDI receipt — the prompt directs unlisted benefit receipt to be treated as zero, so no disability pathway is reachable on these facts." -us,scenario_093,federal_income_tax_before_refundable_credits,39,llm_error,household_unit_or_filing_status,False,,"The joint return here carries only the filers' income: the 27-year-old disabled dependent's $45,000 of wages is excluded from tax-unit AGI, yet that dependent still generates a $500 nonrefundable credit for other dependents. The correct chain is $118,000 of head/spouse wages less $4,058.12 of traditional 401(k) deferrals plus $1,156 of interest = $115,097.88 gross income, less $2,689.62 of student-loan-interest (capped) and traditional IRA adjustments = $112,408.25 AGI, less the 2026 MFJ standard deduction of $32,200 = $80,208.25 taxable, taxed at 10% to $24,800 and 12% above = $9,128.99, less $500 = $8,628.99. Nearly every wrong answer either folded the dependent's $45,000 into the joint return, bolted a second single return for that person onto the household total, or dropped the $500 credit by applying a qualifying-relative gross income or support test to a permanently disabled child. A second cluster double-deducted the $8,389-per-person employer-sponsored insurance premiums from wages already reported net of them, or replaced 2026 OBBBA parameters with pre-TCJA personal exemptions." +us,scenario_093,federal_income_tax_before_refundable_credits,45,llm_error,household_unit_or_filing_status,False,,"The single tax unit is a married-filing-jointly return, with the permanently disabled 27-year-old claimed as a dependent. Only the head's and spouse's income counts: wages net of the $4,058.12 traditional 401(k), plus $1,156 interest, minus $2,500 student loan interest and $189.62 IRA. That gives AGI of $112,408.25 and taxable income of $80,208.25 after the $32,200 2026 standard deduction, $9,128.99 of tax under the permanent TCJA/OBBBA brackets (10% to $24,800, then 12%), and $8,628.99 after the $500 credit for other dependents. Most wrong answers either added the dependent's $45,000 wages to the joint return or added a separate single-return tax for the dependent. Others subtracted employer-sponsored insurance (ESI) premiums from wages, denied the credit for other dependents (ODC) by applying the qualifying-relative gross income test to a disabled qualifying child, assumed the TCJA had expired, or took an auto-loan-interest deduction." us,scenario_093,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_093,payroll_tax,16,llm_error,payroll_tax_base,False,,"Employee payroll tax here is a flat 7.65% on the three earners' full gross wages — 6.2% Social Security on $163,000 (no one approaches the wage base) plus 1.45% Medicare on $163,000, with no Additional Medicare Tax (no individual exceeds $200,000) and no Missouri employee payroll tax — giving $10,106 + $2,363.50 = $12,469.50. Two distinct traps split the wrong answers. Six models subtracted the $8,389 employer-sponsored insurance premium from each worker's FICA base as a Section 125 cafeteria-plan exclusion, cutting the base to $137,833 and landing on $10,544.22; PolicyEngine's payroll tax base is gross wages and salaries, unreduced by ESI premiums, 401(k)/IRA contributions, or student loan interest. The rest identified the correct rule and correct per-person components but submitted a number their own arithmetic does not produce, most of them missing $12,469.50 by tens or hundreds of dollars in the final addition or transcription." -us,scenario_093,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"The Missouri answer turns on three state-specific steps. The tax unit is the head and spouse only: their gross income is $115,097.88 (wages net of $4,058 of traditional 401(k) deferrals plus $1,156 of interest), less $2,689.62 of above-the-line deductions ($2,499.62 of capped student loan interest plus $190 of traditional IRA) for Missouri AGI of $112,408.25, while Dependent 1's $45,000 of wages never enter the return and generate no Missouri liability. That AGI is reduced by the $32,200 conforming standard deduction and by Missouri's federal income tax deduction of 5% of the $8,629 federal liability ($431.45), the rate that applies in the $100,001–$125,000 Missouri AGI band, leaving $79,776.80. Missouri then taxes married filing combined returns by running each spouse's share through its own graduated schedule ($1,453.04 + $1,936.62 = $3,389.65), about $180 less than one schedule on the combined amount. The wrong answers cluster on folding the dependent's wages or a separate dependent tax into the household total, applying one bracket run or a flat 4.7% to combined income, and mishandling the federal income tax deduction as fully phased out, capped at $10,000, or worth 35% of federal liability." +us,scenario_093,payroll_tax,18,llm_error,payroll_tax_base,False,,"The reference is 7.65% of the full $163,000 in gross wages ($53,000 + $65,000 + $45,000). That is 6.2% Social Security ($10,106), since every earner is below the 2026 wage base, plus 1.45% Medicare ($2,363.50). There is no Additional Medicare Tax and no Missouri employee payroll tax. The main trap is the Section 125 assumption: eight models subtracted each worker's $8,389 of employer-sponsored insurance premiums from FICA wages and got $10,544.22 (7.65% of $137,833). No listed fact makes those premiums salary-reduction cafeteria-plan contributions, so the listed gross wages are the FICA base. Most of the remaining models used the right base and rates, and several showed $12,469.50 in their own working, but they added up wrong or submitted a different number from the one they computed." +us,scenario_093,state_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"Only the spouses' income enters the Missouri tax base. The disabled 27-year-old is claimed on the couple's return inside the single tax unit, so the dependent's $45,000 of wages does not join the couple's AGI and does not create a separate Missouri return. From MO AGI of $112,408.25, Missouri allows the $32,200 standard deduction and a federal-tax deduction of 5% ($431.45), because the AGI falls in the $100,001–$125,000 band. That AGI already subtracts the 401(k) and IRA contributions and applies the $2,500 student-loan interest cap; ESI premiums are not subtracted. Missouri then splits the $79,776.80 of taxable income between the spouses by income percentage and applies the 2026 schedule (4.7% top rate) to each share, giving $1,452.33 + $1,935.91. Most models went wrong in one of these ways: taxing the dependent, running one schedule on the joint total, skipping the 5% federal-tax deduction, or using stale rates or deductions or exemptions that don't exist." us,scenario_095,federal_income_tax_before_refundable_credits,3,llm_error,household_unit_or_filing_status,False,,"The head's surviving-spouse status determines the federal filing-status pathway and supplies a standard deduction large enough to eliminate the $15,286.89 of AGI. Social Security is not taxable at this provisional-income level, so it is excluded from AGI; after the half-self-employment-tax deduction, taxable income and income tax before credits are both zero." us,scenario_095,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Federal refundable credits include only the refundable credit components applicable under 2026 law. Age 73 does not independently create a $1,200 refundable senior credit, and all five applicable components—EITC, refundable American Opportunity Credit, refundable CTC, Recovery Rebate Credit, and refundable payroll tax credit—equal zero." us,scenario_095,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -722,8 +731,8 @@ us,scenario_095,payroll_tax,2,llm_error,payroll_tax_base,False,,"The payroll_tax us,scenario_095,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_095,self_employment_tax,6,llm_error,payroll_tax_base,False,,"Self-employment tax here is a single mechanical chain: net earnings from self-employment are 92.35% of the $4,198 of net profit ($3,876.85), that base clears the $400 §1402(b) floor, and the combined 15.3% rate (12.4% OASDI + 2.9% HI) on it produces the reference $593.23. Nothing in the household facts alters it — age 73, Social Security receipt, the 401(k) distribution, and the surviving-spouse status are all irrelevant to §1401 liability, and the above-the-line deduction for half of SE tax reduces income tax, never the SE tax itself. The six wrong answers each break one link: two zero out the tax (one by inverting the $400 comparison, one by treating the half-SE-tax deduction as an offset against SE tax), two mangle the 92.35% base factor (rounding it to 92%, or omitting it and taxing gross $4,198), one submits a number contradicting its own correct arithmetic, and one returns nothing." us,scenario_095,snap,5,llm_error,thresholds_rates,False,,"The household’s $29,605 annual income exceeds the applicable 2026 New Jersey SNAP gross-income eligibility limit for one person, so the SNAP calculation stops with no eligibility and a $0 allotment. Broad-based categorical eligibility, elderly-household deductions, and the minimum-allotment rule do not produce a benefit after this income screen fails." -us,scenario_095,state_income_tax_before_refundable_credits,18,llm_error,state_local_rule,False,,"New Jersey does not compute this household's tax by running the 1.4% bottom bracket over post-exemption income: N.J.S.A. 54A:2-1.1 imposes no gross income tax at all when NJ gross income (Form NJ-1040 line 29, measured after the pension/retirement exclusion) is at or below $20,000 for a qualifying surviving spouse/CU partner, head of household, or joint filer, or $10,000 for a single filer. Here Social Security ($14,022) is excluded from NJ gross income entirely and the age-62+ retirement income exclusion (up to $75,000 for this filing status) removes the full $11,385 401(k) distribution, leaving NJ gross income of $4,198 — below both thresholds — so the statutory exemption zeroes the liability before any bracket, exemption, or medical deduction matters. Every wrong model built a defensible NJ taxable base and then multiplied it by a bracket rate, producing liabilities of $18.46 to $183.29 instead of $0.00. The head's explicit ""surviving spouse"" status, which sets the $20,000 threshold, was used by the models only to size the retirement exclusion and never to trigger the zero-tax provision." -us,scenario_095,state_refundable_credits,3,llm_error,state_local_rule,False,,"New Jersey's refundable gross income tax credits are the NJ EITC (a 40% match on the federal EITC), the NJ Child Tax Credit (requires a child under 6), and the refundable NJ child and dependent care credit (requires work-related care expenses). This 73-year-old childless filer has a federal EITC of $0 because the no-qualifying-child EITC is barred above age 64, and with no children and no care expenses the other two credits are $0, so state_refundable_credits is $0. The trap is reaching outside the income tax: the ANCHOR property-tax relief payment is not a refundable income tax credit, and the household's only housing fact is a $31,000 mortgage balance with $0 property tax and $0 rent. The second trap is manufacturing an ""age-adjusted"" federal EITC base for the state match, when the state match multiplies the federal credit as actually computed." +us,scenario_095,state_income_tax_before_refundable_credits,22,llm_error,thresholds_rates,False,,"New Jersey imposes no income tax when NJ gross income, which excludes Social Security and is measured after the age-62+ pension exclusion, is $10,000 or less for single filers, or $20,000 or less for qualifying widow(er)s and the other joint-type statuses. For this 73-year-old, NJ gross income is $15,583 minus the $11,385 401(k) exclusion, or $4,198. That is below either threshold, so the tax is $0. Almost every model correctly excluded Social Security and the 401(k) distribution. They then treated the leftover $4,198 of self-employment income as taxable, took off exemptions and sometimes medical expenses, and applied the 1.4% bracket. None applied the gross-income threshold that zeroes the liability." +us,scenario_095,state_refundable_credits,4,llm_error,credit_phaseout,False,,"This household has no children, so the NJ EITC is its only possible refundable credit. For a childless filer aged 65 or older, New Jersey computes the credit as 40% of a hypothetical federal childless EITC. The phase-in gives 7.65% × earned income of about $3,901 ($4,198 SE income minus $296.58, the deductible half of SE tax), or $298.45. The phase-out then takes 7.65% × (AGI − non-joint threshold). AGI is about $15,286 because it includes the $11,385 401(k) distribution (Social Security is untaxed because provisional income is below $25,000). Since the 401(k) distribution alone exceeds the roughly $11k threshold, the reduction is larger than the phase-in amount and the credit is $0. The household paid no property tax or rent, so the NJ property tax credit is also $0, and ANCHOR is not an income tax credit." us,scenario_095,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_098,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The $11,100 farm rental loss offsets the $15,540 taxable pension, leaving $4,440 of non-Social-Security income. Provisional income is then below the threshold for taxing Social Security benefits, so AGI remains $4,440; the applicable standard deduction eliminates taxable income and produces zero federal income tax before refundable credits." us,scenario_098,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -749,15 +758,15 @@ us,scenario_099,child2_head_start_eligible,2,llm_error,categorical_eligibility,F us,scenario_099,child2_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility requires both categorical eligibility and financial eligibility; being age four satisfies only the child-category requirement. The household's approximately $159,676 of annual income exceeds California's WIC limit of 185% of the federal poverty level for a four-person household, so child 2 is not eligible." -us,scenario_099,federal_income_tax_before_refundable_credits,39,llm_error,credit_phaseout,False,,"2026 runs on OBBBA-extended parameters, and the whole case turns on three of them: the MFJ standard deduction is $32,200 with a $2,000 non-itemizer charitable deduction stacked on top (a $34,200 floor that beats the household's ~$27,500 of itemizable SALT-plus-charity), the CTC is $2,200 per child and fully absorbable here as a nonrefundable credit ($4,400), and the CDCC rate phases down to 32% at $159,585 of AGI, yielding $1,920 on the $6,000 two-child expense cap. The reference chain is $161,912.80 of wages after $3,087.20 of traditional 401(k) deferrals plus $1,010 interest and $144 dividends = $163,066.80, less $3,481.76 of above-the-line items (the $2,999.76 capital-loss limitation on the $6,478 net loss, $338 educator expense, $144 traditional IRA) = $159,585.05 AGI; less $34,200 = $125,385.05 taxable; $16,980.55 of ordinary tax on the 10/12/22% MFJ schedule (24,800 / 100,800 breakpoints) plus $19.20 on $128 of qualified dividends = $16,999.75; less $6,320 of credits = $10,679.75. Every wrong answer breaks one of three links: assuming the TCJA sunset arrived (personal exemptions of ~$21,200 and 10/15/25% brackets), dropping the $2,000 charitable add-on to the standard deduction, or pricing the credits at pre-OBBBA levels — $2,000-per-child CTC, a fictitious $600–$800 ""nonrefundable portion"" of the CTC, or a 20% CDCC rate." +us,scenario_099,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference uses 2026 law as set by OBBBA. That means a $32,200 standard deduction plus the new $2,000 joint deduction for charitable gifts by non-itemizers, which gives taxable income of $125,385.05. The 2026 brackets are 10% to $24,800, 12% to $100,800, then 22%, which gives $16,999.75 of tax. The full $2,200-per-child CTC ($4,400) is used against that liability. The CDCC is 32% of $6,000 ($1,920): OBBBA's rate falls from 35% by one point per $4,000 of joint AGI over $150,000. The models most often took the CDCC at the old 20% rate, left out the $2,000 charitable deduction, used a $2,000 CTC, or treated only a sliver of the CTC as nonrefundable. A large group applied expired pre-TCJA law (personal exemptions, 15/25% brackets, a $1,000 CTC phased out above $110,000), even though OBBBA made the TCJA structure permanent. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 10,679.75; the corrected value is 10,711.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_099,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The trap is §24(d)'s liability-limitation prong: the refundable CTC is the *lesser* of the earned-income formula (15% of earnings over $2,500), the per-child refundable cap, and the portion of the CTC left unused after the credit offsets tax liability. This married-filing-jointly household has AGI near $159,676 (wages $165,000 plus $1,010 interest, $144 dividends, capital netting limited to a $3,000 loss, less $3,232 of traditional 401(k)/IRA contributions and the $338 educator expense) and roughly $127,000 of taxable income after the 2026 MFJ standard deduction, producing about $17,400 of federal income tax before credits — far more than the $4,400 of 2026 OBBBA CTC ($2,200 × 2), so the entire CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is zero because $159,676 is far beyond the two-child MFJ phase-out, and the CDCC is nonrefundable. Every wrong model treated a refundability cap or the earned-income formula as a standalone entitlement instead of applying the unused-credit ceiling." -us,scenario_099,free_school_meals_eligible,33,llm_error,state_local_rule,False,,"The trap is that school meal tier in California is not decided by the federal NSLP income ladder. The household's countable income of $166,154 against a $33,000 poverty guideline for four gives an FPG ratio of 5.03, far above the 130% free and 185% reduced-price cutoffs, and no SNAP/TANF categorical eligibility exists — every wrong model computed that much correctly. What they all skipped is California's Universal Meals Program, which serves free breakfast and lunch to all enrolled K-12 students statewide regardless of income, forcing school_meal_tier to FREE; applied to the one K-12 child (age 9) at the $7.15 daily subsidy, that produces $1,130.96 of annual free-meal support. Every model stopped at the federal income test instead of checking the state-universal override." +us,scenario_099,free_school_meals_eligible,38,llm_error,state_local_rule,False,,"California runs a universal school meals program: since 2022-23 every public K-12 student gets free breakfast and lunch whatever the household income. PolicyEngine therefore sets school_meal_tier to FREE for Child 1, the 9-year-old K-12 student, even though the household's FPG ratio is 5.03 and it has no SNAP or TANF categorical eligibility. Every wrong model applied the federal NSLP income test, which requires income at or below 130% of FPL for free meals. None applied the state universal-meals override, and all concluded No." us,scenario_099,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_099,payroll_tax,38,llm_error,state_local_rule,False,,"Federal employee FICA is uncontested here: 6.2% Social Security on all $165,000 (both earners are far under the per-worker wage base) plus 1.45% Medicare gives $12,622.50, with no Additional Medicare Tax. The entire spread to $14,767.50 is California's mandatory employee SDI contribution, which SB 951 made uncapped from 2024 and which is set at 1.3% for 2026: 1.3% x $165,000 = $2,145. The prompt explicitly puts ""mandatory employee state payroll taxes"" in scope, so the two dominant failure clusters are dropping CA SDI entirely and carrying a stale SDI rate of 1.1% or 1.2% forward from prior years. A third cluster subtracted the $24,597 of employer-sponsored insurance premiums from the wage base, when payroll taxes apply to the full $165,000 of gross employment income." +us,scenario_099,payroll_tax,40,llm_error,state_local_rule,False,,"Federal FICA is straightforward here: both earners are under the Social Security wage base and the pair is under the $250,000 MFJ Additional Medicare threshold, so SS is $10,230 and Medicare is $2,392.50 on the full $165,000 of gross wages. What separates the reference from the wrong answers is California's mandatory employee SDI. For 2026 it is 1.3% of all wages with no cap, which is $2,145. Models either left SDI out (12,622.50), used an older SDI rate (1.1% gives $14,437.50, 1.2% gives $14,602.50), or wrongly took the employer-sponsored insurance premiums out of the FICA/SDI wage base." us,scenario_099,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -766,7 +775,7 @@ us,scenario_099,spouse_medicaid_eligible,1,parse_contract_failure,missing_output us,scenario_099,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_099,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"CA AGI here is federal AGI of $159,585.05: $165,000 of wages less the $3,088 of traditional 401(k) deferrals, plus $1,010 of interest and $144 of dividends, less the $3,000 capital-loss limitation on the net $6,478 loss and the $144 IRA and $338 educator above-the-line deductions. The household itemizes on the CA return for $20,476.57 — $8,653.50 of real estate taxes (California imposes no SALT cap) plus $11,823.08 of charity after the 2026 0.5%-of-AGI floor trims the $12,621 given — which beats the $11,412 CA joint standard deduction by $9,065, and the 7.5%-of-AGI medical floor of $11,968.88 leaves no medical deduction. Taxable income of $139,108.47 runs to $5,778.21 under the indexed 2026 MFJ brackets, and two personal plus two dependent exemption credits totaling $1,284.46 bring it to $4,493.74. The wrong answers split three ways: taking a standard deduction instead of itemizing, stripping roughly $24,597 of employer-sponsored insurance premiums out of wages already reported net of them, and mis-sizing either the bracket schedule or the exemption credits." +us,scenario_099,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"CA AGI is $159,585. That is gross wages less the $3,088 in 401(k) deferrals, the $144 IRA deduction and the $338 educator deduction, plus interest and dividends, with the capital loss limited to $3,000. The household itemizes $20,477: $11,823 of charity, which is $12,621 minus the 0.5%-of-AGI charitable floor, plus $8,654 of property tax. There is no medical deduction because medical costs of $10,000 ($11,400 with over-the-counter items) are below the 7.5%-of-AGI floor of about $11,969. That leaves $139,108 of taxable income, which reaches only the 8% bracket. The frozen reference prices it on the engine's projected 2026-indexed MFJ schedule, $5,778 of tax, and subtracts $1,284.46 of projected personal and dependent exemption credits for $4,493.74; the release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $5,896.78 less $1,256 on the same income. The wrong answers came from a few repeated mistakes: counting the $2,500 premium twice to claim a medical deduction, subtracting the $24,597 of employer-sponsored insurance premiums from income, taking the standard deduction instead of itemizing, using 2024 brackets rather than the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, ignoring the 0.5% charitable floor, and getting the exemption credits wrong or leaving them out. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,493.74; the corrected value is 4,588.48. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076)." us,scenario_099,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_099,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_100,child1_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child not to qualify for Medicaid. At age 6 and this income, Child 1 qualifies for Montana Medicaid under the OLDER_CHILD category, so Medicaid eligibility categorically precludes CHIP eligibility; the CHIP income ceiling and existing employer-sponsored insurance do not alter that result." @@ -778,7 +787,7 @@ us,scenario_100,child2_early_head_start_eligible,1,llm_error,categorical_eligibi us,scenario_100,child2_head_start_eligible,4,llm_error,age_disability,False,,"Head Start for preschool-age children covers children from age 3 through compulsory school age, so a 5-year-old is squarely inside the band — it is the 6-year-old Child 1 who falls outside it, and Early Head Start that stops below age 3. On top of that, the household's TANF receipt of $505.41 monthly confers categorical eligibility on all members, and its $1,875.40 adjusted gross income is far below the poverty guideline for three people, so income eligibility holds independently. Three models cut Head Start off at age 4 and one defaulted the flag to zero with no derivation; all four therefore never applied the categorical TANF pathway or the income test that both resolve to eligible." us,scenario_100,child2_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Montana Medicaid applies a young-child eligibility pathway to the 5-year-old dependent. The child's household MAGI is 0.07 times FPL, below the applicable young-child income threshold, so the eligibility test returns Yes." us,scenario_100,child2_wic_eligible,11,llm_error,categorical_eligibility,False,,"WIC child categorical eligibility applies only to children under age five; a child whose stated age is 5 has reached the fifth birthday and is outside the child category. The prompt holds age constant for the full year and supplies no existing certification-period status, so low household income cannot establish eligibility after the categorical test fails." -us,scenario_100,federal_refundable_credits,33,llm_error,taxable_income_or_deductions,False,,"Both refundable credits here are earnings-driven, and the trap is the earned-income base: the full $5,915 of gross wages feeds the two-child EITC 40% phase-in ($2,365.89) and the refundable CTC's 15%-of-earnings-above-$2,500 formula ($512.21), for $2,878.10. Elective traditional 401(k)/IRA contributions and the health-premium lines reduce AGI or are plan costs, not the earned-income base, and the $710 FLSA overtime premium is already inside the $5,915. The wrong answers cluster into four groups: subtracting the $3,859 deferral to get $2,056 of ""Box 1"" earned income ($822.40, or lower when premiums are also subtracted), reverting the refundable-CTC threshold to the pre-TCJA $3,000 ($2,803.25), asserting $0 on the theory that low earnings or pre-tax premiums extinguish the credits, and inflating the refundable CTC to the $1,700-per-child cap while skipping the 15% earnings limitation." +us,scenario_100,federal_refundable_credits,35,reference_engine_defect,credit_phaseout,False,,"The reference uses gross wages of $5,915 as earned income for both credits. Those wages already include the $710 FLSA overtime premium. The traditional 401(k) deferral, the IRA contributions and the health premiums do not reduce this figure. With two children, the 40% EITC phase-in gives $2,365.89. The refundable CTC is 15% × ($5,915 − $2,500) = $512.21, using the 2026 $2,500 threshold that OBBBA made permanent, for a total of $2,878.10. The largest group of wrong answers subtracted the $3,859 401(k) deferral, leaving $2,056 of earned income, which gives $822.40 of EITC and no refundable CTC. The second group used the pre-TCJA $3,000 CTC threshold and got $437.25 instead of $512. The rest either returned zero, double-counted overtime, dropped the earned-income limit on the refundable CTC, or submitted a number different from the one they had derived. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,878.10; the corrected value is 822.29. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." us,scenario_100,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Free school meals are awarded through two independent pathways, and this Montana household clears both: it receives $8,625.89 of SNAP, which categorically confers free-meal status, and its countable income of $11,979.68 ($5,914.72 wages plus $6,064.96 TANF) is 44% of the federal poverty guideline for a three-person household, far under the 130% free-meal ceiling. The two children, ages 5 and 6, are both school-age, so the free tier pays a $7.15 daily per-child subsidy for the contiguous-US school year, $2,261.92 annually. The trap is that SNAP and TANF receipt are derived outputs of this same household, not listed input facts, so a model that only credits explicitly stated benefit receipt loses the categorical pathway — but even the wage-only figure of $5,915 sits at roughly 22% of the poverty guideline, so the income test alone still lands in the free tier. Both models returned 0 without running either test." us,scenario_100,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The head qualifies through Montana’s parent/caretaker Medicaid category because the household includes dependent children and MAGI equals 0.07 × FPL, below the category’s 100% FPL limit. This MAGI pathway does not apply an asset test, so the bank balance and employer-sponsored insurance do not negate eligibility." us,scenario_100,head_medicare_eligible,2,llm_error,categorical_eligibility,False,,"Medicare eligibility below age 65 does not follow from a current disability flag alone; it requires a qualifying Medicare pathway, such as entitlement after the applicable Social Security disability waiting period or eligibility based on end-stage renal disease. The 46-year-old head has no listed Medicare-qualifying entitlement or condition, so the disability input does not change the result from ineligible." @@ -786,27 +795,27 @@ us,scenario_100,head_wic_eligible,4,llm_error,categorical_eligibility,False,,"WI us,scenario_100,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_100,payroll_tax,7,llm_error,payroll_tax_base,False,,"The trap is the payroll-tax wage base: PolicyEngine assesses employee OASDI at 6.2% and HI at 1.45% on the full $5,915 of employment income, giving $366.71 + $85.76 = $452.48. None of the listed health inputs touch that base — the $20,644 employer-sponsored insurance premium and the two $564 individually paid premiums are health-coverage and medical-expense inputs, not Section 125 salary reductions — and the $710 FLSA overtime premium is already inside the annual gross wage figure rather than additional to it. Traditional 401(k) and IRA amounts also leave FICA wages untouched, and Montana imposes no mandatory employee-side state payroll tax, so the answer is exactly 7.65% of wages. The wrong answers split into base errors (subtracting premiums to $4,787 or to $0, or adding the overtime premium to reach $6,625) and, for the three models that found the right base, an arithmetic slip, an invented rounding, and a value/explanation mismatch." us,scenario_100,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier. This household is at 44% of the federal poverty guideline and also meets categorical eligibility, so its school_meal_tier is FREE and reduced_price_school_meals_eligible is 0." -us,scenario_100,snap,39,prompt_ambiguity,age_disability,False,,"The reference household is not a wages-only case: PolicyEngine pays it a TANF cash grant, counts that grant as SNAP unearned income (gross $998.31/month against the $492.92 wage figure nearly every model used), and grants a $453.64/month excess shelter deduction alongside the $209 standard and $98.58 earned-income deductions, leaving net income of $237.09 and a household contribution of $71.10/month. Calendar year 2026 also spans a maximum-allotment change, so the three-person ceiling is $785 for most months and $802.60 later, an annual maximum near $9,479 rather than a flat $9,420. Almost every wrong answer collapses net income to near zero by applying only the standard, 20% earned-income, and excess-medical deductions to wages alone, then pays twelve flat months at a guessed maximum — landing at $9,000–$9,420. The remaining errors are a different family: benefits above the maximum-allotment ceiling, monthly figures reported as annual, unexplained trims to a self-computed total, and outright zeros from asserted income or resource ineligibility. Developer adjudication (2026-09-05): the judge (gpt-5.6-sol) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules." +us,scenario_100,snap,46,prompt_ambiguity,age_disability,False,,"The reference comes from three income and deduction steps that partly cancel out. First, SNAP gross income is $998.31/month: $492.92 of wages plus $505.39 of Montana TANF cash, which is the $725.08 payment standard minus 75% of earnings above $200. Second, the head's medical costs are not deductible because the head gets no SSI or SSDI: $2,800 in the bank is over SSI's $2,000 resource limit. So the head does not meet SNAP's disability definition in 7 U.S.C. 2012(j). Third, although no rent is listed, the engine grants the heat-and-eat standard utility allowance to a household without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended (engine defect r30_snap_heat_and_eat_sua, not fixed upstream), so the frozen reference takes a $453.64 excess shelter deduction from the utility allowance. That leaves $237.09 net income, a $71.10 contribution and $713.90/month for January–September (at the $785 maximum); the frozen reference's $733.60/month for October–December uses the engine's projected FY2027 schedule, whereas the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule, including the $785 maximum, for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $713/month ($8,556). Most models counted wages only, took a disabled-member medical deduction and took no shelter deduction, landing at $9,000–$9,420. The record's correction of r30_snap_heat_and_eat_sua also takes no utility-allowance shelter deduction and, with TANF counted and no medical deduction, gives $691 net income, a $208 contribution and $577/month, moving the output from $8,556 to $6,924. Models that did count TANF also took no shelter deduction, as that correction does, and came in above its $6,924; most came in below the frozen reference. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification." us,scenario_100,ssi,9,llm_error,categorical_eligibility,False,,"The generic `is disabled` household fact does not make the head categorically eligible for SSI under PolicyEngine; the controlling SSI-specific indicator is `is_ssi_aged_blind_disabled=False`. All three individuals fail the aged-65-or-older, blind, or qualifying-disabled requirement, so income exclusions, resources, and benefit-rate calculations are never reached and household SSI is $0." -us,scenario_100,state_refundable_credits,38,llm_error,state_local_rule,False,,"The answer is a two-link chain: the federal EITC for a head of household with two qualifying children and $5,915 of wages sits in the 40% phase-in range at $2,365.89 (the $3,859 traditional 401(k) deferral does not reduce EITC earned income), and Montana matches it with a refundable credit at 20% for 2026, giving $473.18. Most wrong answers break the chain at the state rate — using the 3% rate that applied through 2023, the 10% rate that replaced it, or denying Montana has a refundable EITC at all. A second cluster breaks it at the federal base, either subtracting the 401(k) deferral from earned income (yielding $822.40) or zeroing the federal credit outright. The zeros additionally rest on treating the Montana EITC as nonrefundable and therefore extinguished by the absence of Montana tax liability; the credit is refundable and is paid in full." -us,scenario_100,tanf,39,llm_error,state_local_rule,False,,"Montana's FAIM cash grant equals the three-person payment standard of $725.10 per month minus countable income, and countable income here is $219.69 — $492.92 of monthly wages less the $200 flat disregard and 25% of the remainder — giving $505.41 per month and $6,064.96 for the year. The household clears every gate: two dependent children, a $2,800 bank balance under Montana's $3,000 TANF resource limit, an excluded vehicle, and no unearned income, since that same $2,800 exceeds SSI's $2,000 resource limit and zeroes SSI. The wrong answers split two ways: declaring ineligibility on invented asset limits, imputed SSI income, double-counted overtime premium, or ""not modeled"" assertions; and pricing the grant off the frozen $588 three-person maximum (or a 50% disregard) instead of the 2026 standard of $725.10." -us,scenario_101,federal_income_tax_before_refundable_credits,35,llm_error,thresholds_rates,False,,"2026 uses the OBBBA-permanent schedule: a $16,100 single standard deduction, no personal exemption, and 10/12/22 brackets breaking at $12,400 and $50,400. AGI is $101,130 (wages of $103,000 less the $1,870 net long-term capital loss, with the tax-exempt pension, veterans benefits, and employer-plan premiums all outside AGI), leaving $85,030 of taxable income and $1,240 + $4,560 + 0.22 × $34,630 = $13,418.60. Two traps split the field: one group assumed the TCJA rate structure sunset and revived an ~$8,300 standard deduction plus a $5,300 personal exemption at 10/15/25 rates, or fell back on 2024/2025 figures and inflation guesses; a second group subtracted the $1,400 auto-loan interest, which contributes $0 because every qualified-vehicle status input is unlisted and therefore false. A third group derived $85,030 exactly and then submitted a number its own arithmetic never produced." +us,scenario_100,state_refundable_credits,44,reference_engine_defect,credit_phaseout,False,,"Montana's only refundable credit for this household is the state EITC, and for 2026 it equals 20% of the federal EITC. The federal EITC is the two-child 40% phase-in applied to gross earned wages of $5,915, which gives $2,365.89; traditional 401(k) deferrals do not reduce that base. 20% × $2,365.89 = $473.18. Most wrong answers fall into three groups. One group says Montana has no refundable EITC, or no income tax at all, and answers $0. A second group uses the old 3% or 10% rate, which gives $70.98 or $236.60. A third group shrinks the federal EITC base by subtracting the 401(k) deferral, which gives $822.40. Several models also added a $1,200 young-child credit that is not part of this household's 2026 Montana refundable credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 473.18; the corrected value is 164.46. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." +us,scenario_100,tanf,46,llm_error,thresholds_rates,False,,"Montana TANF for this three-person unit (parent plus children aged 6 and 5) uses a monthly payment standard of $725.10 ($8,701.21 a year). From that it subtracts countable earnings: ($5,915/12 − $200) × 75% = $219.69. That leaves $505.41 a month, or $6,064.96 a year. Resources pass: the $2,800 bank balance is under the $3,000 limit, and the vehicle is excluded. SSI is $0 because the $2,800 exceeds SSI's $2,000 resource limit, so no SSI income offsets TANF. Most nonzero answers used an outdated $588 (or other low) payment standard, or a 50% earnings disregard instead of 25%. The $0 answers invented disqualifiers: asset limits below $3,000, counting the vehicle, imputed SSI, double-counted overtime, or a claim that Montana TANF is not modeled." +us,scenario_101,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference is simple ordinary-income tax on $85,030 of taxable income: AGI of $101,130 ($103,000 wages minus the $1,870 capital loss) less the 2026 single standard deduction of $16,100. The rates are 10% to $12,400, 12% to $50,400 and 22% above, which gives $13,418.60 with no nonrefundable credits. The wrong answers fall into three groups. Eight models assumed the TCJA would sunset and used the $8,300 standard deduction, the $5,300 personal exemption and 10/15/25 brackets, but OBBBA made the TCJA structure permanent. Seven models subtracted the listed $1,400 of auto loan interest as an OBBBA qualified vehicle-loan-interest deduction, although the facts establish no qualifying new, U.S.-assembled vehicle. The rest used stale or invented standard-deduction and bracket figures, or submitted numbers that contradict their own arithmetic." us,scenario_101,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_101,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security tax plus 1.45% Medicare tax on the full $103,000 of wages, with no Additional Medicare Tax or mandatory Texas employee payroll tax. This produces $6,386 plus $1,493.50, totaling $7,879.50; health-insurance premiums do not reduce the payroll-tax base unless established as pre-tax employer-plan deductions." us,scenario_101,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident has $0 of state income tax before refundable credits regardless of the listed taxable-income deductions or credits. Taxes from other jurisdictions do not apply because the household resides in Texas." -us,scenario_102,federal_income_tax_before_refundable_credits,31,llm_error,taxable_income_or_deductions,False,,"The whole test is the 2026 married-filing-jointly parameter set: a $32,200 standard deduction plus a $0.20 QBI deduction on the $1 of self-employment income, applied to the full $87,201 of gross income, then taxed at 10% up to $24,800 and 12% above, giving $55,000.80 of taxable income and $6,104.10 of tax with no dependents and no nonrefundable credits. Two traps split the wrong answers: anchoring the standard deduction on the pre-OBBBA $30,000 (or on a lapsed-TCJA ~$16,000 plus restored personal exemptions and a 15% bracket), and subtracting the spouse's $7,989 employer-sponsored insurance premium a second time from wages that are already stated gross, which strips exactly $958.68 of tax at the 12% rate. A third cluster derived a figure within a few dollars of $6,104.10 and then submitted an unrelated number, and one model subtracted estimated withholding — a payment, not a credit — from its liability." +us,scenario_102,federal_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"The correct path uses the OBBBA-permanent TCJA structure for 2026. The married-filing-jointly standard deduction is $32,200 and the 10% bracket ends at $24,800. Gross wages of $87,200 plus $1 of self-employment income are fully taxable; the $7,989 ESI premium is not a stated salary reduction. The only other deduction is a $0.20 QBI deduction, which leaves $55,000.80 of taxable income, tax of $2,480 + 12% × $30,200.80 = $6,104.10, and no nonrefundable credits. Most models went wrong in one of four ways: they carried pre-OBBBA 2025 amounts ($30,000–$30,800 standard deduction, a $23,850–$24,500 10% bracket) forward; they assumed the TCJA sunset (personal exemptions and a 15% bracket); they subtracted the ESI premium from AGI; or they submitted a number that contradicts their own arithmetic." us,scenario_102,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"Refundable federal credits require a qualifying statutory pathway; disability alone does not create a refundable federal income tax credit. With no qualifying children, education expenses, or other credit-triggering facts, and income above the childless EITC range, every refundable component is zero." us,scenario_102,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish North Carolina Medicaid eligibility; the person must qualify through a specific MAGI or traditional categorical pathway and satisfy that pathway's financial rules. The head's household MAGI is $87,201, or 4.03 times FPL, and the engine assigns medicaid_category = NONE, so medical expenses, rent, and $1,001 in assets do not create eligibility." -us,scenario_102,head_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's medicare_eligible is a pure age test: the person must be at least 65, and both adults here are 26, so the value is False for both. The trap is the listed `is_disabled` flag: every wrong model converted it into an under-65 Medicare pathway, but even the statutory under-65 route (42 U.S.C. 426(b)) requires 24 months of entitlement to SSDI/RRB disability benefits, or ESRD/ALS, none of which is a listed household fact — and the prompt directs that unlisted benefit receipt be treated as absent, while Head's $41,200 in wages is far above the substantial-gainful-activity level that gates SSDI entitlement. Disability status by itself confers Medicare neither in PolicyEngine nor in statute, so the correct answer is 0." -us,scenario_102,payroll_tax,16,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on each spouse’s full listed wages of $41,200 and $46,000. The $7,989 employer-sponsored insurance premium is not identified as an employee pretax contribution and therefore does not reduce the stated payroll-tax wage base; the resulting components are $5,406.40 and $1,264.40, totaling $6,670.80. Several models instead deducted that premium, while others derived the correct components but submitted an inconsistent final value." +us,scenario_102,head_medicare_eligible,7,llm_error,age_disability,False,,"Under age 65, a disability flag alone does not make someone eligible for Medicare. The disability route requires 24 months of Social Security Disability Insurance (SSDI) entitlement, or end-stage renal disease or ALS. The household reports no SSDI income, and the head earns $41,200 in wages, far above substantial gainful activity. So the only pathway PolicyEngine can apply is the age-65 test, and a 26-year-old fails it. Every model treated the 'is disabled' flag as enough for Medicare and skipped the SSDI-duration requirement." +us,scenario_102,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee FICA applies to each spouse's full listed gross wages: 7.65% of $41,200 ($3,151.80) plus 7.65% of $46,000 ($3,519.00) equals $6,670.80. Neither spouse reaches the Social Security wage base, combined wages are well below the $250,000 Additional Medicare Tax threshold, and NC has no mandatory employee state payroll tax. The main trap is the $7,989 employer-sponsored insurance premium input. It is not a salary reduction from the listed gross wages, so subtracting it from the spouse's FICA base produces $6,059.64. Most other misses came from models that reached $6,670.80 but submitted a different number, or that used the wrong Medicare rate or the wrong wage total." us,scenario_102,self_employment_tax,2,llm_error,payroll_tax_base,False,,Self-employment tax applies only when net earnings from self-employment reach the filing threshold; the head’s listed $1 produces no liability. There is no minimum $31 self-employment tax or rounding rule that converts this de minimis amount into a positive tax. -us,scenario_102,spouse_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's Medicare eligibility test is an age test: a person qualifies at age 65 or older, and the household's disability flag feeds SSI, the disabled-household SNAP rules, and medical-expense deductions — not Medicare. The statutory under-65 pathway (42 U.S.C. §426(b)) requires 24 months of entitlement to Social Security disability benefits, or an ESRD/ALS diagnosis; this household lists no Social Security disability income and unlisted inputs are zero, so no under-65 pathway opens. The Spouse is 26, so the age test fails and spouse_medicare_eligible is No. Every wrong model converted the bare ""is disabled"" flag into a standalone Medicare qualifier." -us,scenario_102,state_income_tax_before_refundable_credits,25,llm_error,thresholds_rates,False,,"North Carolina's 2026 flat individual income tax rate is 3.99% (the scheduled step down from 4.25%) and its MFJ standard deduction is a fixed, non-indexed $25,500, so the only correct chain is $87,201 federal AGI − $25,500 = $61,701 taxable × 3.99% = $2,461.87. Three traps separate the reference from the wrong answers: carrying forward a stale or invented rate (4.25%, 4.5%, 4.0%, 4.49%) and/or a stale or fabricated standard deduction ($21,500, $23,400, $23,850, $26,400, or the federal $30,000); a six-model cluster that subtracted the spouse's $7,989 employer-sponsored insurance premiums from the stated gross wages to reach a $79,212 AGI, double-counting a Section 125 exclusion already reflected in stated wages; and a group that derived $2,461.87 correctly in its own reasoning and then submitted a larger unexplained figure. Two models denied any NC liability at all, and one returned nothing." +us,scenario_102,spouse_medicare_eligible,7,llm_error,age_disability,False,,"Under-65 Medicare eligibility does not come from a bare disability flag. It requires 24 months of Social Security Disability Insurance entitlement, or end-stage renal disease or ALS. The spouse is 26, receives no SSDI (the benefit is unlisted, so it is $0), and has no ESRD or ALS, so only the age-65 test applies and it fails. All seven models used the same made-up shortcut, ""disabled = Medicare eligible,"" and skipped the SSDI 24-month entitlement step." +us,scenario_102,state_income_tax_before_refundable_credits,26,llm_error,thresholds_rates,False,,"North Carolina's 2026 tax is flat: 3.99% of NC taxable income. For 2026 that rate is set by statute; the revenue triggers in S.L. 2023-134 apply only from 2027. NC taxable income starts from federal AGI of $87,201 ($41,200 + $46,000 wages + $1 self-employment income), minus the $25,500 married-filing-jointly NC standard deduction, which gives $61,701 × 3.99% = $2,461.87. The wrong answers fall into two groups. Thirteen models used a stale or invented rate (4.0%, 4.25%, 4.49%, or 4.5%) and/or the wrong standard deduction ($21,500, $23,400, $23,850, $26,400, $30,000, or none). Eight models subtracted the spouse's $7,989 employer-sponsored insurance premium from wages as a pre-tax exclusion, which turns AGI into $79,212 and the tax into $2,143.11." us,scenario_102,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_104,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"An income of 0.88 times FPL establishes eligibility only within an applicable MAGI category; this 72-year-old belongs to none of those categories and qualifies through no separate New York Medicaid pathway. Immigration-status eligibility and age do not independently create Medicaid eligibility, and SSI receipt provides no categorical route because SSI is zero." us,scenario_104,snap,2,llm_error,thresholds_rates,False,,"An elderly SNAP household is exempt from the gross-income test but must still pass the net-income test. Counting Social Security and veterans benefits and applying the allowable standard and medical deductions leaves annual net income around $23,474, or $1,956 per month, above the one-person net-income limit; rent also produces no excess-shelter deduction because it does not exceed half of adjusted income, so the benefit is $0." -us,scenario_104,state_refundable_credits,39,llm_error,state_local_rule,False,,"The entire $375 is New York's refundable real property tax credit (IT-214), which pays renters and homeowners with low household gross income and is worth up to $375 when the claimant is 65 or older — it requires no earned income, no dependents, and no state tax liability, since 25% of rent counts as deemed real property tax ($1,621 on this household's $6,484). Every wrong model failed at one of two points: most never tested the credit at all, screening New York's refundable credits only for earnings (state EITC) or children (Empire State Child Credit) and stopping when both were absent. The rest found the credit and killed it on the $18,000 household gross income limit by adding the $20,520 of veterans benefits to Social Security for $34,616; NY household gross income here is the $14,096 of Social Security, well under the limit. A third, smaller group invented rent screens — a rent floor, or a monthly-rent ceiling — to dismiss a credit whose computation runs directly off that rent." -us,scenario_107,federal_income_tax_before_refundable_credits,9,llm_error,taxable_income_or_deductions,False,,"Two independent steps drive this household to $0. First, the $6,948 of disability benefits is excluded from AGI and from the provisional-income test, so AGI is wages $6,366 + pension $7,476 + taxable Social Security $1,598 (50% of provisional income $28,197 over the $25,000 single threshold) = $15,439.75. Second, the 2026 single deduction stack is $16,100 basic + $2,050 age-65 addition + $6,000 OBBBA senior deduction = $24,150, with personal exemptions permanently zero. AGI of $15,440 falls $8,710 below that stack, so taxable income and regular tax are both $0. Every wrong model either pulled the non-taxable disability benefits into gross and provisional income, or shrank the deduction stack (fabricated $5,300 personal exemption, understated standard deduction, dropped the $6,000 senior deduction), or both." +us,scenario_104,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"This single 72-year-old New York filer has no earned income, no dependents and no taxable income: Social Security is exempt and veterans benefits are nontaxable. That rules out the NY EITC, Empire State Child Credit and other earnings- or child-based refundable credits. The only candidate, the Real Property Tax Credit for renters, fails its $18,000 household gross income limit because household gross income here is about $34,616. The trap is to add a small nonzero refundable amount after correctly ruling out every refundable credit. The NY household credit is nonrefundable, so it can never count toward state_refundable_credits. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 375.00 is regenerated as 0.00: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9301 (issue #9298) and PolicyEngine/policyengine-us#9313 (the 2025 flat credit tables the sandbox module also applies), after the reference freeze): A renter whose adjusted rent averages more than $450 a month gets no real property tax credit. Reference regenerated (N.Y. Tax Law 606(e)(7)(D); Form IT-214)." +us,scenario_107,federal_income_tax_before_refundable_credits,11,prompt_ambiguity,taxable_income_or_deductions,False,,"PolicyEngine leaves the $6,948 of disability benefits out of gross income. AGI is therefore wages $6,366 + pension $7,476 + taxable Social Security of about $1,598, or $15,439.75. That is below the 2026 single age-65+ standard deduction of $18,150 ($16,100 + $2,050), even before the $6,000 senior deduction, so tax is $0. The wrong answers fall into two groups. One group counted the disability benefits as taxable, which raised taxable Social Security to $5,473 and AGI to $26,263. The other used a pre-TCJA or post-sunset deduction setup, with a $5,300 personal exemption, a lower base standard deduction and no senior deduction. Several models made both errors. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 211.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." us,scenario_107,federal_refundable_credits,2,llm_error,credit_phaseout,False,,"The refundable-credit total is zero because every component is zero, including the childless EITC. The filer’s income exceeds the applicable EITC phaseout limit, and at age 75 the filer also fails the restored maximum-age rule for a childless EITC claimant; the expanded age rules applied only for 2021." us,scenario_107,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 75 does not itself establish Ohio Medicaid eligibility; the person must qualify through a specific Medicaid category and satisfy that pathway's financial rules. The head qualifies through no category, receives no SSI, is not a dependent, and has MAGI income of 2.67 times FPL, so immigration eligibility and $2,000 in bank assets do not produce eligibility." us,scenario_107,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -814,7 +823,7 @@ us,scenario_107,payroll_tax,3,llm_error,other,False,,"Payroll tax is the sum of us,scenario_107,snap,3,llm_error,categorical_eligibility,False,,"SNAP eligibility is screened before calculating deductions and the benefit formula. This household’s roughly $4,125 monthly gross income exceeds the applicable one-person income threshold, so shelter, utility, standard, and earned-income deductions do not produce eligibility and the annual benefit is $0." us,scenario_107,state_income_tax_before_refundable_credits,4,llm_error,thresholds_rates,False,,"Ohio's rate schedule begins with a zero bracket: Ohio taxable nonbusiness income of $26,050 or less is taxed at 0%, and the 2.75% rate applies only above that threshold. This single filer's Ohio taxable income is roughly $11,442 (wages $6,366 + taxable pension $7,476 + taxable Social Security $1,598, less Ohio's deduction for federally taxable Social Security and the $2,400 personal exemption; the $6,948 of disability benefits never enter federal AGI), which sits entirely inside the zero bracket, so Ohio tax before refundable credits is $0. All four models applied a positive marginal rate to the first dollar of Ohio taxable income and then patched the result with credits, producing a positive liability where the statute produces none." us,scenario_107,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_108,federal_income_tax_before_refundable_credits,14,llm_error,taxable_income_or_deductions,False,,"Two independent facts drive the reference to $0, and every wrong answer breaks both. First, the $10,392 of non-Social-Security survivor benefits is not an IRS gross income source, so AGI is the $7,656 of wages alone (the trace's AGI); taxable Social Security is $0 because combined income is $7,656 + 0.5 × $12,934 = $14,123, far under the $25,000 base. Second, 2026 deductions for a single filer age 85 total $24,150 — the $16,100 standard deduction made permanent by OBBBA, the $2,050 additional aged amount, and the $6,000 senior deduction for 2025–2028 — which wipes out taxable income even at the inflated $18,048 AGI every model computed. Each wrong number is the product of an inflated AGI plus a deduction figure that is either a stale pre-OBBBA amount or a TCJA-sunset reconstruction with a personal exemption that OBBBA permanently repealed." +us,scenario_108,federal_income_tax_before_refundable_credits,15,llm_error,taxable_income_or_deductions,False,,"PolicyEngine puts AGI at $7,656, which is wages only. The $10,392 in non-Social Security survivor benefits is not in AGI, and the $12,934 in Social Security survivor benefits is fully untaxed. Every model instead built AGI of $18,048 by counting the $10,392 as taxable pension income, then subtracted a stale, projected or pre-TCJA deduction to get a small positive 10%-bracket tax. Even at their own $18,048, the 2026 OBBBA-era single standard deduction ($16,100) plus the aged add-on ($2,050) is $18,150, and the $6,000 senior deduction for filers 65 and older comes on top. So taxable income is zero and so is tax before refundable credits." us,scenario_108,federal_refundable_credits,2,llm_error,age_disability,False,,"The childless EITC has an age-eligibility limit, and an 85-year-old claimant does not qualify even when earnings fall within the credit’s income range. With no qualifying children and no applicable refundable education, child, recovery rebate, or payroll tax credit, total federal refundable credits are $0." us,scenario_108,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_108,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -823,9 +832,9 @@ us,scenario_108,head_medicare_eligible,1,parse_contract_failure,missing_output,F us,scenario_108,payroll_tax,3,llm_error,other,False,,"Payroll tax equals employee Social Security tax of $474.67 plus employee Medicare tax of $111.01 on $7,656 of wages, for $585.68 total. The decisive step is correctly multiplying the wages by 1.45% and carrying that component into the final sum without transcription or unsupported adjustment." us,scenario_108,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_108,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_108,snap,38,llm_error,categorical_eligibility,False,,"Two rules stand between the reference and every wrong answer. First, Wisconsin confers broad-based categorical eligibility through a TANF-funded non-cash benefit with a 200% FPG gross screen and no asset test: this household's gross income of $2,581.83 is 1.98 times the $1,304.17 one-person poverty guideline, so it is eligible and the 100% FPG net income test that most models used as the gate does not apply. Second, because net income of $2,035.67 makes the 30% expected contribution ($610.50) exceed the $298 maximum allotment, the household falls to the minimum allotment for one- and two-person households — 8% of the maximum, $23.84/month through September and $24.37 after the October allotment adjustment — which sums to $287.68 rather than zero. Thirty-six of the thirty-eight models stopped at either the net income test or the negative formula result; the two that reached the minimum allotment priced it at the unindexed $23/month." -us,scenario_108,state_income_tax_before_refundable_credits,8,llm_error,household_unit_or_filing_status,False,,"Wisconsin exempts Social Security benefits in full, leaving $18,048 of wages plus other survivor benefits, and the head's stated qualifying-surviving-spouse status draws a standard deduction far larger than the single-filer amount — large enough, together with the $700 personal exemption and its $250 age-65 addition, to zero out Wisconsin taxable income before any rate applies. Every wrong model priced the deduction off Wisconsin's single column ($12,760–$13,940) and taxed the $3,800–$4,600 residue at the bottom bracket. Two of them also used the 3.54% bottom rate Wisconsin repealed after tax year 2022 (the bottom rate is 3.50%), but no rate matters when taxable income is zero." -us,scenario_108,state_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The Wisconsin homestead credit is a refundable income tax credit claimed on Schedule H by renters who are age 62+ or disabled, and the 85-year-old disabled head clears that gate. Schedule H household income here is $20,590 — $7,656 of wages plus the $12,934 of tax-exempt Social Security survivor benefits — while the separately listed $10,392 survivor benefit stream stays out of that base, leaving the household well under the $24,680 income ceiling. Property taxes accrued on $7,200 of rent are $1,440, and the credit is 80% x ($1,440 - 8.785% x ($20,590 - $8,060)) = $271.39. The overwhelming shared trap was adding the $10,392 to household income to reach $30,982, declaring the household over the limit, and returning $0; a second cluster never treated the homestead credit as a state refundable credit at all, and the three models that did attempt it eyeballed the amount instead of applying the 8.785% reduction." +us,scenario_108,snap,43,llm_error,categorical_eligibility,False,,"This one-person elderly/disabled Wisconsin household is categorically eligible for SNAP through Wisconsin's broad-based categorical eligibility (TANF non-cash). Its $2,581.83 monthly gross income is under the 200% FPL limit ($2,608), and its $270 in assets pass. Categorical eligibility removes the 100% FPL net income test that almost every model used to zero out the benefit. Because 30% of net income ($611) exceeds the $298 maximum allotment, the eligible one-person household gets the FY2026 minimum allotment of $24 a month, or $288 a year. Most models either failed the household on a net or gross income test or set the benefit to $0 when the formula went negative. The two models that found the minimum used the FY2025 figure of $23 instead of $24." +us,scenario_108,state_income_tax_before_refundable_credits,9,llm_error,state_local_rule,False,,"Every model that answered computed a small positive Wisconsin gross tax, about $110-$165, on roughly $18,048 of Wisconsin income (wages plus the taxable non-Social Security survivor benefits) after the sliding-scale standard deduction and exemptions. None of them then brought that tax to zero with Wisconsin's nonrefundable school property tax credit for renters. That credit is 12% of rent treated as property tax (20% of rent if heat is included, 25% if not, capped at $2,500). On $7,200 of rent it comes to $172.80-$216. That is more than any model's gross tax, so Wisconsin tax before refundable credits is $0. The one model that mentioned the renter credit made it too small and left a balance." +us,scenario_108,state_refundable_credits,46,prompt_ambiguity,state_local_rule,False,,"The full amount is the Wisconsin homestead credit. Rent-based property taxes are 20% × $7,200 = $1,440. Household income is wages of $7,656 plus nontaxable Social Security of $12,934, which equals $20,590. The separate $10,392 survivor-benefits input is not a homestead income source. The credit is 0.8 × ($1,440 − 8.785% × ($20,590 − $8,060)) = $271.39. Most models added the $10,392 survivor benefits, pushing income to $30,982, above the $24,680 cap, and returned $0. Others skipped the homestead credit, looking only at the child-based Wisconsin EITC, or treated the credit as non-refundable. The few that computed a credit used the wrong income base, the wrong rent percentage, or the wrong phase-out parameters. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)." us,scenario_109,child1_chip_eligible,6,llm_error,health_coverage,False,,"CHIP is limited to children who do not qualify for Medicaid. At approximately $30,000 of household income, the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility even though the income is below CHIP's upper limit." us,scenario_109,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Florida's refusal of adult Medicaid expansion does not touch children's coverage: the state's mandatory MAGI pathway for the older-child category (ages 6-18) runs to 133% FPL, and a 9-year-old in a household of five with roughly $21,600 of MAGI sits at 0.72 FPL — barely half the threshold. The only computation required is household MAGI from the $30,000 of self-employment income divided by the family-of-five poverty guideline, then compared with 133% FPL; no asset test, no enrollment fact, and no SSI linkage enters the children's MAGI pathway. Both models returned 0 without ever running that ratio, treating Medicaid eligibility as something the prompt had to assert rather than something the stated income and age determine." us,scenario_109,child2_chip_eligible,6,llm_error,health_coverage,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, not merely under age 19 and below CHIP's upper income limit. At age 6 and this household income, child2 qualifies for Medicaid under Florida's OLDER_CHILD category, so the Medicaid–CHIP mutual-exclusivity rule makes child2 ineligible for CHIP." @@ -838,93 +847,93 @@ us,scenario_109,child3_head_start_eligible,1,llm_error,categorical_eligibility,F us,scenario_109,child3_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"The dispositive rule is Florida's age-based INFANT Medicaid category, which covers children under age 1 at MAGI up to 185% FPL — a far higher threshold than the adult standards, and one that applies in a non-expansion state. The household's MAGI is 0.72 FPL, so the only two conditions that matter, age under 1 and income under the infant threshold, are both satisfied. All three models denied eligibility by routing the infant through adult or generic standards: one gave no derivation, one required an SSI/disability-style marker that MAGI categories do not use, and one computed the income test correctly and then overrode it with a nonexistent asset test plus a Medicaid-expansion and enrollment requirement." us,scenario_109,child3_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is derived from the child's age and household income; it does not require the prompt to explicitly state that the child is WIC-eligible. Child 3 is under age five, and the household's approximately $30,000 annual income is below the 2026 WIC limit of 185% of the federal poverty guideline for a five-person household." us,scenario_109,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The 2026 married-filing-jointly standard deduction does not reduce this household's taxable income to zero after the deductible half of self-employment tax. The resulting pre-credit income tax is fully offset by $2,792.91 of the $6,600 Child Tax Credit allocated as nonrefundable, and tax after nonrefundable credits is floored at zero rather than becoming negative." -us,scenario_109,federal_refundable_credits,36,llm_error,thresholds_rates,False,,"The correct build is EITC $8,231 (2026 MFJ, three qualifying children) plus refundable CTC $3,807.09, where the ACTC is 15% of earned income above $2,500 and earned income is gross self-employment income less the deductible half of SE tax: $30,000 − $2,119.43 = $27,880.57, so 0.15 × ($27,880.57 − $2,500) = $3,807.09. Two traps separate the reference from the wrong answers. First, the refundable CTC is the 15%-of-excess-earnings formula capped at $1,700 per child, not a flat $1,000 per child — the $3,000 shortcut costs $807 and produced the $11,2xx cluster. Second, $27,880.57 sits below the 2026 MFJ three-child EITC phase-out start, so the full $8,231 maximum applies; models that fabricated a $24,000–$29,600 phase-out threshold, or that used gross $30,000 or net SE earnings $27,705 as the ACTC base, shifted the answer in the other direction." +us,scenario_109,federal_refundable_credits,40,llm_error,thresholds_rates,False,,"The reference is the 2026 three-child EITC maximum of $8,231 plus a refundable CTC of $3,807.09. EITC earned income is $30,000 minus the deductible half of SE tax (about $2,119), which is $27,881. That sits on the plateau, below the MFJ phase-out start. The refundable CTC is 15% of ($27,881 − $2,500), which is under the $1,700-per-child cap of $5,100. Models went wrong in three main ways: (1) using the pre-TCJA $1,000-per-child refundable CTC ($3,000), (2) running the 15% ACTC formula on gross $30,000 or on the 92.35% net-earnings base of $27,705 instead of $27,881, and (3) phasing down an EITC that is actually on the plateau. Several also used EITC maximums other than $8,231, or submitted a number that differs from their own arithmetic." us,scenario_109,free_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"The FREE tier here is over-determined: the household's $8,020.55 SNAP benefit confers categorical free-meal eligibility outright, and independently $30,000 for a five-person household is 78% of the federal poverty guideline, far under the 130% free-meal cutoff. The only fact requiring inference is school attendance — never stated explicitly — which follows from the children aged 9 and 6 being K-12 students, generating $7.15 per child per day net of the $0.87 paid portion for $2,261.92 annually. The trap is the instruction to treat unlisted statuses as false: a model that applies it to school enrollment erases both students and skips both qualifying tests. Both models defaulted to 0 on that reading rather than deriving the students from age and then applying either the SNAP categorical pathway or the 130% FPG income test." us,scenario_109,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Low MAGI alone does not establish adult Medicaid eligibility in Florida. The age-30 head is not a dependent, receives no SSI, and fits none of Florida's active Medicaid eligibility categories, so the category is NONE despite MAGI income of 0.72 times FPL." us,scenario_109,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is individual, not household-wide: only a pregnant, postpartum, or breastfeeding woman, an infant, or a child under age five can qualify. The infant satisfies that requirement, but the 30-year-old Head has no listed pregnancy, postpartum, or breastfeeding status, so the Head remains ineligible despite the household meeting the income test." us,scenario_109,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price and free school meals are mutually exclusive output tiers in this calculation. The household’s 0.78 school-meal FPG ratio and categorical eligibility place it in the superior FREE tier, so it receives positive free-meal support and no reduced-price support." us,scenario_109,self_employment_tax,6,llm_error,other,False,,"The Schedule SE computation here is fully determined: net earnings from self-employment are 92.35% of $30,000 = $27,705, entirely below the Social Security wage base, so the tax is $27,705 × 15.3% = $4,238.865, which PolicyEngine reports as $4,238.86. Every model named the correct base fraction, the correct 15.3% combined rate, and correctly noted the wage base is not binding — none missed a rule. The separation is purely in executing the product: each model reported a whole-dollar figure that is $2 to $6 away from $4,238.87, whereas correct rounding of the true product to whole dollars gives $4,239. The trap is arithmetic precision on 0.9235 × 0.153 × $30,000, not any element of the SE tax rule." -us,scenario_109,snap,39,llm_error,thresholds_rates,False,,"The calendar-year SNAP total spans two federal fiscal years, and the October 1 uprating is the trap: January–September uses the FY2026 five-person parameters ($261 standard deduction, $1,183 maximum allotment) giving $1,183 − 0.3 × $1,739 = $661.30/month, while October–December uses the uprated $266.85 standard deduction and $1,209.52 allotment giving $689.62/month, so the annual sum is $8,020.55 rather than 12 × $661.30 = $7,935.60. The second trap is the excess shelter deduction: $800/month rent is below half of the $1,739 adjusted income ($869.50), and no utility expense is listed, so the shelter deduction is $0 and the standard utility allowance does not apply. A third trap is the parameter vintage — many models reached for FY2025 figures ($254 standard deduction, $1,158 allotment) or invented projections instead of the FY2026 values." +us,scenario_109,snap,41,llm_error,thresholds_rates,False,,"This is a simple SNAP case with no shelter deduction. Start with $2,500 of monthly self-employment income. Subtract the $500 earned income deduction (20%) and the $261 FY2026 standard deduction for five people, which leaves $1,739. The 30% contribution is $521.70, and SNAP rounds it up to $522. The FY2026 maximum allotment for five is $1,183, so the benefit is $661 a month, or $7,932 a year. Rent of $800 is below half of adjusted income ($869.50) and no utility costs are listed, so there is no excess shelter deduction. Most wrong answers land within about $600 a year of $7,932. The cause is using stale or guessed FY2026 figures instead of the exact $261 standard deduction and $1,183 maximum, or skipping the round-up of the 30% contribution, which gives $7,935.60 or $7,936. The rest made bigger errors: they invented shelter or utility deductions, subtracted self-employment tax, miscounted household size, or wrongly found the household ineligible. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))." us,scenario_109,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Florida’s non-expansion Medicaid program does not provide a general low-income pathway for a nondisabled 29-year-old adult. The spouse qualifies through no categorical pathway: household MAGI exceeds Florida’s parent/caretaker limit, and the presence of an infant does not establish pregnancy or postpartum status." us,scenario_109,spouse_wic_eligible,8,llm_error,categorical_eligibility,False,,"WIC eligibility for an adult requires the adult herself to be pregnant, breastfeeding, or within the applicable postpartum period; merely living with an infant does not establish any of those statuses. The prompt makes every unlisted status false, so the spouse is not pregnant, postpartum, or breastfeeding and fails categorical eligibility regardless of household income." -us,scenario_110,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The reference builds AGI of $163,421.64 from $95,755.10 of employment income (wages less the $4,244.90 traditional 401(k) deferral only), $30,869.77 of capital gains, $21,301 of taxable interest and $15,694.12 of dividends, less a $198.35 traditional IRA above-the-line deduction — the $6,323 of estate income stays out of AGI but drives a $1,264.59 QBI deduction (20% of it). Taxable income of $145,669.58 then reflects $17,752.06 of deductions: the $16,100 single standard deduction, the $387.47 non-itemizer cash charitable deduction, and that QBI deduction. Nearly every model instead added estate income to AGI, denied the IRA deduction, and took no QBI deduction, landing at the cluster answer of ~$25,841 — $8,173 too much taxable income and $1,943 too much tax. A second cluster assumed a 2026 TCJA sunset and restored personal exemptions, pre-TCJA rate schedules, and the 2%-of-AGI miscellaneous itemized deduction for the $8,730 of unreimbursed employee business expenses, all of which remain repealed." +us,scenario_110,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine leaves estate_income out of IRS gross income, while still treating it as qualified business income for a $1,264.59 §199A deduction (20% of $6,323). Its AGI of $163,421.64 equals wages $95,755.10 + capital gains $30,869.77 + interest $21,301 + dividends $15,694.12 − $198.35, with no estate income. This contradicts IRC §61(a)(15) and §662(a), which include estate and trust distributions in gross income.","The reference builds AGI of $163,421.64 from wages net of the 401(k), capital gains, interest and dividends, less a $198.35 IRA deduction. It leaves the $6,323 estate income out of AGI but deducts 20% of it ($1,264.59) as QBI, alongside the $16,100 standard deduction and the $387 nonitemizer charitable deduction. That puts ordinary taxable income at $104,785.81, entirely within the 22% bracket. The most common answer ($25,841) includes estate income (AGI $169,943), disallows the IRA deduction, and skips the charitable and QBI deductions, which puts $7,259 of ordinary income in the 24% bracket. The other misses come from assuming the TCJA expired (personal exemptions, 2%-floor miscellaneous deductions, 15/25% brackets), outdated standard-deduction and bracket figures, overtime deductions built on an overtime premium that was never listed, subtracting ESI premiums from wages, and final values that contradict the models' own arithmetic. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 23,897.44; the corrected value is 25,747.77. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))." us,scenario_110,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_110,payroll_tax,14,llm_error,payroll_tax_base,False,,"The FICA base here is the stated gross wage figure itself: $100,000 of wages and salaries, well under the $168,600 Social Security wage base and under the $200,000 Additional Medicare Tax threshold, giving 6.2% × $100,000 = $6,200 plus 1.45% × $100,000 = $1,450 = $7,650, with Ohio imposing no mandatory employee state payroll tax. The trap is base-shaving: nine models netted the $6,889 employer-sponsored insurance premium (or, in one case, the $4,245 traditional 401(k) deferral) out of the wage base, even though the household facts state $100,000 as gross wages and salaries and elective deferrals are FICA wages regardless. The remaining misses are threshold invention (Additional Medicare Tax on wages that never reach $200,000), a zeroed base, and answers whose submitted numbers contradict their own correct derivations." +us,scenario_110,payroll_tax,15,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 7.65% of the full $100,000 of gross wages: $6,200 of Social Security tax plus $1,450 of Medicare tax, for $7,650. Wages are below the Social Security wage base and below the $200,000 Additional Medicare Tax threshold, and Ohio has no mandatory employee state payroll tax. The main mistake, made by nine models, was treating the $6,889 employer-sponsored insurance premium as a Section 125 pre-tax salary reduction. The prompt lists no cafeteria plan, and PolicyEngine does not take that premium out of FICA wages. One model subtracted the traditional 401(k) deferral, which stays subject to FICA. Others added an Additional Medicare Tax that does not apply, reported a fabricated $0, or submitted a number that contradicted their own explanation." us,scenario_110,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_110,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Ohio's 2026 schedule is the trap: Ohio AGI of $163,421.64 is reduced only by the $1,900 personal exemption (the tier for OAGI above $80,000) — Ohio grants no standard or itemized deduction — and the tax on the resulting $161,521.64 is $332 plus 2.75% of the excess over $26,050, with no bracket above $100,000 and no nonrefundable credit at this income (the $20 exemption credit stops at $30,000 of OAGI). Two clusters of wrong answers follow. Models that kept a pre-2026 graduated schedule (3.5% above $100,000, or the older 2.765%/3.226%/3.688% table) overshoot by $150–$700, while models that used the flat 2.75% correctly dropped the schedule's $332 base amount and started from unadjusted federal AGI of about $169,900 instead of Ohio AGI, landing in a $3,879–$3,910 cluster. One model reproduced the 2026 schedule and the exemption exactly and lost only $16 by adding the $954 tax-exempt interest to the Ohio base." +us,scenario_110,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"For 2026, Ohio taxes income above $26,050 at a single 2.75% rate plus a fixed $332 base, and there is no 3.5% bracket. At MAGI above $80,000 the personal exemption is $1,900, and Ohio has no standard or itemized deduction. Ohio AGI is $163,421.64 because the $6,889 of pre-tax employer health premiums and the $4,245 401(k) deferral come out of wages. That leaves $161,521.64 of taxable income, and $332 + 2.75% × $135,471.64 = $4,057.47. Most models either dropped the $332 base (flat 2.75% above $26,050) or used the old 2.75%/3.5% schedule, and most also left the employer premiums in AGI (about $169,900), skipped or mis-tiered the exemption, or subtracted deductions Ohio does not allow. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 4,057.47; the corrected value is 4,236.81. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii))." us,scenario_110,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_111,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The IRA distribution produces $18,240 of adjusted gross income, while the Social Security retirement benefit is not taxable under the joint-return provisional-income thresholds. The applicable standard deduction eliminates taxable income, so the tax computation yields $0; both models expressly derived zero taxable income and zero liability but then submitted unsupported positive amounts." us,scenario_111,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Age, blindness, disability, and low assets do not by themselves establish Medicaid eligibility; the applicant must qualify through a specific Washington Medicaid category and satisfy that pathway’s rules. The head qualifies through no pathway: medicaid_category is NONE, SSI receipt is zero, and the 2.02-FPL MAGI exceeds the applicable MAGI-category limits." us,scenario_111,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits or taxable IRA distributions. Because neither spouse has wages or other employee compensation, the employee Social Security, Medicare, Additional Medicare, and mandatory Washington employee payroll-tax bases are all zero, yielding $0." us,scenario_111,snap,1,llm_error,categorical_eligibility,False,,"Washington's 200% gross-income screening rule does not itself establish categorical SNAP eligibility or guarantee the minimum allotment. This two-person household's approximately $43,640 annual income disqualifies it, so the minimum-benefit rule never applies and annual SNAP equals $0." us,scenario_111,spouse_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Blindness and disability open non-MAGI Medicaid pathways but do not establish eligibility by themselves; each pathway retains its own income, categorical, and sometimes resource requirements. The spouse qualifies through none of those pathways, and the MAGI adult pathway also fails because the applicable income level is 2.02 times FPL, above Washington’s 138% FPL expansion threshold." -us,scenario_111,spouse_medicare_eligible,8,llm_error,age_disability,False,,"Medicare's under-65 route is not disability status itself but entitlement to Social Security disability benefits — 24 months of SSDI receipt, or an ESRD/ALS diagnosis. The spouse's `is_disabled` flag drives SSI, the SNAP elderly/disabled deductions, and EITC disability provisions, not Medicare; with every unlisted input set to zero, the spouse has $0 of Social Security disability income, so PolicyEngine's Medicare test reduces to the age-65 threshold and returns False for the 49-year-old while returning True for the 72-year-old head. All eight wrong models treated the demographic disability flag as a standalone Medicare trigger and never checked the benefit-entitlement condition that gates the under-65 pathway." +us,scenario_111,spouse_medicare_eligible,10,llm_error,age_disability,False,,"Under age 65, a person qualifies for Medicare through disability only after 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), or through the ESRD or ALS pathways. PolicyEngine models the same rule: the age-65 test or at least 24 months of Social Security disability receipt. The spouse is 49 and flagged disabled, but the household lists no Social Security disability income, no months of SSDI receipt, and no ESRD or ALS, so the spouse fails every pathway. All ten models treated the disability flag alone as enough for Medicare. They skipped the separate requirement of 24 months of SSDI entitlement." us,scenario_112,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The trap is the $13,000 ""financial assistance"" line: in PolicyEngine `financial_assistance` is a non-taxable cash receipt (counted for benefit means-tests, never in gross income or AGI), so the only taxable items here are $5,923 of wages and $1,920 of farm rent income, reduced by the $3,000 capital-loss limitation on the -$9,944 long-term loss, for AGI of roughly $4,843. That sits far below the 2026 single standard deduction (~$16,100), leaving taxable income of $0, no NIIT (net investment income is a $3,000 loss), and no add-on liabilities, hence $0. Both models handled the $3,000 capital-loss cap and the standard deduction correctly but pushed the $13,000 gift/assistance through as ordinary income, manufacturing $1,743 of taxable income and a $174.30 bracket tax. Excluding the non-taxable transfer collapses the entire computation to zero." us,scenario_112,federal_refundable_credits,21,llm_error,credit_phaseout,False,,"The case turns on one computation: the childless EITC on the phase-in ramp, 7.65% of the head's $5,923 in wages = $453.10. Farm rent is neither earned income nor enough investment income to trip the ~$12,200 disqualification limit, AGI is $4,843 (wages + $1,920 farm rent − the $3,000 capital-loss cap) and sits far below the ~$10,600 single phase-out start, the head is 31 and so inside the 25-64 childless age band, and the $13,000 financial assistance is not taxable income. The wrong answers cluster into four shortcuts: subtracting the $8,389 employer-sponsored insurance premiums from wages to zero out earned income; applying the 7.65% rate to AGI or to a MAGI inflated by the nontaxable financial assistance and declaring the credit phased out; treating ""no qualifying children,"" negative net income, or zero tax liability as automatically zeroing a refundable credit; and deriving $453 correctly and then submitting a different number." us,scenario_112,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Texas does not provide MAGI Medicaid eligibility to a nondependent, childless 31-year-old merely because income is below an income threshold. The head meets the immigration requirement but belongs to none of Texas's active Medicaid categories, so the low 0.30-FPL MAGI level does not create eligibility." us,scenario_112,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying demographic category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 31-year-old head has no listed qualifying status, and unlisted statuses are false, so passing the income test does not establish WIC eligibility." us,scenario_112,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employer-sponsored insurance premiums are not automatically employee pre-tax cafeteria-plan deductions from FICA wages; the listed $5,923 remains subject to employee Social Security and Medicare taxes. PolicyEngine computes $367.22 of Social Security tax and $85.88 of Medicare tax, totaling $453.10, with no Additional Medicare Tax or mandatory Texas employee payroll tax." us,scenario_112,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to positive net earnings from self-employment, after determining the applicable self-employment tax base. The household’s relevant net income is approximately -$2,101, so there are no positive net self-employment earnings and the tax base is zero; treating the $1,920 farm rent entry as a standalone positive self-employment tax base produces the wrong liability." -us,scenario_112,snap,39,llm_error,thresholds_rates,False,,"SNAP countable gross income here is wages ($5,923) plus the $13,000 financial assistance = $18,923 ($1,576.90/month, 121% of the $1,304.17 one-person guideline); the $1,920 of farm rental income is not a countable SNAP income source and the $9,944 capital loss is ignored, so the household clears the gross test, and after the 20% earned-income and standard deductions net income is $1,269.19 (97% of the guideline), clearing the net test — with TANF non-cash broad-based categorical eligibility available as a second pathway and the $10 in bank accounts clearing assets. Because 30% of net income ($380.70) exceeds the $298 one-person maximum allotment, the formula benefit is zero, and the statutory minimum allotment for eligible one- and two-person households (8% of the maximum: $23.84/month, $24.37 after the October FY-rollover adjustment) is what produces $287.68. Wrong answers split into two camps: most inflated countable income with the $1,920 farm rent and failed a gross or net income test, or reached the zero formula benefit and stopped short of the minimum allotment; three models instead dropped the $13,000 financial assistance (or substituted the maximum allotment) and awarded near-maximum benefits." -us,scenario_114,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on the 2026 deduction stack for a 69-year-old itemizer: SALT is $9,431.59 ($4,064 real estate tax plus $5,367.59 of Virginia income tax, far under the raised cap), charitable gifts of $11,447 are cut to $10,913.78 by the 0.5%-of-AGI floor, and medical above the 7.5% floor adds $4,441.86, for $24,787.23 of itemized deductions — on top of which the $4,872.35 QBI deduction and the $4,105.65 senior deduction ($6,000 less 6% of MAGI over $75,000) stack, totaling $33,765.23. AGI is $106,572.56 (85% of Social Security taxable at $19,828.80; only $22 of above-the-line deductions, since the partnership/S-corp income carries no half-SE-tax adjustment), so taxable income is $72,807.33 and the 2026 single schedule (10% to $12,400, 12% to $50,400, 22% above) gives $5,800 + 22% × $22,407.33 = $10,729.61. The wrong answers fall into three families: assuming the TCJA sunset (personal exemption, restored 10/15/25% brackets, no §199A), omitting the senior deduction, and understating itemized deductions — chiefly by leaving the Virginia income tax out of SALT and the $2,000 of OTC health costs out of medical — so the standard deduction looked larger. That last family lands on $11,970, which is precisely the standard-deduction alternative, $1,240.19 above the itemizing result." +us,scenario_112,snap,46,prompt_ambiguity,categorical_eligibility,False,"PolicyEngine's SNAP earned and unearned income source lists leave out the farm_rent_income variable. Its gross income of $1,576.90 equals ($5,923 + $13,000)/12, so the $1,920 of farm rent is excluded, although 7 CFR 273.9(b)(2)(ii) counts net income from rental property (and 273.9(b)(1) counts it as earned income when it is actively managed). Counting the farm rent raises net income to about $1,429/month, above the $1,304 net limit, which makes SNAP $0.","The reference counts only wages ($493.58/month) and the $13,000 of financial assistance ($1,083.33/month) as SNAP income, for $1,576.92/month; it leaves out the $1,920 of farm rent. After the 20% earned-income deduction and the $209 standard deduction, net income is $1,269.19. That is just under the $1,304.17 limit (100% of the poverty guideline), so the household passes the net income test. Because 30% of net income ($380.76) is more than the $298 maximum allotment, this one-person household gets only the minimum allotment of about $23.84/month. Most models added the $160/month of farm rent, which pushed net income to about $1,429 (above the net limit) or applied a 130% gross test despite Texas's 165% broad-based categorical eligibility (BBCE) limit, and answered $0. A few left out the financial assistance and answered about $194/month. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)." +us,scenario_114,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"The correct 2026 result follows OBBBA law. AGI is $106,572.56, with 85% of Social Security taxable. Itemized deductions total $24,787.23: charitable $10,913.78 after the new 0.5%-of-AGI floor, SALT $9,431.59 ($4,064 real estate tax plus $5,367.59 state income tax), and medical $4,441.86 ($10,000 of other expenses plus the $2,434.80 Medicare Part B premium, with non-deductible OTC purchases excluded, above 7.5% of AGI). That total beats the $18,150 aged standard deduction plus the $1,000 non-itemizer charitable deduction. Itemizers still take the $4,105.65 phased senior deduction and the $4,872.35 QBI deduction, leaving $72,807.33 taxed at the permanent TCJA brackets (10% to $12,400, 12% to $50,400, then 22%) for $10,729.61. The wrong answers fall into five groups: (1) applying expired pre-TCJA law (personal exemptions, 15%/25% brackets, no QBI); (2) omitting the senior deduction; (3) understating itemized deductions by dropping state income tax from SALT or missing the Part B premium, and so picking the standard deduction; (4) imputing mortgage interest or self-employment tax that the facts do not support; and (5) reporting numbers unrelated to their own arithmetic." us,scenario_114,federal_refundable_credits,2,llm_error,credit_phaseout,False,,"EITC phaseout uses adjusted gross income or earned income, whichever is greater, rather than wages alone. This filer’s partnership/S-corp income, taxable retirement distributions, pension, taxable Social Security, and interest raise AGI far above the childless EITC limit, so every refundable credit component is zero." us,scenario_114,payroll_tax,4,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes employee Social Security and employee Medicare taxes on the $22,700 of wages, while self-employment tax is reported separately and Virginia adds no mandatory employee payroll tax here. The stated components sum to $1,407.40 + $329.15 = $1,736.55; arbitrary adjustments, duplication, and invented state payroll taxes produce the wrong answers." -us,scenario_114,self_employment_tax,17,llm_error,payroll_tax_base,False,,"The trap is the distinction between partnership/S-corp income and net earnings from self-employment. The household's only business income is the $24,362 listed as ""partnership or S-corp income,"" a separate PolicyEngine input that flows into taxable income but never into the self-employment tax base; self_employment_income is unlisted and therefore 0 under the prompt's ""treat any unlisted numeric input as 0"" rule, so SE tax is $0. That mapping matches substantive law: S-corp distributive shares are not net earnings from self-employment (Rev. Rul. 59-221), and a limited partner's distributive share is excluded by IRC §1402(a)(13); general-partner status is an unlisted fact the prompt directs models to treat as false. Sixteen of seventeen models pushed the $24,362 through 0.9235 × 15.3% anyway, landing near $3,442, and one returned no answer." -us,scenario_114,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Virginia's chain is: federal AGI $106,572.56 → subtract the $19,828.80 of federally taxable Social Security → VA AGI $86,743.77 → subtract $19,419.94 of itemized deductions (charitable gifts of $11,447, real estate taxes of $4,064, and medical costs above the 7.5%-of-AGI floor, with the $2,000 of over-the-counter health expenses counted as medical) and $1,730 of exemptions ($930 personal plus $800 age-65) → taxable income $65,593.83 → $720 accumulated through $17,000 plus 5.75% of the $48,593.83 excess = $3,514.15. The $12,000 age deduction contributes nothing: Virginia reduces it dollar-for-dollar for age-deduction AFAGI above the $50,000 single threshold, and AFAGI here is $86,743.77. Wrong answers cluster into four failures: substituting Virginia's standard deduction for the larger itemized total, restoring the fully phased-out age deduction, treating the $930/$800 exemptions as credits against tax or dropping them, and excluding the $2,000 of over-the-counter health expenses from the medical deduction (the $17,518.04 itemized cluster)." +us,scenario_114,self_employment_tax,18,llm_error,payroll_tax_base,False,,"The household's only business-type input is $24,362 of partnership or S-corp income, and that amount is not net earnings from self-employment. S-corp pass-through income is never subject to SE tax. A partnership distributive share counts only for a general partner, and the prompt says any unlisted status is false, so no general-partner status can be assumed. With zero self-employment income, SE tax is $0. Every substantive wrong answer made the same mistake: each model counted the partnership/S-corp amount as SE earnings, then applied 15.3% to 92.35% of it, which gives about $3,442. Some models then added further arithmetic errors on top of that." +us,scenario_114,state_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"Virginia taxable income here starts from federal AGI ($106,572.56) minus the $19,828.80 Social Security subtraction, giving VA AGI of $86,743.77. The age deduction is zero because a single filer's AFAGI of about $86,744 is more than $12,000 over the $50,000 threshold. From there, Virginia itemized deductions of $19,419.94 apply (the filer itemizes federally), plus exemptions of $930 personal and $800 age-65 taken as deductions from income. That leaves $65,593.83, taxed at $720 plus 5.75% of the amount over $17,000 = $3,514.15. The wrong answers split five ways: (1) using the VA standard deduction instead of itemizing; (2) granting the phased-out $12,000 age deduction; (3) dropping the $800 aged exemption or treating exemptions as tax credits; (4) shrinking the medical deduction by leaving out the $2,000 over-the-counter expenses or adding the 0.5%-of-AGI charitable floor; (5) skipping the Social Security subtraction." us,scenario_114,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_115,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and a reported disability do not by themselves place the head in an Alabama Medicaid eligibility category. PolicyEngine assigns medicaid_category = NONE; the 1.47 FPL MAGI amount therefore cannot establish eligibility through a MAGI category, and immigration eligibility alone is insufficient." us,scenario_115,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_115,snap,5,llm_error,thresholds_rates,False,,"SNAP eligibility uses approximately $23,442 of annual household income here: $18,708 of Social Security plus $4,734 of tax-exempt interest. That income exceeds PolicyEngine’s applicable one-person SNAP eligibility limit, so the calculation stops at ineligibility and returns $0; deductions and minimum-allotment rules do not create a benefit for an ineligible household." -us,scenario_115,state_income_tax_before_refundable_credits,38,llm_error,state_local_rule,False,,"The determinative item is the $4,734 of federally tax-exempt interest: Alabama defines gross income under its own statute rather than starting from federal AGI, and it does not adopt the federal §103 exclusion — only interest on U.S. obligations and Alabama-issued obligations escapes tax — so the full $4,734 lands in Alabama AGI while the $18,708 of Social Security is correctly exempt. A single filer's Alabama deductions here are $3,000 (top-tier standard deduction) plus a $1,500 personal exemption = $4,500; Alabama grants no extra age-65 or blindness addition, and the itemized alternative (medical expenses over the 4% AGI floor, $2,584.80 − $189.36 = $2,395.44) is smaller than the standard deduction, so it loses. Taxable income is $4,734 − $4,500 = $234, taxed at Alabama's 2% first bracket on the first $500, giving $4.68. Every wrong model reached $0 either by keeping the interest out of the Alabama base or by inflating Alabama's deductions/exemptions above $4,734." +us,scenario_115,state_income_tax_before_refundable_credits,45,llm_error,state_local_rule,False,,"Alabama exempts Social Security, but it does not follow the federal exclusion for tax-exempt interest. Only interest on Alabama and U.S. obligations is exempt, and PolicyEngine counts the full $4,734 as Alabama AGI. The single filer gets a $3,000 standard deduction and a $1,500 personal exemption, $4,500 in total. There is no age-65 add-on. That leaves $234 of taxable income, and 2% of $234 is $4.68. Every model either left the interest out entirely, started from a federal AGI of $0 with no add-back, or added the interest back and then overstated the deductions so that they covered all $4,734." us,scenario_115,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_116,snap,1,llm_error,categorical_eligibility,False,,"SNAP’s minimum allotment applies only after a household passes the program’s eligibility tests; it cannot override failure of the gross-income test. At $52,800 of annual household income, this household exceeds the applicable 2026 gross-income limit, so the computation stops with no SNAP benefit." us,scenario_116,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Disability alone does not establish Florida Medicaid eligibility. The spouse receives no qualifying Medicaid category in the engine; with medicaid_category = NONE, the zero MAGI income level and bank assets cannot produce eligibility." -us,scenario_116,spouse_medicare_eligible,3,llm_error,age_disability,False,,"Medicare eligibility for this 42-year-old spouse turns on the age-65 threshold, because the only under-65 pathways — 24 months of Title II SSDI entitlement, ALS, or ESRD — require facts this household does not have. The spouse's $52,800 is veterans benefits, and the prompt's ""treat any unlisted numeric input as 0"" rule zeroes out Social Security disability benefits, so there is no SSDI entitlement to begin a 24-month waiting period. All three models treated the bare is_disabled flag (or VA compensation) as a self-executing Medicare trigger, skipping the SSDI-entitlement predicate that the disability pathway is built on. The disability flag in this household drives SSI and the Medicaid aged/blind/disabled test, not Medicare." +us,scenario_116,spouse_medicare_eligible,4,llm_error,age_disability,False,,"Under age 65, Medicare eligibility comes only from 24 months of Social Security Disability Insurance (Title II) entitlement or from ESRD/ALS. A disability flag on its own does not confer it. The spouse is 42 and receives $52,800 in veterans benefits, which are VA payments and not SSDI. With SSDI income and SSDI months both unlisted, they are 0 under the prompt's rules, so the age test and the SSDI-duration test both fail. All four models treated ""is disabled"" (plus, for one model, VA benefits) as enough for disability-based Medicare, and so answered Yes." us,scenario_117,child3_early_head_start_eligible,5,llm_error,categorical_eligibility,False,,"Early Head Start eligibility requires both an age-qualified child and satisfaction of the program's low-income eligibility rules; being under age 3 is not independently sufficient. Although Child 3 meets the age condition, household income of approximately $250,733 is far above the applicable poverty-based income limit, so the income test produces no eligibility." us,scenario_117,child3_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Age alone does not establish Head Start eligibility; the child must also qualify through an applicable eligibility pathway, principally the program's low-income criteria in this case. This household's approximately $250,733 income exceeds the relevant poverty-based threshold, so Child 3 is not eligible." us,scenario_117,child3_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC child eligibility requires both categorical eligibility and financial eligibility; being age 2 satisfies only the child-category requirement. The household's approximately $250,733 annual income exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household, so Child 3 is not eligible." -us,scenario_117,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The case turns on three 2026 OBBBA mechanics stacked on top of each other: the $25,000 qualified-overtime deduction applies in full because the joint MAGI phaseout starts at $300,000 (not the $150,000 single threshold), the household is better off with the $32,200 MFJ standard deduction plus the $2,000 non-itemizer cash-charitable deduction ($34,200 combined) than itemizing roughly $33,000 of AR income tax and charitable gifts after the 0.5%-of-AGI floor, and that standard-deduction choice makes the $2,589 prior-year state refund nontaxable under the tax-benefit rule. AGI is therefore $248,675 wages less the $530.47 capital-loss deduction = $248,144.53, taxable income $188,944.53, tax $30,991.80 under the 10/12/22 schedule, less $6,600 of CTC (3 children × $2,200; joint phaseout begins at $400,000) and zero CDCC because Head has no earned income. Nearly every wrong answer failed at one of four points: applying a pre-TCJA/sunset regime with personal exemptions and 15/25/28% rates, phasing the overtime deduction out from $150,000 or skipping it entirely, itemizing and consequently taxing the state refund, or fabricating mortgage interest from a bare $501,000 loan balance that carries no stated rate." -us,scenario_117,federal_refundable_credits,4,llm_error,credit_phaseout,False,,"The Additional Child Tax Credit under IRC §24(d) is not a per-child cash grant: it equals only the portion of the Child Tax Credit that survives after the credit is first applied nonrefundably against income tax liability, capped by the per-child refundable limit and by 15% of earned income over $2,500. This household has roughly $250,700 of income and, after ~$25,900 of charitable giving plus mortgage interest and SALT itemized deductions, still owes tens of thousands in federal income tax — far more than its full CTC for three children under 17 — so the entire CTC is absorbed as a nonrefundable credit and the refundable remainder is $0. EITC is fully phased out at this income, and no American Opportunity, Recovery Rebate, or refundable payroll tax credit applies, leaving federal refundable credits at exactly $0. Three of the four models inverted the §24(d) ordering rule and paid out the refundable cap despite ample liability to absorb the credit." +us,scenario_117,federal_income_tax_before_refundable_credits,46,prompt_ambiguity,taxable_income_or_deductions,False,,"Under post-2025 law, start from $250,733 AGI and subtract three deductions: the $32,200 joint standard deduction, the $25,000 qualified-overtime deduction, and the $2,000 non-itemizer charitable deduction. The $35,525 FLSA premium is capped at $25,000 for joint filers, with no phaseout below $300,000 MAGI. That leaves $191,533 of taxable income and $31,561 of bracket tax, less the full $6,600 CTC ($2,200 × 3 children under 17). The CDCC is zero because the head has no earned income. The main trap was the overtime deduction: models omitted it, cut it to the $12,500 single cap, phased it out from the $150,000 single threshold, or took the full premium with no cap. Other errors added to it: TCJA-sunset rate schedules, mortgage interest invented from a bare balance, and a CDCC that the head's zero earnings rule out. Models that came close got the overtime deduction right but itemized instead of taking the standard deduction plus the $2,000 non-itemizer charitable deduction, or claimed a CDCC. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))." +us,scenario_117,federal_refundable_credits,5,llm_error,other,False,,"The refundable CTC (ACTC) is capped at the part of the child tax credit left over after the credit has offset income tax. This couple files jointly with about $250,733 of AGI. That is below the $400,000 CTC phase-out, so the full $2,200-per-child CTC ($6,600) is allowed. Their federal income tax is well over $25,000, which absorbs the whole $6,600 as a nonrefundable credit and leaves nothing to refund. EITC is zero at this income, and no other refundable credit applies, so the total is $0. The wrong answers skipped the tax-liability limit and treated the per-child refundable cap ($1,700 or $1,800), or a made-up fraction of it, as money the family receives automatically." us,scenario_117,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is individual, not household-wide: living with a two-year-old does not make the 44-year-old head categorically eligible. The household's approximately $250,733 annual income also exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household." us,scenario_117,payroll_tax,26,llm_error,payroll_tax_base,False,,"The trap is the 2026 Social Security taxable maximum: $184,500, which caps the employee OASDI tax at $11,439, while the 1.45% Medicare tax runs uncapped on all $248,675 of Spouse wages for $3,605.79, giving $15,044.79. Two secondary traps sit alongside it — the $35,525 FLSA overtime premium is already inside the stated gross wages and must not be added on top, and combined wages fall $1,325 short of the $250,000 married-filing-jointly Additional Medicare Tax threshold, with Arkansas imposing no mandatory employee-side payroll tax. Almost every wrong answer is the correct structure evaluated at a stale or invented wage base (the 2025 $176,100, the 2024 $168,600, or projections between $168,600 and $184,000), and each such answer's shortfall equals 6.2% of the amount by which it understated the taxable maximum. The remaining misses drop the cap entirely and apply a flat 7.65% to full wages ($19,023.64), stack the overtime premium onto wages, invent an Arkansas employee payroll tax, or abandon a correct derivation and submit an unrelated number." us,scenario_117,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_117,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is determined individually and requires both membership in an eligible category—pregnant, postpartum, breastfeeding, infant, or child under five—and satisfaction of the income or adjunctive-eligibility rules. The spouse is not placed in any eligible WIC category by the stated facts, and household income of approximately $250,733 also exceeds the applicable 185% federal poverty guideline." -us,scenario_117,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Arkansas's base here is wages of $248,675 less the $530.47 capital loss — $248,144.53 — with the $2,589 state tax refund excluded and the $35,525 FLSA overtime premium already inside the wage total rather than additive. The only allowable deduction is the $25,900 charitable contribution reduced by the 2026 0.5%-of-AGI floor ($1,240.72), giving $24,659.28: mortgage interest is zero because only the $501,000 balance is listed, the ~$11,800 of medical costs fall below the 7.5%-of-AGI floor of $18,610, unreimbursed employee expenses are not deductible, and Arkansas itemized deductions exclude state income tax. Arkansas's graduated 2026 schedule then yields $8,189.05 on $223,485.25 — an average rate of 3.66%, well under the 3.9% top rate — and $205 of nonrefundable credits ($29 each for two adults and three children, plus $60 additional for qualified individuals) bring it to $7,984.05. The wrong answers cluster around imputed mortgage interest, standard-deduction shortcuts, repealed 4.4%/4.9% top rates or flat top-rate arithmetic, and omitted Arkansas credits." +us,scenario_117,state_income_tax_before_refundable_credits,46,llm_error,thresholds_rates,False,,"Arkansas AGI is $248,144.53, which is the wages less the $530 capital loss. It excludes the $2,589 state refund, and the FLSA overtime premium is already inside gross wages rather than being a separate add-on or a state exclusion. The household itemizes, and its only allowable item is charity: $25,900 minus the 2026 floor of 0.5% of federal AGI ($1,253.67), which leaves $24,646.33. No mortgage interest is allowed because only a loan balance is listed, and medical expenses fall below the floor. Taxable income is $223,498.20, and the engine's 2026 schedule taxes it at $8,189.53, an effective rate of about 3.66%. That is roughly $450 below what a schedule with a 3.9% top rate gives ($8,635). Subtracting $145 in personal credits and the $60 additional tax credit for qualified individuals leaves $7,984.53. The wrong answers fall into four groups: taking the standard deduction instead of itemizing, inventing mortgage interest, using a 3.9%, 4.4% or 4.9% top rate, or leaving out some or all of the $205 in nonrefundable credits." us,scenario_117,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_118,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"New York automatically provides Medicaid eligibility through the SSI-recipient category. The 74-year-old passes SSI's aged, resource, and income tests, receives $9,368 in annual SSI, and therefore qualifies for Medicaid independently of MAGI pathways." us,scenario_118,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earned compensation, not Social Security retirement benefits. With no wages, self-employment income, or other covered earnings listed, the employee-side payroll-tax base is zero and payroll tax is $0." -us,scenario_118,snap,39,llm_error,taxable_income_or_deductions,False,,"This 74-year-old blind and disabled head with $2,800 of Social Security and $70 in assets receives $780.67/month of SSI (itself one of the requested outputs), and SSI is countable unearned income for SNAP, so the unit's gross income is $1,014/month — not the $233.33 of Social Security alone. The elderly/disabled deduction stack (standard deduction plus the uncapped excess shelter allowance built on $598.75/month of mortgage interest and real estate taxes) removes $819.62/month, leaving net income of $194.38, an expected contribution of $58.20, and an allotment of $298 − $58.20 = $239.80/month; nine months at $239.80 plus three months at $248.58 after the FPG/standard-deduction update give $2,903.94. The wrong answers split three ways: models that dropped the imputed SSI zeroed net income and paid the near-maximum allotment (~$3,350–$3,708); models that counted SSI stopped at roughly $405 of deductions and landed near $1,380; and a third group declared the household ineligible and returned $0." -us,scenario_118,ssi,33,llm_error,thresholds_rates,False,,"SSI here is a pure top-up: the 2026 individual Federal Benefit Rate of $994/month ($11,928/year) less countable income, where the $2,800 of Social Security is reduced by the $20-per-month ($240-per-year) general income exclusion to $2,560 countable, giving $780.67/month or $9,368. Three specific traps separate the reference from the wrong answers. First, the 2026 FBR is $994/month — nearly every model anchored on stale or under-projected rates ($943, $967, $983, $990, $991) and landed a few hundred dollars low. Second, PolicyEngine's `ssi` variable is the federal payment alone, so the New York State Supplement Program (a separate variable) must not be added — eleven models built a combined federal-plus-SSP payment standard and overshot. Third, SSI reduces the FBR by countable income rather than cutting off at it, and the income test is monthly: models that compared the $2,800 annual total against a monthly rate, or applied the $20 exclusion once per year, either zeroed the benefit out or shaved $220 off it." -us,scenario_118,state_refundable_credits,30,llm_error,state_local_rule,False,,"The trap is New York's real property tax credit (Tax Law §606(e), Form IT-214): a fully refundable circuit breaker keyed to age, household gross income, and property tax paid — not to earned income, dependents, or any NY tax liability — and available to homeowners (property value ≤ $85,000; mortgage balance here is $82,237) as well as renters. Household gross income for IT-214 includes Social Security, so the $2,800 (even grossed up by SSI) sits far under the $18,000 ceiling, and excess real property tax is $1,633.50 minus 6% of household gross income, which times the rate factor blows through the statutory ceiling. Because a household member is 65 or older (head is 74), that ceiling is $375 rather than the $75 that applies to under-65 households, so the credit lands exactly at $375. Nearly every wrong model either never surfaced §606(e) at all — screening only for the NY EITC, Empire State Child Credit, or CDCC, all of which need earned income or children — or conditioned a refundable credit on there being NY tax to offset; the few that found the credit misapplied its cap or its threshold." -us,scenario_119,child1_chip_eligible,13,llm_error,health_coverage,False,,"Virginia FAMIS covers children under 19 with MAGI up to 200% FPL, and this household's Medicaid MAGI of $52,569 ($55,000 wages + $800 interest less the $3,087 traditional 401(k) and $144 traditional IRA) sits near 193% of the 2026 three-person poverty guideline, so the age-plus-income screen every model ran returns ""eligible."" That screen is only half the test: CHIP reaches a child only as a ""targeted low-income child,"" which requires the child to be uninsured — not covered under a group health plan or other health insurance (42 U.S.C. 1397jj(b)(1)(C)) — and Child 1 is listed as having employer-sponsored insurance. The engine therefore fails Child 1 on CHIP's own categorical/coverage requirement rather than on income, while Medicaid is separately out of reach because 193% FPL exceeds Virginia's 143% limit for children 6-18. Every wrong model stopped at ""under 19 and inside the FAMIS income band"" and never applied the existing-coverage bar." +us,scenario_118,snap,46,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is 74, blind and disabled, has $233.33/month of Social Security and $70 in assets. That qualifies the head for federal SSI of $780.67/month: the $994 benefit rate less $213.33 of countable Social Security after the $20 exclusion. SSI counts as SNAP unearned income, so gross income is $1,014. After the $209 standard deduction, PolicyEngine applies an uncapped elderly/disabled excess shelter deduction. Shelter costs are about $1,013/month: $599 of mortgage interest and property tax plus New York's standard utility allowance. That leaves net income of about $194, a $58.20 expected contribution, and $298 − $58.20 = $239.80/month for January–September. October–December pays $248.58 under FY2027 parameters, for $2,903.94 in total. The wrong answers fall into three groups. About half the models left out SSI, drove net income to $0 and paid roughly the maximum ($3,500/year). About ten counted SSI but used only mortgage interest and property tax as shelter costs (net ≈ $609, about $1,384/year). The rest denied eligibility outright. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)." +us,scenario_118,ssi,35,llm_error,state_local_rule,False,,"The ssi output is the federal SSI payment only. It uses the 2026 FBR of $994/month minus countable unearned income of $213.33/month ($233.33 Social Security less the $20 general exclusion, applied monthly), which gives $780.67/month or $9,368/year. New York's state supplement is not part of ssi. Most wrong answers fall into one of three errors: adding the NY SSP (about $87/month, or $1,044/year), using a stale or understated FBR ($943, $967, $982-$993), or applying the $20 exclusion once a year ($2,780 countable) instead of monthly ($2,560). The $0 answers either treated $2,800 of annual Social Security as exceeding the monthly FBR, ignored the listed aged, blind and disabled status, or computed a positive benefit and then submitted zero." +us,scenario_118,state_refundable_credits,34,llm_error,state_local_rule,False,,"The whole $375 is New York's Real Property Tax Credit (IT-214), which is refundable. A homeowner who pays real property taxes can qualify without NY tax liability, earned income, or rent. From 2025 the credit is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives $375. Most models never considered the RPTC. Others used the wrong cap ($75, $171, or $1,000), used an invented formula, or counted the nonrefundable NY household credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)." +us,scenario_119,child1_chip_eligible,17,llm_error,health_coverage,False,,"Child 1 is 14, so the age test passes. Household MAGI of about $52,569 (roughly 192-197% FPL) is also inside Virginia's FAMIS band, so the income test passes too. The child fails CHIP's categorical test instead. The household facts say the child has employer-sponsored insurance, and a CHIP targeted low-income child cannot be covered under a group health plan. Every answering model ran only the age and income-band screen and ignored the child's ESI coverage flag. None of them applied the no-other-coverage requirement, so all returned eligible." us,scenario_119,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_medicaid_eligible,5,llm_error,health_coverage,False,,"The separating step is the boundary between Virginia's Medicaid and its separate CHIP program. A 14-year-old with no SSI, no disability, and no other categorical hook reaches Medicaid only through the MAGI pathway for children ages 6-18, capped at 143% FPL plus the 5-point disregard; this household sits at 1.92x FPL after the $3,231 of traditional 401(k)/IRA contributions, so medicaid_category is NONE. Coverage from 148% up to about 200% FPL in Virginia is FAMIS, which PolicyEngine reports as CHIP under the separate child1_chip_eligible output, not as Medicaid. Every substantive wrong answer imported the FAMIS/CHIP threshold into the Medicaid question, and one also sized the household at four people instead of three." us,scenario_119,child1_medicare_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_119,child2_chip_eligible,13,llm_error,thresholds_rates,False,,"Virginia's FAMIS ceiling for a child aged 6-18 is 200% of the federal poverty guideline, and the countable income in the trace is the household's ~$55,800 of wages plus taxable interest, with no offset for the $3,087 traditional 401(k), the $144 traditional IRA, or the $6,389 employer-plan premium. Measured against the 2026 three-person poverty guideline, that income is above 200%, which places the 11-year-old above CHIP just as he is above Virginia's 143% child Medicaid limit — the engine's NONE Medicaid category. Every wrong model either shaved the numerator down with retirement or premium exclusions, or stretched the ceiling to 205% by re-adding the 5-point MAGI disregard to a limit that already reflects it, and several never computed the FPL ratio at all and stopped at the under-19 age test." +us,scenario_119,child2_chip_eligible,17,llm_error,thresholds_rates,False,,"Child2 is 11, so CHIP's under-19 age test passes, and the engine rules out Medicaid (category NONE). The deciding step is CHIP's own income test. Measured on the roughly $55,800 of household income in the trace, child2's income level is above Virginia's CHIP child limit, so is_chip_eligible is False. Every model that answered assumed the household was under Virginia's 200-205% FPL FAMIS cap and marked child2 eligible. Most got there by shrinking income to a MAGI of $52,569 (about 192-197% FPL) or lower after subtracting 401(k), IRA or ESI-premium amounts. Two models returned no answer." us,scenario_119,child2_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_medicaid_eligible,5,llm_error,thresholds_rates,False,,"The trap is the boundary between Virginia's children's Medicaid MAGI limit and its FAMIS/CHIP ceiling: Medicaid for children ages 6-18 in Virginia stops at 143% FPL, while FAMIS covers 143% up to 200% FPL (205% with the 5% disregard). Child2's household MAGI lands at 1.92 x FPL, which sits squarely in the CHIP band and above every Medicaid MAGI child category, and with zero SSI, no disability, and no foster or medically-needy pathway the engine assigns medicaid_category NONE. Every substantive wrong answer imported the 200%-205% FAMIS ceiling as the Medicaid threshold, treating FAMIS coverage as Medicaid coverage; two models emitted no value at all for this key." us,scenario_119,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,child2_wic_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_119,federal_income_tax_before_refundable_credits,20,llm_error,credit_phaseout,False,,"The trap is that tax year 2026 stays on the extended TCJA/OBBBA structure and this household's nonrefundable credits exactly exhaust its tax. Pre-credit tax on $28,418.54 of head-of-household taxable income ($52,568.54 AGI less the $24,150 standard deduction) is $3,056.23, and the CDCC at the 2026 35% rate on $1,560 of care expenses ($546), the 10% saver's credit on the $2,000 contribution cap ($200), and the nonrefundable share of the $4,400 CTC ($2,200 each for two children under 17) absorb every dollar, with the $2,089.77 excess flowing to the refundable ACTC. Every wrong answer either reverted 2026 to pre-TCJA law (personal exemptions, $1,000-per-child CTC), inverted the CTC ordering by letting refundable ACTC claim the credit before the nonrefundable portion offsets liability, filed the head as single, or applied no credits at all. Several models derived the correct $0 and then submitted a different number." -us,scenario_119,federal_refundable_credits,39,llm_error,credit_phaseout,False,,"Two steps separate $2,854.01 from the wrong answers. First, the EITC phase-out runs against the greater of AGI and earned income, and earned income is the full $55,000 of gross wages — the $3,087 traditional 401(k) deferral does not reduce it — so the 2026 head-of-household two-child formula gives $7,316 − 21.06% × ($55,000 − $23,890) = $764.23, not zero (the phase-out does not complete until roughly $58,600). Second, the 2026 CTC is $2,200 per child ($4,400), while only $2,310.23 of tax survives the $546 CDCC (OBBBA's 35% rate at this AGI) and the $200 saver's credit on the $4,000 of retirement contributions, so $4,400 − $2,310.23 = $2,089.77 spills into the refundable ACTC — far below both the $1,700-per-child cap and 15% × ($55,000 − $2,500) = $7,875. Models failed by zeroing the EITC, by phasing it from AGI net of the 401(k) or the $6,389 ESI premium, or by using the repealed $2,000-per-child credit and letting an overstated tax liability absorb it." +us,scenario_119,federal_income_tax_before_refundable_credits,21,llm_error,thresholds_rates,False,,"For 2026, OBBBA keeps the TCJA structure. That means a $24,150 head-of-household standard deduction, no personal exemptions, and a $2,200 CTC for each child under 17, so this household's CTC is $4,400. The CDCC rate at this AGI is 35%, giving $546. Tax before credits is $3,056.23. The CDCC and the $200 Saver's credit bring it down to $2,310.23. The nonrefundable CTC is limited only by that remaining liability, so the $4,400 credit wipes it out and the result is $0. The wrong answers came from four mistakes: assuming TCJA expired (personal exemptions and a $1,000-per-child CTC), filing as single, reserving part of the CTC for the refundable ACTC before applying the nonrefundable part, or leaving out credits." +us,scenario_119,federal_refundable_credits,46,reference_engine_defect,credit_phaseout,False,,"This case has two traps. First, the two-child EITC does not phase out completely here. The 2026 credit is $7,316 and phases out at 21.06% above $23,890 (head of household). The phase-out applies to the greater of AGI ($52,569) and earned income, and earned income is the full $55,000 of wages, so the EITC is $7,316 − 0.2106 × ($55,000 − $23,890) = $764.23. Second, the refundable CTC is the leftover part of the 2026 CTC ($2,200 per child, $4,400 total). Regular tax is about $3,056 on $28,419 of taxable income after the $24,150 head-of-household standard deduction. The 35% CDCC ($546) and the 10% Saver's Credit ($200) reduce it to $2,310.23, and the CTC absorbs only that amount. The remaining $2,089.77 is refundable, which is under the $1,700-per-child cap and the 15%-of-earnings-over-$2,500 limit. Models that zeroed the EITC, used AGI or wages net of the 401(k) as the phase-out base, used $2,000 per child, or skipped the CDCC and Saver's Credit before the CTC all missed $2,854.01. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 2,854.01; the corrected value is 3,366.07. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." us,scenario_119,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_medicaid_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_medicare_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_119,payroll_tax,12,llm_error,payroll_tax_base,False,,"The FICA base here is the full $55,000 of wages: 6.2% Social Security = $3,410 (wages are far below the 2026 Social Security wage cap) plus 1.45% Medicare = $797.50, for $4,207.50, with no Additional Medicare Tax and no Virginia mandatory employee payroll tax. The separating step is the $6,389 employer-sponsored insurance premium input, which is employer-paid coverage that does not reduce the employee's FICA wage base — it is not a Section 125 employee salary-reduction contribution. Ten of the twelve wrong answers either subtracted that premium (yielding $3,718.74 = 7.65% x $48,611) or failed to carry their own correct arithmetic into the submitted value." +us,scenario_119,payroll_tax,13,llm_error,payroll_tax_base,False,,"The Social Security and Medicare taxes both apply to the full $55,000 of gross wages. That gives 6.2% × $55,000 = $3,410 plus 1.45% × $55,000 = $797.50, for a total of $4,207.50. No Additional Medicare Tax or Virginia employee payroll tax applies. The household facts never say the head pays the $6,389 employer-sponsored insurance premium through a Section 125 salary reduction, so it does not lower the FICA wage base. Most models assumed a pre-tax cafeteria-plan deduction anyway, taxed $48,611 and got $3,718.74. The rest failed to submit an answer, reported a number that doesn't match their own math, or stated an estimate without computing it." us,scenario_119,reduced_price_school_meals_eligible,7,llm_error,taxable_income_or_deductions,False,,"The school-meal test compares gross household income — $55,000 wages plus $800 taxable interest = $55,800, with no subtraction for employer-sponsored insurance premiums, traditional 401(k), or traditional IRA contributions — against the current-year poverty guideline for a three-person household (head plus two children), yielding a school_meal_fpg_ratio of 2.04. The 2026 three-person 185% reduced-price limit is roughly $50,600, so $55,800 lands above it and the tier is PAID; Virginia has no universal free-meal provision and the household has no SNAP/TANF categorical pathway. Every wrong model pushed the ratio under 1.85 one of three ways: netting pre-tax and premium amounts out of the income base ($46,180), counting a fourth household member, or asserting the 130–185% band without ever dividing income by the guideline. The trap is that the household's income sits just 19 points above the cutoff, so any one of those shortcuts flips the answer." us,scenario_119,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_119,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_119,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Virginia's 2026 chain is: federal AGI of $52,568.54 (wages $55,000 less $3,087 traditional 401(k) and $144 traditional IRA, plus $800 of interest), less the $8,750 head-of-household standard deduction, the $1,560 child and dependent care expense deduction — Virginia subtracts employment-related care costs from Virginia AGI rather than granting a percentage credit — and $2,790 of personal exemptions ($930 × 3), giving $39,468.54 of taxable income and $2,011.94 under the 2%/3%/5%/5.75% schedule. The final step is Virginia's EITC claimed as a 20% nonrefundable credit: the two-child federal EITC is phased down to $764.23 at this income, so the credit is $152.85 and the answer is $1,859.09. Nearly every wrong answer breaks at one of three points — a standard deduction of $8,500, $9,000, $8,000, $4,500, $3,000 or $5,200 instead of $8,750; omitting the $1,560 care deduction or converting it into a nonexistent state credit; or sizing the Virginia EITC off an unphased federal EITC (estimates from $1,115 up to $5,670). A separate cluster double-subtracted the $6,389 of employer-sponsored insurance premiums already excluded from the $55,000 W-2 wage figure, landing on a $46,180 AGI." -us,scenario_119,state_refundable_credits,11,llm_error,state_local_rule,False,,"The trap is Virginia's two-track EITC: the 20% credit under Va. Code § 58.1-339.8 is nonrefundable and capped at Virginia liability, while the refundable version is 15% and is claimed only in lieu of the nonrefundable credit, for taxable years beginning on or after January 1, 2022 but before January 1, 2026. Every wrong model booked a share of the federal EITC as a refundable Virginia credit in tax year 2026, when the refundable election is no longer available and the nonrefundable 20% credit is fully absorbed by this household's Virginia tax on ~$55,800, so it flows entirely through state_income_tax_before_refundable_credits. Virginia's only other individual credit in reach, the Low Income Individual Credit, is nonrefundable and requires VAGI below the federal poverty guideline, which a $55,800 three-person household clears by a wide margin. Nothing in Virginia's 2026 credit set is refundable for this filer, so state_refundable_credits is $0." +us,scenario_119,state_income_tax_before_refundable_credits,46,reference_engine_defect,credit_phaseout,False,"PolicyEngine phases out the federal EITC on $55,000 of gross wages that still include the $3,087 traditional 401(k) deferral ($7,316 − 21.06% × ($55,000 − $23,890) = $764.23 to the cent), but IRC §32(c)(2)(A)(i) counts employee pay as earned income only when it is includible in gross income; the phase-out income should therefore be AGI of $52,568.54, which gives a federal EITC of about $1,276.30, a Virginia 20% nonrefundable EITC of about $255.26, and an output of about $1,756.68.","Virginia taxable income starts from federal AGI ($52,568.54) and subtracts the 2026 $8,750 single/HOH standard deduction, three $930 personal exemptions ($2,790), and the $1,560 child and dependent care expense deduction. That gives $39,468.54 of taxable income and $2,011.94 of bracket tax, and Virginia's 20%-of-federal nonrefundable EITC comes off that before refundable credits. Most models used a stale $8,500 or older standard deduction, skipped the $1,560 childcare deduction, invented Virginia childcare or child credits, reduced AGI by the $6,389 ESI premiums, or dropped the nonrefundable EITC. The models that got both the bracket tax and the EITC step right differ from the reference only in the federal EITC base: they phased it out on AGI, while the reference's $764.23 federal EITC is phased out on $55,000 of gross wages. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated reference_engine_defect (credit_phaseout). The frozen reference is 1,859.09; the corrected value is 1,756.68. PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit. Reference is an engine defect; output excluded from scoring (26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A))." +us,scenario_119,state_refundable_credits,15,llm_error,state_local_rule,False,,"Virginia makes filers choose between a nonrefundable EITC equal to 20% of the federal EITC and a refundable EITC. Both versions cannot be claimed. The household has federal AGI of about $52.6k after the traditional 401(k) and IRA deductions and a federal EITC of about $1.3k. Its Virginia liability is roughly $2,000 (standard deduction and three exemptions, then 5.75% on income above $17k). That liability absorbs the entire nonrefundable credit. PolicyEngine therefore puts the Virginia EITC among the nonrefundable credits that reduce state_income_tax_before_refundable_credits, so va_refundable_credits and state_refundable_credits are $0. Every model that answered treated Virginia's EITC as automatically refundable and put a percentage of the federal EITC into refundable credits, with rates and federal bases that differed from model to model." us,scenario_119,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_120,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"2026 runs on the permanent post-OBBBA schedule, and that is the trap: TCJA rates stay in force (10/12/22/24 with the 24% bracket ending at $201,775), the SALT cap is $40,400 so the full ~$11,896 of Connecticut income tax plus $2,012 of property tax is deductible, charitable gifts face a 0.5%-of-AGI floor that cuts $20,164 to $19,001, mortgage interest prorates by $750,000/$1,000,000 of acquisition debt to $2,808, miscellaneous itemized deductions and personal exemptions remain repealed, and the $6,000 senior bonus deduction is fully phased out at this income. AGI of $232,635 excludes the $1,750 state tax refund and takes the $2,447 farm-rent loss above the line; itemized deductions of $35,717 beat the $18,150 standard deduction, leaving $196,918 of taxable income and $39,858.42 of bracket tax. The final $163.40 is 3.8% NIIT on the full $4,300 of taxable interest plus rental income, which is not netted against the farm-rent loss. The wrong answers split into two camps: models that assumed a TCJA sunset and rebuilt the return on pre-TCJA rates with a personal exemption and 2%-floor employee expenses, and models that kept the expired $10,000 SALT cap or dropped the state income tax from SALT — and nearly all of them omitted the NIIT." +us,scenario_120,federal_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 law as made permanent by the OBBBA. AGI is $232,635: wages are reduced only by the $617 traditional 401(k) deferral, the farm rent loss is netted, and the state refund is not included. Itemized deductions total $35,717: SALT of about $13,908, which is the full Connecticut income tax plus property tax under the $40,400 cap; mortgage interest of $2,808, prorated by the $750k acquisition-debt limit; and charitable gifts of $19,001 after the new 0.5%-of-AGI floor. Taxing $196,918 at the 2026 10/12/22/24% single schedule gives $39,858.42, and NIIT of $163.40 on $4,300 of interest and rental income is added. The wrong answers split into four groups: models that assumed a TCJA sunset (personal exemption, pre-TCJA brackets, miscellaneous deductions); models that used the old $10,000 SALT cap or dropped Connecticut income tax; models that subtracted ESI premiums from wages or skipped the charitable floor; and near-misses that omitted NIIT or computed it on a base netted for the farm rent loss. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_120,federal_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -932,38 +941,38 @@ us,scenario_120,head_medicaid_eligible,1,parse_contract_failure,missing_output,F us,scenario_120,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility applies because the head is age 76, well above the age-65 threshold. Income, assets, disability status, and employer-sponsored insurance do not negate age-based Medicare eligibility." us,scenario_120,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_120,payroll_tax,32,llm_error,state_local_rule,False,,"The separator in this case is Connecticut's Paid Family and Medical Leave contribution — a mandatory employee-only withholding of 0.5% on covered wages, $827.98 on $165,597 — which sits alongside the two federal components ($10,267.01 Social Security at 6.2% and $2,401.16 Medicare at 1.45%) in the requested ""mandatory employee state payroll taxes."" Most wrong models scanned Connecticut for an SDI or employee UI levy, found none, and declared no state employee payroll tax, landing on $12,668.17 exactly $827.98 short. The second trap is the base: PolicyEngine taxes the full $165,597 of wages, while a cluster of models netted out the $7,429 employer-sponsored insurance premium (and in some cases the $617 traditional and $109 Roth 401(k) contributions, neither of which reduces FICA wages), producing $12,890.70 on a $158,168 base. Wages are below both the Social Security wage base and the $200,000 Additional Medicare Tax threshold, so neither of those adjustments applies." +us,scenario_120,payroll_tax,36,llm_error,state_local_rule,False,,"The reference has three parts, all on the full $165,597 of wages: 6.2% Social Security ($10,267.01, since wages are under the 2026 wage base), 1.45% Medicare ($2,401.16), and Connecticut's mandatory 0.5% Paid Leave employee contribution ($827.98). There is no Additional Medicare Tax because wages are under $200,000. The most common mistake was leaving out the CT paid-leave contribution, which gives exactly $12,668.17. The second was subtracting the $7,429 of employer-sponsored insurance premiums (and sometimes 401(k) deferrals) from the payroll-tax base, even though PolicyEngine applies payroll tax to the listed gross wages and traditional 401(k) deferrals are FICA wages under IRC §3121(v)(1)(A)." us,scenario_120,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_120,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"Connecticut's Social Security Benefit Adjustment is the trap: CT keeps only 25% of benefits received ($10,475 of the $41,900) in CT AGI, so the subtraction equals the federally taxable $35,615 less $10,475 = $25,140 and applies at every income level — it is neither zeroed out above the $75,000 threshold nor equal to the full federally taxable amount. That leaves CT AGI of $207,495.12, which the 2%/4.5%/5.5%/6%/6.5% single schedule taxes to $11,237.18, and CT then adds the $250 bottom-bracket phase-out add-back plus $250 low-rate and $180 middle-rate benefit recapture, reaching $11,917.18. Nearly every wrong answer lands on one side or the other of that adjustment — full inclusion drives answers toward $13,000+, full exclusion toward $9,000–$10,600 — and most also dropped the $680 of add-back and recapture or resurrected the fully phased-out $15,000 personal exemption and the property tax credit." +us,scenario_120,state_income_tax_before_refundable_credits,46,reference_engine_defect,taxable_income_or_deductions,False,,"This filer is above Connecticut's $75,000 single threshold. At that income, Connecticut subtracts the federally taxable Social Security that exceeds 25% of total benefits ($35,615 − $10,475 = $25,140). CT AGI is therefore $232,635 − $25,140 = $207,495, and the personal exemption, pension subtraction, personal credit and property tax credit are all fully phased out. The 2026 schedule (2/4.5/5.5/6/6.5%) gives $11,237.18. Adding the $250 2% phase-out add-back and $430 of benefit recapture ($250 low + $180 middle) gives $11,917.18. The models went wrong in five main ways: they denied any Social Security subtraction at this income, subtracted all taxable benefits, used the pre-2024 3%/5% bottom rates, invented deductions or credits, or dropped part of the $680 add-back and recapture. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 11,917.18; the corrected value is 11,919.06. PolicyEngine deducts traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))." us,scenario_120,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_120,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_121,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Two facts drive the $0: PolicyEngine's `disability_benefits` input is a non-taxable income source excluded from AGI, so AGI is wages only ($25,665), and the household's medical costs ($2,200 non-Medicare-Part-B premiums + $30,000 other medical + $400 over-the-counter = $32,600) generate an itemized medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675. That itemized total exceeds AGI, so taxable income floors at $0 and there is no tax to reduce with nonrefundable credits. Every wrong model either taxed the disability benefits, ignored the medical itemized deduction, or both, leaving a positive 10%/12%-bracket residue." +us,scenario_121,federal_income_tax_before_refundable_credits,4,prompt_ambiguity,taxable_income_or_deductions,False,,"Two facts drive the $0: PolicyEngine's `disability_benefits` input is a non-taxable income source excluded from AGI, so AGI is wages only ($25,665), and the household's medical costs ($2,200 non-Medicare-Part-B premiums + $30,000 other medical + $400 over-the-counter = $32,600) generate an itemized medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675. That itemized total exceeds AGI, so taxable income floors at $0 and there is no tax to reduce with nonrefundable credits. Every wrong model either taxed the disability benefits, ignored the medical itemized deduction, or both, leaving a positive 10%/12%-bracket residue. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 875.34. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." us,scenario_121,federal_refundable_credits,3,llm_error,credit_phaseout,False,,"The childless EITC phaseout must use the greater of earned income or adjusted gross income. The $13,044 of disability benefits raises AGI above the 2026 childless EITC income limit, so the EITC and total federal refundable credits are $0; all three models instead treated $17,276 of net wages as both earned income and AGI." us,scenario_121,head_medicaid_eligible,11,llm_error,categorical_eligibility,False,,"The trap is that the `is_disabled` flag by itself opens no Medicaid category in South Carolina. The non-MAGI disability route runs through SSI receipt, and SSI here is $0 because $25,665 in wages plus $13,044 of disability benefits leave countable income far above the SSI federal benefit rate (~$12,000/year) after the $20 general and $65-plus-half earned disregards. The MAGI route fails too: MAGI counts only the taxable wages ($25,665 = 1.61 x FPL for a one-person unit, since child support and the disability benefits are excluded), which sits above SC's parent/caretaker limit, above the 138% FPL expansion limit SC has not adopted, and is moot anyway because no child is present to establish a parent/caretaker category. Every wrong model substituted a program the engine does not implement and that this income fails regardless — a working-disabled buy-in at 250% FPL, a medically needy spend-down against the $30,000 of medical expenses, or a bare disability presumption — instead of testing an actual pathway, so medicaid_category is NONE and eligibility is False." -us,scenario_121,head_medicare_eligible,13,llm_error,age_disability,False,,"PolicyEngine's is_medicare_eligible turns on the age-65 threshold; the head is 53, and neither the is_disabled flag nor the disability_benefits amount is an input to that variable. Every wrong model imported the real-world SSDI pathway (24 months of Title II disability entitlement) and, on top of that, re-labeled the generic disability_benefits input as SSDI — PolicyEngine keeps SSDI in social_security_disability, and no entitlement-duration input exists at all. The prompt further instructs that unlisted facts be treated as false and that nothing be inferred, so assuming a satisfied 24-month entitlement clock manufactures a fact the household record does not contain. The correct derivation is a single age comparison: 53 < 65, so the answer is 0." +us,scenario_121,head_medicare_eligible,16,llm_error,age_disability,False,,"Under age 65, the Medicare disability pathway requires 24 months of Social Security Disability Insurance entitlement under 42 U.S.C. 426(b), or ESRD or ALS. A disability flag alone does not qualify. The head's $13,044 is generic disability-benefits income, not Social Security disability. SSDI receipt and its duration are not listed, so they count as 0, and ESRD is false. That leaves only the age-65 test, which a 53-year-old fails. All 16 models treated ""is disabled"" plus generic disability benefits as SSDI Medicare entitlement. Several also assumed the 24-month waiting period had already passed, even though the prompt says to treat unlisted inputs as 0 and not to infer benefit receipt." us,scenario_121,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_121,payroll_tax,7,llm_error,payroll_tax_base,False,,"The listed employer-sponsored insurance premium is not specified as a pre-tax payroll deduction, so it does not reduce FICA wages under the prompt’s rule that unlisted statuses are false. Applying 6.2% Social Security and 1.45% Medicare to the full $25,665 of wages yields $1,591.23 and $372.14, totaling $1,963.37." us,scenario_121,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_121,snap,23,llm_error,age_disability,False,,"Every wrong answer turns on a single step: reading the bare ""is disabled"" input as making this a SNAP elderly-or-disabled household, which would waive the gross income test and unlock the excess medical deduction. SNAP's disabled-member definition (7 CFR 271.2) requires age 60+ or receipt of SSI, Title II disability or blindness benefits, disability-based Medicaid, a government disability retirement, or VA total-disability compensation; a 53-year-old with $25,665 of wages, employer-sponsored coverage, and an unspecified $13,044 of disability benefits satisfies none of them. The gross income test therefore binds: $25,665 + $13,044 + $1,200 = $39,909, or $3,325.75/month, against a one-person limit of about $1,696/month (130% FPL) and even against a 200% FPL broad-based categorical eligibility ceiling of $2,608/month. Deductions operate only on net income and can never cure a failed gross screen, and the same elderly/disabled status gates the 7 CFR 273.9(d)(3) medical deduction that every model used to zero out net income, so the benefit is $0." +us,scenario_121,snap,30,llm_error,age_disability,False,,"Every model with an answer let the head's ""is disabled"" flag put the household on SNAP's elderly/disabled track. That track skips the gross income test and allows the uncapped excess medical deduction. But SNAP defines a disabled member by benefit receipt under 7 CFR 271.2: SSI, Social Security disability, or comparable government disability or blindness payments. This head gets no SSI and no Social Security disability, only generic disability benefits, so the household is an ordinary one-person unit. Its gross income is about $3,326 a month ($25,665 wages + $13,044 disability benefits + $1,200 child support, divided by 12), well above South Carolina's gross limit of 130% of the poverty line (about $1,696 a month in FY2026), so SNAP is $0 and the $30,000 in medical costs never matters." us,scenario_121,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_121,state_income_tax_before_refundable_credits,8,llm_error,taxable_income_or_deductions,False,,"South Carolina's tax base starts from federal taxable income, and this filer's deductible medical outlays total $32,600 ($30,000 other medical expenses + $2,200 health insurance premiums excluding Medicare Part B + $400 over-the-counter health expenses). Netted against the 7.5%-of-AGI floor (at most $2,903 on the largest possible AGI of $38,709), that leaves roughly $30,000 of itemized medical deduction — nearly double the $16,100 single standard deduction — which wipes out federal taxable income, and SC's subtraction for the disabled filer's disability income removes anything left, so SC taxable income never reaches the top of the zero-rate bracket and the tax is $0. Every wrong model defaulted to the standard deduction and treated the $30,000 medical line as an irrelevant household fact; several compounded that with invented SC rate schedules and a nonexistent personal exemption. The gap between $0 and every submitted number is entirely the unclaimed itemized medical expense deduction — no bracket or rate correction closes it." -us,scenario_122,federal_income_tax_before_refundable_credits,35,llm_error,taxable_income_or_deductions,False,,"AGI is uncontested at $70,646.18 (pension $25,000 + IRA $14,400 + dividends $5,552 + interest $13 + the capped 85% Social Security inclusion of $27,200, less the $1,518.82 net capital loss); the entire case turns on the 2026 below-the-line stack. Correct law stacks $16,100 base + $2,050 (age 65+) + $2,050 (blind) + $6,000 OBBBA senior deduction + $1,000 non-itemizer charitable deduction (IRC §170(p), first effective 2026) = $27,200, beating $14,926 of itemized deductions and leaving $43,446.18 taxable; $1,824 of qualified dividends sits in the 0% band and $41,622.18 of ordinary income taxed at 10% to $12,400 then 12% gives $1,240 + $3,506.66 = $4,746.66. Roughly half the wrong models rebuilt the return under a TCJA sunset that OBBBA cancelled — personal exemptions, an ~$8,300 standard deduction, 15%/25% brackets, and 2%-floor miscellaneous employee expenses — and most of the rest applied only part of the deduction stack, omitting the $6,000 senior deduction, the $1,000 charitable deduction, or both. A third cluster submitted numbers their own worksheets never produced." +us,scenario_121,state_income_tax_before_refundable_credits,8,prompt_ambiguity,taxable_income_or_deductions,False,,"South Carolina's tax base starts from federal taxable income, and this filer's deductible medical outlays total $32,600 ($30,000 other medical expenses + $2,200 health insurance premiums excluding Medicare Part B + $400 over-the-counter health expenses). Netted against the 7.5%-of-AGI floor (at most $2,903 on the largest possible AGI of $38,709), that leaves roughly $30,000 of itemized medical deduction — nearly double the $16,100 single standard deduction — which wipes out federal taxable income, and SC's subtraction for the disabled filer's disability income removes anything left, so SC taxable income never reaches the top of the zero-rate bracket and the tax is $0. Every wrong model defaulted to the standard deduction and treated the $30,000 medical line as an irrelevant household fact; several compounded that with invented SC rate schedules and a nonexistent personal exemption. The gap between $0 and every submitted number is entirely the unclaimed itemized medical expense deduction — no bracket or rate correction closes it. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 212.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)." +us,scenario_122,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"AGI is $70,646.18: $43,446 of non-Social Security income plus $27,200 of taxable Social Security (the 85% cap binds). Under OBBBA's permanent TCJA structure, 2026 deductions total $27,200. That is the $16,100 basic standard deduction, plus $2,050 each for age and blindness, plus the $6,000 senior deduction (MAGI is under $75,000), plus the new $1,000 charitable deduction for single non-itemizers. Taxable income is therefore $43,446.18. The $1,824 of qualified dividends falls in the 0% band, and the remaining $41,622.18 is taxed at 10%/12%, giving $4,746.66. The wrong answers made one of four errors: they assumed the TCJA sunset (personal exemption, itemizing with miscellaneous deductions, a 15% bracket); they dropped the $6,000 senior deduction; they dropped only the $1,000 non-itemizer charitable deduction, which leaves them exactly $120 high; or they zeroed out the tax with credits that do not exist." us,scenario_122,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_122,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Age, blindness, and disability do not by themselves establish Medicaid eligibility under Minnesota’s aged, blind, or disabled pathways. The head qualified through no Medicaid category, received no SSI, and had MAGI equal to 4.85 times FPL, so the eligibility result is No." us,scenario_122,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary earnings, not Social Security benefits, IRA distributions, pensions, dividends, interest, or capital gains. Because the prompt lists no wages and requires unlisted numeric inputs to be zero, the employee payroll-tax base is zero and payroll tax is $0." us,scenario_122,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_122,state_income_tax_before_refundable_credits,39,llm_error,state_local_rule,False,,"The case turns on two Minnesota-specific subtractions taken by a non-itemizer: the full $27,200 of federally taxable Social Security (this filer's $70,646 AGI is below the phaseout threshold, so the subtraction is not reduced) and a $4,977.18 charitable-contribution subtraction for non-itemizers, equal to 50% of contributions above $500 after the 0.5%-of-AGI floor is applied to the $10,808 of gifts. Those come off against a $19,250 standard deduction (base plus the age-65 and blind additions), leaving $19,219 taxed entirely at the 5.35% bottom rate for $1,028.22. The wrong answers cluster into three groups: models that never removed (or only ""partially"" removed) the Social Security subtraction landed at $2,400–$3,200; models that took the Social Security subtraction but skipped the charitable subtraction landed at $1,270–$1,400; and a cluster asserted $0 by inventing senior/disabled subtractions and credits Minnesota does not have." -us,scenario_122,state_refundable_credits,6,llm_error,credit_phaseout,False,,"Minnesota's only refundable credit in play is the renter's credit (Minn. Stat. 290.0693, the former renter's property tax refund moved onto the M1), and it is a hard-capped, fully-phasing-out schedule keyed to ""household income"" as defined in 290A.03 subd. 3: federal AGI plus nontaxable Social Security and tax-exempt interest, with subtractions only for dependents — there is no age-65 or disability reduction of household income. Here AGI is $70,646 (85% of the $32,000 Social Security taxable, $1,519 of net capital losses allowed, $14,400 IRA, $25,000 pension, $5,552 dividends, $13 interest), and adding back $4,800 of nontaxable Social Security and $2,013 of tax-exempt interest gives household income of $77,459, which sits above the top of the 2026 renter's credit schedule, so the credit is zero. Every other Minnesota refundable credit is likewise zero: the working family credit requires earned income (none), and the child credit, K-12 education credit, and child/dependent care credit require qualifying children or care expenses (none). All five substantive answers assumed the renter's credit applied and then reported amounts near its low-income maximum, even though the schedule's maximum state share falls to a few hundred dollars in the top brackets before terminating entirely." -us,scenario_123,child1_chip_eligible,3,llm_error,categorical_eligibility,False,,"CHIP eligibility requires satisfying Pennsylvania’s own age and household-income criteria; being uninsured and ineligible for Medicaid does not independently establish eligibility. Child 1 fails Pennsylvania CHIP’s enrollment age cutoff for this program structure and the approximately $145,002 household income also exceeds the applicable CHIP threshold." +us,scenario_122,state_income_tax_before_refundable_credits,46,llm_error,state_local_rule,False,,"Minnesota's computation starts from federal AGI of $70,646.18. From that it subtracts the full $27,200 of federally taxable Social Security, because AGI is below the single-filer phaseout. It also takes the $4,977.18 nonitemizer charitable subtraction and the $19,300 standard deduction, which includes the age-65 and blind additions and beats itemizing. That leaves $19,169, all taxed at the 5.35% first bracket, for $1,025.54, and no nonrefundable credit applies. Three errors split the wrong answers. Models near $1,300 left out the charitable subtraction. Models near $2,900 to $3,200 skipped or only partly applied the Social Security subtraction and landed in the 6.8% bracket. The $0 answers made up senior or elderly/disabled subtractions or credits, but at this AGI the elderly/disabled subtraction phases out to zero. Models near $1,000 had the right structure but misestimated the deduction amounts, left out the charitable subtraction, added tax-exempt interest, or offset errors with a made-up personal exemption." +us,scenario_122,state_refundable_credits,7,llm_error,credit_phaseout,False,,"The only candidate Minnesota refundable credit is the renter's credit. There are no earnings or children, so the Working Family Credit and Child Tax Credit do not apply. The household's income is about $77,460: AGI plus nontaxable Social Security and tax-exempt interest. At that income the 2026 renter's credit schedule gives nothing, so mn_refundable_credits and state_refundable_credits are both $0. Every model that answered assumed the household falls inside a paying bracket of the renter's credit schedule and granted $1,000-$2,300 from 17% of the $28,235 rent. None of them noticed that the credit is fully phased out at this income." +us,scenario_123,child1_chip_eligible,4,llm_error,thresholds_rates,False,,"Child 1 is 16, so the CHIP age rule (under 19) is met. What decides the case is the income test. The child has to file a return on $45,000 of wages, so that income counts in household MAGI. With the parents' income, household MAGI is about $145,002, which is over 500% FPL for a family of three. That is far above Pennsylvania's CHIP income ceiling (314% FPL), and PolicyEngine's is_chip_eligible applies that ceiling and does not treat the unsubsidized full-cost CHIP buy-in as eligibility. Every model that answered 'Yes' skipped the income ceiling: either it assumed PA's full-cost CHIP makes any income eligible, or it looked only at the child's age." us,scenario_123,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_123,federal_income_tax_before_refundable_credits,38,llm_error,household_unit_or_filing_status,False,,"The trap is the tax-unit boundary combined with enacted 2026 (OBBBA) parameters. PolicyEngine builds tax-unit AGI from filer income only, so the 16-year-old dependent's $45,000 of wages stay out of the joint return entirely and no second return is added to this output: AGI is $100,000 wages + $2 interest = $100,002. From there 2026 supplies a $32,200 joint standard deduction plus the new §170(p) $2,000 non-itemizer cash charitable deduction (itemized deductions total only $21,127.05 — $15,599.99 of charity after the 0.5%-of-AGI floor, $2,457 of real estate tax, $3,070.06 of PA income tax — so the standard path wins), $24,800/$100,800 bracket thresholds giving $7,400.24, and a $2,200 CTC fully absorbed as a nonrefundable credit. Nearly every wrong answer inflated AGI to $145,002, summed the child's own return into the household total, assumed a TCJA sunset with personal exemptions and a $1,000 CTC, imputed mortgage interest from a balance when no interest amount is listed, or dropped the $2,000 charitable add-on." +us,scenario_123,federal_income_tax_before_refundable_credits,45,llm_error,household_unit_or_filing_status,False,,"The output is the liability of one tax unit, the joint return. Its AGI is $100,002 because the dependent child's $45,000 of wages is not the parents' income, and no separate child return is added. Under the 2026 OBBBA rules, the parents take the $32,200 standard deduction plus the $2,000 charitable deduction for non-itemizers. That leaves $65,802 of taxable income and $7,400.24 of tax, and the $2,200 CTC brings it to $5,200.24. The models went wrong in five main ways: counting the child's wages in joint AGI ($11,600.44 or higher), adding the child's own $3,220 tax ($8,420.24), using the expired pre-TCJA rules, leaving out the $2,000 charitable deduction, or giving the head an overtime deduction even though the head is not flagged as paid hourly." us,scenario_123,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The Child Tax Credit is a nonrefundable credit first; the additional child tax credit is only the residual the taxpayer cannot use against liability, further capped at $1,700 per child in 2026 and at 15% of earnings above $2,500. This household's $145,002 of income less the 2026 MFJ standard deduction leaves roughly $112,800 of taxable income and about $14,000 of federal income tax before refundable credits, which absorbs the entire CTC for the 16-year-old, and the one-child EITC is long gone at joint income above $57,000 — so total federal refundable credits are $0. Every wrong model stopped at the earned-income phase-in and the $400,000 MAGI phase-out and paid out a per-child refundable cap ($1,500, $1,600, $1,700, $2,000, or $2,400) as if the ACTC were a standalone entitlement. One model additionally fabricated a $205 EITC at an income level far past its phase-out." us,scenario_123,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. @@ -971,12 +980,12 @@ us,scenario_123,head_medicaid_eligible,2,llm_error,categorical_eligibility,False us,scenario_123,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,local_income_tax,2,llm_error,state_local_rule,False,,"Pennsylvania residence alone does not establish residence or employment in Philadelphia, and the benchmark's local-income-tax output applies the Philadelphia wage tax only when the relevant locality is specified. Because no Philadelphia locality was listed and unlisted facts must be treated as false, neither the head's nor the child's wages enter a Philadelphia wage-tax calculation, yielding $0." -us,scenario_123,payroll_tax,30,llm_error,state_local_rule,False,,"The case turns on a single state component: Pennsylvania withholds a mandatory employee unemployment compensation contribution of 0.07% from all gross covered wages, with no taxable wage base on the employee side. On this household's $145,000 of wages that is $101.50, which sits on top of $8,990 employee Social Security (6.2% of $145,000, entirely under the 2026 wage base) and $2,102.50 employee Medicare (1.45%, uncapped), for $11,194. The overwhelming majority of wrong answers land exactly on $11,092.50 — federal FICA only — because they asserted Pennsylvania imposes no employee-side payroll tax; the remaining misses capped the PA contribution at a $10,000 wage base, used the superseded 0.06% rate, fabricated Additional Medicare Tax on wages far below the $200,000/$250,000 thresholds, or submitted numbers their own arithmetic contradicted." +us,scenario_123,payroll_tax,35,llm_error,state_local_rule,False,,"Most models got federal FICA right: 6.2% Social Security on $145,000 is $8,990 and 1.45% Medicare is $2,102.50, for $11,092.50. The shared trap is Pennsylvania's mandatory employee unemployment compensation contribution. It is 0.07% of all gross wages with no wage cap, so it adds $101.50 ($70 + $31.50) and brings the total to $11,194. Most models said PA has no employee payroll tax or never raised it. The rest capped the PA contribution at $10,000 of wages, used the wrong rate, computed the right total and then submitted a different number, or made arithmetic errors or added a spurious Additional Medicare Tax." us,scenario_123,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires qualification through a specific categorical pathway and its applicable income test. The 39-year-old spouse has no qualifying category and MAGI of 5.31 times FPL, so the spouse fails every available pathway and is not eligible." us,scenario_123,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. -us,scenario_123,state_income_tax_before_refundable_credits,39,llm_error,household_unit_or_filing_status,False,,"The trap is filing-unit scope, not the rate: Pennsylvania's 3.07% flat tax has no standard deduction, exemption, or itemized deductions, so the only question is what income sits in the base. The 16-year-old's $45,000 of wages is a dependent's own income and never enters the parents' tax unit's PA taxable income, leaving $100,000 of wages plus $2 of interest — $100,002 × 0.0307 = $3,070.06. Thirty-two of the wrong models aggregated all three people into a $145,002 household base to get $4,451.56 (several after explicitly deriving the correct $3,070.06 and then adding the child's $1,381.50 back in). The remainder fabricated PA deductions, used federal taxable income, applied a 3% rate, or submitted values contradicting their own arithmetic." +us,scenario_123,state_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"The household is one tax unit: the head and spouse file jointly, and the 16-year-old is their dependent. Pennsylvania's joint return taxes only the spouses' own compensation and interest ($100,000 + $1 + $1 = $100,002) at the flat 3.07% rate, giving $3,070.06. The dependent child's $45,000 of wages is not part of this unit's income. Almost every model added the child's wages to the base ($145,002 × 3.07% = $4,451.56), either by pooling them or by adding a separate return for the child. The few other models invented PA deductions, used the wrong rate or base, or submitted numbers that contradict their own arithmetic." us,scenario_123,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_123,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv index f95b58b8..6fa3629e 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv @@ -45,7 +45,7 @@ us,scenario_118,state_refundable_credits,375.0,50,"PolicyEngine calculated state us,scenario_080,payroll_tax,0.0,3,"PolicyEngine calculated a payroll tax of $0.00 for this Pennsylvania single-adult household with approximately $940 in annual income. The calculation determined that the employee payroll tax component was $0.00, which is the primary driver of the overall spm_unit_payroll_tax value. With annual income of roughly $940, this household falls well below the threshold where Social Security and Medicare payroll taxes would typically apply, resulting in no payroll tax liability for the tax year 2026.", us,scenario_025,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this Ohio household is eligible for WIC (Women, Infants, and Children benefits) in 2026, resulting in a value of False for both household members. WIC eligibility requires applicants to be pregnant women, postpartum women, breastfeeding women, infants, or children under age 5, and this household consists of two adults with no children. Since the household composition does not include any individuals in the qualifying age or life-stage categories, PolicyEngine correctly identified that is_wic_eligible equals [False, False], regardless of the household's income level of approximately $94,925.", us,scenario_005,federal_income_tax_before_refundable_credits,106505.8984375,52,"PolicyEngine calculated the federal income tax before refundable credits as $106,505.90 for this California joint filer household with approximately $585,777 in income. The calculation began with gross income of $541,928.34, which included $388,350 in employment income (after $41,650 in pre-tax 401(k) contributions), $65,936 in taxable interest, $87,642.35 in dividend income, and a net capital loss of $10,910.59. After applying above-the-line deductions of $5,163.84 and taxable income deductions of $33,269.77, the household's taxable income was $503,494.75, which generated $89,095.92 in tax under the standard rate brackets. An additional $11,688 in capital gains tax was applied to the $77,920 in adjusted net capital gains (consisting of qualified dividends and long-term capital gains taxed at preferential rates), and a net investment income tax of $5,721.98 was assessed on the $150,578.34 in net investment income. These components—the ordinary income tax, capital gains tax, and net investment income tax—sum to the final value of $106,505.90.", -us,scenario_042,federal_income_tax_before_refundable_credits,1979.1583251953125,37,"PolicyEngine calculated federal income tax before refundable credits of $1,979.16 for this Wisconsin single filer by applying the standard tax rate schedule to a taxable income of $19,487.65. The taxable income was derived from an adjusted gross income of $43,637.65, which included $44,764.95 in IRS gross income (comprising $6,848 in dividends, $13,655.95 in taxable Social Security benefits, $19,200 in taxable retirement distributions, and $175 in taxable interest income) reduced by $1,127.29 in above-the-line deductions for farm operation losses. From this adjusted gross income, PolicyEngine applied the standard deduction of $18,150 plus an additional senior deduction of $6,000 (totaling $24,150 in deductions), and excluded $928 in qualified dividend income from the taxable income calculation. No additional taxes or credits affected the final amount, as the net investment income tax did not apply and no non-refundable credits were available to reduce the liability.", +us,scenario_042,federal_income_tax_before_refundable_credits,2361.96484375,37,"PolicyEngine computes federal income tax before refundable credits of $2,361.96 for this 79-year-old single filer in Wisconsin, and all of it comes from the main rate schedule. Gross income is $51,708: $19,200 in taxable retirement distributions, $16,846 of taxable Social Security, $6,848 in dividends ($928 qualified and $5,920 non-qualified), $4,886 of taxable private pension, $3,753 of capital gains and $175 of interest. A $1,127.29 farm operations loss is deducted above the line, which gives adjusted gross income of $50,580.71. The $18,150 standard deduction plus the $6,000 additional senior deduction comes to $24,150. That is more than the $12,486.20 available by itemizing, so taxable income is $26,430.71. The main rates apply to taxable income after $4,681 of net capital gain is excluded. Net investment income tax and nonrefundable credits are both zero.", us,scenario_118,local_income_tax,0.0,1,"For a single adult in New York with approximately $2,800 in household income in 2026, PolicyEngine calculated a local income tax liability of $0. This result reflects that the household's income falls below the threshold at which New York's local income tax obligations are triggered. New York's local income tax, which applies in certain jurisdictions within the state, has filing requirements and tax brackets that do not apply to households at this income level. Consequently, no local income tax is owed on this modest income amount.", us,scenario_005,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household in 2026. The household's refundable credits total is the sum of five component credits: the Earned Income Tax Credit (eitc), the refundable American Opportunity Credit (refundable_american_opportunity_credit), the refundable Child Tax Credit (refundable_ctc), the recovery rebate credit (recovery_rebate_credit), and the refundable payroll tax credit (refundable_payroll_tax_credit). Each of these components evaluated to $0 for this household. Given the household's high income of approximately $585,777 with no children and no qualifying education expenses or other circumstances that would trigger these credits, none of the refundable credit provisions applied, resulting in a total refundable credits amount of $0.", us,scenario_100,payroll_tax,452.47613525390625,5,"PolicyEngine calculated the payroll tax for this Montana head-of-household with two children at $452.48 for tax year 2026. This amount comprises two components: employee Social Security tax of $366.71 and employee Medicare tax of $85.76, both assessed on the primary earner's wages. The Social Security tax was computed at the standard 6.2% rate on eligible earnings, while the Medicare tax was applied at 1.45% on the same wage base. Since the household's annual income of approximately $5,915 falls below typical payroll tax exemption thresholds, the relatively modest payroll tax obligation reflects the limited earnings subject to these employment taxes.", @@ -77,7 +77,7 @@ us,scenario_118,head_wic_eligible,0.0,28,"PolicyEngine determined that this 74-y us,scenario_005,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this California household with ~$585,777 in income. The computation aggregates refundable tax credits across all 50 states and select localities, with ca_refundable_credits—the California-specific component—evaluating to $0 since the household resides in California. All other state refundable credit variables (al_refundable_credits, ar_refundable_credits, az_refundable_credits, and so on through wv_refundable_credits) also returned $0, as the household is not subject to those states' refundable credit programs. The final state_refundable_credits value of $0 reflects the sum of these individual state components, indicating that no refundable state tax credits applied to this benchmark household in tax year 2026.", us,scenario_042,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Wisconsin resident with ~$58,434 household income in 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with each state-specific variable (al_refundable_credits, ar_refundable_credits, and so on through wi_refundable_credits) evaluated individually. For this household, wi_refundable_credits evaluated to $0, as did all other state-level refundable credit variables, resulting in a total state_refundable_credits value of $0. This outcome reflects that either Wisconsin offers no refundable state tax credits applicable to this household's circumstances, or the household's income and filing status do not qualify for any available refundable credits in the 2026 tax year.", us,scenario_060,head_medicare_eligible,1.0,1,"PolicyEngine determined that the household head is Medicare eligible (value = 1.0, or True) for the 2026 tax year. This determination is based on the household composition and demographic characteristics: the household consists of a single adult in Texas with no children and an annual income of approximately $62,775. Under Medicare eligibility rules, individuals who have reached age 65 or who qualify through disability or end-stage renal disease status are eligible for the program. PolicyEngine's computation indicates that this household head meets one of these eligibility criteria, resulting in the Medicare eligibility flag being set to True for purposes of calculating tax and benefit obligations in 2026.", -us,scenario_080,snap,3596.039794921875,373,"PolicyEngine calculated an annual SNAP benefit of $3,596.04 for this Pennsylvania household by determining a monthly allotment of $298-$304.68 and annualizing it. The household qualifies for SNAP because it meets all eligibility requirements: the gross income of $301.33 is only 23% of the federal poverty guideline ($1,304-$1,330), it passes the net income test, and it qualifies categorically through TANF non-cash eligibility with assets of $19,827.65 (including $3,570 in vehicle value) below Pennsylvania's limits. The household also satisfies work requirements and immigration status eligibility. The monthly normal allotment of $298-$304.68 represents the maximum benefit for a single-person household in the contiguous United States, and since the household's income is well below the threshold, no reduction is applied, resulting in the full allotment being provided.", +us,scenario_080,snap,3596.039794921875,373,"PolicyEngine calculated an annual SNAP benefit of $3,596.04 for this Pennsylvania household by computing monthly allotments that vary seasonally. The household's monthly SNAP gross income of $301.33 (comprising $300 in financial assistance and $1.33 in dividends) falls well below the federal poverty guideline of $1,304.17 for January–September and $1,330 for October–December, satisfying the gross income test. Although the household's countable assets of $19,827.65 exceed the standard SNAP asset limit, it qualifies categorically through Pennsylvania's TANF non-cash program, which it meets on income and asset grounds. The engine applies Pennsylvania's standard utility allowance of $857 monthly through the heat-and-eat practice, which when combined with the standard deduction ($209 for January–September, $213.68 for October–December) reduces the household's net income to $0 in every month, yielding the maximum monthly allotment for a single-person household: $298 for the nine months of January–September and $304.68 for the three months of October–December, totaling $3,596.04 annually.", us,scenario_005,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this California household in 2026. California does not impose a local income tax at the state level, and while some California municipalities may have local income taxes, PolicyEngine's standard calculation for state-level tax purposes does not include local income tax obligations. Therefore, despite the household's substantial income of approximately $585,777, the local_income_tax variable correctly reflects that no local income tax liability exists under the standard California tax framework for 2026.", us,scenario_042,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Wisconsin household in 2026. Wisconsin does not impose a local income tax at the state level, and PolicyEngine's modeling reflects this absence of a local income tax obligation. For a single adult in Wisconsin with a household income of approximately $58,434, no local income tax liability accrues under current tax law. This result aligns with Wisconsin's tax structure, which relies on state income tax rather than local income taxation.", us,scenario_025,head_medicare_eligible,0.0,1,"PolicyEngine determined that neither adult in this Ohio household is Medicare eligible in 2026, resulting in a value of [False, False]. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and this household's adults do not meet that threshold. The household's joint income of approximately $94,925 and filing status as a married couple filing jointly are relevant to other tax and benefit calculations, but do not affect the age-based Medicare eligibility determination. Since neither household member has reached the qualifying age of 65, PolicyEngine correctly assigned a value of 0.0 (False) for the is_medicare_eligible variable for both adults.", @@ -196,7 +196,7 @@ us,scenario_062,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_100,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this Montana household are eligible for Early Head Start, resulting in a value of [False, False, False]. Early Head Start eligibility in PolicyEngine is typically determined by factors such as the child's age (generally limited to infants and toddlers under age 3) and household income relative to federal poverty thresholds. With a household income of approximately $5,915 and a head-of-household filing status with one adult and two children, the household's circumstances did not satisfy the program's eligibility criteria as modeled for the 2026 tax year. The resulting array indicates that neither child qualified for the program.", us,scenario_007,state_income_tax_before_refundable_credits,755.7783203125,60,"For this single Idaho resident with approximately $54,565 in household income, PolicyEngine calculated $755.78 in state income tax before refundable credits. The calculation began with an adjusted gross income of $49,064.12, from which $13,784.12 in taxable Social Security benefits were subtracted as an Idaho-specific deduction, resulting in an Idaho AGI of $35,280. After applying the standard deduction of $16,100, the household's Idaho taxable income was reduced to $19,180. Idaho's tax brackets and rates were then applied to this taxable income amount to arrive at the final tax liability of $755.78, with all other states contributing zero tax liability.", us,scenario_043,ssi,0.0,25,"For the 66-year-old head of household in Colorado in 2026, PolicyEngine calculated an SSI benefit of $0.00. Although the individual meets the categorical requirement for SSI as an aged, blind, or disabled person (is_ssi_aged_blind_disabled=True), the household income of approximately $4,642 exceeds the SSI income limit, resulting in ineligibility. PolicyEngine determined that the household would take up SSI if eligible (takes_up_ssi_if_eligible=True), but since the income threshold was exceeded, no benefit amount was payable. SSI is evaluated on an individual basis, and in this case the person's countable income eliminated any SSI entitlement despite meeting the non-financial eligibility criteria.", -us,scenario_043,snap,3596.039794921875,460,"PolicyEngine calculated a SNAP benefit of $3,596.04 for this Colorado household by determining that the individual qualifies for a monthly allotment of $298–$304.68 (varying across the tax year, likely due to inflation adjustments to the federal poverty guideline which increased from $1,304.17 to $1,330) and assuming full take-up of benefits across all 12 months. The household met all SNAP eligibility requirements: with gross income of $387.35 and earned income of $340.68, the household's gross income-to-poverty ratio of approximately 0.29–0.30 fell well below the 130% threshold; net income also satisfied program limits; liquid assets of $2,800 were within allowable limits; and the individual qualified for categorical eligibility through TANF non-cash assistance as an elderly or disabled household member. The household also satisfied work requirements and immigration status criteria. The monthly allotment of $298–$304.68 represents the maximum benefit for a single-person household in the contiguous United States, resulting in an annual benefit of approximately $3,596 when multiplied across the tax year.", +us,scenario_043,snap,3576.0,460,"PolicyEngine computes 2026 SNAP for this one-person Colorado household as $3,576 for the year, the sum of twelve monthly benefits of $298 each. The household is eligible in every month. Its gross income is $387.35 a month, of which $340.68 is earned, against a SNAP poverty guideline of $1,304.17 from January through September (a ratio of about 30%) and $1,330 from October through December (about 29%). It passes the gross and net income tests and, with its 66-year-old member counted as elderly or disabled, the asset test on $2,800 of bank assets; it is also categorically eligible through Colorado's TANF non-cash eligibility test. Each month its normal allotment equals the one-person maximum allotment for the contiguous states, $298.", us,scenario_026,child3_wic_eligible,0.0,1,"PolicyEngine determined that the third child in this North Carolina household is not eligible for WIC benefits, resulting in a value of False. WIC eligibility is primarily determined by income thresholds, which are set at 185% of the federal poverty line. For a household of five persons in 2026, this income limit would be approximately $47,000 annually. Since this household's income is approximately $85,209, it exceeds the WIC income threshold by a substantial margin, placing the household above the maximum allowable income for program participation. Consequently, PolicyEngine marked all household members, including the third child, as ineligible for WIC benefits.", us,scenario_081,head_wic_eligible,0.0,1,"PolicyEngine determined that the household head is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False for is_wic_eligible. WIC eligibility in Massachusetts requires that household members meet specific criteria, primarily including the presence of pregnant women, postpartum women, breastfeeding women, infants, or children under age 5. This benchmark household consists of a single adult with no children and a household income of approximately $174,088, which means it lacks the required family composition for WIC eligibility. Since the household does not contain any of the qualifying individuals (pregnant women, postpartum women, breastfeeding women, infants, or children under 5), PolicyEngine correctly classified the head as ineligible for WIC benefits.", us,scenario_100,free_school_meals_eligible,1.0,279,"This Montana household qualifies for free school meals with an annual benefit value of $2,261.92. The household meets eligibility through categorical qualification via SNAP receipt of $8,625.89 annually, which automatically confers free meal status regardless of income level. Additionally, the household's countable income of $11,979.68 (comprising $5,914.72 in employment income and $6,064.96 in TANF benefits) represents 44% of the federal poverty guideline, well below the 130% income threshold for free meals. The two school-age children in the household receive a daily subsidy of $7.15 per child, generating the total annual benefit of $2,261.92 based on standard school year calculations for the contiguous United States.", @@ -266,7 +266,7 @@ us,scenario_007,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_101,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Texas household in 2026. The household, consisting of a single adult with no children and an annual income of approximately $101,830, does not qualify for TANF benefits because the program is designed to provide cash assistance to families with dependent children. Since this household has zero children, it falls outside the eligibility criteria for TANF, resulting in no benefit amount regardless of income level.", us,scenario_044,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Kansas household in 2026 because neither spouse reported self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this joint household's approximately $56,806 in total income derives from wages or other non-self-employment sources, there is no self-employment tax liability. The calculation correctly reflects that self-employment tax applies exclusively to individuals with Schedule C net profit or other qualifying self-employment earnings, which this household does not have.", us,scenario_119,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this Virginia household qualifies for Early Head Start in 2026, resulting in an eligibility array of [False, False, False]. Early Head Start is a federal program specifically designed for children under age 3, and this household's children do not meet the age requirements for the program. With a household income of approximately $55,800 and head-of-household filing status, the family's income level would not have been a disqualifying factor, as Early Head Start serves low-income families and this income falls within typical eligibility thresholds. The determination reflects that the children in this benchmark household simply fall outside the target age range for this particular early childhood intervention program.", -us,scenario_082,state_refundable_credits,650.5,77,"For this New York head-of-household filer with one child and approximately $90,132 household income, PolicyEngine calculated total state refundable credits of $650.50, which consists entirely of New York state credits since all other states yielded zero credits. The New York refundable credits total comprises two components: the New York Child Tax Credit (ny_ctc) of $290.50 and the New York Child and Dependent Care Credit (ny_cdcc) of $360.00. The ny_ctc calculation started with a base amount of $1,000 for the one qualifying child but was reduced by a phase-out of $709.50 based on the household's adjusted gross income of $117,585.15, resulting in the $290.50 credit. The ny_cdcc was calculated by taking the maximum credit of $3,000, multiplying it by the applicable rate of 0.6 (derived from the household's AGI of $117,585.15), and then applying the federal CDCC rate of 0.2, yielding the $360.00 credit based on $3,000 in qualifying dependent care expenses.", +us,scenario_082,state_refundable_credits,667.0,102,"PolicyEngine calculated New York refundable credits of $667 for this head of household with one child aged 1. The Empire State child credit is $307: the $1,000 credit for a child aged three or younger, reduced by $16.50 for each whole $1,000 of federal adjusted gross income ($117,652.65) above the $75,000 threshold, 42 steps or $693. New York's child and dependent care credit is $360: 60% of the federal credit, which is 20% of the $3,000 of care expenses the federal credit counts for one child.", us,scenario_082,state_income_tax_before_refundable_credits,5598.55224609375,77,"For this New York head-of-household filer with approximately $90,132 in household income, PolicyEngine calculated a state income tax before refundable credits of $5,598.55. The calculation began with an adjusted gross income (AGI) of $117,585.15, from which New York deductions of $11,200 and exemptions of $1,000 were subtracted to arrive at a taxable income of $105,385.15. The main New York income tax on this taxable income was $5,442.17, to which a supplemental tax of $156.38 was added, yielding the final total of $5,598.55. The deductions included both itemized deductions of $11,169.24 (comprising $11,057.16 in SALT deduction and $112.07 in charitable deduction) and the standard deduction of $11,200, with the standard deduction being applied as the larger amount.", us,scenario_026,spouse_medicare_eligible,0.0,1,"PolicyEngine determined that the spouse is not Medicare eligible in this 2026 household, resulting in a value of False (0.0). Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and the spouse in this North Carolina household does not meet this age threshold. The household composition of two adults and three children with a combined income of approximately $85,209 does not trigger any alternative pathways to Medicare eligibility, such as end-stage renal disease or ALS status. Therefore, PolicyEngine correctly assigned False to the spouse_medicare_eligible variable, indicating no Medicare coverage eligibility for the spouse in tax year 2026.", us,scenario_026,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible in 2026, returning a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, requiring individuals to be at least 65 years old, along with citizenship or permanent residency status and sufficient work history under Social Security. Based on the household composition provided—a joint filer household with 2 adults and 3 children in North Carolina with a combined income of approximately $85,209—PolicyEngine's calculation indicates that the head of household does not meet the age threshold for Medicare eligibility. The array output [False, False, False, False, False] reflects the eligibility status for all five household members, confirming that none currently qualify for Medicare benefits.", @@ -301,7 +301,7 @@ us,scenario_044,ssi,0.0,25,"PolicyEngine calculated Supplemental Security Income us,scenario_120,payroll_tax,13496.1552734375,11,"PolicyEngine calculated the spm_unit_payroll_tax for this Connecticut resident at $13,496.16 by summing three components of employee payroll taxes. The employee_social_security_tax contributed $10,267.01, calculated on taxable_earnings_for_social_security of $165,597 at the standard 6.2% rate. The employee_medicare_tax added $2,401.16, applied to the full earnings at the standard 1.45% rate. Additionally, Connecticut's employee_state_payroll_tax of $827.98 was included, which consists entirely of the ct_employee_paid_leave_contribution of $827.98, a state-mandated contribution on covered earnings. These three components—federal Social Security, federal Medicare, and Connecticut's paid leave contribution—sum to the total payroll tax liability of $13,496.16.", us,scenario_082,head_wic_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for the Women, Infants, and Children (WIC) program, resulting in a value of False (0.0). WIC eligibility in New York requires that household income fall below 185% of the federal poverty line, and with a household income of approximately $90,132, this benchmark household significantly exceeds the income threshold for a household of two people. Additionally, WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age five; as the head of household in this scenario does not fall into one of these categorical groups, they do not qualify regardless of income. The combination of excess income and ineligible household composition resulted in PolicyEngine computing is_wic_eligible as False for both household members.", us,scenario_044,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Kansas household in 2026. TANF is a needs-based program primarily designed for families with children, and this benchmark household consists of two adults with no children. Since the household does not meet the basic eligibility requirement of having dependent children, PolicyEngine determined no TANF benefit applies regardless of the household's income level of approximately $56,806. The $0 result reflects the program's structural targeting rather than any income-based phase-out or disqualification.", -us,scenario_008,state_refundable_credits,5342.39990234375,69,"For this New Jersey household with joint filing status and six children earning approximately $30,915 annually, PolicyEngine calculated total state refundable credits of $5,342.40, which consists entirely of New Jersey refundable credits since all other states yielded zero. The New Jersey total comprises three components: the New Jersey Earned Income Tax Credit (EITC) of $3,292.40 (based on a phased-in federal EITC of $8,231), the New Jersey Child Tax Credit (CTC) of $2,000 (applied to the household's four dependent children under the age threshold), and the New Jersey Property Tax Credit of $50 (for which the household qualified as renters with at least one spouse age 42 or older). The household's New Jersey taxable income of $18,915 and eligibility status across all three credit programs determined the final refundable credit amount.", +us,scenario_008,state_refundable_credits,5842.39990234375,127,"PolicyEngine calculated New Jersey refundable credits of $5,842.40 for this married couple filing jointly with six children. The New Jersey earned income credit is $3,292.40: 40% of the federal earned income credit, which reaches its $8,231 maximum for a family with three or more children. The New Jersey child tax credit is $2,500 under the schedule New Jersey enacted on June 30, 2026 for tax years 2026 to 2028, which pays more per young child at low taxable income. The New Jersey property tax credit adds $50.", us,scenario_101,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this Texas household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_countable_income of $106,007.53—derived from employment income of $103,000, pension income of $2,307.53, and veterans benefits of $700—yields a school_meal_fpg_ratio of 6.64, meaning the household income is 664% of the federal poverty guideline (spm_unit_fpg of $15,960). Since Texas does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the income threshold for free meals applies: households must have income at or below 130% of the federal poverty guideline. With income far exceeding this 130% threshold, the household is classified in school_meal_tier 2 (paid meals) and receives no subsidy, resulting in a school_meal_net_subsidy of 0.", us,scenario_026,child2_head_start_eligible,0.0,1,"PolicyEngine determined that child 2 is not eligible for Head Start (is_head_start_eligible = False) based on the household's income level relative to the program's eligibility thresholds. Head Start eligibility in the United States is primarily determined by whether household income falls at or below 100% of the federal poverty line, or in some cases up to 130% of the poverty line depending on program rules and state variations. With a household income of approximately $85,209 and three children in North Carolina, this household's income substantially exceeds the federal poverty threshold for a family of five (which was approximately $28,749 in 2026), placing them well above the income cutoff for Head Start participation. Consequently, PolicyEngine marked all three children in the household as ineligible for the program, resulting in a value of False for child 2.", us,scenario_101,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Texas household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $101,830 translates to a school_meal_fpg_ratio of 6.64, meaning household income is 664% of the federal poverty guideline—far exceeding the 185% income threshold required for reduced-price meal eligibility. Since Texas does not offer universal free school meals and the household meets neither the income requirement nor categorical eligibility criteria (such as SNAP or TANF participation), the household is assigned to the PAID school_meal_tier. Consequently, the school_meal_net_subsidy is 0, and the household pays the full price for school meals.", @@ -314,7 +314,7 @@ us,scenario_102,federal_income_tax_before_refundable_credits,6104.09619140625,40 us,scenario_120,state_income_tax_before_refundable_credits,11917.18359375,75,"PolicyEngine calculated Connecticut state income tax before refundable credits of $11,917.18 for this single filer with ~$244,169 household income. The calculation began with an adjusted gross income (AGI) of $232,635.12, from which Connecticut-specific AGI subtractions of $25,140 were applied—primarily a Social Security benefit adjustment of $25,140 based on $41,900 in Social Security benefits and combined income calculations. This produced a Connecticut AGI of $207,495.12, which was then reduced to a taxable income of $207,495.12 after accounting for phase-out add-backs of $250 and recapture adjustments totaling $430 (comprising $250 in low-tax recapture and $180 in middle-tax recapture). The resulting tax before any personal credits and alternative minimum tax was $11,917.18, which remained unchanged after applying the AMT and personal credit calculations, as no credits reduced the liability.", us,scenario_026,child1_early_head_start_eligible,0.0,1,"PolicyEngine determined that none of the three children in this North Carolina household are eligible for Early Head Start in 2026, resulting in a value of False for all household members. Early Head Start eligibility is typically restricted to children under age 3, and this household's children do not meet the age requirements for the program. With a household income of approximately $85,209, the family's income level would not present a barrier to eligibility if age requirements were met, as Early Head Start serves low-income families and this income falls within the range of many state programs. The False determination across all five household members reflects that either the children exceed the age threshold or the household composition does not include infants and toddlers in the target age range for this early childhood education program.", us,scenario_044,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of 0.0 (False). The WIC program is specifically designed to serve pregnant women, postpartum women, breastfeeding women, infants, and children under age 5 who meet income and nutritional risk criteria. Since this benchmark household consists of 2 adults with no children and a household income of approximately $56,806, the spouse does not fall into any of the eligible demographic categories for WIC participation. Therefore, PolicyEngine correctly identified ineligibility based on the household composition and age requirements of the program.", -us,scenario_008,snap,15246.9052734375,541,"PolicyEngine calculated an annual SNAP benefit of $15,246.91 for this New Jersey household of 8 people with ~$30,915 in household income by determining that each of the 8 household members qualifies for a monthly allotment of approximately $1,260–$1,302, which sums to the annual total. The household met all SNAP eligibility requirements: it passed the gross income test (57% of the federal poverty guideline at $2,576.25 in gross income versus the $4,512.50 threshold), the net income test (39% of FPG at $1,763.92 in net income after deducting $812.33 in allowable deductions), and categorical eligibility through TANF non-cash assistance. The monthly allotment was calculated as the maximum allotment for a household of 8 in the contiguous U.S. ($1,789–$1,829) minus the household's expected contribution of approximately $528, yielding the normal allotment of $1,260–$1,302 per month. All 8 household members met work requirements and immigration eligibility criteria, and the household was assumed to take up the benefit if eligible.", +us,scenario_008,snap,15108.0,541,"PolicyEngine computes an annual 2026 SNAP benefit of $15,108 for this eight-person New Jersey household, made up of $1,259 in each of the twelve months. Monthly gross income is $2,576.25: $2,566.67 in earned income (the spouse's $2,233.33 in wages plus the head's self-employment income) and $9.58 in interest. Subtracting a $299 standard deduction and a $513.33 earned income deduction, $812.33 in total, leaves $1,763.92, which rounds to net income of $1,764. The household passes the gross and net income tests and is categorically eligible through TANF non-cash eligibility in New Jersey; gross income is 57% of the poverty guideline from January through September and 55% from October through December. Net income sets an expected contribution of $530, and the $1,789 maximum allotment for eight people minus $530 gives $1,259 a month.", us,scenario_102,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household of two adults with no children and approximately $87,201 in household income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. For a childless joint filer household at this income level, neither the EITC nor the refundable CTC would apply, and absent qualifying education expenses or other specific circumstances, the other refundable credits also did not generate any benefit.", us,scenario_120,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Connecticut resident with approximately $244,169 in household income. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with ct_refundable_credits specifically evaluated for this household's Connecticut residence. Since the household has no children and a high income level, it does not qualify for Connecticut's refundable credits, which are primarily targeted toward lower-income families. All other state-level refundable credit variables (al_refundable_credits, ar_refundable_credits, and so forth through wv_refundable_credits) also evaluated to $0, as the household's state of residence is Connecticut and it does not qualify for any other states' credits. The sum of these zero values across all jurisdictions yields the final state_refundable_credits amount of $0.", us,scenario_008,ssi,0.0,25,"PolicyEngine calculated total household SSI of $0.00 for this New Jersey household in 2026. Although the spouse meets the SSI eligibility criterion of being aged, blind, or disabled (is_ssi_aged_blind_disabled=True), they receive $0.00 in benefits. The head of household and all six children do not meet SSI eligibility requirements, as none are classified as aged, blind, or disabled. Since SSI is an individual benefit computed separately for each household member with no pooling, and only the spouse qualifies but receives no payment, the household's total SSI benefit remains $0.00. The computation trace confirms that all eight household members have zero SSI amounts, consistent with the reference value.", @@ -336,7 +336,7 @@ us,scenario_008,head_wic_eligible,0.0,1,"PolicyEngine determined that only one m us,scenario_102,state_income_tax_before_refundable_credits,2461.869873046875,59,"PolicyEngine calculated the state income tax before refundable credits for this North Carolina household at $2,461.87. The calculation began with the household's adjusted gross income of $87,201, from which PolicyEngine applied North Carolina's standard deduction of $25,500 (applicable to joint filers), resulting in a taxable income of $61,701. This taxable income was then subject to North Carolina's tax rate schedule to arrive at the final tax liability of $2,461.87 before any refundable credits. All other states returned zero tax liability, confirming that only North Carolina income tax applied to this household.", us,scenario_082,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible in 2026, resulting in a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, requiring individuals to be at least 65 years old, along with meeting citizenship and residency requirements. Based on the household composition and income profile provided for this benchmark case, the head of household does not meet the age threshold for Medicare eligibility. Consequently, PolicyEngine correctly classified the head of household as ineligible for Medicare benefits in the 2026 tax year.", us,scenario_120,ssi,0.0,25,"PolicyEngine calculated an SSI benefit of $0.00 for this 76-year-old household head in Connecticut for tax year 2026. Although the individual meets SSI categorical eligibility as aged (is_ssi_aged_blind_disabled=True), the household's income of approximately $244,169 far exceeds the SSI income limits, resulting in complete benefit phase-out. The computation confirms that takes_up_ssi_if_eligible=True, indicating the household would claim the benefit if eligible, but the substantial countable income eliminates any payable amount. SSI is evaluated on an individual basis, and this person's income alone disqualifies them from receiving any benefit payment.", -us,scenario_064,dependent2_chip_eligible,0.0,1,"PolicyEngine determined that dependent2 is not eligible for CHIP (Children's Health Insurance Program) in Wisconsin for tax year 2026. The person is 18 years old, which exceeds CHIP's age eligibility threshold that typically covers children under 19 in most states. Additionally, PolicyEngine's computation confirmed that dependent2 is not Medicaid-eligible (categorized as NONE), meaning the household does not qualify for Medicaid coverage either. Since the person fails both CHIP's age requirement and lacks Medicaid eligibility, they do not meet the criteria for either program, resulting in an is_chip_eligible value of False.", +us,scenario_064,dependent2_chip_eligible,0.0,1,"PolicyEngine determined that Dependent 2 is not eligible for CHIP in Wisconsin for 2026. At 18, Dependent 2 is within CHIP's age limit, which covers children under 19. Two other conditions fail. The household's income is 321.5% of the federal poverty guideline, above Wisconsin's CHIP limit of 306%. And Dependent 2 has employer-sponsored insurance, which PolicyEngine counts as coverage that rules a child out of CHIP. PolicyEngine also finds Dependent 2 not eligible for Medicaid, so the value is False.", us,scenario_026,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this North Carolina household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $85,209 places them at 2.20 times the federal poverty guideline, which exceeds the 185% income threshold required for reduced-price meal eligibility. Since the state does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they do not meet any pathway to reduced-price assistance. Consequently, the household's school_meal_tier is classified as PAID, and the school_meal_net_subsidy is 0, meaning the household must pay the full price for school meals.", us,scenario_044,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (Children's Health Insurance Program) in Kansas for the 2026 tax year, resulting in a value of False. The spouse's age of 61 years old places them outside CHIP's eligibility criteria, which is designed for children and young adults, not seniors. Additionally, PolicyEngine confirmed the spouse is not Medicaid-eligible (engine category: NONE), meaning they do not qualify through the alternative pathway that sometimes provides CHIP access. With both the age requirement and Medicaid eligibility failing, the spouse does not meet any of CHIP's qualifying conditions, yielding the final value of 0.0 (False).", us,scenario_082,child1_medicare_eligible,0.0,1,"PolicyEngine determined that child1_medicare_eligible equals False (0.0) for this benchmark household. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old to qualify for the standard program. The child in this household is under 65 years old, which is the binding constraint that makes them ineligible for Medicare. Additionally, there are no special circumstances in this household—such as end-stage renal disease or ALS—that would grant Medicare eligibility to someone under the age threshold. Therefore, PolicyEngine correctly computed is_medicare_eligible as False for the child.", @@ -392,9 +392,9 @@ us,scenario_064,child1_early_head_start_eligible,0.0,5,"PolicyEngine determined us,scenario_045,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Michigan household because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's income of approximately $36,276 is derived from wage and salary sources rather than self-employment, there is no self-employment income base upon which to calculate the 15.3% self-employment tax rate (which covers Social Security and Medicare taxes for self-employed individuals). Therefore, PolicyEngine correctly determined that no self-employment tax liability applies to this household.", us,scenario_121,federal_income_tax_before_refundable_credits,0.0,8,"For this single adult in South Carolina with household income of approximately $39,909, PolicyEngine calculated federal income tax before refundable credits as $0. The computation shows that income_tax_before_credits, the primary component, equals $0, indicating that after applying the standard deduction and tax brackets, this household's taxable income falls below the threshold requiring federal income tax liability. The net_investment_income_tax also contributes $0, as the adjusted_gross_income of $25,665 does not trigger the 3.8% net investment income surtax. Additionally, income_tax_capped_non_refundable_credits equals $0, meaning there are no non-refundable credits available to reduce any tax liability. The other components—recapture of investment credit, unreported payroll tax, and qualified retirement penalty—are all $0, confirming no additional tax adjustments apply to this household.", us,scenario_083,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Texas household in 2026. The household's total income of approximately $11,010 falls below the self-employment income threshold required to trigger self-employment tax obligations. Self-employment tax is only assessed on net earnings from self-employment that exceed $400 in a given tax year, and this household's income composition and level did not meet that threshold. As a result, no self-employment tax liability was generated for this benchmark household.", -us,scenario_027,snap,287.68316650390625,445,"PolicyEngine calculated a monthly SNAP benefit of $287.68 for this Connecticut household of two adults by aggregating multiple monthly calculations across the tax year. The household qualifies for SNAP because it meets the gross income test (with a ratio of 1.44 times the federal poverty guideline of $1,762.50), has at least one elderly or disabled member, and satisfies categorical eligibility through TANF non-cash assistance, work requirements, and immigration status checks. The benefit amount is determined by the SNAP formula: the maximum allotment for a two-person household in the contiguous US ($546 in most months, $558.24 in later months) minus the expected contribution based on net income ($635.10 in early months, $633.60 in later months), which results in the minimum allotment of $23.84 or $24.37 per month since the formula produces a negative value. The annual total of $287.68 reflects the sum of these monthly minimum allotments across the year, as the household's net income of approximately $2,113–$2,118 per month exceeds what would generate a positive benefit under the standard SNAP calculation.", +us,scenario_027,snap,288.0,445,"PolicyEngine computes an annual 2026 SNAP total of $288 for this Connecticut couple, the sum of twelve monthly benefits of $24. The household is eligible in every month: it has an elderly or disabled member, its monthly gross income of $2,536.25 passes the gross income test, and it is categorically eligible through Connecticut's TANF non-cash rules, with gross income at 1.44 times the $1,762.50 monthly poverty guideline from January through September and 1.41 times the $1,803.33 guideline from October through December, net income at 1.2 and 1.17 times, and $48,000 in assets passing the asset test. Monthly net income of $2,118 sets an expected contribution of $636, which exceeds the $546 maximum allotment for a two-person household, so the $24 minimum allotment applies each month.", us,scenario_121,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this South Carolina single adult with approximately $39,909 in household income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits amount of $0. This outcome reflects that the household does not qualify for any of these refundable credits—likely because the household has no qualifying children for the CTC, does not meet EITC income or work requirements, has no qualifying education expenses for the American Opportunity Credit, and does not qualify for the other refundable credit programs.", -us,scenario_045,state_income_tax_before_refundable_credits,1288.8375244140625,57,"For this Michigan resident with approximately $36,276 in household income, PolicyEngine calculated a state income tax liability of $1,288.84 before refundable credits. The calculation began with an adjusted gross income of $36,275.59, from which Michigan's personal exemption of $5,950 was subtracted to arrive at a taxable income of $30,325.59. This taxable income was then subject to Michigan's tax rate schedule to produce the final tax before refundable credits of $1,288.84. All other states' income tax variables returned zero, confirming that only Michigan's tax applied to this single-adult household.", +us,scenario_045,state_income_tax_before_refundable_credits,1290.9625244140625,62,"PolicyEngine calculated Michigan state income tax before refundable credits of $1,290.96 for this single-adult household with $36,275.59 in employment income. The computation began with adjusted gross income of $36,275.59, from which PolicyEngine subtracted Michigan's personal exemption of $5,900 (the 2026 amount published by Michigan Treasury in February 2026) to arrive at taxable income of $30,375.59. This taxable income was then subject to Michigan's tax rate schedule to produce the final tax liability of $1,290.96 before any refundable credits. All other states' income tax calculations returned zero, as the household resides in Michigan.", us,scenario_064,free_school_meals_eligible,0.0,14,"PolicyEngine determined that this Wisconsin household is not eligible for free school meals, resulting in a value of 0.0. The household's countable income of $117,295—comprising $97,295 in employment income and $20,000 in retirement distributions—was compared against the federal poverty guideline for their household size (SPM unit FPG of $38,680), yielding a school_meal_fpg_ratio of 3.03. This ratio places the household in school_meal_tier 2, meaning their income exceeds the 185% reduced-price eligibility threshold (which would be approximately $71,558). Since Wisconsin does not have universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they are required to pay the full price for school meals. Although PolicyEngine calculated a daily subsidy amount of $0.87 per meal, this subsidy does not apply to households in tier 2, resulting in zero free school meal eligibility.", us,scenario_027,ssi,0.0,25,"PolicyEngine calculated Supplemental Security Income (SSI) of $0.00 for this household in 2026. Although both the head (age 68) and spouse (age 39) meet the categorical requirement for SSI eligibility—both are marked as aged, blind, or disabled—the household receives no SSI benefit. Both individuals are coded as willing to take up SSI if eligible (takes_up_ssi_if_eligible = True for both), indicating that program participation is not the limiting factor. The zero benefit reflects that when income and asset limits are applied on an individual basis with spousal deeming rules, neither person qualifies for a positive SSI payment given the household's total income of approximately $30,435. SSI is computed separately for each person, and in this case, the income counting rules result in both the head and spouse having countable income that exceeds their respective federal benefit rates, yielding no payment to either individual.", us,scenario_102,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in North Carolina for tax year 2026. The head is 26 years old, which exceeds CHIP's age eligibility requirement that typically limits coverage to children and young adults under a specified age threshold. Additionally, with a household income of approximately $87,201, the head's income level places them above CHIP's income eligibility limits, which are generally set at a percentage of the federal poverty level designed to serve lower-income families. Since the head fails both CHIP's age criteria and income criteria, and is also not Medicaid-eligible, PolicyEngine correctly returned is_chip_eligible as False.", @@ -420,7 +420,7 @@ us,scenario_066,payroll_tax,39.779998779296875,5,"For this single adult in Virgi us,scenario_121,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this South Carolina resident with ~$39,909 in household income. The computation aggregates refundable tax credits across all 50 states and select localities, with sc_refundable_credits specifically evaluated for this household's state of residence. Since South Carolina does not currently offer refundable tax credits that this single adult without children qualifies for, sc_refundable_credits returned $0. All other state-level refundable credit variables (from Alabama through Wyoming, plus New York City and Utah) similarly evaluated to $0, resulting in a total state_refundable_credits value of $0.", us,scenario_045,ssi,0.0,25,"PolicyEngine calculated SSI of $0.00 for this household in 2026 because the head of household, age 44, does not meet any of the eligibility criteria for Supplemental Security Income. SSI eligibility requires that a household member be aged 65 or older, blind, or disabled; the engine's inputs indicate that is_ssi_aged_blind_disabled=False for the sole adult in this household. Although the household's income of approximately $36,276 would be relevant to benefit calculation if the person were eligible, the $0 result stems directly from the failure to satisfy the categorical requirement, not from income limits. The computation confirms that even though the household would take up SSI if eligible (takes_up_ssi_if_eligible=True), no benefit is payable when the underlying eligibility condition is not met.", us,scenario_083,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this Texas household in 2026. The household consists of a single adult age 20 with approximately $11,010 in annual income. SSI eligibility requires that a household member meet at least one of three criteria: being aged 65 or older, blind, or disabled. The head of household does not satisfy any of these conditions (is_ssi_aged_blind_disabled=False), and therefore is ineligible for SSI regardless of income level. Since no household member meets SSI's categorical eligibility requirement, PolicyEngine correctly determined that the household receives $0 in SSI benefits.", -us,scenario_045,snap,287.68316650390625,531,"PolicyEngine calculated a monthly SNAP benefit of $287.68 for this Michigan household by determining eligibility and then computing the benefit amount. The household qualified for SNAP because it met all eligibility requirements: the net income test (with snap_net_income of $1,102.06 at 85% of the federal poverty guideline of $1,304.17), the asset test (with $800 in bank account assets), categorical eligibility through TANF non-cash assistance, and work requirements. The benefit amount was calculated using the formula: maximum allotment minus expected contribution, where the snap_max_allotment for a single-person household in the contiguous US was $298 and the snap_expected_contribution was $330.60 (30% of net income). Since this calculation would yield a negative benefit, PolicyEngine applied the snap_min_allotment of $23.84 as the floor for most months, but in later months adjusted the standard deduction and shelter deduction slightly, resulting in a marginally higher minimum allotment of $24.37. The final reported value of $287.68 represents an average across the tax year reflecting these variations in the minimum allotment across different months.", +us,scenario_045,snap,0.0,25,"PolicyEngine computes an annual 2026 SNAP amount of $0 for this one-person Michigan household. The worker earns $3,022.97 a month in wages and pays $433.33 a month ($5,200 a year) in legally obligated child support. With the fix merged in policyengine-us #9586, PolicyEngine's child support parameter for Michigan counts child support paid in SNAP gross income and deducts it when computing net income, so gross income is the full $3,022.97: 231.8% of the federal poverty guideline from January to September ($1,304.17) and 227.3% from October ($1,330). The household has no elderly or disabled member, so it must pass SNAP's 130% gross income test, which it fails, and it is not categorically eligible through Michigan's TANF non-cash program, whose gross income limit is 200% of the guideline. It is therefore ineligible in every month, and the benefit for the year is $0. Its net income, $1,102 a month after the $209 standard deduction, a $604.59 earned income deduction, the $433.33 child support deduction and a $673.98 excess shelter deduction, would pass the net income test, but without an elderly or disabled member the household must also pass the gross income test or qualify categorically.", us,scenario_121,local_income_tax,0.0,1,"For a single adult in South Carolina with a household income of approximately $39,909 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that South Carolina does not impose a local income tax at the state level, and the household's residence does not have a municipal or county income tax obligation. Therefore, no local income tax liability accrues to this benchmark household regardless of their income level.", us,scenario_045,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Michigan household in 2026. The household consists of a single adult with no children and an annual income of approximately $36,276. TANF is a cash assistance program primarily designed for families with dependent children, and this household's composition—lacking any qualifying children—makes it ineligible for benefits under the program's eligibility requirements. Consequently, PolicyEngine derived a benefit value of zero dollars, reflecting the household's lack of qualification for this particular assistance program.", us,scenario_102,free_school_meals_eligible,0.0,15,"PolicyEngine determined that this North Carolina household is not eligible for free school meals, resulting in a value of 0.0. The household's countable income of $87,201 was compared against the federal poverty guideline for a two-person household ($21,640), yielding a school_meal_fpg_ratio of 4.03, which places them in school_meal_tier 2 (paid meals). Since the household's income exceeds 185% of the federal poverty guideline (the threshold for reduced-price meals at $40,036), they do not qualify for any subsidy under the income-based eligibility rules. Additionally, the household does not meet categorical eligibility through programs like SNAP or TANF, and North Carolina does not have a universal free school meals program. Therefore, the household must pay the full price for school meals with no subsidy applied.", @@ -448,7 +448,7 @@ us,scenario_083,head_chip_eligible,0.0,2,"PolicyEngine determined that the head us,scenario_104,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this household as $0. The computation shows that the employee_payroll_tax component, which represents the employee's share of Social Security and Medicare taxes, evaluated to $0. Since the household is identified as a tax unit head (is_tax_unit_head = True), this individual is treated as the primary taxpayer for the unit. The zero payroll tax result indicates that either no wages subject to payroll tax were reported for this household, or the income structure did not trigger standard payroll tax obligations for the 2026 tax year.", us,scenario_121,head_wic_eligible,0.0,14,"PolicyEngine determined that this household is not eligible for WIC (Women, Infants, and Children) benefits because it fails two of the three required eligibility criteria. The household's wic_countable_income of $39,909—derived from employment income of $25,665, child support received of $1,200, and disability benefits of $13,044—exceeds the wic_income_limit of $29,526 for a single-person household in South Carolina, causing it to fail the income test. Additionally, the household does not meet categorical eligibility requirements, as the head of household is 53 years old and WIC categorical eligibility is limited to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5. Although the household meets the nutritional risk criterion (is_wic_at_nutritional_risk = True), this alone is insufficient without meeting either the income or categorical requirements, resulting in an overall eligibility determination of False.", us,scenario_008,child2_chip_eligible,0.0,1,"PolicyEngine determined that child2, a 14-year-old household member, is not eligible for CHIP (child2_chip_eligible = False). The primary reason for this ineligibility is that child2 qualifies for Medicaid under the OLDER_CHILD category, which takes precedence in the eligibility determination. Since CHIP is designed to cover children who do not qualify for Medicaid, child2's Medicaid eligibility automatically disqualifies them from CHIP coverage. The household's income of approximately $30,915 is not the limiting factor in this determination; rather, the existing Medicaid eligibility status is the decisive factor that renders CHIP coverage unnecessary and ineligible.", -us,scenario_066,snap,3596.039794921875,373,"PolicyEngine calculated a 2026 SNAP benefit of $3,596.04 for this Virginia household by aggregating monthly allotments across the year. The household qualifies for SNAP because it meets all eligibility requirements: the gross income of $43.33 monthly (3% of the federal poverty guideline of $1,304.17) passes the gross income test, the household qualifies for categorical eligibility through TANF non-cash assistance, and it satisfies the net income, asset, work requirement, and immigration status tests. The monthly allotment varies slightly across the year, with most months yielding $298 and later months yielding $304.68, reflecting changes in the federal poverty guideline (which increases from $1,304.17 to $1,330 mid-year). Since the household is assumed to take up SNAP if eligible, PolicyEngine sums these monthly allotments across all 12 months of 2026 to arrive at the annual benefit total of approximately $3,596.", +us,scenario_066,snap,3576.0,373,"PolicyEngine computes 2026 SNAP for this one-person Virginia household as $3,576 for the year, the sum of twelve monthly allotments of $298. The household is eligible in every month. It is categorically eligible because it meets Virginia's TANF non-cash gross income, net income and asset tests, the last with assets of $14,000. Its SNAP gross income, all of it earned, is $43.33 a month, about 3% of the monthly poverty guideline. That guideline is $1,304.17 for January through September and $1,330 for October through December. The household also passes the SNAP gross and net income tests and the work requirements. Each month's allotment equals the one-person maximum for the contiguous United States, $298, well above the $24 minimum allotment.", us,scenario_104,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this New York household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $34,616 in household income derives from other sources (such as wages or salaries), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Without self-employment earnings, no self-employment tax liability is generated.", us,scenario_027,spouse_medicare_eligible,0.0,1,"PolicyEngine determined that the spouse in this Connecticut household is not Medicare eligible, returning a value of False (0.0) for the second adult. The household consists of two adults with a combined income of approximately $30,435 in 2026. Medicare eligibility is primarily determined by age—individuals must be 65 years or older to qualify for Medicare based on age alone—and the computation trace shows that while the first household member returned True for is_medicare_eligible, the second member returned False, indicating the spouse does not meet the age threshold or other eligibility criteria for Medicare in that tax year. This binary result reflects PolicyEngine's assessment that only one member of this joint-filing couple qualifies for Medicare benefits.", us,scenario_045,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, along with certain disability or end-stage renal disease criteria. For this single adult in Michigan with a household income of approximately $36,276, the household composition and income level alone do not trigger Medicare eligibility—the determining factor is that the individual does not meet the age or disability requirements that would qualify them for the program. Consequently, PolicyEngine assigned a value of 0.0 (False) to the is_medicare_eligible variable, indicating no Medicare benefits would be available to this household member.", @@ -465,7 +465,7 @@ us,scenario_008,child4_chip_eligible,0.0,1,"PolicyEngine determined that child4, us,scenario_027,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Connecticut household is not eligible for reduced-price school meals (reduced_price_school_meals_eligible = 0.0) because the household qualifies for the superior free meals tier instead. With a household income of approximately $30,435, the school_meal_fpg_ratio is 1.41, meaning the household's income is 141% of the federal poverty guideline. Since the household meets categorical eligibility through an existing benefit program, it qualifies for free meals under the free tier, which covers households at or below 130% of the poverty guideline or those with categorical eligibility. Because the household already receives the free tier benefit, it is not eligible for the reduced-price tier, which serves as a secondary program for households above the free threshold but below the reduced-price income limit (typically 185% of poverty). The school_meal_net_subsidy is therefore 0 for the reduced-price category, as the household's needs are fully met through the free meals program.", us,scenario_045,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Michigan household is not eligible for reduced-price school meals, resulting in a value of 0.0. The household qualifies for the superior FREE tier through Michigan's universal free school meals program, which provides meals at no cost regardless of income. With a school_meal_fpg_ratio of 2.27 (meaning household income is 227% of the federal poverty guideline), the household exceeds the reduced-price income threshold, but this is not the limiting factor. Because the household already qualifies for free meals through the state's universal program—a better benefit than reduced-price eligibility—the reduced_price_school_meals_eligible variable correctly reflects 0, indicating no additional reduced-price eligibility is needed or applicable.", us,scenario_083,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Texas household is not eligible for reduced-price school meals (reduced_price_school_meals_eligible = False) because the household qualifies for the more generous free meals tier instead. With a household income of approximately $11,010, the school_meal_fpg_ratio is 0.69, meaning the income is 69% of the federal poverty guideline—well below the 130% threshold required for free meals. Since the household meets the income requirement for free school meals and does not rely on categorical eligibility pathways, PolicyEngine assigned the household to the FREE school_meal_tier. Consequently, the school_meal_net_subsidy is 0 for reduced-price meals, as the household receives the full subsidy available through the free meals program rather than the partial subsidy of the reduced-price tier.", -us,scenario_104,state_refundable_credits,375.0,51,"PolicyEngine calculated state refundable credits of $375 for this New York resident. The entire amount comes from New York's real property tax credit (ny_real_property_tax_credit), which equals $375. This credit was determined based on the household's characteristics: the taxpayer is 72 years old and has annual rent payments of $6,483.53. New York's real property tax credit is designed to provide relief to eligible renters and homeowners, and in this case the credit formula applied to the household's rent expense and age resulted in the $375 refundable credit. All other states' refundable credits contributed $0 to the total, as the household resides in New York.", +us,scenario_104,state_refundable_credits,0.0,51,"PolicyEngine computes state refundable credits of $0 for this household, a single 72-year-old filer in New York. The total adds up each state's refundable credits. The household's state code resolves to NY, so New York's refundable credits are the ones that can apply, and ny_refundable_credits evaluates to $0 for this household. The filer reports no employment income, $14,096 in Social Security retirement benefits, $20,520 in veterans benefits and $6,483.53 in annual rent. No New York refundable credit is allowed on these facts. Every other state's refundable credit variable, and the New York City refundable credit line, also evaluates to $0, so the total is $0 with no refundable state credit for the year.", us,scenario_008,child5_chip_eligible,0.0,1,"PolicyEngine determined that child5, a 5-year-old in this New Jersey household, is not eligible for CHIP (is_chip_eligible = False). The child qualifies for Medicaid under the YOUNG_CHILD category, which makes them ineligible for CHIP because CHIP is designed to cover children who do not qualify for Medicaid. Since the household's income of approximately $30,915 falls within the Medicaid eligibility threshold for young children, the child's Medicaid eligibility automatically disqualifies them from CHIP coverage.", us,scenario_046,federal_income_tax_before_refundable_credits,499.71484375,40,"PolicyEngine calculated federal income tax before refundable credits of $499.71 for this Oklahoma joint household by first determining taxable income of $44,964.29, which resulted from adjusted gross income of $92,003 reduced by taxable income deductions of $47,038.71 (comprised of the standard deduction of $32,200 plus an overtime income deduction of $14,838.71). Applying 2026 tax rates to this taxable income produced income tax before credits of $4,899.71. The household then claimed non-refundable credits totaling $4,400, consisting entirely of the Child Tax Credit for two qualifying children at $2,200 each, which reduced the tax liability to $499.71. The net investment income tax and other supplemental taxes did not apply to this household, as the $3 in taxable interest income fell below the thresholds for those calculations.", us,scenario_121,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, along with certain disability or end-stage renal disease criteria. Since this benchmark household is a single adult in South Carolina with a household income of approximately $39,909, the household member does not meet the age threshold or other qualifying conditions for Medicare enrollment. Therefore, PolicyEngine correctly assigned a value of 0.0 (False) to the is_medicare_eligible variable, indicating no Medicare eligibility for this household.", @@ -512,7 +512,7 @@ us,scenario_008,child3_medicare_eligible,0.0,1,"PolicyEngine determined that chi us,scenario_046,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0.00 for this Oklahoma household in 2026. Oklahoma does not impose a state income tax, and the household's location within Oklahoma does not have a local income tax that would apply to wage or investment income. Therefore, with no applicable local income tax jurisdiction for this household, the local_income_tax variable correctly returns zero regardless of the household's approximately $92,003 in annual income.", us,scenario_028,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Pennsylvania head of household with three children and approximately $61,277 in household income. This result reflects that Pennsylvania does not impose a local income tax at the state level, which is why the local_income_tax variable resolves to zero regardless of the household's income level or filing status. The calculation is straightforward: with no applicable local income tax policy in Pennsylvania for the 2026 tax year, no tax liability accrues to this household.", us,scenario_104,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. The household head is 72 years old, which exceeds CHIP's age eligibility limit—CHIP primarily serves children and pregnant women, with coverage generally ending at age 19 (or slightly older in some states for pregnant women). Additionally, with a household income of approximately $34,616, the person does not meet CHIP's income thresholds, which are typically set at or below 200-250% of the federal poverty level depending on the state. Since the person fails both CHIP's age requirement and income criteria, they are ineligible for the program.", -us,scenario_122,state_income_tax_before_refundable_credits,1028.216552734375,80,"For this Minnesota resident with approximately $77,460 in household income, PolicyEngine calculated a state income tax of $1,028.22 before refundable credits. The calculation began with an adjusted gross income of $70,646.18, from which Minnesota subtractions of $32,177.18 were applied (primarily a $27,200 Social Security subtraction and a $4,977.18 charitable contribution subtraction), resulting in a taxable income of $19,219. This taxable income was then subject to Minnesota's basic tax calculation, which applied the state's tax brackets and rates to arrive at the final tax liability of $1,028.22. All other states' income taxes were zero, as the household is only subject to Minnesota taxation.", +us,scenario_122,state_income_tax_before_refundable_credits,1025.54150390625,92,"PolicyEngine calculated Minnesota state income tax before refundable credits of $1,025.54 for this single Minnesota resident with ~$77,460 household income in 2026. The calculation began with IRS gross income of $72,165, which was reduced by above-the-line deductions of $1,518.82 (a capital loss) to yield adjusted gross income of $70,646.18. Minnesota taxable income was determined by subtracting both the inflation-adjusted standard deduction of $19,300 (increased due to the filer's age and blindness status) and Minnesota-specific subtractions totaling $32,177.18—comprising a $4,977.18 charitable contribution subtraction and a $27,200 Social Security income subtraction—resulting in taxable income of $19,169. This taxable income was then subject to Minnesota's 2026 tax brackets (the inflation-adjusted amounts published by the Minnesota Department of Revenue in December 2025), producing the basic tax liability of $1,025.54 before any refundable credits.", us,scenario_084,local_income_tax,0.0,1,"For a single adult in North Carolina with a household income of approximately $33,233 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that North Carolina does not impose a local income tax at the state level, and the household's income and circumstances do not trigger any municipal-level income taxes that might apply in specific jurisdictions. The computation therefore directly yields zero local income tax liability for this benchmark household.", us,scenario_028,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Pennsylvania head-of-household with three children and annual household income of approximately $61,277 in 2026. Although the household's `takes_up_snap_if_eligible` variable evaluated to True, indicating willingness to participate in the program, the household's income level exceeded the SNAP eligibility threshold, resulting in no benefit entitlement. The computation trace shows the benefit was evaluated multiple times across different policy scenarios or household configurations, consistently yielding $0, confirming that income ineligibility was the determining factor in this case.", us,scenario_046,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Oklahoma household of four in 2026. Although the household's `takes_up_snap_if_eligible` variable is set to True, indicating the household would participate in the program if qualified, the household's income of approximately $92,003 exceeds the SNAP income eligibility threshold. Under federal SNAP rules, households must meet both gross and net income limits based on family size; this household's income level places it above the maximum allowable for a family of four, making them ineligible regardless of their willingness to participate. Consequently, PolicyEngine determined no SNAP benefits could be claimed, resulting in the $0 benefit value.", @@ -599,7 +599,7 @@ us,scenario_107,head_wic_eligible,0.0,1,"PolicyEngine determined that the househ us,scenario_046,child1_chip_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for CHIP (is_chip_eligible = False) in 2026. The child fails to meet CHIP's eligibility criteria on multiple grounds: at age 10, the child exceeds the maximum age limit for CHIP enrollment in Oklahoma, and the household's income of approximately $92,003 also exceeds CHIP's income threshold for a family of four. Since the child is neither Medicaid-eligible nor meets CHIP's own age and income requirements, PolicyEngine correctly classified the child as ineligible for the program.", us,scenario_085,state_income_tax_before_refundable_credits,0.0,47,"PolicyEngine calculated the state income tax before refundable credits as $0 for this Pennsylvania resident with approximately $58,949 in household income. The computation aggregates income tax liabilities across all 50 states and the District of Columbia, with each state-level variable (such as pa_income_tax_before_refundable_credits) evaluated individually. Since the household resides in Pennsylvania, only pa_income_tax_before_refundable_credits is relevant to the final calculation, and this variable evaluated to $0. All other state income tax variables also returned $0, as the household has no tax liability in those jurisdictions. The final state_income_tax_before_refundable_credits value of $0 reflects that this Pennsylvania household's income falls below the threshold at which Pennsylvania state income tax liability is incurred for the 2026 tax year.", us,scenario_123,payroll_tax,11194.0,11,"PolicyEngine calculated the household's total payroll tax of $11,194 by summing three components across the two earning adults and one child. The employee Social Security tax totaled $8,990, derived from the household's $145,000 in employment income split between the two adults ($100,000 and $45,000 respectively), with the primary earner hitting the Social Security wage base cap. The employee Medicare tax added $2,102.50, calculated at 1.45% on all wages without a cap. Finally, Pennsylvania's employee unemployment compensation contribution contributed $101.50, assessed at the state's rate on the combined employment income of both working adults. These three components—Social Security, Medicare, and state payroll tax—sum to the total spm_unit_payroll_tax of $11,194.", -us,scenario_028,child3_chip_eligible,0.0,1,"PolicyEngine determined that child3 is not eligible for CHIP in Pennsylvania for 2026. The child fails to meet CHIP's eligibility criteria on multiple grounds: child3 is age 10, which falls within CHIP's covered age range, but the household's income of approximately $61,277 exceeds Pennsylvania's CHIP income threshold. Additionally, since child3 is not Medicaid-eligible (categorized as NONE in the engine), the child does not qualify for CHIP through the Medicaid pathway. The combination of excess household income relative to the program's limits and lack of Medicaid eligibility results in the is_chip_eligible value of False.", +us,scenario_028,child3_chip_eligible,0.0,1,"PolicyEngine found Child 3 not eligible for CHIP. The child is 10, under the age limit of 19, and is not Medicaid-eligible, because the household's income of $60,010 (wages plus interest; the child support it receives is not MAGI income) is 182% of the $33,000 poverty guideline for four, above Pennsylvania's 138% Medicaid limit for children aged 6 to 18. That income passes Pennsylvania's CHIP limit of 319% of the guideline (314% plus the 5-point disregard). The deciding condition is coverage: the prompt lists Child 3 as having employer-sponsored insurance, and a child covered by a group health plan is not a targeted low-income child (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii)), so PolicyEngine's disqualifying-coverage test fails and is_chip_eligible is 0.", us,scenario_008,child6_early_head_start_eligible,1.0,1,"PolicyEngine determined that child 6 is eligible for Early Head Start (is_early_head_start_eligible = True) based on the household's income relative to the federal poverty line. The household's annual income of approximately $30,915 falls below the 100% federal poverty line threshold required for Early Head Start eligibility in New Jersey. Early Head Start is a federally-funded program serving infants and toddlers under age 3 from low-income families, and this household's income level qualifies them for enrollment. The first five children in the household array returned False because they either exceed the age requirement for Early Head Start (which serves children under 3 years old) or the household does not meet other program criteria for those specific children, while child 6 meets all eligibility requirements.", us,scenario_046,child2_chip_eligible,0.0,1,"PolicyEngine determined that child2 is not eligible for CHIP (is_chip_eligible = False) because the household's income exceeds CHIP's financial thresholds for Oklahoma in 2026. With a household income of approximately $92,003 and two children, the family's income-to-federal poverty level ratio surpasses the maximum allowed under Oklahoma's CHIP program, which typically caps eligibility at 200-250% of the federal poverty line depending on the child's age and family composition. Additionally, child2 is not Medicaid-eligible (categorized as NONE), meaning the child does not qualify through the Medicaid pathway that would otherwise provide a bridge to CHIP coverage. The combination of excess household income and lack of Medicaid eligibility results in PolicyEngine correctly computing is_chip_eligible as False for this nine-year-old child.", us,scenario_085,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Pennsylvania resident with ~$58,949 in household income. The computation aggregates refundable tax credits across all 50 states and select localities, with pa_refundable_tax_credits—the only potentially applicable component for this household—also evaluating to $0. Pennsylvania does not currently offer refundable tax credits that this single adult without children would qualify for at this income level, resulting in no state-level refundable credits being available to offset the household's tax liability.", @@ -678,18 +678,18 @@ us,scenario_068,payroll_tax,2607.39013671875,5,"PolicyEngine calculated a payrol us,scenario_048,federal_income_tax_before_refundable_credits,0.0,8,"For this single adult in Ohio with approximately $34,626 in household income, PolicyEngine calculated federal income tax before refundable credits as $0. The computation shows that income_tax_before_credits, which is the primary component of this calculation, equals $0, indicating that after applying the standard deduction and tax brackets for the 2026 tax year, no federal income tax liability was generated. The net_investment_income_tax also contributed $0, as the adjusted_gross_income of $7,656 falls below the threshold for this tax. Additionally, no recapture of investment credits, unreported payroll tax, or qualified retirement penalties applied to this household, and income_tax_capped_non_refundable_credits was $0, meaning either no non-refundable credits were available or they were not needed to offset any tax liability.", us,scenario_086,federal_income_tax_before_refundable_credits,3255.5478515625,37,"PolicyEngine calculated federal income tax before refundable credits of $3,255.55 by starting with a gross income of $72,900.40, which included $62,000 in employment income and $10,900.40 in taxable Social Security benefits. After applying above-the-line deductions of $337.50 (educator expenses), the adjusted gross income was $72,562.90. The household claimed the standard deduction of $24,150 for head of household filing status, resulting in a taxable income of $48,412.90. Applying 2026 tax rates to this taxable income produced an income tax before credits of $5,455.55, which was then reduced by $2,200 in non-refundable child tax credits, yielding the final value of $3,255.55.", us,scenario_009,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this North Carolina household in 2026. TANF is a needs-based program with strict eligibility requirements, and this household's income of approximately $110,400 substantially exceeds the income thresholds for TANF eligibility in North Carolina. Additionally, TANF is primarily designed for families with children, and this household has zero children, which further disqualifies them from receiving benefits. As a result, PolicyEngine correctly determined that no TANF assistance applies to this benchmark household.", -us,scenario_028,reduced_price_school_meals_eligible,1.0,3,"PolicyEngine determined that this Pennsylvania household with three children qualifies for reduced-price school meals, resulting in an annual benefit value of $3,009.82. The household's income-to-poverty ratio of 1.82 falls within the federal eligibility threshold for reduced-price meals, which extends to 185% of the federal poverty guideline. Since the state does not offer universal free school meals and the household does not meet categorical eligibility through programs like SNAP or TANF, eligibility is determined solely by the income test. The engine classified the household into the REDUCED school_meal_tier, and the resulting school_meal_net_subsidy of $3,009.82 reflects the annual value of reduced-price meal benefits for the three children in the household.", +us,scenario_028,reduced_price_school_meals_eligible,0.0,19,"PolicyEngine determined that this Pennsylvania household of four is not eligible for reduced-price school meals. School meals count the household's income of $61,277: $60,000 of wages, $10 of interest and $1,267 of child support received, which counts as income for school meals (7 CFR 245.6(a)(5)(ii)). That is 1.86 times the $33,000 poverty guideline for a household of four, above the 185% limit for reduced-price meals, so the value is 0.", us,scenario_068,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Maryland household because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's income of approximately $34,084 is derived from wage and salary sources rather than self-employment, there is no self-employment income base upon which to calculate the 15.3% self-employment tax rate (which covers Social Security and Medicare taxes for self-employed individuals). Therefore, PolicyEngine correctly determined that no self-employment tax liability applies to this household.", us,scenario_086,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this Georgia head-of-household filer with one child and approximately $74,824 in household income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. This outcome reflects that the household's income level and filing status did not qualify for any of the refundable credits available under 2026 tax law.", us,scenario_048,federal_refundable_credits,0.0,6,"For this single adult in Ohio with approximately $34,626 in household income and no children, PolicyEngine calculated federal refundable credits of $0. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0 for this household. The EITC yielded no credit because the household's income and filing status did not qualify for this benefit, the refundable American Opportunity Credit was $0 as there were no qualifying education expenses or students, the refundable CTC was $0 due to the absence of dependent children, and the other refundable credits similarly did not apply to this taxpayer's circumstances.", us,scenario_028,free_school_meals_eligible,0.0,14,"PolicyEngine determined that this Pennsylvania household is not eligible for free school meals (value: 0), though it does qualify for reduced-price meals. The household's countable income of $60,010 (comprising $60,000 in employment income and $10 in interest income) was compared against the federal poverty guideline of $33,000 for the household's size, yielding a ratio of 1.82 times the poverty line. This income level places the household in the reduced-price tier rather than the free meals tier, as the household exceeds the income threshold for free eligibility but falls within the range for reduced-price meals. Since the household does not receive categorical assistance through SNAP or TANF and Pennsylvania does not have universal free school meals, the household does not qualify for free meals despite having three children in K-12 schools.", us,scenario_029,federal_income_tax_before_refundable_credits,0.0,10,"For this single adult in Oklahoma with approximately $312 in household income during tax year 2026, PolicyEngine calculated federal income tax before refundable credits as $0. The computation shows that while the household had $312 in taxable interest income, which contributed to an adjusted gross income of $312, this amount fell below the standard deduction threshold for a single filer, resulting in zero income tax before credits. Additionally, the household had no net investment income tax, recapture of investment credit, unreported payroll tax, qualified retirement penalties, or non-refundable credits to apply, all of which remained at $0. Therefore, the final income tax before refundable credits was $0.", us,scenario_048,payroll_tax,585.6839599609375,5,"For this single adult in Ohio with approximately $34,626 in household income, PolicyEngine calculated a total payroll tax of $585.68 for tax year 2026. This amount consists of two components: employee Social Security tax of $474.67 and employee Medicare tax of $111.01. The Social Security portion represents 6.2% of covered wages up to the annual wage base, while the Medicare portion represents 1.45% of all covered wages with no cap. Since this individual is identified as a tax unit head, they are subject to the standard employee payroll tax rates without any special exemptions or adjustments.", -us,scenario_108,snap,287.68316650390625,484,"PolicyEngine calculated a monthly SNAP benefit of $287.68 for this Wisconsin household by aggregating the individual monthly allotments across multiple benefit months in the tax year. The household qualifies for SNAP because it meets all eligibility requirements: the gross income of $2,581.83 is within acceptable limits at 1.98 times the federal poverty guideline of $1,304.17, assets of $270 fall below the threshold, and the household qualifies through categorical eligibility via TANF non-cash assistance. The normal monthly allotment is calculated as the maximum allotment of $298 (for a single-person household in the contiguous US) minus the expected contribution of $610.50, which is derived from the net income of $2,035.67; however, this produces a negative value, so the benefit is set to the minimum allotment of $23.84 per month for most months, with some months receiving $24.37 based on updated federal poverty guidelines. The annual total of $287.68 reflects the sum of these monthly minimum allotments across the tax year, assuming the household takes up the benefit when eligible.", +us,scenario_108,snap,288.0,484,"PolicyEngine computes an annual 2026 SNAP total of $288 for this one-person Wisconsin household, the sum of twelve monthly allotments of $24, January through December, with the household assumed to take up the benefit. It passes the gross income test, with monthly gross income of $2,581.83 and an elderly or disabled member, passes the asset test with $270 in assets, and is categorically eligible through Wisconsin's TANF non-cash rules. Its monthly net income is $2,036, giving an expected contribution of $611, which exceeds the $298 maximum allotment for one person in the contiguous US. The benefit therefore falls to the $24 minimum allotment in every month. The poverty guideline in the eligibility tests is $1,304.17 from January through September and $1,330 from October through December, which does not change the monthly amount.", us,scenario_009,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this North Carolina household is eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False]`. WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this household consists of two adults with no children. Since the household contains no members in any of the qualifying demographic categories, both household members fail the categorical eligibility requirement for WIC, regardless of their income level of approximately $110,400.", us,scenario_108,ssi,0.0,129,"The 85-year-old household member in Wisconsin receives $0 in SSI for 2026 because their countable income exceeds the maximum benefit amount. Although the individual qualifies for SSI based on age (is_ssi_aged_blind_disabled=True) and passes the resource test with $270 in countable resources, their ssi_countable_income of $2,210.33 far exceeds the ssi_amount_if_eligible of $994. This results in an uncapped_ssi calculation of negative $1,216.33 ($994 − $2,210.33), which is capped at zero since SSI cannot produce a negative benefit. The individual's countable income includes $7,656 in earned income and $23,326 in unearned income, both of which are counted against the SSI benefit limit, leaving no remaining benefit to pay.", us,scenario_123,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program), resulting in `is_chip_eligible = False`. The household's income of approximately $145,002 exceeds Pennsylvania's CHIP income limits, which typically cap eligibility at a percentage of the federal poverty level (generally around 200–250% depending on family size and state policy). Additionally, the head of household is age 39, which places them outside CHIP's age requirements—CHIP is designed for children and young adults, not working-age adults. Since the person fails both CHIP's income criterion and its age-based eligibility rules, PolicyEngine correctly classified them as ineligible for the program.", -us,scenario_068,state_income_tax_before_refundable_credits,1252.9677734375,66,"For this Maryland resident with approximately $34,084 in household income, PolicyEngine calculated a state income tax liability of $1,252.97 before refundable credits. The calculation began with an adjusted gross income (AGI) of $34,083.53, from which Maryland's standard deduction of $3,400 and personal exemptions of $3,200 were subtracted to arrive at a taxable income of $27,483.53. This taxable income was then subject to Maryland's tax rate schedule to produce the final tax before refundable credits of $1,252.97. All other states returned zero tax liability, confirming that only Maryland's income tax applied to this single-adult household.", +us,scenario_068,state_income_tax_before_refundable_credits,1255.3427734375,69,"PolicyEngine calculated Maryland income tax before refundable credits of $1,255.34 for this single filer. Maryland adjusted gross income of $34,083.53 less the $3,350 standard deduction and $3,200 of personal exemptions gives Maryland taxable income of $27,533.53, and Maryland's rate schedule applied to it gives $1,255.34. The standard deduction is the flat 2025 amount, the last Maryland published before the benchmark's 2026-07-03 reference freeze.", us,scenario_086,payroll_tax,4743.0,5,"PolicyEngine calculated a payroll tax of $4,743 for this Georgia head-of-household with one child and approximately $74,824 in household income. This total comprises two components: employee Social Security tax of $3,844 and employee Medicare tax of $899. The Social Security tax was computed by applying the 6.2% employee rate to the applicable wage base (capped at the annual limit), while the Medicare tax was calculated at 2.35% on total wages, reflecting the additional 0.9% high-income Medicare tax threshold that applies to higher earners. These payroll taxes are withheld from the primary earner's wages, as indicated by the tax unit head designation, with no payroll tax liability attributed to the dependent child.", us,scenario_029,federal_refundable_credits,0.0,6,"For this single adult in Oklahoma with approximately $312 in household income in 2026, PolicyEngine calculated federal refundable credits of $0. The calculation examined five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. With such minimal income and no children in the household, this household did not qualify for any of the major federal refundable credits available in 2026.", us,scenario_009,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False. WIC eligibility requires the presence of children under age 5 in the household, and this benchmark household contains 0 children. Since the household does not meet the fundamental categorical requirement for WIC participation, the spouse's eligibility status is False regardless of other factors such as the household's income of approximately $110,400.", @@ -786,14 +786,14 @@ us,scenario_086,child1_head_start_eligible,0.0,6,"PolicyEngine determined that t us,scenario_109,ssi,0.0,25,"PolicyEngine calculated total household SSI for 2026 as $0.00 because none of the five household members—the head (age 30), spouse (age 29), or the three children (ages 9, 6, and 0)—meet SSI's eligibility criteria. SSI is an individual benefit that requires recipients to be aged 65 or older, blind, or disabled; the engine's inputs indicate that all five household members have `is_ssi_aged_blind_disabled=False`, meaning none qualify on these grounds. Although the computation trace shows that all household members have `takes_up_ssi_if_eligible=True`, indicating they would claim the benefit if eligible, the absence of any qualifying condition results in $0.00 SSI for each person and therefore $0.00 for the household overall.", us,scenario_030,federal_income_tax_before_refundable_credits,0.0,8,"For this single adult in Texas with approximately $13,000 in household income, PolicyEngine calculated federal income tax before refundable credits as $0. The computation shows that income_tax_before_credits, the primary component, equals $0, indicating that the household's tax liability before any credits is eliminated entirely. This result reflects that the adjusted_gross_income of $13,000 falls below the standard deduction threshold for a single filer in 2026, which means no taxable income remains after the standard deduction is applied. The other components—net investment income tax, recapture of investment credit, unreported payroll tax, and qualified retirement penalty—all contribute $0 to the final calculation, as none of these additional tax items apply to this household's circumstances.", us,scenario_049,payroll_tax,15134.5,6,"PolicyEngine calculated a total payroll tax of $15,134.50 for this household by summing the employee-side payroll tax components. The employee social security tax contributed $11,439, calculated by applying the 6.2% employee rate to wages up to the 2026 social security wage base limit. The employee Medicare tax added $3,668.50, derived from the standard 1.45% rate on all wages earned by the higher-earning spouse. An additional Medicare tax of $27 was assessed on wages exceeding the $200,000 threshold for single filers (adjusted proportionally for the joint household structure). These three components—social security tax, standard Medicare tax, and additional Medicare tax—sum to the total spm_unit_payroll_tax of $15,134.50.", -us,scenario_109,snap,8020.55419921875,495,"PolicyEngine calculates SNAP month by month and sums the twelve benefits. From January through September, the $261 standard deduction leaves $1,739 of net monthly income; subtracting the $521.70 expected contribution from the $1,183 maximum allotment produces about $661.30 per month. From October through December, updated fiscal-year parameters raise the standard deduction to $266.85 and the maximum allotment to $1,209.52, producing about $689.62 per month after the $519.90 expected contribution. The household passes the income, asset, and work-requirement tests throughout the year, and summing the unrounded monthly benefits yields $8,020.55.", +us,scenario_109,snap,7932.0,495,"PolicyEngine computes SNAP month by month for this five-person Florida household and adds the twelve monthly benefits, each $661, for an annual total of $7,932. Every month, gross income is $2,500 of earned income, the monthly share of the head's $30,000 in self-employment income. Deductions total $761, a $500 earned income deduction plus a $261 standard deduction, leaving net income of $1,739. The expected contribution from that net income is $522, and subtracting it from the $1,183 maximum allotment for five people gives $661. The household passes the gross income, net income, $900 asset and work-requirement tests in every month. The poverty guideline used in the income tests rises from $3,137.50 (January through September) to $3,223.33 (October through December), but the benefit stays $661 each month.", us,scenario_086,child1_medicare_eligible,0.0,1,"PolicyEngine determined that child1_medicare_eligible equals False (represented as 0.0) for this Georgia household. Medicare eligibility is primarily determined by age, and the child in this household does not meet the age threshold of 65 years required for standard Medicare coverage. Additionally, the household does not qualify for Medicare based on disability or end-stage renal disease status, which are the other primary pathways to Medicare eligibility for individuals under 65. Therefore, PolicyEngine correctly classified the child as not Medicare-eligible for the 2026 tax year.", us,scenario_070,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Illinois resident with ~$38,100 household income in 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with il_refundable_credits specifically evaluated for this household since the IL state indicator is true. Since Illinois does not currently offer refundable tax credits that apply to this household's income level and filing status, il_refundable_credits returned $0. All other state-level refundable credit variables (from Alabama through Wyoming, plus New York City) also evaluated to $0, as the household does not reside in those jurisdictions. The sum of these zero values across all states and localities yields the final state_refundable_credits amount of $0.", us,scenario_049,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this New Hampshire joint household in 2026 because neither spouse had self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this household's approximately $262,821 in total income was derived from other sources (such as wages or investment income rather than self-employment), no self-employment tax liability was triggered. The calculation correctly reflects that self-employment tax applies only to individuals with qualifying self-employment earnings, which this household did not report.", us,scenario_086,child1_early_head_start_eligible,0.0,5,"PolicyEngine determined that the child in this Georgia household is not eligible for Early Head Start (value: False). Early Head Start eligibility requires meeting multiple criteria, and this household failed to satisfy the necessary conditions. The child's age of 12 years exceeds the program's age requirements, which target infants and toddlers under age 3. Additionally, the household did not qualify under categorical eligibility pathways (is_head_start_categorically_eligible: False) and failed the income eligibility test (is_head_start_income_eligible: False), with a household income of approximately $74,824. Since the child did not meet the age requirement and lacked both categorical and income-based eligibility, PolicyEngine correctly assigned an ineligibility status of False.", us,scenario_030,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this single adult in Texas with approximately $13,000 in household income. The computation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. For a single filer with no children and this income level, the household does not qualify for the EITC (which requires either qualifying children or age/income restrictions for childless workers), the refundable CTC (which requires qualifying children), or the other refundable credits evaluated.", us,scenario_109,tanf,0.0,1,"PolicyEngine calculated a Temporary Assistance for Needy Families (TANF) benefit of $0 for this Florida household in 2026. Despite the household's modest income of approximately $30,000 with three children, the household did not qualify for TANF benefits under Florida's program rules. This result reflects either that the household's income exceeded Florida's TANF eligibility threshold, or that other household characteristics (such as work requirements, citizenship status, or asset limits) rendered them ineligible for assistance in that tax year. The zero benefit indicates no TANF payment was due to this household under the applicable 2026 Florida TANF policy parameters.", -us,scenario_012,snap,4952.08935546875,351,"PolicyEngine calculated a 2026 SNAP benefit of $4,952.09 for this Mississippi household of three by aggregating monthly allotments across the year. The household qualifies for SNAP because it meets all eligibility tests: gross income of $1,833.33 monthly is 83% of the federal poverty guideline ($2,220.83), net income of $1,257.67 is 57% of the poverty guideline, assets total only $2, and all household members meet work requirements and immigration status criteria. The monthly benefit is calculated as the maximum allotment of $785 (for a three-person household in the contiguous U.S.) minus the expected household contribution of $377.10, which is 30% of net income, yielding a monthly allotment of $407.90. This monthly amount of approximately $407.90 is multiplied across twelve months, with slight variations in later months due to updated standard deductions and poverty guidelines, resulting in the annual total of $4,952.09.", +us,scenario_012,snap,4884.0,351,"PolicyEngine computes an annual 2026 SNAP total of $4,884 for this three-person Mississippi household, the sum of twelve monthly allotments of $407. The head's $22,000 in annual earnings gives gross income of $1,833.33 a month. Deductions total $575.67, a $209 standard deduction plus a $366.67 earned income deduction (20 percent of earnings), leaving net income of $1,258 after rounding to the nearest dollar. All three members meet the work and immigration requirements, the household holds $2 in assets, and its income passes the gross and net income tests measured against a monthly poverty guideline of $2,220.83 in January through September and $2,276.67 in October through December. Each month, 30 percent of net income, rounded up, sets an expected contribution of $378, and subtracting it from the $785 maximum allotment leaves $407.", us,scenario_070,local_income_tax,0.0,1,"For a single adult in Illinois with a household income of approximately $38,100 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that the household does not owe any local income tax liability based on Illinois's tax structure and the applicable local tax rules for the given income level and filing status. The computation shows no intermediate tax amounts or adjustments that would generate a positive local income tax obligation for this benchmark household.", us,scenario_012,ssi,0.0,25,"PolicyEngine calculated SSI as $0.00 for this Mississippi household in 2026 because none of the three household members—the 40-year-old head, 18-year-old spouse, or 10-year-old child—meets SSI's categorical eligibility requirement of being aged 65 or older, blind, or disabled. The engine evaluated each person individually (with spousal deeming rules applied where applicable) and determined that `is_ssi_aged_blind_disabled` was False for all three members based on the provided inputs. Although all household members have `takes_up_ssi_if_eligible` set to True, indicating willingness to claim benefits, this take-up parameter only applies to those who first satisfy the categorical eligibility criteria. Since no member qualifies under the aged, blind, or disabled standard, SSI benefits remain $0.00 across the entire household.", us,scenario_049,state_income_tax_before_refundable_credits,0.0,56,"PolicyEngine calculated the state income tax before refundable credits for this New Hampshire household as $0 because New Hampshire's income tax applies only to dividend and interest income, and the household's taxable income from these sources fell below the filing threshold. The household reported $6,320 in dividend income and $3,501 in interest income, yielding a combined nh_taxable_income of $5,021 after applying the nh_total_exemptions of $4,800 (the base exemption for a joint filer). Since this remaining taxable income of $5,021 is below New Hampshire's income tax filing requirement threshold, no state income tax liability was assessed. All other states in the computation tree returned $0 because the household resides in New Hampshire, which is the only state with jurisdiction to tax this household's income.", @@ -827,7 +827,7 @@ us,scenario_109,child3_wic_eligible,1.0,1,"PolicyEngine determined that the thir us,scenario_049,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this household in 2026. This result reflects that the household, consisting of two adults with no children and a household income of approximately $262,821, does not qualify for TANF assistance. TANF programs are primarily designed to provide cash assistance to families with dependent children, and eligibility is further restricted by income and asset limits that this household substantially exceeds. With no qualifying dependents and income far above state-specific thresholds, the household receives no TANF benefit.", us,scenario_070,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. CHIP eligibility in Illinois requires applicants to meet both age and income thresholds, and this 57-year-old individual fails CHIP's age criterion, as the program is designed for children and young adults rather than adults in their late fifties. Additionally, with a household income of approximately $38,100, the person does not meet CHIP's income requirements for the limited adult coverage categories that do exist in some states. The combination of exceeding the age limit and failing to qualify under income-based criteria resulted in the chip_category classification of 3, which corresponds to ineligibility for the program.", us,scenario_088,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Texas household in 2026 because the household had no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $24,035 in total income did not include self-employment earnings, no self-employment tax liability was generated. The calculation correctly reflects that self-employment tax applies exclusively to individuals operating their own businesses or working as independent contractors, which was not the case for this household.", -us,scenario_030,snap,287.68316650390625,363,"PolicyEngine calculated a monthly SNAP benefit of $287.68 for this Texas household by aggregating benefits across multiple months in 2026. The household qualifies for SNAP because it meets all eligibility requirements: it passes the asset test with $190 in bank account assets, qualifies categorically through TANF non-cash eligibility (with a gross income ratio of 1.6 and net income ratio of 1.27 relative to the federal poverty guideline), satisfies work requirements, and has eligible immigration status. For most months, the household receives the minimum allotment of $23.84 per month for a single-person household in the contiguous US, calculated as the difference between the maximum allotment of $298 and the expected contribution of $497.10 (which exceeds the maximum, triggering the minimum benefit). In later months, the benefit increases slightly to $24.37 as the maximum allotment rises to $304.68 and the expected contribution decreases to $495.60, reflecting minor changes in the household's net income of approximately $1,653–$1,658 throughout the year.", +us,scenario_030,snap,288.0,363,"PolicyEngine computes an annual 2026 SNAP total of $288 for this one-person Texas household, the sum of twelve monthly allotments of $24. The household is eligible every month. Its $190 in bank assets pass the asset test, and it is categorically eligible through TANF non-cash eligibility. Monthly gross income is $2,083.33, including $1,083.33 of earned income, and monthly net income is $1,658. Gross and net income are 1.6 and 1.27 times the $1,304.17 poverty guideline from January through September, and 1.57 and 1.25 times $1,330 from October through December. Each month the expected contribution computed from net income is $498. That exceeds the $298 maximum allotment for a one-person household in the contiguous US, so the benefit is the $24 minimum allotment.", us,scenario_070,head_medicare_eligible,0.0,1,"PolicyEngine determined that the household head is not Medicare eligible in 2026, resulting in a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65. Since this benchmark household consists of a single adult with a household income of approximately $38,100 and no indication of qualifying disability or end-stage renal disease status, the household head does not meet the age threshold or other qualifying conditions for Medicare enrollment. Therefore, PolicyEngine correctly classified the head of household as ineligible for Medicare benefits in the 2026 tax year.", us,scenario_088,state_income_tax_before_refundable_credits,0.0,46,"PolicyEngine calculated state_income_tax_before_refundable_credits as $0.00 for this Texas resident with approximately $24,035 in household income. The computation aggregates income tax liabilities across all 50 states plus Washington D.C. and New York City, with each individual state and local jurisdiction contributing $0.00 to the total. Texas, the resident's state of residence, has no state income tax, which is the primary reason for the zero value. All other state income tax variables—including those for high-tax states like California, New York, and Massachusetts—also returned $0.00 because the household is only subject to taxation in Texas. The final state_income_tax_before_refundable_credits value of $0.00 reflects the sum of these zero components across all jurisdictions.", us,scenario_049,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this New Hampshire household is eligible for WIC (Women, Infants, and Children benefits), resulting in a value of False for both household members. WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this household consists of two adults with no children. Since the household composition does not include any individuals in the eligible age or demographic categories, PolicyEngine correctly computed is_wic_eligible as [False, False], yielding a reference value of 0.0.", @@ -901,7 +901,7 @@ us,scenario_071,head_wic_eligible,0.0,13,"PolicyEngine determined that this hous us,scenario_109,child2_medicare_eligible,0.0,1,"PolicyEngine determined that child 2 is not Medicare eligible in this 2026 Florida household, returning a value of False (0.0). Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and child 2 in this benchmark household is well below that threshold as a dependent child in a family with three children. The computation returned an array of five False values across all household members, indicating that none of the five individuals—the two adults and three children—met Medicare eligibility criteria for the tax year. This result is consistent with the household structure, where only the adult members could potentially approach Medicare eligibility age, and they have not yet reached 65.", us,scenario_013,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this Arizona household as $0. The computation shows that the employee_payroll_tax component, which represents Social Security and Medicare taxes withheld from wages, evaluated to $0. Since the household member is identified as the tax unit head (is_tax_unit_head = True), this individual's payroll tax obligation was assessed directly. With a household income of approximately $30,472, the employee payroll tax calculation resulted in no tax liability, meaning either no wages subject to payroll tax were reported or the income fell below applicable thresholds for this tax year.", us,scenario_089,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household of two adults with no children and approximately $115,866 in household income. The computation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. This outcome reflects that the household's income level and family composition (no qualifying children and no education expenses claimed) place them outside the eligibility thresholds for these refundable credit programs in 2026.", -us,scenario_051,state_income_tax_before_refundable_credits,814.9500122070312,56,"For this Louisiana resident with approximately $40,000 in household income, PolicyEngine calculated a state income tax liability of $814.95 before refundable credits. The calculation began with an adjusted gross income (AGI) of $40,000, from which Louisiana's standard deduction of $12,835 was subtracted to arrive at a taxable income of $27,165. This taxable income was then subject to Louisiana's tax rate schedule, resulting in the final tax liability of $814.95. All other states' income tax calculations returned zero, confirming that only Louisiana's tax applied to this household.", +us,scenario_051,state_income_tax_before_refundable_credits,820.3499755859375,56,"PolicyEngine computes state income tax before refundable credits of $820.35 for this single 25-year-old filer living in Louisiana. The engine evaluates each state's tax, and only Louisiana's is nonzero; every other state's line in the trace is zero. Louisiana adjusted gross income equals federal adjusted gross income of $40,180, which matches the head's $40,000 of employment income plus $180 of capital gains. Subtracting the Louisiana standard deduction of $12,835 leaves Louisiana taxable income of $27,345. Louisiana tax before nonrefundable credits on that income is $820.35, equal to 3 percent of taxable income, and Louisiana tax before refundable credits is the same $820.35, which becomes the state total.", us,scenario_031,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this California resident with approximately $23,853 in household income. The computation aggregates refundable tax credits across all 50 states and select localities, with ca_refundable_credits—the California-specific component—evaluating to $0 since the household resides in California. All other state-level refundable credit variables (al_refundable_credits, ar_refundable_credits, az_refundable_credits, and so forth across all remaining states and NYC) also returned $0, as the household is not subject to those jurisdictions' tax regimes. The sum of these zero values across all state and local components yields the final result of $0 in total state refundable credits for tax year 2026.", us,scenario_031,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this California resident with approximately $23,853 in household income during the 2026 tax year. California does not impose a local income tax at the municipal or county level; instead, the state relies on state income tax and sales taxes for revenue. Since there is no applicable local income tax jurisdiction in California, the local_income_tax variable correctly returns zero regardless of the household's income level. This result reflects the actual tax code structure rather than any exemption or deduction applied to the household's earnings.", us,scenario_109,child1_head_start_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for Head Start (is_head_start_eligible = False) based on the household's income level relative to the program's eligibility thresholds. Head Start eligibility in Florida is primarily determined by whether household income falls at or below 100% of the federal poverty line, or meets other categorical eligibility criteria. With a household income of approximately $30,000 for a family of five (2 adults and 3 children), this household's income-to-poverty ratio exceeded the standard eligibility threshold, resulting in ineligibility for the program. The computation returned False across all five household members, indicating that none of the children qualified for Head Start enrollment under the 2026 policy parameters.", @@ -929,7 +929,7 @@ us,scenario_031,head_wic_eligible,0.0,25,"PolicyEngine determined that this 67-y us,scenario_071,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this New York household is not eligible for free school meals, resulting in a value of 0.0. Although New York has a universal free school meals program, the household's income of $36,542 places it at 2.29 times the federal poverty guideline (which is $15,960 for a single-person household), exceeding the income threshold for free meal eligibility. The household does not meet categorical eligibility through programs like SNAP or TANF, which would otherwise qualify it regardless of income. Consequently, the school_meal_tier was assigned a value of 0, and the school_meal_net_subsidy was calculated as 0, indicating no subsidy benefit for this household.", us,scenario_109,child2_early_head_start_eligible,0.0,1,"PolicyEngine determined that child 2 is not eligible for Early Head Start, returning a value of False (0.0) for the benchmark household. Early Head Start eligibility is primarily determined by age and income thresholds; the program serves infants and toddlers under age 3 in households meeting federal poverty guidelines or other specified income limits. For this Florida household with approximately $30,000 in annual income and three children, child 2 did not meet the program's eligibility criteria—most likely because the child's age exceeded the Early Head Start age limit or the household's income exceeded the applicable threshold for the number of dependents. The array output [False, False, False, False, True] indicates that only the fifth household member (likely the youngest child) qualified for the program, suggesting that age was the binding constraint for the older children in this household.", us,scenario_051,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this 25-year-old single adult in Louisiana for tax year 2026. SSI eligibility requires that a household member meet at least one of three criteria: being aged 65 or older, blind, or disabled. The head of household in this case is age 25.0 and does not meet the aged, blind, or disabled criterion under the engine's inputs, making them ineligible for SSI. Although the household has an income of approximately $40,000, the $0 benefit results from the failure to satisfy SSI's categorical eligibility requirement, not from income limits. PolicyEngine determined that even though the household indicated it would take up SSI if eligible (takes_up_ssi_if_eligible = True), no SSI payment was due because no household member qualified under the program's eligibility rules.", -us,scenario_013,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this single adult in Arizona with a household income of approximately $30,472 in 2026. Although the household's income level would typically fall within the eligibility range for SNAP benefits, the calculation shows that the household is not eligible for any benefit amount. The model determined that `takes_up_snap_if_eligible` is True, indicating the household would participate in the program if eligible, but the income threshold or other eligibility criteria resulted in a benefit of zero dollars. This outcome reflects that the household's income exceeds the net income limit after standard deductions are applied, placing them above the threshold at which SNAP benefits phase out completely.", +us,scenario_013,snap,240.0,418,"PolicyEngine calculated SNAP benefits of $240 for 2026 for this 80-year-old disabled Arizonan who lives alone: $0 in January and February and the $24 monthly minimum from March through December. The household's gross income is $2,539.33 a month. Under SNAP's ordinary rules it fails the net income test, with net income of $2,138 a month, and the asset test, with $58,700 in the bank. From March 2026 Arizona extends SNAP through expanded categorical eligibility to households with gross income up to 200% of the poverty guideline ($2,608.33 a month, $2,660 from October), which treats the income and asset tests as met. The ordinary benefit formula, the $298 maximum less 30% of net income, gives less than nothing, so the household receives the $24 minimum in each of those ten months.", us,scenario_051,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Louisiana household in 2026. The household consists of a single adult with no children and an annual income of approximately $40,000. TANF eligibility and benefit amounts are primarily determined by household composition and income level, with the program designed to serve families with dependent children. Since this household contains no children, it does not meet the basic eligibility requirements for TANF benefits, resulting in a $0 benefit amount.", us,scenario_071,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this New York household is not eligible for reduced-price school meals, resulting in a value of 0.0. The household's income of approximately $36,542 places it at 2.29 times the federal poverty guideline (school_meal_fpg_ratio), which exceeds the reduced-price eligibility threshold. However, the household qualifies for the more generous FREE tier through New York's universal free school meals program, which provides free meals to all students regardless of income or categorical eligibility status. Since the household already receives the better benefit tier (FREE meals), it does not qualify for the reduced-price tier, and the reduced_price_school_meals_eligible variable is set to False with a corresponding school_meal_net_subsidy of 0.", us,scenario_089,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this North Carolina household in 2026. Although the model determined that the household would take up SNAP if eligible (takes_up_snap_if_eligible = True), the household's annual income of approximately $115,866 substantially exceeds the income eligibility threshold for SNAP benefits. SNAP eligibility is primarily determined by gross income limits, which for a two-person household in 2026 are set well below this household's income level. As a result, despite the household's willingness to participate in the program, they do not qualify for any SNAP benefits, yielding a benefit amount of $0.", @@ -1031,7 +1031,7 @@ us,scenario_032,child1_wic_eligible,0.0,1,"PolicyEngine determined that child1_w us,scenario_110,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this Ohio household is not eligible for free school meals, resulting in a value of 0. The household's school_meal_countable_income of $137,949.12—derived from employment income of $100,000 plus investment income of $37,949.12—yields a school_meal_fpg_ratio of 8.64, meaning the household's income is 8.64 times the federal poverty guideline of $15,960 for a single-person household. Since Ohio does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, it must meet the income threshold to qualify for reduced-price or free meals. With an income far exceeding the 185% reduced-price limit (which would be approximately $29,526), the household is classified in school_meal_tier 2 and therefore pays the full price for school meals, resulting in a school_meal_net_subsidy of 0.", us,scenario_072,spouse_medicare_eligible,1.0,1,"PolicyEngine determined that both spouses in this Michigan household are Medicare-eligible, resulting in a value of [True, True] for the 2026 tax year. Medicare eligibility is primarily determined by age, with individuals becoming eligible at age 65 under standard rules. Given that this is a joint household with two adults and a household income of approximately $73,974, PolicyEngine's computation indicates that both members of the couple have reached or exceeded the Medicare eligibility threshold. The resulting array value of [True, True] reflects that each spouse individually meets the criteria for Medicare enrollment, which affects their eligibility for related benefits and tax treatment in the 2026 tax year.", us,scenario_014,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this West Virginia household in 2026. This result reflects that the household, with a joint income of approximately $183,584 and no children, does not qualify for TANF assistance. TANF is primarily designed to provide cash assistance to families with dependent children, and since this benchmark household has zero children, they are categorically ineligible for the program regardless of their income level. Consequently, PolicyEngine's computation trace shows tanf = 0 with no intermediate calculations, as the eligibility criteria are not met.", -us,scenario_000,federal_income_tax_before_refundable_credits,2883.49365234375,40,"PolicyEngine calculated federal income tax before refundable credits of $2,883.49 for this Texas single filer by first determining adjusted gross income of $50,437.11, which included $45 in taxable interest income, $24,000 in taxable pension income, $18,542.70 in taxable Social Security benefits, and $7,849.41 in taxable retirement distributions. The household's taxable income was reduced to $26,095.78 after applying $24,341.33 in deductions, which consisted of $18,341.33 in itemized deductions (primarily state and local tax deductions) plus a $6,000 additional senior deduction, as the filer qualified for an extra standard deduction due to age. Applying 2026 tax brackets to the taxable income of $26,095.78 resulted in an income tax liability of $2,883.49, which became the final value since no net investment income tax, credits, or other adjustments applied.", +us,scenario_000,federal_income_tax_before_refundable_credits,2906.45361328125,26,"PolicyEngine calculated federal income tax before refundable credits of $2,906.45 for this Texas single filer aged 65 or older. Adjusted gross income is $50,437.11: $24,000 of taxable private pension income, $18,542.70 of taxable Social Security, $7,849.41 of taxable retirement distributions and $45 of interest. Itemized deductions come to $17,587.20, with the general sales tax deduction taken from the IRS's 2025 optional sales tax table (the last edition published before the benchmark's 2026-07-03 reference freeze) plus a local component of 20% of it, so the filer takes the larger $18,150 standard deduction; with the $6,000 senior deduction, deductions total $24,150 and taxable income is $26,287.11. The federal rate schedule applied to taxable income gives $2,906.45.", us,scenario_052,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False. WIC eligibility requires that a household meet income thresholds set at 185% of the federal poverty line, and this household's income of approximately $478,826 substantially exceeds that limit. Additionally, WIC is a program designed to serve pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this benchmark household contains only two adults with no children, making it ineligible on demographic grounds as well. The combination of excess income and lack of qualifying household members resulted in the False eligibility determination.", us,scenario_090,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Kansas resident with approximately $7,020 in household income. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with each state-level credit (ks_refundable_credits, al_refundable_credits, ar_refundable_credits, and so on) evaluating to $0. Since the household is located in Kansas, only ks_refundable_credits would be applicable, which also computed to $0. The final state_refundable_credits value of $0 reflects that this household does not qualify for any refundable state tax credits in Kansas for the 2026 tax year.", us,scenario_110,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Ohio household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $175,142 places it at 8.64 times the federal poverty guideline, far exceeding the 185% federal poverty level threshold required for reduced-price meal eligibility. Since the state does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is assigned to the PAID school_meal_tier and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", @@ -1097,7 +1097,7 @@ us,scenario_090,free_school_meals_eligible,0.0,15,"PolicyEngine determined that us,scenario_090,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Kansas household is not eligible for reduced-price school meals (reduced_price_school_meals_eligible = 0.0) because the household qualifies for the superior free meals tier instead. With a household income of approximately $7,020 and a school_meal_fpg_ratio of 0.74, the household's income falls at 74% of the federal poverty guideline, which is well below the 130% threshold required for free meals. Since the household meets the income criterion for free school meals through the free tier (school_meal_tier = FREE), it does not qualify for the reduced-price tier, which serves households with income between 130% and 185% of the poverty guideline. The school_meal_net_subsidy of 0 reflects that the household receives the full free meal benefit rather than a partial subsidy.", us,scenario_111,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Washington State household in 2026. TANF is a needs-based program primarily designed to assist families with dependent children, and this benchmark household has zero children. Since the household does not meet the basic eligibility requirement of having dependent children in the home, PolicyEngine determined no TANF benefit applies regardless of the household's income level of approximately $43,640. The program's structure therefore results in a zero benefit for this household composition.", us,scenario_000,head_wic_eligible,0.0,16,"PolicyEngine determined that this 77-year-old single adult in Texas is not eligible for WIC (Women, Infants, and Children) benefits, despite meeting the nutritional risk criterion. The household's WIC countable income of $111,568.82—derived from interest income ($5,623.41), social security ($26,096), pension income ($24,000), veterans benefits ($48,000), and retirement distributions ($7,849.41)—far exceeds the WIC income limit of $29,526 for a single-person household in the contiguous United States. Additionally, the applicant does not meet categorical eligibility requirements, as WIC category 5 (which applies to individuals aged 77) is not an eligible category for the program. Since the household fails both the income test and categorical eligibility test, the overall WIC eligibility determination is False.", -us,scenario_073,snap,287.68316650390625,473,"PolicyEngine calculated a monthly SNAP benefit of $287.68 for this single-adult Michigan household with approximately $25,754 annual income by determining eligibility through multiple pathways and then computing the benefit amount. The household qualifies for SNAP because it meets the gross income test (with a ratio of 1.65 times the federal poverty guideline of $1,304.17), passes the asset test with $3,600 in combined bank and stock assets, and satisfies categorical eligibility as a TANF non-cash eligible household. The benefit calculation uses the standard SNAP formula: the maximum allotment of $298 for a single-person household in the contiguous US minus the expected contribution of $526.50 (derived from the household's net income of $1,755.47), which yields a minimum allotment of $23.84 per month. Since the household is assumed to take up SNAP if eligible, PolicyEngine applies this minimum allotment across multiple months in the tax year, with most months receiving $23.84 and some receiving $24.37 as the federal poverty guideline adjusts, resulting in the annual total of $287.68.", +us,scenario_073,snap,288.0,473,"PolicyEngine computes an annual 2026 SNAP total of $288 for this single-adult Michigan household, the sum of twelve monthly benefits of $24. In every month the household is eligible: it passes the gross income test, its $3,600 in assets ($1,800 in a bank account and $1,800 in stock) pass the asset test, it has an elderly or disabled member, and it is categorically eligible through Michigan's TANF non-cash rules. Monthly gross income is $2,146.13 and net income is $1,755, giving an expected contribution of $527, which exceeds the $298 maximum allotment for one person in the contiguous US. The benefit therefore falls to the $24 minimum allotment. The poverty guideline is $1,304.17 from January through September and $1,330 from October through December, moving the gross income ratio from 1.65 to 1.61 without changing the $24 benefit.", us,scenario_014,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this West Virginia household is not eligible for reduced-price school meals, resulting in a value of 0.0. The household's income of approximately $183,584 places it at 8.48 times the federal poverty guideline, far exceeding the 185% federal poverty level threshold required for reduced-price meal eligibility. Since the state does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is classified in the PAID school_meal_tier and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", us,scenario_015,federal_income_tax_before_refundable_credits,2969.1259765625,32,"PolicyEngine calculated federal income tax before refundable credits of $2,969.13 for this Indiana single filer by applying the standard 2026 tax rate schedule to a taxable income of $26,809.38. The taxable income was derived by starting with IRS gross income of $45,958.07 (which included $45,000 in employment income, $2,000 in taxable interest income, and a $4,510 long-term capital loss), subtracting above-the-line deductions of $3,048.69 (including $1,041.93 in pre-tax 401(k) contributions and $3,000 in capital loss deductions), and then subtracting the standard deduction of $16,100. The net investment income tax and other additional tax components contributed zero to the final liability, as the household's net investment income was negative at $1,000 and no non-refundable credits were available to reduce the tax.", us,scenario_053,payroll_tax,5123.10400390625,5,"For this single adult in Idaho with approximately $66,969 in household income, PolicyEngine calculated a total payroll tax (spm_unit_payroll_tax) of $5,123.10 by summing two components of the employee_payroll_tax. The employee_social_security_tax portion was $4,152.06, calculated by applying the 6.2% Social Security tax rate to the applicable wage base. The employee_medicare_tax portion was $971.05, derived from the 2.45% Medicare tax rate applied to total wages. Since this individual is identified as the tax unit head (is_tax_unit_head = True), these employee payroll tax obligations represent the full payroll tax liability for the household.", @@ -1115,7 +1115,7 @@ us,scenario_053,state_income_tax_before_refundable_credits,2435.2783203125,63,"P us,scenario_000,head_medicare_eligible,1.0,1,"PolicyEngine determined that the household head is Medicare eligible (value = 1.0, or True) for the 2026 tax year. This determination is based on the household's demographic characteristics and age status as of the relevant tax year. The single adult in this Texas household meets the eligibility criteria established by Medicare rules, which primarily depend on age (typically 65 or older) and citizenship or permanent residency status. With a household income of approximately $111,569, income does not affect Medicare eligibility, as Medicare is a universal program for qualifying individuals regardless of earnings. Therefore, PolicyEngine assigned a value of True to indicate that the head of household qualifies for Medicare benefits in 2026.", us,scenario_091,federal_refundable_credits,0.0,6,"For this single adult in Wisconsin with approximately $35,982 in household income and no children, PolicyEngine calculated federal refundable credits of $0. The calculation examined five potential refundable credit components: the Earned Income Tax Credit (eitc), the refundable portion of the American Opportunity Credit (refundable_american_opportunity_credit), the refundable Child Tax Credit (refundable_ctc), the recovery rebate credit (recovery_rebate_credit), and the refundable payroll tax credit (refundable_payroll_tax_credit). Each of these components evaluated to $0, resulting in a total refundable credits amount of $0. This outcome reflects that the household does not qualify for any of these refundable credits under 2026 tax law—most notably, the EITC requires either dependent children or to fall within specific age and income ranges for childless workers, and the refundable CTC requires dependent children, neither of which apply to this single adult household.", us,scenario_015,payroll_tax,3442.5,5,"PolicyEngine calculated a payroll tax of $3,442.50 for this single Indiana resident with approximately $43,450 in household income. The total payroll tax consists of two components: employee Social Security tax of $2,790 and employee Medicare tax of $652.50. The Social Security tax was computed by applying the 6.2% employee rate to earnings up to the 2026 Social Security wage base, while the Medicare tax was calculated by applying the 1.45% employee rate to total wages. Since this household member is identified as a tax unit head, they are subject to the standard employee payroll tax obligations without any special exemptions or adjustments.", -us,scenario_033,federal_income_tax_before_refundable_credits,3788.988037109375,33,"For this South Dakota joint filer household in 2026, PolicyEngine calculated federal income tax before refundable credits of $3,788.99 by applying 2026 tax rates to a taxable income of $35,708.23. The taxable income was derived from an adjusted gross income of $83,208.23 (which included $96,095 in IRS gross income from employment, self-employment, interest, and Social Security, reduced by $12,886.76 in above-the-line deductions) minus $47,500 in standard and senior deductions. The household's IRS gross income of $96,095 reflected $79,000 in employment income, $16,320 in taxable Social Security benefits, $775 in taxable interest income, and a $11,400 self-employment loss deduction. No additional taxes or credits adjusted this amount, resulting in the final tax liability of $3,788.99.", +us,scenario_033,federal_income_tax_before_refundable_credits,3818.148193359375,34,"PolicyEngine calculated federal income tax before refundable credits of $3,818.15 by applying 2026 tax rates to the household's taxable income of $35,951.23. The taxable income was derived from an adjusted gross income of $83,451.23, which included $96,338 in IRS gross income (comprising $79,000 in employment income, $16,320 in taxable Social Security benefits, $775 in taxable interest income, and $243 in state and local tax refund income) reduced by $12,886.76 in above-the-line deductions (including an $11,400 self-employment loss and $1,486.76 in health savings account contributions). The household then applied the standard deduction plus senior deduction totaling $47,500, resulting in the taxable income figure. No investment income tax, credits, or other adjustments applied, yielding the final tax liability of $3,818.15.", us,scenario_073,head_wic_eligible,0.0,14,"PolicyEngine determined that this 57-year-old Michigan resident is not eligible for WIC (Women, Infants, and Children) benefits, despite meeting multiple eligibility criteria. The household's countable income of $25,753.59, derived from social security ($22,645.59) and pension income ($3,108), falls below the WIC income limit of $29,526 for a single-person household in the contiguous United States, satisfying the income test. The individual also meets categorical eligibility requirements and demonstrates nutritional risk status. However, WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5—categories defined by the wic_category variable. At age 57, this household member falls outside all qualifying age and life-stage categories, making them ineligible for WIC regardless of passing the income and nutritional risk assessments.", us,scenario_015,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Indiana household because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household does not report any such income, no self-employment tax liability is generated. The calculation correctly reflects that self-employment tax applies exclusively to individuals who operate their own business or are self-employed, which does not apply to this household's income profile of approximately $43,450.", us,scenario_091,payroll_tax,2065.5,5,"PolicyEngine calculated a payroll tax liability of $2,065.50 for this single Wisconsin resident with approximately $35,982 in household income. The total payroll tax consists of two components: employee Social Security tax of $1,674.00 and employee Medicare tax of $391.50. The Social Security tax was computed by applying the 6.2% employee rate to the applicable wage base, while the Medicare tax was calculated at the 1.45% employee rate on total wages. Since this individual is classified as a tax unit head, they are subject to the standard employee payroll tax obligations without any special exemptions or adjustments.", @@ -1124,7 +1124,7 @@ us,scenario_033,federal_refundable_credits,0.0,6,"PolicyEngine calculated federa us,scenario_091,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Wisconsin household because the household has no self-employment income to tax. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $35,982 in total income derives from other sources (such as wages or salaries), there is no self-employment income base upon which to apply the 15.3% self-employment tax rate. Consequently, PolicyEngine's computation trace shows self_employment_tax = 0, reflecting the absence of any self-employment activity subject to Social Security and Medicare taxes.", us,scenario_053,state_refundable_credits,155.0,76,"For this Idaho resident in 2026, PolicyEngine calculated total state refundable credits of $155. This amount comes entirely from Idaho's grocery credit (id_grocery_credit), which contributed the full $155. The grocery credit calculation determined that the household qualified for all 12 months of the tax year, with each month meeting the qualifying criteria. The base grocery credit amount (id_grocery_credit_base) was set at $155, and since the household qualified for the full year with no phase-out or reduction applied, this base amount became the final credit value. All other states' refundable credits returned zero, as the household resides in Idaho.", us,scenario_015,state_income_tax_before_refundable_credits,1236.3267822265625,57,"PolicyEngine calculated Indiana state income tax before refundable credits of $1,236.33 for this single filer in 2026. The calculation began with the household's adjusted gross income of $42,909.38, from which Indiana's base exemption of $1,000 was subtracted to arrive at an Indiana AGI of $41,909.38. This taxable income amount was then subject to Indiana's tax rate schedule, resulting in the final tax liability of $1,236.33. All other states' income tax calculations returned zero, confirming that only Indiana's tax applied to this resident.", -us,scenario_091,state_income_tax_before_refundable_credits,843.6614990234375,62,"For this Wisconsin resident with approximately $35,982 in household income, PolicyEngine calculated a state income tax before refundable credits of $843.66. The calculation began with an adjusted gross income (AGI) of $35,012.56, from which PolicyEngine subtracted Wisconsin's standard deduction of $12,067.66 and a personal exemption of $700, resulting in a taxable income of $22,244.90. This taxable income was then subject to Wisconsin's 2026 tax brackets and rates, yielding the final tax liability of $843.66 before any refundable credits could be applied.", +us,scenario_091,state_income_tax_before_refundable_credits,843.6614990234375,73,"PolicyEngine calculated Wisconsin state income tax before refundable credits of $843.66 for this single Wisconsin resident. The calculation began with federal gross income of $35,055.84, composed of $26,073.84 in wages (after $926.16 in pre-tax contributions), $4,806 in taxable interest, and $4,176 in dividends, while excluding $1,170 in capital gain distributions reported without Schedule D. After applying a $43.28 traditional IRA deduction, Wisconsin adjusted gross income reached $35,012.56. Wisconsin taxable income was then determined by subtracting the Wisconsin standard deduction of $12,067.66 and the personal exemption of $700, yielding taxable income of $22,244.90. Applying Wisconsin's tax brackets to this taxable income produced the final tax liability of $843.66 before any refundable credits.", us,scenario_000,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $111,569 places it at 6.99 times the federal poverty guideline, which exceeds the 185% income threshold required for reduced-price meal eligibility. Since Texas does not have universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is assigned to the PAID school_meal_tier and receives no subsidy (school_meal_net_subsidy = 0). Consequently, the household must pay the full price for school meals.", us,scenario_053,local_income_tax,0.0,1,"For this single adult in Idaho with a household income of approximately $66,969 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that Idaho does not impose a local income tax at the state level, and the household's residence does not appear to be subject to any municipal or county-level income taxes that would generate a tax liability. Consequently, despite the household's substantial income, no local income tax obligation was derived from PolicyEngine's computation.", us,scenario_033,payroll_tax,6043.5,5,"PolicyEngine calculated a total payroll tax of $6,043.50 for this South Dakota joint household by summing the employee_social_security_tax and employee_medicare_tax components. The employee social security tax totaled $4,898, split between the two adults as $1,612 and $3,286 respectively, reflecting their individual wage earnings. The employee medicare tax component added $1,145.50, distributed as $377 for one spouse and $768.50 for the other, proportional to their respective incomes. These two payroll tax components—social security and medicare—were combined to arrive at the final spm_unit_payroll_tax value of $6,043.50 for the tax year 2026.", @@ -1224,7 +1224,7 @@ us,scenario_092,state_income_tax_before_refundable_credits,0.0,47,"PolicyEngine us,scenario_033,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this South Dakota household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income is approximately $99,218, which translates to a school_meal_fpg_ratio of 4.05—meaning their income is 405% of the federal poverty guideline. Since this ratio far exceeds the 185% income threshold required for reduced-price meal eligibility, and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they are classified in the school_meal_tier as PAID. Consequently, the school_meal_net_subsidy is 0, and the household must pay the full price for school meals.", us,scenario_016,state_refundable_credits,0.0,45,"PolicyEngine calculated state_refundable_credits as $0 for this Florida resident by summing refundable tax credits across all 50 states and the District of Columbia. Since the household is located in Florida (fl_refundable_credits), which does not offer state-level refundable tax credits, that component contributed $0 to the total. All other state refundable credit variables—including those for high-credit states like California, Colorado, and New York—were also $0 because the household's residence in Florida means it is not subject to those states' tax systems. The final aggregation of all 43 state and local refundable credit components yielded a total of $0, reflecting that Florida has no refundable state tax credits available to this single adult filer with approximately $34,638 in household income.", us,scenario_054,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this North Carolina household in 2026 because none of the three household members—the 40-year-old head of household, the 10-year-old child, and the 2-year-old child—meet SSI's eligibility criteria. SSI requires that a person be aged 65 or older, blind, or disabled; the engine's inputs indicate that all three individuals have `is_ssi_aged_blind_disabled=False`, meaning none qualify under these categorical requirements. Although all household members have `takes_up_ssi_if_eligible=True`, indicating they would claim the benefit if eligible, the absence of any qualifying condition means no SSI payment is generated for any individual. The household's near-zero income does not trigger SSI eligibility on its own; rather, the categorical requirement is the binding constraint that results in the $0.00 total benefit.", -us,scenario_054,snap,6125.68896484375,481,"PolicyEngine calculated an annual SNAP benefit of $6,125.69 for this North Carolina head-of-household family of three by aggregating monthly benefits across the year. The household qualifies for SNAP because it meets all eligibility requirements: the gross income of $1,425 is 63-64% of the federal poverty guideline ($2,220-$2,277), the net income of $931 is 41-42% of the poverty guideline, and the household passes the categorical eligibility test through TANF non-cash assistance. The monthly benefit is calculated as the maximum allotment ($785-$803 depending on the month) minus the expected household contribution, which is 30% of net income ($279-$278), yielding a monthly allotment of approximately $505-$525. The household is assumed to take up the benefit if eligible, and the annual total of $6,125.69 represents the sum of these monthly amounts across all twelve months of 2026.", +us,scenario_054,snap,6060.0,481,"PolicyEngine computes annual SNAP of $6,060 for this three-person North Carolina household as the sum of twelve monthly allotments of $505 each, January through December. The head's $17,100 of self-employment income counts as $1,425 a month of gross earned income. Deductions total $494 a month: a $209 standard deduction plus a $285 earned income deduction, leaving net income of $931. The household passes the gross and net income tests and is categorically eligible through TANF non-cash eligibility, with gross income at 64% and net income at 42% of the monthly poverty guideline of $2,220.83 in January through September (63% and 41% of $2,276.67 in October through December) and $14,000 in assets passing the asset test. Each month the $785 maximum allotment minus a $280 expected contribution gives $505.", us,scenario_092,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Alabama resident with ~$32,380 in household income. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with al_refundable_credits—the Alabama-specific refundable credits variable—evaluated at $0 since the household resides in Alabama (AL = True). All other state-level refundable credit variables, including those for Arkansas, Arizona, California, Colorado, and the remaining states, also returned $0, indicating that either no refundable credits apply to this household's income level and filing status, or Alabama offers no refundable tax credits for single filers in this income range. The sum of these zero values across all jurisdictions yields the final state_refundable_credits amount of $0.", us,scenario_074,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this Louisiana household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_countable_income of $45,252—derived from social security ($33,640), pension income ($11,228), and retirement distributions ($384)—was compared against the federal poverty guideline threshold. With a school_meal_fpg_ratio of 2.84, the household's income exceeds 185% of the federal poverty guideline (the reduced-price meal eligibility threshold), placing them in school_meal_tier 2 (paid meals). Since Louisiana does not have universal free school meals and the household does not meet categorical eligibility through programs like SNAP or TANF, they are required to pay the full price for school meals. Consequently, the school_meal_net_subsidy is 0, confirming no free meal benefit applies.", us,scenario_036,federal_income_tax_before_refundable_credits,4926.39990234375,19,"PolicyEngine calculated the federal income tax before refundable credits as $4,926.40 for this single New Jersey resident with approximately $59,220 in household income. The calculation began with the household's IRS gross income of $59,220, which was treated entirely as taxable pension income. From this amount, PolicyEngine applied the standard deduction of $16,100, resulting in a taxable income of $43,120. The federal income tax main rates were then applied to this taxable income, yielding $4,926.40 in tax liability before any refundable credits. No additional taxes were added from net investment income tax, investment credit recapture, payroll tax, or retirement penalties, and no non-refundable credits were available to reduce the tax.", @@ -1305,7 +1305,7 @@ us,scenario_114,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance us,scenario_054,child1_head_start_eligible,0.0,1,"PolicyEngine determined that neither of the two children in this North Carolina household are eligible for Head Start in 2026, resulting in an eligibility array of [False, False, False]. Head Start eligibility is typically determined by federal income thresholds, which are generally set at or below 100% of the federal poverty line, though states may have additional criteria. Given that this household has approximately $0 in annual income, the ineligibility result suggests that PolicyEngine applied additional eligibility requirements beyond income alone—such as age restrictions (Head Start typically serves children ages 3-5), enrollment status, or other program-specific criteria that were not met by the children in this benchmark household. The three-element array structure indicates PolicyEngine evaluated eligibility for the household head and both children, with all three returning False.", us,scenario_036,free_school_meals_eligible,0.0,12,"PolicyEngine determined that this New Jersey household is not eligible for free school meals, resulting in a value of 0.0. The household's school_meal_fpg_ratio of 3.71 indicates that their countable income of $59,220 is 3.71 times the federal poverty guideline of $15,960 for their household size. Since this ratio exceeds the 185% threshold required for reduced-price meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, they are classified in school_meal_tier 2 (PAID). New Jersey does not offer universal free school meals, so no state-level program provides an alternative pathway to eligibility. Consequently, the household must pay the full price for school meals with no subsidy.", us,scenario_018,state_income_tax_before_refundable_credits,1146.05224609375,64,"PolicyEngine calculated Arizona state income tax before refundable credits of $1,146.05 for this single filer with ~$68,055 household income. The calculation began with an adjusted gross income (AGI) of $61,592.09, from which Arizona deductions of $15,750 were subtracted to arrive at taxable income of $45,842.09. The deduction amount was determined by comparing itemized deductions of $10,786.71 (which included $1,286.71 in state and local taxes and $9,500 in medical expenses) against the standard deduction of $15,750, with the standard deduction selected as the larger amount. Arizona's tax rate schedule was then applied to the taxable income of $45,842.09 to produce the final tax liability of $1,146.05 before any refundable credits.", -us,scenario_093,state_income_tax_before_refundable_credits,3389.65087890625,77,"PolicyEngine calculated Missouri state income tax before refundable credits of $3,389.65 for this joint-filing household with three adults and ~$164,156 in household income. The calculation began with IRS gross income of $115,097.88, which was reduced by above-the-line deductions of $2,689.62 to arrive at Missouri adjusted gross income of $112,408.25. From this amount, Missouri itemized deductions of $12,469.50 were subtracted—comprising the standard deduction of $32,200, Missouri's deduction for federal income taxes paid of $431.45, and allowable itemized deductions including state taxes withheld of $3,409.93—yielding Missouri taxable income of $79,776.80. The tax was then calculated on this taxable income using Missouri's tax brackets and rates, resulting in a total tax of $3,389.65 distributed across the two primary earners in the household ($1,453.04 and $1,936.62 respectively), with the third household member exempt from Missouri income tax.", +us,scenario_093,state_income_tax_before_refundable_credits,3388.24560546875,95,"PolicyEngine calculated Missouri income tax before refundable credits of $3,388.25 for this married couple filing jointly, who claim an adult dependent. The spouses' Missouri adjusted gross income is $112,408.25 ($48,955.40 and $63,452.85) after 401(k) deferrals and above-the-line deductions; the dependent's Missouri adjusted gross income is $0 in the engine's calculation, so that person owes no Missouri tax here. The couple takes the $32,200 standard deduction, larger than Missouri itemized deductions of $12,469.50, and a $431.45 deduction for federal income tax, leaving Missouri taxable income of $79,776.80, allocated between the spouses in proportion to income ($34,743.94 and $45,032.86). Missouri's 2026 brackets, as the Department of Revenue published them in November 2025, give $1,452.33 and $1,935.91, a total of $3,388.25.", us,scenario_075,head_medicare_eligible,0.0,1,"PolicyEngine determined that the household head is not Medicare eligible in 2026, resulting in a value of False (0.0). Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65. Since this benchmark household consists of a single adult with a household income of approximately $109,286 and no indication of disability or end-stage renal disease status, the household head does not meet the age threshold required for Medicare enrollment. Therefore, PolicyEngine correctly classified the head of household as ineligible for Medicare benefits in the 2026 tax year.", us,scenario_114,head_wic_eligible,0.0,17,"PolicyEngine determined that this household is not eligible for WIC benefits (is_wic_eligible = False) because it failed the income test, despite meeting the nutritional risk criteria. The household's wic_countable_income of $85,732—derived from employment income ($22,700), social security ($23,328), pension income ($30,000), retirement distributions ($9,600), and interest income ($104)—substantially exceeds the wic_income_limit of $29,526 for a single-person household in the contiguous United States. Additionally, the household does not meet categorical eligibility requirements, as the head of household is 69 years old, placing them in WIC category 5, which does not qualify for the program. Although PolicyEngine identified that the household meets nutritional risk criteria (is_wic_at_nutritional_risk = True), this alone is insufficient for WIC eligibility without satisfying both the income test and categorical requirements.", us,scenario_036,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $59,220 translates to a school_meal_fpg_ratio of 3.71, meaning the household income is 371% of the federal poverty guideline. Since this ratio substantially exceeds the 185% income threshold required for reduced-price meal eligibility, and the household does not meet categorical eligibility through programs like SNAP or TANF, the household is classified in the PAID school_meal_tier. Consequently, the school_meal_net_subsidy is 0, and the household must pay the full price for school meals.", @@ -1352,7 +1352,7 @@ us,scenario_114,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determin us,scenario_115,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household with approximately $23,442 in annual income. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. For a single adult with no children and this income level, the household does not qualify for the EITC (which requires either dependent children or specific age/income thresholds for childless workers), the refundable CTC (which requires qualifying children), or the other refundable credits evaluated.", us,scenario_018,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (Children's Health Insurance Program) in Arizona for 2026, resulting in an is_chip_eligible value of False. The determination is based on chip_category = 3, which indicates the person fails to meet CHIP's eligibility criteria. Since CHIP is a program designed for children and teenagers, and this household consists of a single adult age 53, the person exceeds the maximum age requirement for CHIP enrollment. Additionally, with a household income of approximately $68,055, the person's income level places them outside CHIP's income-based eligibility thresholds for the applicable category. The combination of age ineligibility and income ineligibility results in the False determination.", us,scenario_002,reduced_price_school_meals_eligible,0.0,3,"This household is not eligible for reduced-price school meals, resulting in a value of 0. With a household income of approximately $134,820, the income-to-federal poverty guideline ratio is 5.12, which far exceeds the 185% threshold required for reduced-price meal eligibility. Washington State does not offer universal free school meals, and the household does not qualify for categorical eligibility through programs like SNAP or TANF. Consequently, PolicyEngine assigns the household a school_meal_tier of PAID, meaning they must pay the full price for school meals, and the school_meal_net_subsidy is 0.", -us,scenario_076,state_income_tax_before_refundable_credits,6806.78662109375,70,"PolicyEngine calculated Idaho state income tax before refundable credits of $6,806.79 for this head-of-household filer in 2026. The calculation began with adjusted gross income of $164,220, from which PolicyEngine subtracted $1,000 in household and dependent care expense deductions and applied the standard deduction of $24,150, resulting in taxable income of $138,270. An additional $800 qualified business income deduction was applied to arrive at the final taxable income figure. This taxable income was then subject to Idaho's tax rate schedule for head-of-household filers (filing status 3), producing the tax liability of $6,806.79 before any refundable credits.", +us,scenario_076,state_income_tax_before_refundable_credits,6818.34423828125,116,"PolicyEngine calculated Idaho income tax before refundable credits of $6,818.34 for this head-of-household filer. Federal adjusted gross income of $164,220 less Idaho's $1,000 household and dependent care deduction gives Idaho adjusted gross income of $163,220. Subtracting the $24,150 standard deduction and an $800 qualified business income deduction on $4,000 of qualified business income leaves Idaho taxable income of $138,270. Idaho taxes income above the head-of-household zero-rate threshold of $9,622, the 2025 amount the Idaho State Tax Commission published and the last located before the benchmark's 2026-07-03 reference freeze, at 5.3%: ($138,270 - $9,622) x 5.3% = $6,818.34.", us,scenario_037,local_income_tax,0.0,1,"For a single adult in North Carolina with a household income of approximately $41,943 in 2026, PolicyEngine calculated a local income tax of $0. This result reflects that North Carolina does not impose a local income tax at the county or municipal level—only the state income tax applies to residents' earned income. Since there is no local income tax obligation in the jurisdiction, the local_income_tax variable correctly returns zero regardless of the household's income level.", us,scenario_093,dependent1_wic_eligible,0.0,1,"PolicyEngine determined that none of the three household members are eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False, False]`. WIC eligibility requires applicants to meet specific categorical requirements—typically being pregnant, postpartum, breastfeeding, or a child under age 5—combined with income limits set at 185% of the federal poverty line. Since this household consists of 3 adults with no children and a household income of approximately $164,156, the members do not satisfy the categorical requirements for WIC participation. Therefore, PolicyEngine correctly computed that dependent1_wic_eligible equals 0.0, indicating no eligible household members.", us,scenario_115,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this Alabama household as $0. The computation shows that the employee_payroll_tax component, which represents Social Security and Medicare taxes withheld from wages, evaluated to $0. Since the household is identified as a tax unit head (is_tax_unit_head = True), this individual is treated as the primary taxpayer for payroll tax purposes. Despite the household having an income of approximately $23,442, which would typically be subject to payroll taxes, the employee_payroll_tax calculation resulted in no tax liability, leading to the final payroll tax value of $0.", @@ -1380,7 +1380,7 @@ us,scenario_115,state_refundable_credits,0.0,46,"PolicyEngine calculated state_r us,scenario_055,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Florida household in 2026. Florida does not impose a state income tax, and the local_income_tax variable, which captures local-level income taxes imposed by municipalities or counties, also resulted in $0 because Florida municipalities do not levy local income taxes on wages or salaries. Therefore, despite the household's income of approximately $157,137, no local income tax liability was generated under Florida's tax structure.", us,scenario_115,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Alabama household in 2026. Alabama does not impose a state income tax, and the household's location within Alabama does not have a local income tax obligation that would apply to wage or salary income. Given the household's annual income of approximately $23,442, no local income tax liability was generated under the applicable tax rules for the jurisdiction.", us,scenario_037,head_wic_eligible,0.0,12,"PolicyEngine determined that this household is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of False. The primary disqualifying factor is that the household fails the income test: with countable income of $41,943.38, it exceeds the WIC income limit of $29,526 for a single-person household in a contiguous US state. Additionally, the household does not meet categorical eligibility requirements, as the head of household is 44 years old and therefore outside the age ranges that qualify for WIC (which serves pregnant women, postpartum women, infants, and children). Although PolicyEngine determined the household meets nutritional risk criteria, this alone is insufficient for eligibility without satisfying either the income test or categorical requirements.", -us,scenario_020,federal_income_tax_before_refundable_credits,68056.7109375,56,"PolicyEngine calculated the federal income tax before refundable credits as $68,056.71 for this single Texas filer in 2026. Starting with gross income of $305,859 (comprising $300,000 in employment income, $4,392 in short-term capital gains, $1,435 in taxable interest, and $32 in dividends), the household reduced adjusted gross income to $300,150.19 by claiming $5,708.82 in above-the-line alimony deductions. The filer itemized deductions, claiming $17,090.34 in state and local tax deductions rather than the $16,100 standard deduction, which reduced taxable income to $283,059.84. Applying 2026 tax rates to this taxable income produced an income tax of $68,056.71, which was then increased by $222.64 in net investment income tax (calculated on the $5,859 in investment income) and $4.59 in capital gains tax, yielding the final value of $68,056.71 before refundable credits.", +us,scenario_020,federal_income_tax_before_refundable_credits,68056.7109375,56,"PolicyEngine calculated federal income tax before refundable credits of $68,056.71 for this Texas single filer. IRS gross income of $305,859 ($300,000 of wages, $4,392 of short-term capital gains, $1,435 of taxable interest and $32 of dividends) less a $5,708.82 alimony deduction gives adjusted gross income of $300,150.19. Itemized deductions of $17,090.34, all of them state and local taxes (the $13,834 of real estate taxes plus a general sales tax amount PolicyEngine projects from the IRS's 2023 optional sales tax table), exceed the $16,100 standard deduction; the $196 charitable gift falls below the 0.5%-of-AGI floor that applies to itemizers. Taxable income is $283,059.84. Tax on it is $67,834.07, including $4.59 on the qualified dividends taxed at capital gains rates, and the $222.64 net investment income tax brings the total before refundable credits to $68,056.71.", us,scenario_093,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False). The head is 51 years old, which exceeds CHIP's age limit of 18 years; CHIP is a program designed for children and young adults, and individuals over this threshold are categorically ineligible regardless of income or other factors. Additionally, the household income of approximately $164,156 substantially exceeds CHIP's income thresholds, which vary by state but are typically set at 200–400% of the federal poverty level. Since the head fails both CHIP's age requirement and income requirement, PolicyEngine correctly returned a value of False for is_chip_eligible.", us,scenario_055,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Florida household in 2026. Although the household's take-up parameter (`takes_up_snap_if_eligible`) was set to True, indicating the household would claim benefits if eligible, the household's annual income of approximately $157,137 substantially exceeds the income eligibility threshold for SNAP. Under federal SNAP rules, households must meet gross income limits (typically 130% of the federal poverty line) to qualify for the program, and this single-adult household's income places it well above that threshold. As a result, the household is ineligible for SNAP benefits regardless of its willingness to participate, yielding a benefit amount of $0.", us,scenario_003,state_income_tax_before_refundable_credits,0.0,46,"PolicyEngine calculated state_income_tax_before_refundable_credits as $0.00 for this Texas household by aggregating the income tax liabilities across all 50 states and the District of Columbia. Since the household is domiciled in Texas, which has no state income tax, tx_income_tax_before_refundable_credits contributed $0.00 to the total. All other state-level income tax variables—including those for high-tax states like California, New York, and Massachusetts—also returned $0.00 because the household's tax residency is exclusively in Texas. The summation of these 51 zero-valued state and local tax components yielded the final result of $0.00 for the aggregate state income tax liability before refundable credits.", @@ -1461,7 +1461,7 @@ us,scenario_020,head_medicare_eligible,0.0,1,"PolicyEngine determined that the h us,scenario_056,self_employment_tax,613.2224731445312,1,"For a single adult in New Jersey with approximately $1,499 in household income during tax year 2026, PolicyEngine calculated a self-employment tax liability of $613.22. This amount reflects the standard self-employment tax obligation, which is assessed on net earnings from self-employment at a combined rate of 15.3% (12.4% for Social Security and 2.9% for Medicare). The calculation applies to the portion of self-employment income that exceeds the applicable threshold and is subject to these payroll taxes. The resulting self-employment tax of $613.22 represents the household's federal obligation for this tax component in 2026.", us,scenario_003,spouse_medicare_eligible,0.0,1,"PolicyEngine determined that neither spouse in this Texas household is Medicare eligible in 2026, resulting in a value of [False, False] for the is_medicare_eligible variable. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old to qualify for the program. Since this benchmark household consists of two adults with a combined household income of approximately $197,459, PolicyEngine's computation indicates that both spouses fall below the age threshold for Medicare eligibility. Consequently, the reference value for spouse_medicare_eligible is 0.0, reflecting that neither household member qualifies for Medicare benefits in the 2026 tax year.", us,scenario_095,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this New Jersey single adult household with approximately $29,605 in income. The computation aggregates refundable tax credits across all 50 states and select localities, with each state-level variable (al_refundable_credits, ar_refundable_credits, and so on through wv_refundable_credits) evaluated individually. Since the household resides in New Jersey, only nj_refundable_credits was potentially applicable, but this variable also evaluated to $0. The final state_refundable_credits value of $0 reflects that this household does not qualify for any refundable state tax credits under 2026 tax law.", -us,scenario_038,snap,7286.9443359375,507,"PolicyEngine calculated an annual SNAP benefit of $7,286.94 for this Louisiana household of four by determining a monthly allotment and multiplying across the tax year. The household qualifies for SNAP because it meets all eligibility requirements: gross income of $1,916.04 monthly (72% of the federal poverty guideline of $2,679.17), net income of $1,309.83 after deductions (49% of the poverty guideline), assets of $500, and categorical eligibility through TANF non-cash assistance. The monthly benefit is calculated as the maximum allotment of $994 (for a four-person household in the contiguous US) minus the household's expected contribution of $392.70, which is 30% of net income, yielding a monthly allotment of $601.30. Multiplying this monthly amount by 12 months produces the annual benefit of $7,286.94, assuming the household takes up the benefit if eligible.", +us,scenario_038,snap,7212.0,507,"PolicyEngine computes annual SNAP of $7,212 for this Louisiana household of four as the sum of twelve monthly allotments of $601, January through December 2026. Monthly gross income is $1,916.04, all of it earned. Subtracting deductions of $606.21, a $223 standard deduction plus a $383.21 earned income deduction (20% of earnings), leaves $1,309.83, which rounds to net income of $1,310. The household passes the gross income, net income and asset tests, with $500 in assets, and is categorically eligible through TANF non-cash eligibility. Against a poverty guideline of $2,679.17 in January through September and $2,750 in October through December, gross income is 72% then 70% of the guideline and net income 49% then 48%. The expected contribution is 30% of net income, $393, which subtracted from the $994 maximum allotment leaves $601 each month.", us,scenario_020,free_school_meals_eligible,0.0,16,"PolicyEngine determined that this Texas household is not eligible for free school meals, resulting in a value of 0.0. The household's countable income for school meal purposes totaled $301,467, derived from employment income of $300,000 plus investment income of $1,467, which when compared to the federal poverty guideline of $15,960 for a single-person household yields a school_meal_fpg_ratio of 18.89. This ratio places the household in school_meal_tier 2 (paid meals), as the income far exceeds the 185% reduced-price eligibility threshold of approximately $29,526. Since Texas does not have universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household must pay the full price for school meals, resulting in a net subsidy of zero.", us,scenario_038,ssi,0.0,25,"PolicyEngine calculated SSI (Supplemental Security Income) as $0.00 for this Louisiana household in 2026 because none of the four household members meet SSI's eligibility criteria. The head of household (age 25), spouse (age 18), and both children (ages 7 and 6) all have is_ssi_aged_blind_disabled set to False, meaning none qualify under the program's requirement that beneficiaries be aged 65 or older, blind, or disabled. Although all household members indicated willingness to take up SSI if eligible (takes_up_ssi_if_eligible = True for each), the absence of any qualifying condition under the engine's inputs resulted in zero SSI benefits across all four individuals. The household's combined income of approximately $22,992 is therefore not the limiting factor; rather, the lack of any aged, blind, or disabled household member is the sole reason for the $0.00 result.", us,scenario_076,child2_medicare_eligible,0.0,1,"PolicyEngine determined that child2_medicare_eligible is False (represented as 0.0) for the second child in this household. Medicare eligibility is primarily determined by age, requiring individuals to be at least 65 years old, and this household's second child does not meet that threshold. The computation returned a three-element boolean array [False, False, False] representing the Medicare eligibility status for each household member—the head of household and both children—all of whom fall below the age requirement for Medicare coverage in the 2026 tax year.", @@ -1571,7 +1571,7 @@ us,scenario_057,state_income_tax_before_refundable_credits,0.0,47,"PolicyEngine us,scenario_098,local_income_tax,0.0,1,"For a single adult in Illinois with a household income of approximately $35,035 in 2026, PolicyEngine calculated a local income tax of $0. Illinois does not impose a local income tax on residents; instead, the state relies on state income tax and other revenue sources for funding. Since there is no local income tax liability in Illinois regardless of income level, the local_income_tax variable correctly returns zero for this household.", us,scenario_098,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this Illinois household as $0. The computation shows that the employee_payroll_tax component, which represents Social Security and Medicare taxes withheld from wages, evaluated to $0. Since the household member is identified as the tax unit head (is_tax_unit_head = True), this individual's payroll tax obligation was assessed directly. Despite the household having approximately $35,035 in income, the employee_payroll_tax returned zero, indicating that either no wages subject to payroll tax were present in the income composition or the income was structured in a way that exempted it from standard payroll tax withholding.", us,scenario_057,state_refundable_credits,33.20000076293945,49,"PolicyEngine calculated state refundable credits of $33.20 for this Louisiana household by aggregating refundable tax credits across all 50 states and DC. Since the household is located in Louisiana (LA = True), only the Louisiana refundable credits component was non-zero. Louisiana's refundable credits of $33.20 derive entirely from the Louisiana Earned Income Tax Credit (EITC), which is calculated as 5% of the federal EITC. With a federal EITC of $664, the state EITC equals $33.20 (5% × $664), and this amount comprises the household's total state refundable credits for the 2026 tax year.", -us,scenario_117,federal_income_tax_before_refundable_credits,24391.796875,44,"PolicyEngine calculated federal income tax before refundable credits of $24,391.80 by starting with a tax-before-credits amount of $30,991.80 and subtracting $6,600 in non-refundable credits. The tax-before-credits figure was derived by applying 2026 tax rates to a taxable income of $188,944.53, which was computed from an adjusted gross income of $248,144.53 reduced by $59,200 in deductions (combining the standard deduction of $32,200, an overtime income deduction of $25,000, and a charitable deduction of $2,000). The household's adjusted gross income of $248,144.53 came from employment income of $248,675 offset by a $530.47 capital loss deduction. The $6,600 non-refundable credit represents the child tax credit for the three dependent children in the household, calculated at $2,200 per child.", +us,scenario_117,federal_income_tax_before_refundable_credits,24961.33984375,45,"PolicyEngine calculated federal income tax before refundable credits of $24,961.34 for this Arkansas joint filer household by starting with an IRS gross income of $251,263.83, which included $248,675 in employment income, a $2,588.82 state and local tax refund, and a $530.47 capital loss. After applying $530.47 in above-the-line deductions for the capital loss and $59,200 in taxable income deductions (the standard deduction of $32,200 plus a $25,000 overtime income deduction and $2,000 charitable deduction), the household's taxable income was $191,533.36. Applying 2026 tax rates to this taxable income yielded $31,561.34 in income tax before credits. PolicyEngine then subtracted $6,600 in non-refundable child tax credits (the maximum of $2,200 per child for three qualifying children), resulting in income tax before refundable credits of $24,961.34.", us,scenario_001,federal_income_tax_before_refundable_credits,0.0,9,"PolicyEngine calculated federal income tax before refundable credits as $0 for this Virginia joint-filing household with approximately $44,289 in household income. The computation shows that income_tax_before_credits, the primary component, equals $0, indicating that the household's tax liability before any credits is eliminated entirely. With an adjusted gross income of $31,402.21, the household falls below the standard deduction threshold for joint filers in 2026, resulting in no taxable income. The other components—net investment income tax, recapture of investment credit, unreported payroll tax, and qualified retirement penalty—all contribute $0, as none of these additional tax items apply to this household. Therefore, the final value of income_tax_before_refundable_credits is $0.", us,scenario_021,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals, resulting in a value of 0.0. Although the household's income of approximately $27,508 places it at 1.72 times the federal poverty guideline—within the income threshold for reduced-price meals—the household has no children and therefore no school-age dependents who could benefit from the program. The engine correctly classified the school_meal_tier as REDUCED based on the income calculation, but since there are no eligible children in the household to receive meals, the school_meal_net_subsidy is 0, yielding a final eligibility value of 0.", us,scenario_038,child2_medicare_eligible,0.0,1,"PolicyEngine determined that the second child in this Louisiana household is not Medicare eligible, returning a value of False (represented as 0.0). Medicare eligibility is primarily determined by age, requiring individuals to be 65 years or older, and this household's second child does not meet that threshold. The computation returned an array of four boolean values—one for each household member—with all values set to False, indicating that none of the household members, including the second child, qualify for Medicare based on the eligibility criteria evaluated for the 2026 tax year.", @@ -1584,7 +1584,7 @@ us,scenario_001,federal_refundable_credits,0.0,6,"For this Virginia household of us,scenario_098,ssi,0.0,25,"For the 80-year-old head of household in Illinois, PolicyEngine determined an SSI benefit of $0.00 for tax year 2026. Although the individual meets the categorical requirement for SSI as aged (is_ssi_aged_blind_disabled=True), the household's income of approximately $35,035 exceeds the SSI income limits, resulting in ineligibility for benefits. The computation trace shows that takes_up_ssi_if_eligible evaluated to True, indicating the household would claim SSI if eligible, but the income test prevented any benefit from being awarded. Since SSI is calculated on an individual basis rather than as a pooled household total, this person's own income rendered them ineligible despite meeting the age and disability status requirements.", us,scenario_117,payroll_tax,15044.787109375,5,"PolicyEngine calculated the household's payroll tax liability at $15,044.79 by summing the employee payroll taxes for the tax unit. The calculation breaks down into two components: employee social security tax of $11,439 and employee Medicare tax of $3,605.79. These amounts were computed based on the household's wage income of approximately $250,733, with the social security tax applied at 6.2% up to the annual wage base limit and the Medicare tax applied at 1.45% on all wages. The computation identified one tax unit head among the five household members (the primary adult), and the payroll taxes were attributed entirely to the second household member, indicating that this individual earned the reported wages subject to these employment taxes.", us,scenario_001,payroll_tax,2223.393798828125,5,"PolicyEngine calculated the household's payroll tax liability at $2,223.39 for tax year 2026 by computing employee payroll taxes on the tax unit head's earnings. The total employee payroll tax of $2,223.39 consists of two components: employee Social Security tax of $1,801.97 and employee Medicare tax of $421.43. These amounts were derived from the primary earner's wages (the tax unit head, marked as True in the is_tax_unit_head array), while the second household member contributed no payroll tax, indicating either no employment income or earnings below the threshold for tax liability. The calculation applies the standard 2026 payroll tax rates to the applicable wage base for each program.", -us,scenario_022,federal_income_tax_before_refundable_credits,11131.3271484375,64,"PolicyEngine calculated federal income tax before refundable credits of $11,131.33 for this single California resident by first determining IRS gross income of $109,149.16 from employment income ($52,635), taxable pension income ($30,989.41), taxable Social Security ($17,624.75), rental income ($4,550), and other sources. The household's adjusted gross income of $109,149.16 was then reduced by taxable income deductions of $34,515.86, which included itemized deductions ($29,654.81 comprising charitable deductions of $14,847.67 and SALT deductions of $14,807.14) plus a qualified business income deduction of $910 and an additional senior deduction of $3,951.05, yielding a taxable income of $74,633.30. PolicyEngine applied the 2026 federal tax brackets to this taxable income to calculate income tax before refundable credits of $11,131.33, which was lower than the alternative calculation using the standard deduction ($13,442.38), confirming that itemizing was the optimal choice for this taxpayer.", +us,scenario_022,federal_income_tax_before_refundable_credits,11113.5703125,64,"PolicyEngine calculated federal income tax before refundable credits of $11,113.57 for this single California resident aged 65 or older. Adjusted gross income is $109,149.16: $52,635 of wages, $30,989.41 of taxable pension income, $17,624.75 of taxable Social Security, $4,550 of rental income and $3,300 of other income. Itemized deductions of $29,735.52, made up of $14,887.85 of state and local taxes and $14,847.67 of charitable gifts, exceed the $18,150 standard deduction; with the $910 qualified business income deduction and a $3,951.05 senior deduction, total deductions are $34,596.57 and taxable income is $74,552.59. The state and local tax amount rests on PolicyEngine's estimate of California income tax withheld, computed with California's 2025 tax brackets, the last California published before the benchmark's 2026-07-03 reference freeze. The federal rate schedule applied to taxable income gives $11,113.57.", us,scenario_098,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Illinois household in 2026. This household, consisting of a single adult with no children and an annual income of approximately $35,035, does not qualify for TANF benefits because the program is specifically designed to provide cash assistance to families with dependent children. Since this benchmark household contains zero children, it falls outside the eligibility criteria for TANF, resulting in a benefit amount of zero.", us,scenario_077,head_medicare_eligible,0.0,1,"PolicyEngine determined that the head of household is not Medicare eligible (is_medicare_eligible = False) for the 2026 tax year. Medicare eligibility in the United States is primarily determined by age, with individuals becoming eligible at age 65, along with certain disability or end-stage renal disease criteria. For this single adult in Louisiana with a household income of approximately $18,039, the household composition and income level alone do not trigger Medicare eligibility; rather, the determining factor is the individual's age, which falls below the standard eligibility threshold of 65 years. Consequently, PolicyEngine assigned a value of 0.0 (False) to the is_medicare_eligible variable, indicating this household member does not qualify for Medicare benefits under 2026 rules.", us,scenario_022,federal_refundable_credits,0.0,6,"For this single adult in California with approximately $112,259 in household income and no children, PolicyEngine calculated federal refundable credits of $0. The calculation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. This outcome reflects that the household's income level and family composition do not qualify for any of the refundable credits available under 2026 tax law.", @@ -1599,14 +1599,14 @@ us,scenario_098,head_wic_eligible,0.0,13,"PolicyEngine determined that this 80-y us,scenario_022,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this California household because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $112,259 in income derives from other sources (such as wages or salaries), there is no self-employment tax liability. The calculation correctly reflects that self-employment tax applies exclusively to individuals who operate their own business or are self-employed, which does not apply to this household's income profile.", us,scenario_057,tanf,0.0,1,"PolicyEngine calculated a Temporary Assistance for Needy Families (TANF) benefit of $0 for this Louisiana household in 2026. With a household income of approximately $38 and no children present, the household does not meet the eligibility requirements for TANF, which is a program specifically designed to provide cash assistance to families with dependent children. Since this benchmark household consists of two adults with no children, they are categorically ineligible for the program regardless of their income level, resulting in a benefit amount of zero.", us,scenario_038,child1_early_head_start_eligible,0.0,1,"PolicyEngine determined that neither child in this Louisiana household is eligible for Early Head Start in 2026, resulting in a value of False for both children. Early Head Start eligibility is typically restricted to children under age 3, and this household's children do not meet the age requirement for the program. With a household income of approximately $22,992, the family's income level would generally fall within the range that qualifies for federally-funded early childhood programs, but age eligibility is the binding constraint in this case. Therefore, PolicyEngine correctly assigned an eligibility status of False to both child records in the household.", -us,scenario_117,state_income_tax_before_refundable_credits,7984.05419921875,146,"For this Arkansas joint-filing household with ~$250,733 in income, PolicyEngine calculated $7,984.05 in state income tax before refundable credits. The calculation began with employment income of $248,675 and a capital loss of $530.47, yielding adjusted gross income of $248,144.53. After applying itemized deductions of $24,659.28, the household's taxable income was reduced to $223,485.25. This taxable income was then subjected to Arkansas's tax brackets for married filing jointly status, resulting in a tax liability of $8,189.05 before credits. Finally, PolicyEngine applied $205 in non-refundable credits—comprising $145 in personal credits ($58 for the two adults and $87 for the three dependent children) and $60 in additional tax credits for qualified individuals—to arrive at the final amount of $7,984.05.", +us,scenario_117,state_income_tax_before_refundable_credits,7984.53369140625,147,"For this Arkansas joint-filing household with approximately $250,733 in income, PolicyEngine calculated a state income tax before refundable credits of $7,984.53. The household's Arkansas adjusted gross income of $248,144.53 was reduced by itemized deductions of $24,646.33, resulting in taxable income of $223,498.20. This taxable income was subject to Arkansas's tax brackets, generating a tax before non-refundable credits of $8,189.53. PolicyEngine then applied $205 in non-refundable credits—comprising $145 in personal credits (for the two spouses and three dependent children) and $60 in additional tax credits for qualified individuals—reducing the final tax liability to $7,984.53.", us,scenario_001,state_income_tax_before_refundable_credits,0.0,47,"PolicyEngine calculated the state income tax before refundable credits as $0 for this Virginia household with approximately $44,289 in household income. The computation aggregates income tax liabilities across all 50 states plus Washington D.C. and New York City, with each jurisdiction's component (such as va_income_tax_before_refundable_credits) evaluating to $0. Since the household resides in Virginia, only the Virginia state income tax component would typically apply, and that component also computed to $0, resulting in a total state income tax liability of $0 before any refundable credits are applied. This outcome reflects either that the household's income falls below Virginia's taxable threshold for the 2026 tax year or that applicable deductions and exemptions fully offset the tax liability for a joint filer with no dependents at this income level.", us,scenario_077,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this household is not eligible for reduced-price school meals (value: 0), despite identifying the household as having a school_meal_tier of REDUCED. The household's income of approximately $18,039 places it at 1.63 times the federal poverty guideline (school_meal_fpg_ratio), which falls within the range for reduced-price meal eligibility. However, the household does not qualify for categorical eligibility through programs like SNAP or TANF, and Louisiana does not have universal free school meals. Critically, the school_meal_net_subsidy calculated to 0, meaning no subsidy benefit was available to apply, which resulted in the final eligibility determination of 0 despite meeting the income-based tier criteria.", us,scenario_057,head_wic_eligible,0.0,1,"PolicyEngine determined that neither adult in this Louisiana household is eligible for WIC (Women, Infants, and Children) benefits in 2026, resulting in a value of False for both household members. WIC eligibility is restricted to pregnant women, postpartum women, breastfeeding women, infants, and children under age 5, and this benchmark household consists of two adults with no children. Since the household contains no pregnant women, postpartum women, breastfeeding women, infants, or young children, neither member meets the categorical requirements for WIC participation regardless of their income level of approximately $38,000.", us,scenario_117,state_refundable_credits,0.0,46,"PolicyEngine calculated state_refundable_credits as $0 for this Arkansas household in 2026. The computation aggregates refundable tax credits across all 50 states and the District of Columbia, with ar_refundable_credits specifically evaluated for Arkansas (the household's state of residence) returning $0. All other state-level refundable credit variables—including those for Alabama, Arizona, California, Colorado, and all remaining states—also evaluated to $0. Since every component of the state refundable credits calculation yielded zero, the total state_refundable_credits value is $0, indicating this household qualifies for no refundable state tax credits in 2026.", us,scenario_038,free_school_meals_eligible,1.0,258,"PolicyEngine determined that this Louisiana household qualifies for free school meals with an annual benefit value of $2,261.92. The household meets eligibility through categorical qualification via SNAP receipt ($7,286.94 annually), which automatically confers free meal status regardless of income thresholds. Additionally, the household's income of $22,992.50 represents 70% of the federal poverty guideline (school_meal_fpg_ratio of 0.70), placing it well below the 130% income threshold required for free meals. With two children in K-12 school (is_in_k12_school showing True for both children), the household receives a daily subsidy of $7.15 per child, yielding the total annual benefit of $2,261.92 after accounting for the school meal paid daily subsidy of $0.87.", us,scenario_022,state_income_tax_before_refundable_credits,2439.650146484375,75,"PolicyEngine calculated California state income tax before refundable credits of $2,439.65 for this single California resident with ~$112,259 household income. The calculation began with an adjusted gross income of $109,149.16, from which California-specific AGI subtractions of $17,624.75 (primarily taxable social security) were deducted to arrive at California AGI of $91,524.41. From this AGI, California deductions totaling $23,676.67 were subtracted—comprising a standard deduction of $5,706 and itemized deductions of $23,676.67 (which included $14,847.67 in charitable contributions and $8,829 in real estate taxes)—resulting in taxable income of $67,847.75. This taxable income was then subject to California's tax rate schedule, producing a tax before credits of $2,752.58, which was reduced by non-refundable credits of $312.93 (California exemptions based on the household member's aged/blind status) to yield the final tax liability of $2,439.65.", -us,scenario_078,federal_income_tax_before_refundable_credits,24772.693359375,34,"PolicyEngine calculated the federal income tax before refundable credits as $24,772.69 for this Maryland single filer by first determining adjusted gross income of $198,505 from employment income. The household's taxable income was reduced to $134,061.22 through itemized deductions totaling $64,443.77, which consisted of $28,210 in interest deduction and $36,233.77 in state and local tax (SALT) deduction. These itemized deductions exceeded the standard deduction of $16,100, making itemization the more favorable option. Applying the 2026 federal tax brackets to the taxable income of $134,061.22 resulted in an income tax liability of $24,772.69 before any refundable credits or adjustments.", +us,scenario_078,federal_income_tax_before_refundable_credits,24164.45703125,35,"PolicyEngine calculated federal income tax before refundable credits of $24,164.46 for this Maryland single filer by applying 2026 tax rates to a taxable income of $131,526.91. The taxable income was derived from an adjusted gross income of $200,136.91—which included $198,505 in employment income plus $1,631.91 in state and local tax refund income—reduced by $68,610 in itemized deductions. The household itemized rather than taking the standard deduction of $16,100, claiming $40,400 in state and local tax deductions and $28,210 in interest deductions, which resulted in a lower tax liability of $24,164.46 compared to $36,766.86 if the standard deduction had been used. No additional taxes or credits adjusted this amount, as net investment income tax and other components were zero.", us,scenario_117,local_income_tax,0.0,1,"PolicyEngine calculated a local_income_tax value of $0.00 for this Arkansas household in 2026. Arkansas does not impose a local income tax on residents, which is why the local_income_tax variable resolves to zero regardless of the household's income level of approximately $250,733. This result reflects the state's tax structure, where income taxation occurs only at the state and federal levels, with no additional local income tax liability assessed by municipalities or counties.", us,scenario_098,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) based on the chip_category value of 3. CHIP eligibility requires meeting both age and income thresholds, and this 80-year-old household member fails CHIP's age criterion, as the program is designed for children and young adults, not seniors. With a household income of approximately $35,035 and no dependent children in the household, the person does not qualify under any CHIP category. The resulting chip_category of 3 reflects ineligibility, leading to the final determination that is_chip_eligible equals False.", us,scenario_001,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Virginia household in 2026. Virginia does not impose a local income tax at the state level, and the household's locality of residence does not levy an additional local income tax on wages or other income sources. Therefore, regardless of the household's joint income of approximately $44,289, no local income tax liability was generated. This result reflects the tax structure in Virginia, where income taxation is handled exclusively at the state and federal levels.", @@ -1633,7 +1633,7 @@ us,scenario_022,ssi,0.0,25,"PolicyEngine calculated Supplemental Security Income us,scenario_039,payroll_tax,0.0,3,"PolicyEngine calculated the payroll tax (spm_unit_payroll_tax) for this Virginia household as $0. The computation shows that the employee_payroll_tax component, which represents Social Security and Medicare taxes withheld from wages, evaluated to $0. This result appears inconsistent with the household's reported income of approximately $84,625, as typical employee payroll tax rates (6.2% for Social Security and 1.45% for Medicare) would normally apply to earned wages at this income level. The household was identified as a tax unit head (is_tax_unit_head = True), but this classification alone does not explain why payroll tax obligations would be zero given the substantial household income reported.", us,scenario_057,head_chip_eligible,0.0,1,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False). The person is age 43, which places them outside CHIP's target population of children. Additionally, PolicyEngine's eligibility determination found that this person is already eligible for Medicaid under the SSI_RECIPIENT category. Since CHIP is designed to cover children who do not qualify for Medicaid, individuals who are already Medicaid-eligible are categorically ineligible for CHIP regardless of other factors. Therefore, the household receives a CHIP eligibility value of 0.0 (False).", us,scenario_117,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Arkansas household in 2026. The household's annual income of approximately $250,733 substantially exceeds the income eligibility thresholds for TANF assistance, which are designed to serve low-income families. Given this high household income level, the family does not qualify for any TANF benefits under Arkansas's program rules, resulting in the zero benefit value.", -us,scenario_078,state_income_tax_before_refundable_credits,6936.33740234375,66,"PolicyEngine calculated Maryland state income tax before refundable credits of $6,936.34 for this single filer with ~$200,137 household income. The calculation began with adjusted gross income of $198,505, from which Maryland deductions of $54,670 were subtracted to arrive at taxable income of $143,835. The deductions consisted of itemized deductions totaling $54,670, which included $28,210 in interest deductions and $26,460 in capped property taxes, exceeding the standard deduction of $3,400 and therefore making itemization the beneficial choice. Maryland's tax rate schedule was then applied to the taxable income of $143,835 to produce the final tax liability of $6,936.34 before any refundable credits.", +us,scenario_078,state_income_tax_before_refundable_credits,6936.87646484375,87,"PolicyEngine calculated Maryland state income tax before refundable credits of $6,936.88 for this single filer with ~$200,137 household income. The calculation began with adjusted gross income of $200,136.91, from which Maryland allowed a subtraction of $1,631.91 (salt_refund_income), resulting in Maryland AGI of $198,505. The household claimed itemized deductions totaling $54,659.73—comprising $28,210 in interest deductions, $26,460 in capped property taxes, and a $3,350 Maryland standard deduction adjustment—which reduced taxable income to $143,845.27. This taxable income was then subjected to Maryland's tax brackets and rate structure, yielding the final tax liability of $6,936.88 before any refundable credits.", us,scenario_117,head_wic_eligible,0.0,1,"PolicyEngine determined that none of the five household members are eligible for WIC (Women, Infants, and Children) benefits, resulting in `is_wic_eligible = [False, False, False, False, False]`. WIC eligibility requires that applicants meet categorical requirements (being pregnant, postpartum, breastfeeding, an infant, or a child under age 5) and satisfy income limits set at 185% of the federal poverty line. With a household income of approximately $250,733 annually in Arkansas, this household substantially exceeds the WIC income threshold, which for a family of five would be roughly $50,000–$55,000 depending on the specific composition and poverty guidelines applied. The high household income alone disqualifies all members from WIC participation, regardless of whether any household members meet the categorical requirements for the program.", us,scenario_022,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this California household in 2026. The household consists of a single adult with no children and an annual income of approximately $112,259. TANF eligibility is restricted to families with dependent children, and since this household has zero children, it does not qualify for the program regardless of income level. Therefore, PolicyEngine correctly determined that no TANF benefit applies to this household.", us,scenario_039,self_employment_tax,0.0,1,"PolicyEngine calculated a self-employment tax of $0 for this Virginia household in 2026 because the household has no self-employment income. Self-employment tax is only assessed on net earnings from self-employment activities, and since this benchmark household's approximately $84,625 in household income derives from other sources (such as wages or salaries), there is no self-employment tax liability. The calculation correctly reflects that self-employment tax applies only to individuals who operate their own business or are self-employed, which does not apply to this household's income profile.", @@ -1704,7 +1704,7 @@ us,scenario_099,head_wic_eligible,0.0,1,"PolicyEngine determined that no househo us,scenario_099,spouse_wic_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for WIC (Women, Infants, and Children) benefits, resulting in a value of 0.0 (False). The household's income of approximately $159,676 exceeds the WIC income eligibility threshold, which typically caps at 185% of the federal poverty line for a family of four. Since the benchmark household's income substantially surpasses this limit, PolicyEngine correctly classified the spouse as ineligible for the program. The array output [False, False, False, False] indicates that none of the four household members meet WIC eligibility criteria based on the income test.", us,scenario_059,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Florida household in 2026. Florida does not impose a state income tax, and the local_income_tax variable, which captures taxes levied by local jurisdictions within states, returned zero because Florida municipalities do not assess local income taxes on residents. With a household income of approximately $62,704 and joint filing status, the household has no local income tax liability under Florida's tax structure.", us,scenario_078,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Maryland household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $200,137 places it at 12.44 times the federal poverty guideline, far exceeding the 185% federal poverty level threshold required for reduced-price meal eligibility. Since Maryland does not have universal free school meals and the household meets neither the income requirement nor categorical eligibility criteria (such as SNAP or TANF participation), the household is assigned to the PAID school_meal_tier. Consequently, the school_meal_net_subsidy is 0, and the household must pay the full price for school meals.", -us,scenario_023,state_income_tax_before_refundable_credits,6.7984771728515625,68,"For this California resident with approximately $25,443 in household income, PolicyEngine calculated a state income tax before refundable credits of $6.80. The calculation began with an adjusted gross income of $22,534.34, from which California's standard deduction of $5,706 was subtracted to arrive at a taxable income of $16,828.34. This taxable income was then subject to California's tax brackets, resulting in a tax before credits of $223.27. Finally, PolicyEngine applied non-refundable credits totaling $216.47—comprising $156.47 in exemptions and $60.00 in renter credits—which reduced the tax liability to the final amount of $6.80.", +us,scenario_023,state_income_tax_before_refundable_credits,12.776718139648438,69,"PolicyEngine calculated California income tax before refundable credits of $12.78 for this single filer. California adjusted gross income of $22,534.34 less the $5,706 standard deduction gives taxable income of $16,828.34, and the California rate schedule gives tax of $225.78 before credits. Nonrefundable credits of $213, a $153 personal exemption credit and a $60 renter's credit (the household pays $24,000 of rent and its income is under the renter's credit limit), leave $12.78. California's 2026 bracket thresholds, exemption credits and credit limits equal the 2025 amounts California published, the last before the benchmark's 2026-07-03 reference freeze; the 2026 indexing factor rests on June 2026 prices, published after it.", us,scenario_117,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (is_chip_eligible = False) in Arkansas for the 2026 tax year. The spouse, age 34, fails CHIP's eligibility criteria on the basis of age, as CHIP provides coverage for children and pregnant women, not adults over 19 (with limited exceptions for pregnancy). Additionally, the household income of approximately $250,733 substantially exceeds CHIP income thresholds for any covered category. Since the spouse is neither Medicaid-eligible nor meets CHIP's own categorical and income requirements, PolicyEngine correctly returned a value of 0.0 (False) for spouse_chip_eligible.", us,scenario_002,federal_refundable_credits,0.0,6,"PolicyEngine calculated federal refundable credits of $0 for this benchmark household of two adults in Washington State with approximately $134,820 in household income. The refundable credits aggregate comprises five component credits: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components evaluated to $0 for the household. The EITC phases out at higher income levels and does not apply to this household; the American Opportunity and Child Tax Credits do not apply because there are no qualifying students or dependent children; the Recovery Rebate Credit does not apply in 2026 as it was specific to prior pandemic relief; and the refundable payroll tax credit similarly does not apply. Consequently, the household receives no refundable federal tax credits.", us,scenario_039,free_school_meals_eligible,0.0,15,"PolicyEngine determined that this Virginia household is not eligible for free school meals, resulting in a value of 0. The household's school_meal_fpg_ratio of 3.68—calculated by dividing countable income of $58,674.97 by the federal poverty guideline of $15,960—places them well above the 185% threshold (1.85) required for reduced-price meals and far above the 130% threshold (1.30) for free meals. Since Virginia does not offer universal free school meals and the household meets neither categorical eligibility requirements nor income thresholds, they are classified in school_meal_tier 2 (PAID), meaning they pay the full price for school meals. Consequently, the school_meal_net_subsidy is 0, and the household receives no free school meal benefit.", @@ -1713,7 +1713,7 @@ us,scenario_002,payroll_tax,0.0,3,"PolicyEngine calculated a payroll tax of $0 f us,scenario_117,child1_chip_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for CHIP (is_chip_eligible = False) in Arkansas for the 2026 tax year. The household's income of approximately $250,733 substantially exceeds Arkansas's CHIP income limits, which are set at a percentage of the federal poverty level and result in much lower thresholds for eligibility. Additionally, at age 16, child1 falls within CHIP's covered age range, but the income-based disqualification is the binding constraint preventing eligibility. Since the household income far surpasses the program's income threshold, child1 does not meet CHIP's eligibility criteria regardless of other factors.", us,scenario_079,federal_income_tax_before_refundable_credits,0.0,8,"PolicyEngine calculated federal income tax before refundable credits as $0 for this Arizona joint-filing household with approximately $7,560 in annual income. The computation begins with income_tax_before_credits, which evaluated to $0, indicating that the household's taxable income fell below the threshold requiring tax liability for their filing status. The net_investment_income_tax component also contributed $0, as this 3.8% tax on net investment income applies only to higher-income taxpayers. Additional components—recapture_of_investment_credit, unreported_payroll_tax, and qualified_retirement_penalty—each summed to $0, as none of these special tax adjustments applied to this household. Finally, income_tax_capped_non_refundable_credits was $0, meaning no non-refundable credits were available to reduce an already-zero tax liability, resulting in a final value of $0.", us,scenario_059,snap,0.0,25,"PolicyEngine calculated a SNAP (Supplemental Nutrition Assistance Program) benefit of $0 for this Florida household of two adults with no children and approximately $62,704 in annual household income. The household's income level exceeds the SNAP eligibility threshold for a two-person household without dependents, which is substantially lower than their reported income. Although the model indicates the household would take up SNAP benefits if eligible (takes_up_snap_if_eligible = True), the income disqualification prevents any benefit from being awarded. Therefore, despite the household's willingness to participate in the program, their earnings place them above the income limit, resulting in zero SNAP benefits for the 2026 tax year.", -us,scenario_023,state_refundable_credits,148.3108673095703,49,"PolicyEngine calculated state refundable credits of $148.31 for this California resident in 2026. The entire amount comes from the California Earned Income Tax Credit (CA EITC), as the household is eligible for this program. The CA EITC calculation is based on the filer's adjusted earnings of $17,442.65, which represents the income used to determine the credit amount under California's EITC formula. All other states' refundable credit programs returned zero values, as the household is only subject to California's tax rules. The final state refundable credits value of $148.31 represents the total refundable tax benefits available to this household at the state level for the 2026 tax year.", +us,scenario_023,state_refundable_credits,95.1540069580078,49,"PolicyEngine computes state refundable credits of $95.15 for this single California filer in 2026. The whole amount is California's refundable credits, which consist entirely of the California Earned Income Tax Credit (CalEITC) of $95.15; every other state's refundable credits are zero. The filer, who has no children, is eligible for CalEITC. The credit calculation draws on two income measures. The first is the filer's adjusted earnings of $17,442.65, which equal earned income and come entirely from employment income. The second is adjusted gross income of $22,534.34, which is higher than earnings. With eligibility and these two income amounts as inputs, the CalEITC formula yields $95.15, which passes through as California's refundable credits and the reported total.", us,scenario_039,reduced_price_school_meals_eligible,0.0,3,"PolicyEngine determined that this Virginia household is not eligible for reduced-price school meals, resulting in a value of 0. The household's income of approximately $84,625 translates to a school_meal_fpg_ratio of 3.68, meaning household income is 368% of the federal poverty guideline—well above the 185% threshold required for reduced-price meal eligibility. Since Virginia does not offer universal free school meals and the household does not qualify for categorical eligibility through programs like SNAP or TANF, the household is classified in the PAID school_meal_tier. Consequently, the school_meal_net_subsidy is 0, and the household pays the full price for school meals.", us,scenario_079,federal_refundable_credits,0.0,6,"For this Arizona joint household with approximately $7,560 in annual income and no children in tax year 2026, PolicyEngine calculated federal refundable credits of $0. The computation evaluated five potential refundable credit components: the Earned Income Tax Credit (EITC), the refundable American Opportunity Credit, the refundable Child Tax Credit (CTC), the Recovery Rebate Credit, and the refundable payroll tax credit. Each of these components returned $0, resulting in a total refundable credits value of $0. With no qualifying children and income below typical EITC phase-in thresholds for joint filers, the household did not qualify for the primary refundable credits available under 2026 tax law.", us,scenario_099,child2_wic_eligible,0.0,1,"PolicyEngine determined that child 2 is not eligible for WIC benefits, resulting in a value of False (0.0). WIC eligibility in California requires that household income fall below 185% of the federal poverty line. With a household income of approximately $159,676 annually, this benchmark household substantially exceeds the income threshold for a family of four, which would be around $49,000 at 185% of the poverty line for 2026. Since the household's income far surpasses this limit, child 2 does not qualify for WIC assistance regardless of other potential eligibility factors.", @@ -1756,7 +1756,7 @@ us,scenario_059,head_chip_eligible,0.0,1,"PolicyEngine determined that the head us,scenario_002,tanf,0.0,1,"PolicyEngine calculated a TANF (Temporary Assistance for Needy Families) benefit of $0 for this Washington household in 2026. This result reflects that the household, with a joint income of approximately $134,820 and no children, does not qualify for TANF assistance. TANF is a needs-based program primarily designed to support families with dependent children, and this benchmark household's substantial income and lack of qualifying dependents place it well above any eligibility thresholds. Consequently, PolicyEngine's computation trace shows no intermediate calculations—the benefit determination is straightforward at $0.", us,scenario_040,local_income_tax,0.0,1,"PolicyEngine calculated a local income tax of $0 for this Arizona household in 2026. Arizona does not impose a local income tax at the state level, and the household's residence in Arizona means they are not subject to any municipal or county-level income taxes in that jurisdiction. Therefore, despite the household's joint income of approximately $56,400, no local income tax liability was generated, resulting in the computed value of $0.", us,scenario_023,head_chip_eligible,0.0,2,"PolicyEngine determined that the head of household is not eligible for CHIP (is_chip_eligible = False) because they fall into chip_category 3, which represents individuals who are already eligible for Medicaid and therefore ineligible for CHIP. The person, a 28-year-old with household income of approximately $25,443, qualifies for Medicaid under the WORKING_DISABLED_BUY_IN category. Since CHIP is designed to cover children who do not qualify for Medicaid, individuals who are already Medicaid-eligible are categorically excluded from CHIP eligibility regardless of their income level. This ineligibility is a direct result of the person's existing Medicaid eligibility status rather than any income threshold consideration.", -us,scenario_079,snap,2428.017333984375,616,"PolicyEngine calculated a monthly SNAP benefit of $2,428.02 for this Arizona household of two adults by aggregating individual monthly allotments of $198.90 and $212.64 across the tax year 2026. The household qualified for SNAP based on meeting all eligibility criteria: their snap net income of $1,157 (derived from snap gross income of $1,511 minus deductions of $354) was 66% of the federal poverty guideline of $1,762.50, their gross income was 86% of the FPG, they passed the asset test with $1,045.29 in assets, and they met categorical eligibility through SSI receipt of $881 and TANF non-cash eligibility. The benefit amount was calculated using the standard SNAP formula: the maximum allotment for a two-person household in the contiguous US ($546 in most months, $558.24 in later months) minus the expected contribution of approximately $347, yielding the normal allotment that was then subject to the minimum allotment floor of $23.84–$24.37. The household's assumption of taking up SNAP if eligible resulted in the full calculated benefit being included in the annual total.", +us,scenario_079,snap,2376.0,616,"PolicyEngine computes an annual 2026 SNAP total of $2,376 for this Arizona couple, the sum of twelve monthly allotments of $198. Each month, gross income is $1,511, all unearned: $881 of SSI ($440.50 each) plus $630 of Social Security for the head. Deductions of $354, a $209 standard deduction plus a $145 excess medical expense deduction, leave net income of $1,157. Gross income is 86% of the $1,762.50 monthly poverty guideline January through September and 84% of $1,803.33 October through December, and the household passes the income tests, the asset test with $1,045.29 in assets, and categorical eligibility, which PolicyEngine checks against its SSI and TANF non-cash eligibility. The $546 maximum allotment minus a $348 expected contribution yields $198 every month, above the $24 minimum.", us,scenario_099,child1_chip_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for CHIP (is_chip_eligible = False) because the household's income exceeds CHIP's financial thresholds for California in 2026. With a household income of approximately $159,676, the family surpasses the income limit for CHIP eligibility, which typically caps out at a percentage of the federal poverty level (often around 200-250% depending on the state and program year). Additionally, since child1 is not Medicaid-eligible (categorized as NONE in the engine), they do not qualify through the Medicaid pathway that sometimes provides automatic CHIP eligibility. The combination of excess household income and lack of Medicaid eligibility results in PolicyEngine assigning a value of False to is_chip_eligible for this child.", us,scenario_117,child1_head_start_eligible,0.0,1,"PolicyEngine determined that child1 is not eligible for Head Start (is_head_start_eligible = False) based on the household's income level relative to the program's eligibility thresholds. Head Start eligibility in Arkansas is primarily determined by whether household income falls below 100% of the federal poverty line or meets other categorical eligibility criteria. With a household income of approximately $250,733 annually for a family of five, this benchmark household substantially exceeds the income threshold required for Head Start participation. Consequently, PolicyEngine returned a value of False for the child's Head Start eligibility status, indicating the household does not qualify for this federal early childhood education program.", us,scenario_059,spouse_chip_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for CHIP (Children's Health Insurance Program) in 2026, resulting in a value of False. The spouse's age of 78 years exceeds CHIP's eligibility criteria, which are designed for children and young adults, not seniors. Additionally, the spouse does not meet Medicaid eligibility requirements (categorized as NONE), which would be a prerequisite pathway to CHIP coverage in many cases. Since the spouse fails both CHIP's age-based criteria and lacks Medicaid eligibility, PolicyEngine correctly computed is_chip_eligible as False for this household member.", diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json new file mode 100644 index 00000000..c8706075 --- /dev/null +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_wording-amendments.json @@ -0,0 +1,3127 @@ +{ + "note": "Wording-only amendments of published text for cases GPT-6.1 Sol re-opened (finish_gpt61sol.py load_amendments): each replaces its old text, found exactly once, with the new text. None changes a class, an exclusion or a score. Triage applies them; export binds this file; the freeze commits it beside the adjudication record.", + "amendments": [ + { + "case_id": "us__scenario_008__state_refundable_credits", + "field": "reasoning", + "old": "The judge labeled one model's row reference_later_law because it applied", + "new": "The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied", + "reason": "The re-judge labels the case and two rows reference_later_law, not one row; the decision itself is unchanged." + }, + { + "case_id": "us__scenario_008__state_refundable_credits", + "field": "reasoning", + "old": "; the row is a model error.", + "new": "; the rows are model errors.", + "reason": "Follows the previous amendment: the decision scores both rows as model errors." + }, + { + "case_id": "us__scenario_056__snap", + "field": "reasoning", + "old": "the alternative value is the engine's zero-hours result, not a certified entitlement.", + "new": "the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "reason": "The re-judge's case note says mortgage interest is not a SNAP shelter cost; the decision named that unlisted input (reference_basis) but described only the hours reading. The added sentence is the pinned exclusion record's own alternative_reading for this output." + }, + { + "case_id": "us__scenario_067__ssi", + "field": "reasoning", + "old": "Under the alternative reading nine of the twelve models that paid SSI land within 10% of $1,368 (six exactly);", + "new": "Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05);", + "reason": "The count described the 2026-09-05 board; the re-judge's case note counts all 17 models that pay SSI on this board. The three outliers are unchanged." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "or used unindexed zero-bracket thresholds ($4,489, $4,811).", + "new": "or used another zero-bracket threshold: $4,489, or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920.", + "reason": "The re-judge calls the $4,811 zero-rate threshold a model error, but the release's Idaho convention c_id_hold_2025 (reference_outputs.csv.meta.json) holds the published 2025 $4,811 for 2026, and the exclusion's corrected value applies it; the frozen reference's $4,920 is the engine's price-index projection. Corrected on review: Supersedes the existing 053 case-note amendment whose new text is \"or used another zero-bracket threshold: the 2024 $4,489, or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920.\": no record file dates $4,489 (c_id_hold_2025 names only the 2025 $4,811 single amount), so the year is dropped. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The model used the 2025 zero-rate threshold of $4,811 instead of the inflation-indexed 2026 single threshold of $4,920. That overstated the income taxed at 5.3% by $109 and the tax by $5.79.", + "new": "The model used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. That taxed $109 more income at 5.3% than the frozen reference, $5.79 of tax.", + "reason": "The re-judge calls the $4,811 zero-rate threshold a model error, but the release's Idaho convention c_id_hold_2025 (reference_outputs.csv.meta.json) holds the published 2025 $4,811 for 2026, and the exclusion's corrected value applies it; the frozen reference's $4,920 is the engine's price-index projection.", + "model": "claude-opus-5.5" + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also used the 2025 zero-rate threshold of $4,811 instead of $4,920.", + "new": "It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920.", + "reason": "The re-judge calls the $4,811 zero-rate threshold a model error, but the release's Idaho convention c_id_hold_2025 (reference_outputs.csv.meta.json) holds the published 2025 $4,811 for 2026, and the exclusion's corrected value applies it; the frozen reference's $4,920 is the engine's price-index projection.", + "model": "gpt-6.1-sol" + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The model subtracted the $5,000 health-insurance premiums from Idaho income, but Idaho taxable income here is AGI minus only the $16,100 standard deduction. It also used the 2025 zero-rate threshold of $4,811 instead of $4,920. Together these understated the taxed income by $4,891.", + "new": "The model subtracted the $5,000 of health insurance premiums from Idaho income, which the exclusion's corrected value subtracts under Idaho Code 63-3022P (assuming the premiums are not paid through a pre-tax salary reduction) and the frozen reference, AGI minus only the $16,100 standard deduction, leaves out. It also used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Together these taxed $4,891 less income than the frozen reference; its $2,176.07 is $0.013 above the exclusion's corrected value, $2,176.057.", + "reason": "The re-judge calls the $4,811 zero-rate threshold a model error, but the release's Idaho convention c_id_hold_2025 (reference_outputs.csv.meta.json) holds the published 2025 $4,811 for 2026, and the exclusion's corrected value applies it; the frozen reference's $4,920 is the engine's price-index projection. The row also said Idaho allows no premium subtraction here; the exclusion record's engine defect is exactly that omission (Idaho Code 63-3022P). Corrected on review: Supersedes the existing 053 gpt-6-astra amendment whose new text begins \"The model subtracted the $5,000 of health insurance premiums from Idaho income, which Idaho Code 63-3022P allows\" and ends \"is within a cent of the exclusion's corrected value, $2,176.06.\". reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction, so the subtraction is the corrected value's only on that assumption, and 2,176.07 - 2,176.057 = 0.013 is more than a cent.", + "model": "gpt-6-astra" + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "With household gross income under $18,000 and $1,634 in property taxes, 50% of the excess is far above the cap, so a 74-year-old gets the $375 elderly maximum.", + "new": "With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a 74-year-old gets a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "The formula, 50% of ($1,634 minus the income-band percentage of household gross income), gives at least about $420, so the full $375 cap applies.", + "new": "With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375, so nothing reduces it to $63.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "Real property taxes paid by a homeowner qualify directly, and $1,634 in taxes on household gross income under $18,000 produces the $375 elderly-capped credit.", + "new": "Real property taxes paid by a homeowner qualify directly, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "The model correctly got $452 as 50% of the property taxes above 6% of a $12,168 household gross income. It then capped that at an income-banded $171 instead of the flat $375 maximum for a household with a member 65 or older, so the credit is $375.", + "new": "The model used the repealed household-gross-income formula, getting $452 as 50% of the property taxes above 6% of a $12,168 household gross income. It then capped that at an income-banded $171, but a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375 from a flat table, so the model's figure is not a correct step; the model's cap error stays the diagnosis." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The model computed an uncapped RPTC of $733 but applied the $75 maximum for households with no one 65 or older. The head is 74, so the $375 elderly maximum applies.", + "new": "The model used the repealed household-gross-income formula to get $733 and then applied a $75 maximum it tied to having no dependents. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the flat $375 for a filer 65 or older applies.", + "reason": "The row presents the model's $733 as the uncapped RPTC and states a $75 maximum for households with no one 65 or older as the rule, but the adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed and gives only the federal-AGI-$0 row (a filer 65 or older receives $375). The $75-cap diagnosis stays, in the model's own terms from prompt.md (no dependents)." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "For homeowners, the RPTC is based on real property taxes paid, and $1,634 in taxes on very low household income reaches the $375 elderly maximum.", + "new": "For homeowners, the RPTC is based on real property taxes paid, and with federal AGI of $0 and $1,634 in taxes, above 3.5% of that AGI, this 65+ filer receives a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "The model correctly found the $375 elderly-capped RPTC but added a $75 NY household credit.", + "new": "The model used the repealed household-gross-income rule to reach $375, the right RPTC for a filer 65 or older with federal AGI of $0, but added a $75 NY household credit.", + "reason": "The row calls the model's $375 a correct finding, but the model's reasoning in the case's prompt.md reaches it from a combined income including SSI of $13,176, the repealed household-gross-income rule; the adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) gives $375 from a flat table for a filer 65 or older with federal AGI of $0. The added household credit stays the diagnosis." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "It missed the refundable Real Property Tax Credit, which pays a 65+ homeowner with household gross income under $18,000 and $1,634 in property taxes the $375 maximum.", + "new": "It missed the refundable Real Property Tax Credit, which pays a 65+ homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "For a 74-year-old homeowner paying $1,634 in property taxes on household gross income under $18,000, 50% of the excess taxes is capped at the $375 elderly maximum.", + "new": "For a 74-year-old homeowner with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, the credit is a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It is available to homeowners 65 or older with household gross income under $18,000, and here it reaches the $375 cap.", + "new": "It applies here: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "The model correctly computed an uncapped RPTC of $768 (50% of $1,634 minus 3.5% of $2,800), but compared it to a nonexistent $1,000 senior maximum. The elderly maximum is $375, so the credit is $375.", + "new": "The model used the repealed household-gross-income formula, getting $768 (50% of $1,634 minus 3.5% of $2,800), and compared it to a $1,000 senior maximum that does not apply here. A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375 from a flat table, so the model's figure is not a correct step; the model's cap error stays the diagnosis." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "The listed facts (age 74, owner paying $1,634 in real property taxes, household gross income under $18,000) qualify for the refundable Real Property Tax Credit at its $375 elderly maximum.", + "new": "The listed facts (age 74, owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0) qualify for the refundable Real Property Tax Credit at a flat $375 for a filer 65 or older.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "50% of the $1,634 in property taxes above the small income-based floor is capped at $375 for a household member 65 or older.", + "new": "With federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "The Real Property Tax Credit still applies to this 74-year-old homeowner with household gross income under $18,000 and pays the $375 cap.", + "new": "The Real Property Tax Credit still applies to this 74-year-old homeowner with federal AGI of $0 and property taxes above 3.5% of it, and pays a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "The model identified the RPTC and household gross income of $12,168 but applied the $75 cap for households with no one 65 or older. Because the head is 74, the maximum is $375, and 50% of the excess property taxes (about $452) reaches it.", + "new": "The model identified the RPTC but used the repealed household-gross-income rule, with an estimated $12,168 of household gross income, and capped the credit at $75 for a homeowner 65 or older. The head is 74 with federal AGI of $0 and property taxes above 3.5% of it, so the credit is a flat $375.", + "reason": "The row credits the model with identifying household gross income and states the $75 cap for households with no one 65 or older and the 50%-of-excess amount as the rule, but the adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed and a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375. The $75-cap diagnosis stays, as the model stated it in prompt.md (a cap for a homeowner 65 or older)." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The RPTC is 50% of the property taxes above the income-band percentage of household gross income (3.5% below $3,000), which gives $768 here, capped at the $375 elderly maximum.", + "new": "From 2025 the RPTC is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, a filer 65 or older receives $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The maximum for a household with a member 65 or older is $375, and the 50%-of-excess-taxes amount exceeds that, so the credit is $375.", + "new": "A filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "It pays this 74-year-old homeowner with $1,634 in property taxes and household gross income under $18,000 the $375 elderly maximum.", + "new": "It pays this 74-year-old homeowner, whose $1,634 in property taxes is above 3.5% of a federal AGI of $0, a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "A 74-year-old owner paying $1,634 in real property taxes with household gross income under $18,000 qualifies for the refundable RPTC at its $375 elderly cap.", + "new": "A 74-year-old owner paying $1,634 in real property taxes, above 3.5% of a federal AGI of $0, qualifies for the refundable RPTC at a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "The model found no eligibility factors, overlooking that real estate taxes of $1,634 paid by a 74-year-old homeowner with household gross income under $18,000 trigger the refundable RPTC at its $375 elderly maximum.", + "new": "The model found no eligibility factors, overlooking that a 74-year-old homeowner with federal AGI of $0 and $1,634 in real estate taxes, above 3.5% of that AGI, receives the refundable RPTC at a flat $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says that from 2025 the credit is keyed to federal AGI and a flat table and that a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375; the row states the repealed household-gross-income rule the record says the judge applied." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "The model cited the head's income and age as reasons for no credit, but age 74 is what raises the Real Property Tax Credit cap from $75 to $375. The low income makes 50% of the excess property taxes exceed that cap, so the credit is $375.", + "new": "The model cited the head's income and age as reasons for no credit, but they are what qualify the head for the Real Property Tax Credit: a filer 65 or older with federal AGI of $0 and property taxes above 3.5% of it receives a flat $375, so the credit is $375.", + "reason": "The row states a $75 cap for filers under 65 and the 50%-of-excess formula as the rule, but the adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed and gives only the federal-AGI-$0 row (a filer 65 or older receives $375 from a flat table); the diagnosis that the model's income and age reasons are what qualify the head stays." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "case_annotation", + "old": "A homeowner who pays real property taxes and whose household gross income is under $18,000 qualifies, so NY tax liability, earned income, and rent do not matter.", + "new": "A homeowner who pays real property taxes can qualify without NY tax liability, earned income, or rent.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed: from 2025 the credit is keyed to federal AGI and a flat table, and a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375. The rationale states the repealed rule (the under-$18,000 limit, the 50%-of-excess formula and its $768 and $421–452 figures) as the governing one." + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "field": "case_annotation", + "old": "The credit is 50% of the property taxes above a small percentage of household gross income (3.5%–6.5%, depending on the income band). Here that is well over the cap no matter how income is counted: $768 at $2,800, or about $421–452 if SSI is added. A household member age 65 or older gets a $375 maximum instead of $75, so the answer is $375.", + "new": "From 2025 the credit is keyed to federal AGI and a flat table: with federal AGI of $0 and $1,634 in property taxes, above 3.5% of that AGI, this 74-year-old filer receives $375.", + "reason": "The adjudication record (us_adjudications.json, reasoning; reference_basis N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025) says the household-gross-income rule is repealed: from 2025 the credit is keyed to federal AGI and a flat table, and a filer 65 or older with federal AGI of $0 and property tax above 3.5% of it receives $375. The rationale states the repealed rule (the under-$18,000 limit, the 50%-of-excess formula and its $768 and $421–452 figures) as the governing one." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-fable-5", + "old": "It then paid a made-up $293/month maximum instead of the FY2026 $298 for January–September and the FY2027 $304.68 for October–December.", + "new": "It then paid a made-up $293/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States the FY2027 $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The $293 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It paid a $293/month maximum for all 12 months instead of $298 for January–September and $304.68 for October–December.", + "new": "It paid a $293/month maximum for all 12 months instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The $293 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It then used the FY2025 $292/month maximum instead of $298 (January–September) and $304.68 (October–December).", + "new": "It then used the FY2025 $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The stale $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "It also counted the $924 capital gains and used a flat $300 maximum instead of $298 for January–September and $304.68 for October–December.", + "new": "It also counted the $924 capital gains and used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The $300 maximum and capital-gains errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "It also counted the $924 capital gains, and it used a flat $300 maximum instead of $298 (January–September) and $304.68 (October–December).", + "new": "It also counted the $924 capital gains, and it used a flat $300 maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The $300 maximum and capital-gains errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "It reached zero net income through BBCE and the energy-subsidy SUA, but used a stale $292/month maximum instead of $298 (January–September) and $304.68 (October–December).", + "new": "It reached zero net income through BBCE and the energy-subsidy SUA, but used a stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The stale $292, $198 and $600 errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "kimi-k3", + "old": "It paid the stale $292/month maximum instead of $298 for January–September and $304.68 for October–December.", + "new": "It paid the stale $292/month maximum instead of the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "States $304.68 as the correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The stale $292 and income errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "It never applied the $857 heating SUA from the energy subsidy that zeroes net income, and never used the actual maximums of $298 (January–September) and $304.68 (October–December).", + "new": "It never applied the $857 heating SUA from the energy subsidy that zeroes net income, and never used the FY2026 $298 maximum, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "Calls $304.68 an actual October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The SUA and missing-maximum errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "The correct maximums are $298 for January–September (FY2026) and $304.68 for October–December (FY2027).", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States the FY2027 $304.68 as a correct maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The outdated $291 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The correct maximums are $298 for January–September and $304.68 for October–December.", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States $304.68 as a correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The FY2025 $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "The correct maximums are $298 for January–September and $304.68 for October–December.", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States $304.68 as a correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The FY2025 $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "The correct maximums are $298 for January–September and $304.68 for October–December.", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States $304.68 as a correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The stale $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "grok-4.5", + "old": "The correct maximums are $298 for January–September and $304.68 for October–December.", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States $304.68 as a correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The stale $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "grok-4.6", + "old": "The correct maximums are $298 for January–September and $304.68 for October–December.", + "new": "The correct maximum is the FY2026 $298, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; the frozen reference's $304.68 for October–December is the engine's projected FY2027 maximum.", + "reason": "States $304.68 as a correct October-December maximum. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The FY2025 $292 error stays." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It paid $298 for all 12 months and missed the October 2026 FY2027 increase to $304.68 for October–December. It also reached zero net income the wrong way:", + "new": "Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum. But it reached zero net income the wrong way:", + "reason": "Calls paying FY2026's $298 for October-December an error; the model's 3,576 (predictions.csv) equals $298 x 12, the revisions[17] recheck value (3,576). meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. Its income and SUA errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "grok-4.7", + "old": "It paid $298 for all 12 months and missed the October 2026 FY2027 increase to $304.68. It also counted only", + "new": "Its $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December; the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum. But it counted only", + "reason": "Calls paying FY2026's $298 for October-December an error; the model's 3,576 (predictions.csv) equals the revisions[17] recheck value (3,576 = $298 x 12). meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. Its income-counting errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It paid $298 for all 12 months and knowingly left out the October 2026 cost-of-living update. From October the FY2027 maximum is $304.68, so annual SNAP is 298×9 + 304.68×3 = $3,596.04.", + "new": "It paid $298 for all 12 months and knowingly left out any October 2026 cost-of-living update, which is what the release's SNAP convention (c_snap_hold_fy2026) does: October–December 2026 hold the FY2026 schedule because USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze. The frozen reference instead uses the engine's projected FY2027 maximum of $304.68 from October, giving 298×9 + 304.68×3 = $3,596.04. Its route to zero net income is wrong, though: $301.33 of countable income less the $209 standard deduction leaves $92.33, which the frozen reference zeroes only with the $857 heating SUA.", + "reason": "Calls paying FY2026 for October-December an error and states $304.68 and $3,596.04 as correct; the model's 3,576 equals the revisions[17] recheck value (3,576). meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The added last sentence keeps a real error from the model's own reasoning in prompt.md ('Countable income minus the standard deduction leaves net income of about zero'; $301.33 - $209 = $92.33 per the case note) so the row does not read as fully correct; drop it if new content is unwanted." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It paid the FY2026 $298 maximum for all 12 months, missing that the FY2027 increase to $304.68 applies to October–December 2026.", + "new": "It paid the FY2026 $298 maximum for all 12 months, which matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference instead uses the engine's projected FY2027 increase to $304.68 for October–December 2026.", + "reason": "Calls paying FY2026's $298 for October-December an error; the model's 3,576 equals the revisions[17] recheck value (3,576 = $298 x 12). meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It missed that the FY2027 maximum of $304.68 applies to October–December 2026.", + "new": "That matches the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 for October–December 2026; the frozen reference's $304.68 for those months is the engine's projected FY2027 maximum.", + "reason": "Calls paying FY2026's $298 for October-December an error; the model's 3,576 equals the revisions[17] recheck value (3,576 = $298 x 12). meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-5.5", + "old": "It missed that the $2,000 energy subsidy triggers the $857 heating SUA, which with the uncapped disabled shelter deduction zeroes net income, and it also missed the October increase to $304.68.", + "new": "The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 × 12 = $3,240, $12 above the model because it used $96 of net income rather than $92. Its $298 maximum for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum.", + "reason": "Calls keeping the FY2026 $298 (the model's stated '2026 maximum $298' for 12 months, prompt.md) for October-December an error. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The SUA clause is kept per the brief. Revised on independent review." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It missed that the $2,000 energy subsidy triggers the $857 heating SUA, which through the uncapped disabled excess shelter deduction zeroes net income, and it also missed the October increase to $304.68.", + "new": "The frozen reference applies the $857 heating SUA that the $2,000 energy subsidy triggers, which zeroes net income; the exclusion's corrected value withholds that SUA from a household without an elderly or disabled member, as the model did, and pays $270 a month ($3,240), $1 above the model's $269. Keeping one maximum for all 12 months is consistent with the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum.", + "reason": "Calls not applying an October increase an error. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The SUA clause is kept per the brief. Revised on independent review." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It also counted the $924 capital gains as SNAP income and ignored the October increase to $304.68.", + "new": "It also counted the $924 capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum.", + "reason": "Calls the FY2026 $3,576 annual maximum ($298 x 12, per the row's own reconstruction) an error for ignoring October. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The SUA and capital-gains errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It also counted capital gains as SNAP income and missed the October increase to $304.68.", + "new": "It also counted capital gains as SNAP income. Its $3,576 annual maximum ($298 × 12) matches the release's SNAP convention (c_snap_hold_fy2026); the frozen reference's October increase to $304.68 is the engine's projected FY2027 maximum.", + "reason": "Calls the model's '$3,576 annual maximum' (prompt.md reasoning; $298 x 12) an error for missing October. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $304.68 is the frozen reference's engine projection. The SUA and capital-gains errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It missed that the energy subsidy triggers the $857 heating SUA, which with the uncapped disabled shelter deduction zeroes net income, and it ignored the October increase to $304.68.", + "new": "Its $3,240 equals the exclusion's corrected value, which withholds the heat-and-eat SUA from a household without an elderly or disabled member; the model reached it by applying no SUA although it called the household disabled, a reading under which the record says the energy subsidy would still confer the SUA. Holding $298 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026). The frozen reference instead applies the $857 heating SUA the energy subsidy triggers, which zeroes net income, and uses the engine's projected FY2027 $304.68 for October–December.", + "reason": "The model's 3,240 (predictions.csv) equals reference_exclusions.json alternative_value 3,240.0 (r30_snap_heat_and_eat_sua; revisions[17] recheck: $270 x 12, net $92, contribution $28), so calling its no-SUA route and its FY2026 October-December maximum errors contradicts rule (a); meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze). If the SUA clause should stay per 'keep missed SUA', use only the October-December half. Revised on independent review." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct result is zero net income after the $857 SUA and the maximum allotment, $3,596.04.", + "new": "The frozen reference reaches zero net income after the $857 SUA and pays the maximum allotment, $3,596.04, using the engine's projected FY2027 $304.68 for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 all year, which on that path gives $298 × 12 = $3,576.", + "reason": "States the frozen $3,596.04 (298x9 + 304.68x3) as the correct result. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); revisions[17] rechecks this output at 3,576 on 2.15.17 with the conventions. The asset-test errors stay." + }, + { + "case_id": "us__scenario_080__snap", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "The facts fully define a one-person unit with $300/month financial assistance and $1.33 dividends, which under BBCE with the $857 SUA gets the maximum allotment of $3,596.04.", + "new": "The facts fully define a one-person unit with $300/month financial assistance and $1.33 dividends, which under BBCE with the $857 SUA gets the maximum allotment: $3,596.04 in the frozen reference, which uses the engine's projected FY2027 $304.68 for October–December, or $298 × 12 = $3,576 under the release's SNAP convention (c_snap_hold_fy2026), which holds FY2026 all year.", + "reason": "States the frozen $3,596.04 as the maximum allotment. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); revisions[17] rechecks this output at 3,576 on 2.15.17 with the conventions." + }, + { + "case_id": "us__scenario_080__snap", + "field": "case_annotation", + "old": "The reference depends on three steps, and each wrong model missed at least one.", + "new": "The reference depends on three steps, and each wrong model departed from at least one.", + "reason": "Once the third step is attributed to the engine's FY2027 projection, 'missed' would call the five $3,576 answers ($298 x 12, predictions.csv: claude-fable-5.1, claude-opus-5.5, gemini-3.7-flash, gpt-6-astra, grok-4.7) errors for paying FY2026 in October-December, which c_snap_hold_fy2026 (meta revisions[0]) prescribes. Revised on independent review." + }, + { + "case_id": "us__scenario_080__snap", + "field": "case_annotation", + "old": "Third, calendar 2026 spans FY2026 ($298/month for January–September) and FY2027 ($304.68/month for October–December), so the maximum allotment is not one figure times 12; 16 models used a stale maximum ($291–$293) or $298 for all 12 months.", + "new": "Third, the frozen reference pays FY2026's $298/month for January–September and the engine's projected FY2027 $304.68/month for October–December, whereas the release's SNAP convention (c_snap_hold_fy2026) holds FY2026's $298 for all 12 months (USDA published FY2027 on 2026-08-21, after the 2026-07-03 freeze); of the 16 models that answered 12 times a monthly maximum, 11 used a stale maximum ($291–$293) and 5 used $298 for all 12 months, which matches the convention.", + "reason": "States FY2027's $304.68 as governing October-December and counts $298 for all 12 months as a miss. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); revisions[17] rechecks 3,576 ($298 x 12). Counts from predictions.csv: 3,516 x2, 3,504 x8, 3,492 x1 (=11 stale) and 3,576 x5, totalling the judge's 16." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It then applied $785 × 12, missing the higher October–December FY2027 allotment.", + "new": "It then applied $785 × 12; holding the $785 maximum for October–December matches the release's SNAP convention (c_snap_hold_fy2026), and the frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule.", + "reason": "Calls holding the FY2026 $785 maximum for October-December (model's '$9,420 ($785/mo)', prompt.md) an error. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). Its TANF, medical, standard-deduction and shelter errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "Its $8,300 (~$692/month) is below even its own wages-only formula ($785 − 30% × $185 ≈ $730), and it ignored the higher October–December FY2027 allotment.", + "new": "Its $8,300 (~$692/month) is below even its own wages-only formula ($785 − 30% × $185 ≈ $730). The frozen reference's higher October–December allotment uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "Calls not applying the FY2027 October-December allotment an error. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The TANF, shelter and formula errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "That produced $92 net income instead of $237.09, and it applied $757.50 to all 12 months instead of $713.90 (January–September) and $733.60 (October–December).", + "new": "That produced $92 net income and $757.50/month for all 12 months, against the frozen reference's $237.09 net income, $713.90 for January–September and $733.60 for October–December; the $733.60 uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as the correct October-December amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The medical, TANF and shelter errors stay; $713.90 is attributed to the frozen reference, not the convention. Revised on independent review." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It then applied an $805 maximum to all 12 months, although January–September 2026 uses the FY2026 $785 maximum.", + "new": "It then applied an $805 maximum to all 12 months, although the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 $785 maximum for all 12 months of 2026.", + "reason": "Limits the FY2026 $785 maximum to January-September, implying the engine's projected FY2027 maximum governs October-December. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The $805 error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "glm-5.3", + "old": "It also applied an $800 maximum to all 12 months, although January–September uses the FY2026 $785 maximum, which gives $713.90/month.", + "new": "It also applied an $800 maximum to all 12 months instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months; at $785 the frozen reference pays $713.90/month for January–September.", + "reason": "Limits the FY2026 $785 maximum to January-September, implying the FY2027 projection governs October-December. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The $800 error stays; $713.90 stays attributed to the frozen reference." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "The result was $753/month instead of $713.90 for January–September and $733.60 for October–December.", + "new": "The result was $753/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as the correct October-December amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The medical, TANF and shelter errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-sol", + "old": "That inflated net income and cut the benefit to $618/month instead of $713.90 for January–September and $733.60 for October–December.", + "new": "Measured against the frozen reference, leaving out the $453.64 deduction raised net income, and its benefit came to $618/month instead of the frozen reference's $713.90 for January–September and $733.60 for October–December; the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as the correct October-December amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The missed-shelter error stays. Folded in: Calls the model's net income 'inflated' although it is higher only than the frozen reference's defect-based $237.09; us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap): correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). This old lies wholly inside the v3 amendment's new text for this row. Revised on independent review." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It also took a medical deduction for a head who is not SNAP-disabled, and ended at $647/month instead of $713.90 (January–September) and $733.60 (October–December).", + "new": "It also took a medical deduction for a head who is not SNAP-disabled, and ended at $647/month instead of the frozen reference's $713.90 (January–September) and $733.60 (October–December); the latter uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as the correct October-December amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The shelter and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It then applied a single $710.92 monthly amount to all 12 months, missing the rise to $733.60/month for October–December 2026 under FY2027 parameters.", + "new": "It then applied a single $710.92 monthly amount to all 12 months; the frozen reference's rise to $733.60/month for October–December 2026 uses the engine's projected FY2027 parameters, which the release's SNAP convention (c_snap_hold_fy2026) replaces with the FY2026 schedule all year.", + "reason": "Calls not raising the October-December amount an error and states the FY2027 $733.60 as governing. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The TANF and shelter errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "This household has gross income at 45% of the poverty guideline and $237.09 net income, which gives $713.90–$733.60 per month.", + "new": "This household has gross income at 45% of the poverty guideline, and the frozen reference's $237.09 net income gives $713.90–$733.60 per month; the $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as a correct monthly amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The zero-benefit error stays. Revised on independent review." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.3", + "old": "That yields $713.90–$733.60 per month.", + "new": "That yields the frozen reference's $713.90–$733.60 per month; its $733.60 for October–December uses the engine's projected FY2027 schedule, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States $733.60 as a correct monthly amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The no-calculation error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "It also applied a medical deduction the non-SNAP-disabled head doesn't qualify for, which put its $6,400 far below the correct $8,625.89.", + "new": "It also applied a medical deduction the non-SNAP-disabled head doesn't qualify for, which put its $6,400 far below the frozen reference's $8,625.89, which uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 all year.", + "reason": "Calls the frozen $8,625.89 (713.90x9 + 733.60x3) the correct value. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The SSI and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "qwen3.8-max", + "old": "The household is also broad-based categorically eligible through TANF non-cash status, and vehicle value is excluded, so SNAP is $8,625.89, not zero.", + "new": "The household is also broad-based categorically eligible through TANF non-cash status, and vehicle value is excluded, so SNAP is positive, not zero: the frozen reference gives $8,625.89, using the engine's projected FY2027 schedule for October–December, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year.", + "reason": "States the frozen $8,625.89 as the SNAP amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). The resource-test error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "kimi-k2.6", + "old": "It gave no SNAP value and no explanation, so there was no answer to score against the $8,625.89 annual benefit.", + "new": "It gave no SNAP value and no explanation, so there was no answer to score against the frozen reference's $8,625.89 annual benefit.", + "reason": "Presents the frozen $8,625.89 (which carries the engine's projected FY2027 October-December amounts) as 'the annual benefit'; attribution only. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze)." + }, + { + "case_id": "us__scenario_100__snap", + "field": "case_annotation", + "old": "That leaves $237.09 net income, a $71.10 contribution, $713.90/month for January–September (at the $785 maximum) and $733.60/month for October–December (FY2027).", + "new": "That leaves $237.09 net income, a $71.10 contribution and $713.90/month for January–September (at the $785 maximum); the frozen reference's $733.60/month for October–December uses the engine's projected FY2027 schedule, whereas the release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule, including the $785 maximum, for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $713/month ($8,556).", + "reason": "States the FY2027 $733.60 as the October-December amount. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $733.60 (on an $802.60 maximum) is the frozen reference's engine projection (revisions[17] recheck: 8,556). Developer adjudication sentence untouched. Revised on independent review." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "That leaves $139,108 of taxable income, which reaches only the 8% bracket of the 2026-indexed MFJ schedule and produces $5,778 of tax. Subtracting the $1,284.46 in personal and dependent exemption credits gives $4,493.74.", + "new": "That leaves $139,108 of taxable income, which reaches only the 8% bracket. The frozen reference prices it on the engine's projected 2026-indexed MFJ schedule, $5,778 of tax, and subtracts $1,284.46 of projected personal and dependent exemption credits for $4,493.74; the release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $5,896.78 less $1,256 on the same income.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "using 2024 or 2025 brackets that were not indexed for 2026,", + "new": "using 2024 brackets rather than the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026,", + "reason": "The re-judge counts the 2025 brackets as a mistake, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 amounts FTB published for 2026; only the 2024 brackets remain an error." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It priced taxable income with the 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and used the 2025 exemption credits ($1,256). The 2026-indexed schedule (8% starting at ~$117,961) and $1,284.46 of credits apply instead. Together with deducting full charity without the 0.5% AGI floor, this overstated net tax by $122.", + "new": "It priced taxable income with the 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and used the 2025 exemption credits ($1,256); the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. It also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense.", + "reason": "The re-judge calls the 2025 brackets and credits errors, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds them for 2026. The exclusion's alternative_reading allows charity without the federal floor and phases out the IRA deduction (corrected value 4,588.48), and revisions[17] adds back the $337.50 educator expense for a fully law-correct 4,615.48; this model answered 4,615.38.", + "model": "gpt-6-astra" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It used the 'latest published' 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and the 2025 exemption credits of $1,256. The 2026-indexed schedule and $1,284.46 of credits apply instead. Its taxable income also omitted the 0.5%-of-AGI charitable floor, and the stale brackets overstated net tax by $122.", + "new": "It used the 'latest published' 2025 CA MFJ brackets (8% starting at $115,084), giving $5,871.38 on $138,791, and the 2025 exemption credits of $1,256; the release's California convention (c_ca_hold_2025) holds exactly these amounts for 2026, where the frozen reference uses the engine's projected 2026 schedule and $1,284.46 of credits. To reach that taxable income it also deducted full charity and left out the IRA and educator deductions, which is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity). Its $4,615.38 is $121.64 above the frozen reference, $26.90 above the exclusion's corrected value, $4,588.48, which fixes only the first two, and $0.10 below the $4,615.48 that also adds back the educator expense.", + "reason": "The re-judge calls the 2025 brackets and credits stale, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds them for 2026. The exclusion's alternative_reading allows charity without the federal floor and phases out the IRA deduction (corrected value 4,588.48), and revisions[17] adds back the $337.50 educator expense for a fully law-correct 4,615.48; this model answered 4,615.38. Revised on independent review.", + "model": "gpt-6.1-sol" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then priced that income with 2025 brackets that were not indexed, giving ~$5,860 of pre-credit tax versus $5,778 under the 2026 schedule. The bracket error drives the $72 overstatement.", + "new": "It then priced that income with the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, giving ~$5,860 of pre-credit tax where the frozen reference's projected 2026 schedule gives $5,778 on $139,108. That schedule difference drives its $72 overstatement against the frozen reference; against the exclusion's corrected value, $4,588.48, it is $22.48 low, mainly because its answer then implies about $1,294 of exemption credits where the convention's 2025 amounts give $1,256.", + "reason": "The re-judge calls the 2025 brackets an error, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds them for 2026 (credits $153/$475, $1,256 here). With ~$5,859.86 of 2025-schedule tax on $138,647, the $4,566 answer implies $1,293.86 of credits; reference_exclusions.json's corrected value is 4,588.48.", + "model": "gpt-6-sol" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Those brackets start 9.3% at $136,700, whereas under the 2026 schedule this income stays in the 8% bracket for $5,778 of tax before the $1,284.46 credits.", + "new": "Those brackets start 9.3% at $136,700, whereas under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected schedule, this income stays in the 8% bracket (the frozen reference gives $5,778 of tax before its $1,284.46 credits).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. The 2025 joint 8% bracket runs to $145,448, so the 2024-bracket error stands.", + "model": "gpt-5.4-nano" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It got taxable income roughly right (~$138,791 after $21,275 itemized) but priced it at ≈$6,940, about $1,160 more than the 2026 CA MFJ brackets give. Those brackets keep ~$139k in the 8% bracket for $5,778 of tax.", + "new": "It got taxable income roughly right (~$138,791 after $21,275 itemized) but priced it at ≈$6,940, about $1,160 more than the frozen reference's $5,778 and about $1,070 more than the $5,871 that the 2025 brackets, which the release's California convention (c_ca_hold_2025) holds for 2026, give on $138,791. Both that schedule and the engine's projected one keep ~$139k in the 8% bracket.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. Revised on independent review.", + "model": "claude-fable-5" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its taxable income (~$138,309) was close to the correct $139,108, but it guessed bracket tax at ~$6,200 instead of the $5,778 the 2026 CA MFJ schedule produces.", + "new": "Its taxable income (~$138,309) was close to the frozen reference's $139,108, but it guessed bracket tax at ~$6,200, above both the $5,778 the frozen reference's projected 2026 CA MFJ schedule produces and the $5,897 that the 2025 schedule, which the release's California convention (c_ca_hold_2025) holds for 2026, gives on $139,108.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. Revised on independent review.", + "model": "claude-opus-4.7" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Under the 2026 MFJ schedule, income near $139k sits in the 8% bracket for $5,778 of tax.", + "new": "Under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026, as under the frozen reference's projected 2026 MFJ schedule, income near $139k sits in the 8% bracket (for $5,778 of tax in the frozen reference).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "claude-opus-4.8" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its taxable income of ~$138,000 was about right, but it priced it at ~$7,700, roughly $1,900 above the 2026 CA MFJ schedule ($5,778 at $139,108).", + "new": "Its taxable income of ~$138,000 was about right, but it priced it at ~$7,700, roughly $1,900 above the frozen reference's projected 2026 CA MFJ schedule ($5,778 at $139,108) and $1,800 above the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,897 there).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "claude-opus-5" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct personal-plus-dependent credits are $1,284.46.", + "new": "The personal-plus-dependent credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "claude-opus-5" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "while the correct credits are $1,284.46.", + "new": "while the credits are $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "claude-sonnet-5" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "(the total should be $1,284.46).", + "new": "(the total is $1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026, and $1,284.46 in the frozen reference).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "deepseek-v4-flash-0731" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its answer implies ~$5,845 of pre-credit tax, which is above the $5,778 the 2026 CA MFJ brackets produce on $139,108 of taxable income.", + "new": "Its answer implies ~$5,845 of pre-credit tax, above the $5,778 the frozen reference's projected 2026 CA MFJ brackets produce on $139,108 of taxable income but in line with the $5,844 behind the exclusion's corrected value ($4,588.48 plus the $1,256 of 2025 exemption credits that the release's California convention, c_ca_hold_2025, holds for 2026); against that value its $76.52 overstatement is the credit shortfall.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. Revised on independent review.", + "model": "gemini-3-flash-preview" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct path subtracts $20,477 of itemized deductions to reach $139,108 of taxable income, then $5,778 of tax less $1,284.46 of exemption credits.", + "new": "The frozen reference subtracts $20,477 of itemized deductions to reach $139,108 of taxable income, then $5,778 of tax on the engine's projected 2026 schedule less $1,284.46 of exemption credits; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48.", + "reason": "reference_exclusions.json: corrected value 4,588.48 (r02 and r11 with every convention); the frozen 4,493.74 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces. Revised on independent review.", + "model": "gemini-3.1-flash-lite-preview" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct computation is $139,108 taxable, $5,778 of tax, less $1,284.46 of credits, for $4,493.74.", + "new": "The frozen reference computes $139,108 taxable, $5,778 of tax, less $1,284.46 of credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48.", + "reason": "reference_exclusions.json: corrected value 4,588.48 (r02 and r11 with every convention); the frozen 4,493.74 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces. Revised on independent review.", + "model": "gemini-3.5-flash-lite" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The two personal and two dependent exemption credits total $1,284.46.", + "new": "The two personal and two dependent exemption credits total $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($1,284.46 in the frozen reference's projection).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "gpt-6-luna" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its bracket and credit estimates then understated net tax by a further ~$85 compared with the 2026 schedule ($5,778 of tax, $1,284.46 of credits).", + "new": "Its bracket and credit estimates then understated net tax by ~$86 against the frozen reference's projected 2026 schedule ($5,778 of tax, $1,284.46 of credits) and by ~$233 against the 2025 schedule and credits that the release's California convention (c_ca_hold_2025) holds for 2026 ($4,615.38 on $138,791), which is its whole gap to the fully law-correct $4,615.48 that also adds back the educator expense.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. On $138,791 the 2025 schedule gives $5,871.38 - $1,256 = $4,615.38 against this model's $4,382; the projected schedule gives about $4,468. Revised on independent review.", + "model": "gpt-5.6-sol" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It gave a round ~$4,500 guess based on '~165k income' without working out itemized deductions, 2026 brackets or exemption credits. The exact computation is $139,108.47 taxable, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74.", + "new": "It gave a round ~$4,500 guess based on '~165k income' without working out itemized deductions, brackets or exemption credits. The frozen reference computes $139,108.47 taxable, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48.", + "reason": "reference_exclusions.json: corrected value 4,588.48 (r02 and r11 with every convention); the frozen 4,493.74 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces. Revised on independent review.", + "model": "grok-4.3" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then priced that income at ~$5,890, about $160 more than the 2026 CA MFJ brackets produce (~$5,726), for a net overstatement of $106.", + "new": "It then priced that income at ~$5,890, about $160 more than the frozen reference's projected 2026 CA MFJ brackets produce (~$5,726) and about $46 more than the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives (~$5,844), for a net overstatement of $106 against the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "grok-4.5" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It subtracted only $576 of exemption credits instead of $1,284.46 (two personal credits of about $150 each plus two dependent credits of about $490 each). It also priced its $138,491 of taxable income at $5,866, above what the 2026 schedule gives, and deducted charity without the 0.5% AGI floor.", + "new": "It subtracted only $576 of exemption credits instead of $1,256 at the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 (two personal credits of $153 and two dependent credits of $475; the frozen reference's projection gives $1,284.46). It also priced its $138,491 of taxable income at $5,866, above what either that 2025 schedule (~$5,847) or the frozen reference's projected schedule gives; its charity deduction without the 0.5% AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. Revised on independent review.", + "model": "inkling" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct computation is $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74.", + "new": "The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48.", + "reason": "reference_exclusions.json: corrected value 4,588.48 (r02 and r11 with every convention); the frozen 4,493.74 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces. Revised on independent review.", + "model": "kimi-k2.6" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct computation is $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74.", + "new": "The frozen reference computes $139,108.47 of taxable income, $5,778.21 of tax, less $1,284.46 of exemption credits, for $4,493.74, on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the IRA phase-out and charity-floor defects, is $4,588.48.", + "reason": "reference_exclusions.json: corrected value 4,588.48 (r02 and r11 with every convention); the frozen 4,493.74 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces. Revised on independent review.", + "model": "kimi-k3" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also stated that no nonrefundable credits apply, but CA's personal and dependent exemption credits ($1,284.46) are not reduced at ~$160k of AGI.", + "new": "It also stated that no nonrefundable credits apply, but CA's personal and dependent exemption credits ($1,256 at the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026; $1,284.46 in the frozen reference) are not reduced at ~$160k of AGI.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "grok-build-0.1" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "In fact $139,108 of taxable income generates $5,778 of bracket tax, and the $1,284.46 of exemption credits offsets only part of it.", + "new": "In fact $139,108 of taxable income generates $5,897 of bracket tax under the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($5,778 on the frozen reference's projected schedule), and the $1,256 of exemption credits ($1,284.46 in the frozen reference) offsets only part of it.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "minimax-m3" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then priced $138,063 at only $4,301.50, although 2026 brackets give about $5,700.", + "new": "It then priced $138,063 at only $4,301.50, although the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,813 (the frozen reference's projected 2026 brackets, about $5,700).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income.", + "model": "qwen-3.7-max" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It invented a $34,000 standard deduction instead of itemizing $20,477, and it priced $130,894 at $8,100 when 2026 CA MFJ brackets give about $5,160 at that income.", + "new": "It invented a $34,000 standard deduction instead of itemizing $20,477, and it priced $130,894 at $8,100 when the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $5,240 at that income.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]; latest_c_ca_hold_2025.py: 2025 thresholds, $153/$475 credits) holds California's 2025 schedule and credits for 2026; this excluded output's frozen reference keeps the engine's projection, and revisions[17] gives 4,640.78 (= $5,896.78 - $1,256) with conventions on the same taxable income. On $130,894 the 2025 joint schedule gives $5,239.62.", + "model": "qwen3.8-max" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The 2026 indexed MFJ schedule taxes this income at a top rate of 9.3%, giving $41,208.45, and the MFJ exemption credits, reduced by $6 per $2,500 of excess AGI, leave $156.93 to subtract.", + "new": "The frozen reference's projected 2026 MFJ schedule taxes this income at a top rate of 9.3%, giving $41,208.45, and its projected MFJ exemption credits, reduced by $6 per $2,500 of excess AGI, leave $156.93 to subtract. The release's California convention (c_ca_hold_2025) holds the 2025 schedule and credits FTB published, which give $41,369.87 of tax and $150 of credits on the same income.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also kept the 2025 bracket schedule instead of the indexed 2026 thresholds and used a $222 exemption credit instead of $156.93.", + "new": "It also used the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 exemption credit is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI.", + "reason": "The re-judge calls the 2025 schedule an error, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 brackets and credits ($153 each, cut $6 per $2,500 over $504,411) for 2026. On its $498,606 base the 2025 schedule gives $39,247.67 less $222 (7 phase-out steps at $520,901), exactly its $39,025.67; at $536,764.50 the credits are $150 (revisions[17]: 41,219.87 = 1.755.4 + r19).", + "model": "gpt-6-astra" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also held the 2025 bracket schedule constant instead of indexing to 2026 and used $222 of exemption credits instead of $156.93.", + "new": "It also held the 2025 bracket schedule and exemption credit amounts, which the release's California convention (c_ca_hold_2025) holds for 2026, where the frozen reference uses the engine's projected 2026 schedule; its $222 of credits is the 2025 amount phased down at its own understated AGI, against $156.93 in the frozen reference and $150 under the convention at $536,764.50 of AGI.", + "reason": "The re-judge calls holding the 2025 schedule an error, but c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 brackets and credits ($153 each, cut $6 per $2,500 over $504,411) for 2026. On its $498,606 base the 2025 schedule gives $39,247.67 less $222 (7 phase-out steps at $520,901), exactly its $39,025.67; at $536,764.50 the credits are $150 (revisions[17]: 41,219.87 = 1.755.4 + r19).", + "model": "gpt-6.1-sol" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its bracket and exemption-credit math on that base tracks the 2026 schedule, so the whole miss comes from the understated income base.", + "new": "Its bracket and exemption-credit math on that base is consistent with the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 ($38,635.79 of tax on $492,027 before credits) as well as with the frozen reference's projected schedule, so the whole miss comes from the understated income base.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection. On $492,027 the 2025 schedule gives $38,635.79 and the projected one about $38,474, so the $38,343 answer fits either, implying $292.79 or $131.37 of credits, both within two personal credits.", + "model": "gpt-6-sol" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It lands near the target only because its bracket math overstated tax on that base (~$38,800 vs ~$34,856 on the 2026 schedule), and it used $560 of exemption credits instead of $156.93.", + "new": "It lands near the target only because its bracket math overstated tax on that base (~$38,800 vs ~$34,856 on the frozen reference's projected 2026 schedule and ~$35,017 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), and it used $560 of exemption credits instead of $156.93.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "claude-opus-4.7" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its bracket math then understated tax even on that base (~$35,900 on the 2026 schedule, not $32,000), and it ignored the $156.93 exemption credits.", + "new": "Its bracket math then understated tax even on that base (~$35,900 on the frozen reference's projected 2026 schedule and ~$36,068 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026, not $32,000), and it ignored the $156.93 exemption credits.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "claude-sonnet-5" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "But its bracket math produced $42,151 where the 2026 MFJ schedule gives about $40,942 on that base.", + "new": "But its bracket math produced $42,151 where the frozen reference's projected 2026 MFJ schedule gives about $40,942 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,104.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "deepseek-v4-flash-0731" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It gave no derivation, and $31,215 matches about $415,700 of taxable income on the 2026 MFJ schedule, roughly $106,000 below $521,425.72.", + "new": "It gave no derivation, and $31,215 matches about $415,700 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $413,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $106,000 to $108,000 below $521,425.72.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gemini-3.1-flash-lite-preview" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct path taxes $521,425.72 at up to 9.3% for $41,208.45 and subtracts $156.93 of exemption credits.", + "new": "The frozen reference taxes $521,425.72 at up to 9.3% for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40.", + "reason": "reference_exclusions.json: corrected value 40,920.40 (r02 and r11 with every convention); the frozen 41,051.51 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces (+168.36, revisions[17]).", + "model": "gemini-3.1-flash-lite-preview" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It gave no derivation, and $33,816 matches about $443,600 of taxable income on the 2026 MFJ schedule, roughly $78,000 below $521,425.72.", + "new": "It gave no derivation, and $33,816 matches about $443,600 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $441,800 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $78,000 to $80,000 below $521,425.72.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gemini-3.5-flash-lite" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct result taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits.", + "new": "The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40.", + "reason": "reference_exclusions.json: corrected value 40,920.40 (r02 and r11 with every convention); the frozen 41,051.51 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces (+168.36, revisions[17]).", + "model": "gemini-3.5-flash-lite" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The 2026 MFJ schedule taxes that at about $40,650, but its $42,150 overshoots by roughly $1,500 through misapplied bracket thresholds.", + "new": "The frozen reference's projected 2026 MFJ schedule taxes that at about $40,650 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 at about $40,820, so its $42,150 overshoots by roughly $1,300 to $1,500 through misapplied bracket thresholds.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gemini-3.6-flash" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its reasoning mentions 'pre-tax deductions' plus standard/itemized deductions, and $36,918 matches about $477,000 of taxable income on the 2026 MFJ schedule, roughly $44,400 below $521,425.72.", + "new": "Its reasoning mentions 'pre-tax deductions' plus standard/itemized deductions, and $36,918 matches about $477,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $475,200 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $44,400 to $46,300 below $521,425.72.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gemini-3.7-flash" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then reported a round $40,000 rather than the ~$40,450 its own base gives after the $156.93 exemption credits.", + "new": "It then reported a round $40,000 rather than the ~$40,450 its own base gives after the $156.93 exemption credits on the frozen reference's projected amounts (about $40,620 after $150 of credits on the 2025 amounts that the release's California convention, c_ca_hold_2025, holds for 2026).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gemini-3.8-flash" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its taxable income of about $522,970 is within roughly $1,550 of $521,425.72, but its bracket math gave $41,942 where the 2026 MFJ schedule yields about $41,352 on that base. It also claimed no nonrefundable credits, when $156.93 of phased-down exemption credits apply.", + "new": "Its taxable income of about $522,970 is within roughly $1,550 of $521,425.72, but its bracket math gave $41,942 where the frozen reference's projected 2026 MFJ schedule yields about $41,352 on that base and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $41,513. It also claimed no nonrefundable credits, when phased-down exemption credits apply: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "glm-5.3" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It offered only a generic statement, and $20,677 matches about $302,000 of taxable income on the 2026 MFJ schedule.", + "new": "It offered only a generic statement, and $20,677 matches about $302,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $300,500 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gpt-5.4-mini" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It gave no concrete derivation, and $37,099 matches about $479,000 of taxable income on the 2026 MFJ schedule, roughly $42,500 below $521,425.72.", + "new": "It gave no concrete derivation, and $37,099 matches about $479,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $477,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $42,500 to $44,300 below $521,425.72.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "gpt-5.4-nano" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct computation taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits.", + "new": "The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40.", + "reason": "reference_exclusions.json: corrected value 40,920.40 (r02 and r11 with every convention); the frozen 41,051.51 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces (+168.36, revisions[17]).", + "model": "gpt-5.4-nano" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It gave no derivation, and $48,720 matches about $604,000 of taxable income on the 2026 MFJ schedule, roughly $82,000 above $521,425.72.", + "new": "It gave no derivation, and $48,720 matches about $604,000 of taxable income on the frozen reference's projected 2026 MFJ schedule (about $602,100 on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026), roughly $80,600 to $82,500 above $521,425.72.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "grok-4.3" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then claimed about $33,544 of tax on $565,375 of taxable income, where the 2026 MFJ schedule gives about $45,300.", + "new": "It then claimed about $33,544 of tax on $565,375 of taxable income, where the frozen reference's projected 2026 MFJ schedule gives about $45,300 and the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 about $45,457.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "qwen-3.7-max" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It then computed only $29,423 of tax on that amount, while the 2026 MFJ schedule gives about $38,800: about $6,549 through the 8% bracket plus 9.3% above roughly $148,700.", + "new": "It then computed only $29,423 of tax on that amount, while the 2025 schedule that the release's California convention (c_ca_hold_2025) holds for 2026 gives about $38,968: $6,403.94 through the 8% bracket plus 9.3% above $145,448 (the frozen reference's projected schedule gives about $38,800: about $6,549 through the 8% bracket plus 9.3% above roughly $148,700).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection. The fix module's 2025 joint thresholds put the top of the 8% bracket at $145,448.", + "model": "qwen3.8-max" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The only nonrefundable credit here is the phased-down $156.93 MFJ exemption credit, subtracted from $41,208.45 of tax on $521,425.72 of taxable income.", + "new": "The only nonrefundable credit here is the phased-down MFJ exemption credit, subtracted from the tax on $521,425.72 of taxable income: $156.93 from $41,208.45 on the frozen reference's projected 2026 amounts, or $150 from $41,369.87 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "claude-haiku-4.5" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "At this income every marginal dollar is taxed at 9.3%, giving $41,208.45 before the $156.93 exemption credits.", + "new": "At this income every marginal dollar is taxed at 9.3%, giving $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "claude-opus-5" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Tax is $41,208.45 before the $156.93 of exemption credits.", + "new": "Tax is $41,208.45 before $156.93 of exemption credits on the frozen reference's projected 2026 amounts, or $41,369.87 before $150 on the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "minimax-m3" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "The correct computation taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits, giving $41,051.51.", + "new": "The frozen reference taxes $521,425.72 for $41,208.45 and subtracts $156.93 of exemption credits on the engine's projected 2026 amounts, giving $41,051.51; the exclusion's corrected value, which uses the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 and fixes the engine defects, is $40,920.40.", + "reason": "reference_exclusions.json: corrected value 40,920.40 (r02 and r11 with every convention); the frozen 41,051.51 uses the projected schedule that c_ca_hold_2025 (revisions[1]) replaces (+168.36, revisions[17]).", + "model": "kimi-k2.6" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also treated the exemption credits as fully phased out, when $156.93 remains after the $6-per-$2,500 reduction.", + "new": "It also treated the exemption credits as fully phased out, when some remain after the $6-per-$2,500 reduction: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "claude-sonnet-5.5" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also said no nonrefundable credits apply at this AGI, but $156.93 of MFJ exemption credits survive the phase-down.", + "new": "It also said no nonrefundable credits apply at this AGI, but MFJ exemption credits survive the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "deepseek-v4-pro-0813" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its $310 exemption credit ignores the $6-per-$2,500 phase-down, which leaves $156.93.", + "new": "Its $310 exemption credit ignores the $6-per-$2,500 phase-down, which leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "deepseek-v4.1-flash" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also used about $280 of exemption credits instead of the $156.93 that survives the phase-down.", + "new": "It also used about $280 of exemption credits instead of what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "grok-4.5" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "It also claimed $314 of exemption credits where the $6-per-$2,500 phase-down leaves $156.93.", + "new": "It also claimed $314 of exemption credits where the $6-per-$2,500 phase-down leaves $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026 ($156.93 in the frozen reference).", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "grok-4.7" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "old": "Its $258 exemption credit is also larger than the $156.93 that survives the phase-down.", + "new": "Its $258 exemption credit is also larger than what survives the phase-down: $150 at the frozen reference's AGI under the 2025 amounts that the release's California convention (c_ca_hold_2025) holds for 2026, and $156.93 in the frozen reference.", + "reason": "c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds the 2025 schedule and credits for 2026; revisions[17] credits it with +168.36 (1.755.4 + r19 = 41,219.87: $41,369.87 of tax less $150 of credits at the frozen AGI), while the frozen reference keeps the engine's projection.", + "model": "kimi-k3" + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The closest group disallowed the $721 IRA deduction under the active-participant phaseout, and some also dropped the $580 non-itemizer charitable deduction or claimed auto-loan interest, which the reference does not allow.", + "new": "The closest group disallowed the $721 IRA deduction under the active-participant phaseout, as the exclusion's corrected value ($30,702.59) does under 26 U.S.C. 219(g) and the frozen reference, which has no active-participant phase-out, does not; some also dropped the $580 non-itemizer charitable deduction or claimed auto-loan interest, which the reference does not allow.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The note listed disallowing the IRA deduction among the failures." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "The reference deducts the full $721 because no plan-coverage flag is listed and unlisted booleans default to false. The two readings (401(k) deferrals imply active participation, or the unlisted coverage flag is false) explain the entire $158.63 gap, which is 22% × $721.", + "new": "The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. That $721 explains the entire $158.63 gap to the frozen reference (22% × $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row presented the defect as a reading of an unlisted coverage flag." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "The reference deducts the full $721 because no plan-coverage flag is listed and unlisted booleans are false. Everything else matches, so the $158.63 gap is exactly 22% of $721 and comes down to those two readings.", + "new": "The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion; the exclusion's corrected value disallows it under 26 U.S.C. 219(g), as the model did. Everything else matches, so the $158.63 gap to the frozen reference comes from the $721 (22% of $721 is $158.62), and its $30,702.54 is $0.05 below the corrected value, $30,702.59.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row presented the defect as a reading of an unlisted coverage flag." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "Its joint AGI of $247,385 leaves out the $721 traditional IRA contribution, which corresponds to the active-participant reading because the spouse makes 401(k) deferrals. The reference deducts the full $721 because no plan-coverage flag is listed and unlisted booleans are false. That difference alone produces the $158.63 gap.", + "new": "Its joint AGI of $247,385 leaves out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That difference alone produces the $158.63 gap to the frozen reference, and its $30,702.54 is $0.05 below the corrected value.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row presented the defect as a reading of an unlisted coverage flag. Revised on independent review." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It started from $247,385, leaving out the $721 traditional IRA contribution under the reading that the spouse's 401(k) deferrals make them an active participant. The reference deducts the full $721 because no plan-coverage flag is listed and unlisted booleans are false. That is the only difference, and it produces the $158.63 gap.", + "new": "It started from $247,385, leaving out the $721 traditional IRA contribution, as the exclusion's corrected value, $30,702.59, does: under 26 U.S.C. 219(g) the spouse's 401(k) deferrals make the spouse an active participant, and above the joint phase-out range the contribution is not deductible. The frozen reference deducts the full $721 because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion. That is the only difference, and it produces the $158.63 gap to the frozen reference; its $30,702.54 is $0.05 below the corrected value.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row presented the defect as a reading of an unlisted coverage flag. Revised on independent review." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "The $30,830 also leaves out the $721 above-the-line traditional IRA deduction and the $580 OBBBA non-itemizer cash charitable deduction, so its taxable income of $190,185 is $1,301 too high.", + "new": "The $30,830 also leaves out the $580 OBBBA non-itemizer cash charitable deduction and the $721 above-the-line traditional IRA deduction, so its taxable income of $190,185 is $1,301 above the frozen reference's; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), leaving only the $580 missing.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It also left out the $721 traditional IRA deduction and the $580 non-itemizer cash charitable deduction, so net deductions were $373 too low and tax was $82 too high.", + "new": "It also left out the $580 non-itemizer cash charitable deduction and the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Against the frozen reference net deductions were $373 too low and tax $82 too high; against the corrected value, $30,702.59, net deductions were $348 too high and tax $76.59 too low.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 30626." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It also left out the $721 IRA deduction and the $580 non-itemizer charitable deduction, so taxable income was $215,185 instead of $188,884.", + "new": "It also left out the $580 non-itemizer charitable deduction and the $721 IRA deduction, so taxable income was $215,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "It also left out the $721 IRA deduction, which puts taxable income $15,687 too low.", + "new": "It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income is $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 27092.78, 22% of $16,408 below the corrected value." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "After also leaving out the $721 IRA deduction, taxable income was $15,687 too low.", + "new": "It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income was $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 27092.78, 22% of $16,408 below the corrected value." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It excluded $16,408 of employer-plan premiums from wages that never contained them, which put AGI at $230,977 instead of $246,664. With the $721 IRA deduction also missing, taxable income came out $15,687 too low.", + "new": "It excluded $16,408 of employer-plan premiums from wages that never contained them, which put AGI at $230,977 instead of $246,664, the frozen reference's AGI net of a $721 IRA deduction. The model left out that deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), so taxable income came out $15,687 below the frozen reference's and the whole gap to the corrected value comes from the $16,408.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 27092.78, 22% of $16,408 below the corrected value." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "That is $15,687 below the correct $188,884; the $721 IRA deduction is also missing.", + "new": "That is $15,687 below the frozen reference's $188,884. It also left out the $721 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so the whole gap to the corrected value comes from the $16,408.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row gave the frozen $188,884 as correct and called leaving out the IRA deduction an error." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It claimed the $928 of auto loan interest as a qualified vehicle-loan-interest deduction, which the reference does not allow for this household, and it left out the $721 traditional IRA deduction. Taxable income ended up $207 too low, producing the $45.53 shortfall.", + "new": "It claimed the $928 of auto loan interest as a qualified vehicle-loan-interest deduction, which the reference does not allow for this household, and it left out the $721 traditional IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)). Taxable income ended up $207 below the frozen reference's, producing the $45.53 shortfall; against the corrected value, $30,702.59, the $928 deduction alone leaves it $204.21 short.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 30498.38." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It never took the OBBBA non-itemizer cash charitable deduction of $580 (available from 2026) or the $721 above-the-line IRA deduction, so taxable income was $1,301 too high.", + "new": "It never took the OBBBA non-itemizer cash charitable deduction of $580 (available from 2026) or the $721 above-the-line IRA deduction, so taxable income was $1,301 above the frozen reference's. The frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so against the corrected value, $30,702.59, only the $580 is missing and its $30,830.14 is $127.55 high.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error; predictions.csv has this model at 30830.14." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "It left out the $580 non-itemizer cash charitable deduction and the $721 IRA deduction, so taxable income was $190,185 instead of $188,884.", + "new": "It left out the $580 non-itemizer cash charitable deduction and the $721 IRA deduction, so taxable income was $190,185 instead of the frozen reference's $188,884; the frozen reference takes the $721, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), so of the two omissions only the $580 is an error.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called leaving out the IRA deduction an error." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It correctly deducted the $721 IRA contribution and used the $32,200 standard deduction, but it never applied the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium or the $580 non-itemizer charitable deduction.", + "new": "It deducted the $721 IRA contribution, as the frozen reference does although the exclusion's corrected value disallows it under the active-participant phase-out (26 U.S.C. 219(g)), and used the $32,200 standard deduction, but it never applied the $25,000 qualified overtime deduction on the spouse's $44,647 FLSA premium or the $580 non-itemizer charitable deduction.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row called deducting the IRA contribution correct." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "Its $33,473 corresponds to taxable income of about $202k, some $13k above the correct $188,884, so it did not apply the full stack of $32,200 standard, $25,000 overtime, $580 charitable and $721 IRA deductions.", + "new": "Its $33,473 corresponds to taxable income of about $202k, some $13k above the frozen reference's $188,884, so it did not apply the full stack of $32,200 standard, $25,000 overtime and $580 charitable deductions (the frozen reference also deducts the $721 IRA contribution, which the exclusion's corrected value disallows under the active-participant phase-out, 26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row gave the frozen $188,884 as correct and counted the IRA deduction in the required stack." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "It stated taxable income of about $229,000, roughly $40k above the correct $188,884, because it did not take the $25,000 overtime deduction on top of the $32,200 standard deduction.", + "new": "It stated taxable income of about $229,000, roughly $40k above the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), because it did not take the $25,000 overtime deduction on top of the $32,200 standard deduction.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row gave the frozen $188,884, which includes the $721 IRA deduction, as the correct taxable income." + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Its $20,460 corresponds to taxable income of only about $143k, some $46k below the correct $188,884, so its deduction stack was badly overstated.", + "new": "Its $20,460 corresponds to taxable income of only about $143k, some $46k below the frozen reference's $188,884 (which the exclusion's corrected value raises by disallowing the $721 IRA deduction), so its deduction stack was badly overstated.", + "reason": "reference_exclusions.json (scenario_049 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), its alternative_reading makes a 401(k) deferrer (the spouse defers $15,436 per prompt.md) an active participant who deducts nothing above $149,000 joint (corrected value 30,702.59), and revisions[17] finds no 219(g) or active-participant code in the engine. The row gave the frozen $188,884, which includes the $721 IRA deduction, as the correct taxable income." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Idaho taxable income is federal AGI of $66,968.68 (full gross wages; the ESI premiums do not reduce it) minus the $16,100 standard deduction, which gives $50,868.68. Idaho's 2026 flat 5.3% rate (cut from 5.695% by 2025 HB 40) applies only above the indexed single zero-rate bracket of $4,920: 0.053 × $45,948.68 = $2,435.28.", + "new": "The frozen reference's Idaho taxable income is federal AGI of $66,968.68 (full gross wages; the ESI premiums do not reduce it) minus the $16,100 standard deduction, which gives $50,868.68; it leaves out Idaho's subtraction of the head's $5,000 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes on the assumption that they are not paid through a pre-tax salary reduction. Idaho's 2026 flat 5.3% rate (cut from 5.695% by 2025 HB 40) applies only above the single zero-rate bracket, which the frozen reference sets at the engine's projected $4,920: 0.053 × $45,948.68 = $2,435.28. The corrected value uses the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026: 0.053 × ($45,868.68 − $4,811) = $2,176.06; gpt-6-astra's $2,176.07 is $0.013 above it.", + "reason": "The note gave AGI minus the standard deduction and the indexed $4,920 bracket as Idaho's 2026 law; predictions.csv has gpt-6-astra at 2176.07. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched); reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "The model applied 5.3% to its entire taxable income and omitted Idaho's $4,920 single zero-rate bracket. It also subtracted the $950 auto loan interest from AGI, so AGI came out as $66,019 instead of $66,969. The correct tax is 0.053 × ($50,869 − $4,920) = $2,435.", + "new": "The model applied 5.3% to its entire taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference). It also subtracted the $950 auto loan interest from AGI, so AGI came out as $66,019 instead of $66,969. The frozen reference's tax is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row gave the projected $4,920 as Idaho's bracket and the frozen $2,435 as correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The model used Idaho's repealed graduated brackets (1%–6.5%, plus an invented 7.15% tier) instead of the flat 5.3% rate above the $4,920 zero bracket.", + "new": "The model used Idaho's repealed graduated brackets (1%–6.5%, plus an invented 7.15% tier) instead of the flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's zero bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The model used the superseded 5.695% rate and a stale $4,673 zero-bracket threshold instead of 5.3% above $4,920.", + "new": "The model used the superseded 5.695% rate and a stale $4,673 zero-bracket threshold instead of 5.3% above $4,811, the threshold the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920.", + "reason": "The row graded the model's zero bracket against the projected $4,920. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "The model applied the superseded 5.695% rate with no zero-rate bracket instead of 5.3% above $4,920.", + "new": "The model applied the superseded 5.695% rate with no zero-rate bracket instead of 5.3% above the zero-rate threshold ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as the threshold the model should have used. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The correct computation is 5.3% × ($66,969 − $16,100 − $4,920) = $2,435.", + "new": "The frozen reference's computation is 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket, which overstated the tax by about $261.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference.", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's $4,920 single zero-rate bracket, which overstated the tax by about $261.", + "new": "The model applied 5.3% to the full $50,869 taxable income and omitted Idaho's single zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference), which overstated the tax by about $261 against the frozen reference.", + "reason": "The row gave the projected $4,920 as Idaho's bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "Idaho instead applies 5.3% to taxable income above the $4,920 zero bracket after the $16,100 standard deduction.", + "new": "Idaho instead applies 5.3% to taxable income above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference) after the $16,100 standard deduction; the exclusion's corrected value also subtracts the $5,000 of premiums, assuming they are not paid pre-tax.", + "reason": "The row gave the frozen computation, without the premium subtraction and with the projected $4,920, as Idaho's rule. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "The correct computation is 5.3% × ($50,869 − $4,920).", + "new": "The frozen reference's computation is 5.3% × ($50,869 − $4,920); the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "The correct result is 0.053 × ($50,869 − $4,920) = $2,435.", + "new": "The frozen reference's result is 0.053 × ($50,869 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen $2,435 correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The model applied a superseded rate of about 5.69% to all of roughly $49,852 of taxable income instead of 5.3% on only the income above the $4,920 zero-rate bracket.", + "new": "The model applied a superseded rate of about 5.69% to all of roughly $49,852 of taxable income instead of 5.3% on only the income above the zero-rate bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as the bracket the model should have used. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "Its $3,450 is about 6.8% of the correct $50,869 base, so it either used a rate above Idaho's flat 5.3% or skipped most of the $16,100 standard deduction. Either way, it applied no $4,920 zero-rate bracket; the correct tax is $2,435.", + "new": "Its $3,450 is about 6.8% of the frozen reference's $50,869 base, so it either used a rate above Idaho's flat 5.3% or skipped most of the $16,100 standard deduction. Either way, it applied no $4,920 zero-rate bracket; the frozen reference's tax is $2,435, and the exclusion's corrected value is $2,176.06.", + "reason": "The row called the frozen $50,869 and $2,435 correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It also omitted the $4,920 zero-rate bracket; the correct computation is 5.3% × ($50,869 − $4,920).", + "new": "It also omitted the $4,920 zero-rate bracket; the frozen reference's computation is 5.3% × ($50,869 − $4,920), and the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation correct. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.2", + "old": "Idaho actually subtracts only the $16,100 standard deduction, leaving $50,869, which is taxed at 5.3% above $4,920 for $2,435.", + "new": "The frozen reference subtracts only the $16,100 standard deduction, leaving $50,869, which it taxes at 5.3% above the engine's projected $4,920 for $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row gave the frozen computation as what Idaho subtracts and taxes. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Its $2,050 is an effective rate of about 4.0% on the $50,869 base, well short of the correct 5.3% × $45,949 = $2,435 under Idaho's flat tax above the $4,920 zero bracket.", + "new": "Its $2,050 is an effective rate of about 4.0% on the $50,869 base, well short of the frozen reference's 5.3% × $45,949 = $2,435 (above the engine's projected $4,920 zero bracket) and of the exclusion's corrected value, $2,176.06.", + "reason": "The row called the frozen $2,435 correct and the projected $4,920 Idaho's bracket. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "Its $1,195 is an effective rate of about 2.3% on the $50,869 taxable base, far below Idaho's flat 5.3% on the $45,949 above the $4,920 zero bracket, which yields $2,435.", + "new": "Its $1,195 is an effective rate of about 2.3% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06.", + "reason": "The row gave the frozen computation as Idaho's tax. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "Its $1,588 is an effective rate of only about 3.1% on the $50,869 taxable base, far below Idaho's flat 5.3% on the $45,949 above the $4,920 zero bracket, which yields $2,435.", + "new": "Its $1,588 is an effective rate of only about 3.1% on the $50,869 taxable base, far below Idaho's flat 5.3%, which the frozen reference applies to the $45,949 above the engine's projected $4,920 zero bracket for $2,435; the exclusion's corrected value is $2,176.06.", + "reason": "The row gave the frozen computation as Idaho's tax. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "The model used stale parameters: the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100, and the 2024 zero-rate threshold of $4,489 instead of the 2026 figure of $4,920. Both errors enlarged the income taxed at 5.3% by $781.", + "new": "The model used stale parameters: the 2025 standard deduction of $15,750 instead of the 2026 figure of $16,100, and a $4,489 zero-rate threshold instead of the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026 (the frozen reference uses the engine's projected $4,920). Both enlarged the income taxed at 5.3%, by $672 against the convention's threshold and $781 against the frozen reference.", + "reason": "The row gave the projected $4,920 as Idaho's 2026 figure. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched). Revised on independent review." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "The model used a zero-rate threshold of $4,946 instead of the 2026 indexed single threshold of $4,920. That understated the income taxed at 5.3% by $26 and the tax by $1.36.", + "new": "The model used a zero-rate threshold of $4,946, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $135 less income at 5.3% than the $4,811 threshold does, and $26 less than the frozen reference ($1.36 of tax).", + "reason": "The row gave the projected $4,920 as the 2026 indexed threshold. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The model applied 'progressive' rates to the $50,869 taxable income, but Idaho has a single flat 5.3% rate above the $4,920 zero bracket.", + "new": "The model applied 'progressive' rates to the $50,869 taxable income, but Idaho has a single flat 5.3% rate above the zero bracket ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as Idaho's zero bracket. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "The model used a zero-rate threshold of about $4,800 instead of the 2026 indexed single threshold of $4,920, taxing $46,069 at 5.3% instead of $45,949. That overstated the tax by $6.38.", + "new": "The model used a zero-rate threshold of about $4,800, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, taxing $46,069 at 5.3% instead of $46,058 under the convention or $45,949 in the frozen reference. That put its tax $6.38 above the frozen reference and $0.60 above the $2,441.06 that the $4,811 threshold gives before the premium subtraction.", + "reason": "The row gave the projected $4,920 as the 2026 indexed threshold. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched); revisions[17] gives 2,441.057 for this output under the conventions before the premium subtraction." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "The model used a $16,150 standard deduction instead of $16,100 and a $4,942 zero-rate threshold instead of $4,920, understating the income taxed at 5.3% by $72. It then rounded down, landing $4.28 below $2,435.28.", + "new": "The model used a $16,150 standard deduction instead of $16,100 and a $4,942 zero-rate threshold, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920, understating the income taxed at 5.3% by $72 against the frozen reference and $181 against the convention. It then rounded down, landing $4.28 below the frozen reference's $2,435.28.", + "reason": "The row graded the model's threshold against the projected $4,920. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "It then arrived at $2,702 through an unexplained adjustment instead of computing 5.3% × ($66,969 − $16,100 − $4,920) = $2,435.", + "new": "It then arrived at $2,702 through an unexplained adjustment instead of computing the frozen reference's 5.3% × ($66,969 − $16,100 − $4,920) = $2,435; the exclusion's corrected value, $2,176.06, also subtracts the $5,000 of premiums (assuming they are not paid pre-tax) and uses the $4,811 zero bracket that c_id_hold_2025 keeps for 2026.", + "reason": "The row gave the frozen computation as the one the model should have done. reference_exclusions.json (scenario_053 state, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 2,176.057, which revisions[17] reproduces as 0.053 x (45,868.68 - 4,811) and notes that it assumes the $5,000 is not paid through a pre-tax salary reduction; c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched)." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It deducted the full $12,621 of charity without the 0.5%-of-AGI floor ($798) and left the $144 IRA and $338 educator deductions out of CA AGI ($160,066 instead of $159,585), putting taxable income at $138,791.", + "new": "It deducted the full $12,621 of charity and left the $144 IRA and $338 educator deductions out of CA AGI ($160,066), putting taxable income at $138,791; that is how the record corrects the engine's California defects (the federal charity floor, the IRA phase-out and educator-expense conformity), where the frozen reference applies the $798 floor and both deductions ($159,585 of CA AGI, $139,108 taxable).", + "reason": "The row called defect-free treatments errors. reference_exclusions.json (scenario_099) records the IRA phase-out and federal charity floor as engine defects, and revisions[17].excluded_outputs_rechecked (scenario_099) adds educator-expense conformity and gives the fully law-correct 4,615.48 on this taxable income; companion to the amendment of this row's bracket sentence." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "It then reported $36,767, far below the ~$48,050 that the 9.3% bracket gives on its own $595,000 base.", + "new": "It then reported $36,767, far below what the California schedule gives on its own $595,000 base (about $48,050 on the frozen reference's projected 2026 schedule, and slightly more on the 2025 schedule that the release's California convention, c_ca_hold_2025, holds for 2026).", + "reason": "The ~$48,050 is the frozen reference's projected 2026 schedule; c_ca_hold_2025 (reference_outputs.csv.meta.json revisions[1]) holds California's 2025 amounts, whose lower thresholds give more tax on the same base. The model's inconsistency stays the diagnosis." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It itemized $22,294.58 including charity and employee expenses instead of $15,338.79, so taxable income was $498,606 rather than $521,425.72.", + "new": "It itemized $22,294.58 including charity and employee expenses, which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading ($17,280 of real estate taxes, $2,574 of charity and $3,429.98 of employee expenses above 2% of AGI, less the $989.40 high-income limitation, 6% of AGI above $504,411), where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing $15,338.79; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference.", + "reason": "reference_exclusions.json (scenario_005) records as an engine defect that PolicyEngine applies the federal charitable floor and the suspension of miscellaneous deductions to California itemized deductions; 17,280 + 2,574 + (13,848 − 2% × 520,901) − 6% × (520,901 − 504,411) = 22,294.58 from prompt.md's amounts, and 17,280 − 6% × (536,764.50 − 504,411) = 15,338.79. Revised on independent review." + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It itemized $22,294.58 (charity plus employee expenses above 2%, limited) instead of the $15,338.79 real-estate-tax-only figure, so taxable income was $498,606 rather than $521,425.72.", + "new": "It itemized $22,294.58 (charity plus employee expenses above 2%, limited), which is California's itemized deduction at its $520,901 of AGI under the exclusion's corrected reading, where the frozen reference applies the federal charity floor and suspends the employee expenses, allowing the $15,338.79 real-estate-tax-only figure; with its understated AGI, taxable income came to $498,606 against $521,425.72 in the frozen reference.", + "reason": "reference_exclusions.json (scenario_005) records as an engine defect that PolicyEngine applies the federal charitable floor and the suspension of miscellaneous deductions to California itemized deductions; 17,280 + 2,574 + (13,848 − 2% × 520,901) − 6% × (520,901 − 504,411) = 22,294.58 from prompt.md's amounts." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-fable-5", + "old": "It left out the $453.64 utility-allowance excess shelter deduction, so its $89 net income understates the correct $237.09.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), and its $89 net income understates the $691 that correction gives (the frozen reference's is $237.09).", + "reason": "Calls omitting the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). TANF, medical and $758 x 12 arithmetic errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It also used a $214 standard deduction instead of $209 and assumed no shelter deduction, when the utility allowance gives $453.64.", + "new": "It also used a $214 standard deduction instead of $209. It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.", + "reason": "Presents the utility allowance's $453.64 deduction as due, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The $214 standard deduction, TANF and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct steps are $998.31 gross (wages plus TANF), minus the $209 standard, $98.58 earned-income and $453.64 excess shelter deductions. That gives $237.09 net and $713.90/month for January–September.", + "new": "The frozen reference's steps are $998.31 gross (wages plus TANF), minus the $209 standard, $98.58 earned-income and $453.64 excess shelter deductions, giving $237.09 net and $713.90/month for January–September. Its $453.64 comes from engine defect r30_snap_heat_and_eat_sua: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended. The record's correction takes no utility-allowance shelter deduction and gives $691 net ($998.31 − $209 − $98.58, rounded), a $208 contribution and $577/month.", + "reason": "Presents the $453.64 deduction, $237.09 net and $713.90/month as the correct steps, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The no-formula error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "But a missing deduction can't lower a benefit already computed without it, and the utility allowance actually gives a $453.64 shelter deduction.", + "new": "But a missing deduction can't lower a benefit already computed without it. Taking no shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 utility-allowance shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.", + "reason": "Says the utility allowance 'actually gives' the $453.64 deduction, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF, $768 maximum and self-reduction errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "It used the outdated $766 maximum allotment instead of FY2026's $785 and worked from wages only, leaving out $505.39/month of TANF and the $453.64 excess shelter deduction.", + "new": "It used the outdated $766 maximum allotment instead of FY2026's $785 and worked from wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The $766 maximum, TANF and self-reduction errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5", + "old": "It counted only wages and applied only the standard and earned-income deductions, leaving out both the $505.39/month of TANF income and the $453.64 excess shelter deduction.", + "new": "It counted only wages and applied only the standard and earned-income deductions, leaving out the $505.39/month of TANF income. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended.", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The TANF and formula errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It also left out the $505.39/month of TANF from gross income and the $453.64 excess shelter deduction.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF from gross income.", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay; the next sentence's $237.09 is attributed to the frozen reference by v3." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It counted wages only, leaving out $505.39/month of TANF, and left out the $453.64 utility-allowance shelter deduction. That gives $72.67 net instead of $237.09, with a $784 maximum applied flat for 12 months.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it counted wages only, leaving out $505.39/month of TANF. That gives $72.67 net instead of the $691 the correction gives (the frozen reference has $237.09), with a $784 maximum applied flat for 12 months.", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The $228 standard deduction, medical, TANF and $784 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It also left out the $505.39/month of TANF income and the $453.64 shelter deduction, which together give $237.09 net and a $71.10 monthly contribution.", + "new": "It also left out the $505.39/month of TANF income, which with no utility-allowance shelter deduction gives $691 net and a $208 monthly contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error and presents $237.09 net and $71.10 as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "It also subtracted a $100.67 medical deduction for a head who is not SNAP-disabled and left out both the $505.39/month of TANF and the $453.64 shelter deduction.", + "new": "It also subtracted a $100.67 medical deduction for a head who is not SNAP-disabled and left out the $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The overtime, medical, TANF, $797 maximum and $221 standard deduction errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "It correctly counted TANF as unearned income, but left out the $453.64 excess shelter deduction from the utility allowance, which left its net income at $602 instead of $237.09.", + "new": "It correctly counted TANF as unearned income, though at $458.54/month rather than $505.39, and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). Its net income came to $602, against $691 under that correction and the frozen reference's $237.09.", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, $204 standard deduction and $768 maximum errors stay. Revised on independent review." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "That explanation understated TANF ($335/month vs $505.39), took a $140.67 medical deduction for a head who is not SNAP-disabled, and left out the $453.64 excess shelter deduction.", + "new": "That explanation understated TANF ($335/month vs $505.39) and took a $140.67 medical deduction for a head who is not SNAP-disabled. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Lists leaving out the $453.64 shelter deduction among errors, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The value mismatch, TANF, medical, $198 standard deduction and $768 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It counted wages only, leaving out $505.39/month of TANF, and left out the $453.64 shelter deduction.", + "new": "It counted wages only, leaving out $505.39/month of TANF. It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF and $805 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "It also left out $505.39/month of TANF from gross income and the $453.64 excess shelter deduction. That gives $54 net income instead of $237.09, with a $781 maximum instead of $785.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out $505.39/month of TANF from gross income. That gives $54 net income instead of the $691 the correction gives (the frozen reference has $237.09), with a $781 maximum instead of $785.", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF and $781 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "It ignored the 30% of net income ($71.10/month on $237.09 of net income from wages plus TANF) that must be subtracted.", + "new": "It ignored the 30% of net income that must be subtracted: $208/month on the $691 of net income from wages plus TANF under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10/month on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 net and the $71.10 contribution as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The no-contribution error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The correct steps give $237.09 net income, a $71.10 contribution and $713.90/month for January–September.", + "new": "The record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, gives $691 net income, a $208 contribution and $577/month; the frozen reference's $237.09 net income, $71.10 contribution and $713.90/month for January–September also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09, $71.10 and $713.90 as the correct steps, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The self-contradicting $6,128 error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "It counted TANF as unearned income but left out the $453.64 utility-allowance excess shelter deduction, which pushed implied net income to ~$544 and the benefit to $604.75/month.", + "new": "It counted TANF as unearned income and took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended); its implied net income was ~$544 and its benefit $604.75/month.", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and $768 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It applied a disabled-head medical deduction the head doesn't qualify for (no SSI or SSDI), computed income from wages only and left out $505.39/month of TANF, and assumed no shelter deduction instead of $453.64.", + "new": "It assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it applied a disabled-head medical deduction the head doesn't qualify for (no SSI or SSDI) and computed income from wages only, leaving out $505.39/month of TANF.", + "reason": "Calls assuming no shelter deduction instead of $453.64 an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It never subtracted 30% of the $237.09 net income that comes from wages plus TANF after deductions.", + "new": "It never subtracted 30% of net income: $691 from wages plus TANF after the standard and earned-income deductions under the record's correction of engine defect r30_snap_heat_and_eat_sua, or the frozen reference's $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 as the net income from wages plus TANF, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The value mismatch, $768 maximum and no-contribution errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It deducted medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF and the $453.64 shelter deduction, and used the FY2025 $204 standard deduction and $768 maximum. That gave $97 net income instead of $237.09 and $739/month applied flat for 12 months.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used the FY2025 $204 standard deduction and $768 maximum. That gave $97 net income instead of the $691 the correction gives (the frozen reference has $237.09) and $739/month applied flat for 12 months.", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF, $204 and $768 errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "glm-5.2", + "old": "The head isn't SNAP-disabled without SSI or SSDI, and SNAP income also includes $505.39/month of TANF, giving $237.09 net.", + "new": "The head isn't SNAP-disabled without SSI or SSDI, and SNAP income also includes $505.39/month of TANF, giving $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 as the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The premium-as-medical, TANF and $800 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "glm-5.3", + "old": "It correctly took no medical deduction, but left out the $505.39/month of TANF income and the $453.64 excess shelter deduction and used a $215 standard deduction instead of $209.", + "new": "It correctly took no medical deduction and, as the record's correction of engine defect r30_snap_heat_and_eat_sua does, no utility-allowance shelter deduction (the frozen reference's $453.64 excess shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), but it left out the $505.39/month of TANF income and used a $215 standard deduction instead of $209.", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The TANF, $215 standard deduction and $800 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "It also ignored the $71.10 monthly contribution from $237.09 of net income.", + "new": "It also ignored the monthly contribution: $208 on $691 of net income under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction, or the frozen reference's $71.10 on $237.09, which also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $71.10 on $237.09 as the correct contribution, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The above-maximum and no-contribution errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.5", + "old": "The head receives no SSI or SSDI and gets no SNAP medical deduction, and wages plus $505.39/month of TANF leave $237.09 net.", + "new": "The head receives no SSI or SSDI and gets no SNAP medical deduction, and wages plus $505.39/month of TANF leave $691 net under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 as the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The premium-as-medical, TANF and $767 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It left out the $505.39/month of TANF that brings net income to $237.09, which requires a $71.10 monthly contribution.", + "new": "It left out the $505.39/month of TANF that brings net income to $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, which requires a $208 monthly contribution; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 net and $71.10 as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The zero-net and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It also ignored $505.39/month of TANF income, so net income is $237.09 and the benefit is $713.90/month, not the $785 maximum.", + "new": "It also ignored $505.39/month of TANF income, so under the record's correction of engine defect r30_snap_heat_and_eat_sua net income is $691 and the benefit $577/month, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 net and $713.90/month as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The actual unearned income is $505.39/month of TANF, offset by a $453.64 shelter deduction.", + "new": "The actual unearned income is $505.39/month of TANF; the record's correction of engine defect r30_snap_heat_and_eat_sua takes no utility-allowance shelter deduction, while the frozen reference offsets the TANF with a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents the $453.64 shelter deduction as applying, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The SSI and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-astra", + "old": "With $505.39/month of TANF, net income is $237.09 and the contribution is $71.10/month, so the benefit is below the $785 maximum.", + "new": "With $505.39/month of TANF, net income is $691 and the contribution $208/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, so the benefit is below the $785 maximum; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 net and $71.10 as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It computed income from wages only with no shelter costs, leaving out both $505.39/month of TANF and the $453.64 utility-allowance shelter deduction.", + "new": "It computed income from wages only with no shelter costs, leaving out $505.39/month of TANF; taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua, since the frozen reference's $453.64 comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The TANF error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6-sol", + "old": "It counted TANF cash as income but applied only the earned-income and standard deductions, leaving out the $453.64 excess shelter deduction from the utility allowance.", + "new": "It counted TANF cash as income and applied only the earned-income and standard deductions, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference also takes a $453.64 excess shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). The correction gives $691 net income and $577/month, $41 below its $618; at the $785 maximum, $618 implies about $555 of net income, so its estimated TANF was about $135/month below the $505.39.", + "reason": "Calls leaving out the $453.64 shelter deduction an error, yet the record's r30 correction applies only the earned-income and standard deductions; us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap): correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). Revised on independent review." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It counted TANF income but explicitly assumed no shelter deduction, dropping the $453.64 utility-allowance excess shelter deduction.", + "new": "It counted TANF income and explicitly assumed no shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does; the frozen reference's $453.64 utility-allowance excess shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls assuming no shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.3", + "old": "It returned $0 without doing any SNAP calculation, although the household has gross income at 45% of the poverty guideline, broad-based categorical eligibility and $237.09 net income.", + "new": "It returned $0 without doing any SNAP calculation, although the household has gross income at 45% of the poverty guideline and broad-based categorical eligibility; the record's correction of engine defect r30_snap_heat_and_eat_sua gives $691 net income and $577/month, and the frozen reference, whose $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended), has $237.09 net income.", + "reason": "Presents $237.09 as the household's net income, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The no-calculation error stays." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.5", + "old": "It deducted $1,688/year of medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF and the $453.64 shelter deduction, and used an outdated $198 standard deduction and $766 maximum instead of $209 and $785. That produced $668/year of net income instead of $237.09/month.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it deducted $1,688/year of medical expenses for a head who receives no SSI or SSDI and is not SNAP-disabled, left out $505.39/month of TANF, and used an outdated $198 standard deduction and $766 maximum instead of $209 and $785. That produced $668/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month).", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF, $198 and $766 errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.6", + "old": "With $505.39/month of TANF, net income is $237.09. It also paid the outdated $766 maximum for all 12 months instead of $785 minus the $71.10 contribution.", + "new": "With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). It also paid the outdated $766 maximum for all 12 months instead of $785 minus the $208 contribution under that correction (the frozen reference's is $71.10).", + "reason": "Presents $237.09 net and the $71.10 contribution as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF and $766 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-4.7", + "old": "It subtracted a $1,688/year disabled-member medical deduction the head doesn't qualify for (no SSI or SSDI), counted wages only and left out $505.39/month of TANF, and left out the $453.64 shelter deduction. It also used a $787 maximum instead of FY2026's $785, which gave $536/year of net income instead of $237.09/month.", + "new": "It took no utility-allowance shelter deduction, as the record's correction of engine defect r30_snap_heat_and_eat_sua does (the frozen reference's $453.64 shelter deduction comes from that defect: the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended). But it subtracted a $1,688/year disabled-member medical deduction the head doesn't qualify for (no SSI or SSDI) and counted wages only, leaving out $505.39/month of TANF, which gave $536/year of net income instead of the $691/month the correction gives (the frozen reference has $237.09/month). It also used a $787 maximum instead of FY2026's $785.", + "reason": "Calls leaving out the $453.64 shelter deduction an error and $237.09 the correct net, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical, TANF and $787 maximum errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "grok-build-0.1", + "old": "It also used a $182 medical deduction the non-SNAP-disabled head doesn't get to assume zero net income, when TANF-inclusive net income is $237.09.", + "new": "It also used a $182 medical deduction the non-SNAP-disabled head doesn't get to assume zero net income, when TANF-inclusive net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 as the TANF-inclusive net income, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The single-month and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "inkling", + "old": "It also left out $505.39/month of TANF and the $453.64 shelter deduction, which together with wages give $237.09 net and a $71.10 contribution.", + "new": "It also left out $505.39/month of TANF, which with wages gives $691 net and a $208 contribution under the record's correction of engine defect r30_snap_heat_and_eat_sua, which takes no utility-allowance shelter deduction; the frozen reference's $237.09 net and $71.10 contribution also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the $453.64 shelter deduction an error and presents $237.09 net and $71.10 as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "kimi-k3", + "old": "With $505.39/month of TANF, net income is $237.09, so the benefit is $713.90/month, not the $785 maximum.", + "new": "With $505.39/month of TANF, net income is $691, so the benefit is $577/month under the record's correction of engine defect r30_snap_heat_and_eat_sua, not the $785 maximum; the frozen reference's $237.09 net and $713.90/month also subtract a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents $237.09 net and $713.90/month as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "minimax-m3", + "old": "It also skipped the $209 standard deduction, left out TANF and the shelter deduction, and used an outdated $766 maximum instead of $785.", + "new": "It also skipped the $209 standard deduction, left out TANF, and used an outdated $766 maximum instead of $785. Taking no utility-allowance shelter deduction matches the record's correction of engine defect r30_snap_heat_and_eat_sua; the frozen reference's $453.64 shelter deduction comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Calls leaving out the shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The full-net subtraction, standard deduction, TANF and $766 errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "annotation", + "model": "ox-alpha", + "old": "With $505.39/month of TANF and the $453.64 shelter deduction, net income is $237.09, not $3.", + "new": "With $505.39/month of TANF, net income is $691 under the record's correction of engine defect r30_snap_heat_and_eat_sua, not $3; the frozen reference's $237.09 also subtracts a $453.64 shelter deduction that comes from that defect (the engine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended).", + "reason": "Presents the $453.64 deduction and $237.09 net as correct, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The medical and TANF errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "case_annotation", + "old": "Third, a $453.64 excess shelter deduction from the utility allowance applies even though no rent is listed.", + "new": "Third, although no rent is listed, the engine grants the heat-and-eat standard utility allowance to a household without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended (engine defect r30_snap_heat_and_eat_sua, not fixed upstream), so the frozen reference takes a $453.64 excess shelter deduction from the utility allowance.", + "reason": "States that the $453.64 deduction applies, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). Developer adjudication sentence untouched." + }, + { + "case_id": "us__scenario_100__snap", + "field": "case_annotation", + "old": "Most models counted wages only, took a disabled-member medical deduction and left out the shelter deduction, landing at $9,000–$9,420.", + "new": "Most models counted wages only, took a disabled-member medical deduction and took no shelter deduction, landing at $9,000–$9,420. The record's correction of r30_snap_heat_and_eat_sua also takes no utility-allowance shelter deduction and, with TANF counted and no medical deduction, gives $691 net income, a $208 contribution and $577/month, moving the output from $8,556 to $6,924.", + "reason": "Calls leaving out the shelter deduction an error, but per us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap), correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The wages-only and medical errors stay." + }, + { + "case_id": "us__scenario_100__snap", + "field": "case_annotation", + "old": "Models that did count TANF left out the shelter deduction and came in too low.", + "new": "Models that did count TANF also took no shelter deduction, as that correction does, and came in above its $6,924; most came in below the frozen reference.", + "reason": "Calls leaving out the shelter deduction an error and the results too low; us_adjudications.json reasoning and reference_outputs.csv.meta.json revisions[17].excluded_outputs_rechecked (scenario_100 snap): correcting engine defect r30_snap_heat_and_eat_sua (defect text: reference_exclusions.json scenario_080 snap) removes the utility-allowance shelter deduction and gives net $691, a $208 contribution and $577/month (8,556 to 6,924). The TANF-counting rows (deepseek-v4-pro 7,048, gemini-3.5-flash 7,257, gpt-6-sol 7,416, gpt-6.1-sol 7,764.08, deepseek-v4-pro-0813 9,037.20; predictions.csv) are all above 6,924 and four are below 8,625.89." + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "It then reported $6,222, which contradicts its own post-credit figure of $5,684, and it also skipped the 0.5%-of-AGI floor on charitable deductions.", + "new": "It then reported $6,222, which contradicts its own post-credit figure of $5,684; its charity deduction without the 0.5%-of-AGI floor matches the exclusion's corrected reading, where the frozen reference applies the federal floor.", + "reason": "reference_exclusions.json (scenario_099) records the federal charity floor applied to California as an engine defect (r11); this model reaches the law-correct taxable income (revisions[17] recheck), as the gpt-5.6-sol row now says. Added on independent review." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "The one difference is that it disallowed the $43 traditional IRA contribution under the active-participant phase-out, while the reference subtracts $43.28 above the line (AGI $504,778.56). That extra $43 of income taxed at 32% is the $13.89 gap.", + "new": "The one difference is that it disallowed the $43 traditional IRA contribution under the active-participant phase-out, as the exclusion's corrected value does under 26 U.S.C. 219(g), while the frozen reference subtracts $43.28 above the line (AGI $504,778.56) because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That extra $43 of income taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). prompt.md and base-predictions.csv have this model at 104225.3, the corrected value; the row did not say so." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "The one difference is that it treated the $43 traditional IRA contribution as phased out, while the reference subtracts $43.28 above the line. That $43 taxed at 32% is the $14 gap.", + "new": "The one difference is that it treated the $43 traditional IRA contribution as phased out, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That $43 taxed at 32% is the $14 gap to the frozen reference, and its $104,225 is $0.26 below the corrected value, $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). prompt.md and base-predictions.csv have this model at 104225, the corrected value to rounding; the row did not say so." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "The one difference is that it did not deduct the $43 traditional IRA contribution, while the reference subtracts $43.28 above the line. That amount taxed at 32% is the $13.89 gap.", + "new": "The one difference is that it did not deduct the $43 traditional IRA contribution, which the exclusion's corrected value also disallows under the active-participant phase-out (26 U.S.C. 219(g)), while the frozen reference subtracts $43.28 above the line because PolicyEngine deducts traditional IRA contributions without that phase-out, the engine defect behind this exclusion. That amount taxed at 32% is the $13.89 gap to the frozen reference, and its $104,225.30 is $0.04 above the corrected value, $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). prompt.md and base-predictions.csv have this model at 104225.3, the corrected value; the row did not say so." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It also left out the $218.42 NIIT and skipped the $43 IRA deduction.", + "new": "It also left out the $218.42 NIIT. It skipped the $43 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference takes it.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row called skipping the IRA deduction an error without naming the frozen reference; the model's reasoning in prompt.md disallows it for a 401(k)-covered filer at this MAGI. The NIIT clause is left to the caveat (the exclusion's note names r25_niit_in_federal_output as an unlisted input)." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "but the traditional 401(k) and IRA deductions total only about $969 against $500,000 of wages. Taxable income is still $472,578.56, and the tax is $104,211.41.", + "new": "but the traditional 401(k) deferral and IRA contribution total only about $969 against $500,000 of wages. Taxable income is still $472,578.56 in the frozen reference, whose tax is $104,211.41; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row treated the IRA contribution as a deduction and gave the frozen taxable income and tax as the household's." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "It reached the right AGI ($504,779)", + "new": "It reached the frozen reference's AGI ($504,779), which takes the $43 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)),", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row called the frozen AGI, which includes the $43.28 IRA deduction (prompt.md: the model subtracted $969 of 401(k) and IRA contributions), the right AGI." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct path is ordinary tax of $103,865.78 on $471,730.56, plus $127.20 on qualified dividends and $218.42 NIIT.", + "new": "The frozen reference's path is ordinary tax of $103,865.78 on $471,730.56, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, without the IRA deduction, is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row gave the frozen derivation, which takes the IRA deduction, as the correct path." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The $123,753 is about $19,500 above the correct $104,211.41, which fits", + "new": "The $123,753 is about $19,500 above the frozen reference's $104,211.41 and the exclusion's corrected value, $104,225.26, a gap that fits", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row gave the frozen 104,211.41 as correct; 123,753 is 19,541.59 above it and 19,527.74 above the corrected value." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "its $105,744 is $1,533 above the correct figure, which reflects", + "new": "its $105,744 is $1,533 above the frozen reference's $104,211.41 ($1,519 above the exclusion's corrected value, $104,225.26), a gap that reflects", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row gave the frozen 104,211.41 as the correct figure; 105,744 is 1,532.59 above it and 1,518.74 above the corrected value." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "The correct path is $472,578.56 of taxable income taxed on the 2026 MFJ brackets for $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT.", + "new": "The frozen reference's path is $472,578.56 of taxable income taxed on the 2026 MFJ brackets for $103,865.78, plus $127.20 on qualified dividends and $218.42 NIIT; the exclusion's corrected value, which disallows the $43 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $104,225.26.", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row gave the frozen derivation, whose $472,578.56 of taxable income takes the IRA deduction, as the correct path; 113,421 is about $9,200 above either value." + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "regular tax on the correct $472,578.56 of taxable income is $103,993.", + "new": "regular tax on the frozen reference's $472,578.56 of taxable income is $103,993; that taxable income takes the $43 IRA deduction, which the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_052 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g); IRS Notice 2025-67), and its alternative_reading makes a 401(k) deferrer (the head defers $926 per prompt.md) an active participant who deducts nothing above $149,000 joint: frozen value 104,211.41, corrected value 104,225.26 (the $43.28 IRA deduction at 32%). The row gave the frozen taxable income, which takes the IRA deduction, as correct." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It also left out the $1,118 traditional IRA deduction that the reference takes above the line. The net result was taxable income of $150,590.50 instead of $149,972.27, which gave $22,290.71.", + "new": "It also left out the $1,118 traditional IRA deduction, which the frozen reference takes above the line and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The net result was taxable income of $150,590.50, against the frozen reference's $149,972.27, which gave $22,290.71, $109.94 below the corrected value, $22,400.65.", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. prompt.md has this model at 22,290.71, 109.94 below the corrected value." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "and left out the $1,118 IRA deduction. Together these overstate ordinary tax and give $22,783.", + "new": "and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). The 2025 thresholds overstate ordinary tax, so its $22,783 is $382.35 above the corrected value, $22,400.65 ($628.30 above the frozen reference).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. prompt.md has this model at 22,783.00: 382.35 above the corrected value and 628.30 above the frozen 22,154.70." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "This overstated the result by about $2,050, on top of leaving out the $1,118 IRA deduction.", + "new": "This overstated the result by about $2,050. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. The row's $21,837 ordinary tax on $147,331 is already the no-IRA figure; 24,450 is 2,049.35 above the corrected value." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "That kept AGI at $183,290.50 instead of $182,172.27. Every other step matches the reference, and $1,118 at the 22% marginal rate accounts for the $246 gap.", + "new": "That kept AGI at $183,290.50, where the frozen reference has $182,172.27. Every other step matches the reference, and $1,118 at the 22% marginal rate accounts for the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-4.6", + "old": "It also left out the $1,118 IRA deduction, so taxable income was overstated at $153,291.", + "new": "It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so only the smaller standard deduction overstated its $153,291 of taxable income.", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. Without the IRA deduction taxable income is 184,872.25 - 1,582 - 32,200 = 151,090.25 (prompt.md gross income and capital loss), so the model's 153,291 is 2,200.75 higher, the $30,000 standard deduction." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "It did take the $1,118 IRA deduction, but it overstated pre-Social Security income by $100", + "new": "It took the $1,118 IRA deduction, as the frozen reference does, although the exclusion's corrected value disallows it because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). It also overstated pre-Social Security income by $100", + "reason": "The row presents taking the $1,118 IRA deduction as correct. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "With everything else matching, the $1,118 at 22% explains the $246 excess.", + "new": "With everything else matching, the $1,118 at 22% explains the $246 gap to the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, calling it an excess, but prompt.md has this model at 22,401.00, $0.35 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4.1-flash", + "old": "It also left out the $1,118 IRA deduction, which pushed taxable income to $153,290.50.", + "new": "It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)); its taxable income was $153,290.50.", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "never actually subtracted the $32,200 standard deduction or the $1,118 IRA deduction. This overstated taxable income by more than $33,000.", + "new": "never actually subtracted the $32,200 standard deduction. It also skipped the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)). This overstated taxable income by more than $33,000 against the frozen reference and more than $32,000 against the corrected value.", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. Without the IRA deduction taxable income is 184,872.25 - 1,582 - 32,200 = 151,090.25 (prompt.md), 32,774.75 below the model's $183,865." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "That income produces about $21,591 of ordinary tax plus $564 on qualified dividends.", + "new": "That income produces about $21,591 of ordinary tax plus $564 on qualified dividends in the frozen reference, which subtracts the $1,118 IRA deduction; the exclusion's corrected value disallows that deduction because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), for about $21,837 of ordinary tax.", + "reason": "The row gives the frozen reference's $21,591 ordinary tax, which rests on the $1,118 IRA deduction, as what the income produces. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. 22,400.65 - 564 = 21,836.65." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "and it left out the $1,118 IRA deduction.", + "new": "and it left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It also made a $10 AGI arithmetic slip and left out the $1,118 IRA deduction.", + "new": "It also made a $10 AGI arithmetic slip and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "That, plus leaving out the $1,118 IRA deduction, inflated AGI to $184,370.50.", + "new": "That inflated AGI to $184,370.50. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. Without the IRA deduction AGI is $183,290.50 (the engine-defect rows), so the $184,370.50 is inflated by the $1,080 double count alone." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "far below the correct $149,972", + "new": "far below the frozen reference's $149,972 (which the exclusion's corrected value raises by disallowing the $1,118 IRA deduction, because neither spouse has compensation)", + "reason": "The row calls the frozen reference's taxable income, which rests on the $1,118 IRA deduction, correct. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "Everything else matches, and the IRA amount at 22% explains the $246 difference.", + "new": "Everything else matches, and the IRA amount at 22% explains the $246 difference from the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, but prompt.md has this model at 22,401.00, $0.35 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "That kept AGI at $183,290.50 instead of $182,172.27 and raised tax by $246 at the 22% rate.", + "new": "That kept AGI at $183,290.50, where the frozen reference has $182,172.27, and put tax $246 above the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, saying it raised tax, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "That produces the $246 excess at the 22% rate.", + "new": "That produces the $246 gap to the frozen reference at the 22% rate. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, calling it an excess, but prompt.md has this model at 22,401.00, $0.35 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "The remaining steps match, and the gap is 22% × $1,118.", + "new": "The remaining steps match, and the gap to the frozen reference is 22% × $1,118. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "That left AGI at $183,290.50 instead of $182,172.27 and added $246 of tax.", + "new": "That left AGI at $183,290.50, where the frozen reference has $182,172.27, and tax $246 above the frozen reference. The exclusion's corrected value, $22,400.65, also treats the contribution as non-deductible because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, saying it added tax, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "Every other step matches, and the difference is $1,118 taxed at 22%.", + "new": "Every other step matches, and the difference from the frozen reference is $1,118 taxed at 22%. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,401 is $0.35 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, but prompt.md has this model at 22,401.00, $0.35 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "That overstated tax by $246 at the 22% marginal rate.", + "new": "That put tax $246 above the frozen reference at the 22% marginal rate. The exclusion's corrected value, $22,400.65, also leaves the contribution out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, calling it an overstatement, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "That adds $246 at the 22% rate.", + "new": "That adds $246 at the 22% rate against the frozen reference. The exclusion's corrected value, $22,400.65, also leaves the deduction out because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), and the model's $22,400.71 is $0.06 above it.", + "reason": "The row frames the answer only as a gap from the frozen reference, but prompt.md has this model at 22,400.71, $0.06 from the exclusion's corrected value. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "but 401(k) deferrals need wages and the only deductions available are the $1,582 capital loss, the $1,118 IRA contribution and the $32,200 standard deduction.", + "new": "but 401(k) deferrals need wages, and the only deductions available are the $1,582 capital loss and the $32,200 standard deduction; the frozen reference also deducts the $1,118 IRA contribution, which the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists the $1,118 IRA contribution among the deductions available. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "and left out the $1,118 IRA deduction. That inflated AGI to $184,371 and taxable income to $152,171, where the reference has $149,972.27.", + "new": "which inflated AGI to $184,371 and taxable income to $152,171, where the frozen reference has $149,972.27. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. Without the IRA deduction AGI is $183,290.50 (the engine-defect rows), so the $184,371 is inflated by the $1,080 double count alone." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "and left out the $1,118 IRA deduction.", + "new": "and left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "This partly cancelled its omission of the $1,118 IRA deduction, but the result was still $464 below the reference's taxable income.", + "new": "It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)), so its taxable income was $464 below the frozen reference's and the whole gap to the corrected value, $22,400.65, comes from the second $1,582 subtraction.", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118. The row calls leaving it out an omission the double count cancels. prompt.md has this model at 22,052.67, 347.98 below the corrected value; 22% of 1,582 is 348.04." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "It left out the $1,582 capital loss and the $1,118 IRA deduction, and overstated ordinary tax at $24,968.", + "new": "It left out the $1,582 capital loss and overstated ordinary tax at $24,968. It also left out the $1,118 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists leaving out the $1,118 IRA deduction as an error. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "That means it subtracted a deduction beyond the $1,582 capital loss, the $1,118 IRA contribution and the $32,200 standard deduction.", + "new": "That means it subtracted a deduction beyond the $1,582 capital loss and the $32,200 standard deduction, and beyond the $1,118 IRA contribution that the frozen reference also subtracts and the exclusion's corrected value disallows because neither spouse has compensation (26 U.S.C. 219(b)(1)(B), 219(c)).", + "reason": "The row lists the $1,118 IRA contribution among the deductions the law allows. reference_exclusions.json (scenario_003 federal, r01_ira_compensation) records as the engine defect that PolicyEngine deducts traditional IRA contributions above the compensation limit (26 U.S.C. 219(a), 219(b)(1)(B), 219(c)); prompt.md lists no wages or self-employment income for either spouse, and the corrected value, 22,400.654297 (us_adjudications.json: frozen 22,154.70, corrected 22,400.65; revisions[17] reproduces it with r01_v2 on 2.15.17), deducts none of the $1,118." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "The $2,164 IRA deduction is allowed, the $4,148 state refund is excluded, and the net capital loss is capped at $3,000.", + "new": "The frozen reference allows the $2,164 IRA deduction, excludes the $4,148 state refund and caps the net capital loss at $3,000. It allows the IRA deduction because PolicyEngine applies no active-participant phase-out, the engine defect behind this exclusion: under 26 U.S.C. 219(g) both spouses' traditional 401(k) deferrals make them active participants, their modified AGI for 219(g), $543,076, is far above $149,000, the top of the 2026 joint phase-out range, and the exclusion's corrected value, $107,198.34, allows no IRA deduction.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The note stated the defect, the IRA deduction without the phase-out, as the law." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "(d) came within a few hundred dollars but missed the $1,070 non-itemizer deduction, the 401(k) proration, the IRA deduction, or the refund exclusion.", + "new": "(d) came within a few hundred dollars but missed the $1,070 non-itemizer deduction, the 401(k) proration or the refund exclusion; several also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The note listed leaving out the IRA deduction among the failures." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct standard-deduction path gives $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax.", + "new": "The standard-deduction path gives the frozen reference's $503,495 of taxable income and $89,096 + $11,688 + $5,722 of tax; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen figures, which reflect the $2,164 IRA deduction, as correct." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The correct path uses the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction and totals $106,506 with NIIT.", + "new": "The correct path uses the $32,200 standard deduction plus the $1,070 non-itemizer charity deduction. With NIIT the frozen reference totals $106,506; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen $106,506, which includes the $2,164 IRA deduction, as the correct total." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "None of these exist in 2026 law, which leaves $503,495 taxable under the standard deduction.", + "new": "None of these exist in 2026 law. Under the standard deduction the frozen reference leaves $503,495 taxable, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen $503,495, which is net of the $2,164 IRA deduction, as what 2026 law leaves taxable." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "correct AGI is $536,765.", + "new": "the frozen reference's AGI is $536,765, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen AGI, which is net of the $2,164 IRA deduction, as correct." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "Ordinary tax is then $89,096 at the 2026 brackets.", + "new": "The frozen reference's ordinary tax is then $89,096 at the 2026 brackets, on taxable income net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen ordinary tax, which reflects the $2,164 IRA deduction, as the result of the 2026 rules." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "The correct figure is $89,095.92 of ordinary tax on $425,575, plus $11,688 on $77,920 of qualified dividends, plus $5,721.98 NIIT.", + "new": "The frozen reference's figure is $89,095.92 of ordinary tax on $425,575, plus $11,688 on $77,920 of qualified dividends, plus $5,721.98 NIIT; its taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen figures, which reflect the $2,164 IRA deduction, as correct." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "far below the $106,506 that the 2026 permanent brackets, the $33,270 standard-plus-charity deduction and $5,722 NIIT produce.", + "new": "far below the frozen reference's $106,506, which the 2026 permanent brackets, the $33,270 standard-plus-charity deduction and $5,722 NIIT produce together with a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row attributed the frozen $106,506, which includes the $2,164 IRA deduction, to the 2026 rules alone." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "It also left out the $2,164 IRA deduction and reported a rounded $105,000.", + "new": "It also left out the $2,164 IRA deduction, which the frozen reference takes and the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)), and reported a rounded $105,000.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row called leaving out the IRA deduction an error." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It kept full $46,308 401(k) deferrals and disallowed the $2,164 IRA deduction.", + "new": "It kept full $46,308 401(k) deferrals. It disallowed the $2,164 IRA deduction, as the exclusion's corrected value does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row called disallowing the IRA deduction an error." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The correct total is $89,096 of ordinary tax, $11,688 of qualified-dividend tax and $5,722 of NIIT.", + "new": "The frozen reference's total is $89,096 of ordinary tax, $11,688 of qualified-dividend tax and $5,722 of NIIT; the exclusion's corrected value, $107,198.34, is higher because it disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen total, which reflects the $2,164 IRA deduction, as correct." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "It disallowed the $2,164 traditional IRA deduction. The resulting $1,653 overstatement of taxable income at 32% explains the $529 excess.", + "new": "It disallowed the $2,164 traditional IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)); the frozen reference allows it because PolicyEngine applies no such phase-out. The resulting $1,653 of taxable income above the frozen reference's, taxed at 32%, explains its $529 excess over the frozen reference; the answer is $163.42 below the corrected value.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row called disallowing the IRA deduction an error; predictions.csv has this model at 107034.92." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "The correct derivation reaches $503,495 of taxable income after the $32,200 standard deduction and $1,070 charity deduction, with tax of $89,096 + $11,688 + $5,722.", + "new": "The frozen reference reaches $503,495 of taxable income after the $32,200 standard deduction and $1,070 charity deduction, with tax of $89,096 + $11,688 + $5,722; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen figures, which reflect the $2,164 IRA deduction, as the correct derivation." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "and disallowing the IRA deduction. Taxable income was $506,218 instead of $503,495.", + "new": "and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.66 above the corrected value.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row listed disallowing the IRA deduction among the errors; predictions.csv has this model at 107377." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "Together these overstated taxable income and pushed tax $571 above the correct total.", + "new": "Together these overstated taxable income and pushed tax $571 above the frozen reference's total; the exclusion's corrected value, $107,198.34, which disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)), is $121.34 above the answer.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen total, which includes the $2,164 IRA deduction, as correct; predictions.csv has this model at 107077." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "correct AGI is $536,765 on wages of $388,350.", + "new": "the frozen reference's AGI is $536,765 on wages of $388,350, net of a $2,164 IRA deduction that the exclusion's corrected value disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen AGI, which is net of the $2,164 IRA deduction, as correct." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "The correct result is regular tax on $503,495 plus NIIT with no AMT.", + "new": "The frozen reference's result is regular tax on $503,495 plus NIIT with no AMT; that taxable income is net of a $2,164 IRA deduction that the exclusion's corrected value, $107,198.34, disallows under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen taxable income, which is net of the $2,164 IRA deduction, as the correct result." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "and disallowing the IRA deduction, which overstated taxable income by $3,423.", + "new": "and disallowing the IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Together these put taxable income $3,423 above the frozen reference's, and the answer is $332.66 above the corrected value.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row listed disallowing the IRA deduction among the errors; predictions.csv has this model at 107531." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "and disallowing the $2,164 IRA deduction. Taxable income was $506,218 instead of $503,495.", + "new": "and disallowing the $2,164 IRA deduction, as the exclusion's corrected value, $107,198.34, does under the active-participant phase-out (26 U.S.C. 219(g)) and the frozen reference does not. Taxable income was $506,218 instead of the frozen reference's $503,495; the answer is $178.98 above the corrected value.", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row listed disallowing the IRA deduction among the errors; predictions.csv has this model at 107377.32." + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "In fact no interest was paid, SALT is capped, and the standard-plus-charity deduction of $33,270 leaves $503,495 taxable. That produces $106,506 of tax including NIIT.", + "new": "In fact no interest was paid, SALT is capped, and the standard-plus-charity deduction of $33,270 leaves $503,495 taxable in the frozen reference, which produces $106,506 of tax including NIIT; the exclusion's corrected value, $107,198.34, also disallows the $2,164 IRA deduction under the active-participant phase-out (26 U.S.C. 219(g)).", + "reason": "reference_exclusions.json (scenario_005 federal, r02_ira_219g) records as the engine defect that PolicyEngine deducts traditional IRA contributions without the active-participant phase-out (26 U.S.C. 219(g)), with corrected value 107,198.34 (the adjudication record's reasoning gives the same pair, frozen 106,505.90 and corrected 107,198.34); its alternative_reading makes a 401(k) deferrer an active participant who deducts nothing above $149,000 joint, and prompt.md lists traditional 401(k) deferrals and $1,082 of traditional IRA contributions for each spouse. reference_outputs.csv.meta.json revisions[17] (excluded_outputs_rechecked) finds no 219(g) or active-participant code in the engine, makes both spouses active participants under 219(g)(5)(A)(i) and puts ira_219g_magi at $543,076, so the engine's $2,163.84 deduction is wrong. The row gave the frozen figures, which reflect the $2,164 IRA deduction, as fact." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Together they come to $17,090.34, which is under the $40,400 SALT cap and above the $16,100 standard deduction, so itemizing wins. Once the filer itemizes, the $196 gift falls below the 0.5%-of-AGI floor.", + "new": "The frozen reference projects that table amount from the IRS's 2023 edition, for $17,090.34 in all, which is under the $40,400 SALT cap and above the $16,100 standard deduction, so it itemizes and the $196 gift falls below the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) uses the IRS 2025 tables instead, whose $1,595 for a single Texas filer at this income leaves SALT below the standard deduction, so the household takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[2]: policyengine-us 1.755.4 projected 2026 tables from its 2023 edition with chained CPI." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "deduct $17,090.34 of itemized SALT for taxable income of $283,059.84; tax that at 2026 rates, with qualified dividends at 15%; then add $222.64 of NIIT.", + "new": "deduct the $16,296 of standard and nonitemizer charitable deductions for taxable income of $283,854.19; tax that at 2026 rates, with qualified dividends at 15%; then add $222.64 of NIIT, for $68,334.73, or $68,112.09 without it, the exclusion's corrected value. The frozen reference instead itemizes $17,090.34, for taxable income of $283,059.84 and a total of $68,056.71.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "stopped at the standard deduction, some also taking the $196 nonitemizer charitable deduction, and skipped the sales-tax election.", + "new": "took the standard deduction, as the convention does; those that also took the $196 nonitemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or of $68,112.09 without it.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT. prompt.md: claude-fable-5.1, claude-opus-5.5 and kimi-k3 answered 68,334.59; gpt-5.6-sol 68,112.00; gpt-6-astra and gpt-6.1-sol 68,111.95." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It missed the general sales tax table deduction available in no-income-tax Texas, which brings itemized SALT to $17,090.34 and makes itemizing the better choice. That gives taxable income of $283,059.84, and the $794 difference at 35% explains the $278 overshoot.", + "new": "Under the release's sales tax convention (c_irs_sales_tax_2025) that is the household's choice, because the IRS 2025 table leaves SALT below the standard deduction; the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84, and the $794 difference at 35% explains its $278 gap to that reference. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT prompt.md: claude-fable-5.1 answered 68,334.59." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It never added Texas's general sales tax table deduction to the $13,834 of property tax. The combined $17,090.34 of SALT beats the standard deduction and lowers taxable income to $283,059.84.", + "new": "That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction even with Texas's general sales tax deduction added to the $13,834 of property tax; only the frozen reference's projected sales tax brings SALT to $17,090.34 and lowers taxable income to $283,059.84. Its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT prompt.md: claude-opus-5.5 answered 68,334.59." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "It judged the itemized amount to be only $13,834 and took the standard deduction plus the $196 nonitemizer charity deduction. It missed the Texas general sales tax table deduction, which raises itemized SALT to $17,090.34 and taxable income falls to $283,059.84.", + "new": "It treated the itemized amount as $13,834 without the sales tax and took the standard deduction plus the $196 nonitemizer charity deduction, which is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025): the IRS 2025 table leaves SALT below the standard deduction either way. Only the frozen reference's projected sales tax raises itemized SALT to $17,090.34 and lowers taxable income to $283,059.84; its $68,334.59 is $0.14 below the $68,334.73 the conventions give with NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT prompt.md: kimi-k3 answered 68,334.59." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It took the standard deduction plus the $196 nonitemizer charitable deduction instead of itemizing $17,090.34 of SALT (property plus sales tax table). It also left out the $222.64 net investment income tax that belongs in this total.", + "new": "It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT (property plus a projected sales tax). It also left out the $222.64 net investment income tax, which the frozen reference includes in this total; its $68,112.00 is $0.09 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gpt-5.6-sol answered 68,112.00. reference_exclusions.json treats NIIT inclusion as an unlisted input, so the row states the frozen reading rather than a rule." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It used the standard deduction plus the $196 nonitemizer charitable deduction, missing the Texas sales tax table deduction that makes itemized SALT of $17,090.34 larger. It also left out the $222.64 NIIT.", + "new": "It used the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference's projected sales tax makes itemized SALT of $17,090.34 larger. It also left out the $222.64 NIIT, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gpt-6-astra answered 68,111.95." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It took the standard deduction plus the $196 nonitemizer charitable deduction instead of itemizing $17,090.34 of SALT, which includes the Texas sales tax table deduction. It also omitted the $222.64 net investment income tax.", + "new": "It took the standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice: the IRS 2025 table leaves SALT below the standard deduction, where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax. It also omitted the $222.64 net investment income tax, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion's corrected value, $68,112.09, which applies every convention and leaves the NIIT out.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gpt-6.1-sol answered 68,111.95." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "It never added the Texas general sales tax deduction from the IRS optional table, which raises itemized SALT to $17,090.34, above the standard deduction. The extra ~$990 of taxable income at 35% accounts for its ~$347 overstatement.", + "new": "The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax raises itemized SALT to $17,090.34, and the extra ~$990 of taxable income at 35% accounts for its ~$347 gap to that reference. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% is nearly all of its $69.27 gap to the $68,334.73 the conventions give with NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT prompt.md: claude-fable-5 answered 68,404.00 on $284,050 of taxable income." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "Adding the Texas general sales tax table deduction brings itemized SALT to $17,090.34 and taxable income to $283,059.84.", + "new": "The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it even with the sales tax added; only the frozen reference's projected sales tax brings itemized SALT to $17,090.34 and taxable income to $283,059.84. It left out the $196 nonitemizer charitable deduction, so its taxable income is $195.81 above the convention's $283,854.19; that $68.53 of tax at 35% matches its $68.46 gap to the $68,334.73 the conventions give with NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; revisions[17] gives 68,334.73 on policyengine-us 2.15.17 with conventions and NIIT prompt.md: gpt-5.5 answered 68,403.19 on $284,050 of taxable income." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.7", + "old": "It used the standard deduction, for taxable income of $284,050, missing the sales tax table deduction that makes itemized SALT of $17,090.34 the larger deduction.", + "new": "It used the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the larger deduction), but it left out the $196 nonitemizer charitable deduction, for taxable income of $284,050 instead of the convention's $283,854.19.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: grok-4.7 answered 68,268.00 on $284,050 of taxable income." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It also took the standard deduction without adding the Texas sales tax table deduction that makes itemized SALT of $17,090.34 the better choice. Its taxable income of $289,759 is $6,700 too high.", + "new": "It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax makes itemized SALT of $17,090.34 the better choice), but not the $196 nonitemizer charitable deduction. Its taxable income of $289,759 is $5,904.81 above the convention's $283,854.19 and $6,699.16 above the frozen reference's $283,059.84.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 289,759 = 305,859 - 16,100. prompt.md: claude-sonnet-5.5 answered 70,401.30 on $289,759 of taxable income." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "glm-5.3", + "old": "It also took the standard deduction without adding the Texas sales tax table deduction, which lifts itemized SALT to $17,090.34, above $16,100.", + "new": "It also took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (only the frozen reference's projected sales tax lifts itemized SALT to $17,090.34, above $16,100), but not the $196 nonitemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: glm-5.3 answered 70,468.34 on $289,759 of taxable income (305,859 - 16,100)." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "and took the standard deduction without adding the Texas sales tax table deduction to property tax.", + "new": "and took the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's projected sales tax, added to property tax, makes itemizing larger.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: ox-alpha answered 70,178.70 on $289,759 of taxable income (305,859 - 16,100)." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "uses the standard deduction instead of $17,090.34 of itemized SALT.", + "new": "uses the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 289,759 = 305,859 - 16,100. prompt.md: gpt-5.6-terra answered 70,179.00." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-luna", + "old": "It also takes the standard deduction instead of itemizing $17,090.34 of real estate plus sales tax.", + "new": "It also takes the $16,100 standard deduction without the $196 nonitemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of real estate plus a projected sales tax.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 289,759 = 305,859 - 16,100. prompt.md: gpt-6-luna answered 70,401.00, which gpt-6-sol's 70,401.34 on $289,759 matches." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "The alimony deduction lowers AGI to $300,150, and itemized SALT of $17,090.34 (including the sales tax table amount) exceeds the standard deduction.", + "new": "The alimony deduction lowers AGI to $300,150, and the nonitemizer charitable deduction adds $196 to the standard deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where only the frozen reference's itemized SALT of $17,090.34 (including a projected sales tax table amount) exceeds it.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gpt-6-sol answered 70,401.34 on $289,759 of taxable income (305,859 - 16,100)." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "It used the standard deduction instead of itemizing $17,090.34 of SALT and omitted the $222.64 NIIT.", + "new": "It used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax), but not the $196 nonitemizer charitable deduction, and it omitted the $222.64 NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gpt-5.6-luna answered 65,244.40, which is 67,834.07 - 35% x (283,059.84 - 275,661): AGI of 291,761 after the ESI subtraction less 16,100, without the $196." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.8-flash", + "old": "used the 2025 $15,000 standard deduction instead of itemizing $17,090.34 of SALT (property plus sales tax table).", + "new": "used the 2025 $15,000 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 nonitemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT, property plus a projected sales tax table amount).", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: gemini-3.8-flash answered 71,341.70 on $290,859 of taxable income (305,859 - 15,000)." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "inkling", + "old": "treated the $196 of charity as an above-the-line deduction, and skipped the $5,709 alimony deduction. It then took the standard deduction instead of itemizing $17,090.34 of SALT.", + "new": "took the $196 of charity above the line rather than as the nonitemizer charitable deduction, which lowers taxable income by the same $196, and skipped the $5,709 alimony deduction. Its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: inkling answered 67,479.00 with the $196 above the line and the $16,100 standard deduction." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash-lite", + "old": "used the standard deduction instead of itemizing $17,090.34 of SALT,", + "new": "used the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice (the frozen reference itemizes $17,090.34 of SALT with a projected sales tax),", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "As a result it never itemized the $17,090.34 of real estate plus sales tax that beats the standard deduction.", + "new": "Not itemizing is still the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction under the $40,400 cap; only the frozen reference's projected sales tax lifts real estate plus sales tax to $17,090.34, above it.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It capped SALT at $10,000 instead of the $40,400 2026 cap and ignored the sales tax table deduction.", + "new": "It capped SALT at $10,000 instead of the $40,400 2026 cap and ignored the sales tax table deduction; neither changes its $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but it left out the $196 nonitemizer charitable deduction.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT. prompt.md: claude-opus-4.7 answered 70,813.00; its reasoning counts the $196 only in its $10,196 itemized total and takes $343,007 - $16,100 of taxable income." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "and capped SALT at $10,000, missing the sales-tax itemization.", + "new": "and capped SALT at $10,000 instead of $40,400.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT, so leaving out the sales tax is not an error." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "instead of taxing $283,059.84 of taxable income and adding NIIT.", + "new": "instead of taxing $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the $16,100 standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), and adding NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5", + "old": "The correct figure is $17,090.34 of SALT: $13,834 of real estate tax plus the sales tax table amount. Its $72,700 answer is an unsupported estimate rather than tax on $283,059.84 of taxable income plus $222.64 NIIT.", + "new": "Itemized deductions are SALT alone: $13,834 of real estate tax plus the sales tax table amount, which the frozen reference projects for $17,090.34 in all; the release's sales tax convention (c_irs_sales_tax_2025) takes the IRS 2025 table, whose $1,595 for this filer leaves SALT below the $16,100 standard deduction. Its $72,700 answer is an unsupported estimate rather than tax on $283,854.19 of taxable income under the convention ($283,059.84 in the frozen reference) plus $222.64 NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "so the correct figure is regular tax on $283,059.84 plus NIIT.", + "new": "so the correct figure is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "Under current law, regular tax on $283,059.84 applies with no AMT.", + "new": "Under current law, regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or on $283,059.84 in the frozen reference, which itemizes a projected sales tax.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "so the answer is regular tax on $283,059.84 plus NIIT.", + "new": "so the answer is regular tax on $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax), plus NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "instead of regular tax on $283,059.84 of taxable income plus NIIT.", + "new": "instead of regular tax on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), or $283,059.84 in the frozen reference, which itemizes a projected sales tax, plus NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "so taxable income is $283,059.84.", + "new": "so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax).", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "and taxable income is $283,059.84.", + "new": "and taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction ($283,059.84 in the frozen reference, which itemizes a projected sales tax).", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "Correct itemized deductions are only $17,090.34 of SALT, which leaves $67,834 of regular tax plus NIIT.", + "new": "Correct itemized deductions are only SALT, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction leaves $68,112.09 of regular tax plus NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The correct itemized total is $17,090.34 of SALT, giving tax of $67,834.07 plus $222.64 NIIT.", + "new": "Itemized deductions are SALT alone, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025), so the $16,100 standard deduction plus the $196 nonitemizer charitable deduction gives tax of $68,112.09 plus $222.64 NIIT; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $67,834.07 plus NIIT.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.3", + "old": "It did not deduct alimony, itemize $17,090.34 of SALT, tax the resulting $283,059.84 at 2026 brackets, or add the $222.64 NIIT.", + "new": "It did not deduct alimony, add the $196 nonitemizer charitable deduction to its standard deduction, tax the resulting $283,854.19 at 2026 brackets, or add the $222.64 NIIT; its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), where the frozen reference itemizes $17,090.34 of SALT with a projected sales tax and taxes $283,059.84.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19. prompt.md: grok-4.3's reasoning names the standard deduction on ~$306k of AGI." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.5", + "old": "so taxable income is $283,059.84, not $148,321.", + "new": "so taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 nonitemizer charitable deduction, or $283,059.84 in the frozen reference, which itemizes a projected sales tax, not $148,321.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "regular tax on $283,059.84 applies with no AMT.", + "new": "regular tax applies with no AMT, on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025) or $283,059.84 in the frozen reference, which itemizes a projected sales tax.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "The correct derivation is AGI of $300,150 after alimony, less $17,090.34 of itemized SALT, for taxable income of $283,059.84, tax of $67,834.07 and NIIT of $222.64.", + "new": "The correct derivation under the release's sales tax convention (c_irs_sales_tax_2025) is AGI of $300,150 after alimony, less the $16,100 standard deduction and the $196 nonitemizer charitable deduction, for taxable income of $283,854.19, tax of $68,112.09 and NIIT of $222.64; the frozen reference itemizes $17,090.34 of SALT with a projected sales tax, for $283,059.84 and $67,834.07.", + "reason": "c_irs_sales_tax_2025 (reference_outputs.csv.meta.json revisions[2]) replaces the engine's projection from its 2023 table with the IRS 2025 tables; revisions[17] (excluded_outputs_rechecked, scenario_020) puts the 2025 TX size-1 $300k+ cell at $1,595, so SALT stays below the $16,100 standard deduction, and reference_exclusions.json's 68,112.09 = 68,056.71 - 222.64 + 35% x (17,090.34 - 16,296), the standard deduction plus the $196 gift, with every convention and no NIIT; 300,150.19 - 16,296 = 283,854.19." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "and the federal-conforming 2026 standard deduction of $16,100 brings it to $19,180.", + "new": "and the federal-conforming 2026 standard deduction of $16,100 brings the frozen reference's taxable income to $19,180. The frozen reference leaves out Idaho's subtraction of the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the exclusion's corrected value takes, for taxable income of $17,100.", + "reason": "The rationale gave the frozen $19,180, which leaves out the premium subtraction, as Idaho taxable income. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811)." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Idaho's 2026 zero-rate band: the first $4,920 of single taxable income is taxed at 0%, and only the remaining $14,260 is taxed at 5.3%, giving $755.78.", + "new": "Idaho's 2026 zero-rate band. The frozen reference taxes the first $4,920 of single taxable income, the engine's projected band, at 0% and only the remaining $14,260 at 5.3%, giving $755.78. The exclusion's corrected value uses the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026, for 5.3% × ($17,100 − $4,811) = $651.32, which gpt-6-astra matches.", + "reason": "The rationale gave the engine's projected $4,920 as Idaho's 2026 zero-rate band and the frozen $755.78 as the tax. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811)); reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); predictions.csv.gz in the snapshot run has gpt-6-astra at 651.32." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "case_annotation", + "old": "Near misses used the 2025 zero-rate band ($4,811) or a guessed one ($4,990 or $5,000) instead of $4,920.", + "new": "Near misses differ from the frozen reference only in the zero-rate band: a guessed one ($4,990 or $5,000), or the $4,811 that the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920.", + "reason": "The rationale called the $4,811 zero-rate band a model error and gave the projected $4,920 as the right one. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811)), and the exclusion's corrected value applies it." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "It correctly reached $19,180 of Idaho taxable income.", + "new": "It reached the frozen reference's $19,180 of Idaho taxable income.", + "reason": "The row called the frozen $19,180, which leaves out the $2,080 premium subtraction, correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811)." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5", + "old": "The correct result applies 5.3% only to the $14,260 above Idaho's $4,920 zero-rate band.", + "new": "The frozen reference applies 5.3% only to the $14,260 above the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation correct and gave the projected $4,920 as Idaho's band. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-fable-5.1", + "old": "It correctly derived $19,180 of taxable income but taxed all of it at 5.3%. It left out Idaho's 2026 zero-rate band on the first $4,920 of single taxable income, so only $14,260 is taxed, for $755.78.", + "new": "It derived the frozen reference's $19,180 of taxable income but taxed all of it at 5.3%. It left out Idaho's 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920 and so taxes only $14,260, for $755.78. The exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32.", + "reason": "The row called the frozen $19,180 correct and gave the projected $4,920 and the frozen $755.78 as Idaho's 2026 band and tax. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "and results in $755.78 of tax.", + "new": "and gives $755.78 of tax in the frozen reference; the exclusion's corrected value, which also subtracts the $2,080 of premiums and uses the $4,811 zero-rate band that c_id_hold_2025 keeps for 2026, is $651.32.", + "reason": "The row gave the frozen $755.78 as the tax. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "It correctly reached $19,180 of Idaho taxable income.", + "new": "It reached the frozen reference's $19,180 of Idaho taxable income.", + "reason": "The row called the frozen $19,180, which leaves out the $2,080 premium subtraction, correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811)." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.7", + "old": "The correct step taxes $19,180 − $4,920 = $14,260 at 5.3%.", + "new": "The frozen reference taxes $19,180 − $4,920 = $14,260 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation and the projected $4,920 correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-4.8", + "old": "The correct figure is 5.3% × ($19,180 − $4,920).", + "new": "The frozen reference's figure is 5.3% × ($19,180 − $4,920) = $755.78, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation and the projected $4,920 correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-opus-5.5", + "old": "It used the 2025 zero-rate threshold of $4,811 instead of the inflation-indexed 2026 figure of $4,920. That overstated the income taxed at 5.3% by $109, giving $761.56 instead of $755.78.", + "new": "It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate threshold, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $109 more at 5.3% than the frozen reference, giving $761.56 against its $755.78.", + "reason": "The row calls the $4,811 zero-rate threshold a model error, but c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811)), and the exclusion's corrected value applies it. The model's difference from the corrected value is the premium subtraction: reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); 0.053 x 2,080 = 110.24, and the snapshot run's predictions.csv.gz has this model at 761.56 = 0.053 x (19,180 - 4,811)." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "claude-sonnet-5.5", + "old": "It correctly reached $19,180 of taxable income but applied 5.3% to all of it. Idaho taxes only the $14,260 above the $4,920 zero-rate band, giving $755.78.", + "new": "It reached the frozen reference's $19,180 of taxable income but applied 5.3% to all of it. The frozen reference taxes only the $14,260 above the engine's projected $4,920 zero-rate band, giving $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and taxes only the income above the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen $19,180 correct and gave the projected $4,920 and the frozen $755.78 as Idaho's rule and tax. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro", + "old": "The correct step taxes $19,180 − $4,920 at 5.3%.", + "new": "The frozen reference taxes $19,180 − $4,920 at 5.3%, with the engine's projected $4,920 zero-rate band; the exclusion's corrected value, $651.32, taxes $17,100 − $4,811 at 5.3%, after the $2,080 premium subtraction and with the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation and the projected $4,920 correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "Idaho's $4,920 zero-rate band", + "new": "Idaho's zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference)", + "reason": "The row gave the projected $4,920 as Idaho's zero-rate band. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-flash-lite-preview", + "old": "After the $16,100 deduction and the $4,920 zero-rate band, tax is 5.3% × $14,260 = $755.78, about half of its $1,582.", + "new": "In the frozen reference, after the $16,100 deduction and the engine's projected $4,920 zero-rate band, tax is 5.3% × $14,260 = $755.78, about half of its $1,582; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row gave the frozen computation and the projected $4,920 as the tax. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "The correct computation is 5.3% × $14,260.", + "new": "The frozen reference's computation is 5.3% × $14,260 = $755.78, above the engine's projected $4,920 zero-rate band; the exclusion's corrected value is 5.3% × $12,289 = $651.32, after the $2,080 premium subtraction and the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen computation correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "Neither figure applies 5.3% to the $14,260 above Idaho's $4,920 zero-rate band.", + "new": "Neither figure applies 5.3% only to income above Idaho's zero-rate band: the frozen reference taxes the $14,260 above the engine's projected $4,920, and the exclusion's corrected value taxes the $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction.", + "reason": "The row gave the projected $4,920 as Idaho's zero-rate band and the frozen $14,260 as the amount taxed. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "which exceeds the $16,100 standard deduction plus the $4,920 zero-rate band, leaving $14,260 taxed at 5.3%.", + "new": "which exceeds the $16,100 standard deduction plus the zero-rate band. The frozen reference, with the engine's projected $4,920 band, leaves $14,260 taxed at 5.3%; the exclusion's corrected value also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026, leaving $12,289 taxed at 5.3%, for $651.32.", + "reason": "The row gave the projected $4,920 as Idaho's zero-rate band and the frozen $14,260 as the amount taxed. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "The $35,280 private pension remains in Idaho income, and after the $16,100 standard deduction and $4,920 zero-rate band, $14,260 is taxed at 5.3%.", + "new": "The $35,280 private pension remains in Idaho income. In the frozen reference, after the $16,100 standard deduction and the engine's projected $4,920 zero-rate band, $14,260 is taxed at 5.3%, for $755.78; the exclusion's corrected value, $651.32, also subtracts the $2,080 of premiums and uses the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row gave the projected $4,920 as Idaho's zero-rate band and the frozen $14,260 as the amount taxed. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.5", + "old": "It used a guessed $5,000 zero-rate band instead of Idaho's inflation-indexed 2026 single amount of $4,920. That understated taxable income at 5.3% by $80, giving $751.54 instead of $755.78.", + "new": "It used a guessed $5,000 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,180 at 5.3%, giving $751.54, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32.", + "reason": "The row gave the projected $4,920 as Idaho's 2026 single amount and graded the model against it. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "It correctly reached $19,180 but applied 5.3% to the entire amount. It left out Idaho's $4,920 zero-rate band, which leaves only $14,260 taxable.", + "new": "It reached the frozen reference's $19,180 but applied 5.3% to the entire amount. It left out Idaho's zero-rate band: the frozen reference excludes the engine's projected $4,920, leaving $14,260 taxable, and the exclusion's corrected value, after the $2,080 premium subtraction, excludes the $4,811 that c_id_hold_2025 keeps for 2026, leaving $12,289.", + "reason": "The row called the frozen $19,180 correct and gave the projected $4,920 as Idaho's band. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-astra", + "old": "It subtracted $2,080 of health insurance premiums from Idaho income, but in this computation Idaho income is $35,280 with only the Social Security subtraction. It also used the unindexed 2025 zero-rate band of $4,811 instead of 2026's $4,920, giving $651.32.", + "new": "It subtracted the head's $2,080 of health insurance premiums from Idaho income, the Idaho subtraction (Idaho Code 63-3022P) that the engine leaves out; the frozen reference's Idaho income of $35,280 has only the Social Security subtraction. It also used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920. Its $651.32 is the exclusion's corrected value.", + "reason": "The row calls the premium subtraction and the $4,811 band model errors, but its answer is the exclusion's corrected value. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811)); the snapshot run's predictions.csv.gz has this model at 651.32." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6-sol", + "old": "It used an approximate $4,990 zero-rate band instead of Idaho's 2026 single amount of $4,920. That taxed $14,190 instead of $14,260 and gave $752 instead of $755.78.", + "new": "It used an approximate $4,990 zero-rate band, where the release's Idaho convention (c_id_hold_2025) keeps $4,811 for 2026 and the frozen reference uses the engine's projected $4,920. That taxed $14,190 at 5.3% and gave $752, against $14,260 and $755.78 in the frozen reference and $14,369 with the $4,811 band before the premium subtraction; the exclusion's corrected value also subtracts the head's $2,080 of health insurance premiums and taxes $12,289, for $651.32.", + "reason": "The row gave the projected $4,920 as Idaho's 2026 single amount and graded the model against it. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "It used the 2025 zero-rate band of $4,811 instead of the inflation-indexed 2026 amount of $4,920. It taxed $14,369 instead of $14,260, giving $761.56.", + "new": "It did not subtract the head's $2,080 of health insurance premiums (Idaho Code 63-3022P), which the frozen reference also leaves out and the exclusion's corrected value takes, so its $761.56 is $110.24 above the corrected value, $651.32. It used the $4,811 zero-rate band, which the release's Idaho convention (c_id_hold_2025) keeps for 2026, where the frozen reference uses the engine's projected $4,920; that taxed $14,369 at 5.3%, against $14,260 in the frozen reference.", + "reason": "The row calls the $4,811 zero-rate band a model error, but c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811)), and the exclusion's corrected value applies it. The model's difference from the corrected value is the premium subtraction: reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); 0.053 x 2,080 = 110.24, and the stage's fold/us/predictions.csv has this model at 761.56 = 0.053 x (19,180 - 4,811)." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "grok-4.6", + "old": "instead of taxing $14,260 at 5.3% above the $4,920 zero-rate band.", + "new": "instead of taxing at 5.3% only the income above the zero-rate band: $14,260 above the engine's projected $4,920 in the frozen reference, or $12,289 above the $4,811 that c_id_hold_2025 keeps for 2026, after the $2,080 premium subtraction, in the exclusion's corrected value.", + "reason": "The row gave the projected $4,920 as the zero-rate band the model should have used. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "kimi-k3", + "old": "The correct steps subtract the $16,100 standard deduction to reach $19,180 and then exclude the $4,920 zero-rate band.", + "new": "The frozen reference subtracts the $16,100 standard deduction to reach $19,180 and then excludes the engine's projected $4,920 zero-rate band; the exclusion's corrected value also subtracts the $2,080 of premiums, for $17,100, and excludes the $4,811 band that c_id_hold_2025 keeps for 2026.", + "reason": "The row called the frozen $19,180 and the projected $4,920 correct. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "minimax-m3", + "old": "Idaho does: it taxes income above a $4,920 zero-rate band at a flat 5.3%, which here produces $755.78.", + "new": "Idaho does: it taxes income above a zero-rate band at a flat 5.3%. With the engine's projected $4,920 band the frozen reference is $755.78; the exclusion's corrected value, with the $4,811 band that the release's Idaho convention (c_id_hold_2025) keeps for 2026 and the $2,080 premium subtraction, is $651.32.", + "reason": "The row gave the projected $4,920 as Idaho's band and the frozen $755.78 as Idaho's tax. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "ox-alpha", + "old": "It correctly derived $19,180 of Idaho taxable income but applied 5.3% to all of it. It left out the 2026 zero-rate band on the first $4,920 of single taxable income.", + "new": "It derived the frozen reference's $19,180 of Idaho taxable income but applied 5.3% to all of it. It left out the 2026 zero-rate band for single filers: $4,811 under the release's Idaho convention (c_id_hold_2025), where the frozen reference uses the engine's projected $4,920.", + "reason": "The row called the frozen $19,180 correct and gave the projected $4,920 as the 2026 band. reference_exclusions.json (scenario_007 state_income_tax_before_refundable_credits, r07_idaho_health_premiums) records the omitted Idaho premium subtraction (Idaho Code 63-3022P), corrected value 651.32, which reference_outputs.csv.meta.json revisions[17] reproduces as 0.053 x (17,100 - 4,811); c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "field": "annotation", + "model": "qwen3.8-max", + "old": "instead of the 5.3% flat rate above the $4,920 zero-rate band.", + "new": "instead of the 5.3% flat rate above the zero-rate band ($4,811 under the release's Idaho convention, c_id_hold_2025; the engine's projected $4,920 in the frozen reference).", + "reason": "The row gave the projected $4,920 as the zero-rate band the model should have used. c_id_hold_2025 (reference_outputs.csv.meta.json revisions[4]) holds Idaho's 2026 single zero-rate threshold at the published 2025 $4,811, and the frozen $4,920 is the engine's projection kept on this excluded output (excluded_outputs_untouched; revisions[17] recheck: 0.053 x (4,920.03 - 4,811))." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "case_annotation", + "old": "It counts no NJ unemployment, workforce or supplemental workforce fund employee contribution.", + "new": "It counts no NJ unemployment, workforce or supplemental workforce fund employee contribution, although New Jersey workers pay 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds; the exclusion's corrected value counts them ($113.90 more), for $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The note gave the frozen reference's TDI-plus-FLI component as New Jersey's mandatory employee payroll tax without saying the reference omits contributions New Jersey requires." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "case_annotation", + "old": "Most wrong models either stopped at FICA ($2,050.20) or added the 0.3825–0.425% UI/WF/SWF contribution, often with outdated or inflated TDI/FLI rates (0% TDI, 0.09% FLI, or 0.33% FLI).", + "new": "Most models either stopped at FICA ($2,050.20) or added the 0.3825–0.425% UI/WF/SWF contribution, often with outdated or inflated TDI/FLI rates (0% TDI, 0.09% FLI, or 0.33% FLI). gpt-6-astra and gpt-6-sol match the exclusion's corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The note counted adding the UI/WF/SWF contribution among the wrong answers; predictions.csv has gpt-6-astra and gpt-6-sol at 2276.66, the corrected value." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "claude-fable-5", + "old": "Added the NJ UI (0.3825%) and workforce/SWF (0.0425%) employee contributions, which the payroll_tax output does not include. It also set the TDI employee rate to 0% instead of 0.19% ($50.92) and used a stale 0.09% FLI rate instead of 0.23% ($61.64), giving $128.65 of NJ tax instead of $112.56.", + "new": "Added the NJ UI (0.3825%) and workforce/SWF (0.0425%) employee contributions, which New Jersey requires and the frozen reference leaves out. It also set the TDI employee rate to 0% instead of 0.19% ($50.92) and used a stale 0.09% FLI rate instead of 0.23% ($61.64), giving $128.65 of NJ tax, against the frozen reference's $112.56 and the $226.46 NJ component of the exclusion's corrected value.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the UI and workforce contributions an error and gave the frozen $112.56 as the NJ figure; the corrected value's NJ component is 2,276.66 - 2,050.20 = 226.46." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "claude-haiku-4.5", + "old": "The correct NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), which is $112.56.", + "new": "The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), which is $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row gave the frozen TDI-plus-FLI $112.56 as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "claude-sonnet-5", + "old": "The correct NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56, and wages are well below the cap.", + "new": "The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value also counts the worker unemployment and workforce contributions New Jersey requires ($113.90), for $226.46. Wages are well below the cap.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row gave the frozen TDI-plus-FLI $112.56 as the correct NJ addition, although the model's own estimate included the UI contribution." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "deepseek-v4-pro-0813", + "old": "Put NJ employee payroll taxes at $154.10 (about 0.575% of wages) instead of $112.56. That figure reflects counting UI/workforce contributions and/or wrong rates, when only TDI at 0.19% plus FLI at 0.23% applies.", + "new": "Put NJ employee payroll taxes at $154.10 (about 0.575% of wages). That is $41.54 above the frozen reference's TDI at 0.19% plus FLI at 0.23% ($112.56) and $72.36 below the $226.46 of the exclusion's corrected value, which adds the worker unemployment and workforce contributions New Jersey requires and the frozen reference leaves out.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said only TDI and FLI apply and counted the UI/workforce contributions as a possible error; the model gave no breakdown of its $154.10, which is $72.36 below the corrected NJ component of $226.46." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gemini-3-flash-preview", + "old": "It counted the employee UI contribution, which the output excludes, and left out TDI, instead of applying TDI at 0.19% plus FLI at 0.23% ($112.56).", + "new": "It counted the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and left out TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the UI contribution an error and gave the frozen TDI-plus-FLI component as the NJ answer." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gemini-3.1-pro-preview", + "old": "It wrongly included the employee UI contribution and misstated the rates; the correct NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56.", + "new": "It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and misstated the rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, and the model's $2,188 is $88.66 below the corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called including the UI contribution wrong and gave the frozen $112.56 as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "It counted the UI employee contribution, which the output excludes, and omitted TDI (0.19%, $50.92).", + "new": "It counted the UI employee contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI (0.19%, $50.92). Its $2,174 is $102.66 below the exclusion's corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI contribution and counted it as an error." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "It included the employee UI contribution, which the output does not count, and omitted TDI, instead of TDI at 0.19% plus FLI at 0.23% ($112.56).", + "new": "It included the employee UI contribution, which New Jersey requires and the frozen reference leaves out, and omitted TDI. The frozen reference's NJ component is TDI at 0.19% plus FLI at 0.23% ($112.56); the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which the model's $138.02 misses by $88.44.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the UI contribution an error and gave the frozen TDI-plus-FLI component as the NJ answer." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gemini-3.7-flash", + "old": "It included the UI and workforce development contributions, which the output excludes, and left out TDI (0.19%, $50.92).", + "new": "It included the UI and workforce development contributions, which New Jersey requires and the frozen reference leaves out, and left out TDI (0.19%, $50.92). Its $2,189.52 is $87.14 below the exclusion's corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI and workforce contributions and counted them as an error." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-5.4-mini", + "old": "the correct total is $2,050.20 + $112.56 NJ TDI/FLI.", + "new": "the frozen reference's total is $2,050.20 + $112.56 NJ TDI/FLI ($2,162.76), and the exclusion's corrected value adds $113.90 of NJ worker unemployment and workforce contributions, for $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row gave the frozen 2,162.76 as the correct total." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-5.5", + "old": "Applied a combined 0.9425% NJ rate ($252.59) that includes the UI/WF/SWF employee contribution, which the output excludes, along with an overstated FLI rate. The correct NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56.", + "new": "Applied a combined 0.9425% NJ rate ($252.59) that includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, along with an overstated FLI rate. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $252.59 exceeds by $26.13.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI/WF/SWF contribution and gave the frozen $112.56 as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "That is 0.425% UI/WF/SWF, which the output does not count, plus TDI 0.19% and FLI at 0.33% instead of the 2026 rate of 0.23%.", + "new": "That is 0.425% UI/WF/SWF, which New Jersey requires and the frozen reference leaves out, plus TDI 0.19% and FLI at 0.33% instead of the 2026 rate of 0.23%, so it exceeds the $226.46 of the exclusion's corrected value by $26.80.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output does not count the UI/WF/SWF contribution and counted it as an error." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "That includes the 0.425% UI/WF/SWF contribution, which the output excludes, and overstates FLI at 0.33% and TDI at 0.23%; the correct NJ addition is 0.19% TDI plus 0.23% FLI, or $112.56.", + "new": "That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, and overstates FLI at 0.33% and TDI at 0.23%. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI, or $112.56; the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI/WF/SWF contribution and gave the frozen $112.56 as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-5.6-terra", + "old": "That includes the UI/WF/SWF employee contribution, which the output excludes, and misstates TDI/FLI; the correct NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56.", + "new": "That includes the UI/WF/SWF employee contribution, which New Jersey requires and the frozen reference leaves out, and misstates TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $202.34 falls $24.12 short of.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI/WF/SWF contribution and gave the frozen $112.56 as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-6-astra", + "old": "Used the correct TDI (0.19%) and FLI (0.23%) rates but also added the 0.425% UI/workforce contribution ($113.90). The output counts only TDI and FLI, so NJ tax should be $112.56, not $226.46.", + "new": "Used the correct TDI (0.19%) and FLI (0.23%) rates and added the 0.425% UI/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. The frozen reference counts only TDI and FLI ($112.56); the model's $226.46 NJ addition, for $2,276.66, is the exclusion's corrected value.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the added contribution an error, but predictions.csv has this model at 2276.66, the corrected value." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-6-luna", + "old": "That includes the 0.425% UI/WF/SWF contribution, which the output excludes, with overstated TDI/FLI rates, instead of 0.19% TDI plus 0.23% FLI ($112.56).", + "new": "That includes the 0.425% UI/WF/SWF contribution, which New Jersey requires and the frozen reference leaves out, with overstated TDI/FLI rates. The frozen reference's NJ addition is 0.19% TDI plus 0.23% FLI ($112.56); the exclusion's corrected value adds the 0.425% contribution ($113.90), for $226.46, which its $263.98 exceeds by $37.52.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI/WF/SWF contribution and gave the frozen $112.56 as the NJ answer." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-6-sol", + "old": "Applied TDI at 0.19% and FLI at 0.23% correctly but also added the 0.425% unemployment/workforce contribution ($113.90), which the output does not include. That doubled the NJ addition to $226.46 instead of $112.56.", + "new": "Applied TDI at 0.19% and FLI at 0.23% correctly and added the 0.425% unemployment/workforce contribution ($113.90), which New Jersey requires and the frozen reference leaves out. Its NJ addition of $226.46, for $2,276.66, is the exclusion's corrected value; the frozen reference's is $112.56.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the added contribution an error, but predictions.csv has this model at 2276.66, the corrected value." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "The UI/workforce piece is not part of the payroll_tax output, so NJ tax should be $112.56.", + "new": "That 0.3825% is the worker unemployment contribution, which New Jersey requires and the frozen reference leaves out; the model did not add the 0.0425% workforce share ($11.39), so its $2,265.27 is $102.51 above the frozen reference's $2,162.76 and $11.39 below the exclusion's corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row called the UI contribution an error and gave the frozen $112.56 as the NJ figure; predictions.csv has this model at 2265.27, $11.39 (0.0425% of $26,800) below the corrected value." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "grok-4.6", + "old": "Added the NJ employee UI contribution (0.3825%), which the output excludes, plus about $75 of TDI/FLI. The correct NJ addition is only TDI at 0.19% plus FLI at 0.23%, or $112.56.", + "new": "Added the NJ employee UI contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, plus about $75 of TDI/FLI. The frozen reference's NJ addition is TDI at 0.19% plus FLI at 0.23%, or $112.56; the exclusion's corrected value adds UI and the 0.0425% workforce share ($113.90), for $226.46, and the model's $2,228 is $48.66 below the corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI contribution and that only TDI and FLI apply." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "grok-4.7", + "old": "Included a 0.425% UI/workforce contribution, which the output does not count, and overstated TDI (0.23% vs 0.19%) and FLI (0.33% vs 0.23%). That gives $263.98 of NJ tax instead of $112.56.", + "new": "Included a 0.425% UI/workforce contribution, which New Jersey requires and the frozen reference leaves out, and overstated TDI (0.23% vs 0.19%) and FLI (0.33% vs 0.23%). That gives $263.98 of NJ tax, $151.42 above the frozen reference's $112.56 and $37.52 above the $226.46 of the exclusion's corrected value.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output does not count the UI/workforce contribution and gave the frozen $112.56 as the NJ figure." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "inkling", + "old": "Added the NJ UI employee contribution (0.3825%), which the output excludes, used a stale 0.09% FLI rate instead of 0.23%, and omitted TDI at 0.19%. That gave about $127 of NJ tax instead of $112.56.", + "new": "Added the NJ UI employee contribution (0.3825%), which New Jersey requires and the frozen reference leaves out, used a stale 0.09% FLI rate instead of 0.23%, and omitted TDI at 0.19% and the 0.0425% workforce share. That gave about $127 of NJ tax, against the frozen reference's $112.56 and the $226.46 of the exclusion's corrected value; its $2,177 is $99.66 below the corrected value, $2,276.66.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI contribution and gave the frozen $112.56 as the NJ figure." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "kimi-k3", + "old": "Added $257.95 of NJ contributions, including the UI/workforce employee contribution, which the output excludes, and inflated TDI/FLI rates. The correct NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64).", + "new": "Added $257.95 of NJ contributions, including the UI/workforce employee contribution, which New Jersey requires and the frozen reference leaves out, and inflated TDI/FLI rates. The frozen reference's NJ addition is TDI at 0.19% ($50.92) plus FLI at 0.23% ($61.64), or $112.56; the exclusion's corrected value adds the unemployment and workforce contributions ($113.90), for $226.46, which its $257.95 exceeds by $31.49.", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes the UI/workforce contribution and gave the frozen TDI-plus-FLI component as the correct NJ addition." + }, + { + "case_id": "us__scenario_008__payroll_tax", + "field": "annotation", + "model": "qwen-3.7-max", + "old": "For 2026 the output counts TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64), and excludes UI.", + "new": "For 2026 the frozen reference counts TDI at 0.19% ($50.92) and FLI at 0.23% ($61.64) and leaves out UI, which New Jersey requires; the exclusion's corrected value counts UI at 0.3825% and the workforce funds at 0.0425% ($113.90), for $2,276.66, which its $2,152.71 misses by $123.95: TDI, FLI and the workforce share ($11.39).", + "reason": "reference_exclusions.json (scenario_008 payroll_tax, r05_nj_worker_ui) records as the engine defect that PolicyEngine leaves the New Jersey worker unemployment (0.3825%) and workforce (0.0425%) contributions out of employee payroll tax (N.J.S.A. 43:21-7(d)(1)), corrected value 2,276.66 ($113.90 more on the $26,800 of wages). The row said the output excludes UI; predictions.csv has this model at 2152.71, which is FICA plus 0.3825% UI." + }, + { + "case_id": "us__scenario_023__snap", + "field": "case_annotation", + "old": "That gives $298 − $262.80 = $35.20/month for January–September and $304.68 − $256.50 = $48.18/month for October–December after the FY2027 update, or $461.34 for the year.", + "new": "That gives the frozen reference's $298 − $262.80 = $35.20/month for January–September and $304.68 − $256.50 = $48.18/month for October–December under the engine's projected FY2027 schedule, or $461.34 for the year. The release's SNAP convention (c_snap_hold_fy2026) holds the FY2026 schedule for all 12 months, under which the current engine, with the SNAP rounding fixes applied with the convention, gives $34/month ($408).", + "reason": "States the FY2027 $48.18 and the $461.34 total as the benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). Same correction as the earlier amendment of the old rationale; the rest of the rationale, including its reading of the unlisted input, stays." + }, + { + "case_id": "us__scenario_023__snap", + "field": "case_annotation", + "old": "Under the 7 CFR 271.2 definition, which PolicyEngine follows, the member must receive", + "new": "Under the 7 CFR 271.2 definition, the member must receive", + "reason": "Presents the engine's SNAP disability test as the 7 CFR 271.2 definition, but the record shows it departs from it: under policyengine-us 1.755.4, which produced the frozen reference, meeting SSI's disability criterion alone switches on the elderly-or-disabled SNAP rules (reference_exclusions.json scenario_023 snap note; us_adjudications.json reasoning), and revisions[17] excluded_outputs_rechecked scenario_023 snap says 2.15.17 does not model the 7 CFR 271.2(11) route (receipt of disability-based Medi-Cal) and that SSI criteria stopped conferring SNAP status only with #9345. The statement of the definition itself stays." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "ox-alpha", + "old": "The model correctly capped the shelter deduction at $744, but it used the outdated $204 standard deduction instead of $209 and applied the FY2026 $298 maximum to all 12 months. From October 2026 the FY2027 update raises the maximum to $304.68 and lowers the contribution to $256.50, giving $48.18/month for October–December, so the correct total is $461.34, not $402.", + "new": "The model correctly capped the shelter deduction at $744 and applied the FY2026 $298 maximum to all 12 months, as the release's SNAP convention (c_snap_hold_fy2026) does, but it used the outdated $204 standard deduction instead of $209. The frozen reference's $48.18/month for October–December ($304.68 maximum, $256.50 contribution) uses the engine's projected FY2027 schedule, which is how it reaches $461.34; with the convention's FY2026 schedule all year, the current engine gives $408 ($34 × 12, which also reflects the SNAP rounding fixes) against its $402.", + "reason": "Calls applying the FY2026 $298 maximum to all 12 months an error and states the FY2027 update and $461.34 as correct. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The $204 standard-deduction error stays. Same correction as the earlier amendment of this row; the row's class (period_annualization) is unchanged." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "deepseek-v4-flash-0731", + "old": "and the annual benefit is $461, not $1,995.", + "new": "and the frozen reference's annual benefit is $461, not $1,995. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3.5-flash", + "old": "That gives $35.20/month for most of the year, not about $200.", + "new": "That gives the frozen reference's $35.20/month for January–September, not about $200. The frozen reference's higher October–December amount uses the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "'For most of the year' presents the engine's projected FY2027 October-December change ($48.18) as the rule. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gemini-3.6-flash", + "old": "and the annual benefit is $461, not $1,980.", + "new": "and the frozen reference's annual benefit is $461, not $1,980. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "glm-5.3", + "old": "and the benefit to $461/year.", + "new": "and the frozen reference's benefit to $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The shelter-deduction and annual-versus-monthly errors stay." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-5.4-nano", + "old": "and receives $461 for the year.", + "new": "and receives $461 for the year in the frozen reference, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The missed-eligibility error stays." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-5.6-luna", + "old": "and the benefit is $35.20/month for most of 2026.", + "new": "and the frozen reference's benefit is $35.20/month for January–September 2026, with a higher October–December amount from the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "'For most of 2026' presents the engine's projected FY2027 October-December change ($48.18) as the rule. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-5.6-sol", + "old": "giving $461/year instead of $2,424.", + "new": "giving the frozen reference's $461/year instead of $2,424. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "gpt-6.1-sol", + "old": "and the annual benefit is $461.", + "new": "and the frozen reference's annual benefit is $461, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "grok-4.5", + "old": "and the benefit is $461/year.", + "new": "and the frozen reference's benefit is $461/year, whose October–December months use the engine's projected FY2027 schedule; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "inkling", + "old": "which gives $461/year.", + "new": "which gives the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The disability-reading comparison stays, left to the caveat (unlisted input meets_ssi_disability_criteria)." + }, + { + "case_id": "us__scenario_023__snap", + "field": "annotation", + "model": "qwen3.8-max", + "old": "which yields $461/year.", + "new": "which yields the frozen reference's $461/year. That $461 uses the engine's projected FY2027 schedule for October–December; the release's SNAP convention (c_snap_hold_fy2026) holds FY2026 for all 12 months, under which the current engine, with the SNAP rounding fixes, gives $34/month ($408).", + "reason": "States the frozen $461 as the annual benefit. meta.json revisions[0] c_snap_hold_fy2026 holds FY2026 for Oct-Dec 2026 (FY2027 published 2026-08-21, after the freeze); $461.34 = 35.20x9 + 48.18x3, whose $48.18 is the frozen reference's engine projection (revisions[17] excluded_outputs_rechecked scenario_023 snap: latest_conventions gives 408 = $34 x 12). The unsupported income-and-resources claim stays." + } + ] +} \ No newline at end of file diff --git a/app/bun.lock b/app/bun.lock index 30fb4ec8..f6bc76e2 100644 --- a/app/bun.lock +++ b/app/bun.lock @@ -17,6 +17,7 @@ "@types/react-dom": "^19.2.3", "eslint": "^9.38.0", "eslint-config-next": "^16.2.6", + "fast-check": "^4.10.2", "next": "^16.2.6", "tailwindcss": "^4.2.4", "typescript": "~5.9.3", @@ -1063,6 +1064,8 @@ "extend-shallow": ["extend-shallow@2.0.1", "", { "dependencies": { "is-extendable": "^0.1.0" } }, "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug=="], + "fast-check": ["fast-check@4.10.2", "", { "dependencies": { "pure-rand": "^8.0.0" } }, "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw=="], + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], "fast-glob": ["fast-glob@3.3.1", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.4" } }, "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg=="], @@ -1609,6 +1612,8 @@ "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "pure-rand": ["pure-rand@8.4.2", "", {}, "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng=="], + "query-string": ["query-string@9.3.1", "", { "dependencies": { "decode-uri-component": "^0.4.1", "filter-obj": "^5.1.0", "split-on-first": "^3.0.0" } }, "sha512-5fBfMOcDi5SA9qj5jZhWAcTtDfKF5WFdd2uD9nVNlbxVv1baq65aALy6qofpNEGELHvisjjasxQp7BlM9gvMzw=="], "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], diff --git a/app/next.config.ts b/app/next.config.ts index 2f6491cf..b245cee5 100644 --- a/app/next.config.ts +++ b/app/next.config.ts @@ -1,7 +1,21 @@ import type { NextConfig } from "next"; +// Note URLs that moved. The BBCE note, first published September 23, was +// republished as of October 5 on release dashboard-data-20260930; its old URL +// stays a permanent link to it. +export const noteRedirects = [ + { + source: "/notes/2026-09-23-five-snap-households-bbce", + destination: "/notes/2026-10-05-five-snap-households-bbce", + permanent: true, + }, +]; + const nextConfig: NextConfig = { reactStrictMode: true, + async redirects() { + return noteRedirects; + }, }; export default nextConfig; diff --git a/app/package.json b/app/package.json index 070be755..a414799c 100644 --- a/app/package.json +++ b/app/package.json @@ -20,6 +20,7 @@ "@types/react-dom": "^19.2.3", "eslint": "^9.38.0", "eslint-config-next": "^16.2.6", + "fast-check": "^4.10.2", "next": "^16.2.6", "tailwindcss": "^4.2.4", "typescript": "~5.9.3" diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf index 135c63a9..2630f496 100644 Binary files a/app/public/paper/policybench.pdf and b/app/public/paper/policybench.pdf differ diff --git a/app/public/paper/web/figures/positive_zero_scatter.png b/app/public/paper/web/figures/positive_zero_scatter.png index a10e64cb..787a6ac2 100644 Binary files a/app/public/paper/web/figures/positive_zero_scatter.png and b/app/public/paper/web/figures/positive_zero_scatter.png differ diff --git a/app/public/paper/web/index.html b/app/public/paper/web/index.html index 98ab5fe7..fe427714 100644 --- a/app/public/paper/web/index.html +++ b/app/public/paper/web/index.html @@ -129,7 +129,8 @@

Table of contents

  • Data and scenario construction
  • Results

    Abstract

    -

    PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset, scored through policyengine.py 4.16.1. It evaluates 39 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 84% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 66.9% weighted exact by 22.3 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-09-05), the top-scoring US model on exact match is GPT-5.6 Sol at 89.2%, while GPT-5.4 nano is lowest at 62.6%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 73.1% exact, below the 77.8% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 8,783 rows whose legacy threshold score is below 1. That universe contains 8,780 of the snapshot’s 8,780 exact-match misses and 3 exact hits; 1,605 additional rows with bounded score below 100 were not selected or annotated. Reference-suspect flags in the selected universe were adjudicated against primary sources and engine traces, and the frozen release has no known reference-computation defects. The 0.9% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.

    +

    PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.15.17. It evaluates 46 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 86% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 70.0% weighted exact by 25.0 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-09-30), the top-scoring US model on exact match is GPT-6 Sol at 95.0%, while GPT-5.4 nano is lowest at 65.3%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 76.2% exact, below the 81.0% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 7,860 rows whose legacy threshold score is below 1. That universe contains 7,856 of the snapshot’s 7,856 exact-match misses and 4 exact hits; 2,107 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-09-29 with policyengine-us 2.15.17, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 28 outputs whose references rest on engine defects the audits found and upstream has not fixed, and the 28 whose references depend on an input the prompt does not state. The 0.7% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at https://policybench.org.

    Introduction

    Household tax-and-benefit estimation sits between arithmetic and policy discussion. Each case has numeric labels, but generating those labels requires filing status, household composition, income concepts, program thresholds, and jurisdiction-specific rules. PolicyBench measures whether models can map household records to those outputs.

    Most current language-model evaluations do not test this mapping directly. General math benchmarks emphasize symbolic manipulation and exact final answers (Cobbe et al. 2021; Hendrycks et al. 2021), while generic question-answering benchmarks emphasize recall or instruction following. PolicyBench instead asks whether models can transform household facts into policy outputs without access to tools or simulators.

    @@ -162,11 +163,11 @@

    The nearest public-benefits benchmark is the concurrent Public Benefits Bench, built by Vals AI with the Center for Civic Futures and Code for America (Vals AI 2026). It evaluates models on 459 expert-validated SNAP guidance scenarios (230 in the published test set) spanning all 50 US states plus Guam and the Virgin Islands, and grades free-text answers against expert-written rubric criteria using a judge model that agreed with SNAP policy experts on 80.6% of grading decisions. Where PolicyBench isolates parametric knowledge by withholding tools, Public Benefits Bench varies tool access directly: across models, multi-turn conversation raises pass rates by roughly 7.6 percentage points and web search by about 6.9. The two benchmarks probe complementary failure surfaces — conversational guidance quality for a single program with tools available, versus no-tool numeric estimation across tax and transfer outputs scored deterministically against microsimulation references — and reach consistent conclusions. The best model passes 71.7% of rubric criteria under Public Benefits Bench’s strongest condition, and its authors conclude that “no general-purpose AI model performs well enough to be trusted with SNAP benefits guidance,” matching the deployability gaps PolicyBench reports for unaided calculation.

    Finally, PolicyBench depends on two infrastructure literatures that are not themselves large language model (LLM) benchmarks. One is structured-output reliability, since the benchmark relies on multi-output JSON responses and parse coverage rather than free-form prose (Shorten et al. 2024). The other is tax-benefit microsimulation, where systems such as EUROMOD provide the methodological precedent for evaluating policy rules over household microdata (Sutherland and Figari 2013). There is also operational work on applying and evaluating AI systems in public-benefits settings, including caseworker-assist and Supplemental Nutrition Assistance Program (SNAP)-focused evaluations (Nava Labs 2026, 2025; ZenML LLMOps Database 2025). PolicyBench combines these strands into a public cross-model benchmark over household-level tax and benefit outputs.

    Benchmark design

    -

    The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 39 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.

    +

    The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. 10 of the 46 models received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.

    Headline metric: exact match

    The headline metric is the exact-match rate: the share of requested outputs the model gets right to the dollar (or, for eligibility flags, to the boolean). This is the deployability bar. A tax filer, a benefit estimator, or a caseworker cannot ship “close” — a prediction off by $50 is no more usable than one off by $500, because neither matches the bottom line a downstream system or a household needs. For currency amounts, “exact” means within 1 currency unit of the reference value after numeric parsing. For binary outputs, “exact” means the parsed value is exactly the same 0/1 flag as the reference.

    -

    Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 84% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 84% unweighted, leaving little room above the floor to separate models that actually understand the rules.

    -

    The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 84% row share to 66.9%. An always-zero baseline therefore scores 66.9% weighted exact rather than the 84% it earns unweighted, and the top model clears that baseline by 22.3 points — a margin comparable to within-1%. Across the 39-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.

    +

    Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 86% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 86% unweighted, leaving little room above the floor to separate models that actually understand the rules.

    +

    The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 86% row share to 70.0%. An always-zero baseline therefore scores 70.0% weighted exact rather than the 86% it earns unweighted, and the top model clears that baseline by 25.0 points — a margin comparable to within-1%. Across the 46-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.

    The exact rate is aggregated to a household score and then to a country score, with the same population-derived output weights that the within-1% companion and the bounded score use. Equal household weight preserves comparability across scenarios and slices. The leaderboard reports US results on this fixed test set.

    Near-miss companion: within 1%

    The within-1% hit rate is the near-miss-tolerant companion to the headline: the share of requested outputs the model gets within one percent of the PolicyEngine reference value. For currency amounts, “within 1%” means |pred − ref| ≤ 0.01 × |ref| when the reference is nonzero and |pred| ≤ 1 (the same one-currency-unit absolute tolerance used by exact match) when the reference is zero. Binary outputs are requested as integer 0/1 eligibility flags and scored identically to the exact-match metric.

    @@ -178,8 +179,8 @@

    Secondary me

    Aggregation proceeds in three steps. First, each household-output prediction receives a 0-100 score. Second, each household receives one model score: requested output rows are weighted by output-group weights constructed from the full weighting population, then renormalized within the household over outputs that are actually requested for that household. Person-level coverage flags are scored at the person row; their output-group weight is split across the relevant people in the household and is based on PolicyEngine value proxies where available. Third, the country score averages those household scores with equal household weight. The leaderboard reports US results on this fixed test set. Equal-output-group scores and other alternative views are reported as sensitivity checks.

    The benchmark requires each model response to include numeric answers and one explanation per requested output. Both the exact-match rate and the bounded score use only the numeric answers. Explanations are retained for scenario exploration and qualitative error analysis; they should not be interpreted as faithful traces of model reasoning.

    Because explanations are required, the canonical task measures policy estimation under a public-facing structured-response contract, not isolated arithmetic accuracy. Prompt fairness is part of the benchmark contract. The current release uses one prompt template per country, with no model-specific tuning. Models receive the same household facts and requested outputs. They receive no web or external tool access. The prompt sets unlisted numeric inputs to 0, unlisted boolean or status facts to false, and household characteristics as constant over the tax-benefit year. Provider-specific differences are limited to structured-output transport, request shape, and the reasoning setup in Table 2. The harness sends no reasoning-control or sampling parameters. It leaves reasoning effort, temperature, and related decoding controls at provider defaults. The frozen GPT-5.5 wave uses provider-default reasoning effort.

    -

    The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 1.2 points above that row. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.

    -

    Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 39 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.

    +

    The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 0.9 points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.

    +

    Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). 10 of the 46 models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated.

    One row, GLM-5.3-Flash (preview), answered as a cloaked OpenRouter listing called “Ox Alpha” and is labeled a preview. It was publicly callable under the canonical whole-scenario request and forced-tool transport, and its date is its OpenRouter listing date. After the row’s run, Z.ai identified the listing as GLM-5.3-Flash (OpenRouter 2026); the row carries the disclosed name and keeps the preview qualifier, because whether the checkpoint that answered is the one later released under that name has not been stated, and it is distinct from the GLM-5.3 row, which is Z.ai’s larger model called under its own id. Its artifact key stays ox-alpha, the name it was queried under.

    Frozen snapshot and open-set status

    Leaderboard positions are version-sensitive, so manuscript claims refer to the frozen source-run exports in Table 1 rather than to the live site. The committed source-run exports are the manuscript artifacts. The public site exposes the current prompts, predictions, explanations, and reference outputs for transparency. This makes the public leaderboard open-set: models, model providers, or benchmark users could learn from the released cases before later runs. Protected leaderboard claims would require a separate held-out or rotating set.

    @@ -205,11 +206,11 @@

    Frozen snapshot an Snapshot date -2026-09-05 +2026-09-30 Model response date -2026-06-12 to 2026-09-05 +2026-06-12 to 2026-09-30 Policy period @@ -221,7 +222,7 @@

    Frozen snapshot an Frozen export SHA-256 prefix -US 2cce2598dbf9 +US 1e029aaa87d1 Snapshot manifest @@ -237,7 +238,7 @@

    Frozen snapshot an US reference outputs SHA-256 prefix -b9136a15e285 +e8bbba8fd3e9 Benchmark spec SHA-256 prefix @@ -249,7 +250,7 @@

    Frozen snapshot an Scoring code SHA-256 prefix -fdb87bf9e1f9 +baaf2506800b US run label @@ -260,12 +261,12 @@

    Frozen snapshot an 42 -PolicyEngine.py -4.16.1 +Reference engine +policyengine-us 2.15.17 -PolicyEngine-US -policyengine-us 1.755.4 +PolicyEngine.py (recorded for provenance) +6.1.2 US dataset @@ -277,7 +278,7 @@

    Frozen snapshot an Models -39 +46 Output groups @@ -298,12 +299,12 @@

    Frozen snapshot an -
    +
    -Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through the transport its model card records, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the provider rejects a forced tool or where the model card selects JSON. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for ten rows; tool choice for nine rows. Reasoning setup and timeouts for every row, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date. +Table 2: Model run configuration in the frozen manuscript snapshot. Every model answers through its row’s transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (tool_choice set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. Supervised-run fingerprints pin answer contract, request shape, and completion ceiling for 17 rows; tool choice for 16 rows; reasoning setup and timeouts for seven rows. Reasoning setup and timeouts for the other ten fingerprinted rows, and all fields for the other 29 rows, are the harness registry as frozen in the snapshot’s serving-configuration file. Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1’s date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3’s weights, announced at its API launch (Willison 2026), shipped on Hugging Face on July 27, 2026 under a custom license (Tech Times 2026). GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run (OpenRouter 2026), and its Released date is its August 20, 2026 OpenRouter listing date.
    @@ -332,7 +333,7 @@

    Frozen snapshot an

    - + @@ -341,7 +342,7 @@

    Frozen snapshot an

    - + @@ -350,7 +351,7 @@

    Frozen snapshot an

    - + @@ -359,7 +360,7 @@

    Frozen snapshot an

    - + @@ -368,7 +369,7 @@

    Frozen snapshot an

    - + @@ -377,34 +378,52 @@

    Frozen snapshot an

    - + + + + + + + + + + - + - + - + + + + + + + + + + - + @@ -413,7 +432,7 @@

    Frozen snapshot an

    - + @@ -422,160 +441,196 @@

    Frozen snapshot an

    - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -584,7 +639,7 @@

    Frozen snapshot an

    - + @@ -593,7 +648,7 @@

    Frozen snapshot an

    - + @@ -602,79 +657,88 @@

    Frozen snapshot an

    - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -688,7 +752,9 @@

    Frozen snapshot an

    Data and scenario construction

    United States

    -

    The US benchmark is built from the certified PolicyEngine populace dataset (populace_us_2024) using PolicyEngine US. The dataset contains 160,858 people in 75,112 households, with household, tax-unit, SPM-unit, family, marital-unit, and person records for PolicyEngine calculations. The sampled households are filtered to keep a single-tax-unit, single-family, single-Supplemental Poverty Measure (SPM)-unit structure with at least one adult and a supported filing status. Of the 75,112 households, 63,128 (84.0%) pass the filter and form the eligible draw (this filter-pass rate is coincidentally close to the 84% zero-output share discussed under the benchmark design; the two figures are unrelated). The 16.0% excluded by the filter include multi-tax-unit households (e.g., adult roommates), multi-family households, multi-SPM-unit households, and households whose head reports a filing status outside the supported set. These excluded compositions are exactly the kind of cases where federal/state credit allocations and benefit-unit rules become hardest, so the eligible draw is a tractable subset rather than the full distribution of US households. Prompts include nonzero promptable raw inputs across relevant entities rather than a hand-curated summary, so the models see many of the same facts the simulator receives. Filing status is not stated in the prompt; the reference computation infers it from tax-unit role flags. Models therefore see the same household facts that drive the reference filing-status assignment, but they do not receive that assignment as a label.

    +

    The US benchmark is built from the certified PolicyEngine populace dataset (populace_us_2024) using PolicyEngine US. The dataset contains 160,858 people in 75,112 households, with household, tax-unit, SPM-unit, family, marital-unit, and person records for PolicyEngine calculations. The sampled households are filtered to keep a single-tax-unit, single-family, single-Supplemental Poverty Measure (SPM)-unit structure with at least one adult and a supported filing status. Of the 75,112 households, 63,128 (84.0%) pass the filter and form the eligible draw (this filter-pass rate is coincidentally close to the 86% zero-output share discussed under the benchmark design; the two figures are unrelated). The 16.0% excluded by the filter include multi-tax-unit households (e.g., adult roommates), multi-family households, multi-SPM-unit households, and households whose head reports a filing status outside the supported set. These excluded compositions are exactly the kind of cases where federal/state credit allocations and benefit-unit rules become hardest, so the eligible draw is a tractable subset rather than the full distribution of US households. Prompts include nonzero promptable raw inputs across relevant entities rather than a hand-curated summary, so the models see many of the same facts the simulator receives. Filing status is not stated in the prompt; the reference computation infers it from tax-unit role flags. Models therefore see the same household facts that drive the reference filing-status assignment, but they do not receive that assignment as a label.

    +

    Disability in the household facts

    +

    The household facts carry one disability fact, while the programs apply several. The certified microdata marks a person as disabled when they report any of the six disability-difficulty items in the Current Population Survey, and the prompt passes that flag through as is disabled: 33 of the 177 people in the benchmark households carry it. The law defines disability program by program, and policyengine-us follows it with a separate input for each determination. SSI requires its own disability criterion and no substantial gainful activity (42 U.S.C. 1382c(a)(3)). SNAP counts a member as disabled through receipt-based routes, among them receiving SSI or Social Security disability benefits and a qualifying veterans’ status (7 U.S.C. 2012(j)); policyengine-us approximates the SSI route with SSI disability status, which does not require receiving SSI. Medicare’s disability route before age 65 requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), the route policyengine-us models. The tax code uses permanent and total disability for dependents (26 U.S.C. 152), retirement on disability for the credit for the elderly and disabled (26 U.S.C. 22), and incapacity for self-care for the child and dependent care credit (26 U.S.C. 21). The general flag enters none of these determinations; in the reference engine it reaches mainly state provisions, and in SNAP only the work-requirement and student rules. One of those state provisions is California’s Working Disabled Program, a Medi-Cal pathway for people with disabilities who work: the program uses SSI’s definition of disability (42 CFR 435.540(a)), and policyengine-us tests the general flag. No benchmark person carries any of the program-specific inputs. The certified build never sets SSI’s disability criterion, and the months of Social Security disability receipt are integer-typed, which the prompt builder does not expose. The references of a person the prompt calls disabled therefore take the non-disabled path through each of these determinations unless another listed fact establishes it: listed Social Security disability income, for example, makes a person disabled for SNAP. A careful reader of is disabled could take either path, so wherever the other reading moves a reference, the output is excluded from scoring for every model (Section 4.3): SSI and SNAP outputs under SSI’s disability criterion, the SNAP output whose engine defect turns on whether its head is a disabled member, and one California Medicaid output, which the other reading moves once the engine’s Working Disabled Program test reads SSI’s definition. Separately, the prompt lists Social Security disability income but not how long it has been received, so the Medicare eligibility of the five household heads with that income, which turns on 24 months of it, is excluded whether or not the prompt calls them disabled. Where the other reading moves nothing, the output stays scored. The next scenario refresh defines the prompt’s disability fact and states the program-specific determinations as separate facts (PolicyEngine/policybench#165, PolicyEngine/microcosm#876).

    The current US release requests 18 scored output groups spanning federal income tax, refundable credits, payroll and self-employment tax, state and local income tax, Supplemental Nutrition Assistance Program (SNAP), Supplemental Security Income (SSI), Temporary Assistance for Needy Families (TANF), school-meal eligibility, and person-level coverage eligibility for the Special Supplemental Nutrition Program for Women, Infants, and Children (WIC), Medicaid, the Children’s Health Insurance Program (CHIP), Medicare, Head Start, and Early Head Start; several of these categories span two scored groups each (federal and state refundable credits, payroll and self-employment tax, state and local income tax, free and reduced-price school meals), which is how the named categories reconcile to the 18 output groups. The source run also requested the Affordable Care Act (ACA) Premium Tax Credit (PTC), but explanation audits showed that the prompt could be misleading when households lacked plan-specific Marketplace information. We therefore preserve those raw responses but exclude PTC from the canonical scored leaderboard until the prompt contract is revised.

    The output scope is intentionally narrower than the full PolicyEngine model. Table 3 summarizes the inclusion rule. The benchmark asks for WIC eligibility rather than a WIC dollar amount; WIC dollar values are used only as impact-weight proxies for coverage flags, not as requested model outputs.

    @@ -734,16 +800,20 @@

    United States

    -

    Reference-output credibility

    +

    Reference-output credibility

    PolicyBench treats PolicyEngine outputs as benchmark reference outputs, not as administrative records. The reference source is nevertheless stronger than an ad hoc answer key: PolicyEngine is open source, used for household calculators and reform analysis, and externally checked in specific domains. No. 10 Downing Street’s data science team adapted PolicyEngine’s open-source microsimulation model for experimental policy simulation, with validation against external projections before use (Woodruff 2026; Ghenis 2026). In the US, PolicyEngine (2024) reports matching the National Bureau of Economic Research (NBER) TAXSIM-35 model (Feenberg and Coutts 1993) to the cent on the vast majority of cases for the 2021 tax year across hundreds of thousands of tax units per state, with state-specific differences documented in the integration tests. We do not restate that comparison as a single percentage because the published source uses qualitative phrasing rather than a headline accuracy number. PolicyEngine has also signed a memorandum of understanding (Ghenis and Makarchuk 2025) with the Federal Reserve Bank of Atlanta for future validation work against its Policy Rules Database (2026). The Atlanta Fed sources are a caveat rather than evidence of completed validation for this benchmark: they document planned collaboration and the comparison source, not finished checks of the frozen PolicyBench outputs. Taken together, these sources support using PolicyEngine as a transparent reference implementation with partial external validation, but they do not validate every benchmark output.

    -

    This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and three audit waves, the model-assisted review covered 8,783 rows whose legacy threshold score is below 1. That selected universe contains 8,780 exact-match misses and 3 exact hits; 1,605 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Reviewers raised 35 reference-suspect flags in total. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s reference outputs are generated with the fixed engine, policyengine-us 1.755.4. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. The asymmetry of this record is itself evidence about the reference: even among the cells the reviewing models found most suspicious, 31 of 35 flags (89%) resolved to model-side errors, set against thousands of confirmed model errors overall and four reference defects. Table 4 summarizes the reviewed discrepancy classes and outcomes; no unresolved reference-computation defect is known in the frozen snapshot.

    +

    This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered 7,860 rows whose legacy threshold score is below 1. That selected universe contains 7,856 exact-match misses and 4 exact hits; 2,107 additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine’s computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer’s hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot’s scored reference outputs come from the fixed engine, policyengine-us 2.15.17. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. Table 4 summarizes the reviewed discrepancy classes and outcomes.

    +

    The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged 39 references. Adjudication against primary sources and sandbox runs of the reference engine affirmed 5 of them, traced 8 to inputs the prompt never lists, replaced 6 with a regenerated reference, and confirmed 20 as defects in policyengine-us 1.755.4, the version behind the references then. The judge’s stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause’s rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, 8 had been fixed in policyengine-us after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and #9586): capital gain distributions, New York’s renter cap, the CalEITC’s lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, 15 in all, and policyengine-us 2.15.17 contains all 8 fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin’s 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin’s published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model: 28 outputs in 20 households are recorded as engine-defect exclusions across 11 root causes, and 8 of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). They stay excluded until references built on a fixed engine replace them. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household’s survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires.

    +

    The same wave tested a failure mode no judge flagged. policyengine-us fills a parameter’s values past its last encoded year by projecting it with a price index, so a 2026 reference can rest on a forecast rather than on the amount a government published. Every reference was recomputed with each projected 2026 value held at its last encoded value, and each projected amount behind a moved output was then sourced. Where the government published the 2026 amount before the freeze, the reference takes it: Minnesota’s, Michigan’s and Missouri’s brackets, deductions and exemptions, and Maryland’s withholding allowance, which reaches federal tax through the state and local tax deduction. Wisconsin had published its 2026 brackets and deductions too, but every Wisconsin output they move is excluded for an engine defect, so they enter only those outputs’ corrected values. Where no 2026 amount published before the freeze was found, the reference takes the last amount published: California’s indexed amounts, whose 2026 factor rests on June 2026 prices; the IRS optional sales tax tables, whose 2026 edition had not been found by September 22; Idaho’s zero-rate threshold; Maryland’s return deduction; and SNAP’s FY2027 figures, published on 2026-08-21. Illinois’s exemption and Minnesota’s child credit projections matched the published amounts. These corrections change parameter values and no rule, so the affected references were regenerated with the same engine version rather than excluded: 23 references, most of them SNAP. With the upstream fixes, the September 22 release regenerated 26 references in 24 households, 13 of them SNAP. The reference sidecar records each regenerated value with the fix that produced it, and the fixes, the recomputation of every reference under each, and the verification reports are committed with the snapshot (reference_audit/2026-09-22).

    +

    The SNAP convention as first built also corrected the engine’s SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine’s household size, eligibility and floored net income, the Axiom rules engine’s encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey’s state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income’s cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect is not fixed upstream: the engine still grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, and the scored output it moves is excluded. That household’s prompt calls its head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits; read the other way, the allowance would stand, and so would the reference the publication conventions and upstream SNAP fixes give ($3,576, which the output would carry if scored), so the output is excluded on either reading. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine’s values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA’s SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan’s broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0.

    +

    PolicyBench rebuilt the references on 2026-09-29 with policyengine-us 2.15.17, the newest release when it began sweeping the references that day (uploaded 00:23 UTC), in place of 1.755.4. It computes each scored reference with policyengine_us.Simulation. The snapshot also records policyengine.py 6.1.2 for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside 2.15.17. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA’s FY2027 figures as published values. policyengine-us 2.15.17 counts Maryland county income tax in its state income tax. PolicyBench’s state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP’s work rules read, so the scenario builder also passes each prompt’s stated usual weekly hours under that input’s name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes 4 scored references (Table 4). Two take law published before the freeze that policyengine-us 1.755.4 did not encode: New Jersey’s child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona’s broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York’s Empire State child credit phase-out. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household’s excluded SNAP output also flagged a scored output the move did not change: the head’s Medicaid eligibility. The head’s income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California’s Working Disabled Program requires SSI’s definition of disability, the unlisted input that already excludes the household’s SNAP. policyengine-us 2.15.17 tests the general disability flag there and gives 1 under either reading; with the program’s test reading SSI’s definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another 2 references move by less than the dollar tolerance. The 56 excluded outputs keep the values they were decided on (52 computed with policyengine-us 1.755.4, 4 with 2.15.17), and PolicyBench re-reviewed the 19 of them that move on 2.15.17; all stay excluded. policyengine-us 2.17.0, the newest release when PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as 2.15.17 for all 1,984 outputs under the same conventions and adapter. The reference sidecar lists every change, and reference_audit/2026-09-28 holds the recomputation of every output, the investigations and the reviews.

    The audit tooling itself required hardening, which we report because it changes how much the annotations can be trusted. The derivation narratives displayed alongside reference values are generated from engine computation traces. After earlier model-written narratives were found to misattribute eligibility pathways, mechanical validators now enforce that each narrative states the engine’s actual mechanism — for example, that a child ineligible for CHIP because they qualify for Medicaid is never described as failing a CHIP income test. The failure-audit judge reads these grounded narratives together with engine facts and must return a per-model diagnosis naming the specific rule or computation the model missed. It is barred from re-adjudicating the reference in prose: hedged or reference-adjudicating verdicts are mechanically rejected and re-judged, and genuine reference doubt is routed to a structured flag that triggers the manual adjudication described above.

    -

    After freezing the snapshot and completing response-contract repairs, we annotated the 8,783 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 8,780 of the snapshot’s 8,780 exact-match misses and 3 exact hits. The 1,605 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). A third class, prompt_ambiguity, is reserved for outputs the snapshot does not score: eleven outputs in ten households, whose reference depends on an engine input the certified household data never carried (the SSI disability criterion, and months of SSDI receipt) and the prompt therefore never listed. For each, the reference was recomputed with the engine version that produced it under the reading a careful reader could take of the stated facts, and it moved; the output is removed from scoring for every model, so no model gains or loses from it, and of the 293 annotated rows on those outputs, 287 carry prompt_ambiguity as description rather than as a scored class and the rest are answers that never parsed. Every model is therefore scored on 1,973 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 11 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Two judge models produced the verdicts, and both are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions, and Claude Opus 5, called through the Claude Code CLI, judged every case a September 2026 addition joined (350 of 668 cases, each recorded in a per-case provenance sidecar in the audit tree). Judge verdicts classify misses after scoring and change no score. The two unlisted inputs behind the exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build, and months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose. The prompt lists a general is disabled fact, drawn from the six CPS disability-difficulty items, and never a program-specific determination. The exclusion set was derived mechanically rather than from which models missed: every reference was recomputed under each alternative reading and every output that moved was excluded, which is why it includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit).

    +

    After freezing the snapshot and completing response-contract repairs, we annotated the 7,860 rows whose legacy threshold score is below 1 and their scenario-output cases. Table 5 reports the final classifications for that legacy-threshold audit universe. It contains 7,856 of the snapshot’s 7,856 exact-match misses and 4 exact hits. The 2,107 other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: llm_error (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or parse_contract_failure (a missing or unparsable answer, classified deterministically). Two further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. prompt_ambiguity marks 28 outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine’s Working Disabled Program test read SSI’s definition, as the program does). reference_engine_defect marks the 28 outputs where that engine misapplies the law on stated facts. Together they remove 56 outputs in 39 households, and of the 2,111 annotated rows on those outputs, 2,056 carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP’s October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on 1,928 of its 1,984 requested outputs. The exclusion record (reference_exclusions.json) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records 69 cases of developer adjudication, kept beside the annotations with the judge’s verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the 116 cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29 and September 30 additions joined or a reference revision changed (260 of 674 cases), its own row’s cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (meets_ssi_disability_criteria), false for every person in the certified June 2026 microdata build; months of SSDI receipt (months_receiving_social_security_disability), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (weekly_hours_worked_before_lsr), which policyengine-us 1.755.4 defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin’s homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers’ child, which decides whose income counts in that dependent’s Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general is disabled fact and never a program-specific determination (Section 4.2). Every exclusion but the California Medicaid output was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect’s sandbox fix, moved on policyengine-us 1.755.4 or, for the state and local tax refund reading, on 2.15.17. On 1.755.4, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On 2.15.17, PolicyBench re-ran four of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves three previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same 2.15.17 calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and three scored outputs move by less. PolicyBench did not re-run the other sweeps on 2.15.17. On 2.15.17, the California Medicaid output stays at 1 under either reading of SSI’s disability criterion, because the engine’s Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household’s excluded SNAP output, and excluded it because the program itself requires SSI’s definition of disability.

    -Table 4: Development discrepancy review before the frozen snapshot. +Table 4: Discrepancy review across the audit waves.
    @@ -783,6 +853,26 @@

    Reference-output credibil

    + + + + + + + + + + + + + + + + + + + +
    Claude Fable 5 claude-fable-5 2026-06-091,973/​1,9731,928/​1,928 forced tool 1 output/​request thinking does not engage under forced tool callClaude Fable 5.1 claude-fable-5-1 2026-09-011,973/​1,9731,928/​1,928 JSON whole scenario provider default; 16,384-token shared budgetClaude Haiku 4.5 claude-haiku-4-5-20251001 2025-10-151,973/​1,9731,928/​1,928 forced tool 1 output/​request provider defaultClaude Opus 4.7 claude-opus-4-7 2026-04-161,973/​1,9731,928/​1,928 forced tool 1 output/​request provider defaultClaude Opus 4.8 claude-opus-4-8 2026-05-281,973/​1,9731,928/​1,928 forced tool 1 output/​request provider defaultClaude Opus 5 claude-opus-5 2026-07-241,973/​1,9731,928/​1,928 forced tool whole scenario thinking does not engage under forced tool call
    Claude Opus 5.5claude-opus-5-52026-09-211,928/​1,928JSONwhole scenarioprovider default; 16,384-token shared budget
    Claude Sonnet 4.6 claude-sonnet-4-6 2026-02-171,973/​1,9731,928/​1,928 forced tool 1 output/​request provider default
    Claude Sonnet 5 claude-sonnet-5 2026-06-301,973/​1,9731,928/​1,928 forced tool 1 output/​request thinking does not engage under forced tool call
    Claude Sonnet 5.5claude-sonnet-5-52026-09-281,928/​1,928JSONwhole scenarioprovider default; 16,384-token shared budget
    DeepSeek V4 Flash 0731 openrouter/​deepseek/​deepseek-v4-flash-0731 2026-07-31†1,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 98,304-token shared budgetDeepSeek V4 Pro deepseek/​deepseek-v4-pro 2026-04-24†1,973/​1,9731,928/​1,928 JSON whole scenario provider default; 16,384-token shared budgetDeepSeek V4 Pro 0813 openrouter/​deepseek/​deepseek-v4-pro-0813 2026-08-13†1,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 98,304-token shared budget
    DeepSeek V4.1 Flashdeepseek/​deepseek-flash2026-09-10†1,928/​1,928JSONwhole scenarioprovider default; 16,384-token shared budget
    Gemini 3 Flash Preview gemini/​gemini-3-flash-preview 2025-12-171,973/​1,9731,928/​1,928 JSON whole scenario provider default
    Gemini 3.1 Flash Lite Preview gemini/​gemini-3.1-flash-lite-preview 2026-03-031,973/​1,9731,928/​1,928 JSON whole scenario provider default
    Gemini 3.1 Pro Preview gemini/​gemini-3.1-pro-preview 2026-02-191,973/​1,9731,928/​1,928 JSON whole scenario provider default
    Gemini 3.5 Flash gemini/​gemini-3.5-flash 2026-05-191,973/​1,9731,928/​1,928 JSON whole scenario provider default
    Gemini 3.5 Flash-Lite gemini/​gemini-3.5-flash-lite 2026-07-211,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    Gemini 3.6 Flash gemini/​gemini-3.6-flash 2026-07-211,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    Gemini 3.7 Flash gemini/​gemini-3.7-flash 2026-08-131,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    Gemini 3.8 Flash gemini/​gemini-3.8-flash 2026-09-021,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GLM-5.2 openrouter/​z-ai/​glm-5.2 2026-06-13†1,834/​1,9731,795/​1,928 JSON 3 outputs/​request provider default; 16,384-token shared budget
    GLM-5.3 openrouter/​z-ai/​glm-5.3 2026-08-14†1,896/​1,9731,857/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GPT-5.4 mini gpt-5.4-mini 2026-03-171,973/​1,9731,928/​1,928 forced tool whole scenario provider default
    GPT-5.4 nano gpt-5.4-nano 2026-03-171,973/​1,9731,928/​1,928 forced tool whole scenario provider default
    GPT-5.5 gpt-5.5 2026-04-231,973/​1,9731,928/​1,928 forced tool 3 outputs/​request provider default; 16,384-token shared budget
    GPT-5.6 Luna gpt-5.6-luna 2026-07-091,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GPT-5.6 Sol gpt-5.6-sol 2026-07-091,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GPT-5.6 Terra gpt-5.6-terra 2026-07-091,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GPT-6 Astra gpt-6-astra 2026-09-041,973/​1,9731,928/​1,928forced toolwhole scenarioprovider default; 16,384-token shared budget
    GPT-6 Lunagpt-6-luna2026-09-221,928/​1,928forced toolwhole scenarioprovider default; 16,384-token shared budget
    GPT-6 Solgpt-6-sol2026-09-221,928/​1,928forced toolwhole scenarioprovider default; 16,384-token shared budget
    GPT-6.1 Solgpt-6.1-sol2026-09-291,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budgetGrok 4.3 xai/​grok-4.3 2026-04-171,973/​1,9731,928/​1,928 forced tool whole scenario provider defaultGrok 4.5 xai/​grok-4.5 2026-07-081,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budgetGrok 4.6 xai/​grok-4.6 2026-08-121,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    Grok 4.7xai/​grok-4.72026-09-211,928/​1,928forced toolwhole scenarioprovider default; 16,384-token shared budget
    Grok Build 0.1 xai/​grok-build-0.1 2026-05-291,973/​1,9731,928/​1,928 forced tool whole scenario provider default
    Inkling openrouter/​thinkingmachines/​inkling 2026-07-15†1,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 49,152-token shared budget
    Kimi K2.6 openrouter/​moonshotai/​kimi-k2.6 2026-04-20†1,551/​1,9731,538/​1,928 JSON 3 outputs/​request provider default; 16,384-token shared budget
    Kimi K3 openrouter/​moonshotai/​kimi-k3 2026-07-16†1,910/​1,9731,870/​1,928 JSON whole scenario provider default; 49,152-token shared budget
    MiniMax M3 openrouter/​minimax/​minimax-m3 2026-06-01†1,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    GLM-5.3-Flash (preview) openrouter/​stealth/​ox-alpha 2026-08-201,973/​1,9731,928/​1,928 forced tool whole scenario provider default; 16,384-token shared budget
    Qwen3.7-max openrouter/​qwen/​qwen3.7-max 2026-05-191,973/​1,9731,928/​1,928 JSON 3 outputs/​request provider default; 16,384-token shared budget
    Qwen3.8-Max openrouter/​qwen/​qwen3.8-max 2026-08-031,973/​1,9731,928/​1,928 JSON whole scenario provider default; 98,304-token shared budget US payroll and overtime Reviewed during development; upstream data fixes and prompt clarifications were applied before the frozen snapshot.
    US federal income tax (September 2026 wave)Engine defects confirmed: the traditional IRA deduction ignores the compensation limit and the active-participant phase-out (IRC 219(b), 219(g)); estate income never enters gross income; elective deferrals count as EITC and refundable-CTC earned income. Affected outputs excluded. Capital gain distributions reported without Schedule D never reached AGI, a defect fixed upstream after the freeze; the references it alone moves are regenerated with the fix. The taxability of employment disability benefits and whether the output includes the net investment income tax excluded as unlisted inputs. The educator-expense flag dissolved against the $350 2026 cap.
    US state income tax and credits (September 2026 wave)Engine defects confirmed in New Jersey worker contributions, Wisconsin’s elected retirement exclusion, Wisconsin’s capital gain exclusion (which leaves out capital gain distributions), Idaho’s health-premium subtraction, California itemized-deduction conformity, and Massachusetts dividends taxed without the short-term capital loss offset; affected outputs excluded. The CalEITC’s missing second lookup at adjusted gross income and New York’s renter cap were fixed upstream after the freeze; the affected references are regenerated with the fixes. Wisconsin homestead income turned on the unlisted type of survivor benefits (excluded as an unlisted input). The New Jersey EITC, New York real property tax credit and Idaho child tax credit flags dissolved against the statute (Idaho’s credit expired after 2025).
    US SNAP, SSI and CHIP (September 2026 wave)Whether listed mortgage interest is on the occupied home, unlisted hours worked, survivor-benefit taxability, and whether an adult tax dependent is the filers’ child (three Medicaid outputs) excluded as unlisted inputs. SNAP October-December references regenerated with the FY2026 figures, the last published before the freeze. SNAP references regenerated for every month of 2026 with the engine’s SNAP rounding defects fixed as policyengine-us fixed them after the freeze (cents in the allotment, an unrounded minimum benefit, floored net income, unrounded income standards). One Michigan SNAP reference regenerated with the fix for the SNAP child support option, whose engine values carried the opposite meaning (Michigan deducts child support from net income, and the engine excluded it from gross income; fixed upstream in policyengine-us #9586 on 2026-09-24). The heat-and-eat utility allowance granted without an elderly or disabled member, which P.L. 119-21 ended, excluded as an engine defect. The CHIP flag dissolved: employer coverage bars CHIP.
    US projected 2026 parameters (September 2026 wave)No judge flag: a recomputation with every projected 2026 value held found references resting on price-index projections. Wisconsin, Minnesota, Michigan and Missouri amounts and Maryland’s withholding allowance had been published before the freeze; California’s indexed amounts, the IRS sales tax tables, Idaho’s threshold and Maryland’s return deduction had not. Scored references regenerated with the published or last-published amounts; Wisconsin’s reach only the corrected values of Wisconsin outputs excluded for engine defects.
    US references on policyengine-us 2.15.17 (September 2026 upgrade)Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us 1.755.4 and 2.15.17 to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes 4 scored references, for law published before the freeze that 1.755.4 did not encode or misapplied: New Jersey’s 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona’s 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York’s Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: 2.15.17 counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the 19 excluded outputs that move; all stay excluded. The re-review of a California household’s excluded SNAP output flagged its head’s Medicaid eligibility, which turns on the same unlisted input, SSI’s disability criterion, through the Working Disabled Program; PolicyBench excluded it.
    @@ -822,18 +912,18 @@

    Reference-output credibil US -8783 -8082 -701 +7860 +7208 +652 0 0 0 Total -8783 -8082 -701 +7860 +7208 +652 0 0 0 @@ -847,7 +937,7 @@

    Reference-output credibil

    Results

    United States leaderboard

    -

    The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-5.6 Sol (89.2 exact, 95% CI 86.3–91.7; 91.5 within-1%), GPT-6 Astra (88.0 exact, 95% CI 85.2–90.7; 89.8 within-1%), and Claude Fable 5.1 (86.9 exact, 95% CI 83.8–89.8; 89.9 within-1%).

    +

    The US leaderboard for the frozen manuscript snapshot is shown in Table 6. The top three models in that snapshot are GPT-6 Sol (95.0 exact, 95% CI 93.2–96.6; 95.9 within-1%), Claude Opus 5.5 (93.7 exact, 95% CI 91.6–95.6; 94.9 within-1%), and GPT-5.6 Sol (93.6 exact, 95% CI 91.3–95.6; 94.3 within-1%).

    @@ -887,53 +977,53 @@

    United States leaderboard 0 -GPT-5.6 Sol -89.2 -86.3–91.7 -1–3 -91.5 -94.2 -95.8 +GPT-6 Sol +95.0 +93.2–96.6 +1–2 +95.9 +97.1 +98.0 1 -GPT-6 Astra -88.0 -85.2–90.7 +Claude Opus 5.5 +93.7 +91.6–95.6 1–4 -89.8 -93.3 -96.3 +94.9 +96.9 +97.6 2 -Claude Fable 5.1 -86.9 -83.8–89.8 -2–5 -89.9 -94.0 +GPT-5.6 Sol +93.6 +91.3–95.6 +1–6 +94.3 95.9 +96.9 3 -Kimi K3 -86.7 -82.6–90.1 -1–7 -88.9 -92.4 +GPT-6 Luna +92.1 +89.8–94.3 +3–9 93.8 +96.7 +97.4 4 -GPT-5.6 Luna -84.6 -81.9–87.3 -4–10 -86.3 -90.9 -92.7 +Claude Sonnet 5.5 +92.1 +89.6–94.3 +3–8 +93.1 +96.1 +97.0 @@ -946,328 +1036,313 @@

    United States leaderboard

    Uncertainty

    Each leaderboard score is a mean over a sample of 100 households, so the rankings carry sampling uncertainty. We quantify it with a household-level bootstrap: the 100 households are resampled with replacement 10,000 times, and each model’s population-weighted exact-match rate is recomputed on every resample. The resampling unit is the household, carried with its full output vector, so within-household correlations across outputs are preserved. The same resampled set is used for every model in each replicate (a paired bootstrap), so the rank ranges in Table 6 reflect genuine ordering uncertainty rather than independent per-model noise.

    -

    Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 8 of the other 38 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

    +

    Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 8 of the other 45 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

    Cost and latency

    -

    Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans \\\\$0.002 (GPT-5.4 nano) to \\\\$0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-5.6 Sol at 89.2% exact, sits mid-pack at \\\\$0.090 per household and 36 s.

    +

    Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.002 (GPT-5.4 nano) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.0% exact, sits mid-pack at $0.027 per household and 23 s.

    -Table 7: Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: provider-reported where the provider returns one, otherwise reconstructed at the configured list price at request time. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household’s summed request time. A dash marks usage or wall-clock latency that was not measured per call. +Table 7: Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household’s summed request time. A dash marks usage or wall-clock latency that was not measured per call.
    -
    -
    - - - +
    +
    - - - - - - + + + + + - + + + + + + + + + + + + - - + + - + + + + + + + + + + + + - - + + - - - + + - + + + + + + - - + + - - + - - + + - - + - + + + + + + + - - - + + - - - + + - - - + + - + + + + + + - - + + - - + - - + + - - + - - + + - - - - - - - - - - + + - - - - - + + + + - - - - - - - - - - + + + + + + + + - + + + + + + - - + + - - + - - + + - - + - - + + + + + + + + - - - + + - - - - - + + + + - - - + + - - - - - + + + + - - - + + - - - + + - - - + + - - - - - - - - - - + + - - + - - + + - - + - - + + - - + - - + + - - + - - + + - - - - - - - - - - + + - + + + + + + - - + + - - + - - + + - - + - - + + - - + - - + +
    ModelExact (%)Cost / householdLatency (median)
    ModelExact (%)Cost /​ householdLatency (median)
    0GPT-6 Sol95.0$0.02723 s
    Claude Opus 5.593.7$0.06724 s
    GPT-5.6 Sol89.2\$0.09093.6$0.090 36 s
    1GPT-6 Luna92.1$0.00229 s
    Claude Sonnet 5.592.1$0.03420 s
    GPT-6 Astra88.0\$0.12691.7$0.126 32 s
    2 Claude Fable 5.186.9\$0.25790.8$0.257 49 s
    3GPT-6.1 Sol90.6$0.02334 s
    Kimi K386.7\$0.47090.4$0.470 1023 s
    4
    GPT-5.6 Luna84.6\$0.01989.3$0.019 14 s
    5
    GLM-5.3-Flash (preview)84.188.5 — 242 s
    Grok 4.788.3$0.260547 s
    6 Inkling84.1\$0.09188.2$0.091 250 s
    7 GPT-5.584.1\$0.26288.2$0.262 —
    8 GPT-5.6 Terra83.5\$0.03987.7$0.039 19 s
    9DeepSeek V4.1 Flash87.1$0.02485 s
    Grok 4.683.1\$0.08786.4$0.087 219 s
    10
    Gemini 3.8 Flash82.2\$0.06885.9$0.068 56 s
    11
    Grok 4.581.3\$0.07385.1$0.073 112 s
    12Claude Fable 580.4\$0.54198 s
    13 Claude Opus 580.3\$0.06784.0$0.067 16 s
    14Gemini 3.7 Flash79.7\$0.0149 sClaude Fable 583.6$0.54198 s
    15DeepSeek V4 Flash 073179.7\$0.003195 s
    16 DeepSeek V4 Pro 081379.5\$0.19983.4$0.199 808 s
    DeepSeek V4 Flash 073183.3$0.003195 s
    17Gemini 3.7 Flash83.1$0.0149 s
    Gemini 3.6 Flash79.4\$0.05583.1$0.055 26 s
    18
    Gemini 3.5 Flash-Lite78.6\$0.00382.0$0.003 3 s
    19
    Gemini 3.1 Pro Preview78.5\$0.08581.6$0.085 45 s
    GLM-5.381.1$0.169134 s
    20 Claude Opus 4.777.8\$0.29381.0$0.293 53 s
    21GLM-5.377.8\$0.169134 sGemini 3 Flash Preview81.0$0.04858 s
    22 Grok 4.377.5\$0.01580.6$0.015 22 s
    23Gemini 3 Flash Preview77.5\$0.04858 sGrok Build 0.180.4$0.045135 s
    24 Claude Sonnet 4.677.1\$0.18680.3$0.186 114 s
    25 Gemini 3.5 Flash76.8\$0.05680.0$0.056 24 s
    26 DeepSeek V4 Pro76.4\$0.01179.8$0.011 170 s
    27Grok Build 0.176.3\$0.045135 s
    28 Gemini 3.1 Flash Lite Preview76.1\$0.00279.7$0.002 3 s
    29
    Qwen3.7-max73.9\$0.13977.4$0.139 —
    30
    GLM-5.273.5\$0.08277.0$0.082 —
    31
    Claude Opus 4.873.1\$0.25576.2$0.255 55 s
    32
    MiniMax M372.6\$0.00775.7$0.007 —
    33Claude Haiku 4.572.0\$0.05548 s
    34 Qwen3.8-Max71.8\$0.22475.2$0.224 861 s
    35Claude Haiku 4.574.7$0.05548 s
    GPT-5.4 mini70.6\$0.00673.7$0.006 5 s
    36
    Claude Sonnet 569.9\$0.17272.6$0.172 64 s
    37
    Kimi K2.664.7\$0.39167.4$0.391 —
    38
    GPT-5.4 nano62.6\$0.00265.3$0.002 6 s
    - -
    -

    Cost preserves each call’s recorded total, including provider-reported charges where available and request-time reconstructions otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5’s usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

    +

    Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5’s usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

    Simple baselines

    -

    Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 66.9% on the headline exact-match rate — below the 84% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 22.3 points, and the within-1% companion shows the same separation with near-miss tolerance.

    +

    Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 70.0% on the headline exact-match rate — below the 86% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.0 points, and the within-1% companion shows the same separation with near-miss tolerance.

    @@ -1305,16 +1380,16 @@

    Simple baselines

    0 Always zero -66.9 -66.9 -66.9 +70.0 +70.0 +70.0 1 Median reference by output -60.9 -60.9 -64.2 +63.9 +63.9 +67.1 @@ -1364,51 +1439,51 @@

    Sensitivity to benchmark view

    0 Exact-match headline -GPT-5.6 Sol (89.2) -GPT-6 Astra (88.0) -Claude Fable 5.1 (86.9) +GPT-6 Sol (95.0) +Claude Opus 5.5 (93.7) +GPT-5.6 Sol (93.6) 1 Within-1% companion -GPT-5.6 Sol (91.5) -Claude Fable 5.1 (89.9) -GPT-6 Astra (89.8) +GPT-6 Sol (95.9) +Claude Opus 5.5 (94.9) +GPT-5.6 Sol (94.3) 2 Bounded score -GPT-6 Astra (96.3) -Claude Fable 5.1 (95.9) -GPT-5.6 Sol (95.8) +GPT-6 Sol (98.0) +Claude Opus 5.5 (97.6) +GPT-6 Luna (97.4) 3 Equal-output-group score -GPT-6 Astra (97.5) -Claude Fable 5.1 (97.3) -GPT-5.6 Sol (97.2) +Claude Opus 5.5 (98.1) +GPT-6 Sol (98.1) +GPT-6 Astra (98.1) 4 Amount outputs only -Claude Fable 5.1 (96.6) -GPT-6 Astra (96.5) -GPT-5.6 Sol (95.8) +Claude Opus 5.5 (97.6) +GPT-6 Astra (97.5) +GPT-6.1 Sol (97.3) 5 Positive reference cases only -GPT-5.5 (87.8) -GPT-6 Astra (86.7) -Gemini 3.5 Flash (85.5) +GPT-6 Astra (95.0) +GPT-6.1 Sol (92.4) +Claude Opus 5.5 (91.6) 6 Zero reference cases only -Gemini 3.8 Flash (99.1) -Claude Fable 5.1 (99.1) -GPT-6 Astra (98.9) +Claude Opus 5.5 (99.5) +GPT-6 Sol (99.5) +Gemini 3.8 Flash (99.2) @@ -1457,23 +1532,23 @@

    Sensitivity to benchmark view

    0 Household-weighted (default) -GPT-6 Astra (96.3) -Claude Fable 5.1 (95.9) -GPT-5.6 Sol (95.8) +GPT-6 Sol (98.0) +Claude Opus 5.5 (97.6) +GPT-6 Luna (97.4) 1 Equal weights -GPT-6 Astra (97.1) -Claude Fable 5.1 (97.1) -GPT-5.6 Sol (96.9) +Claude Opus 5.5 (98.2) +GPT-6 Sol (98.1) +GPT-6 Astra (97.9) 2 Budget-weighted -GPT-6 Astra (95.4) -Claude Fable 5.1 (95.1) -GPT-5.6 Sol (94.8) +GPT-6 Sol (97.6) +GPT-6 Luna (96.6) +Claude Opus 5.5 (96.5) @@ -1524,42 +1599,42 @@

    Hardest benchmark targets 0 federal_income_tax_before_refundable_credits -78.1 -51.1 -53.7 -64.3 +82.7 +61.6 +63.6 +71.8 15 state_income_tax_before_refundable_credits -79.8 -54.7 -57.4 -68.5 +84.2 +64.3 +66.7 +76.1 16 state_refundable_credits -82.2 -78.7 -78.8 -79.5 +85.5 +81.9 +82.0 +82.8 13 snap -86.7 -79.6 -80.9 -84.4 +89.5 +84.2 +85.5 +88.0 1 federal_refundable_credits -89.6 -83.8 -84.9 -87.4 +91.9 +86.8 +88.0 +90.4 @@ -1609,58 +1684,58 @@

    Zero and positive cases

    0 federal_income_tax_before_refundable_credits -65.3 -37.2 -90.9 +72.7 +43.1 +93.4 1 state_income_tax_before_refundable_credits -69.5 -40.1 -91.2 +76.9 +48.5 +91.8 2 state_refundable_credits -80.7 -12.1 +83.9 +17.2 98.8 3 snap -84.9 -28.3 +88.4 +36.5 96.8 4 federal_refundable_credits -89.0 -47.4 -95.1 +91.7 +58.8 +95.8 5 payroll_tax -90.9 -86.7 -98.2 +92.1 +88.4 +98.5 6 person_medicaid_eligible -91.2 -79.5 -94.9 +92.8 +83.1 +95.6 7 person_chip_eligible -96.2 +96.4 — -96.2 +96.4 @@ -1682,12 +1757,12 @@

    Zero and positive cases

    Open-weight time lag

    -

    The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 76.4% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 86.7% exact trails three closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).

    +

    The leaderboard also measures time. Following the lag construction the UK AI Security Institute applied to cyber capabilities (UK AI Security Institute 2026), the open-weight lag is the time between the closed frontier first reaching a score and the leading open-weight model matching it. Two cuts of the frozen roster give the trajectory. Among models released before July 2026, the leading open-weight model is DeepSeek V4 Pro at 79.8% exact, whose weights shipped on April 24, 2026 (DeepSeek 2026); the earliest-released closed model above that score is Gemini 3 Flash Preview, released December 17, 2025 (TechCrunch 2025) — a lag of 128 days, about 4.2 months. On the full roster, Kimi K3 at 90.4% exact trails 8 closed models; the earliest released of them, GPT-5.6 Sol, generally available July 9, 2026 (TechCrunch 2026), puts the lag at 7 days by API availability, or 18 days dated to Kimi K3’s July 27 weight release on Hugging Face (Tech Times 2026).

    Both figures are lower bounds in one direction: a closed model outside the evaluated roster that crossed either score earlier would lengthen the corresponding lag. They also carry the uncertainty of any single-benchmark, single-domain estimate — the AISI analysis reports substantially longer lags on cyber tasks — and a lag measured at a moving frontier compresses mechanically when a strong open-weight release lands days after a frontier release.

    Failure modes

    The benchmark surfaces a few recurring failure patterns.

    First, models miss positive tax and benefit quantities more often than zero cases. The lowest-scoring outputs by bounded score (Table 11) require the model to choose the right income concepts, exclusions, program thresholds, and sequencing before applying any final subtraction. Positive benefit cases remain difficult, so the result should not be read as a general claim that benefits are easy.

    -

    Second, joint accuracy across interacting components can be lower than marginal accuracy on either component. Table 13 shows within-10% accuracy for federal_refundable_credits, state_refundable_credits, and the conjunction of both within the same household. The joint hit rate can be no higher than either marginal and is strictly lower than both for every model except Claude Fable 5.1, GPT-5.6 Sol, and GPT-6 Astra. Leaderboard scores that average across outputs therefore overstate how often a model gets a single household’s federal/state credit allocation jointly correct.

    +

    Second, joint accuracy across interacting components can be lower than marginal accuracy on either component. Table 13 shows within-10% accuracy for federal_refundable_credits, state_refundable_credits, and the conjunction of both within the same household. The joint hit rate can be no higher than either marginal and is strictly lower than both for every model except Claude Fable 5.1, GPT-6.1 Sol, Claude Opus 5.5, GPT-6 Astra, GPT-6 Sol, Grok 4.7, and GPT-5.6 Sol. Leaderboard scores that average across outputs therefore overstate how often a model gets a single household’s federal/state credit allocation jointly correct.

    @@ -1725,275 +1800,324 @@

    Failure modes

    1 Claude Fable 5.1 -98.0 -90.0 -90.0 +100.0 +93.8 +93.8 -25 -GPT-5.6 Sol +33 +GPT-6.1 Sol 99.0 -86.0 -86.0 +92.8 +92.8 -27 +6 +Claude Opus 5.5 +100.0 +91.8 +91.8 + + +30 GPT-6 Astra -98.0 -86.0 -86.0 +100.0 +88.7 +88.7 + + +31 +GPT-6 Luna +99.0 +89.7 +88.7 -23 -GPT-5.5 -95.0 -85.0 -82.0 +32 +GPT-6 Sol +99.0 +87.6 +87.6 -16 -Gemini 3.6 Flash -95.0 -82.0 -81.0 +37 +Grok 4.7 +99.0 +87.6 +87.6 -18 +28 +GPT-5.6 Sol +99.0 +86.6 +86.6 + + +26 +GPT-5.5 +95.9 +87.6 +85.6 + + +21 Gemini 3.8 Flash -95.0 -84.0 -81.0 +95.9 +86.6 +84.5 -30 +36 Grok 4.6 -96.0 -83.0 -81.0 +96.9 +85.6 +84.5 -10 -DeepSeek V4 Pro 0813 -93.0 -81.0 -79.0 +13 +DeepSeek V4.1 Flash +95.9 +85.6 +83.5 -32 -Inkling -94.0 -81.0 -79.0 +12 +DeepSeek V4 Pro 0813 +94.8 +84.5 +82.5 -11 -Gemini 3 Flash Preview -92.0 -82.0 -78.0 +19 +Gemini 3.6 Flash +95.9 +83.5 +82.5 -17 +20 Gemini 3.7 Flash -90.0 -83.0 -78.0 +91.8 +85.6 +81.4 -26 -GPT-5.6 Terra -95.0 -79.0 -77.0 +39 +Inkling +94.8 +83.5 +81.4 -34 -Kimi K3 -94.0 -79.0 -77.0 +14 +Gemini 3 Flash Preview +91.8 +85.6 +80.4 +41 +Kimi K3 +94.8 +82.5 +80.4 + + 0 Claude Fable 5 -91.0 -82.0 -76.0 +92.8 +84.5 +79.4 - + 9 +Claude Sonnet 5.5 +99.0 +80.4 +79.4 + + +11 DeepSeek V4 Pro -90.0 -80.0 -76.0 +91.8 +83.5 +79.4 -13 -Gemini 3.1 Pro Preview -90.0 -82.0 -76.0 +27 +GPT-5.6 Luna +94.8 +81.4 +79.4 -24 -GPT-5.6 Luna -94.0 -79.0 -76.0 +29 +GPT-5.6 Terra +95.9 +81.4 +79.4 -29 +35 Grok 4.5 -92.0 -78.0 -76.0 +93.8 +81.4 +79.4 -36 -GLM-5.3-Flash (preview) -90.0 -81.0 -75.0 +16 +Gemini 3.1 Pro Preview +91.8 +82.5 +78.4 -14 -Gemini 3.5 Flash -90.0 -79.0 -74.0 +43 +GLM-5.3-Flash (preview) +91.8 +83.5 +78.4 -5 -Claude Opus 5 -87.0 -81.0 -73.0 +17 +Gemini 3.5 Flash +91.8 +81.4 +77.3 -6 -Claude Sonnet 4.6 -93.0 -79.0 -73.0 +5 +Claude Opus 5 +88.7 +83.5 +76.3 -12 -Gemini 3.1 Flash Lite Preview -86.0 -79.0 -73.0 +7 +Claude Sonnet 4.6 +93.8 +81.4 +76.3 -8 +10 DeepSeek V4 Flash 0731 -88.0 -79.0 -72.0 +89.7 +82.5 +75.3 15 -Gemini 3.5 Flash-Lite -86.0 -79.0 -72.0 +Gemini 3.1 Flash Lite Preview +86.6 +81.4 +75.3 -22 -GPT-5.4 nano -86.0 -79.0 -72.0 +18 +Gemini 3.5 Flash-Lite +87.6 +81.4 +75.3 -31 -Grok Build 0.1 -88.0 -78.0 -72.0 +25 +GPT-5.4 nano +87.6 +81.4 +75.3 +38 +Grok Build 0.1 +90.7 +81.4 +75.3 + + 4 Claude Opus 4.8 -84.0 -79.0 -70.0 +85.6 +81.4 +73.2 - -28 + +34 Grok 4.3 -84.0 -79.0 -70.0 +85.6 +81.4 +73.2 - -20 + +23 GLM-5.3 -86.0 -76.0 -69.0 +87.6 +79.4 +72.2 - -21 + +24 GPT-5.4 mini -81.0 -79.0 -69.0 +82.5 +81.4 +72.2 - -7 + +8 Claude Sonnet 5 -80.0 -79.0 -68.0 +81.4 +81.4 +71.1 + + +42 +MiniMax M3 +81.4 +81.4 +70.1 3 Claude Opus 4.7 -85.0 -78.0 -67.0 +86.6 +79.4 +69.1 -35 -MiniMax M3 -80.0 -79.0 -67.0 - - -19 +22 GLM-5.2 -80.0 -72.0 -65.0 +81.4 +74.2 +68.0 - + 2 Claude Haiku 4.5 -75.0 -79.0 -64.0 +76.3 +81.4 +67.0 - -37 + +44 Qwen3.7-max -78.0 -79.0 -63.0 +79.4 +81.4 +66.0 - -38 + +45 Qwen3.8-Max -67.0 -79.0 -56.0 +68.0 +81.4 +58.8 - -33 + +40 Kimi K2.6 -45.0 -48.0 -29.0 +46.4 +50.5 +30.9 @@ -2004,23 +2128,25 @@

    Failure modes

    -

    Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 701 cells (0.9%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.

    +

    Third, structured-output reliability is part of the benchmark contract. Missing or unparseable numeric values are not dropped. Appendix A documents parser recovery and the supervised runner’s bounded retry rounds. The 652 cells (0.7%) that still returned no parsable value after retries remain in the canonical files and score zero as parse-contract failures. Appendix A gives the per-model breakdown.

    Limitations

    PolicyBench is not a substitute for a production tax-and-benefit calculator. Several caveats matter:

    • output-contract reliability required a repair workflow; raw provider responses are retained within the canonical prediction files wherever the transport exposed them (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)), and cells that never parsed score zero
      • -
    • request shape is not identical across models: 10 of 39 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
      • +
    • request shape is not identical across models: 10 of 46 models answered the prompt template over subsets of the requested outputs rather than the whole scenario (Table 2), an accommodation that, if it moves scores at all, should favor those models; later additions must answer the whole-scenario request or are listed as not scorable
    • zero-heavy outputs require separate positive-case interpretation
    • the choice of headline metric (exact match), near-miss companion (within-1%), and secondary tracking metric (bounded score) is one benchmark view; all three are reported, but downstream evaluations should also consider error-magnitude metrics (mean absolute error, mean absolute percentage error) where appropriate
    • the public scenario explorer exposes the current test set and reference outputs, so open-set leakage is a prominent limitation rather than a minor implementation detail
    • the 100-household manuscript snapshot should be treated as a preview until larger frozen runs are published
      • -
    • the annotations cover the 8,783-row legacy-threshold audit universe, including 8,780 exact-match misses and 3 exact hits, but not 1,605 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
      • -
    • one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (SSI disability status and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address; affected cells score models against the encoded inputs
      • +
    • the annotations cover the 7,860-row legacy-threshold audit universe, including 7,856 exact-match misses and 4 exact hits, but not 2,107 additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value
      • +
    • one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (disability, which the prompt states as one general fact where each program applies its own determination, and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address (Section 4.2); outputs whose reference moves under the other reading of either convention are excluded, while the mis-encoded scenario’s cells score models against its encoded inputs
      • +
    • the reference engine carries known defects: 28 outputs, recorded as engine-defect exclusions, rest on 11 root causes that the audit found in policyengine-us 1.755.4 and that 2.15.17 does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, the engine-upgrade review covered only the outputs the upgrade moved, and on 2.15.17 PolicyBench re-ran four of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain
      • +
    • the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP’s FY2026 figures rather than USDA’s FY2027 figures of 2026-08-21; California’s 2025 indexed amounts; the IRS’s 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey’s 2026 child tax credit schedule and Arizona’s SNAP categorical eligibility limit only after 1.755.4
    • benchmark success should not be interpreted as policy-advice readiness

    The current paper is therefore an evaluation of model performance under a specific structured-output benchmark, not a general certification of tax or benefit competence.

    Conclusion

    -

    PolicyBench shows a consistent pattern across the US benchmark. Models often identify non-applicability, but positive tax and benefit amounts remain difficult, especially for multi-step income tax and positive benefit cases. In the frozen manuscript snapshot, GPT-5.6 Sol is the top-scoring US model.

    +

    PolicyBench shows a consistent pattern across the US benchmark. Models often identify non-applicability, but positive tax and benefit amounts remain difficult, especially for multi-step income tax and positive benefit cases. In the frozen manuscript snapshot, GPT-6 Sol is the top-scoring US model.

    These results support a narrow conclusion: unaided frontier models still struggle to reproduce selected household-level microsimulation outputs under a structured public benchmark. They do not show that models cannot assist policy analysis, and they do not validate PolicyEngine outputs as administrative truth. They suggest that future evaluations should separate no-tool estimation, tool-using system design, and reference-output validation more explicitly.

    Next steps are to expand country coverage, increase frozen sample sizes, and add protected or rotating evaluation sets so that public rankings are less exposed to open-set leakage. The benchmark should also continue reporting sensitivity views, because scores are useful summaries only when their weighting choices are visible.

    Appendix A: Structured-output audit

    @@ -2059,7 +2185,7 @@

    Appendix Final parse coverage -701 of the 76,947 canonical model-output rows (0.9%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (422), GLM-5.2 (139), GLM-5.3 (77), and Kimi K3 (63). They score zero as parse-contract failures. A further 61 rows (Grok 4.3 (56), Kimi K2.6 (4), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 762 rows in all fall short of the numeric-plus-explanation contract. Every other row has both. +652 of the 88,688 canonical model-output rows (0.7%) have no parsable numeric value after retries. The failures comprise Kimi K2.6 (390), GLM-5.2 (133), GLM-5.3 (71), and Kimi K3 (58). They score zero as parse-contract failures. A further 60 rows (Grok 4.3 (55), Kimi K2.6 (4), and Claude Haiku 4.5 (1)) carry a parsed numeric value but no explanation; they are scored on the numeric value and counted here as explanation-contract violations, so 712 rows in all fall short of the numeric-plus-explanation contract. Every other row has both. Preservation rule @@ -2072,7 +2198,7 @@

    Appendix

    -

    Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 701 cells (0.9% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (422), GLM-5.2 (139), GLM-5.3 (77), and Kimi K3 (63). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.

    +

    Model waves after June ran under a resumable supervised runner that retries failed or timed-out scenarios in bounded rounds. No separate retry artifacts exist; repairs are folded into each model’s canonical prediction file. After retries, 652 cells (0.7% of canonical rows) still returned no parsable numeric value. They comprise Kimi K2.6 (390), GLM-5.2 (133), GLM-5.3 (71), and Kimi K3 (58). These cells remain in the canonical files, with their raw responses preserved wherever the transport exposed one (no raw payload is retained for Claude Fable 5 (1,984 rows), GLM-5.3 (78 rows), and Kimi K3 (64 rows)). They score zero as parse-contract failures, and the denominator is never reduced.

    Competing interests

    The author leads PolicyEngine, which develops the microsimulation software used to produce the benchmark reference outputs, and is a founder of the Axiom Foundation, a nonprofit developing deterministic rules-as-code infrastructure. Both organizations’ missions are advanced by the finding that unaided language models do not reliably reproduce rule calculations.

    diff --git a/app/src/app/paper/page.tsx b/app/src/app/paper/page.tsx index 9cc6b5a8..7660b6ae 100644 --- a/app/src/app/paper/page.tsx +++ b/app/src/app/paper/page.tsx @@ -3,11 +3,11 @@ import type { Metadata } from "next"; import Link from "next/link"; import SiteHeader from "../../components/SiteHeader"; +import paperSnapshot from "../../paperSnapshot.json"; -const SNAPSHOT_DATE_LABEL = "Snapshot 2026-09-05"; +const SNAPSHOT_DATE_LABEL = `Snapshot ${paperSnapshot.snapshotDate}`; -const PAPER_DESCRIPTION = - "PolicyBench paper: the 2026-09-05 manuscript snapshot reporting the household-impact-weighted exact-match rate, with model responses collected from June 12 through September 1 and PolicyEngine reference outputs on the US populace microdata."; +const PAPER_DESCRIPTION = `PolicyBench paper: the ${paperSnapshot.snapshotDate} manuscript snapshot reporting the household-impact-weighted exact-match rate, with model responses collected between ${paperSnapshot.responseWindow}, against PolicyEngine reference outputs on the US populace microdata.`; export const metadata: Metadata = { title: "Paper", @@ -38,9 +38,11 @@ export const metadata: Metadata = { }, }; +// paperSnapshot.json is written by scripts/freeze_snapshot.py from the paper +// snapshot manifest; the cache keys follow the rendered files' hashes. const manuscriptPaths = { - pdf: "/paper/policybench.pdf", - web: "/paper/web/index.html?v=20260901-refreeze", + pdf: `/paper/policybench.pdf?v=${paperSnapshot.pdfVersion}`, + web: `/paper/web/index.html?v=${paperSnapshot.webVersion}`, }; const ssrnUrl = process.env.NEXT_PUBLIC_POLICYBENCH_SSRN_URL; @@ -49,9 +51,10 @@ export default function PaperPage() { <>

    Benchmarking no-tool tax-and-benefit estimation in frontier language - models. This page embeds the 2026-09-05 scored manuscript snapshot: a - 100-household public preview reporting the household-impact-weighted - exact-match rate against PolicyEngine reference outputs. + models. This page embeds the {paperSnapshot.snapshotDate} scored + manuscript snapshot: a 100-household public preview reporting the + household-impact-weighted exact-match rate against PolicyEngine + reference outputs.

    diff --git a/app/src/components/ExclusionNote.tsx b/app/src/components/ExclusionNote.tsx index 420a9217..c372e97f 100644 --- a/app/src/components/ExclusionNote.tsx +++ b/app/src/components/ExclusionNote.tsx @@ -32,10 +32,14 @@ export default function ExclusionNote({ isBinary: boolean; currencySymbol: "$" | "£"; }) { - const unlistedInput = exclusion?.unlistedInput ?? pred.excludedInput; - const reason = describeExclusionReason( - exclusion?.reasonCode ?? pred.excludedReason, - ); + const reasonCode = exclusion?.reasonCode ?? pred.excludedReason; + const isDefect = reasonCode === "reference_engine_defect"; + const isLaterLaw = reasonCode === "reference_law_published_after_freeze"; + const unlistedInput = + isDefect || isLaterLaw + ? undefined + : (exclusion?.unlistedInput ?? pred.excludedInput); + const reason = describeExclusionReason(reasonCode); return (
    - {exclusion ? ( + {exclusion && isLaterLaw ? ( + <> +

    + {exclusion.published ? `${exclusion.published}. ` : ""} + {exclusion.alternativeReading} Under the published figure the value + is{" "} + + {formatValue(exclusion.alternativeValue, isBinary, currencySymbol)} + + ; the frozen{" "} + + {formatValue(exclusion.frozenValue, isBinary, currencySymbol)} + {" "} + used the engine’s projection. +

    +

    + {exclusion.law ? `Law: ${exclusion.law}. ` : ""} + Both values computed with {exclusion.engineVersion}; excluded by the + developers on {exclusion.decidedOn}. + {exclusion.note ? ` ${exclusion.note}` : ""} +

    + + ) : exclusion && isDefect ? ( + <> +

    + {exclusion.defect ? `${exclusion.defect}. ` : ""} + {exclusion.alternativeReading} Applying that rule gives{" "} + + {formatValue(exclusion.alternativeValue, isBinary, currencySymbol)} + {" "} + rather than the frozen{" "} + + {formatValue(exclusion.frozenValue, isBinary, currencySymbol)} + + . +

    +

    + {exclusion.law ? `Law: ${exclusion.law}. ` : ""} + Corrected value computed with {exclusion.engineVersion} and a + sandbox fix of the rule; excluded by the developers on{" "} + {exclusion.decidedOn}. + {exclusion.upstream ? ` Upstream: ${exclusion.upstream}.` : ""} + {exclusion.note ? ` ${exclusion.note}` : ""} +

    + + ) : exclusion ? ( <>

    {exclusion.alternativeReading} Under that reading the reference is{" "} diff --git a/app/src/components/Methodology.tsx b/app/src/components/Methodology.tsx index 522e040b..f5079ced 100644 --- a/app/src/components/Methodology.tsx +++ b/app/src/components/Methodology.tsx @@ -13,6 +13,12 @@ import { AUDIT_SELECTION_RULE, summarizeAuditUniverse, } from "../lib/auditUniverse"; +import { excludedOutputEngineSentence } from "../lib/referenceEngine"; +import { + capitalizeFirst, + chunkedServingModels, + numberWord, +} from "../lib/servingConfig"; const versionRegistry = parseDataVersionRegistry(versionRegistryJson); @@ -94,6 +100,22 @@ export default function Methodology({ country === "uk" ? "UK transfer households" : "populace households"; const referenceOutputSource = country === "uk" ? "PolicyEngine-UK" : "PolicyEngine-US"; + // The engine release behind this board's references, as its payload records + // it; archived boards name their own. + const referenceEngineVersion = + benchData.policyengineBundles?.[country]?.model_version ?? null; + // The current board's chunked rows, from the frozen serving configuration, + // against the board's own model count. + const chunkedRowsPhrase = `${capitalizeFirst( + numberWord(chunkedServingModels().length), + )} of the ${noToolsModels.length} models`; + const excludedOutputsSentence = + currentBoard && country === "us" + ? excludedOutputEngineSentence( + benchData.referenceExclusions, + referenceEngineVersion, + ) + : null; const benchmarkDescription = currentBoard ? country === "uk" ? "This app shows the current no-tools UK benchmark on a fixed test set, with PolicyEngine reference outputs computed by PolicyEngine-UK for fiscal year 2026-27." @@ -156,13 +178,13 @@ export default function Methodology({ Each model sees the same household facts and requested outputs and must return every scored output plus a short explanation for each, with no external tools — no calculator, search, or PolicyEngine - access. The answer instructions follow the provider's transport: - a forced answer-schema tool call where the provider accepts one, and a - JSON object where the provider rejects a forced tool or the model card - selects JSON for the family (the older Gemini and DeepSeek rows).{" "} + access. The answer instructions follow each row's transport: a + forced answer-schema tool call, or a JSON object where the model card + or its family default selects JSON, for most such rows because the + provider rejects a forced tool call.{" "} {currentBoard ? ( <> - Ten of the 39 models answer the same facts in subsets of one or + {chunkedRowsPhrase} answer the same facts in subsets of one or three outputs per request, an accommodation that predates the whole-scenario rule. The per-model transport and request shape are recorded in the paper's serving-configuration table and the @@ -194,7 +216,7 @@ export default function Methodology({ {country === "uk" ? "PolicyEngine-UK computes the PolicyEngine reference output for every household-variable pair in UK fiscal year 2026-27. The displayed variables define the benchmark scope for this snapshot." - : "PolicyEngine-US computes the PolicyEngine reference output for every household-variable pair in tax year 2026. The displayed variables define the benchmark scope for this snapshot."} + : `PolicyEngine-US${referenceEngineVersion ? ` (policyengine-us ${referenceEngineVersion})` : ""} computes each scored PolicyEngine reference output in tax year 2026.${excludedOutputsSentence ? ` ${excludedOutputsSentence}` : ""} The displayed variables define the benchmark scope for this snapshot.`} @@ -244,9 +266,10 @@ export default function Methodology({ - Each frozen row uses its recorded per-call cost: provider-reported - where the provider returns one, otherwise reconstructed at the - configured list price at request time. List-price overrides apply at + Each frozen row uses its recorded per-call cost: reconstructed from + token counts at the list price configured at request time, or the + provider-reported charge where no reconstruction was available. + List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. diff --git a/app/src/components/ModelLeaderboard.tsx b/app/src/components/ModelLeaderboard.tsx index f990a310..c89a4806 100644 --- a/app/src/components/ModelLeaderboard.tsx +++ b/app/src/components/ModelLeaderboard.tsx @@ -24,12 +24,19 @@ import ServingSensitivityChip from "./ServingSensitivityChip"; import { NEXT_BOARD_HREF, SENSITIVITY_DOC_HREF, + servingSensitivityCounts, servingSensitivityFor, } from "../lib/servingSensitivity"; import { rankWithFallbackScore, rankWithRecomputedScores, } from "../lib/leaderboardRows"; +import { + capitalizeFirst, + joinWithAnd, + jsonContractClaudeModels, + numberWord, +} from "../lib/servingConfig"; import { SENSITIVITY_VIEWS, modelScoresForView, @@ -240,6 +247,14 @@ export default function ModelLeaderboard({ // metric, weighting or program filter the reader has selected. const showSensitivity = selectedView === "us" && isCurrentBoard(versionId, liveVersionId); + // The Claude rows the live board serves as JSON, from the frozen serving + // configuration: each one's API rejects a forced tool call. + const forcedToolRejectingClaudeLabels = joinWithAnd( + jsonContractClaudeModels().map((model) => MODEL_LABELS[model] ?? model), + ); + // The auto re-runs, and the board rows among them that ran without + // extended thinking (src/lib/servingSensitivity.ts). + const sensitivityCounts = servingSensitivityCounts(); const unfilteredExact = (model: string): number => { const row = baseNoTools.find((entry) => entry.model === model); return row?.exact ?? row?.score ?? 0; @@ -306,19 +321,22 @@ export default function ModelLeaderboard({ Serving sensitivity · August 2026

    - Three Claude rows ran without extended thinking: this board forces + {capitalizeFirst(numberWord(sensitivityCounts.thinkingSuppressed))}{" "} + Claude rows ran without extended thinking: this board forces the answer tool call for every row whose model card selects the - tool contract, which switches Claude's thinking off. Claude - Fable 5.1 rejects forced calls, so its row selects JSON, answers as - a JSON object and reasons either way; its marker compares - transports. The{" "} - tool_choice: auto re-runs are marked on the four rows + tool contract, which switches Claude's thinking off.{" "} + {forcedToolRejectingClaudeLabels} reject forced calls, so their + rows select JSON, answer as a JSON object and reason either way; + Fable 5.1's marker compares transports. + The{" "} + tool_choice: auto re-runs are marked on the{" "} + {numberWord(sensitivityCounts.reruns)} rows (open a marker for the score and why); the board itself is unchanged. The{" "} sensitivity note {" "} - has all four runs, and the{" "} + has all {numberWord(sensitivityCounts.reruns)} runs, and the{" "} next board version {" "} diff --git a/app/src/components/NotesContent.tsx b/app/src/components/NotesContent.tsx index fe48f203..0f42b1a0 100644 --- a/app/src/components/NotesContent.tsx +++ b/app/src/components/NotesContent.tsx @@ -3,10 +3,27 @@ import { Fragment, type ReactNode } from "react"; import { notes, type NoteFact, type PolicyBenchNote } from "../notes"; -const PLACEHOLDER = /\{([A-Za-z][A-Za-z0-9]*)\}/g; +// A fact placeholder: `{key}`, or `{key:words}` to spell out a whole number +// under ten in running prose ("four households", not "4 households"). +const PLACEHOLDER = /\{([A-Za-z][A-Za-z0-9]*)(?::(words))?\}/g; + +type FactFormat = "words" | undefined; + +const NUMBER_WORDS = [ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", +]; export const NOTES_INTRO = - "Dated records of board changes and findings. Each note names the data release its numbers come from; a test in the repository checks every number against the frozen snapshot of that release."; + "Dated records of board changes and findings. Each note names the data release its numbers come from, and a test in the repository checked every number against the frozen snapshot of that release."; export function formatNoteDate(date: string): string { return new Intl.DateTimeFormat("en-US", { @@ -21,8 +38,17 @@ export function formatNoteDate(date: string): string { // a whole-number rate (88.0) keeps its decimal like every other row. const ONE_DECIMAL_FACT = /(Exact|Rate)$/; -function factText(key: string, fact: NoteFact): string { +function factText(key: string, fact: NoteFact, format?: FactFormat): string { if (Array.isArray(fact)) return fact.join(", "); + if ( + format === "words" && + typeof fact === "number" && + Number.isInteger(fact) && + fact >= 0 && + fact < NUMBER_WORDS.length + ) { + return NUMBER_WORDS[fact]; + } if (key === "referenceAnnual" && typeof fact === "number") { return String(Math.round(fact)); } @@ -39,11 +65,14 @@ export function interpolateNoteText( note: PolicyBenchNote, paragraph: string, ): string { - return paragraph.replace(PLACEHOLDER, (_, key: string) => { - const fact = note.facts[key]; - if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); - return factText(key, fact); - }); + return paragraph.replace( + PLACEHOLDER, + (_, key: string, format: FactFormat) => { + const fact = note.facts[key]; + if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); + return factText(key, fact, format); + }, + ); } function NoteLink({ href, children }: { href: string; children: ReactNode }) { @@ -66,6 +95,7 @@ function factNode( note: PolicyBenchNote, key: string, fact: NoteFact, + format?: FactFormat, ): ReactNode { if (Array.isArray(fact)) { return fact.map((item, index) => { @@ -94,7 +124,7 @@ function factNode( ); } - return factText(key, fact); + return factText(key, fact, format); } function NoteParagraph({ @@ -110,9 +140,14 @@ function NoteParagraph({ const index = match.index; if (index > cursor) content.push(paragraph.slice(cursor, index)); const key = match[1]; + const format = match[2] as FactFormat; const fact = note.facts[key]; if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); - content.push({factNode(note, key, fact)}); + content.push( + + {factNode(note, key, fact, format)} + , + ); cursor = index + match[0].length; } if (cursor < paragraph.length) content.push(paragraph.slice(cursor)); diff --git a/app/src/components/ScenarioExplorer.tsx b/app/src/components/ScenarioExplorer.tsx index 842f386f..06650559 100644 --- a/app/src/components/ScenarioExplorer.tsx +++ b/app/src/components/ScenarioExplorer.tsx @@ -20,6 +20,7 @@ import { } from "../modelMeta"; import { binaryFlag } from "../lib/scoring"; import { + describeExcludedAuditNote, findReferenceExclusion, isExcludedOutput, predictionStatus, @@ -49,6 +50,8 @@ function formatBoolean(value: 0 | 1 | null): string { const FAILURE_SOURCE_LABELS: Record = { llm_error: "LLM", prompt_ambiguity: "Prompt ambiguity", + reference_engine_defect: "Reference defect", + reference_later_law: "Later law", }; function formatFailureLabel(value?: string): string | null { @@ -1199,6 +1202,16 @@ export function DetailContent({

    Audit tags
    + {excluded && pred.annotation ? ( +

    + {describeExcludedAuditNote( + exclusion?.reasonCode ?? pred.excludedReason, + )} +

    + ) : null} {pred.annotation ? (

    {pred.annotation} diff --git a/app/src/data.artifact.json b/app/src/data.artifact.json index 3997e07b..618a1eea 100644 --- a/app/src/data.artifact.json +++ b/app/src/data.artifact.json @@ -1,9 +1,9 @@ { - "version": 1, - "repo": "PolicyEngine/policybench", - "tag": "dashboard-data-20260905c", "asset": "dashboard-data.json", - "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260905c/dashboard-data.json", - "sha256": "838bb3757db372fc473daf717616c1faea9254ecadfd1581d6217b1c796890a6", - "bytes": 109225250 + "bytes": 126109700, + "repo": "PolicyEngine/policybench", + "sha256": "d1cae7456cf91ab6fa04644a4d5d570e359522386a7c52f9645d5923bfc11258", + "tag": "dashboard-data-20260930", + "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260930/dashboard-data.json", + "version": 1 } diff --git a/app/src/data.versions.json b/app/src/data.versions.json index c62fa6e8..0eedb003 100644 --- a/app/src/data.versions.json +++ b/app/src/data.versions.json @@ -1,30 +1,30 @@ { - "version": 1, "default": "1.1", + "version": 1, "versions": [ { - "id": "1.1", - "label": "1.1", - "description": "Corrected ground truth (policyengine-us 1.755.4); same household facts and requested outputs for every model; answer transport and request shape per model as recorded in the serving-configuration table - 39 models", - "snapshotLabel": "Snapshot 2026-09-05", "artifact": { "pointer": "live" - } + }, + "description": "Scored reference outputs from policyengine-us 2.15.17 (the 56 excluded outputs keep the values they were decided on: 52 from policyengine-us 1.755.4, 4 from 2.15.17, and the 19 that move on 2.15.17 were re-reviewed and stay excluded); same household facts and requested outputs for every model; answer transport and request shape per model as recorded in the serving-configuration table - 46 models", + "id": "1.1", + "label": "1.1", + "snapshotLabel": "Snapshot 2026-09-30" }, { - "id": "1.0", - "label": "1.0", - "description": "June snapshot - 13 models", - "snapshotLabel": "Snapshot 2026-06-25", "artifact": { - "version": 1, + "asset": "dashboard-data.json", + "bytes": 36259601, "repo": "PolicyEngine/policybench", + "sha256": "72d4b524fb6df8b3aa5e98638ec32ff13fb7bc87a4494edade02d0ae3e7fba5c", "tag": "dashboard-data-20260625", - "asset": "dashboard-data.json", "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260625/dashboard-data.json", - "sha256": "72d4b524fb6df8b3aa5e98638ec32ff13fb7bc87a4494edade02d0ae3e7fba5c", - "bytes": 36259601 - } + "version": 1 + }, + "description": "June snapshot - 13 models", + "id": "1.0", + "label": "1.0", + "snapshotLabel": "Snapshot 2026-06-25" } ] } diff --git a/app/src/lib/predictionStatus.ts b/app/src/lib/predictionStatus.ts index ac3d4f1d..b0b79fbe 100644 --- a/app/src/lib/predictionStatus.ts +++ b/app/src/lib/predictionStatus.ts @@ -66,9 +66,37 @@ export function findReferenceExclusion( const EXCLUSION_REASON_LABELS: Record = { reference_depends_on_unlisted_input: "the reference depends on an input the household facts never listed", + reference_engine_defect: + "the engine that produced the reference misapplies the law on the stated facts", + reference_law_published_after_freeze: + "the reference depends on a figure published after the references were frozen", }; export function describeExclusionReason(reasonCode?: string): string { if (!reasonCode) return EXCLUSION_REASON_LABELS.reference_depends_on_unlisted_input; return EXCLUSION_REASON_LABELS[reasonCode] ?? reasonCode.replaceAll("_", " "); } + +// What the frozen reference of an excluded output rests on, as the exclusion +// note above the audit notes describes it. +const EXCLUDED_REFERENCE_BASIS: Record = { + reference_depends_on_unlisted_input: + "which assumes one reading of the unlisted input described above", + reference_engine_defect: "which carries the engine defect described above", + reference_law_published_after_freeze: + "which uses the engine's projection described above", +}; + +/** + * The line shown before the audit note of an excluded output. The audit note + * compares the answer with the frozen reference, which the exclusion sets + * aside, so an answer the note calls wrong can be right under the rule the + * exclusion states. + */ +export function describeExcludedAuditNote(reasonCode?: string): string { + const basis = + EXCLUDED_REFERENCE_BASIS[ + reasonCode ?? "reference_depends_on_unlisted_input" + ] ?? "which the exclusion above sets aside"; + return `This audit note compares the answer with the frozen reference, ${basis}.`; +} diff --git a/app/src/lib/referenceEngine.ts b/app/src/lib/referenceEngine.ts new file mode 100644 index 00000000..13692f7b --- /dev/null +++ b/app/src/lib/referenceEngine.ts @@ -0,0 +1,67 @@ +/** + * What the methodology copy says about the engine behind excluded outputs. + * An excluded output keeps the value its exclusion was decided on, computed + * with the engine version its record names (the payload's + * referenceExclusions[].engineVersion), which can be older than the engine + * behind the scored references. + */ +import type { ReferenceExclusion } from "../types"; + +/** + * The live board's engine upgrade: excluded outputs whose value moves on the + * new engine were re-reviewed and stay excluded. The count is the length of + * the reference sidecar's engine_upgrade.excluded_outputs_rechecked, which the + * payload does not carry; tests/test_disclosures.py checks both fields against + * the frozen sidecar. + */ +export const ENGINE_UPGRADE_RECHECK = { + engineVersion: "2.15.17", + rechecked: 19, +}; + +function compareVersions(a: string, b: string): number { + const left = a.split(".").map(Number); + const right = b.split(".").map(Number); + for (let i = 0; i < Math.max(left.length, right.length); i += 1) { + const diff = (left[i] ?? 0) - (right[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} + +/** Excluded outputs counted by the policyengine-us version of their value, + * oldest version first. */ +export function excludedOutputsByEngine( + exclusions: ReferenceExclusion[], +): Array<[string, number]> { + const counts = new Map(); + for (const exclusion of exclusions) { + const version = exclusion.engineVersion.replace(/^policyengine-us /, ""); + counts.set(version, (counts.get(version) ?? 0) + 1); + } + return [...counts.entries()].sort(([a], [b]) => compareVersions(a, b)); +} + +/** + * "The 56 excluded outputs keep the values they were decided on (52 computed + * with policyengine-us 1.755.4, 4 with 2.15.17), and PolicyBench re-reviewed + * the 19 of them that move on 2.15.17; all 19 stay excluded." + */ +export function excludedOutputEngineSentence( + exclusions: ReferenceExclusion[] | undefined, + referenceEngineVersion: string | null, +): string | null { + if (!exclusions || exclusions.length === 0) return null; + const groups = excludedOutputsByEngine(exclusions).map( + ([version, count], index) => + index === 0 + ? `${count.toLocaleString("en-US")} computed with policyengine-us ${version}` + : `${count.toLocaleString("en-US")} with ${version}`, + ); + let sentence = `The ${exclusions.length.toLocaleString("en-US")} excluded outputs keep the values they were decided on (${groups.join(", ")})`; + if (referenceEngineVersion === ENGINE_UPGRADE_RECHECK.engineVersion) { + const { rechecked, engineVersion } = ENGINE_UPGRADE_RECHECK; + sentence += `, and PolicyBench re-reviewed the ${rechecked} of them that move on ${engineVersion}; all ${rechecked} stay excluded`; + } + return `${sentence}.`; +} diff --git a/app/src/lib/servingConfig.ts b/app/src/lib/servingConfig.ts new file mode 100644 index 00000000..e8381ae8 --- /dev/null +++ b/app/src/lib/servingConfig.ts @@ -0,0 +1,87 @@ +/** + * Counts and rosters the app copy states about the live board's serving + * treatments, read from the frozen serving configuration + * (src/model-serving-config.json, which prepare-data copies from + * paper/snapshot/20260501/model_serving_config.json) rather than typed by hand. + */ +import frozenServingConfig from "../model-serving-config.json"; + +export type ServingTreatment = { + answer_contract: string; + request_shape: string; + tool_choice: string | null; +}; + +export type ServingConfiguration = { + models: Record; +}; + +export const SERVING_CONFIG = + frozenServingConfig as unknown as ServingConfiguration; + +/** Rows that answer subsets of the requested outputs in each request. */ +export function chunkedServingModels( + config: ServingConfiguration = SERVING_CONFIG, +): string[] { + return Object.entries(config.models) + .filter(([, treatment]) => treatment.request_shape !== "whole scenario") + .map(([model]) => model); +} + +/** + * Claude rows on the JSON answer contract. A Claude row defaults to the tool + * contract (policybench/model_cards.py answer_contract_for); each Claude card + * that sets JSON does so because Anthropic's API rejects a forced tool call + * for that model. + */ +export function jsonContractClaudeModels( + config: ServingConfiguration = SERVING_CONFIG, +): string[] { + return Object.entries(config.models) + .filter( + ([model, treatment]) => + model.startsWith("claude-") && treatment.answer_contract === "json", + ) + .map(([model]) => model); +} + +const NUMBER_WORDS = [ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + "twenty", +]; + +/** A whole number as a word up to twenty, as digits beyond. */ +export function numberWord(value: number): string { + return Number.isInteger(value) && value >= 0 && value < NUMBER_WORDS.length + ? NUMBER_WORDS[value] + : value.toLocaleString("en-US"); +} + +export function capitalizeFirst(text: string): string { + return text.charAt(0).toUpperCase() + text.slice(1); +} + +/** "A", "A and B", "A, B and C". */ +export function joinWithAnd(items: string[]): string { + if (items.length <= 1) return items.join(""); + return `${items.slice(0, -1).join(", ")} and ${items[items.length - 1]}`; +} diff --git a/app/src/lib/servingSensitivity.ts b/app/src/lib/servingSensitivity.ts index 0f875106..ec41b205 100644 --- a/app/src/lib/servingSensitivity.ts +++ b/app/src/lib/servingSensitivity.ts @@ -1,10 +1,12 @@ /** * The tool_choice: auto sensitivity runs of the Claude rows - * (sensitivity/claude-thinking-2026-08.md), scored on the same 1,973 outputs + * (sensitivity/claude-thinking-2026-08.md), scored on the same outputs * the board scores. The board's forced answer-tool call switches Claude's * extended thinking off (other reasoning-by-default providers reason * regardless); Claude Fable 5.1 rejects forced tool calls and answers as JSON, - * reasoning in both runs, so its comparison is one of transport. Scores are + * reasoning in both runs, so its comparison is one of transport. Claude Opus + * 5.5 and Claude Sonnet 5.5 reject forced calls the same way and have no auto + * re-run. Scores are * the pinned three-decimal measurements from sensitivity/data/*.json; round * only for display. Each entry's "would rank" is derived from the live board * rows at render time, never typed by hand. @@ -33,7 +35,7 @@ const FORCED_TOOL = export const SERVING_SENSITIVITY: Record = { "claude-fable-5": { - autoExact: 87.542, + autoExact: 91.521, boardTreatment: `${FORCED_TOOL}, one output per request`, autoTreatment: "re-run with tool_choice: auto and the whole household in one request", @@ -41,14 +43,14 @@ export const SERVING_SENSITIVITY: Record = { noteHref: SENSITIVITY_DOC_HREF, }, "claude-opus-5": { - autoExact: 86.201, + autoExact: 90.013, boardTreatment: FORCED_TOOL, autoTreatment: "re-run with tool_choice: auto", thinkingSuppressedOnBoard: true, noteHref: SENSITIVITY_DOC_HREF, }, "claude-sonnet-5": { - autoExact: 80.775, + autoExact: 84.803, boardTreatment: `${FORCED_TOOL}, one output per request`, autoTreatment: "re-run with tool_choice: auto and the whole household in one request", @@ -56,7 +58,7 @@ export const SERVING_SENSITIVITY: Record = { noteHref: SENSITIVITY_DOC_HREF, }, "claude-fable-5.1": { - autoExact: 88.183, + autoExact: 91.728, boardTreatment: "rejects forced tool calls, so its row answers as a JSON object and reasons at the provider default", autoTreatment: @@ -72,6 +74,19 @@ export function servingSensitivityFor( return SERVING_SENSITIVITY[model]; } +/** How many rows carry an auto re-run, and how many of those board rows ran + * without extended thinking; the leaderboard's sensitivity copy states both. */ +export function servingSensitivityCounts( + entries: Record = SERVING_SENSITIVITY, +): { reruns: number; thinkingSuppressed: number } { + const values = Object.values(entries); + return { + reruns: values.length, + thinkingSuppressed: values.filter((entry) => entry.thinkingSuppressedOnBoard) + .length, + }; +} + /** Signed one-decimal delta from unrounded inputs (display rounding only). */ export function formatDelta(autoExact: number, boardExact: number): string { const delta = autoExact - boardExact; diff --git a/app/src/model-serving-config.json b/app/src/model-serving-config.json index 8edd3088..99dcbe8c 100644 --- a/app/src/model-serving-config.json +++ b/app/src/model-serving-config.json @@ -13,7 +13,7 @@ }, "evidence_summary": { "registry": 29, - "run_state": 10 + "run_state": 17 }, "models": { "claude-fable-5": { @@ -171,6 +171,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "claude-opus-5.5": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "a32cbcf7da538881617ce1ed524024c8488e9f962b27939307b60928031065f4", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "77ce7395dafec31dcbf1d8b14941aa51750046a44ee501b0d414f6e89f6d023c", + "source_rows_sha256": "a32cbcf7da538881617ce1ed524024c8488e9f962b27939307b60928031065f4", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "claude-opus-5-5", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 600, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "claude-opus-5-5", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 600, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "claude-sonnet-4.6": { "answer_contract": "tool", "evidence": { @@ -213,6 +270,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "claude-sonnet-5.5": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "1c14b56193ec04e905fa39896b5199b0546e327dc3a18aeb1da51642de422402", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/sonnet55", + "source_predictions_sha256": "b747a404243ddaa3d509ab662f5564b613ae481c4e8baf5b83767583f0d3f602", + "source_rows_sha256": "1c14b56193ec04e905fa39896b5199b0546e327dc3a18aeb1da51642de422402", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "claude-sonnet-5-5", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 600, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "claude-sonnet-5-5", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 600, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "deepseek-v4-flash-0731": { "answer_contract": "tool", "evidence": { @@ -336,6 +450,63 @@ "shared_completion_budget_tokens": 98304, "tool_choice": "forced" }, + "deepseek-v4.1-flash": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "8ab3c0e72c8daa6613223fb0242d7b88e8340daf1e44859bb4cac3ca57503323", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/dsflash41", + "source_predictions_sha256": "0504729cc50c336db24eca623e9c53ffb2f51ffe41202a960e764a21ec8d18d4", + "source_rows_sha256": "8ab3c0e72c8daa6613223fb0242d7b88e8340daf1e44859bb4cac3ca57503323", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "deepseek/deepseek-flash", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "deepseek/deepseek-flash", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "gemini-3-flash-preview": { "answer_contract": "json", "evidence": { @@ -841,6 +1012,177 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "gpt-6-luna": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "66deb97c4aac19e7e85f0128841d87bd7523394fbd9b439ffee45d7df40d5433", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "02537c603f784519510655092cce346eedf55776049fd68720503c1cfaa34eca", + "source_rows_sha256": "66deb97c4aac19e7e85f0128841d87bd7523394fbd9b439ffee45d7df40d5433", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6-luna", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6-luna", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, + "gpt-6-sol": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "33491207380157db2dbf606b4e13177daf9ea6867b167f51d8e8d44f56682353", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "67699eed8dc234e9801ee7a5d537fafd7e133f59425527c97ab9def6fe5de85e", + "source_rows_sha256": "33491207380157db2dbf606b4e13177daf9ea6867b167f51d8e8d44f56682353", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6-sol", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6-sol", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, + "gpt-6.1-sol": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "6d20d23add25271aebf0ee05cce7075817d2ff5286eeea868ea7834238b5c992", + "kind": "run_state", + "rows_match": true, + "run": "adds202609/gpt61sol", + "source_predictions_sha256": "e1e7662d5afa5a886d1b9b39fd0488b9d11941ebf10ddc88d821b24f3861d2aa", + "source_rows_sha256": "6d20d23add25271aebf0ee05cce7075817d2ff5286eeea868ea7834238b5c992", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6.1-sol", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6.1-sol", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, "grok-4.3": { "answer_contract": "tool", "evidence": { @@ -932,6 +1274,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "grok-4.7": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "f225f0d3e8c7aae04ffd220cbbb1e9fe9ce48c8da350fbb16e5679104c27e866", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/grok47", + "source_predictions_sha256": "f8142eadad320820a97b469da2007dc052fcc041090475ee6620ab743cbd2748", + "source_rows_sha256": "f225f0d3e8c7aae04ffd220cbbb1e9fe9ce48c8da350fbb16e5679104c27e866", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "xai/grok-4.7", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 1800, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "xai/grok-4.7", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 1800, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, "grok-build-0.1": { "answer_contract": "tool", "evidence": { @@ -1129,7 +1528,7 @@ "tool_choice": null } }, - "registry_commit": "d1d841e99284757e8274ee5bffe1d26472fe73fe", + "registry_commit": "cddd01812ad6d6311502ef997f93634f70c28fda", "sources": [ "policybench.config.MODELS", "policybench.model_cards", diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 57ef3307..4187cc64 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -6,18 +6,24 @@ export const MODEL_ORDER = [ "claude-fable-5.1", "claude-fable-5", + "claude-opus-5.5", "claude-opus-5", "claude-opus-4.8", "claude-opus-4.7", + "claude-sonnet-5.5", "claude-sonnet-5", "claude-sonnet-4.6", "claude-haiku-4.5", "grok-4.3", "grok-4.5", "grok-4.6", + "grok-4.7", "ox-alpha", "grok-build-0.1", "gpt-6-astra", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", @@ -34,6 +40,7 @@ export const MODEL_ORDER = [ "gemini-3.1-flash-lite-preview", "deepseek-v4-pro-0813", "deepseek-v4-pro", + "deepseek-v4.1-flash", "deepseek-v4-flash-0731", "deepseek-v4-flash", "kimi-k3", @@ -49,18 +56,24 @@ export const MODEL_ORDER = [ export const MODEL_LABELS: Record = { "claude-fable-5.1": "Claude Fable 5.1", "claude-fable-5": "Claude Fable 5", + "claude-opus-5.5": "Claude Opus 5.5", "claude-opus-5": "Claude Opus 5", "claude-opus-4.8": "Claude Opus 4.8", "claude-opus-4.7": "Claude Opus 4.7", "claude-haiku-4.5": "Claude Haiku 4.5", + "claude-sonnet-5.5": "Claude Sonnet 5.5", "claude-sonnet-5": "Claude Sonnet 5", "claude-sonnet-4.6": "Claude Sonnet 4.6", "grok-4.3": "Grok 4.3", "grok-4.5": "Grok 4.5", "grok-4.6": "Grok 4.6", + "grok-4.7": "Grok 4.7", "ox-alpha": "GLM-5.3-Flash (preview)", "grok-build-0.1": "Grok Build 0.1", "gpt-6-astra": "GPT-6 Astra", + "gpt-6-sol": "GPT-6 Sol", + "gpt-6.1-sol": "GPT-6.1 Sol", + "gpt-6-luna": "GPT-6 Luna", "gpt-5.6-sol": "GPT-5.6 Sol", "gpt-5.6-terra": "GPT-5.6 Terra", "gpt-5.6-luna": "GPT-5.6 Luna", @@ -77,6 +90,7 @@ export const MODEL_LABELS: Record = { "gemini-3.1-flash-lite-preview": "Gemini 3.1 Flash Lite Preview", "deepseek-v4-pro-0813": "DeepSeek V4 Pro 0813", "deepseek-v4-pro": "DeepSeek V4 Pro", + "deepseek-v4.1-flash": "DeepSeek V4.1 Flash", "deepseek-v4-flash-0731": "DeepSeek V4 Flash 0731", "deepseek-v4-flash": "DeepSeek V4 Flash", "kimi-k3": "Kimi K3", @@ -107,7 +121,9 @@ export type ProviderKey = export const MODEL_RELEASE_DATES: Record = { "claude-fable-5.1": "2026-09-01", "claude-fable-5": "2026-06-09", + "claude-opus-5.5": "2026-09-21", "claude-opus-5": "2026-07-24", + "claude-sonnet-5.5": "2026-09-28", "claude-sonnet-5": "2026-06-30", "claude-opus-4.8": "2026-05-28", "claude-opus-4.7": "2026-04-16", @@ -125,16 +141,21 @@ export const MODEL_RELEASE_DATES: Record = { "gpt-5.4-nano": "2026-03-17", "gpt-5.5": "2026-04-23", "gpt-6-astra": "2026-09-04", + "gpt-6-sol": "2026-09-22", + "gpt-6.1-sol": "2026-09-29", + "gpt-6-luna": "2026-09-22", "gpt-5.6-sol": "2026-07-09", "gpt-5.6-terra": "2026-07-09", "gpt-5.6-luna": "2026-07-09", "grok-4.3": "2026-04-17", "grok-4.5": "2026-07-08", "grok-4.6": "2026-08-12", + "grok-4.7": "2026-09-21", "ox-alpha": "2026-08-20", "grok-build-0.1": "2026-05-29", "deepseek-v4-pro-0813": "2026-08-13", "deepseek-v4-pro": "2026-04-24", + "deepseek-v4.1-flash": "2026-09-10", "deepseek-v4-flash-0731": "2026-07-31", "deepseek-v4-flash": "2026-04-24", "kimi-k2.6": "2026-04-20", diff --git a/app/src/notes/2026-09-03-six-snap-households.json b/app/src/notes/2026-09-03-six-snap-households.json index 0e4d5996..3bfecf84 100644 --- a/app/src/notes/2026-09-03-six-snap-households.json +++ b/app/src/notes/2026-09-03-six-snap-households.json @@ -7,7 +7,8 @@ "Five of the six qualify through broad-based categorical eligibility: Connecticut, Michigan, Texas, and Wisconsin confer SNAP eligibility on households receiving a TANF-funded non-cash benefit, with gross-income limits of 200% of poverty (CT, MI, WI) and 165% (TX) and the net-income and asset tests waived. The sixth, scenario_112 in Texas, passes the federal tests; the models counted farm-rent income the rules exclude.", "{categoricalOnlyCount} of the {eligibleCount} eligible households qualify only through categorical eligibility: {categoricalIncomeCount} because income exceeds the federal limits (one of those also exceeds the asset limit), {categoricalAssetCount} because savings alone exceed the federal asset limit. On the {categoricalAssetCount} asset cases the models compute benefits near the reference: Sol within 1% on three and 10% on one, Fable 5.1 within 1% on all four.", "Across the 100 households, Sol's SNAP explanations mention categorical eligibility in {solCategoricalMentions} and assets in {solAssetMentions}; Fable 5.1's mention broad-based categorical eligibility in {fableBbceMentions}; Kimi K3's mention categorical eligibility in {kimiCategoricalMentions}.", - "Pathways were recomputed with policyengine-us {pathwayEngineVersion} from the frozen scenarios and agree with the frozen reference SNAP values for all 100 households; the reference itself was generated with policyengine-us {referenceEngineVersion}." + "Pathways were recomputed with policyengine-us {pathwayEngineVersion} from the frozen scenarios and agree with the frozen reference SNAP values for all 100 households; the reference itself was generated with policyengine-us {referenceEngineVersion}.", + "A later note, published October 5, corrects this one. From release dashboard-data-20260922c on, PolicyBench scores the SNAP amounts of {laterScoredCount:words} of these {deniedCount:words} households at ${laterReference} each, and scores a Michigan worker who pays child support at $0: PolicyEngine counts that child support in gross income, as Michigan does (policyengine-us #9586), and the worker does not qualify. PolicyBench no longer scores a Texas household's SNAP amount, which PolicyEngine computed with hours of work the prompt does not list." ], "facts": { "eligibleCount": 20, @@ -30,7 +31,9 @@ "fableBbceMentions": 28, "kimiCategoricalMentions": 17, "pathwayEngineVersion": "1.723.0", - "referenceEngineVersion": "1.755.4" + "referenceEngineVersion": "1.755.4", + "laterScoredCount": 4, + "laterReference": 288 }, "mentionRegexes": { "categorical": "categorical", @@ -93,6 +96,14 @@ { "label": "SNAP minimum-allotment parameters", "href": "https://github.com/PolicyEngine/policyengine-us/tree/main/policyengine_us/parameters/gov/usda/snap/min_allotment" + }, + { + "label": "Later release dashboard-data-20260922c", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922c" + }, + { + "label": "Later note on these households (October 5)", + "href": "/notes/2026-10-05-five-snap-households-bbce" } ], "boardSnapshot": "2026-09-01", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json new file mode 100644 index 00000000..ca8fb480 --- /dev/null +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -0,0 +1,70 @@ +{ + "slug": "2026-09-22-gpt-6-sol-debuts-first", + "date": "2026-09-22", + "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", + "paragraphs": [ + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol leads at {solExact}% of answers within $1, weighted by household impact, #{solRank} of {nModels} and {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", + "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", + "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", + "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", + "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139).", + "A later release, dashboard-data-20260922c, corrects the reference for one output: the SNAP amount of a Michigan worker who pays child support. PolicyEngine had subtracted that child support from the worker's gross income; with its fix (policyengine-us #9586) the worker does not qualify, and the reference is $0 instead of $288. The later release scores the same outputs, regenerates the same references and excludes the same outputs as this one. On it, GPT-6 Sol scores {laterSolExact}% and Claude Opus 5.5 {laterOpusExact}%, and Claude Opus 5's tool_choice auto re-run scores {laterOpus5AutoExact}%. The note's other scores, gaps, ranks and costs stay the same. An interim release, dashboard-data-20260922b, excluded that output while the fix was open." + ], + "facts": { + "solExact": 94.2, + "solRank": 1, + "nModels": 42, + "solLead": 1.3, + "opusExact": 92.9, + "opusRank": 2, + "sol56Rank": 3, + "sol56Exact": 92.7, + "lunaExact": 91.4, + "lunaRank": 4, + "lunaCost": 0.0019, + "solCost": 0.0271, + "opusCost": 0.0675, + "opus5BoardExact": 83.4, + "opus5AutoExact": 89.3, + "opusGap": 9.6, + "scoredOutputs": 1932, + "totalOutputs": 1984, + "regenerated": 26, + "excluded": 52, + "laterSolExact": 94.3, + "laterOpusExact": 93.0, + "laterOpus5AutoExact": 89.4 + }, + "data": [ + { + "label": "Dashboard data release", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922" + }, + { + "label": "Later release dashboard-data-20260922c", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922c" + }, + { + "label": "Reference audit note", + "href": "/notes/2026-09-22-reference-audit" + }, + { + "label": "Claude thinking sensitivity note", + "href": "https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md" + }, + { + "label": "Paper serving-configuration table", + "href": "/paper" + }, + { + "label": "GPT-6 Sol model page", + "href": "/model/gpt-6-sol" + }, + { + "label": "Claude Opus 5.5 model page", + "href": "/model/claude-opus-5.5" + } + ], + "boardSnapshot": "2026-09-22", + "release": "dashboard-data-20260922" +} diff --git a/app/src/notes/2026-09-22-reference-audit.json b/app/src/notes/2026-09-22-reference-audit.json new file mode 100644 index 00000000..8d03040f --- /dev/null +++ b/app/src/notes/2026-09-22-reference-audit.json @@ -0,0 +1,65 @@ +{ + "slug": "2026-09-22-reference-audit", + "date": "2026-09-22", + "title": "A stronger judge audits the references", + "paragraphs": [ + "Claude Opus 5.5 judged the misses in the cases the September 22 additions joined, and it flagged {flagged} references as suspect. Developers settled each flag against statutes, regulations, agency publications and runs of the reference engine. The adjudication affirmed {affirmed}, replaced {regeneratedFlagged} with a regenerated reference, traced {unlistedFlagged} to an input the prompt never states, and confirmed {defectFlagged} as defects in policyengine-us {engineVersion} that are not yet fixed upstream.", + "Flags sample defects without bounding them. For each defect we implemented the rule as a fix on the same engine version and recomputed all {totalOutputs} references under it. {upstreamFixed} of the root causes had been fixed in policyengine-us after the references were frozen, so the {regeneratedByFix} scored references they move are regenerated with the fix. One of those fixes as first built went further than the upstream fix, applying Wisconsin's capital gain exclusion to capital gain distributions; upstream still leaves them out, so that part counts as a defect not yet fixed. Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model: {defectOutputs} outputs in {defectHouseholds} households are recorded as engine-defect exclusions across {defectRootCauses} root causes, and {defectUnflagged} of them had never been flagged. An output such a defect moves that was already excluded for an unstated input keeps that record. Also excluded are {unlistedOutputs} outputs whose reference turns on an input the prompt never states, such as who paid for the coverage behind a disability benefit.", + "The audit also tested a failure no judge flagged. policyengine-us extends a parameter past its last encoded year with a price-index projection, so a 2026 reference can rest on a forecast where a government published an amount. Recomputing every reference with each projection held found forecasts behind references in SNAP, California, the IRS sales tax tables, Wisconsin, Minnesota, Michigan, Missouri, Maryland and Idaho. The rule the benchmark now states: a scored reference follows from the household facts and from law published before the references were frozen on 2026-07-03. The scored references among them were regenerated with the published amounts, or with the last ones published before the freeze; Wisconsin's published amounts reach only the corrected values of Wisconsin outputs excluded for engine defects.", + "Separating the SNAP convention's parameter holds from its arithmetic exposed engine defects in SNAP itself: the engine kept cents in the monthly allotment where federal rules require whole dollars, set the minimum benefit at $23.84 a month where the rule rounds it to $24, and rounded net income down. policyengine-us fixed each after the freeze, so the SNAP references apply those fixes. One SNAP defect is not fixed upstream: the engine grants the heat-and-eat utility allowance to households without an elderly or disabled member, which a July 2025 law ended, and the scored SNAP output it moves is excluded.", + "In all, {regenerated} references are regenerated, {regeneratedSnap} of them SNAP, and every model is scored on {scoredOutputs} of its {totalOutputs} requested outputs. The exclusion record, the adjudications and the reference sidecar, which lists each regenerated value with its source, ship with the release.", + "A later release, dashboard-data-20260922c, corrects the reference for one output: the SNAP amount of a Michigan worker who pays child support. policyengine-us subtracted that child support from gross income, while Michigan counts it in gross income and deducts it only when computing net income. PolicyEngine fixed this after the audit (policyengine-us #9586), so the later release regenerates the output with the fix: counted in gross income, the worker's income exceeds Michigan's broad-based categorical eligibility limit, and the reference is $0 instead of $288. That makes {laterUpstreamFixed} root causes fixed upstream; every other count in this note stays the same. An interim release, dashboard-data-20260922b, excluded that output while the fix was open." + ], + "facts": { + "flagged": 39, + "affirmed": 5, + "regeneratedFlagged": 6, + "unlistedFlagged": 8, + "defectFlagged": 20, + "engineVersion": "1.755.4", + "totalOutputs": 1984, + "defectOutputs": 28, + "defectHouseholds": 20, + "defectRootCauses": 11, + "defectUnflagged": 8, + "unlistedOutputs": 24, + "regenerated": 26, + "regeneratedSnap": 13, + "upstreamFixed": 7, + "regeneratedByFix": 15, + "scoredOutputs": 1932, + "laterUpstreamFixed": 8 + }, + "data": [ + { + "label": "Dashboard data release", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922" + }, + { + "label": "Later release dashboard-data-20260922c", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922c" + }, + { + "label": "Exclusion record", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json" + }, + { + "label": "Developer adjudications", + "href": "https://github.com/PolicyEngine/policybench/blob/main/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json" + }, + { + "label": "Reference sidecar with the regenerated values", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json" + }, + { + "label": "Paper audit section", + "href": "/paper" + }, + { + "label": "Later note on the SNAP households that qualify through BBCE (October 5)", + "href": "/notes/2026-10-05-five-snap-households-bbce" + } + ], + "boardSnapshot": "2026-09-22", + "release": "dashboard-data-20260922" +} diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json new file mode 100644 index 00000000..58c61e09 --- /dev/null +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -0,0 +1,124 @@ +{ + "slug": "2026-09-29-claude-sonnet-5-5-debuts-fifth", + "date": "2026-09-29", + "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", + "paragraphs": [ + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 scores {sonnetExact}% of answers within $1, weighted by household impact, #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with it, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", + "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", + "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", + "The move changes {scoredChanges:words} scored references. New Jersey's child tax credit schedule for 2026 to 2028, which the state approved on June 30, raises one household's state refundable credits from ${njBefore} to ${njAfter}. Arizona raised the income limit for its broad-based categorical eligibility from {azLimitBefore}% to {azLimitAfter}% of the poverty guideline effective March, and the new limit gives an Arizona household ${azAfter} of SNAP where the reference was ${azBefore}. policyengine-us now counts child support received as school-meal income, which ends a Pennsylvania household's eligibility for reduced-price meals. A fix to the rounding in New York's Empire State child credit phase-out (policyengine-us #9425) raises a New York household's state refundable credits from ${nyBefore} to ${nyAfter}.", + "PolicyBench stops scoring {newExclusions:words} federal income tax outputs. policyengine-us {engineVersion} counts the whole of a listed state and local tax refund as income. Federal law counts the refund only to the extent the refunded tax lowered the household's federal tax in the year the household paid it, and the prompts do not say whether it did. PolicyBench also stops scoring one California household head's Medicaid eligibility, which its re-review of the household's excluded SNAP output flagged. On disability, the prompt says only that the head is disabled. The head's income, {medicaidIncomePercent}% of the poverty guideline, is above the {expansionLimitPercent}% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, California's Medicaid program. Medi-Cal's Working Disabled Program requires SSI's definition of disability, which the prompt does not state, and policyengine-us {engineVersion} tests the general disability flag instead. The same unstated fact already keeps the household's SNAP out of scoring. PolicyBench now scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes {excluded}. Another {withinTolerance:words} references move by less than $1, and PolicyBench re-reviewed the {rechecked} excluded outputs whose values moved; all stay excluded.", + "Together, the new references and the Medicaid exclusion raise the exact rate of every one of the {incumbents} earlier models, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", + "The Arizona household's income keeps it from qualifying for SNAP under the program's ordinary tests, so it qualifies only through broad-based categorical eligibility, and all {nModels} models answer $0 for it. PolicyBench's October 5 note on such households counts it among five held back by income, with {bbceIncomeHeldCount:words} in {bbceIncomeHeldStates}. That note's data list every model's answer for those five households and for the {bbceAssetHeldCount:words} held back by savings, including the three models this release adds." + ], + "facts": { + "nModels": 45, + "sonnetExact": 92.1, + "sonnetRank": 5, + "lunaExact": 92.1, + "lunaRank": 4, + "lunaSonnetGapBelow": 0.02, + "grokExact": 88.3, + "grokRank": 11, + "flashExact": 87.1, + "flashRank": 15, + "solExact": 95.0, + "opusExact": 93.7, + "sol56Exact": 93.6, + "sonnetCost": 0.0343, + "lunaCost": 0.0019, + "flashCost": 0.0242, + "grokCost": 0.2597, + "grokTimeoutSeconds": 1800, + "flashAnswers": 1984, + "previousEngine": "1.755.4", + "engineVersion": "2.15.17", + "upstreamFixed": 8, + "conventions": 9, + "scoredChanges": 4, + "njBefore": "5,342.40", + "njAfter": "5,842.40", + "azBefore": 0, + "azAfter": 240, + "nyBefore": "650.50", + "nyAfter": 667, + "newExclusions": 3, + "scoredOutputs": 1928, + "totalOutputs": 1984, + "excluded": 56, + "medicaidIncomePercent": 141, + "expansionLimitPercent": 138, + "withinTolerance": 2, + "rechecked": 19, + "incumbents": 42, + "driftMin": 0.14, + "driftMax": 0.83, + "medicaidMatched": 20, + "medicaidMissed": 22, + "riserCount": 5, + "riserOne": "GLM-5.3-Flash (preview)", + "riserOnePassed": "Inkling", + "riserTwo": "Claude Opus 5", + "riserTwoPassed": "Claude Fable 5", + "riserThree": "DeepSeek V4 Pro 0813", + "riserThreePassed": "DeepSeek V4 Flash 0731 and Gemini 3.7 Flash", + "riserFour": "Grok Build 0.1", + "riserFourPassed": "Claude Sonnet 4.6 and Gemini 3.5 Flash", + "riserFive": "DeepSeek V4 Pro", + "riserFivePassed": "Gemini 3.1 Flash Lite Preview", + "grokOnboardingTimeoutSeconds": 600, + "azLimitBefore": 185, + "azLimitAfter": 200, + "engineUploadedUtc": "00:23", + "checkEngine": "2.17.0", + "checkPypiReadUtc": "14:58", + "bbceIncomeHeldStates": "Connecticut, Texas, Michigan and Wisconsin", + "bbceIncomeHeldCount": 4, + "bbceAssetHeldCount": 4 + }, + "data": [ + { + "label": "Dashboard data release", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260929" + }, + { + "label": "Reference upgrade record", + "href": "https://github.com/PolicyEngine/policybench/blob/main/reference_audit/2026-09-28/README.md" + }, + { + "label": "Reference sidecar with every change", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json" + }, + { + "label": "Exclusion record", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json" + }, + { + "label": "Paper serving-configuration table", + "href": "/paper" + }, + { + "label": "DeepSeek V4.1 Flash release note (September 10)", + "href": "https://api-docs.deepseek.com/news/news260910" + }, + { + "label": "Claude Sonnet 5.5 model page", + "href": "/model/claude-sonnet-5.5" + }, + { + "label": "Grok 4.7 model page", + "href": "/model/grok-4.7" + }, + { + "label": "DeepSeek V4.1 Flash model page", + "href": "/model/deepseek-v4.1-flash" + }, + { + "label": "Note on the SNAP households that qualify through BBCE (October 5)", + "href": "/notes/2026-10-05-five-snap-households-bbce" + } + ], + "boardSnapshot": "2026-09-29", + "release": "dashboard-data-20260929" +} diff --git a/app/src/notes/2026-10-05-five-snap-households-bbce.json b/app/src/notes/2026-10-05-five-snap-households-bbce.json new file mode 100644 index 00000000..f8e5d0cd --- /dev/null +++ b/app/src/notes/2026-10-05-five-snap-households-bbce.json @@ -0,0 +1,159 @@ +{ + "slug": "2026-10-05-five-snap-households-bbce", + "date": "2026-10-05", + "title": "Most models answer $0 for households that qualify for SNAP under their states' higher income limits", + "paragraphs": [ + "For each of {householdCount:words} households that qualify for SNAP food benefits, most of the {nModels} models on PolicyBench answer $0. SNAP pays every eligible household of one or two people at least a minimum benefit, which in these households' states is ${minimumMonthly} a month through September 2026 and ${minimumFromOctober} from October. Each of the {householdCount:words} has one or two people, so an answer of $0 says the household does not qualify, and that answer could keep someone who qualifies from applying. Of the {answers} answers PolicyBench requests for these households, {zeroAnswers} come to $0 and {hits:words} match the reference.", + "PolicyBench grades answers against references from PolicyEngine, the open-source tax and benefit model, and the references follow law published before PolicyBench's July 3 reference freeze. USDA published the ${minimumFromOctober} minimum on August 21, so the references keep ${minimumMonthly} through December and put each household at that minimum for each month it qualifies: ${referenceAmount} for 2026 for {fullYearCount:words} households, and ${azReference} for an Arizona resident who qualifies from March.", + "Each household qualifies only through broad-based categorical eligibility (BBCE). Ordinarily, SNAP tests a household's gross income against {snapGrossLimit}% of the federal poverty guideline, its net income after deductions for costs such as housing, and its savings. Under BBCE, a state extends SNAP to households that receive a non-cash benefit funded by Temporary Assistance for Needy Families (TANF), or that the state has authorized to receive one. Arizona, Connecticut, Michigan, and Wisconsin open that benefit, and with it SNAP, to households with gross income up to {bbceGrossLimitHigh}% of the guideline, and Texas opens it to those up to {bbceGrossLimitTx}%. Arizona raised its limit from {azLimitBefore}% to {bbceGrossLimitHigh}% starting with benefit month March 2026. None of these states tests net income for the non-cash benefit, and only Texas keeps an asset limit for it.", + "All {householdCount:words} households have gross income above {snapGrossLimit}% of the guideline, but SNAP exempts households with an elderly or disabled member from its ordinary gross income test. Four of the {householdCount:words} have such a member: a Connecticut couple on Social Security disability income, a disabled Michigan resident living alone, an {wiAge}-year-old surviving spouse in Wisconsin, and an {azAge}-year-old Arizona resident on Social Security and a pension. The fifth, a {txAge}-year-old Texas resident with wages and ${financialAssistance} of financial assistance, fails that test. The four exempt households fail SNAP's ordinary net income test instead, and the Connecticut couple and the Arizona resident also hold ${ctSavings} and ${azSavings} in savings, above SNAP's asset limit. Each household's gross income is under its state's BBCE limit, the Arizona resident's from March. BBCE leaves SNAP's benefit formula in place, and at these incomes the formula pays nothing, so each household gets the minimum.", + "The Michigan resident's prompt lists employer-sponsored insurance premiums, and PolicyEngine treats the employer as paying them. Had the resident paid them, SNAP would count them as a medical expense, and the household would pass the net income test and get the same minimum without BBCE. GPT-5.5 gets the Michigan resident's amount right by that route: its explanation deducts medical premiums, puts net income below the net income limit, and applies the minimum.", + "No prompt mentions the non-cash benefit, and each mentions TANF only to ask for the household's TANF amount. Every prompt tells the model to treat any unlisted household fact or status \"as false\", to assume \"program take-up when required\", and not to infer unlisted \"benefit receipt\". The rules on unlisted statuses and benefit receipt tell a model the household lacks the non-cash benefit, while the take-up rule can tell it the household takes that benefit up. PolicyEngine applies BBCE to any household eligible for the non-cash benefit, without checking whether the household receives it or holds the state's authorization to receive it. A model that requires receipt or authorization falls back on SNAP's ordinary tests, and each of the {householdCount:words} fails at least one. Among the explanations for the {zeroAnswers} answers of $0, TANF appears in {zeroTanfMentions:words}: Claude Sonnet 4.6 writes that Texas has no BBCE and that the Texas resident receives neither TANF nor Supplemental Security Income.", + "When savings rather than income hold a household back, most models answer above $0. PolicyBench also asks models about households in New Jersey, North Carolina, Virginia, and Pennsylvania that pass SNAP's income tests but hold savings above its asset limit, which BBCE removes in those states. Of the answers models give for those {assetOnlyCount:words} households, {assetOnlyAbove0Share}% come in above $0, against {incomeAbove0Share}% for the {householdCount:words} held back by income. Of the {nModels} models, {splitModels} answer above $0 for each household held back by savings and $0 for each of the {householdCount:words} held back by income, among them GPT-5.6 Sol and GPT-6 Luna, #{sol56Rank} and #{lunaRank} on PolicyBench. PolicyBench does not score the Pennsylvania household's SNAP amount, because PolicyEngine grants it the heat-and-eat utility allowance, which a 2025 law narrowed. With or without that allowance, the household qualifies for a benefit.", + "PolicyBench asks each model to explain every answer, and the explanations mention BBCE less often for the households held back by income. For the households held back by savings, {assetOnlyBbceMentions} of {assetOnlyExplanations} explanations mention BBCE, by name or as categorical eligibility, and {assetOnlyBbceAssets} of those also mention assets. For the {householdCount:words} held back by income, {incomeBbceMentions} of {explanations} explanations mention BBCE, and {incomeBbceZeros} of those still end at $0. Of those {incomeBbceZeros}, {incomeBbceZeroOverLimit:words} write that the household's income exceeds its state's BBCE limit: {overLimitAz:words} for the Arizona resident and {overLimitTx:words} for the Texas resident. PolicyEngine puts both households under their states' limits, the Arizona resident from March. Another {netLimitOnly:words} cite a net income limit or test, though none of these households' states applies one under BBCE. The last {formulaOnly:words}, from Gemini 3 Flash Preview and Gemini 3.1 Pro Preview, write that the Connecticut couple meets BBCE, then answer $0 because the benefit formula pays nothing, and neither applies the minimum. Of the {incomeBbceAbove0} that mention BBCE and answer above $0, {incomeBbceHits:words} match the reference, and {staleMinimum:words} apply the minimum at its amount through September 2025, ${fy2025Minimum} a month, and answer ${fy2025MinimumAnnual}.", + "GPT-6 Astra gets {astraHits:words} of the {householdCount:words} households right, more than any other model, and cites the state's BBCE limit of {bbceGrossLimitHigh}% of the guideline each time. It answers $0 for the Texas resident, the one household that fails SNAP's ordinary gross income test, and for the Arizona resident. GPT-6.1 Sol gets {sol61Hits:words} right, the Michigan and Wisconsin households, citing each state's categorical eligibility limit. Claude Opus 5.5 answers ${opusOn108} for the Wisconsin surviving spouse, citing Wisconsin's BBCE and its {bbceGrossLimitHigh}% limit. For the Connecticut couple, the disabled Michigan resident, and the Arizona resident, it answers $0, citing SNAP's ordinary net income test, which BBCE removes in all three states. GPT-6 Sol, which leads PolicyBench overall, answers $0 for {sol6ZeroCount:words} of the {householdCount:words} and mentions BBCE in {sol6BbceMentions:words} of its SNAP explanations for PolicyBench's {boardHouseholds} households.", + "GPT-6 Sol, Claude Opus 5.5, and Claude Sonnet 5.5 miss the Texas resident's amount because they count less income than PolicyEngine does. GPT-6 Sol answers ${sol6On030}, and Claude Opus 5.5 and Claude Sonnet 5.5 each answer ${opusOn030}. All three compute an ordinary benefit from wages alone. PolicyEngine also counts the ${financialAssistance} of financial assistance as income, which puts the household above SNAP's ordinary limits and leaves BBCE as its only route. The prompt labels that money only \"financial assistance\", without naming its source, and SNAP counts some kinds of assistance as income but excludes others, such as educational assistance.", + "All {nModels} models answer $0 for the Arizona resident, whose gross income falls between Arizona's old and new BBCE limits. PolicyBench's references use policyengine-us {engineVersion}, which encodes the raise. GPT-6 Astra, GPT-6.1 Sol, Gemini 3.5 Flash, and Gemini 3.8 Flash write that the resident's income exceeds Arizona's categorical eligibility limit; Gemini 3.5 Flash and Gemini 3.8 Flash, along with Gemini 3 Flash Preview, give Arizona's limit as {azLimitBefore}%, the limit before March.", + "This note corrects PolicyBench's September 3 note. That note counted {sept3HouseholdCount:words} households, {fullYearCount:words} of them among the {householdCount:words} here, at ${sept3Reference} each, an amount PolicyBench computed from PolicyEngine's unrounded minimum. SNAP rounds the minimum to the nearest dollar, and so does PolicyEngine (policyengine-us #9162). One of the {sept3HouseholdCount:words}, a Michigan worker who pays child support, does not qualify. Michigan counts that child support in gross income and deducts it when computing net income, which puts the worker's gross income above Michigan's {bbceGrossLimitHigh}% BBCE limit; PolicyEngine subtracted it from gross income until policyengine-us #9586. The version of this note published September 23 counted the worker among its households. PolicyBench no longer scores another of the {sept3HouseholdCount:words}, a second Texas household, whose reference assumed {assumedHours} hours of work a week that the prompt does not list. At 0 hours, as the prompt's rule for unlisted numbers reads, PolicyEngine gives that household $0, the answer GPT-5.6 Sol, Claude Fable 5.1, and Kimi K3 gave in the September 3 note. The September 3 note also says SNAP excludes that household's farm rent, but SNAP counts rent, farm rent included, net of the costs of doing business: as unearned income under 7 CFR 273.9(b)(2)(ii), or as earned income under 273.9(b)(1)(ii) when a household member averages at least 20 hours a week managing the property. PolicyEngine counts farm rent from policyengine-us #9671, which postdates the references' engine, {engineVersion}. The September 3 note described BBCE as covering households that receive the non-cash benefit, but PolicyEngine computed that note's amounts by applying BBCE to every household eligible for that benefit. It also said Connecticut, Michigan, Texas, and Wisconsin waive the asset test, but Texas keeps a ${bbceAssetLimitTx} asset limit." + ], + "facts": { + "householdCount": 5, + "nModels": 46, + "minimumMonthly": 24, + "minimumFromOctober": 25, + "referenceAmount": 288, + "fullYearCount": 4, + "azReference": 240, + "answers": 230, + "zeroAnswers": 190, + "hits": 7, + "snapGrossLimit": 130, + "bbceGrossLimitHigh": 200, + "bbceGrossLimitTx": 165, + "azLimitBefore": 185, + "wiAge": 85, + "azAge": 80, + "txAge": 23, + "financialAssistance": 12000, + "ctSavings": 48000, + "azSavings": 58700, + "zeroTanfMentions": 1, + "assetOnlyCount": 4, + "assetOnlyAbove0Share": 77, + "incomeAbove0Share": 17, + "splitModels": 12, + "sol56Rank": 3, + "lunaRank": 4, + "assetOnlyBbceMentions": 62, + "assetOnlyExplanations": 182, + "assetOnlyBbceAssets": 54, + "incomeBbceMentions": 29, + "explanations": 228, + "incomeBbceZeros": 13, + "incomeBbceZeroOverLimit": 6, + "overLimitAz": 4, + "overLimitTx": 2, + "netLimitOnly": 5, + "formulaOnly": 2, + "incomeBbceAbove0": 16, + "incomeBbceHits": 6, + "staleMinimum": 7, + "fy2025Minimum": 23, + "fy2025MinimumAnnual": 276, + "astraHits": 3, + "sol61Hits": 2, + "opusOn108": 288, + "sol6ZeroCount": 4, + "sol6BbceMentions": 1, + "boardHouseholds": 100, + "sol6On030": "1,208.40", + "opusOn030": 1208, + "engineVersion": "2.15.17", + "sept3HouseholdCount": 6, + "sept3Reference": "287.68", + "assumedHours": 40, + "bbceAssetLimitTx": 5000 + }, + "mentionRegexes": { + "bbce": "broad-based|\\bbbce\\b|categorical(?:ly)?[- ]eligib|expanded categorical", + "tanf": "tanf", + "assets": "asset|resource", + "netLimit": "\\bnet[- ](?:income[- ])?(?:limit|test|screen|threshold|ceiling|rule)|(? { "rows whose legacy threshold score is below 1", ); expect(summarizeAuditUniverse(us)).toEqual({ - annotatedRowCount: 8_783, - legacyThresholdRowCount: 8_783, - exactMissCount: 8_780, - annotatedExactMissCount: 8_780, - annotatedExactHitCount: 3, - unannotatedBelowFullBoundedScoreCount: 1_605, + annotatedRowCount: 7_860, + legacyThresholdRowCount: 7_860, + exactMissCount: 7_856, + annotatedExactMissCount: 7_856, + annotatedExactHitCount: 4, + unannotatedBelowFullBoundedScoreCount: 2_107, }); }); }); diff --git a/app/tests/boardScope.test.ts b/app/tests/boardScope.test.ts index bfb944a3..41e4c668 100644 --- a/app/tests/boardScope.test.ts +++ b/app/tests/boardScope.test.ts @@ -4,8 +4,10 @@ import { renderToStaticMarkup } from "react-dom/server"; import Methodology from "../src/components/Methodology"; import ArchivedBoardNotice from "../src/components/ArchivedBoardNotice"; +import rawData from "../src/data-summary.json"; +import servingConfig from "../src/model-serving-config.json"; import { isCurrentBoard, modelPageHref } from "../src/lib/boardScope"; -import type { BenchData } from "../src/types"; +import type { BenchData, DashboardBundle } from "../src/types"; describe("isCurrentBoard", () => { test("only the live version is the current board", () => { @@ -62,15 +64,19 @@ describe("ArchivedBoardNotice", () => { }); describe("dataset switching", () => { - const data = { - country: "us", - modelStats: [], - programStats: [], - scenarios: [], - scenarioPredictions: {}, - heatmap: [], - failureModes: [], - } as unknown as BenchData; + // The live board's payload summary (prepare-data writes it from the frozen + // release) and the frozen serving configuration: the copy's counts must + // come from these, never from hand-typed numbers. + const data = (rawData as unknown as DashboardBundle).countries.us as BenchData; + const boardModels = data.modelStats.filter( + (row) => row.condition === "no_tools", + ).length; + const chunked = Object.values(servingConfig.models).filter( + (treatment) => treatment.request_shape !== "whole scenario", + ).length; + const words = ["zero", "one", "two", "three", "four", "five", "six", "seven", + "eight", "nine", "ten", "eleven", "twelve"]; + const rosterPhrase = `${words[chunked][0].toUpperCase()}${words[chunked].slice(1)} of the ${boardModels} models`; function render(versionId: string): string { return renderToStaticMarkup( @@ -83,9 +89,18 @@ describe("dataset switching", () => { ); } + test("the configuration and payload agree on the board's roster", () => { + expect(Object.keys(servingConfig.models).sort()).toEqual( + data.modelStats + .filter((row) => row.condition === "no_tools") + .map((row) => row.model) + .sort(), + ); + }); + test("the current-board roster sentence appears only for the live version", () => { - expect(render("1.1")).toContain("Ten of the 39 models"); - expect(render("1.0")).not.toContain("Ten of the 39 models"); + expect(render("1.1")).toContain(rosterPhrase); + expect(render("1.0")).not.toContain(rosterPhrase); expect(render("1.0")).toContain("archived snapshot"); }); @@ -94,6 +109,12 @@ describe("dataset switching", () => { expect(live).toContain("This app shows the current no-tools US benchmark"); expect(live).toContain("Current benchmark scope"); expect(live).toContain("Latest United States run in this app evaluates"); + // The reference engine comes from the board's own payload. + const engine = data.policyengineBundles?.us?.model_version; + expect(engine).toBeTruthy(); + expect(live).toContain( + `PolicyEngine-US (policyengine-us ${engine}) computes each scored PolicyEngine reference output`, + ); const archived = render("1.0"); expect(archived).toContain("This app is showing the archived 1.0 board"); @@ -101,4 +122,21 @@ describe("dataset switching", () => { expect(archived).toContain("The archived 1.0 run evaluates"); expect(archived).not.toContain("Current benchmark scope"); }); + + test("states the engine behind the excluded outputs from the payload", () => { + const exclusions = data.referenceExclusions ?? []; + const byEngine = new Map(); + for (const exclusion of exclusions) { + const version = exclusion.engineVersion.replace("policyengine-us ", ""); + byEngine.set(version, (byEngine.get(version) ?? 0) + 1); + } + const engine = data.policyengineBundles?.us?.model_version as string; + const [older] = [...byEngine.keys()].filter((version) => version !== engine); + const html = render("1.1").replaceAll("'", "'"); + expect(html).toContain( + `The ${exclusions.length} excluded outputs keep the values they were decided on (${byEngine.get(older)} computed with policyengine-us ${older}, ${byEngine.get(engine)} with ${engine})`, + ); + expect(html).toContain(`that move on ${engine}; all`); + expect(render("1.0")).not.toContain("excluded outputs keep the values"); + }); }); diff --git a/app/tests/dataVersions.test.ts b/app/tests/dataVersions.test.ts index 7d333445..6a5cb324 100644 --- a/app/tests/dataVersions.test.ts +++ b/app/tests/dataVersions.test.ts @@ -93,7 +93,7 @@ describe("parseDataVersionRegistry", () => { expect( registry.versions.find((version) => version.id === registry.default) ?.snapshotLabel, - ).toBe("Snapshot 2026-09-05"); + ).toBe("Snapshot 2026-09-30"); }); }); diff --git a/app/tests/expandPage.test.ts b/app/tests/expandPage.test.ts index 6b31474e..b9480736 100644 --- a/app/tests/expandPage.test.ts +++ b/app/tests/expandPage.test.ts @@ -70,25 +70,25 @@ test("expand page derives its exact-score headline from the live summary", async new URL("../src/app/expand/page.tsx", import.meta.url), ).text(); - expect(leader.model).toBe("gpt-5.6-sol"); - expect(leader.exact).toBeCloseTo(89.1611, 4); + expect(leader.model).toBe("gpt-6-sol"); + expect(leader.exact).toBeCloseTo(95.003, 3); expect(pageSource).toContain("const leader = headlineExactLeader(dashboard);"); expect(pageSource).toContain( "const leaderLabel = MODEL_LABELS[leader.model] ?? leader.model;", ); expect(pageSource).toContain("The best model, {leaderLabel}, computes"); expect(pageSource).toContain("{leader.exact.toFixed(1)}% of"); - expect(pageSource).not.toContain("89.2%"); + expect(pageSource).not.toContain("95.0%"); expect(expectedCopy).toBe( - "GPT-5.6 Sol computes 89.2% of requested outputs exactly", + "GPT-6 Sol computes 95.0% of requested outputs exactly", ); }); test("expand page derives Medicaid error frequencies from the bundled rows", async () => { const accuracy = medicaidEligibilityAccuracy(rawData as DashboardBundle); - expect(accuracy.median).toBeCloseTo(93.78531073446328); - expect(misclassificationFrequency(accuracy.median)).toBe("about 1 in 16 people"); - expect(misclassificationFrequency(accuracy.weakest)).toBe("about 1 in 3 people"); + expect(accuracy.median).toBeCloseTo(95.3757225433526); + expect(misclassificationFrequency(accuracy.median)).toBe("about 1 in 22 people"); + expect(misclassificationFrequency(accuracy.weakest)).toBe("about 1 in 4 people"); const pageSource = await Bun.file( new URL("../src/app/expand/page.tsx", import.meta.url), ).text(); diff --git a/app/tests/heatmapMetric.test.ts b/app/tests/heatmapMetric.test.ts index 8a45d4ad..4cd8339d 100644 --- a/app/tests/heatmapMetric.test.ts +++ b/app/tests/heatmapMetric.test.ts @@ -36,11 +36,11 @@ describe("program heatmap metric", () => { test("exact and bounded rates differ where partial credit is large", () => { const federal = fable51("federal_income_tax_before_refundable_credits"); - expect(Math.round(heatmapValue(federal, "exact"))).toBe(69); - expect(Math.round(heatmapValue(federal, "score"))).toBe(93); + expect(Math.round(heatmapValue(federal, "exact"))).toBe(79); + expect(Math.round(heatmapValue(federal, "score"))).toBe(94); const stateTax = fable51("state_income_tax_before_refundable_credits"); - expect(Math.round(heatmapValue(stateTax, "exact"))).toBe(63); - expect(Math.round(heatmapValue(stateTax, "score"))).toBe(92); + expect(Math.round(heatmapValue(stateTax, "exact"))).toBe(73); + expect(Math.round(heatmapValue(stateTax, "score"))).toBe(94); }); test("the rendered heatmap shows exact-match cells and says so", () => { @@ -78,7 +78,7 @@ describe("program heatmap metric", () => { const boundedRates = entries.map((e) => Math.round(e.score)).sort((a, b) => a - b); expect(cells).toEqual(exactRates); expect(cells).not.toEqual(boundedRates); - expect(cells).toContain(69); + expect(cells).toContain(79); expect(html).toContain('aria-pressed="true"'); }); diff --git a/app/tests/modelMeta.test.ts b/app/tests/modelMeta.test.ts index 6985d570..8e7fd4a8 100644 --- a/app/tests/modelMeta.test.ts +++ b/app/tests/modelMeta.test.ts @@ -27,6 +27,8 @@ describe("model metadata", () => { ["glm-5.2", "GLM-5.2", "zai"], ["minimax-m3", "MiniMax M3", "minimax"], ["qwen-3.7-max", "Qwen 3.7 Max", "alibaba"], + ["grok-4.7", "Grok 4.7", "xai"], + ["deepseek-v4.1-flash", "DeepSeek V4.1 Flash", "deepseek"], ] as const)("maps %s to its label and provider", (model, label, provider) => { expect(MODEL_LABELS[model]).toBe(label); expect(getProviderForModel(model)).toBe(provider); diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 6f042fa1..2454d674 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -4,6 +4,7 @@ import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { + NoteArticle, NotesPageContent, interpolateNoteText, } from "../src/components/NotesContent"; @@ -31,7 +32,9 @@ describe("notes", () => { const resolved = note.paragraphs.map((paragraph) => interpolateNoteText(note, paragraph), ); - expect(resolved.join(" ")).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + expect(resolved.join(" ")).not.toMatch( + /\{[A-Za-z][A-Za-z0-9]*(?::words)?\}/, + ); } }); @@ -44,7 +47,127 @@ describe("notes", () => { expect(markup).not.toContain("at 88% of answers"); expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); + expect(markup).toContain( + "Most models answer $0 for households that qualify for SNAP under their states' higher income limits", + ); + expect(markup).toContain("release dashboard-data-20260922"); + // The BBCE note is on the frozen release. + expect(markup).toContain("release dashboard-data-20260930"); + // Cents stay as written; whole-dollar facts get thousands separators. + expect(markup).toContain("GPT-6 Sol answers $1,208.40"); + expect(markup).toContain( + "six households, four of them among the five here, at $287.68 each", + ); + expect(markup).toContain("$12,000 of financial assistance"); + expect(markup).toContain("hold $48,000 and $58,700 in savings"); + // Whole-number shares are not board rates, so they keep no decimal. + expect(markup).toContain("77% come in above $0, against 17%"); + // The September 22 notes close with the later release's figures; board + // rates keep one decimal there too. + expect(markup).toContain( + "A later release, dashboard-data-20260922c, corrects the reference for one output", + ); + expect(markup).toContain("Claude Opus 5.5 93.0%"); expect(markup).toContain("Claude Fable 5.1 added"); - expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*(?::words)?\}/); + }); + + test("a words placeholder spells out whole numbers under ten", () => { + const note = { + ...notes[0], + facts: { small: 4, large: 12, zero: 0, cents: "4.50" }, + paragraphs: [ + "{small:words} and {large:words} and {zero:words} and {cents:words} and {small}.", + ], + }; + expect(interpolateNoteText(note, note.paragraphs[0])).toBe( + "four and 12 and zero and 4.50 and 4.", + ); + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note, titleLevel: "h1" }), + ); + expect(markup).toContain("four and 12 and zero and 4.50 and 4."); + }); + + test("the BBCE note links each household by description", () => { + const note = notes.find( + (entry) => entry.slug === "2026-10-05-five-snap-households-bbce", + ); + expect(note).toBeDefined(); + expect(note!.date).toBe("2026-10-05"); + expect(note!.release).toBe("dashboard-data-20260930"); + const description = interpolateNoteText(note!, note!.paragraphs[0]); + expect(description.length).toBeLessThan(600); + expect(description).toEndWith( + "Of the 230 answers PolicyBench requests for these households, 190 come to $0 and seven match the reference.", + ); + expect(note!.paragraphs[1]).toStartWith( + "PolicyBench grades answers against references from PolicyEngine", + ); + expect(note!.paragraphs[1]).toContain( + "USDA published the ${minimumFromOctober} minimum", + ); + expect(note!.paragraphs[2]).toStartWith( + "Each household qualifies only through broad-based categorical eligibility (BBCE).", + ); + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note: note!, titleLevel: "h1" }), + ); + expect(markup).toContain(">Connecticut couple"); + expect(markup).toContain(">Arizona resident"); + expect(markup).toContain( + "For each of five households that qualify for SNAP food benefits, most of the 46 models on PolicyBench answer $0.", + ); + expect(markup).toContain( + "$288 for 2026 for four households, and $240 for an Arizona resident who qualifies from March", + ); + expect(markup).toContain( + "$24 a month through September 2026 and $25 from October", + ); + expect(markup).toContain("GPT-6 Astra gets three of the five households right"); + expect(markup).toContain( + "GPT-6.1 Sol gets two right, the Michigan and Wisconsin households", + ); + expect(markup).toContain( + "The version of this note published September 23 counted the worker among its households.", + ); + expect(markup).not.toContain("revised this note"); + expect(markup).not.toContain("updated this note"); + expect(markup).toContain(">policyengine-us #9586 (SNAP child support treatment)"); + expect(markup).not.toContain("scenario_045"); + expect(markup).not.toMatch(/>scenario_\d+<\/a>/); + }); + + test("the September 3 note links the later correction", () => { + const note = notes.find( + (entry) => entry.slug === "2026-09-03-six-snap-households", + ); + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note: note!, titleLevel: "h1" }), + ); + expect(markup).toContain( + "A later note, published October 5, corrects this one. From release dashboard-data-20260922c on, PolicyBench scores the SNAP amounts of four of these six households at $288 each, and scores a Michigan worker who pays child support at $0: PolicyEngine counts that child support in gross income, as Michigan does (policyengine-us #9586), and the worker does not qualify.", + ); + expect(markup).toContain( + 'href="https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260922c"', + ); + expect(markup).not.toContain("scenario_045 and scenario_112"); + expect(markup).toContain( + 'href="/notes/2026-10-05-five-snap-households-bbce"', + ); + expect(markup).not.toContain("2026-09-23-five-snap-households-bbce"); + }); + + test("only the September 3 note carries the unrounded-reference footnote", () => { + const footnoted = notes.filter((note) => { + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note, titleLevel: "h1" }), + ); + return markup.includes(`id="${note.slug}-reference-annual-note"`); + }); + // The five-household note's $288 is a rounded reference, so it has none. + expect(footnoted.map((note) => note.slug)).toEqual([ + "2026-09-03-six-snap-households", + ]); }); }); diff --git a/app/tests/programChart.test.ts b/app/tests/programChart.test.ts index 5322dcdb..76da16e3 100644 --- a/app/tests/programChart.test.ts +++ b/app/tests/programChart.test.ts @@ -21,7 +21,7 @@ describe("program bars", () => { const federal = bars.find( (b) => b.variable === "federal_income_tax_before_refundable_credits", )!; - expect(federal.exact).toBeCloseTo(69.0, 1); + expect(federal.exact).toBeCloseTo(79.3, 1); expect(federal.weightShare).toBeCloseTo(0.191, 2); const total = bars.reduce((sum, b) => sum + (b.weightShare ?? 0), 0); expect(total).toBeCloseTo(1, 3); @@ -37,10 +37,10 @@ describe("program bars", () => { ); expect(html.match(/

  • 69.0%<"); + expect(html).toContain(">79.3%<"); expect(html).toContain(">19%<"); - expect(html).toContain("69.0% exact"); - expect(html).toContain("width:69%"); + expect(html).toContain("79.3% exact"); + expect(html).toContain("width:79.26829268292683%"); }); test("narrow rows stack the bar under the label; wide rows reserve a bar track", () => { diff --git a/app/tests/redirects.test.ts b/app/tests/redirects.test.ts new file mode 100644 index 00000000..cf2d74fe --- /dev/null +++ b/app/tests/redirects.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; + +import nextConfig, { noteRedirects } from "../next.config"; +import { getNote, notes } from "../src/notes"; + +describe("note redirects", () => { + test("the September 23 BBCE URL redirects permanently to the October 5 note", async () => { + const redirects = await nextConfig.redirects!(); + expect(redirects).toEqual(noteRedirects); + expect(redirects).toContainEqual({ + source: "/notes/2026-09-23-five-snap-households-bbce", + destination: "/notes/2026-10-05-five-snap-households-bbce", + permanent: true, + }); + }); + + test("every redirect leaves a URL no note owns for a note that exists", async () => { + const redirects = await nextConfig.redirects!(); + const slugs = new Set(notes.map((note) => note.slug)); + for (const redirect of redirects) { + const source = redirect.source.replace(/^\/notes\//, ""); + const destination = redirect.destination.replace(/^\/notes\//, ""); + expect(redirect.source.startsWith("/notes/")).toBe(true); + expect(redirect.destination.startsWith("/notes/")).toBe(true); + expect(slugs.has(source)).toBe(false); + expect(getNote(destination)).toBeDefined(); + expect(redirect.permanent).toBe(true); + } + }); + + test("no note links a redirected URL", async () => { + const redirects = await nextConfig.redirects!(); + const sources = new Set(redirects.map((redirect) => redirect.source)); + for (const note of notes) { + for (const entry of note.data) { + expect(sources.has(entry.href)).toBe(false); + } + } + }); +}); diff --git a/app/tests/referenceEngine.test.ts b/app/tests/referenceEngine.test.ts new file mode 100644 index 00000000..703f2cd5 --- /dev/null +++ b/app/tests/referenceEngine.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import fc from "fast-check"; + +import rawData from "../src/data-summary.json"; +import { + ENGINE_UPGRADE_RECHECK, + excludedOutputEngineSentence, + excludedOutputsByEngine, +} from "../src/lib/referenceEngine"; +import type { DashboardBundle, ReferenceExclusion } from "../src/types"; + +const board = (rawData as unknown as DashboardBundle).countries.us; +const exclusions = board.referenceExclusions ?? []; + +function compare(a: string, b: string): number { + const left = a.split(".").map(Number); + const right = b.split(".").map(Number); + for (let i = 0; i < Math.max(left.length, right.length); i += 1) { + const diff = (left[i] ?? 0) - (right[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} + +const version = fc + .tuple(fc.nat(3), fc.nat(800), fc.nat(20)) + .map((parts) => parts.join(".")); +const exclusionList = fc.array( + version.map( + (v) => ({ engineVersion: `policyengine-us ${v}` }) as ReferenceExclusion, + ), + { maxLength: 60 }, +); + +describe("excludedOutputsByEngine properties", () => { + test("the counts partition the exclusions by version, oldest first", () => { + fc.assert( + fc.property(exclusionList, (list) => { + const groups = excludedOutputsByEngine(list); + // Every exclusion is counted once, under its own version. + expect(groups.reduce((sum, [, count]) => sum + count, 0)).toBe( + list.length, + ); + for (const [v, count] of groups) { + expect(count).toBeGreaterThan(0); + expect( + list.filter((e) => e.engineVersion === `policyengine-us ${v}`) + .length, + ).toBe(count); + } + // Versions are distinct and in numeric (not string) order. + const versions = groups.map(([v]) => v); + expect(new Set(versions).size).toBe(versions.length); + for (let i = 1; i < versions.length; i += 1) { + expect(compare(versions[i - 1], versions[i])).toBeLessThan(0); + } + // Order of the input does not matter. + expect(excludedOutputsByEngine([...list].reverse())).toEqual(groups); + }), + ); + }); + + test("the sentence states every count once and the total", () => { + fc.assert( + fc.property(exclusionList, (list) => { + const sentence = excludedOutputEngineSentence(list, null); + if (list.length === 0) { + expect(sentence).toBeNull(); + return; + } + expect(sentence).toStartWith( + `The ${list.length.toLocaleString("en-US")} excluded outputs keep`, + ); + for (const [v, count] of excludedOutputsByEngine(list)) { + expect(sentence).toContain(`${count.toLocaleString("en-US")} `); + expect(sentence).toContain(v); + } + expect(sentence).not.toContain("re-reviewed"); + }), + ); + }); + + test("numeric version order beats string order", () => { + expect( + excludedOutputsByEngine([ + { engineVersion: "policyengine-us 2.15.17" }, + { engineVersion: "policyengine-us 1.755.4" }, + { engineVersion: "policyengine-us 2.9.0" }, + ] as ReferenceExclusion[]).map(([v]) => v), + ).toEqual(["1.755.4", "2.9.0", "2.15.17"]); + }); +}); + +describe("excluded-output engines against the Python record", () => { + test("the payload's counts are the ones the paper renders from the exclusion record", () => { + // paper_results.excluded_outputs_by_engine_version counts the frozen + // reference_exclusions.json; the rendered paper states its counts. + const html = readFileSync( + new URL("../public/paper/web/index.html", import.meta.url), + "utf8", + ).replace(/<[^>]+>/g, ""); + const match = html.match( + /keep the values they were decided on \((\d+) computed with policyengine-us ([\d.]+), (\d+) with ([\d.]+)\)/, + ); + expect(match).not.toBeNull(); + const [, olderCount, older, newerCount, newer] = match!; + expect(excludedOutputsByEngine(exclusions)).toEqual([ + [older, Number(olderCount)], + [newer, Number(newerCount)], + ]); + }); + + test("the live sentence names the reference engine's recheck", () => { + const sentence = excludedOutputEngineSentence( + exclusions, + ENGINE_UPGRADE_RECHECK.engineVersion, + ); + expect(sentence).toContain( + `re-reviewed the ${ENGINE_UPGRADE_RECHECK.rechecked} of them that move on ${ENGINE_UPGRADE_RECHECK.engineVersion}`, + ); + }); +}); diff --git a/app/tests/scenarioExplorer.test.ts b/app/tests/scenarioExplorer.test.ts index be881f39..e786c070 100644 --- a/app/tests/scenarioExplorer.test.ts +++ b/app/tests/scenarioExplorer.test.ts @@ -133,6 +133,31 @@ describe("prediction detail for excluded outputs", () => { expect(html).not.toContain("Not yet reviewed"); }); + test("says the audit note of an excluded output compares with the frozen reference", () => { + const html = renderDetail(excludedRow()); + + expect(html).toContain('data-testid="excluded-audit-caveat"'); + expect(html).toContain( + "This audit note compares the answer with the frozen reference, which assumes one reading of the unlisted input described above.", + ); + const defect = renderDetail( + excludedRow({ excludedReason: "reference_engine_defect" }), + {}, + ); + expect(defect).toContain( + "This audit note compares the answer with the frozen reference, which carries the engine defect described above.", + ); + }); + + test("a scored miss carries no excluded-output caveat", () => { + const html = renderDetail( + excludedRow({ scored: true, excludedReason: undefined, excludedInput: undefined }), + {}, + ); + expect(html).not.toContain("excluded-audit-caveat"); + expect(html).toContain("never the SSI disability criterion"); + }); + test("falls back to the row's own exclusion fields without a release record", () => { const html = renderDetail(excludedRow(), {}); @@ -200,3 +225,49 @@ describe("predictionStatus", () => { expect(findReferenceExclusion([SSI_EXCLUSION], null, "ssi")).toBeUndefined(); }); }); + +describe("engine-defect exclusion note", () => { + const DEFECT_EXCLUSION: ReferenceExclusion = { + scenarioId: "scenario_049", + variable: "federal_income_tax_before_refundable_credits", + reasonCode: "reference_engine_defect", + rootCause: "r02_ira_219g", + defect: + "The engine deducts traditional IRA contributions without the active-participant phase-out", + law: "26 U.S.C. 219(g); IRS Notice 2025-67", + upstream: "PolicyEngine/policyengine-us#0000", + alternativeReading: + "Under 219(g) a covered participant above $149,000 of joint MAGI deducts nothing.", + frozenValue: 30543.91, + alternativeValue: 30702.59, + engineVersion: "policyengine-us 1.755.4", + decidedOn: "2026-09-22", + note: "", + }; + + test("states the defect, the law, the corrected value and the upstream issue", async () => { + const { default: ExclusionNote } = await import("../src/components/ExclusionNote"); + const html = renderToStaticMarkup( + createElement(ExclusionNote, { + pred: excludedRow({ + groundTruth: 30543.91, + prediction: 30702.59, + excludedReason: "reference_engine_defect", + excludedInput: "r02_ira_219g", + }), + exclusion: DEFECT_EXCLUSION, + isBinary: false, + currencySymbol: "$", + }), + ); + expect(html).toContain("misapplies the law"); + expect(html).toContain("active-participant phase-out"); + expect(html).toContain("26 U.S.C. 219(g)"); + expect(html).toContain("$30,703"); + expect(html).toContain("$30,544"); + expect(html).toContain("PolicyEngine/policyengine-us#0000"); + // The engine-defect note never presents the root cause as an unlisted input. + expect(html).not.toContain("never listed"); + expect(html).not.toContain(" `claude-${s}`), + ), + treatment, + { maxKeys: 20 }, + ) + .map((models) => ({ models }) as ServingConfiguration); + +function isSubsequence(items: string[], of: string[]): boolean { + let at = 0; + for (const item of of) if (item === items[at]) at += 1; + return at === items.length; +} + +describe("serving configuration helpers", () => { + test("worked example", () => { + const example = { + models: { + "claude-a": { + answer_contract: "json", + request_shape: "whole scenario", + tool_choice: null, + }, + "claude-b": { + answer_contract: "tool", + request_shape: "one output per request", + tool_choice: "forced", + }, + "gemini-c": { + answer_contract: "json", + request_shape: "three outputs per request", + tool_choice: null, + }, + }, + } as ServingConfiguration; + expect(chunkedServingModels(example)).toEqual(["claude-b", "gemini-c"]); + expect(jsonContractClaudeModels(example)).toEqual(["claude-a"]); + }); + + test("chunked rows: exactly the rows not sent whole, in roster order", () => { + fc.assert( + fc.property(config, (cfg) => { + const chunked = chunkedServingModels(cfg); + const models = Object.keys(cfg.models); + expect(isSubsequence(chunked, models)).toBe(true); + for (const model of models) { + expect(chunked.includes(model)).toBe( + cfg.models[model].request_shape !== "whole scenario", + ); + } + }), + ); + }); + + test("JSON Claude rows: exactly the Claude rows on the JSON contract", () => { + fc.assert( + fc.property(config, (cfg) => { + const rows = jsonContractClaudeModels(cfg); + const models = Object.keys(cfg.models); + expect(isSubsequence(rows, models)).toBe(true); + for (const model of models) { + expect(rows.includes(model)).toBe( + model.startsWith("claude-") && + cfg.models[model].answer_contract === "json", + ); + } + }), + ); + }); + + test("the live helpers agree with the paper's serving-configuration table", () => { + // Python renders the table from the same frozen configuration + // (paper/index.qmd); the rows are matched by provider id. + const html = readFileSync( + new URL("../public/paper/web/index.html", import.meta.url), + "utf8", + ); + const rows = new Map(); + for (const row of html.matchAll(/]*>([\s\S]*?)<\/tr>/g)) { + const cells = [...row[1].matchAll(/([\s\S]*?)<\/td>/g)].map((c) => + // The table breaks long ids with zero-width spaces. + c[1].replace(/<[^>]+>/g, "").replace(/\u200b/g, "").trim(), + ); + if (cells.length === 7 && (cells[4] === "JSON" || cells[4] === "forced tool")) { + rows.set(cells[1], { contract: cells[4], shape: cells[5] }); + } + } + const models = servingConfig.models as Record< + string, + { provider_id: string } + >; + expect(rows.size).toBe(Object.keys(models).length); + const byTable = (predicate: (row: { contract: string; shape: string }) => boolean) => + Object.entries(models) + .filter(([, t]) => predicate(rows.get(t.provider_id)!)) + .map(([model]) => model); + expect(chunkedServingModels()).toEqual( + byTable((row) => row.shape !== "whole scenario"), + ); + expect(jsonContractClaudeModels()).toEqual( + byTable((row) => row.contract === "JSON").filter((m) => + m.startsWith("claude-"), + ), + ); + // tests/test_disclosures.py checks each of these rows' model card + // records that the API rejects forced tool use. + expect(jsonContractClaudeModels()).toEqual([ + "claude-fable-5.1", + "claude-opus-5.5", + "claude-sonnet-5.5", + ]); + }); + + test("number words and lists", () => { + expect(numberWord(10)).toBe("ten"); + expect(numberWord(45)).toBe("45"); + expect(capitalizeFirst("three")).toBe("Three"); + expect(joinWithAnd(["A"])).toBe("A"); + expect(joinWithAnd(["A", "B"])).toBe("A and B"); + expect(joinWithAnd(["A", "B", "C"])).toBe("A, B and C"); + }); +}); + +describe("leaderboard serving copy", () => { + function render(versionId: string): string { + const programOptions = buildProgramOptions(board); + return renderToStaticMarkup( + createElement(ModelLeaderboard, { + data: board, + selectedView: "us", + dashboard: bundle, + versionId, + liveVersionId: "1.1", + programOptions, + activeProgramIds: new Set(programOptions.map((option) => option.variable)), + activeProgramSummary: "All programs", + onResetPrograms: () => {}, + onToggleProgram: () => {}, + onSelectOnlyProgram: () => {}, + }), + ).replace(/\s+/g, " "); + } + + test("names the Claude rows that reject forced calls from the configuration", () => { + const labels = joinWithAnd( + jsonContractClaudeModels().map((model) => MODEL_LABELS[model] ?? model), + ); + expect(labels).toBe("Claude Fable 5.1, Claude Opus 5.5 and Claude Sonnet 5.5"); + const html = render("1.1"); + expect(html).toContain(`${labels} reject forced calls, so their rows select JSON`); + expect(render("1.0")).not.toContain("reject forced calls"); + }); + + test("states the re-run counts the sensitivity data records", () => { + // The August runs are the forced-tool rows, whose thinking was off; the + // Fable 5.1 run compares transports on a row that reasons. + const counts = servingSensitivityCounts(); + const august = Object.values(augustSummary.runs).map((run) => run.model); + expect(Object.keys(SERVING_SENSITIVITY).sort()).toEqual( + [...august, fable51Summary.model].sort(), + ); + expect( + Object.entries(SERVING_SENSITIVITY) + .filter(([, entry]) => entry.thinkingSuppressedOnBoard) + .map(([model]) => model) + .sort(), + ).toEqual([...august].sort()); + expect(counts).toEqual({ + reruns: august.length + 1, + thinkingSuppressed: august.length, + }); + const html = render("1.1"); + expect(html).toContain( + `${capitalizeFirst(numberWord(counts.thinkingSuppressed))} Claude rows ran without extended thinking`, + ); + expect(html).toContain( + `re-runs are marked on the ${numberWord(counts.reruns)} rows`, + ); + expect(html).toContain(`has all ${numberWord(counts.reruns)} runs`); + }); + + test("the counts follow the entries", () => { + fc.assert( + fc.property(fc.array(fc.boolean(), { maxLength: 12 }), (flags) => { + const entries = Object.fromEntries( + flags.map((flag, i) => [ + `m${i}`, + { ...SERVING_SENSITIVITY["claude-opus-5"], thinkingSuppressedOnBoard: flag }, + ]), + ); + expect(servingSensitivityCounts(entries)).toEqual({ + reruns: flags.length, + thinkingSuppressed: flags.filter(Boolean).length, + }); + }), + ); + }); +}); diff --git a/app/tests/servingSensitivity.test.ts b/app/tests/servingSensitivity.test.ts index ee3dd318..31fe8a7f 100644 --- a/app/tests/servingSensitivity.test.ts +++ b/app/tests/servingSensitivity.test.ts @@ -24,12 +24,14 @@ const rows = bundle.countries.us.modelStats.filter( ); function renderFable5Chip(): string { + const sensitivity = servingSensitivityFor("claude-fable-5")!; + const board = rows.find((row) => row.model === "claude-fable-5")!; return renderToStaticMarkup( createElement(ServingSensitivityChip, { modelLabel: "Claude Fable 5", - boardExact: 80.4266, - sensitivity: servingSensitivityFor("claude-fable-5")!, - wouldRank: 3, + boardExact: board.exact ?? board.score, + sensitivity, + wouldRank: wouldRank(sensitivity.autoExact, rows), }), ); } @@ -57,11 +59,11 @@ describe("serving sensitivity", () => { wouldRank: rank, }), ); - expect(rank).toBe(3); - expect(html).toContain("auto 87.5 · #3"); + expect(rank).toBe(7); + expect(html).toContain("auto 91.5 · #7"); expect(html).toContain("switches Claude's extended thinking off"); - expect(html).toContain("would rank #3"); - expect(html).toContain("(+7.1 against its 80.4% on the unfiltered board)"); + expect(html).toContain("would rank #7"); + expect(html).toContain("(+7.9 against its 83.6% on the unfiltered board)"); expect(html).toContain("sensitivity/claude-thinking-2026-08.md"); expect(html).toContain("issues/139"); }); @@ -71,12 +73,12 @@ describe("serving sensitivity", () => { const html = renderToStaticMarkup( createElement(ServingSensitivityChip, { modelLabel: "Claude Fable 5.1", - boardExact: 86.9, + boardExact: 90.8, sensitivity, wouldRank: wouldRank(sensitivity.autoExact, rows), }), ); - expect(html).toContain("auto 88.2 · #2"); + expect(html).toContain("auto 91.7 · #6"); expect(html).toContain("rejects forced tool calls"); expect(html).toContain("compares transports"); expect(html).toContain('href="/notes/2026-09-01-claude-fable-5-1-added"'); @@ -89,8 +91,11 @@ describe("serving sensitivity", () => { ); expect(source).not.toContain("SENSITIVITY_EXACT"); expect(source).toContain(" { expect(SERVING_SENSITIVITY[run.model].autoExact).toBe(run.sensitivity.exact); } // 88.183 − 86.945 = 1.238 → +1.2; rounding first would have said +1.3. - expect(formatDelta(88.183, fable51Summary.board.exact)).toBe("+1.2"); + expect(formatDelta(88.183, 86.945)).toBe("+1.2"); expect(formatDelta(88.2, 86.945)).toBe("+1.3"); + // The pinned Fable 5.1 pair on the frozen board: 91.728 − 90.828 = 0.900. + expect( + formatDelta(fable51Summary.sensitivity.exact, fable51Summary.board.exact), + ).toBe("+0.9"); expect(formatDelta(80.775, 80.8)).toBe("−0.0"); const fable5 = Object.values(augustSummary.runs).find( (run) => run.model === "claude-fable-5", )!; - expect(formatDelta(fable5.sensitivity.exact, fable5.board.exact)).toBe("+7.1"); + expect(formatDelta(fable5.sensitivity.exact, fable5.board.exact)).toBe("+7.9"); }); test("the open panel stays inside the viewport, horizontally and vertically", () => { diff --git a/app/vercel.json b/app/vercel.json index 20364d65..41dec69b 100644 --- a/app/vercel.json +++ b/app/vercel.json @@ -3,5 +3,10 @@ "framework": "nextjs", "bunVersion": "1.x", "installCommand": "bunx bun@1.3.11 install --frozen-lockfile", - "buildCommand": "bunx bun@1.3.11 run build" + "buildCommand": "bunx bun@1.3.11 run build", + "git": { + "deploymentEnabled": { + "codex/issue165-prompt-contract-v2": false + } + } } diff --git a/docs/adds0928/brief_S3_prose_and_pins.md b/docs/adds0928/brief_S3_prose_and_pins.md new file mode 100644 index 00000000..2b2be5b0 --- /dev/null +++ b/docs/adds0928/brief_S3_prose_and_pins.md @@ -0,0 +1,81 @@ +# Stage 3 brief: prose, notes, paper and pins for release dashboard-data-20260929 + +You are working in `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2`, branch `adds-0928-stage2`, in place. The release data is already frozen and committed (HEAD 49175d0f). Your job is every documentation, app-copy, note, paper and test-pin change the release needs, so that the full test suite, the app checks and the paper render pass. Do not change data, references, exclusions, adjudications, the payload or the frozen snapshot, except through the documented regeneration commands below. + +Read the repository's `CLAUDE.md`. Also read `docs/adds0928/stage2_design.md` ("Before rendering, write a new September 28 note...") and `reference_audit/2026-09-28/README.md`. + +## What changed in this release (verified facts; cite only these or what you verify yourself) + +- **Release.** `dashboard-data-20260929`: 45 models (the 42 of 20260922c plus Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash), 100 households, 1,984 outputs, 1,929 scored, 55 excluded (was 1,932 and 52). The payload sha256 is `e7d5e056b53c0d6d406bb3afb389aeaf80ebb611932ce72c8b3c3ceaef5ddad6` (`app/src/data.artifact.json`). +- **Reference engine.** References now come from policyengine-us **2.15.17**, the newest on PyPI (uploaded 2026-09-29 00:23 UTC). Max's ruling, 2026-09-28: "we should be using the latest pe for this always!" + - policyengine.py 6.1.2 is recorded for provenance only. Its certified US bundle is policyengine-us 2.2.1, and it will not import next to 2.15.17. PolicyBench has always computed references with `policyengine_us.Simulation`. + - The pre-freeze publication rule still holds, with the nine conventions re-expressed for 2.15.17. Read `reference_audit/2026-09-28/README.md` for the exact changes. Do not restate mechanisms beyond what that README and its record say. +- **Reference changes against 20260922c.** + - 4 scored references changed: 008 NJ refundable credits 5,342.40→5,842.40; 013 AZ SNAP 0→240; 028 PA reduced-price school meals 1→0; 082 NY refundable credits 650.50→667.00. + - 3 outputs newly excluded as unlisted-input readings: 033, 078 and 117 federal income tax (taxability of a listed state and local tax refund). + - 2 moved by under $1: 078 and 117 state income tax. + - 19 excluded outputs were re-reviewed and stay excluded. +- **Staged results** (weighted exact-match share; from `results/local/adds0928-v3/data-board45.json` and `incumbent-drift.json`; recompute from the frozen payload rather than trusting these): + - Claude Sonnet 5.5: 91.973%, rank 4 of 45. It is 0.015 points ahead of GPT-6 Luna (91.958%), so describe that as about level, never as a clear lead. + - Grok 4.7: 88.228%, rank 12. DeepSeek V4.1 Flash: 87.274%, rank 15. + - The top 3 are GPT-6 Sol 94.813, Claude Opus 5.5 93.507 and GPT-5.6 Sol 93.280. + - Every incumbent's exact rate rose 0.36 to 0.54 points under the new references. The top eight incumbents keep their order, and 6 incumbents change rank. + - Cost per household: Sonnet 5.5 about $0.034, Grok 4.7 about $0.26, DeepSeek V4.1 Flash about $0.024 (confirm from the payload). +- **Treatments.** Sonnet 5.5 uses the JSON answer contract with adaptive thinking; see `policybench/model_cards.py` and `model_serving_config.json`. Grok 4.7 uses a 1,800-second request timeout. DeepSeek V4.1 Flash is served through the moving `deepseek-flash` alias. The design doc's risk section says how to disclose that; follow it. +- **BBCE follow-up** (for the BBCE note update and the release note). Verify each figure from the frozen predictions before using it. + - The four income-held BBCE households (CT 027, TX 030, MI 073, WI 108; reference $288 each) now have 45 models' answers. + - Sonnet 5.5 answers CT $388 (citing BBCE), TX $1,208, MI $0 and WI $0. Its TX explanation says "I counted wages only and treated the assistance amounts as excluded", the same income error and amount as Claude Opus 5.5 and GPT-6 Sol. + - Grok 4.7: CT $276 (citing BBCE), TX $0, MI $0, WI $0. + - DeepSeek V4.1 Flash: CT $0, TX $588, MI $0, WI $0. + - None of the three gets any of the four exactly right. + - The references now add a fifth household held back by income: Arizona 013. Arizona raised its expanded categorical eligibility gross limit from 185% to 200% of the poverty guideline from benefit month 03/2026, and the reference is $240 ($24 a month, March to December). Every one of the 45 models answers $0 for it. + - The asset-held households (008, 054, 066, 080): Sonnet 5.5 answers above $0 for all four, Grok 4.7 for all four, and DeepSeek V4.1 Flash for three of four. The savings-versus-income split in the note holds for the new models. + +## Deliverables + +1. **A new release note** in `app/src/notes/`, dated 2026-09-29, registered in `app/src/notes/index.ts`, following the existing notes' JSON structure (read `2026-09-22-gpt-6-sol-debuts-first.json` and `2026-09-22-reference-audit.json`) and the voice and fact-pinning of `tests/test_notes.py`. + - Cover the three additions and the move to the newest PolicyEngine, including what changed and why, the new exclusions, and the incumbent drift. + - Every number in the note must be a fact the tests recompute from committed data. Add the tests. + - Title in sentence case. +2. **A dated update to the BBCE note** (`2026-09-23-five-snap-households-bbce.json`). Add an update paragraph with the new models' results and the Arizona household, and link the new release. Keep its historical figures, which are tied to release 20260922b/c, as they are, and keep its tests passing. If the update needs data files, add them under `notes/data/` with tests. +3. **The paper and its render.** + - Update `paper/index.qmd` and `docs/paper.md` for 45 models, 1,929 scored, 55 excluded, the reference engine, the 2026-09-28 upgrade and the response window. + - Render with `uv run python paper/render_paper.py` (or `.venv/bin/python`, see below), then re-pin with `scripts/freeze_snapshot.py --rendered-only`. + - Keep historical statements about earlier releases true to those releases. +4. **The benchmark card, methodology copy and other docs.** + - Update `docs/benchmark_card.md`, the app components that state counts or the engine version (for example `app/src/components/Methodology.tsx`, `ExclusionNote.tsx`) and `app/src/data.versions.json`'s description, which still says policyengine-us 1.755.4. + - Update the sensitivity docs whose tables `tests/test_sensitivity_evidence.py` compares against the rescored summaries (already committed), and the cost report the costs test checks. + - Refresh the app copy of the serving config (`app/src/model-serving-config.json`) from `paper/snapshot/20260501/model_serving_config.json`, as prepare-data does. +5. **Test pins.** + - Update tests that pin facts of the live release (counts such as 42 models, 63 adjudications, 7,545 rows, 39 usage rows, 1,932/52) to this release's values. Recompute each new value from the frozen artifacts; never copy a failing assertion's number blindly. + - Leave pins that belong to earlier releases' notes or records alone. + - `tests/test_paper_results.py::test_frozen_roster_has_42_display_names_and_release_dates` becomes 45. The new models' display names and release dates are already in `policybench/paper_results.py` and `app/src/modelMeta.ts`: Grok 4.7 released 2026-09-21, DeepSeek V4.1 Flash 2026-09-10, and Sonnet 5.5 as recorded there. + +## Voice (public copy: notes, paper, card, methodology copy) + +Before writing any prose, read Max's voice guide `~/.claude/projects/-Users-maxghenis/memory/voice_max.md` (its "Current rules") and the working model `~/.claude/projects/-Users-maxghenis/memory/user_max_working_model.md`, and apply them to every string you write or change. In particular: +- active voice, with the actor in the subject; +- one thought per sentence, with varied rhythm; +- no self-referential clauses and no empty "X, not Y" antithesis; +- no superlatives or intensifiers, and numbers over adjectives; +- sentence-case headings; +- finished work introduces itself fresh: the paper and card state what the benchmark does now, with no draft or review lineage; +- release notes and corrections take the change as their subject and keep the numbers; +- no policy positions, not even hints. + +When a reviewer would flag one sentence, reread the whole document for the pattern. Keep each note's existing structure and register (third person, "PolicyBench …"). + +## Rules + +- **No fabricated claims.** State only what you read in committed code or data, or computed yourself. When a number comes from the payload, compute it. +- **Voice.** Match the existing notes and paper: plain, specific, third person ("PolicyBench …"). Use sentence-case headings. Don't guess at unrecorded figures. +- **Workflow.** + - Run `ruff format` and `ruff check .` (reference_audit/ is excluded). + - Run the full `pytest -q`, and the app's `bun install --frozen-lockfile`, `bun run test`, `bun run lint` and `bun run build` in `app/`. + - Use the worktree's `.venv` (Python 3.12, policyengine-us 2.15.17) with `OPENBLAS_NUM_THREADS=1` and `PYTHONPATH=$PWD`. + - The machine is shared and heavily loaded, so run the big steps serially. +- **Commits.** + - Make coherent commits on `adds-0928-stage2`, each message ending with `Co-Authored-By: Claude Opus 5.5 `. + - Do not push, open PRs, upload release assets, or post anything. + - Do not edit `reference_audit/`, the frozen snapshot data files, the references or the adjudications. +- **Report.** End with a report listing every file changed, every test pin changed with old→new and where the new value came from, and the final results of pytest and the app checks. Write it to `docs/adds0928/report_S3.md` and commit it. diff --git a/docs/adds0928/brief_S3b_review_fixes.md b/docs/adds0928/brief_S3b_review_fixes.md new file mode 100644 index 00000000..c347a3f3 --- /dev/null +++ b/docs/adds0928/brief_S3b_review_fixes.md @@ -0,0 +1,76 @@ +# Stage 3b brief: fix the review findings for release dashboard-data-20260929 + +Work in `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2` (branch `adds-0928-stage2`, in place). Stage 3 (commits after 464649b6) wrote the release's prose, notes, paper and pins. Two adversarial reviews requested changes, and the lead has consolidated them below. + +Keep following `docs/adds0928/brief_S3_prose_and_pins.md`, including its Voice and Rules sections, and Max's voice guide `~/.claude/projects/-Users-maxghenis/memory/voice_max.md`. Use the worktree `.venv` with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`, and run the heavy steps serially. + +## Decisions already made (apply them; do not reopen) + +1. **Engine wording.** Keep policyengine-us 2.15.17 as the recorded engine. State it as a dated fact: "policyengine-us 2.15.17, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded 00:23 UTC)". Add the verification: policyengine-us 2.17.0, the newest release at publication (uploaded 2026-09-29 12:21 UTC), gives the same value for all 1,984 outputs under the same conventions and adapter. + - Evidence: `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/out/latest_final_2170.csv` and `.log`, from `sweep_latest.py --fix fixes/latest_final.py` run in a policyengine 6.1.2 + policyengine-us 2.17.0 venv. + - Copy both files into `reference_audit/2026-09-28/verification/`, add a README line, and add a test that the CSV's `recomputed` column equals the committed reference for every scored output. + - Remove every unanchored "newest", in the card, paper, release note, README and app copy. +2. **Upgrade date.** Use 2026-09-29 for the upgrade everywhere. The references were rebuilt at 2026-09-29 11:57 UTC and the release is 20260929. Call the three models "the September 29 additions" (release 20260929) consistently, in place of "September 28 additions". +3. **What 2.15.17 computes.** It computes each *scored* reference. Everywhere the copy says 2.15.17 computes every reference or every household-variable pair, say "each scored reference". Add one sentence in the card, the paper and the app Methodology copy: the 55 excluded outputs keep the values they were decided on (52 computed with policyengine-us 1.755.4, 3 with 2.15.17), and the 19 of them that move on 2.15.17 were re-reviewed and stay excluded. Apply the same to `app/src/data.versions.json`'s description. + +## Findings to fix + +**Release note (`app/src/notes/2026-09-29-...json`):** +- Paragraph 7 calls the Arizona household "a fifth household to do so". That is wrong: other scored households also qualify only through BBCE. Arizona is the fifth household *held back by income* in the BBCE note's sense, where the other four are CT 027, TX 030, MI 073 and WI 108. Rewrite it as the reviewer suggested, and fix "the other four". +- The DeepSeek V4.1 Flash cost is priced at the provider's standard (peak) list rate, per `policybench/config.py`. Say so ("at DeepSeek's standard list price"). Make no claim about off-peak billing unless you verify DeepSeek's current off-peak policy and the run's timestamps from a primary source this session. +- Tie the remaining prose facts to data in the tests: the title's ordinal equals the ordinal of `facts['sonnetRank']`, and the engine version equals the sidecar's. + +**Drift claims.** The note's drift range, its three neighbor swaps, "GPT-6 Sol still leads" and "every one of the 42 earlier models" need a real check. +- Commit a small fixture holding the 42 no_tools exact scores of release `dashboard-data-20260922c`, taken from the live 22c payload. Its sha256 is `01e7e72b3a6bdd2d3178ba32625ff769d5b81dc07541af6ea8da2c852774ddcc`, and it is available as the stage base or in git at 3220a7a: `paper/snapshot/.../data.json.gz` wraps the country payload. +- Record the source sha in a meta file. +- Test the note's drift facts against that fixture and the frozen payload with no git access. Replace the vacuous assertion. + +**Paper (`paper/index.qmd`, `docs/paper.md`):** +- Drop the unnamed "five audit waves" count ("Across development and each later audit ..."). +- "the reviewers excluded" becomes "the investigation found, and PolicyBench excluded, ...". +- `docs/paper.md`: the manifest's `reference_output_refresh` block records the reference runtime's default dataset (build populace-us-2024-spm-20260915, from the policyengine.py 6.1.2 bundle), which reference computation does not read. The households' source build is populace-us-2024-5da5a95-20260611 (`scenarios.csv.meta.json`). Say that accurately. +- Fix the pre-existing render bug: the cost-and-latency sentence renders a literal `\\$0.002` and the cost table renders `\$0.034`, in both HTML and PDF. Find the escaping in `paper/index.qmd` or `policybench/paper_results.py`, fix it, and add a test on the rendered HTML. + +**Manifest reproducibility note (`scripts/freeze_snapshot.py`, the `build_manifest` template).** +- It says the references "were generated with policyengine.py X and policyengine-us Y against the certified PolicyEngine US populace dataset (build ...)". That is a mechanism error. PolicyBench computes each reference with `policyengine_us.Simulation` from the household's own listed inputs, and records policyengine.py only for provenance. +- Rewrite the template sentence accurately: name policyengine-us Y and policyengine.py X (provenance), and say the households were drawn from the populace build in `scenarios.csv.meta.json`. Read that build from the scenarios meta, not the runtime bundle. +- Keep the manifest's existing keys. If you add a field for the households' build, test it. +- Regenerate the manifest through the documented freeze (the next section), not by hand. + +**App copy.** +- `Methodology.tsx`: compute the chunked-row count from `model-serving-config.json` (rows whose request shape is not "whole scenario") and the model total from the payload's modelStats. +- `ModelLeaderboard.tsx`: build the list of Claude rows that reject forced tool calls (JSON contract) from the serving config. +- Update `app/tests` to compare against the config and payload, not hand-typed numbers. + +**Paper-results invariants (`policybench/paper_results.py`, the engine_upgrade_* properties).** +- Assert that scored changes + within-tolerance + new exclusions == `len(engine_upgrade_revision['changed'])`. +- Add a Hypothesis property test over synthetic revisions and exclusion sets checking that the partition is exact and disjoint. hypothesis is a dev dependency. + +**Adjudication `judge_previous` dates (data fix).** +- The upgrade's refresh wrote each replaced judge class under `judge_previous` with a `judged_on` equal to the entry's `adjudicated_on`. For some entries that pairs claude-opus-5-5 with a date before Opus 5.5 judged anything, which is impossible. +- Fix it. Take each previous verdict's actual `judged_at_utc` from `/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/audit/cases//verdict.meta.json` when that verdict's classes match the recorded previous classes. Otherwise rename the field to `adjudicated_on`, which is what it holds. +- Add a test: every `judge_previous` date is on or after its judge model's release date (`MODEL_RELEASE_DATES` or the equivalent registry), and every current judge date is on or after the previous one. +- Also comment the judge-provenance test (older judges' counts fall because Opus 5.5 re-judged their cases after the reference revisions). +- Edit the staged record `results/local/adds0928-v3/publish/us_full_run_20260612_policyengine_4_16_1_populace/annotations/us_adjudications.json`. The committed `annotations/.../us_adjudications.json` is produced by the freeze. + +## Regenerating after data-adjacent fixes + +The adjudication fix and the manifest template fix change frozen artifacts, so rerun the documented chain after them, in this order: + +```bash +V=$PWD/results/local/adds0928-v3 +.venv/bin/python scripts/finish_adds0928.py --stage-dir $V --step triage +.venv/bin/python scripts/finish_adds0928.py --stage-dir $V --step export # keeps the 22c replay gate +.venv/bin/python scripts/freeze_adds0928.py --stage-dir $V --dry-run +.venv/bin/python scripts/freeze_adds0928.py --stage-dir $V +.venv/bin/python paper/render_paper.py && .venv/bin/python scripts/freeze_snapshot.py --rendered-only +``` + +If the payload hash changes (annotations feed the payload), report the new sha256. `app/src/data.artifact.json` must then match it, and the freeze updates it. + +## Done means + +- Every finding above is fixed, or explicitly answered in your report with evidence. +- `pytest -q` passes, the app checks pass (`bun run test`, `lint`, `build`), `ruff check .` and `ruff format --check .` pass, and the paper renders. +- Coherent commits, each ending with the `Co-Authored-By: Claude Opus 5.5 ` line. No push. +- Return, as your final answer: every change with file and finding, every pin moved (old → new → source), the final payload sha256, and the test and app results. diff --git a/docs/adds0928/brief_S3c_review_fixes.md b/docs/adds0928/brief_S3c_review_fixes.md new file mode 100644 index 00000000..064dc0af --- /dev/null +++ b/docs/adds0928/brief_S3c_review_fixes.md @@ -0,0 +1,87 @@ +# Stage 3c brief: exclude scenario_023 head_medicaid_eligible; harden the gates + +Work in `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2` (branch `adds-0928-stage2`, HEAD f7ced3b3), in place. The lead's pre-merge review of PR #182 confirmed one blocker and several minors. Evidence is in `docs/adds0928/review_pr182_2026-09-29.json`; read it first. + +Also read: +- `docs/adds0928/brief_S3b_review_fixes.md`, for the regeneration chain, the Voice rules and the other Rules; +- the repo `CLAUDE.md`; +- Max's voice guide `~/.claude/projects/-Users-maxghenis/memory/voice_max.md`. + +Python: `.venv/bin/python` with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. The machine is disk-starved and heavily loaded. Run the heavy steps serially, never make copies of large files, and delete any scratch you create. + +## The decision (already made; apply it) + +`scenario_023 head_medicaid_eligible` joins the exclusions as `reference_depends_on_unlisted_input`, with `unlisted_input` = `meets_ssi_disability_criteria`. This is the same input and reason that already exclude this household's SNAP. + +The basis is on the record: +- The excl_snap_ssi_disability investigation (`reference_audit/2026-09-28/clusters.json`) flags it out of cluster: "the lead should review it". +- Its independent review agrees (`verification/reviews/excl_snap_ssi_disability.md`). +- The investigation's own analysis supports it. The head's MAGI is 141.2% of FPL, above the 138% adult limit, so Medi-Cal is reachable only through a disability pathway. California's 250% Working Disabled Program requires the federal (SSA) definition of disability, 42 CFR 435.540(a), and that definition is the unlisted input. +- Reading A (the head does not meet the SSI/SSA disability criteria) gives 0. Reading B gives 1, which is the engine's value on 2.15.17, category WORKING_DISABLED_BUY_IN. +- Rule 4 of `reference_audit/2026-09-28/README.md` says such an output is excluded. + +Before recording anything, verify both values yourself on 2.15.17 with `fixes/latest_final.py`, using the probe approach in the review evidence. Record exactly what you computed. + +## 1. Record and apply the exclusion + +- **`reference_exclusions.json`.** Add the entry, following the scenario_023 SNAP entry's schema: + - `frozen_value` 1.0, `alternative_value` 0.0; + - `engine_version` "policyengine-us 2.15.17"; + - `decided_on` "2026-09-29"; + - `alternative_reading` and `note`, each stating what you computed and the legal basis. + - Exclusion entries decided before this wave stay unchanged. +- **Audit record (`reference_audit/2026-09-28/`).** + - Record the disposition of the out-of-cluster flag: a `reconciliations`-style entry in `clusters.json`, plus an `audit_exclusions` list in `final_actions.json` naming this output and its basis. + - Add `verification/reviews/pr182_review_023.md` with the review's reasoning and your computed values. + - Update `README.md`: counts (56 exclusions, 1,928 scored, 28 unlisted-input; recompute every count), a short "Also excluded on review" paragraph, and rule text if needed. + - The engine-upgrade revision's `changed` list stays exactly the engine changes. This exclusion is a separate, listed audit decision, not an engine change. +- **Case annotations.** The published case note for this output (`annotations/.../us_case_notes.csv`, the row that calls the 22 zero answers `llm_error` "None of them considered the employed-disabled buy-in") must not survive as a scored-error note. Let triage rebuild it the way it handles every other excluded output. Check the result. + +## 2. Harden the gates (review minors; each needs a failing-then-passing test) + +In `scripts/finish_adds0928.py`: +- **`reference_revision()`.** Also compare `impact_weight` (and every other column) against the 22c base. Any difference not listed is refused. +- **Exclusion set.** Require exact set equality in `resolve_base`, and after the freeze in `resolve_live_base`/`export`: the 22c exclusion keys (from BASE_COMMIT), plus the revision's newly excluded keys, plus the `audit_exclusions` keys in `final_actions.json`, and nothing else. A swapped or extra exclusion must be refused. Replace the count-only check (`52 + added`). +- **Replay gate.** Add a test where an incumbent's modelStats drift on the 22c references, and the export must refuse. The review showed that turning `replay_base_references` into a no-op leaves all 64 tests passing. +- **Drift attribution.** The drift report and every public claim that says "the only cause is the reviewed reference revision" must now name both causes: the revision and the one audit exclusion. The replay still proves incumbents reproduce on the 22c references. Word it accurately. + +In `tests/`: +- **`test_reference_upgrade.py:125`.** A 0/1 flag change of exactly 1 must not be exempt as "within $1". Match `paper_results.moves_beyond_tolerance`. +- **`test_disclosures.py`.** Split the paper-claim tests at lines 605 and 636 so the HTML assertions run in CI, and only the PDF part skips without pypdf/pdftotext. +- **Sweep-timing test (`test_reference_upgrade.py`, the pin-commit check).** After the squash merge the pin commit leaves main's history. Make the skip message say exactly that. Also assert, without git, that the recorded pin time precedes the sweep start and follows 2.15.17's upload, so the check keeps running. +- **Sidecar reasons.** The 19 `excluded_outputs_rechecked` reasons are cut at 600 characters. + - Store the reviewers' full `corrected_per_output` text instead, taken deterministically from `clusters.json`. + - Use the documented install path (`scripts/install_adds0929_references.py` accepts record-text-only changes), and fix the truncation in `build_references_latest.py`. + - Add a test that each recorded reason equals the full reviewer text. + - If regenerating the sidecar needs a full 1,984-output recompute on this machine, patch the text with a small committed script instead. The install check must still prove that only record text changed. + +## 3. Regenerate and update every public figure + +Run the chain in `brief_S3b_review_fixes.md` ("Regenerating after data-adjacent fixes"): +- triage, export (keeping the 22c replay gate), freeze dry-run, freeze; +- `paper/render_paper.py`; +- `freeze_snapshot.py --rendered-only`. + +Then update everything that states a now-stale figure, recomputing each value from the frozen artifacts: +- **The release note.** Recompute Sonnet 5.5's rank. If it is no longer 4th, rename the note file and its slug to match, e.g. `2026-09-29-claude-sonnet-5-5-debuts-fifth.json`, and update `app/src/notes/index.ts` and every reference. The title's ordinal is test-pinned to `facts['sonnetRank']`. Describe the gap to its neighbor as about level if it is under 0.1 points. +- **Other copy:** the BBCE note update (check whether anything moved), the paper, `docs/paper.md`, `docs/benchmark_card.md`, the app Methodology copy, `app/src/data.versions.json`, and the sensitivity docs. +- **Test pins:** recompute every one; never copy a failing number. + +Report the new payload sha256. The release asset will be replaced by the lead, not by you. + +## Rules + +- Do not push, upload, open PRs, or call any model or provider API. +- Make coherent commits, each ending with `Co-Authored-By: Claude Opus 5.5 `. +- **Done means:** + - `pytest -q -m "not slow"` passes. + - The app checks pass: `bun install --frozen-lockfile`, `bun run test`, `bun run lint`, `bun run build` in `app/`. + - `ruff check .` and `ruff format --check .` pass. + - The paper renders. +- **Final answer:** + - every change, with its file and finding; + - every moved pin, as old → new, with the source of the new value; + - the computed values for the exclusion; + - the new ranks of the three additions and of the top five; + - the new payload sha256; + - the test and app results, verbatim. diff --git a/docs/adds0928/report_S2_prep.md b/docs/adds0928/report_S2_prep.md new file mode 100644 index 00000000..06ddedc7 --- /dev/null +++ b/docs/adds0928/report_S2_prep.md @@ -0,0 +1,241 @@ +# September 28 stage 2 preparation + +Prepared in the assigned workspace only. No judge calls, worker changes, +publication, release upload, push, or merge were performed. + +## Design and deliverables + +- `scripts/finish_adds0928.py`: separate prepare/fold, Opus 5.5 judge, triage, + and strict export steps. All outputs stay under a scratch stage directory. +- `scripts/snapshot_adds0928.py`: copy completed scenario CSVs without reading + live supervisor state; produce explicitly synthetic rehearsal state. +- `scripts/freeze_adds0928.py`: local release-build adapter, gated on a strict + export receipt and complete serving evidence. Not executed during preparation. +- `docs/adds0928/stage2_design.md`: design, risks, exact rehearsal commands, + and the complete fold → judge → triage → export → release-build sequence. +- `tests/test_finish_adds0928.py`: completion, isolation, coverage, provenance, + adjudication, receipt, export, and snapshot-copy regression tests. + +The base is the committed September 22c snapshot: 42 models, 1,984 requested +outputs per model, 52 exclusions, and 1,932 scored outputs. Its country payload +recombines to live SHA256 +`01e7e72b3a6bdd2d3178ba32625ff769d5b81dc07541af6ea8da2c852774ddcc`. +The new release is planned as `dashboard-data-20260929`, with 45 models. + +Audit cases use the historical legacy-threshold miss selector, grouping all +wrong models by household/output. A new answer changes the prompt and invalidates +its copied verdict. Unchanged verdicts survive; missing-output-only cases need +no paid judge. New-model cases require schema-valid, complete verdicts with +hash-bound requested/reported `claude-opus-5-5` provenance. Reference flags and +nonfinal scored diagnoses block export until evidence-backed adjudication. + +## PARTIAL rehearsal + +The immutable CSV snapshot contains the following observations. Each rank below +compares one addition with the 42 incumbents on exactly that addition's copied +households, using the board's population-weighted household exact-match metric. +These are **PARTIAL rehearsal results**, not a released 45-model ranking. + +| Model | Copied households | Scored outputs | Exact match | Rank among 43 | +|---|---:|---:|---:|---:| +| Claude Sonnet 5.5 | 100 | 1,932 | 91.454010% | 4 | +| Grok 4.7 | 49 | 973 | 86.537884% | 12 | +| DeepSeek V4.1 Flash | 95 | 1,836 | 87.457366% | 13 | + +Sonnet has 100 valid household CSVs in this copy, but the CSV-only procedure +does not read or certify its actual supervisor completion state or treatment. +Completion order is not necessarily representative, so the smaller cohorts do +not estimate final ranks. + +The staged **45-model PARTIAL payload** uses the 49 households completed by +all three additions (992 requested outputs, 19 exclusions, 973 scored outputs): + +| Model | Exact match on common cohort | Rank among 45 | +|---|---:|---:| +| Claude Sonnet 5.5 | 91.650727% | 4 | +| Grok 4.7 | 86.537884% | 13 | +| DeepSeek V4.1 Flash | 86.318837% | 15 | + +The `--early --partial` command exited 0. The standard exporter and nonstrict +dashboard-schema gate passed. Output: +`results/local/adds0928-rehearsal/stage/PARTIAL-data-board45.json`. +There is no `release-ready.json`; no paid audit was executed. + +The ordinary `--early` invocation exited 1 with: + +```text +refusing incomplete additions: grok47: 49/100, stopped_reason=None; dsflash41: 95/100, stopped_reason=None +``` + +Only the scratch states were temporarily changed to `total=completed`, retaining +`synthetic_partial=true`. The rehearsal then used `--early --partial`. +The source-copy states were restored to `total=100` afterward; the stage keeps +its explicitly synthetic input copies and hashes for reproducibility. +Production mode additionally requires all 100 households, exact output-key +coverage and actual matching treatment fingerprints. Partial/early stages can +never resume into a release stage or obtain `release-ready.json`. + +Scratch evidence is under `results/local/adds0928-rehearsal/`: source copies in +`completed-csvs`, per-file hashes alongside those copies, `refusal.log`, +`dry-run.log`, and staged artifacts under `stage/`. Full original CSV transcripts +remain in the copies; PARTIAL scoring omits the unused repeated `raw_response` +column to reduce memory use on the shared machine. + +## Breakages and remaining release work + +- Fixed stale 39-model/September 5 driver constants by pinning September 22c + predictions, references, exclusions and live payload identity. +- Fixed the snapshot-copy assumption that filenames are household IDs: + `scenario_006.csv` can contain `scenario_007`; filenames are queue indices. +- Fixed country-payload recombination and canonical JSON serialization required + by the freezer. Its preflight now rejects incompatible bytes before mutation. +- Added key coverage checks beyond row counts, strict all-three completion, + scratch/source overlap protection, complete verdict coverage and Opus metadata + validation, and immutable evidence hashes across export and freeze. +- Added preservation of current judge classes in adjudications, matching + exclusion decisions, and regenerated case counts. Full exports must preserve + every incumbent model statistic, including Fable 5's historical batch usage. +- All three registrations, price overrides and response-cost/token fields are + present. Provider-reported cost fields are blank; recorded costs are reconstructed + from usage and prices. No model metadata or new-model cost-field blocker was found. +- The existing freezer omitted these three supervised runs. The adapter requires + their copied states and fingerprints and preserves incumbent serving evidence. + Actual new-run fingerprints remain unverified in this CSV-only rehearsal. +- DeepSeek responses report only `deepseek-flash`, with observed fingerprint + `aeb56401ca74e127821c4f9126dcb669`; they do not independently report V4.1. + Preserve the dated onboarding/provider-list evidence, response IDs, aliases, + fingerprints and timestamps. Do not claim immutable response-version proof. +- Keep the registered Grok release date, September 21: the September 2 value + in the older context is API-object creation, not public launch. +- The final note, manuscript prose, frozen roster pins, paper rendering and app + checks require the completed, judged 45-model payload. This prep does not + refreeze or change any published snapshot/pointer relative to the registration + branch. The existing paper roster/metadata test is expected to need refreezing. + +## Validation + +The targeted suite passed **207 tests**, including all 57 new driver/freezer +tests and the existing fold, audit, adjudication, reference provenance, model +card, judge provenance and cost/latency tests: + +```bash +OPENBLAS_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$PWD" \ + /Users/maxghenis/PolicyEngine/policybench/.venv/bin/python -m pytest -q \ + tests/test_finish_adds0928.py tests/test_fold_board.py tests/test_audit.py \ + tests/test_adjudications.py tests/test_reference_provenance.py \ + tests/test_model_cards.py tests/test_judge_provenance.py tests/test_cost_latency.py +``` + +Ruff formatting/checks passed for the three scripts and new test file. +The actual paid judge and full release build remain unexecuted, as required for +this preparation. Python 3.14 from the existing environment satisfies the +project's `>=3.10` requirement; imports use this workspace's code. + +The separately run +`tests/test_paper_results.py::test_frozen_roster_has_42_display_names_and_release_dates` +failed as expected: `MODEL_DISPLAY_NAMES` includes the three registered additions, +while the untouched frozen snapshot contains 42 models. This inherited mismatch +must be resolved by the final 45-model refreeze, not by changing the prep snapshot. +Its log is `results/local/adds0928-rehearsal/paper-roster.log`. + +## Git handoff + +The initial checkout was clean at `c7aaffe9bc6f`. The required registration base +is `05f472a`, the head of `add-claude-sonnet-5.5`. + +The managed sandbox denied creation of the linked worktree's `index.lock` +because its Git directory is in the caller's repository, outside the assigned +workspace. No approval mechanism is available. An independent Git directory +inside `results/local/stage2-history.git` holds the local `adds-0928-stage2` +branch based on `05f472a`; its object store reads existing objects without +writing the caller's repository. A commit bundle is supplied for recovery. + +Driver, design and tests commit: **`3a339d0`**, +`Prepare stage two for the September 28 model additions`. +The report is a separate coherent follow-up commit. The bundle is +`results/local/adds0928-stage2.bundle` and contains both commits with `05f472a` +as its prerequisite. + +To inspect that branch within this workspace: + +```bash +git --git-dir=results/local/stage2-history.git --work-tree=. log -3 --oneline +git --git-dir=results/local/stage2-history.git --work-tree=. status --short +``` + +No files were written to the requested external chief-of-staff state paths; +the design and this report are committed here instead. + +## Exact sequence once all three runs reach 100/100 + +Run from this assigned workspace. Check the remote pointer still names the +September 22c base before starting. All three runs must have `completed=total=100` +and null `stopped_reason`; the driver verifies these and copies their inputs. +Use a fresh stage directory, and run every heavy step sequentially. + +```bash +export OPENBLAS_NUM_THREADS=1 +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONPATH="$PWD" +export GIT_DIR="$PWD/results/local/stage2-history.git" +export GIT_WORK_TREE="$PWD" +PB_PY=/Users/maxghenis/PolicyEngine/policybench/.venv/bin/python +PB_SOURCE=/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609 +PB_AUDIT=/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit +PB_STAGE="$PWD/results/local/adds0928-final" +PB_RUN=us_full_run_20260612_policyengine_4_16_1_populace + +gh api 'repos/PolicyEngine/policybench/contents/app/src/data.artifact.json?ref=main' \ + --jq '.content | @base64d' + +# Fold, copy the audit, and prepare changed cases without making judge calls. +"$PB_PY" scripts/finish_adds0928.py --step prepare \ + --runs-root "$PB_SOURCE" --stage-dir "$PB_STAGE" \ + --audit-seed "$PB_AUDIT/audit" --grounding "$PB_AUDIT/grounding.csv" + +# Run this command inside the selected Claude subscription lane. +"$PB_PY" scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step judge + +"$PB_PY" scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step triage +``` + +If triage stops, inspect `$PB_STAGE/reference-flags.csv` and +`$PB_STAGE/unresolved-rows.csv`. Investigate and record evidence-backed decisions +in `$PB_STAGE/publish/$PB_RUN/annotations/us_adjudications.json`, preserving the +current judge's exact verdict, then rerun `triage`. A newly changed reference or +exclusion requires a reviewed reference revision; this additive driver refuses +such drift. Do not continue past a failed gate. + +```bash +"$PB_PY" scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step export +"$PB_PY" scripts/freeze_adds0928.py --stage-dir "$PB_STAGE" --dry-run +"$PB_PY" scripts/freeze_adds0928.py --stage-dir "$PB_STAGE" +"$PB_PY" scripts/sensitivity_by_variable.py +"$PB_PY" scripts/rescore_sensitivity_summaries.py \ + --release dashboard-data-20260929 +``` + +At this point, write the September 28 note from the final results and update the +paper, benchmark card, methodology/model-count prose, release-specific test +pins, and alias/treatment disclosures. This editorial step depends on actual +final scores and any investigated flags. Preserve historical notes' own pins. +Then run: + +```bash +"$PB_PY" paper/render_paper.py +"$PB_PY" scripts/freeze_snapshot.py --rendered-only +"$PB_PY" -m ruff format . +"$PB_PY" -m ruff check . +"$PB_PY" -m ruff format --check . +"$PB_PY" -m pytest -q +bun install --cwd app --frozen-lockfile +(cd app && bun run test) +(cd app && bun run lint) +(cd app && bun run build) +git diff --check +``` + +Review and commit the resulting local release build. These commands stop before +upload, push or merge. In an ordinary checkout recovered from the bundle, omit +the two workspace-local `GIT_DIR`/`GIT_WORK_TREE` exports and use its normal Git +metadata. See the design note for the copy-only rehearsal and full risk analysis. diff --git a/docs/adds0928/report_S3.md b/docs/adds0928/report_S3.md new file mode 100644 index 00000000..d9067565 --- /dev/null +++ b/docs/adds0928/report_S3.md @@ -0,0 +1,123 @@ +# Stage 3 report: prose, notes, paper and pins for release dashboard-data-20260929 + +Written by the stage-3 implementer (Claude Code workflow wf_c63fb075-f87, Opus 5.5), then a voice pass. The harness kept the implementer from writing this file, so the lead recorded its returned report here. + +## Commits + +- b3ef6247 Pin release 20260929's counts in the tests, sensitivity doc and app +- e615e5e8 State the 45-model board and the policyengine-us 2.15.17 references in the card and app copy +- 0833778f Update the paper for the 45-model board on policyengine-us 2.15.17, render and re-pin it +- 10edbe9c Add the September 29 release note and update the BBCE note for release 20260929 +- 7e192f6d Put the actor first in the paper's engine paragraph and the card's engine sentence; re-render and re-pin +- 756ce4f2 Voice pass on the release-20260929 prose; re-render and re-pin the paper + +## Files changed + +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fourth.json (new release note) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/notes/index.ts +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/notes/2026-09-23-five-snap-households-bbce.json (dated update: 2 paragraphs, 10 update* facts, 4 links; 20260922b figures unchanged) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/notes/data/bbce_households_20260929.csv (+ .meta.json; 45 models x 5 income-held households incl. AZ 013) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/notes/data/bbce_asset_households_20260929.csv (+ .meta.json; 45 x 4 savings-held households) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/scripts/bbce_households_20260929.py (new; writes both CSVs) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/paper/index.qmd +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/policybench/paper_results.py (engine_upgrade_* properties, previous_policyengine_us_version, reference_rebuilt_date; regenerated_reference_* restricted to convention/upstream_fix; dataset_* read from the scenario draw) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/public/paper/policybench.pdf, app/public/paper/web/index.html, app/public/paper/web/figures/positive_zero_scatter.png (render) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/paper/snapshot/20260501/manifest.json (rendered_paper_artifacts only, via --rendered-only) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/paperSnapshot.json (re-pin) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/docs/benchmark_card.md (new Reference outputs section; audit/exclusion/adjudication counts; snapshot date; Sonnet 5.5 and DeepSeek V4.1 Flash in the rejects-forced-tool list) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/docs/paper.md +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/docs/artifacts.md +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/sensitivity/claude-thinking-2026-08.md +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/components/Methodology.tsx (Ten of the 45; engine version read from the board payload) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/components/ModelLeaderboard.tsx +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/data.versions.json (description: Reference outputs from policyengine-us 2.15.17) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/lib/servingSensitivity.ts +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/src/model-serving-config.json (refreshed from frozen file by prepare-data) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_adjudications.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_disclosures.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_paper_results.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_report_costs.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_sensitivity_evidence.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_snapshot_artifacts.py +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/tests/test_notes.py (release-note test pinning every sentence; drift-baseline rebuild test vs git cb312fd7; BBCE update test) +- /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/app/tests/auditUniverse.test.ts, boardScope.test.ts, dataVersions.test.ts, expandPage.test.ts, heatmapMetric.test.ts, programChart.test.ts, servingSensitivity.test.ts + +## Test pins changed (old → new; source) + +- tests/test_adjudications.py::test_committed_record_is_applied_to_the_frozen_annotations (entries): 63 → 68 (annotations/.../us_adjudications.json) +- test_adjudications (adjudicated classes prompt_ambiguity/llm_error/engine_defect): 24/11/28 → 27/13/28 (us_adjudications.json; new check: llm_error by reference_verdict = affirmed 5, regenerated 6, none 2 (007 federal, 008 NJ later-law resolutions)) +- test_adjudications (excluded_from_scoring entries; manifest reference_exclusions.outputs): 52; 52 → 55; 55 (us_adjudications.json; manifest) +- test_adjudications (manifest developer_adjudications.cases / by_judge_verdict): 63 / llm_error 44, prompt_ambiguity 2, reference_engine_defect 11, reference_model_issue_fixed 6 → 68 / llm_error 49, prompt_ambiguity 2, reference_data_issue_fixed 1, reference_model_issue_fixed 16 (paper/snapshot/20260501/manifest.json (equals entries' judge classes)) +- tests/test_paper_results.py::test_frozen_roster_has_45_display_names_and_release_dates (renamed from _42_): 42 → 45 (+ Sonnet 5.5 2026-09-28, Grok 4.7 2026-09-21, DeepSeek V4.1 Flash 2026-09-10) (payload modelStats; policybench/paper_results.py registry) +- test_paper_results::test_parse_contract_failure_counts_come_from_frozen_dashboard: kimi-k2.6 394; total 657 → kimi-k2.6 391; total 654 (payload rows on scored outputs (3 newly excluded outputs leave the count)) +- test_paper_results::test_audit_universe_counts_come_from_frozen_rows_and_annotations: 7,545 / 7,541 / 7,541 / 4 / 1,843 → 7,796 / 7,792 / 7,792 / 4 / 2,027 (payload + us_audit_row_annotations.csv) +- test_paper_results::test_contract_violations_are_counted_both_ways: grok-4.3 56; 61; 718 → grok-4.3 55; 60; 714 (payload) +- test_paper_results::test_serving_evidence_caption_comes_from_frozen_configuration: 13/13/12/13; three rows → 16/16/15/16; six rows; summary registry 29, run_state 16 (paper/snapshot/20260501/model_serving_config.json) +- test_paper_results::test_serving_evidence_counts_exclude_legacy_or_unrecorded_fields: 12 -> 13; 12 → 15 -> 16; 15 (model_serving_config.json) +- test_paper_results::test_joint_credit_accuracy_exceptions_come_from_frozen_table: Astra [100,89.7,89.7], GPT-6 Sol [99,88.7,88.7], GPT-5.6 Sol [99,87.6,87.6]; 5 exceptions → Astra [100,88.7,88.7], GPT-6 Sol [99,87.6,87.6], Grok 4.7 [99,87.6,87.6], GPT-5.6 Sol [99,86.6,86.6]; 6 exceptions (Grok 4.7 added) (payload federal/state refundable credit predictions) +- test_paper_results::test_joint_credit_accuracy_prose_tracks_changed_table_exceptions: 4 exceptions → 5 exceptions incl. Grok 4.7 (same table) +- test_paper_results::test_judge_provenance_is_frozen_in_the_manifest: opus-5 183, opus-5-5 171, sol 314, total 668 → opus-5 132, opus-5-5 239 (judged_on includes 2026-09-29), sol 303, total 674 (manifest audit_annotation_artifacts.judge_provenance) +- test_paper_results::test_joint_credit_table_orders_ties_deterministically: 87.6 tie: GPT-5.6 Sol, GPT-6 Luna → 88.7 tie: GPT-6 Astra, GPT-6 Luna; 87.6 tie: GPT-6 Sol, Grok 4.7 (joint table, ties broken by model id) +- test_paper_results::test_excluded_outputs_are_outside_the_scored_audit_universe: 52 / '52 outputs' / '36 households' / unlisted 24 / 1,932 / 1793 / 616 / n 1932 → 55 / '55 outputs' / '39 households' / unlisted 27 / 1,929 / 2041 / 780 / n 1929; SALT-refund unlisted input 3; regenerated 26 kept (household 24, SNAP 13 added) (reference_exclusions.json, annotations, payload, sidecar convention+upstream_fix revisions) +- test_paper_results::test_engine_upgrade_counts_come_from_the_reference_sidecar (new): - → 2.15.17 from 1.755.4; date 2026-09-28; scored changes 4; within tolerance 2; new exclusions 3; rechecked 19; NJ 5342.40->5842.40, AZ 0->240, PA 1->0, NY 650.5->667 (reference_outputs.csv.meta.json engine_upgrade revision) +- test_paper_results::test_dataset_build_is_the_one_the_households_came_from (new): - → populace-us-2024-5da5a95-20260611 (scenarios.csv.meta.json and population_weights.json) +- tests/test_snapshot_artifacts.py::test_snapshot_deviation_audit_annotations_are_complete_and_final: 7,545/7,541/7,541/4/9,388/1,843; llm_error 6,888, parse 657 → 7,796/7,792/7,792/4/9,823/2,027; llm_error 7,142, parse 654 (payload + annotations) +- tests/test_disclosures.py::test_audit_disclosures_use_the_frozen_legacy_threshold_universe: 7,545/7,541/7,541/4/1,843 → 7,796/7,792/7,792/4/2,027 (payload + annotations) +- tests/test_report_costs.py::test_frozen_report_discloses_recorded_subtotal_and_published_total: 39 / 42 / 416.523 / 480.602 → 42 / 45 / 448.339 / 512.418 (analysis/usage_summary.csv, payload modelStats; lines already in frozen analysis/report.md) +- tests/test_sensitivity_evidence.py::test_doc_table_ranks_match_the_frozen_45_model_board (renamed): '42-model board (2026-09-22)', 42 rows → '45-model board (2026-09-29)', 45 rows (payload; doc table values from rescored sensitivity/data/*.json) +- tests/test_notes.py CURRENT_RELEASE_SNAPSHOT / SUPERSEDED_RELEASES: 2026-09-22 / (no 22c) → 2026-09-29 / + dashboard-data-20260922c (manifest snapshot_date) +- app/tests/auditUniverse.test.ts: 7,545/7,545/7,541/7,541/4/1,843 → 7,796/7,796/7,792/7,792/4/2,027 (app data summary from frozen payload) +- app/tests/boardScope.test.ts: Ten of the 42 models → Ten of the 45 models; + engine sentence 'PolicyEngine-US (policyengine-us 2.15.17) computes' (serving config (10 chunked of 45); payload policyengineBundles) +- app/tests/dataVersions.test.ts: Snapshot 2026-09-22 → Snapshot 2026-09-29 (app/src/data.versions.json (written by freezer)) +- app/tests/expandPage.test.ts: leader 94.278; '94.3%' → 94.813; '94.8%' (data summary) +- app/tests/heatmapMetric.test.ts: Fable 5.1 federal exact 76 → 79 (data summary heatmap (82 scored federal outputs)) +- app/tests/programChart.test.ts: 76.5; width 76.47058823529412% → 79.3; width 79.26829268292683% (data summary) +- app/tests/servingSensitivity.test.ts: rank 6; 'auto 90.9 · #6'; '+7.4 against its 83.5%'; Fable 5.1 'auto 91.1 · #6'; delta +7.4 → rank 7; 'auto 91.4 · #7'; '+7.5 against its 83.9%'; 'auto 91.6 · #7'; delta +7.5 (rescored sensitivity summaries (servingSensitivity.ts autoExact 91.414/89.905/84.695/91.646)) + +## Results + +- pytest (implementer): 955 passed, 6 skipped, 11 warnings in 67.26s (0:01:07) +- app checks: bun install --frozen-lockfile: ok (no changes); bun run test: 149 pass, 0 fail; bun run lint: exit 0 (eslint --max-warnings=0); bun run build: exit 0, compiled; the new note page /notes/2026-09-29-claude-sonnet-5-5-debuts-fourth prerenders. Also ruff format --check (103 files formatted), ruff check (all passed), git diff --check clean. Final paper PDF sha256 69a9679a32a217669cb06aec46309fd067b8ba21288055d37216237d25499935 (web index.html key 4bb46077... replaced by the final render; manifest and paperSnapshot.json re-pinned). +- paper rendered: True +- pytest (after voice pass): 955 passed, 6 skipped, 11 warnings in 108.28s (after re-render and re-pin; OPENBLAS_NUM_THREADS=1, PYTHONPATH=$PWD, worktree .venv). App: bun test tests shows 149 passed, 0 failed. ruff format and check are clean on tests/test_notes.py. Left unchanged, and outside a wording-only pass: docs/paper.md still says reference_output_refresh carries "the certified US populace dataset's build id". However, the implementer's own comment in policybench/paper_results.py says that block records the reference runtime's default dataset (populace-us-2024-spm-20260915), which can differ from the certified build the households were sampled from. That fact needs checking. Commit 756ce4f2 on adds-0928-stage2 is not pushed. + +## Voice pass edits + +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: superlative / adjective without a number +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: sentence a reader would read twice +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: passive voice / inanimate actor where PolicyBench acts +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: passive participle where an actor exists +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: sentence a reader would read twice (double 'from', dangling 'which') +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: name the actor and action (actorless abstraction) +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: split a 50-word sentence; passive voice where an actor exists +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: sentence a reader would read twice (the colon list was attached to the wrong clause) +- 2026-09-29-claude-sonnet-5-5-debuts-fourth.json: be-verb identity replaced by a real verb; name the actor +- 2026-09-23-five-snap-households-bbce.json: sentence a reader would read twice +- 2026-09-23-five-snap-households-bbce.json: split a sentence a reader would read twice; be-verb replaced with a verb +- index.qmd: passive voice where an actor exists; sentence a reader would read twice +- index.qmd: superlative in a label; passive chain; sentence starting with a numeral +- index.qmd: inanimate actor replaced by the real actor +- index.qmd: sentences a reader would read twice (garden path 'read from 40 hours', stacked relative clauses); passive voice +- index.qmd: passive voice where an actor exists; inanimate actor +- index.qmd: passive voice where an actor exists; sentence a reader would read twice +- benchmark_card.md: passive voice; be-verb identity; ambiguous 'it' +- benchmark_card.md: sentence a reader would read twice (a list of causes followed a colon after 'references') +- benchmark_card.md: sentence a reader would read twice (an internal class label used as prose) +- benchmark_card.md: inanimate actor; passive voice where an actor exists +- paper.md: passive participle where an actor exists +- test_notes.py: pinned phrase updated to carry the same fact + +## Open problems the implementer reported + +- docs/adds0928/report_S3.md is NOT committed. The harness refused the write ('Subagents should return findings as text, not write report files'), so this output carries the report: all files, every pin old->new with its source, and the results. The orchestrator should write and commit it if the release needs that file. +- Where the brief differs from the frozen artifacts (the artifacts win). (1) Incumbent drift is +0.3621 (Kimi K2.6) to +0.5349 (GPT-6 Sol). That rounds to 0.36 to 0.53, not the brief's 0.54, and the note says 0.53. (2) Among the 42 incumbents, only three neighbor pairs swap, at positions 16/17, 25/26 and 31/32: Opus 5 over Fable 5, Gemini 3 Flash Preview over Opus 4.7, and DeepSeek V4 Pro over Gemini 3.1 Flash Lite Preview. So six models change rank, which matches the brief, and the top 15 keep their order, not only the top eight. The note names the three swaps. (3) With the engine-upgrade revision in the sidecar, paper_results.regenerated_reference_count would have become 34. I restricted it to the September 22 convention and upstream-fix revisions (26) and added engine_upgrade_* properties. Every other staged figure checked out: Sonnet 91.973 (#4, 0.0155 above Luna 91.958), Grok 88.228 (#12), Flash 87.274 (#15), the top three, the costs ($0.0343, $0.2597, $0.0242) and all the BBCE figures. +- Frozen manifest mechanism error, left unfixed. The reproducibility note says 'Reference outputs were generated with policyengine.py 6.1.2 and policyengine-us 2.15.17 against the certified PolicyEngine US populace dataset (populace-us-2024-spm-20260915...)'. In fact the references come from policyengine_us.Simulation on each household's own inputs, and policyengine.py is recorded only for provenance. The sentence comes from the build_manifest template in scripts/freeze_snapshot.py, and correcting it needs that template fixed plus a full refreeze. I did not edit the frozen manifest. +- Dataset build mismatch. The manifest's reference_output_refresh records populace-us-2024-spm-20260915, which is the policyengine.py 6.x bundle's default dataset. The households and the population weights come from populace-us-2024-5da5a95-20260611. paper_results.dataset_* now read the scenario draw's build, and a new test pins this. The manifest field itself is unchanged. +- Pre-existing paper render bug, not fixed. The cost-and-latency sentence renders a literal '\\\\$0.002' and the cost table renders '\$0.034', in both the HTML and the PDF. The previous render has the same problem. +- DeepSeek alias provenance review is still open. The stage-2 design asks for a substantive review before release. Every one of the row's 1,984 answers reports only 'deepseek-flash' and one system fingerprint (aeb56401ca74e127821c4f9126dcb669). The release note discloses this and labels the row from DeepSeek's September 10 release note. +- The release note and the BBCE update link the tag dashboard-data-20260929, which won't resolve until the release assets are uploaded. Upload stays outside this stage. +- Judgment calls a reviewer should confirm. (a) 'Across development and five audit waves': I raised it from four because the September 28 additions had their own judge wave, the same way PR #174 went from three to four. (b) The benchmark card now leaves out the September 22 regenerated-reference counts and the 22b/22c release history, following the fresh-introduction voice rule; the paper keeps that history. +- Environment change: I installed the project's locked dev and docs extras into the worktree .venv (uv sync --frozen --extra dev --extra docs --inexact) so Quarto could run Jupyter. policyengine-us stayed at 2.15.17 and policyengine at 6.1.2. prepare-data also downloaded the June 1.0 payload into app/.cache, which is untracked. +- On 'using my voice?': before writing any prose I read voice_max.md (current rules and standing bans) and user_max_working_model.md. I applied them to every string I added or changed: actor-first active voice, numbers over adjectives, no empty antithesis (Sonnet vs Luna is stated as '#4... GPT-6 Luna, #5, also rounds to 92.0%, less than 0.02 points behind'), no self-reference, sentence-case title, change-as-subject for the release and BBCE notes, and present-state wording in the paper and card. I also did one full-document pass. No independent Opus or Astra voice or fact review has run yet, and the CLAUDE.md rule calls for one before merge. + +Two adversarial reviews (facts; tests) followed; their findings are consolidated in brief_S3b_review_fixes.md. diff --git a/docs/adds0928/review_evidence/probe023.py b/docs/adds0928/review_evidence/probe023.py new file mode 100644 index 00000000..9f6635cf --- /dev/null +++ b/docs/adds0928/review_evidence/probe023.py @@ -0,0 +1,51 @@ +import json, copy, importlib.util, sys +from pathlib import Path +import pandas as pd +WT = Path("/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2") +from policybench.ground_truth import _extract_person_value, _pe_variable_for_output +from policybench.scenarios import scenario_from_dict +from policyengine_us import CountryTaxBenefitSystem, Simulation +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +spec = importlib.util.spec_from_file_location("lf", Path("/private/tmp/claude-501/-Users-maxghenis-Library-Application-Support-Claude-scratch-workspaces-ee3763f4-e7ea-4177-bcaf-1362266768c1-4457aa9d-2dc0-4bcf-a128-e50adae6be25-scratch-2026-09-22-c3da5e/3dd3d124-a1eb-44ed-ac89-ce3df9700101/scratchpad/fixes/latest_final.py")) +lf = importlib.util.module_from_spec(spec); sys.modules["lf"]=lf; spec.loader.exec_module(lf) + +sc = pd.read_csv(WT/"paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/scenarios.csv") +row = sc[sc.scenario_id=="scenario_023"].iloc[0] +scenario = scenario_from_dict(json.loads(row["scenario_json"])) +base = scenario.to_pe_household() + +class wdp_ssa(Variable): + pass + +class ca_wdp_disability_eligible(Variable): + value_type = bool + entity = Person + label = "WDP disability per SSA definition (pre-SGA)" + definition_period = YEAR + defined_for = StateCode.CA + def formula(person, period, parameters): + return person("meets_ssi_disability_criteria", period) | person("is_blind", period) | (person("social_security_disability", period) > 0) + +class law_reform(Reform): + def apply(self): + lf.reform.apply(self) + self.update_variable(ca_wdp_disability_eligible) + +def run(label, system, ssi_crit): + sit = copy.deepcopy(base) + for pid, p in sit["people"].items(): + p["meets_ssi_disability_criteria"] = {"2026": ssi_crit} + sim = Simulation(tax_benefit_system=system, situation=sit) + pe_var = _pe_variable_for_output("head_medicaid_eligible", "us") + v = float(_extract_person_value(sim.calculate(pe_var, 2026), scenario, "head_medicaid_eligible")) + cat = sim.calculate("medicaid_category", 2026) + print(label, "pe_var=", pe_var, "head_medicaid_eligible=", v, "category=", list(cat), "ca_wdp_eligible=", list(sim.calculate("ca_wdp_eligible", 2026))) + +ref_sys = CountryTaxBenefitSystem(reform=lf.reform) +law_sys = CountryTaxBenefitSystem(reform=law_reform) +run("latest_final, readingA (ssi_crit False):", ref_sys, False) +run("latest_final, readingB (ssi_crit True): ", ref_sys, True) +run("WDP-per-SSA, readingA:", law_sys, False) +run("WDP-per-SSA, readingB:", law_sys, True) diff --git a/docs/adds0928/review_evidence/rescore023.py b/docs/adds0928/review_evidence/rescore023.py new file mode 100644 index 00000000..a6bf9200 --- /dev/null +++ b/docs/adds0928/review_evidence/rescore023.py @@ -0,0 +1,32 @@ +import pandas as pd, json, warnings +warnings.filterwarnings("ignore") +from policybench.analysis import weighted_hit_rate_scores_by_model +S='paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/' +r=pd.read_csv(S+'reference_outputs.csv') +ex={(x['scenario_id'],x['variable']) for x in json.load(open(S+'reference_exclusions.json'))['exclusions']} +gt=r[[ (a,b) not in ex for a,b in zip(r.scenario_id,r.variable)]].copy() +p=pd.read_csv(S+'predictions.csv.gz', low_memory=False) +def score(g): + s=weighted_hit_rate_scores_by_model(g,p,{},country='us').set_index('model')['weighted_exact']*100 + return s +base=score(gt) +d=json.load(open('results/local/adds0928-v3/data-board45.json')) +ms=d['countries']['us'].get('modelStats') +pub={m['model']:m for m in ms} if isinstance(ms,list) else ms +k=next(iter(pub.values())) +print('modelStats keys', list(k.keys())[:20]) +key=[c for c in k if 'exact' in c.lower()] +print(key) +mask=(gt.scenario_id=='scenario_023')&(gt.variable=='head_medicaid_eligible') +excl=score(gt[~mask]) +g0=gt.copy(); g0.loc[mask,'value']=0.0 +zero=score(g0) +out=pd.DataFrame({'base':base,'excl023':excl,'ref0':zero}) +if key: + out['published']=pd.Series({m:v[key[0]] for m,v in pub.items()}) +out['rank_base']=out.base.rank(ascending=False,method='min') +out['rank_excl']=out.excl023.rank(ascending=False,method='min') +out['rank_ref0']=out.ref0.rank(ascending=False,method='min') +pd.set_option('display.width',200) +print(out.sort_values('base',ascending=False).round(3).to_string()) +print('delta excl range', (out.excl023-out.base).min(), (out.excl023-out.base).max()) diff --git a/docs/adds0928/review_pr182_2026-09-29.json b/docs/adds0928/review_pr182_2026-09-29.json new file mode 100644 index 00000000..73a5b32c --- /dev/null +++ b/docs/adds0928/review_pr182_2026-09-29.json @@ -0,0 +1,116 @@ +{ + "approve": false, + "confirmed": [ + { + "severity": "major", + "title": "Wave's own investigator and reviewer flagged scored scenario_023 head_medicaid_eligible as resting on the unlisted-input reading; no disposition recorded", + "file": "reference_audit/2026-09-28/clusters.json", + "line": 1396, + "evidence": "The excl_snap_ssi_disability investigation summary (clusters.json:1396) says: \"Out of cluster: the scored scenario_023 head_medicaid_eligible (1) follows reading B, because ca_wdp_disability_eligible reads the broad is_disabled flag ... Under reading A the law gives 0 (42 CFR 435.540(a)), so the lead should review it.\" The independent review (clusters.json:1470, verification/reviews/excl_snap_ssi_disability.md:9) says the engine puts the head in WORKING_DISABLED_BUY_IN and that \"This strengthens the investigator's out-of-cluster flag.\" final_actions.json has no entry for this output, clusters.json `reconciliations` covers only 078 federal, and grep of README.md and docs/adds0928/*.md turns up no disposition. The output is still scored: reference_outputs.csv:379 has `scenario_023,head_medicaid_eligible,1.0,6612.93994140625`. The same household's SNAP is excluded for exactly this ambiguity (reference_exclusions.json: scenario_023 snap, unlisted_input meets_ssi_disability_criteria). The prompt input is just `is_disabled: true` (scenarios.csv; prompts.py:106 renders it as \"is disabled\"). In predictions.csv.gz, 22 of 45 models answer 0 and are scored wrong, among them gpt-6-sol, claude-opus-5.5 and the new claude-sonnet-5.5 and grok-4.7; 21 answer 1. us_case_notes.csv:171 publishes the misses as llm_error (\"None of them considered the employed-disabled buy-in\").", + "failure_scenario": "The release publishes exact rates (with this 0/1 output weighted by $6,612.94) that count 22 models wrong for giving the value the benchmark itself treats as the stated-facts reading for this household's SNAP. The public case note calls those answers model errors. The PR's audit record, which the PR body describes as each cluster fully reviewed, leaves an explicit reviewer flag on a scored reference unresolved. The value predates this PR, but this PR re-publishes it and scores the three new models on it.", + "verdict": { + "real": true, + "severity": "blocker", + "reasoning": "I could not refute this. I reproduced every part of it, and the effect on the release is larger than the finding says.\n\n1. **The flag exists and nothing disposes of it.**\n - clusters.json:1396 has the investigator's \"Out of cluster ... the lead should review it\".\n - clusters.json:1470 (also verification/reviews/excl_snap_ssi_disability.md:9) has the reviewer's \"for the lead ... strengthens the investigator's out-of-cluster flag\".\n - final_actions.json has four keys: engine, approved, new_exclusions, excluded_rechecked. Its only scenario_023 entry is the SNAP recheck.\n - clusters.json `reconciliations` holds only 078 federal.\n - `git grep` at f7ced3b3 for \"out-of-cluster|Out of cluster|for the lead|lead should|435.540\" finds only those two clusters.json lines and their review file. Nothing in docs/adds0928 or README.md addresses the flag.\n - sweep_moves.csv:372 and verification/latest_final_2170.csv:374 / 2171.csv:374 all have the output at 1.0 with moved=False. Nothing re-examined it.\n\n2. **The reference follows the benchmark's own alternative reading.**\n - The engine rule (2.15.17, ca_wdp_disability_eligible.py) is `is_disabled | is_blind | ssdi>0`. It never reads meets_ssi_disability_criteria.\n - The legal rule differs. LA DPSS 250% WDP policy, which the engine itself cites, requires the person to \"Meet the federal definition of disability for Social Security disability programs\", with no SGA screen. 42 CFR 435.540(a) likewise requires \"the same definition of disability as used under SSI\". That is exactly meets_ssi_disability_criteria, which the engine describes as \"before the substantial gainful activity screen\".\n - I ran a probe on latest_final. I copied the fixes to the scratchpad because the committed 2026-09-28/fixes/latest_c_irs_sales_tax_2025.py loads r19_irs_sales_tax_2025.json, which exists only under 2026-09-22/fixes.\n - Reading A: head_medicaid_eligible=1, category 11 (WORKING_DISABLED_BUY_IN).\n - Reading B: 1, category 8 (SENIOR_OR_DISABLED).\n - With WDP disability made to follow the SSA definition: reading A gives **0** (category NONE); reading B gives 1.\n - So under law the output turns on the same unlisted input that excludes this household's SNAP (reference_exclusions.json: scenario_023 snap, meets_ssi_disability_criteria).\n - The published SSI case note for the same household (us_case_notes.csv:179) says \"The head's unlisted SSI disability status is false under the prompt instructions\". That contradicts the Medicaid reference of 1 and case note 171.\n - The engine-input sweep of 2026-09-05 could not catch this, because the engine's WDP ignores meets_ssi_disability_criteria.\n\n3. **Predictions match the finding.** In paper/snapshot/.../predictions.csv.gz for scenario_023 head_medicaid_eligible, 22 of 45 models answer 0, 21 answer 1 and 2 give no answer. The models answering 0 include gpt-6-sol, claude-opus-5.5, claude-sonnet-5.5 and grok-4.7. us_case_notes.csv:171 labels 24 answers as llm_error (22 zeros plus the 2 missing), and it was re-authored in this PR (blame 4cf19978, this PR's freeze commit). It asserts that disability plus work alone places the head in the buy-in, which the DPSS text contradicts.\n\n4. **Why blocker rather than major.** I rescored with policybench.analysis.weighted_hit_rate_scores_by_model(country='us') on the snapshot's scored references and predictions. This reproduces every published data-board45 modelStats `exact` value exactly.\n - I then applied the benchmark's own remedy for unlisted-input dependence and excluded this one output, as the household's SNAP already is. That moves exact rates by up to \u00b10.29 points.\n - It also flips the PR's headline claim. Claude Sonnet 5.5 falls to 5th (92.081) behind GPT-6 Luna (92.097). Today the release note is titled \"Claude Sonnet 5.5 debuts fourth\" (app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fourth.json), and the PR body lists rank 4 and says it \"sits 0.015 points above GPT-6 Luna\".\n - Other ranks move too: inkling goes from 10 to 12, and deepseek-v4-pro-0813 from 23 to 21.\n - If the reference were set to 0 (the reading-A law value) instead, Sonnet would stay 4th. So the public rank claim hangs on an unresolved reviewer flag about a scored reference that, by the benchmark's own rules, should be excluded.\n\n**Caveats.**\n- The value (1.0) predates this PR (board 20260922c). This PR still re-publishes it, scores the three new models on it, and makes a new rank claim that depends on it.\n- The finding overstates the PR body slightly. The body says each cluster had \"an investigation and an independent adversarial review\" and that the one conflict is reconciled. It does not say every flag was resolved.\n\n**Evidence files:** /private/tmp/claude-501/-Users-maxghenis-Library-Application-Support-Claude-scratch-workspaces-ee3763f4-e7ea-4177-bcaf-1362266768c1-4457aa9d-2dc0-4bcf-a128-e50adae6be25-scratch-2026-09-22-c3da5e/3dd3d124-a1eb-44ed-ac89-ce3df9700101/scratchpad/probe023.py and rescore023.py." + } + } + ], + "refuted": [], + "minors": [ + { + "severity": "minor", + "title": "reference_revision() only compares the `value` column; unlisted impact_weight and exclusion-set changes get through", + "file": "scripts/finish_adds0928.py", + "line": 689, + "evidence": "The docstring (lines 662-664) says the snapshot may differ from 22c 'only by one committed engine_upgrade revision whose `changed` list is exactly the set of outputs whose value differs ... Anything else is refused'. The PR's Invariants section makes the same claim. The code only diffs `set_index(KEY)[\"value\"]` (lines 684-698). It never compares the CSV's impact_weight column (analysis.py:262/742/900/962 read it for weighting) or reference_exclusions.json against the 22c base or the revision. Exclusions get a count check only (`52 + added`, lines 223-235), and only in resolve_base, which runs before the freeze. The post-freeze path, resolve_live_base (lines 270-283), has no exclusion check at all. I ran a scratch script (/private/tmp/claude-501/rev182/gap.py) that builds a snapshot whose revision lists one value change but also changes an unlisted impact_weight from 1.0 to 5000.0 and swaps the base exclusion for a different one. Output: `reference_revision accepted: t listed changes: 1`. The committed data is clean. impact_weight has 0 diffs against `git show 3220a7a6:.../reference_outputs.csv`. The exclusions are the 22c 52 plus exactly the 3 keys the revision lists with cause excluded_reference_depends_on_unlisted_input (none removed, none modified). Earlier sidecar revisions 1-17 are unchanged.", + "failure_scenario": "A later rebuild or re-install changes a boolean impact_weight, or swaps one exclusion for another, alongside a valid engine_upgrade revision. reference_revision() accepts it. The count check still passes before the freeze and is absent after it. The replay then credits the resulting incumbent modelStats drift to 'the reviewed reference revision', and the export publishes it with the PR's claim that the revision is the only cause." + }, + { + "severity": "minor", + "title": "The incumbent replay gate behind the 'only cause is the revision' claim has no test; disabling it leaves all 64 tests passing", + "file": "tests/test_finish_adds0928.py", + "line": 449, + "evidence": "replay_base_references (scripts/finish_adds0928.py:702-728) is the only check behind the PR body's public claim: 'The only cause is the reviewed reference revision: re-exported on the 22c references, every incumbent reproduces its live modelStats exactly (the export gate)'. The PR also lists it under Invariants. No test calls it with a drifting incumbent. Mutation check: a pytest plugin that replaces driver.replay_base_references with a no-op still gives `64 passed` on tests/test_finish_adds0928.py. The only strict-export test (test_strict_export_recombines_to_freeze_bytes_and_binds_evidence) calls the real reference_revision() on the committed snapshot. That snapshot now carries the revision, so export() takes the revision branch: running the test writes stage/replay-20260922c and stage/incumbent-drift.json under basetemp. Its stubbed export_full_run returns the same `payload` object for both the real export and the replay, so the replay comparison always passes. As a result, the no-revision gate `require(not changed, \"incumbent modelStats drift\")` (line 751) is no longer run by any test. The claim itself holds on the real artifacts. The replay's data.json (results/local/adds0928-v3/replay-20260922c/.../data.json) matches `git show 3220a7a6:.../data.json.gz` for all 42 incumbents' modelStats (Fable's 4 usage fields patched) and heatmap rows. So no number is wrong today.", + "failure_scenario": "A future edit breaks the replay: it compares only new models, skips writing the base references, or turns drift into a warning. Every test still passes. The next release could then publish incumbent score changes caused by something other than the reviewed revision, with the same 'only cause' claim and no CI signal." + }, + { + "severity": "minor", + "title": "The sweep-timing check against git will silently skip on main after the squash merge", + "file": "tests/test_reference_upgrade.py", + "line": 344, + "evidence": "reference_audit/2026-09-28/verification/sweep_timing.json pins reference_sweep.pin_commit = be90e543d116 ('Pin policyengine.py 6.1.2 and policyengine-us 2.15.17'). test_the_pin_commit_time_in_the_timing_record_is_gits runs `git log -1 ` and calls pytest.skip('git history unavailable') on failure. `git merge-base --is-ancestor be90e543 origin/main` fails: the commit exists only on this PR branch. The repo squash-merges (origin/main commits are single-parent '(#179)', '(#178)', ...). The new `fetch-depth: 0` (ci.yml:26-29) fetches refs/heads/* and tags, not refs/pull/*. The check passes on the PR's merge ref today. After the squash merge and the branch cleanup, main's CI can no longer resolve the commit and the test skips. BASE_COMMIT 3220a7a6 is on origin/main, so the finish-driver gates this CI change was made for are unaffected.", + "failure_scenario": "After merge and branch deletion, the differential check behind the published sweep-timing record (2.15.17 pinned at 01:58 UTC, which supports 'newest release when the sweep began') stops running in CI without a failure. A later edit to sweep_timing.json's pin_commit fields would go unchecked." + }, + { + "severity": "minor", + "title": "Sidecar's excluded_outputs_rechecked reasons are the reviewers' text cut at exactly 600 characters, dropping the reviewed basis", + "file": "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json", + "line": 1051, + "evidence": "All 19 `reason` strings in revisions[-1].excluded_outputs_rechecked (meta.json:940 onward), and in final_actions.json excluded_rechecked, are exactly 600 characters long. Each is a prefix of that cluster's reviewer corrected_per_output reason, which runs 638 to 1,905 characters (computed by startswith against clusters.json). Examples: scenario_005 federal ends \"salt_refund_income=0 on 2.15.17 gives ex\". scenario_100 snap (meta.json:1051, final_actions.json:175) ends \"Keep the outp\", which drops the reviewer's \"Do not drop the unlisted-input basis ... MWD ... 8,844\". That review had agree=false, and this was its point of disagreement. scenario_118 snap stops partway through its third reason, before the reviewer's instruction to label 2,868 as the default-county engine value \"not ... the law-correct stated-facts value\".", + "failure_scenario": "README.md says the sidecar's excluded_outputs_rechecked \"gives each one's 2.15.17 value and reason\", and the builder docstring says the reviewed reason is recorded. As published, the recorded reason for scenario_100 reads as if the output stays excluded only on the r30 port at 6,924. The reviewer's unlisted-input basis and the 118 corrections appear only in clusters.json, not in the reference record." + }, + { + "severity": "minor", + "title": "Review-backing test exempts 0/1 eligibility flips as \"within $1\", so an unreviewed flip passes", + "file": "tests/test_reference_upgrade.py", + "line": 125, + "evidence": "Lines 125-126 exempt any change with `abs(regenerated - previous) <= 1.0` from needing an approved or added entry. The review-action check (lines 131-145) iterates only approved|added, so the exemption covers a 0/1 flag change of exactly 1. The builder (build_references_latest.py:134-137, round() for _eligible) and paper_results.moves_beyond_tolerance (\"any change for a 0/1 flag\") both treat such a flip as beyond tolerance. Mutation run on a scratch copy: I removed scenario_028 from final_actions.approved and relabeled its sidecar change as cause engine_upgrade_within_1. `pytest tests/test_reference_upgrade.py -k \"every_changed_reference_is_listed_and_reviewed or every_output_that_moves\"` then gave 2 passed.", + "failure_scenario": "A future revision could ship a reference eligibility flip, which changes the exact-match score of every model, labeled as a within-$1 move with no approved action. The PR body's invariant \"every change is listed and review-backed\" would still show green. The build guard would catch it only if the build script were rerun." + }, + { + "severity": "minor", + "title": "PR body understates the Sonnet 5.5 vs GPT-6 Luna gap as 0.015 points", + "file": "/private/tmp/claude-501/-Users-maxghenis-Library-Application-Support-Claude-scratch-workspaces-ee3763f4-e7ea-4177-bcaf-1362266768c1-4457aa9d-2dc0-4bcf-a128-e50adae6be25-scratch-2026-09-22-c3da5e/3dd3d124-a1eb-44ed-ac89-ce3df9700101/scratchpad/pr_20260929_body.md", + "line": 11, + "evidence": "Frozen payload modelStats: claude-sonnet-5.5 exact 91.97320, gpt-6-luna 91.95770. Recomputed gap = 0.015508856 (python over paper/snapshot/.../data.json.gz). Rounded to three decimals that is 0.016, not 0.015. The release note itself says 'less than 0.02 points' (sonnetLunaGapBelow = ceil to hundredths), which is correct.", + "failure_scenario": "The public PR description states a gap that is truncated rather than rounded (0.015 vs 0.0155 to 0.016). It is tiny, but it is a published number that does not match the payload at its stated precision. Fix: say 0.016 or 'under 0.02', matching the note." + }, + { + "severity": "minor", + "title": "PR body says narratives were rewritten for '10 changed outputs'; only 9 outputs changed", + "file": "/private/tmp/claude-501/-Users-maxghenis-Library-Application-Support-Claude-scratch-workspaces-ee3763f4-e7ea-4177-bcaf-1362266768c1-4457aa9d-2dc0-4bcf-a128-e50adae6be25-scratch-2026-09-22-c3da5e/3dd3d124-a1eb-44ed-ac89-ce3df9700101/scratchpad/pr_20260929_body.md", + "line": 39, + "evidence": "git diff 3220a7a6..f7ced3b3 -- annotations/.../us_case_reference_explanations.csv rewrites 10 rows. The references themselves change on only 9 outputs (reference_outputs.csv diff: 008 NJ, 013 AZ, 028 PA, 033/078/117 federal, 078/117 state, 082 NY; the sidecar engine_upgrade.changed has 9 entries). The 10th rewritten narrative is scenario_064 dependent2_chip_eligible, whose reference stays 0.0. The same line calls 064 'the CHIP narrative that misled the judge', so the body contradicts itself.", + "failure_scenario": "The public PR body misstates the number of changed outputs (9 versus 10) and presents the 064 CHIP narrative rewrite as a changed output. Fix: say '10 narratives: the 9 changed outputs plus 064 CHIP'." + }, + { + "severity": "minor", + "title": "PR 'Verdict writer' invariant relies on code that is not in the repository", + "file": "/private/tmp/claude-501/-Users-maxghenis-Library-Application-Support-Claude-scratch-workspaces-ee3763f4-e7ea-4177-bcaf-1362266768c1-4457aa9d-2dc0-4bcf-a128-e50adae6be25-scratch-2026-09-22-c3da5e/3dd3d124-a1eb-44ed-ac89-ce3df9700101/scratchpad/pr_20260929_body.md", + "line": 54, + "evidence": "The Invariants list claims: 'Verdict writer. Exact model coverage, served model = claude-opus-5-5 from the transcript, no hedging, and each verdict bound to its prompt's sha256.' The only code that does this is /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/provisional/write_verdicts.py, which is gitignored. `git ls-files | grep -i verdict` returns no such writer, and no committed test checks the property. I checked the outcome independently: all 221 cases with new-model rows, plus the 9 changed outputs, have verdict.meta.json sidecars in results/local/adds0928-v3/audit/cases naming claude-opus-5-5. The claim is true, but a reader of the repo cannot verify it.", + "failure_scenario": "A reviewer or auditor reading the PR takes 'Verdict writer' to be a committed, tested gate like the other invariants listed with repo paths. It is local tooling, so nothing in CI enforces it. Fix: name it as local or uncommitted tooling, or commit the writer and a test for it." + }, + { + "severity": "minor", + "title": "Two new paper-claim tests always skip in CI, including their HTML checks", + "file": "tests/test_disclosures.py", + "line": 605, + "evidence": "`test_paper_abstract_scopes_the_exclusions_to_what_the_audits_found` (line 605) and `test_paper_takes_its_engine_times_from_the_timing_record` (line 636) each build a tuple `(html_text, _pdf_running_text()/_pdf_text())` before they assert anything. `_pdf_text()` (lines 108-120) calls `pytest.importorskip(\"pypdf\")` when `pdftotext` is not on PATH. pypdf is not in pyproject.toml or uv.lock (grep count 0), and the CI test job installs no poppler. I simulated CI with `env PATH=/usr/bin:/bin .venv/bin/python -m pytest tests/test_disclosures.py -k \"abstract_scopes or engine_times_from or dollar_amounts or rendered_pdf\"`, which gave `1 passed, 6 skipped ... SKIPPED [6] tests/test_disclosures.py:120: could not import 'pypdf'`. I then mapped the skip marks in the CI log for PR #182's test job (run 36615138858, `975 passed, 11 skipped`) onto `--collect-only` order. Both new tests are among the 11 skips. Locally they pass, because /opt/homebrew/bin/pdftotext exists, so the committed index.html and policybench.pdf are correct today. The existing HTML/PDF pair (`test_rendered_html_prints_dollar_amounts_without_escapes` and `test_rendered_pdf_...`) splits HTML from PDF so the HTML half runs in CI; these two tests do not.", + "failure_scenario": "The abstract's exclusion sentence (the 28 engine-defect and 27 unlisted-input counts) and the two clock-time sentences ('uploaded 00:23 UTC', 'checked PyPI on 2026-09-29 at 14:58 UTC') appear only in the rendered app/public/paper/web/index.html and policybench.pdf, because the qmd renders them inline from `r.*`. The paper CI job renders HTML to paper/out and never compares it with the committed render. If a later change moves an exclusion count or the timing record and the committed render goes stale, CI still passes: both tests skip before any assertion on the served HTML. The site would then publish stale numbers. Fix: check the HTML in a separate, unconditional test, and keep the PDF half behind the skip (or add pypdf to the dev extras)." + } + ], + "checked": [ + { + "lens": "gates", + "checked": "I read scripts/finish_adds0928.py in full: resolve_base, base_commit_blob, resolve_live_base, reference_revision, replay_base_references, export and main. I also read scripts/freeze_adds0928.py, scripts/install_adds0929_references.py, the freeze_snapshot.py diff, the ci.yml diff and tests/test_finish_adds0928.py.\n\nTest runs:\n- tests/test_finish_adds0928.py: 64 passed.\n- The same file with the replay gate replaced by a no-op: still 64 passed.\n- PR #182 CI (gh pr checks): all green.\n\nChecks against git and the release that came out sound:\n- BASE_COMMIT 3220a7a6 is on origin/main.\n- The pinned BASE_REFERENCE_SHA256 values match `git show 3220a7a6:` for all three files.\n- The 22c data.json.gz at BASE_COMMIT rewraps to BASE_SHA256 01e7e72b.\n- The committed snapshot data.json.gz rewraps to d146473d. That equals app/src/data.artifact.json and the uploaded release asset (GitHub asset digest sha256:d146473d, size 123324948).\n- The uploaded predictions.csv.gz digest ef2c003a equals the snapshot file and the manifest pin.\n- Staged reference and scenario files equal the committed ones. stage.json and release-ready.json pins (2043 files) all verify.\n- scenarios.csv and its sidecar are unchanged from 22c.\n\nReference revision against 22c: exactly 9 values differ, all listed with matching values, no NaNs, and impact_weight is unchanged. Exclusions are the 22c 52 plus exactly the 3 revision-listed keys; earlier sidecar revisions are unchanged.\n\nReplay and drift: the replay's data.json reproduces 22c modelStats (Fable's 4 usage fields patched from 22c by design) and heatmap for all 42 incumbents. Against 22c, the staged payload changes incumbent modelStats only through the reference revision: n goes from 1932 to 1929, and exact rises 0.362 to 0.535 points. On the 22c references, incumbents' per-row annotation text and failureSource/failureSubtype labels differ from 22c, from the re-judging. modelStats and heatmap do not depend on those labels, so no number moves.\n\nPost-freeze path: prepare is refused once the pointer moves. A re-export still gates against the 22c asset read from git, and the staged-vs-committed reference check at freeze time closes the gap between the files the gate reads and the files the export uses.\n\nCI: fetch-depth 0 is enough for the BASE_COMMIT reads. No other workflow runs pytest.\n\nScratch files are in /private/tmp/claude-501/rev182; the worktree is untouched (git status clean)." + }, + { + "lens": "references", + "checked": "I computed each of these myself, read-only, with scratch files only in the scratchpad.\n(1) reference_outputs.csv vs sweep_moves.csv: all 1,984 keys match. Every value in the reference column matches except 3 rows (scenario_023 state_refundable_credits, scenario_112 federal_refundable_credits and payroll_tax) that differ by at most 6e-14; sweep_moves is written to fewer digits.\n(2) Counts against base commit 3220a7a6:\n- 4 scored changes: 008 NJ 5342.40 to 5842.40; 013 AZ SNAP 0 to 240; 028 PA reduced-price meals 1 to 0; 082 NY 650.50 to 667.\n- 3 new exclusions: 033, 078 and 117 federal.\n- 2 moves under $1: 078 MD state +0.54 and 117 AR state +0.48.\n- 19 rechecked; exclusions go from 52 to 55; 1,929 scored.\nAll of these match README.md, the sidecar `changed` list, and the PR body.\n(3) latest_final_2170.csv and latest_final_2171.csv each reproduce all 1,929 scored references bit-exactly. They give each of the 19 rechecked outputs its sidecar value_on_2_15_17, no other excluded output moves, and both equal sweep_moves' final column everywhere. The 2171 sha256 matches sweep_timing.json.\n(4) I recomputed all 1,984 outputs myself on the installed policyengine-us 2.15.17 with fixes/latest_final.py and the committed builder (r19 table copied as the build docstring says). Every scored reference reproduced exactly; the only differences were the 19 rechecked excluded outputs.\n(5) Review backing:\n- Every changed or newly excluded output traces to a cluster with an investigation and a review. The 16 review files equal clusters.json reviews apart from the runner field: 6 workflow and 10 subfleet, as the README says.\n- The approved values equal the reviewers' proposed references. The salt-refund review (agree=false) agrees with exclusion, and the enum mismatch is explained. The 078 federal reconciliation cites that review.\n- For the two moves under $1, no model's exact-match hit changes on either output among the 45 in predictions.csv.gz.\n(6) tests/test_reference_upgrade.py: 14 passed.\nWorktree: /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2." + }, + { + "lens": "alias", + "checked": "I found no defects in the stated-hours alias. Every claim the PR makes about it holds when checked against the engine.\n\n**How policyengine-us 2.15.17 reads the two variables** (read from the installed package in the worktree's .venv):\n- `hours_worked_last_week` is a pure input with no formula. Its only reader is `monthly_hours_worked`, which feeds Medicaid community engagement, AR Medicaid work requirements, DC GAC, IL AABD, MD TCA and MS WD.\n- `weekly_hours_worked_before_lsr` is a pure input with `default_value = 0`. It is read directly by the four SNAP work-rule variables (ABAWD, general, registration-exempt, student work exception), by several state CCAPs and by the HI and MA TANF dependent-care deductions. It also reaches `weekly_hours_worked` (its sum with the behavioural response), which other CCAPs and the DC and MT TANF work tests read.\n- No variable file reads both chains, and neither variable has a formula derived from the other, so setting both cannot double count hours.\n- The 1.755.4 triage venv has `default_value = 40`. Upstream commit 82745ca239 (\"Default weekly_hours_worked_before_lsr to 0\", 2026-08-10) was merged as PR #9261 on 2026-08-12. The comment at `policybench/scenarios.py:146-153`, the README and the paper sentence are all accurate.\n\n**The alias changes only scenario_066's reference.** I ran 2.15.17 plus `fixes/latest_final.py` with the sweep's `build_situation` in a scratchpad copy:\n- Scope: all 56 scenarios with stated hours, 1,220 outputs. Every other scenario is untouched, because the alias fires only when `hours_worked_last_week` is in the person's inputs.\n- Alias on versus off, exactly one output moves: scenario_066 snap, 3576.0 with the alias and 0.0 without.\n- A second run over all 100 scenarios (1,984 outputs) gave every scored output exactly equal to the committed `reference_outputs.csv`.\n- Setting unlisted hours to 40 as well moved only scenario_056 snap (95 to 1140) and scenario_112 snap (0 to 288). Both are excluded outputs, which matches the audit's `latest_alt_unlisted_hours_40` claim.\n\n**Other paths:**\n- Prompts render from `person.inputs` (`prompts.py:331`), so the alias never reaches a prompt. The target stays in `EXCLUDED_INPUT_VARIABLES` in any case.\n- No snapshot scenario carries `weekly_hours_worked_before_lsr` as an input, so the \"target not in inputs\" guard never suppresses the alias.\n- The prompt label \"usual weekly hours worked\" has been unchanged since 2026-05-01, so what models saw is consistent with the new reference.\n- `build_references_latest.py` gets its situations from `Scenario.to_pe_household` and asserts the alias is present for scenario_066.\n\n**Test coverage:**\n- `tests/test_reference_upgrade.py` never mentions hours, so it does not cover the alias. It only compares committed CSVs, including the 2.17.0 verification, which could only match 066's 3,576 if the alias was in the run.\n- The alias is covered instead by `tests/test_scenarios.py` (stated 12, 40 and 60 hours copied to the engine name; unstated or zero hours left at the default). I ran `pytest tests/test_scenarios.py -k hours`: 6 passed.\n- `tests/test_snapshot_artifacts.py:906` pins the sidecar's builder note.\n\nThe PR body's line about the alias is accurate." + }, + { + "lens": "claims", + "checked": "I recomputed every public number against the frozen payload paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz, its predictions.csv.gz and the base files at 3220a7a6. Nothing an app reader sees is wrong.\n\n**Board figures (all match)**\n- 45 models. Weighted exact rates and ranks: Sonnet 5.5 91.97 (#4), GPT-6 Luna 91.96 (#5), Grok 4.7 88.23 (#12), DeepSeek V4.1 Flash 87.27 (#15).\n- Top three: GPT-6 Sol 94.81, Opus 5.5 93.51, GPT-5.6 Sol 93.28.\n- Cost per household, recomputed from tokens at the config rates: $0.0343, $0.2597 and $0.0242. GPT-6 Luna is $0.0019.\n- Each new model has 1,984 rows, 100 calls and no errors. DeepSeek reports resolved model 'deepseek-flash' with a single fingerprint.\n\n**Incumbent drift and the reference change**\n- Incumbent drift is +0.362 (Kimi K2.6) to +0.535, with exactly 3 neighbor swaps (Opus 5 over Fable 5, Gemini 3 Flash Preview over Opus 4.7, DeepSeek V4 Pro over Gemini 3.1 Flash Lite). No incumbent prediction changed between base and head (0 of 83,328 rows).\n- No incumbent matched the old values of 033, 078 or 117 federal. All 42 matched AZ 013 SNAP at $0, and all 45 now answer $0 against $240.\n- The reference diff has exactly 9 changes: 4 scored (NJ 5,342.40 to 5,842.40; AZ 0 to 240; PA 1 to 0; NY 650.50 to 667), 3 newly excluded, and 2 under $1. The sub-$1 moves flip no model's exact match.\n- Exclusions: 55 in 39 households, 28 engine-defect and 27 unlisted-input. 52 are recorded on 1.755.4 and 3 on 2.15.17. 19 excluded outputs moved and were rechecked, and each engine-defect recheck says the defect is still present. Every excluded output's frozen_value equals reference_outputs.csv.\n\n**Engine-version checks**\n- latest_final_2170.csv and latest_final_2171.csv equal the 2.15.17 'final' column for all 1,984 outputs (max difference 0.0).\n- sweep_timing is consistent: 2.15.17 was uploaded 00:23, the sweep began 01:42, and 2.16.0 came at 04:10. Live PyPI at 20:42 UTC still shows 2.17.1 as newest.\n- `import policyengine` 6.1.2 fails next to 2.15.17 with 'not certified', and its bundle pins policyengine-us 2.2.1 and populace-us-2024-spm-20260915.\n- The nine convention modules change parameters only (no variable or formula overrides). The eight upstream fixes are backed by ancestry evidence in port_state_result.json and the SNAP hold module.\n\n**BBCE note update**\n- Sonnet answers $388 for 027 (citing categorical eligibility) and $1,208 for 030 (wages only), and $0 for the other two.\n- Grok answers $276 for 027 (citing BBCE) and $0 for the rest. Flash answers $588 for 030 and $0 for the rest.\n- On the savings-held households, Sonnet and Grok answer above $0 on all 4 and Flash on 3.\n- None of the three new models hits the four $288 households. AZ is $24 \u00d7 10 months = $240, and all 45 models answer $0.\n\n**Other checks**\n- The release_20260922c_exact fixture equals the base snapshot's modelStats exactly.\n- Paper, docs/paper.md, benchmark_card, sensitivity doc and Methodology copy: I checked every figure against the `r` object and the serving config. That covers 7,796 / 7,792 / 2,027 / 2,041 / 53 / 68 adjudications; 303/132/239 of 674 judged cases; 10 chunked rows; 8 of 14 JSON rows recorded as rejections; and the sensitivity ranks #7/#9/#19/#7. The all-1,984 sensitivity figures recompute to 87.1, 85.8, 80.3 and 88.0.\n- Opus 5.5 judged all 221 cases with new-model rows and all 9 changed outputs. All 23 judge reference-suspect flags are adjudicated.\n- No stale 42-model, 1,932 or 52 figure is presented as current in app/src, docs, paper sources, the rendered HTML or the PDF. The Sept 23 BBCE note's 42 is dated.\n- The staged data-board45.json has sha256 d146473d\u2026, and its US section equals the frozen payload.\n- Targeted tests pass: tests/test_notes.py -k 'release_20260929 or bbce_note_update or previous_release_scores or bbce_households_note_facts' (4 passed) and tests/test_sensitivity_evidence.py (9 passed).\n\nThe only defects are three minor ones in the PR body text, listed as findings." + }, + { + "lens": "tests", + "checked": "Diff scope: 3220a7a6..f7ced3b3 -- tests/ app/tests/ (26 files). No test function was deleted: the three removed names are renames (42 to 45 board, Fable line to local Claude models). Nothing was newly marked slow, skip or xfail. The one early-return guard change (dashboard-data-20260922c added to SUPERSEDED_RELEASES) disables recomputation for no note, because no note carries that release. The new 20260929 note and the BBCE update tests run their full bodies because the manifest tag is dashboard-data-20260929.\n\nI recomputed every changed pin independently from the frozen artifacts, and each matched. Data sources: paper/snapshot/.../data.json.gz, reference_exclusions.json, reference_outputs.csv and its sidecar, annotations/*, the manifest, and the staged audit case tree.\n- Board: 45 no_tools models, n = 1929 each; 86,805 scored rows = 45 x 1,929.\n- Exclusions: 55 (28 engine defect, 27 unlisted input) in 39 households; by engine 52 on 1.755.4 and 3 on 2.15.17; 1,984 reference rows.\n- Annotations: 9,837 rows. Scored rows: 7,796, of which 7,142 llm_error and 654 parse failures (kimi-k2.6 391, glm-5.2 133, glm-5.3 71, kimi-k3 59). Excluded-output rows: 2,041, of which 780 prompt_ambiguity.\n- Audit universe from the payload: threshold < 100 on 7,796 rows, exact < 100 on 7,792, bounded < 100 on 9,823, so 2,027 are unannotated.\n- Missing explanations: grok-4.3 55, kimi-k2.6 4, claude-haiku-4.5 1 (60 in total), so 714 contract violations.\n- Adjudications: 68 cases. Adjudicated class: 28 engine defect / 27 prompt ambiguity / 13 llm_error. Judge verdict: 49 / 2 / 1 / 16. 55 excluded, 54 re-judged, 63 with a published verdict.\n- Judge provenance: recomputed with freeze_snapshot.audit_judge_provenance on results/local/adds0928-v3/audit/cases as 674 cases (132 Opus 5 / 239 Opus 5.5 / 303 GPT-5.6 Sol), with the same judged-on dates.\n- Serving config: evidence kinds 29 registry / 16 run_state. The app copy is byte-identical to the frozen one.\n- Heatmap: Fable 5.1 federal exact = 65/82 = 79.268.\n- Previous-release fixture: equals the base-commit snapshot's 42 exact scores (max diff 0). Incumbent drift is 0.362 to 0.535, with three neighbor swaps.\n- The 2.17.1 later check file matches 2.17.0 on all 1,984 outputs.\n- Narrative reference_value equals the reference CSV on all 1,984 outputs.\n- app/src/data-summary.json equals the frozen payload, and the staged data-board45.json sha256 equals d146473d.\n\nTests I ran, all passing:\n- pytest: test_paper_results, test_reference_exclusions, test_report_costs, test_sensitivity_evidence, test_reference_audit, test_reference_upgrade, test_adjudications, test_disclosures, test_model_cards, test_install_references, test_snapshot_artifacts, test_notes (one skip, which predates this PR), test_finish_adds0928, test_policyengine_runtime, test_scenarios.\n- bun: the 10 changed or new app test files (74 pass).\n\nReviewed as sound:\n- The loosened checks: the _revisions() filter and superseded-key handling in test_reference_audit (only scenario_078 federal tax is superseded), the regenerated_at_utc date rule, and the forced-tool regex widening (both documents still match the original pattern).\n- The hand-typed copy checks moved to derived rendering in boardScope and servingConfig tests, which render the sentences from the config and payload.\n\nCI on head f7ced3b3 is green, gh reports the PR MERGEABLE, and the test job fetches full history, so the git-differential tests ran and did not skip. The only gap found is the CI-vacuous pair reported above." + } + ], + "missing_lenses": [] +} \ No newline at end of file diff --git a/docs/adds0928/stage2_design.md b/docs/adds0928/stage2_design.md new file mode 100644 index 00000000..bc44958d --- /dev/null +++ b/docs/adds0928/stage2_design.md @@ -0,0 +1,241 @@ +# September 28 additions: stage 2 design + +This note and the drivers live in the assigned checkout. The requested external +state-note path is outside the job's writable workspace. No worker checkout, +live run directory, release asset or production pointer is changed by staging. + +## Base and fold + +Use `dashboard-data-20260922c`, SHA-256 +`01e7e72b3a6bdd2d3178ba32625ff769d5b81dc07541af6ea8da2c852774ddcc`: +42 models, 100 households, 1,984 requested outputs per model, 52 exclusions and +1,932 scored outputs. The committed snapshot under +`paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/` +contains the base predictions, reference outputs and their provenance sidecar, +exclusions, scenarios and frozen country payload. Recombining that payload must +match the live release hash. Use the checkout's matching annotations and +`us_adjudications.json`; never use the old driver's 39-model September 5 base +or its mutable `reference_v12` scratch directory. + +[PR #178](https://github.com/PolicyEngine/policybench/pull/178), merged as +`cb312fd775b36af644b10803c076a0d0efc79f2e`, establishes this base. Its difference +from 20260922b was the regenerated Michigan `scenario_045` SNAP reference of +$0 after the upstream child-support fix; it restored that output to scoring. +There are 26 regenerated references. The dated 20260922c release name preserves +the September 22 snapshot; this new addition uses `dashboard-data-20260929` and +the September 28 snapshot date. + +`scripts/finish_adds0928.py` generalizes `finish_opus55.py` and +`judge_stages.py`. It requires all three named supervised runs, each with +`completed == total > 0`, `stopped_reason is None`, and predictions present. +The production fold additionally requires complete reference-key coverage, +known model metadata, costs and treatment provenance. Copies of inputs and all +subsequent outputs stay under `--stage-dir`. `fold_board` combines predictions; +the official exporter scores against the copied, unchanged references and +exclusions. All 42 incumbent `modelStats` must stay identical. Fable 5's historical +batch usage must survive export, as must zero-cost Ox Alpha usage. + +`--early` skips judging and cannot produce a release-ready receipt. Normal +`--early` refuses partial runs. The explicit `--early --partial` rehearsal uses +synthetic completion state in copied scenarios only, scores the common completed +household cohort for the 45-model comparison, and also reports each addition +against the 42 incumbents on that addition's own completed cohort. Every such +score and payload is marked **PARTIAL**, and publication is blocked. + +Reproduce the CSV-only rehearsal in new scratch directories: + +```bash +export OPENBLAS_NUM_THREADS=1 +PB_SOURCE=/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609 +PB_COPY="$PWD/results/local/adds0928-partial-copy" +PB_EARLY_STAGE="$PWD/results/local/adds0928-partial-stage" +uv run python scripts/snapshot_adds0928.py \ + --runs-root "$PB_SOURCE" --out-dir "$PB_COPY" + +# Expected nonzero exit: the synthetic copied states still have total=100. +uv run python scripts/finish_adds0928.py \ + --runs-root "$PB_COPY" --stage-dir "$PB_EARLY_STAGE" --early + +# Temporarily change only the copied states; restore them even if export fails. +uv run python - "$PB_COPY" "$PB_EARLY_STAGE" <<'PY' +import json +import subprocess +import sys +from pathlib import Path + +paths = sorted(Path(sys.argv[1]).glob("*/run/run_state.json")) +original = {path: path.read_bytes() for path in paths} +try: + for path in paths: + state = json.loads(original[path]) + assert state["synthetic_partial"] + state["total"] = state["completed"] + path.write_text(json.dumps(state, indent=2) + "\n") + subprocess.run([ + sys.executable, "scripts/finish_adds0928.py", + "--runs-root", sys.argv[1], "--stage-dir", sys.argv[2], + "--early", "--partial", + ], check=True) +finally: + for path, content in original.items(): + path.write_bytes(content) +PY +``` + +## Audit and triage + +The unit of audit is `(country, scenario_id, variable)`, grouping every wrong +model's answer and explanation for that output. `prepare_audit` adds the household +facts, reference value, reference derivation and existing grounding. When a new +model joins a case, the rendered prompt changes and its old verdict and provenance +sidecar are invalidated. Unchanged prompts retain verdicts. New wrong outputs +become new case directories. Cases containing only missing/unparseable answers +are classified deterministically without a judge call. + +The existing audit selects `threshold_score_single_prediction < 1`, including +excluded reference outputs for explanatory coverage. This is the historical +legacy-threshold miss set, not every headline exact-match miss. Scoring itself +uses the 1,932 nonexcluded outputs and population-weighted household exact rates. + +Copy the historical unified audit into the stage, preserving valid verdicts and +hash-bound metadata. Use `scripts/run_audit_claude.sh` with +`AUDIT_MODEL=claude-opus-5-5`, `AUDIT_PARALLEL=1`, and an explicitly selected +Claude subscription lane. It invokes Claude without tools, validates the full +verdict schema, and records requested/reported model, CLI version, timestamp, +session and verdict hash. On September 22, lane admission failure meant Opus 5.5 +native Workflow subagents supplied verdicts through `judge_stages.py write`; +that was the same prepare/validate/collect contract, not another judge model. + +Missing and hedged verdicts are rejudged with a bounded retry count. A standing +`reference_suspect` flag is an investigation, not a retry target. `--step triage` +collects fresh row annotations and case notes, including new wrong-model counts, +applies the staged adjudications, and stops on unresolved cases. Investigators +must check the stated facts, primary authority and engine trace, independently +review the conclusion, and write the evidence and original judge class to the +staged adjudication record. An affirmed reference clears the flag with its +reason; an engine defect fixed upstream is regenerated; an unfixed formula +defect or necessary unlisted input is excluded for every model. Apply the existing +published-law/held-value conventions; do not silently change the scoring universe. + +The historical `triage/build_records.py`, `regen_references.py`, +`make_on_convention.py`, `package_audit.py` and sweeps produced the September 22 +reference revisions. Their committed, reproducible equivalents live under +`reference_audit/2026-09-22/`. They are evidence and templates; rerunning their +hardcoded historical paths is not part of an additive release. A newly confirmed +reference change requires a separately reviewed revision and fresh incumbent +comparisons. The additive release helper deliberately refuses changed references. + +## Commands after all three runs complete + +Run from this checkout, sequentially. The source paths below are read-only. The +stage must be new for `prepare`; subsequent commands resume that same stage. + +```bash +export OPENBLAS_NUM_THREADS=1 +export PYTHONPATH="$PWD" +export PYTHONDONTWRITEBYTECODE=1 +# This managed workspace needs its local Git metadata for correct registry pins. +# An ordinary checkout restored from the bundle uses its normal Git metadata. +if [ -d results/local/stage2-history.git ]; then + export GIT_DIR="$PWD/results/local/stage2-history.git" + export GIT_WORK_TREE="$PWD" +fi +PB_RUN=us_full_run_20260612_policyengine_4_16_1_populace +PB_SOURCE=/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609 +PB_AUDIT_SOURCE=/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit +PB_STAGE="$PWD/results/local/adds0928-final" + +# Stop and revise the base if this pointer no longer names September 22c. +gh api 'repos/PolicyEngine/policybench/contents/app/src/data.artifact.json?ref=main' \ + --jq '.content | @base64d' + +uv run python scripts/finish_adds0928.py \ + --runs-root "$PB_SOURCE" --stage-dir "$PB_STAGE" \ + --audit-seed "$PB_AUDIT_SOURCE/audit" \ + --grounding "$PB_AUDIT_SOURCE/grounding.csv" --step prepare + +# Execute this step inside the selected Claude subscription lane. +AUDIT_MODEL=claude-opus-5-5 AUDIT_PARALLEL=1 \ + uv run python scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step judge + +uv run python scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step triage +``` + +If triage stops, investigate the staged `reference-flags.csv` and +`unresolved-rows.csv`; record reviewed resolutions in +`$PB_STAGE/publish/$PB_RUN/annotations/us_adjudications.json`, preserving the judge's +actual verdict. Then rerun `triage`. Do not invent adjudications to unblock export. + +```bash +uv run python scripts/finish_adds0928.py --stage-dir "$PB_STAGE" --step export +uv run python scripts/freeze_adds0928.py --stage-dir "$PB_STAGE" --dry-run +uv run python scripts/freeze_adds0928.py --stage-dir "$PB_STAGE" +uv run python scripts/sensitivity_by_variable.py +uv run python scripts/rescore_sensitivity_summaries.py \ + --release dashboard-data-20260929 +``` + +`export` writes the strict 45-model payload and a hash-bound `release-ready.json`. +`freeze_adds0928.py` is a **local build**, with no upload code. It checks that +receipt, unchanged references and incumbent predictions, complete 100/100 input +copies, and staged adjudications. It configures the existing freezer in process, +pins the staged audit provenance, preserves incumbent frozen serving evidence, +and requires copied run-state evidence for the additions. It updates the local +pointer/version label, freezes the compact snapshot and annotations, caches the +payload for offline app builds, and creates deterministic `predictions.csv.gz` +beside the staged payload. It does not invoke historical `publish_opus55.py`, +whose hardcoded scratch paths would write outside this workspace even in dry-run. + +Before rendering, write a new September 28 note from final recomputed results; +update the paper source, benchmark card, methodology/model-count prose and tests, +serving-sensitivity prose, and release-specific fact pins. Keep historical notes' +own release pins and findings. Account for Sonnet's JSON/adaptive thinking +treatment and DeepSeek's moving alias. The roster count is 45; scored output and +exclusion counts stay 1,932/52 unless separately reviewed triage changes them. + +```bash +uv run python paper/render_paper.py +uv run python scripts/freeze_snapshot.py --rendered-only +uv run ruff format scripts/finish_adds0928.py scripts/freeze_adds0928.py +uv run ruff check . +uv run ruff format --check . +uv run pytest -q +bun install --cwd app --frozen-lockfile +(cd app && bun run test) +(cd app && bun run lint) +(cd app && bun run build) +git diff --check +``` + +Review the generated diff and commit coherent changes on the assigned branch. +The two future release assets are `dashboard-data.json` (the staged payload, +renamed at upload time) and `predictions.csv.gz`. This job stops before upload, +push, merge or production publication. Recheck release-tag availability before +any later publication; choose a fresh suffix if the intended tag was taken. + +## Risks and gates + +- A partial cohort's score and rank cannot estimate a final rank without + uncertainty; completion order need not be representative. Synthetic state is + rehearsal evidence only. No release receipt is created for it. +- The DeepSeek `deepseek-flash` alias can change. Retain per-response reported + model/version evidence and response timestamps; an alias alone does not prove + V4.1 Flash. If responses report only that alias, preserve the dated onboarding + and provider-model-list evidence, disclose the limit, and obtain a substantive + provenance review before release. Do not relabel the alias as an observed + immutable version or rewrite historical V4 board rows. +- Costs can be present yet mispriced. Verify new model registry, dates, input, + output/cache prices and response usage against the registered treatments. +- Card defaults can differ from the actual run fingerprint (notably Sonnet's + JSON contract and Grok's 1,800-second timeout). Do not edit running workers or + relabel old responses. The freezer must use actual copied treatment evidence. +- Copy the audit, grounding, exclusions and reference sidecar together. Stale + case notes undercount wrong models; absent exclusions silently change scores; + a `by_model` directory in the export bundle can hide incumbent predictions. +- Judge flags may require real reference work. Rejudging an already adjudicated + case can change its judge class; preserve the new exact judge class alongside + the independently justified adjudication. Never permit incumbent score drift + merely to finish an additive release. +- Local release building intentionally changes tracked snapshot/pointer files; + preparation and partial dry runs do not. Large payloads, dataframe exports and + paper builds run serially with `OPENBLAS_NUM_THREADS=1` on the shared machine. diff --git a/docs/artifacts.md b/docs/artifacts.md index 6842f1ee..eb13112c 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -1,6 +1,6 @@ # Dashboard data artifacts -The dashboard payload (`data.json`, ~100MB at 39 models) is a generated +The dashboard payload (`data.json`, ~126MB at 46 models) is a generated artifact, not source. The manuscript snapshot keeps the frozen per-country copy as a deterministic gzip (`paper/snapshot//runs//data.json.gz`, read through `policybench.snapshot_payload`) because the plain export passed diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 38aec324..238fbd34 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -49,11 +49,14 @@ PolicyBench has one canonical evaluation mode. Structured responses are collected through the transport each model card records: a forced answer-schema tool call (`submit_outputs`) where the card -selects the tool transport and the provider accepts a forced call, or the same -fields returned as a JSON object — because the provider rejects a forced tool -(Kimi K3, Qwen 3.8 Max, Claude Fable 5.1) or because the card selects JSON -for the family (the older Gemini and DeepSeek rows). The per-model transport -is in the snapshot's `model_serving_config.json`. +selects the tool transport, or the same fields returned as a JSON object where +it selects JSON. For most JSON rows the card records that the provider rejects +a forced tool call: Claude Fable 5.1, Claude Opus 5.5, Claude Sonnet 5.5, +DeepSeek V4.1 Flash (whose card says the same of the V4 rows), Kimi K2.6, Kimi +K3, Qwen 3.7 Max and Qwen 3.8 Max. The cards of DeepSeek V4 Pro and GLM-5.2 +select JSON without recording a rejection, and the older Gemini rows answer as +JSON by the Gemini family default. The per-model transport is in the +snapshot's `model_serving_config.json`. This is an output format, not a capability — nothing executes, no result is returned to the model, and each response is a single round trip. The benchmark remains no-tool in every response transport. @@ -62,41 +65,139 @@ The headline score uses the numeric answers only. Explanations are retained for auditing, scenario exploration, and qualitative error analysis; they should not be described as faithful reasoning traces. +## Reference outputs + +PolicyBench computes each scored US reference by running +`policyengine_us.Simulation` from policyengine-us 2.15.17, the newest release +when PolicyBench began sweeping the references on 2026-09-29 (uploaded 00:23 +UTC). The 56 excluded outputs keep the values they were decided on (52 computed +with policyengine-us 1.755.4, 4 with 2.15.17), and PolicyBench re-reviewed the +19 of them that move on 2.15.17; all 19 stay excluded. The manifest also +records policyengine.py 6.1.2 for provenance. Its certified US bundle carries +policyengine-us 2.2.1, and policyengine.py does not load beside 2.15.17. + +A scored reference follows from the stated facts and from law published before +PolicyBench froze the references on 2026-07-03. Where policyengine-us projects +a 2026 amount with a price index, or carries one published after the freeze, +the reference takes the amount published before the freeze or, where none was, +the last one published. Nine publication conventions set those amounts, and each +changes parameter values only: SNAP's October to December months, for example, +hold the FY2026 figures rather than USDA's FY2027 figures of 2026-08-21. +policyengine-us 2.15.17 counts Maryland county income tax in its state income +tax; an adapter takes it out again, because PolicyBench's state income tax +output leaves local tax out. policyengine-us 2.17.0, the newest release when +PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as +2.15.17 for all 1,984 outputs under the same conventions and adapter. + +Four changes between policyengine-us 1.755.4 and 2.15.17 each moved one scored +reference: New Jersey's child tax credit schedule for 2026 to 2028 (P.L.2026, +c.26, approved June 30, 2026), Arizona's broad-based categorical eligibility +limit for SNAP (200% of the poverty guideline from March 2026, up from 185%), +child support received counting as income for school meals (7 CFR +245.6(a)(5)(ii)), and the rounding of New York's Empire State child credit +phase-out. Three federal income tax outputs left scoring (below), and two state +income tax references moved by less than $1. The reference sidecar's +`engine_upgrade` revision lists every change, and `reference_audit/2026-09-28/` +records the investigation and the independent review behind each. + ## Audit scope -The frozen US annotations cover 8,783 scored rows selected because their -legacy threshold score is below 1 (293 further annotated rows sit on the eleven +The frozen US annotations cover 7,860 scored rows selected because their +legacy threshold score is below 1 (2,111 further annotated rows sit on the 56 excluded outputs and are description, not audit). This audit universe contains -8,780 of the snapshot's 8,780 exact-match misses and three exact hits. Another -1,605 scored rows have a bounded score below 100 but fall outside the legacy-threshold selection -and have no audit annotation. Two judge models produced the verdicts, both of -them board rows: GPT-5.6 Sol through the Codex CLI for 318 cases, and Claude -Opus 5 through the Claude Code CLI for the 350 cases a September 2026 addition -joined; the manifest's audit_annotation_artifacts.judge_provenance block -carries the tally. Verdicts change no score. A judge verdict outside the final -classes is resolved by a recorded developer adjudication -(annotations/.../us_adjudications.json). This snapshot carries eleven, one per -excluded output: each affirms `prompt_ambiguity` and removes the output from -scoring, keeping the judge's original verdict beside the decision and the -reasoning: Claude Opus 5 judged six of the eleven (its prompt-ambiguity reading -of scenario_064 SSI, and llm_error on the five Medicare outputs) and GPT-5.6 Sol -judged five (llm_error on the three SNAP and two other SSI outputs). - -Eleven outputs in ten households are excluded from scoring for every model +7,856 of the snapshot's 7,856 exact-match misses and four exact hits. Another +2,107 scored rows have a bounded score below 100 but fall outside the +legacy-threshold selection and have no audit annotation. Three judge models +produced the verdicts, all of them board rows: GPT-5.6 Sol through the Codex +CLI for 298 cases, Claude Opus 5 through the Claude Code CLI for the 116 cases +the September 5 additions joined that no later judge re-judged, and Claude Opus +5.5 for 260 cases the September 22, September 29 and September 30 additions joined or a +reference revision changed; the +manifest's audit_annotation_artifacts.judge_provenance block carries the tally. +Verdicts change no score. A judge verdict outside the final classes, and every +reference-suspect flag, is resolved by a recorded developer adjudication +(annotations/.../us_adjudications.json), which keeps the judge's +verdict (the case's current verdict.json; a flag an earlier judge run raised is +kept and says so) beside the decision and the reasoning. This snapshot carries +69: one for each excluded output, one for each flagged reference the +adjudication affirmed or replaced with a regenerated reference, and two for +cases whose misses the judge attributed to the reference applying later law; +the adjudication records them as model errors because that law predates the +freeze. + +The September 22 audit implemented each defect it confirmed in policyengine-us +1.755.4 as a sandbox fix on that engine version and recomputed every +reference under it. Eight of those root causes were fixed in policyengine-us +after the references were frozen (#8839, #9162, #9301, #9313, #9318, #9363 and +#9586: capital gain distributions, New York's renter cap, the CalEITC's lookup +at adjusted gross income, the engine's SNAP rounding, and the SNAP child +support option, whose engine values carried the opposite meaning: the engine +excluded child support paid from gross income in Michigan, which deducts it +from net income). For the September 22 references, PolicyBench applied those +fixes on 1.755.4 to the 15 scored references they move; policyengine-us 2.15.17 +contains all eight, so the references now take them from the engine. An output +one of them moves that an unfixed cause also moves stays excluded. The capital +gain fix as first built also applied Wisconsin's capital gain exclusion to the +distributions, which the upstream fix does not; that part is recorded as its +own defect, not fixed upstream. + +Fifty-six outputs in 39 households are excluded from scoring for every model (`reference_exclusions.json` beside the frozen references, pinned by the -manifest): their reference depends on an engine input the certified household -data never carried and the prompt therefore never listed. The SSI disability -criterion is false for every person in the June 2026 build, and months of SSDI -receipt is never carried and never promptable. Each excluded reference was -recomputed with policyengine-us 1.755.4 under the reading a careful reader could -take of the stated `is disabled` or SSDI-income fact, and it moved. Exclusion is -symmetric: rows that matched the frozen reference leave the score with rows that -did not, so every model is scored on 1,973 of its 1,984 requested outputs. The -rows on those outputs stay annotated (287 as `prompt_ambiguity`, six that never -parsed as `parse_contract_failure`) as description; no scored row carries the -ambiguity class. Do not read a $0 SSI reference -for a disabled under-65 household member as a finding about that person's SSI -eligibility. +manifest). The September 22 audit recomputed every reference on +policyengine-us 1.755.4 under each defect's sandbox fix and excluded every +output a defect not fixed upstream moved by more than a dollar: 28 are recorded +as engine-defect exclusions across the eleven such root causes (among them the +IRA deduction's compensation limit and phase-out, estate income, and the +heat-and-eat SNAP utility allowance that P.L. 119-21 ended for households +without an elderly or disabled member), and an output such a defect moves that +was already excluded for an unstated input keeps that record. On +policyengine-us 2.15.17 each of these outputs keeps the value its defect +produced, or it moved and a re-review found the defect still present. +Twenty-eight depend on an input the prompt never states, such as whether a +person meets SSI's definition of disability, months of SSDI receipt, weekly +hours worked, the type of survivor benefits, who paid for the coverage behind a +disability benefit, whether an adult tax dependent is the filers' child, or +whether a listed state and local tax refund counts as income, which turns on +whether the refunded tax reduced federal tax in the year the household paid it +(26 U.S.C. 111(a)); each was recomputed under the other reading a careful +reader could take, on the engine version that produced its reference, and it +moved. One of them, a California household head's Medicaid eligibility, moved +only once the engine read SSI's definition of disability where the law does. +The head's income is above the limit for the adult expansion group, so only a +disability pathway leads to Medi-Cal, and California's Working Disabled Program +requires SSI's definition of disability (42 CFR 435.540(a)). policyengine-us +2.15.17 tests the general disability flag instead and gives 1 under either +reading; with the program's test reading SSI's definition of disability, the +head qualifies only by meeting it. +Exclusion is symmetric: rows that matched the frozen reference leave the score +with rows that did not, so every model is scored on 1,928 of its 1,984 +requested outputs. The rows on those outputs stay annotated as description: +each carries its exclusion's class, except the 55 answers that never parsed, +which stay parse_contract_failure; no scored row carries a descriptive class. +The prompt states disability as one general fact +(any of the six Current Population Survey disability-difficulty items); SSI, +SNAP, Medicare and the tax code each apply their own determination, which no +benchmark person carries, so a disabled person's references take the +non-disabled path unless another listed fact establishes the determination +(manuscript section "Disability in the household facts"). Do not read a $0 SSI +reference for a disabled under-65 household member as a finding about that +person's SSI eligibility. + +On policyengine-us 2.15.17, PolicyBench re-ran four of the September 22 sweeps +behind these exclusions over every output: the IRA deduction limit fix, the net +investment income tax definition, and the readings for mortgage residence and +40 unlisted weekly hours. It ran a new sweep for the state and local tax refund +reading. Set against the same 2.15.17 calculation without its fix or reading, +no sweep moves a scored output by more than the $1 exact-match tolerance, and +three scored outputs move by less. The other ten defect fixes and the other +readings (SSI's definition of disability, months of SSDI receipt, survivor +benefits, disability coverage, adult dependents and Massachusetts bank +interest) ran on 1.755.4 only. The engine-upgrade review covered the outputs +the upgrade moved, and through them it flagged the California Medicaid output +above, which the reading of SSI's definition of disability does not move on +2.15.17 because the engine's Working Disabled Program test reads the general +disability flag. An output that one of those fixes or readings would move on +2.15.17, but did not move on 1.755.4, could still be scored. Canonical runs require numeric answers and explanations for each requested output. If future prompt-contract ablations omit explanations, they should be @@ -173,7 +274,7 @@ Discipline for private files: - Run evaluations on the private split by passing the private manifest explicitly (`--scenario-manifest .../scenarios-private.csv`); the eval and analyze commands need no other changes. -- Activation is a snapshot decision: the current 2026-09-05 snapshot scores +- Activation is a snapshot decision: the current 2026-09-30 snapshot scores 100 public households whose scenario manifest was generated on 2026-06-12 from a 125-household request split with seed 1042. It does not report protected scores. The first snapshot that reports protected scores should @@ -197,9 +298,9 @@ the site's types and leaderboard already filter on `condition`. ## Cost basis -Each frozen row uses its recorded per-call cost: provider-reported where the -provider returns one, otherwise reconstructed at the configured list price at -request time. List-price overrides apply at request time, not retroactively to +Each frozen row uses its recorded per-call cost: reconstructed from token counts +at the list price configured at request time, or the provider-reported charge +where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Published model costs retain these recorded totals rather than repricing past calls at today's rates. diff --git a/docs/gpt61sol/brief_S3_prose_and_pins.md b/docs/gpt61sol/brief_S3_prose_and_pins.md new file mode 100644 index 00000000..5fd6adde --- /dev/null +++ b/docs/gpt61sol/brief_S3_prose_and_pins.md @@ -0,0 +1,63 @@ +# Stage 3 brief: note, paper, card and pins for release dashboard-data-20260930 + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver`, branch `gpt61sol-driver`. The release data is already frozen and committed (`git log --oneline -3`). Your job is every documentation, app-copy, note, paper and test-pin change the release needs, so that the full test suite, the app checks and the paper render pass. Do not change data, references, exclusions, adjudications, the payload or the frozen snapshot, except through the documented regeneration commands below. + +Read the repository's `CLAUDE.md`, `docs/gpt61sol/design.md` (the commands after the freeze are near its end) and, as the model for this job, `docs/adds0928/brief_S3_prose_and_pins.md` with its report `docs/adds0928/report_S3.md`. This release is smaller than that one: one model joins, and nothing else moves. + +## What changed in this release (verified facts; cite only these or what you verify yourself) + +- **Release.** `dashboard-data-20260930`: 46 models, the 45 of `dashboard-data-20260929` plus GPT-6.1 Sol. 100 households, 1,984 outputs, 1,928 scored, 56 excluded, all as in 20260929. The payload sha256 is in `app/src/data.artifact.json`. +- **Nothing else moved.** The references, the exclusions and every incumbent's statistics are release 20260929's, byte for byte; the driver and the freeze refuse anything else (`scripts/finish_gpt61sol.py` `incumbent_drift`, `scripts/freeze_gpt61sol.py` `verify_incumbent_stats`, `verify_references`). The release adds no adjudication. +- **Staged result** (weighted exact-match share; recompute from the frozen payload rather than trusting these): + - GPT-6.1 Sol: 90.595%, rank 8 of 46, at about $0.023 a household. All 1,928 scored answers parsed. + - Its neighbours: Claude Fable 5.1 90.828% (rank 7) and Kimi K3 90.435% (rank 9). + - The top three keep their places: GPT-6 Sol 95.003%, Claude Opus 5.5 93.697%, GPT-5.6 Sol 93.572%. GPT-6 Sol costs about $0.027 a household. + - So GPT-6.1 Sol scores about 4.4 points below GPT-6 Sol. State the gap; give no reason for it unless you can show one from committed data. +- **Treatment.** `policybench/model_cards.py` `gpt-6.1-sol`: the forced tool contract on the Responses API with provider-default reasoning, like GPT-6 Sol. `policybench/paper_results.py` and `app/src/modelMeta.ts` give its display name and its release date, 2026-09-29, the day OpenAI announced it. Prices are in `policybench/config.py`. +- **Failure-reason judges.** GPT-6.1 Sol joined 134 audit cases (cases where it missed), and an Opus 5.5 judge re-read each. `docs/gpt61sol/judge_provenance.json` records how each of those 134 verdicts was produced; read its note and use only what it says: + - 60 ran isolated: from an empty directory outside the repository, with no tools and a token login, with an empty session context. + - 74 ran through the earlier runner from inside the repository. Their transcripts show no file read, search or shell call, but each one's context carried the login's account e-mail, the repository's git status and a skill listing. + - Scores do not depend on judge verdicts. The verdicts feed the published failure reasons (case notes and row annotations). + - The note and the paper must disclose the 60 and 74 split plainly, and link the record. + - A follow-up release (`dashboard-data-20260930b`, branch `judge-isolation-20260930b`) is planned for verdicts carried over from earlier releases. Do not describe it as done, and do not promise what it will do. +- **Wording amendments.** The release applies 394 wording-only amendments to published failure reasons in re-judged cases (`results/local/gpt61sol-v1/wording-amendments.json`, read only; the policy is in `docs/gpt61sol/design.md`). No class, exclusion or score changes through them. Mention them only if the paper or card already describes this mechanism for an earlier release; do not invent a description. + +## Deliverables + +1. **A new release note** in `app/src/notes/`, dated 2026-09-30, registered in `app/src/notes/index.ts`, following the structure of `2026-09-29-claude-sonnet-5-5-debuts-fifth.json` and `2026-09-22-gpt-6-sol-debuts-first.json`, and the fact-pinning of `tests/test_notes.py` (see `test_release_20260929_note` and its `_facts` helper). + - Cover: GPT-6.1 Sol's score, rank and cost against GPT-6 Sol and its neighbours; its treatment; that nothing else in the release moved; and the judge disclosure above. + - Every number in the note must be a fact a test recomputes from committed data. Add the tests. `CURRENT_RELEASE_SNAPSHOT` and the earlier release's "current release" tests need the same handover the 20260929 note got from 20260922c (`test_previous_release_scores_rebuild_from_this_snapshot` and its neighbours). + - Title in sentence case. The slug follows the earlier ones: `2026-09-30-gpt-6-1-sol-debuts-`. +2. **The paper and its render.** + - First run `scripts/sensitivity_by_variable.py` and `scripts/rescore_sensitivity_summaries.py --release dashboard-data-20260930`, as the design note lists. + - Update `paper/index.qmd` and `docs/paper.md` for 46 models and the response window, and add the GPT-6.1 Sol row wherever the roster or serving table is prose. + - Render with `paper/render_paper.py`, then re-pin with `scripts/freeze_snapshot.py --rendered-only`. + - Keep statements about earlier releases true to those releases. +3. **The benchmark card, methodology copy and other docs.** Update `docs/benchmark_card.md`, the app components that state counts, `app/src/data.versions.json`'s description, and the app copy of the serving config (`app/src/model-serving-config.json`, refreshed from `paper/snapshot/20260501/model_serving_config.json` as prepare-data does). Update the sensitivity docs and the cost report the tests compare. +4. **Test pins.** Update tests that pin facts of the live release (45 models and whatever follows from it) to this release's values. Recompute each new value from the frozen artifacts; never copy a failing assertion's number. Leave pins that belong to earlier releases alone. `tests/test_paper_results.py::test_frozen_roster_has_45_display_names_and_release_dates` becomes 46. + +## Voice (public copy: note, paper, card, methodology copy) + +Before writing any prose, read Max's voice guide `~/.claude/projects/-Users-maxghenis/memory/voice_max.md` (its "Current rules") and the working model `~/.claude/projects/-Users-maxghenis/memory/user_max_working_model.md`, and apply them to every string you write or change. In particular: +- active voice, with the actor in the subject; +- one thought per sentence, with varied rhythm; +- no self-referential clauses and no empty "X, not Y" antithesis; +- no superlatives or intensifiers, and numbers over adjectives; +- sentence-case headings; +- the paper and card state what the benchmark does now, with no draft or review lineage; +- a release note takes the change as its subject and keeps the numbers; +- no policy positions, and no speculation about why a model scores as it does. + +Keep the notes' register (third person, "PolicyBench …"). + +## Rules + +- **No fabricated claims.** State only what you read in committed code or data, or computed yourself. Messaging docs and briefs, this one included, are claims: verify each figure above before you use it. +- **Save progress as you go.** Commit each deliverable as soon as its own tests pass, in the order above. Don't hold everything for one commit at the end: a job that stops on a usage limit must leave its finished parts committed. Start by checking `git log` and `git status` for work an earlier attempt left, and continue it. +- **Workflow.** + - Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. + - Run `ruff format` and `ruff check .`. + - Run the full `pytest -q`, serially (the machine is shared; a parallel run has been killed for memory), and the app's `bun install --frozen-lockfile`, `bun run test`, `bun run lint` and `bun run build` in `app/`. +- **Commits.** Each message ends with `Co-Authored-By: Claude Opus 5.5 `. Do not push, open PRs, upload release assets, or post anything. Do not edit the frozen snapshot data files, the references or the adjudications by hand. +- **Never delete anything you did not create.** Never `rm -rf` a shared directory or a glob. +- **Report.** End with a report listing every file changed, every test pin changed with old → new and where the new value came from, and the final results of pytest and the app checks. Write it to `docs/gpt61sol/report_S3.md` and commit it. diff --git a/docs/gpt61sol/brief_after_rejudge.md b/docs/gpt61sol/brief_after_rejudge.md new file mode 100644 index 00000000..44809540 --- /dev/null +++ b/docs/gpt61sol/brief_after_rejudge.md @@ -0,0 +1,43 @@ +# Brief: finish the GPT-6.1 Sol stage after the re-judge (restate, amendments, triage, export) + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver` (branch `gpt61sol-driver`). Read the repo `CLAUDE.md`, `docs/gpt61sol/design.md`, `docs/gpt61sol/brief_rejudge_unhardened.md` and the progress log `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/gpt61sol/rejudge_report.md`. + +State: all 134 new verdicts in `results/local/gpt61sol-v1` now come from the hardened runner. +- 17 ran on subfleet lane claude-10. +- The trial, `us__scenario_000__federal_income_tax_before_refundable_credits`, ran on Max's first setup-token login. +- The other 116 ran on his second setup-token login. Its sidecars declare `claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30`. + +The old verdicts are in `rejected-verdicts/`. + +If the runner stopped early on a usage limit, some to-do cases have no verdict. Check the tail of `.../gpt61sol/rejudge_run2.log` and the count of present verdicts. In that case: +- Restore those cases' old verdicts from `rejected-verdicts/`. Take the entry whose `reason.txt` is the unhardened-judge reason with today's timestamp. +- Record exactly which verdicts are old and which are new. +- Continue. Max agreed to ship a disclosed mix rather than wait. + +Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. Make no judge calls. The judging is done, and this brief needs no model calls. + +## Steps + +1. **Validate.** Every present verdict must pass the driver's `validate_verdicts`. Every new one must carry `judge_isolation`, with `judge_effort` xhigh, Opus 5.5 served, and `prompt_sha256` bound. Read at least 10 of the new transcripts (`claude.transcript.jsonl` beside each verdict). Each must show exactly one tool call, `StructuredOutput`, and no userEmail, credential_org, skill or git context. +2. **Restate** the adjudication records of re-judged cases that have one, with `scripts/restate_gpt61sol_adjudications.py`. +3. **Rebuild the wording amendments** for the re-judged cases in `wording-amendments.json`: 288 in 11 cases (005, 007, 053 and 099 state income tax; 008 payroll tax; 008 state refundable credits; 020 federal income tax; 023, 080 and 100 snap; 067 ssi). + - Apply the same rules that made them: `docs/gpt61sol/design.md`, the amendment policy, and `finish_gpt61sol.py load_amendments`. + - Each amendment must match its new text exactly once and must be true. + - Drop the ones the new text makes unnecessary. Add any the new text needs. No published case note or row annotation may state something the record contradicts. For Idaho 007, no text may call the release's $4,811 convention a model error. + - Never change a class, an exclusion or a score. + - Report the before and after counts per case. +4. **Triage:** `--step triage`. If a flag needs a new decision, STOP and report it. +5. **Export:** `--step export`. Report: + - GPT-6.1 Sol's exact rate and rank; + - the no-drift result for all 45 incumbents; + - the new payload sha256; + - old against new class distribution for the 134, with the number of changed classes and changed flags. +6. **Checks.** Run `pytest -q -m "not slow"`; only the frozen-roster test may fail. Run the two driver test files including the slow tests, and `tests/test_run_audit_claude.py`. Run `ruff check .` and `ruff format --check .`. +7. **Commit** tracked changes coherently. Each message ends with `Co-Authored-By: Claude Opus 5.5 `. Append each step's result to the progress log as you go. + +## Rules + +- Never delete anything you did not create. Never `rm -rf` a shared directory or a glob. +- Don't push, open PRs or upload. +- Stop before the freeze. +- Final answer: everything asked for in steps 1–6, and `git status`. diff --git a/docs/gpt61sol/brief_driver.md b/docs/gpt61sol/brief_driver.md new file mode 100644 index 00000000..84894451 --- /dev/null +++ b/docs/gpt61sol/brief_driver.md @@ -0,0 +1,69 @@ +# Brief: the GPT-6.1 Sol additions driver + +> Superseded base (2026-09-29): the values below are PR #182's head. #182 changed +> after review and merged as `d616e67c`; the driver now pins that release (56 +> exclusions, 1,928 scored, payload `a5cb9989…`). See `design.md` and +> `brief_repin_prepare.md`. + +Work in this worktree (branch `add-gpt-6.1-sol`, from the head of PolicyEngine/policybench#182). Read the repository's `CLAUDE.md` first. Then read `scripts/finish_adds0928.py`, `scripts/freeze_adds0928.py`, `tests/test_finish_adds0928.py`, the freeze script's tests, and `docs/adds0928/stage2_design.md`. + +## What the release is + +PolicyBench is adding one model, GPT-6.1 Sol (`gpt-6.1-sol`, registry key `gpt-6.1-sol`, run slug `gpt61sol`), to the 45-model board of release `dashboard-data-20260929`. That release is PR #182, which is being merged now. The references do not change. Every one of the 45 incumbents' `modelStats` must come out byte-identical. + +The base (read these from this worktree; they equal #182's head): + +| Item | Value | +|---|---| +| `app/src/data.artifact.json` tag | `dashboard-data-20260929` | +| Payload sha256 | `d146473d9bd7776638c59a0a20774dbe9026d8bcee0f2201e114146609ddf246` (123,324,948 bytes) | +| Models | 45 | +| Outputs | 1,984 (100 households); 1,929 scored, 55 excluded | +| `reference_outputs.csv` sha256 | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `reference_outputs.csv.meta.json` | `5469664726adef3675f4cb4504021bd5c24c8acbfa00b21d943854cd84f14bda` | +| `reference_exclusions.json` | `ae28ade59705e6314f4d1b5fbd906ec58af0d503e39a59a7e13b597679676f67` | +| `scenarios.csv` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | +| `scenarios.csv.meta.json` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | + +Recompute every hash yourself from the files in `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/`, and stop if any differs from this table. + +## Deliverables + +1. **`scripts/finish_gpt61sol.py`**, adapted from `finish_adds0928.py`, with the same steps (`prepare`, `judge`, `triage`, `export`) and the same `--early`/`--partial` rehearsal. + - `MODELS = {"gpt61sol": "gpt-6.1-sol"}`, and 46 models after the fold. + - `BASE_TAG = "dashboard-data-20260929"` with the sha above. Require 45 base models and 55 exclusions. + - Keep `RELEASE_TAG` as one constant. Set it to `"dashboard-data-20260930"`. The lead may change it at freeze time, so nothing else may hard-code the tag. + - **No reference revision.** Remove `reference_revision`, `replay_base_references` and the drift report. + - Pin the five reference files above in a `BASE_REFERENCE_SHA256`-style table. + - Require the committed files to match their pins in `resolve_base` and again in `export`. + - Export must require zero incumbent `modelStats` drift: all 45, with nothing tolerated. Keep the Fable 5 historical-usage carry-over and the zero-cost Ox Alpha handling if the exporter still needs them; read why they exist in `stage2_design.md` and the code first. + - **After the freeze,** the working-tree snapshot is the new release, so a re-export must read the base payload from git. + - Keep `base_commit_blob` and `resolve_live_base`, with `BASE_COMMIT` pointing at the commit whose tree holds release 20260929. + - Set `BASE_COMMIT = "f7ced3b37643ecfdb90ec383339d0244b7017bbb"` (#182's head) with the comment `# PR #182 head; the lead repoints this to its merge commit on main`. The lead will replace it after the merge. + - Seed the audit from the 20260929 stage: `--audit-seed /Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/results/local/adds0928-v3/audit`. Its grounding comes from wherever `finish_adds0928.py` read it; trace this through the code and the stage's `stage.json`, and document the exact prepare command in the design doc. + - Carried-over verdicts must stay bound to their prompt sha256, as now. + - Any case whose prompt changes because GPT-6.1 Sol's answer joins it must be re-judged. + - The judge must cover all 46 models in a case. +2. **`scripts/freeze_gpt61sol.py`**, adapted from `freeze_adds0928.py`. `NEW_MODELS` is GPT-6.1 Sol only, and it keeps the treatment validation. + - The reference sidecar, exclusions and adjudications must not change unless triage records an adjudication. + - Drop the manifest pin catch-up for a reference revision; there is none. +3. **Tests.** + - Add `tests/test_finish_gpt61sol.py` and a freeze test, adapted from the existing ones, with synthetic data where the originals use it. + - Include: the 45-incumbent no-drift gate refuses any change; a reference file that misses its pin is refused; incomplete or stopped runs are refused; the wrong model in a run state is refused; `base_commit_blob` names a missing commit. + - Where a property holds for all inputs, add a Hypothesis test. For example, the fold keeps every incumbent row byte-identical and adds exactly 1,984 rows for the new model with the reference key set. `hypothesis` is a dev dependency. + - The existing `finish_adds0928`/`freeze_adds0928` tests must keep passing unchanged. +4. **`docs/gpt61sol/design.md`**, short: + - what the release is and what the gates enforce; + - the exact command sequence (prepare, judge, triage, export, freeze dry-run, freeze, paper render, `freeze_snapshot.py --rendered-only`), with the real paths: + - runs root `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609`; + - run `gpt61sol/run`; + - stage dir `results/local/gpt61sol-v1` in this worktree. + +## Rules + +- Do not run `prepare`, `judge`, `triage`, `export` or the freeze against real data. Do not touch `results/`, the snapshot, the references, the annotations or the payload pointer. The lead runs the pipeline once GPT-6.1 Sol's run finishes. +- Use the Python at `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, which has policyengine-us 2.15.17 and hypothesis. Run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`, serially; the machine is shared. +- The frozen-roster test `tests/test_paper_results.py::test_frozen_roster_has_45_display_names_and_release_dates` fails on this branch until the refreeze. That is expected; leave it. +- Before you finish, run `ruff check .`, `ruff format --check .` and `pytest -q -m "not slow"`. Report every failure other than the one above. +- Make coherent commits, each message ending with `Co-Authored-By: Claude Opus 5.5 `. Do not push, open PRs, upload anything, or call any model API. +- Final answer: the files you added and changed, the gates with the tests that pin each one, the exact pipeline commands, and the test results. diff --git a/docs/gpt61sol/brief_driver_review.md b/docs/gpt61sol/brief_driver_review.md new file mode 100644 index 00000000..64d8253d --- /dev/null +++ b/docs/gpt61sol/brief_driver_review.md @@ -0,0 +1,21 @@ +# Brief: independent review of the GPT-6.1 Sol release driver + +Review, read-only, the release driver for adding GPT-6.1 Sol to PolicyBench release `dashboard-data-20260929` (45 models). The new release is additions-only: the references must not change, and all 45 incumbents' `modelStats` must come out byte-identical. + +- Worktree: `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver`, branch `gpt61sol-driver`, HEAD 229730ba. +- Diff: `git diff origin/main..HEAD`. origin/main is d616e67c, release 20260929 as merged. +- Start by reading `docs/gpt61sol/design.md`, `scripts/finish_gpt61sol.py`, `scripts/freeze_gpt61sol.py`, `tests/test_finish_gpt61sol.py` and `tests/test_freeze_gpt61sol.py`. + +Try to break it. Could any of these get through? +- a reference or exclusion change; +- incumbent drift; +- a stale or unbound judge verdict; +- a verdict from the wrong model or covering the wrong set of models; +- a dropped adjudication; +- a tag or roster mistake; +- a wrong pin: recompute every pin from the committed files; +- a test that cannot fail. Mutate the code in a temp copy (`mktemp -d` and `git worktree` or `git archive`), never in the worktree. + +Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=`. The stage in `results/local/gpt61sol-v1` is live and another job is writing to it: read it, never write it. + +First line of your answer: APPROVE or REQUEST CHANGES. Then list each finding with its severity (blocker, major or minor), its evidence (the commands and their output) and a fix. diff --git a/docs/gpt61sol/brief_judge_triage.md b/docs/gpt61sol/brief_judge_triage.md new file mode 100644 index 00000000..e504d81c --- /dev/null +++ b/docs/gpt61sol/brief_judge_triage.md @@ -0,0 +1,40 @@ +# Brief: judge and triage the GPT-6.1 Sol stage + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver` (branch `gpt61sol-driver`, HEAD 229730ba). Read the repo `CLAUDE.md` and `docs/gpt61sol/design.md` first. + +`prepare` has already run into `results/local/gpt61sol-v1`. It found 134 cases that need Opus 5.5, every one because GPT-6.1 Sol answered wrong and joined it; 540 carry over. You run in a Claude subscription lane, so judge calls bill this lane's account. Never use ANTHROPIC_API_KEY: unset it if it is set. + +Setup: + +```bash +PB_PY=/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python +PB_STAGE="$PWD/results/local/gpt61sol-v1" +export OPENBLAS_NUM_THREADS=1 PYTHONPATH="$PWD" +unset ANTHROPIC_API_KEY +``` + +## Steps + +1. **Judge at width 4.** + - Run `AUDIT_MODEL=claude-opus-5-5 AUDIT_PARALLEL=4 AUDIT_PYTHON="$PB_PY" bash scripts/run_audit_claude.sh "$PB_STAGE/audit"`. Read that script first to confirm the arguments. + - Then run `"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step judge`. It validates every verdict: exact model coverage, a hash-bound sidecar showing the served model `claude-opus-5-5`, and no hedging. It retries whatever is missing or invalid, one at a time. + - If the judge step exits after three passes, investigate and resume. Do not weaken validation. +2. **Triage:** `"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step triage`. + - About 54 of the 134 re-judged cases have adjudication records, 46 of them scoring exclusions. If a re-judge changes an adjudicated case's class, restate the new class in its staged record (`$PB_STAGE/publish/us_full_run_20260612_policyengine_4_16_1_populace/annotations/us_adjudications.json`), exactly as `design.md` and the 20260929 records did: the current judge class goes in, the replaced class moves to `judge_previous`, dated from its sha256-bound sidecar. + - Keep the judge's exact class. Never invent a class or a decision to unblock triage. + - Report any `reference-flags.csv` row that asks for a new decision (a reference defect or a new exclusion) and STOP: an exclusion that moves has no path in this release. +3. **Stop after triage passes.** Do not run export or the freeze. + +## Rules + +- Run heavy steps serially, apart from the judge width above. +- Deletion: never delete anything you did not create in this task. Never `rm -rf` a shared directory or a glob. +- Don't push, don't open PRs, and don't upload anything. +- Commit nothing unless you change tracked files. Stage outputs are git-ignored. + +**Final answer:** +- the judge counts: judged, retried, and the served-model tally from the sidecars; +- the class distribution of the 134 new verdicts; +- every adjudication record you restated, with its old and new class and why; +- the triage output, verbatim; +- any flags you stopped on. diff --git a/docs/gpt61sol/brief_rejudge_unhardened.md b/docs/gpt61sol/brief_rejudge_unhardened.md new file mode 100644 index 00000000..7e464234 --- /dev/null +++ b/docs/gpt61sol/brief_rejudge_unhardened.md @@ -0,0 +1,46 @@ +# Brief: re-judge the unhardened GPT-6.1 Sol verdicts with the hardened runner, then export again + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver` (branch `gpt61sol-driver`, HEAD e221b4ce). Read the repo `CLAUDE.md` and `docs/gpt61sol/design.md` first. + +The previous fix job committed the review fixes and hardened `scripts/run_audit_claude.sh`: each judge now runs from an empty directory outside the repo, with tools disallowed, on the lane login only. It then re-judged 17 verdicts and exported. It was cancelled before it could write its report, but the worktree is clean and the stage `results/local/gpt61sol-v1` holds its export. + +Of the 134 new verdicts in the stage, 117 still come from the old, unhardened runner: their sidecars have no `judge_isolation`, and `prompt_sha256_source` says the prompt binding was stamped from the transcript. A separate audit found that each of those 117 judge sessions ran inside the repo, with an injected workflow-authoring skill message in context. PolicyBench will publish every new verdict with one clean provenance, so re-judge all 117. + +Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. Unset ANTHROPIC_API_KEY; judges must bill this lane's login only. + +## Steps + +1. List the 117 new verdicts whose sidecars lack `judge_isolation`, and check the count. + - Do NOT touch carried-over verdicts. In particular, another session owns and will re-judge 086 federal_refundable_credits, 045 snap, 014 federal_refundable_credits, 015 federal_refundable_credits and 015 head_medicaid_eligible. + - Move each of the 117 old verdicts aside into `rejected-verdicts/`, following the fix job's pattern. +2. Re-judge them with the hardened runner at width 4. Then run `--step judge`. Every new verdict must pass validation, with `judge_isolation` present, `claude-opus-5-5` served, `prompt_sha256` bound, and no hedging. +3. Check isolation from the transcripts: none of the 134 judge sessions may show a tool call or an injected skill or git-status message. Report what you checked. +4. Restate the adjudication records for re-judged cases that have one, with `scripts/restate_gpt61sol_adjudications.py`. +5. Re-check `wording-amendments.json` against the new text. + - Each amendment must still apply exactly once and still be true. + - Drop any a re-judge made unnecessary. + - Add any that the new judge text requires. The Idaho 007 state rule applies: no published text may call the release's $4,811 convention a model error. + - Never change a class, an exclusion or a score. +6. Run `--step triage`, then `--step export`. Report: + - GPT-6.1 Sol's exact rate and rank; + - that the no-drift gate passed for all 45 incumbents; + - the class distribution of the 134 new verdicts, old against new, with counts of changed classes and changed flags; + - any triage flag. If a flag needs a new decision, STOP and report it. +7. Stop before the freeze. + +## Rules + +- Checks: `pytest -q -m "not slow"` must pass, except the expected frozen-roster test, plus the two driver test files including the slow ones. `ruff check .` and `ruff format --check .` must pass. +- Commit tracked changes coherently. Each message ends with `Co-Authored-By: Claude Opus 5.5 `. +- Never delete anything you did not create. Never `rm -rf` a shared directory or a glob. +- Do not push, open PRs or upload. +- Write the final report to the `-o` path as you go. The previous job was cancelled before it wrote one. + +## Running on the desktop login (Max, 2026-09-30) + +Max asked for this job to run on the active desktop account, because no other Opus lane is free. The hardened `scripts/run_audit_claude.sh` refuses the desktop login's config directory on purpose. That guard came after 22 verdicts billed the desktop account by accident. + +- Add an explicit, narrow opt-in. `JUDGE_ALLOW_DESKTOP_LOGIN=1` lets the runner use the desktop login's config directory. All the other isolation stays as it is: empty cwd, tools off, allowlisted environment, and no API key. +- Record the opt-in in each verdict's sidecar, for example `judge_account_declared: "desktop login (JUDGE_ALLOW_DESKTOP_LOGIN, Max 2026-09-30)"`, so the published provenance says exactly where each verdict ran. +- Test both paths. Without the variable, the runner still refuses the desktop directory. With it, the runner accepts it and records it. +- Use the opt-in only if this job's lane is the desktop login. Otherwise keep the lane-only path. diff --git a/docs/gpt61sol/brief_repin_prepare.md b/docs/gpt61sol/brief_repin_prepare.md new file mode 100644 index 00000000..7fa82004 --- /dev/null +++ b/docs/gpt61sol/brief_repin_prepare.md @@ -0,0 +1,51 @@ +# Brief: repin the GPT-6.1 Sol driver to release 20260929 as merged, then run `prepare` + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver` (branch `gpt61sol-driver`). Read the repo `CLAUDE.md`, `docs/gpt61sol/brief_driver.md` and `docs/gpt61sol/design.md` first. + +The driver (`scripts/finish_gpt61sol.py`, `scripts/freeze_gpt61sol.py` and their tests) was built against PR #182's head `f7ced3b3`. After review, #182 changed before it merged: +- It excluded `scenario_023 head_medicaid_eligible` as an audit exclusion. That makes 56 exclusions (28 engine defect, 28 unlisted input) and 1,928 scored outputs. +- It rewrote the reference sidecar's reasons. +- It hardened `finish_adds0928.py`, adding `check_exclusions` and `audit_exclusions` in `reference_audit/2026-09-28/final_actions.json`. +- It merged main's #181 (provenance computed once per run). + +It merged as `d616e67c` on origin/main, and release `dashboard-data-20260929` is now latest. The payload sha256 is `a5cb9989d78cb18d040fec2f1f5d0775df15b9b917ae99d883d8701b7fa480a7` (123,362,946 bytes). + +## Do + +1. **Merge.** Run `git fetch origin && git merge origin/main`. Resolve any conflicts; the driver files are new, so there should be few. +2. **Repin every base constant to the merged release, computing each value from the files now committed:** + - `BASE_COMMIT = "d616e67c33b6f80dabf5cb7329f069f9a1de069d"`; + - `BASE_SHA256`; + - the reference-file hash table; + - the counts: 45 base models, 56 exclusions, 1,928 scored, 1,984 outputs. + + Remove the "#182 head" placeholder comment. The no-reference-revision rule stays: the committed references must equal release 20260929's, byte for byte. + + The audit seed's prompts must still re-render byte-identically from the committed snapshot. The 023 case's prompt may have changed, since its adjudication and case note changed; handle this deliberately. A case whose prompt changed is re-judged. Re-check the grounding pin as well. +3. **Tests.** Update every test pin with a recomputed value, never a copied one. Run `tests/test_finish_gpt61sol.py`, `tests/test_freeze_gpt61sol.py`, `tests/test_finish_adds0928.py`, then `pytest -q -m "not slow"`, `ruff check .` and `ruff format --check .`. Only `tests/test_paper_results.py::test_frozen_roster_has_45_display_names_and_release_dates` may fail, because GPT-6.1 Sol is registered but not yet frozen. +4. **Commit** coherently. Each message ends with `Co-Authored-By: Claude Opus 5.5 `. +5. **Run `prepare` on the real data**, exactly as `docs/gpt61sol/design.md` documents: + - runs root `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609` (the run is `gpt61sol/run`: 100/100, model gpt-6.1-sol); + - audit seed `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/results/local/adds0928-v3/audit`; + - grounding `/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/grounding.csv`; + - stage dir `results/local/gpt61sol-v1` in this worktree. + + Do NOT run `judge`, `triage`, `export` or the freeze. + + Report: + - how many cases need judging (new, and re-judged because a prompt changed), with a few examples; + - how many carry over; + - GPT-6.1 Sol's staged exact rate and rank from the early export, if the driver produces one. + +## Rules + +- Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. +- Run heavy steps serially. +- Deletion: never delete anything you did not create in this task. Never `rm -rf` a shared directory or a glob. Put scratch files in a fresh `mktemp -d` and delete only that. +- Do not push, open PRs, upload, or call any model or provider API. +- **Final answer:** + - every pin, old → new, with its source; + - the commits; + - the test and ruff results, verbatim; + - the prepare report; + - `git status`, which must be clean except for the git-ignored stage. diff --git a/docs/gpt61sol/brief_review_fixes.md b/docs/gpt61sol/brief_review_fixes.md new file mode 100644 index 00000000..59991c6c --- /dev/null +++ b/docs/gpt61sol/brief_review_fixes.md @@ -0,0 +1,58 @@ +# Brief: fix the GPT-6.1 Sol driver review findings, then export + +Work in place in `/Users/maxghenis/PolicyEngine/policybench-wt/gpt61sol-driver` (branch `gpt61sol-driver`). Read the repo `CLAUDE.md` and `docs/gpt61sol/design.md` first. + +An independent Opus review requested changes; its full text is in `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/gpt61sol/driver_review.md`. The reviewer had no shell, so verify every finding by running code before you change anything. The judge and triage job's report is in `.../gpt61sol/judge_report.md`, and its outputs are in the stage `results/local/gpt61sol-v1`. It may have left a committed or untracked `scripts/restate_gpt61sol_adjudications.py` and its test; review and keep them, fixing finding 7. + +Python: `/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python`, run with `OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD`. + +## Fix, each with a test that fails before the fix and passes after + +1. **Major: the adjudication record must not rewrite decisions it has no reason to touch.** + - Export binds `prompt-changes.json` in `release-ready.json`. + - `verify_adjudication_record` (freeze) and triage require each committed entry's non-judge fields to stay byte-identical, key order included. + - Judge fields may change only for case ids in the bound `changed ∪ added` lists. Use the `JUDGE_FIELDS` set from the restate script as the one definition. + - Replace `test_triage_may_restate_a_rejudged_class` with a pair: a re-judged case may be restated, and a rewrite of an incumbent-only case is refused. +2. **Major: every judged verdict is bound to its own bytes.** + - For every non-parse-only case, require `verdict.meta.json`'s `verdict_sha256` to equal the digest of `verdict.json`, and require `prompt_sha256` to be present and to match `prompt.md`. + - Cases that carry over from the seed must also keep `verdict.json` byte-identical to the seed's. Bind the seed digests in `stage.json` at prepare. + - Add a mutation test: edit a carried-over verdict and it must show as pending or refused. +3. **Minor: stamp `prompt_sha256` on the new Opus 5.5 verdicts.** Stamp it in the sidecar after each judge pass, or in the runner. Require it for every verdict that names GPT-6.1 Sol. Apply it to the 134 existing new verdicts, and check that each stamped hash matches `prompt.md` as it is now. +4. **Minor: the freeze's baseline for dropped adjudications** must come from git at `BASE_COMMIT` (`base_commit_blob`), not from the working-tree file the freeze overwrites. +5. **Minor: the freeze's `verify_receipt`** must also check `base_tag == BASE_TAG` and `base_sha256 == BASE_SHA256`. +6. **Minor: local-only tests.** Commit a small digest of the seed (case id, prompt sha256, verdict sha256 for each judged case) and test the committed digest against the real seed when it is present. Say in the design table which checks can run only on this machine. +7. **Minor: the restate script** must verify the new record in memory before it writes anything. + +## Then + +- Re-run `judge` (validation only; nothing should be pending) and `triage`, so the stage satisfies the stricter gates. +- Run `export`. Report GPT-6.1 Sol's exact rate and rank, and confirm the no-drift gate passed for all 45 incumbents. +- Stop before the freeze. + +## Rules + +- Checks: `pytest -q -m "not slow"` must pass, except the expected frozen-roster test, plus the two driver test files including the slow ones. `ruff check .` and `ruff format --check .` must pass. +- Commit coherently. Each message ends with `Co-Authored-By: Claude Opus 5.5 `. +- Never delete anything you did not create. Never `rm -rf` a shared directory or a glob. +- Do not push, open PRs, upload, or call a provider API. The judge's Claude CLI, which uses this lane's subscription, is allowed only if a verdict really must be redone. +- Final answer: per finding, the commit (or a rebuttal with evidence); the export result; the test results, verbatim; and `git status`. + +## Also from the judge/triage report (`.../gpt61sol/judge_report.md`; read it in full) + +8. **Judge independence.** + - Nine of the 134 new judges read the adjudication decision records while judging: 039 federal, 039 state, 042 state, 049 federal, 056 snap, 076 state, 091 state, 110 federal and 112 snap. A tenth (117 state) searched the repo. Eight read the live policyengine-us checkout. + - Harden `scripts/run_audit_claude.sh` so each judge runs: + - from a fresh empty directory outside the repo; + - with file, search, web and shell tools disallowed (check the installed `claude --help` for the exact flags); + - on this lane's own credentials. It must never fall back to the desktop login: 22 kept verdicts billed the desktop account. Refuse to run if the lane's config dir is not set, and record the account the call used in the sidecar if the CLI reports it. + - Add a test for the command line the script builds. + - Then find every new verdict whose judge transcript shows a file read, search or shell call, not only the ten above. Re-judge each with the hardened runner, restate its adjudication with the restate script, and re-run triage. + - Report which verdicts changed. +9. **Published wording.** + - Idaho 007 state: the new judge explanation calls the $4,811 zero bracket a model error, but the release's Idaho convention sets $4,811. The report also names out-of-date decision wording in 008 state refundable credits, 056 snap, 057 ssi and 067 ssi. + - No published case note or row annotation may state something the record contradicts. + - Re-judging with the hardened runner may resolve 007. If not, or for the stale decision wording, add a narrow mechanism: a stage file listing wording-only amendments (case id, field, old text, new text, reason) for re-judged cases only. Export binds that file, and finding 1's gate allows exactly the listed changes and nothing else. Test it both ways. + - Never change a decision's class, exclusion status or scoring. +10. **Two scored decisions the re-judge disputes.** 008 state refundable credits: the judge calls it later law, but New Jersey's law passed June 30, before the July 3 freeze. 056 state refundable credits: the judge calls it an engine defect, but the affirmation rules that out. The decisions stand. Confirm the published case notes for them read correctly. + +Do items 8 and 9 before export. The judge's Claude CLI is allowed for the re-judges. diff --git a/docs/gpt61sol/design.md b/docs/gpt61sol/design.md new file mode 100644 index 00000000..72607196 --- /dev/null +++ b/docs/gpt61sol/design.md @@ -0,0 +1,234 @@ +# GPT-6.1 Sol addition: design + +## What the release is + +This release adds GPT-6.1 Sol (`gpt-6.1-sol`, run slug `gpt61sol`) to the +45-model board of release `dashboard-data-20260929` (PR #182, merged to main +as `d616e67c`). The new board has 46 models. The 100 households, the 1,984 +requested outputs per model and the references do not change: 1,928 outputs +are scored and 56 are excluded (28 engine defects, 28 unlisted inputs). +There is no reference revision. Every incumbent's `modelStats` entry must come +out byte-identical to the 20260929 payload. + +The base is pinned in `scripts/finish_gpt61sol.py`. Each value was +recomputed from the files committed at `d616e67c`. The payload and +predictions were also checked against the published release assets' digests. + +| Item | Value | +|---|---| +| Base tag, payload sha256 | `dashboard-data-20260929`, `a5cb9989…80a7` (123,362,946 bytes; the committed `data.json.gz` rewraps to it) | +| Models, outputs, exclusions, scored | 45, 1,984, 56 and 1,928 | +| `reference_outputs.csv` | `e8bbba8f…2466` | +| `reference_outputs.csv.meta.json` | `816fef53…1a4b` | +| `reference_exclusions.json` | `bf4e6a24…81c2` | +| `scenarios.csv` | `71b16212…858a` | +| `scenarios.csv.meta.json` | `03a66e90…aebb` | +| `BASE_COMMIT` | `d616e67c…` (the merge of PR #182 on main) | + +The new release's tag is `RELEASE_TAG` in `scripts/finish_gpt61sol.py`, and +nothing else names it. `freeze_gpt61sol.py --tag` defaults to it, and the +commands below read it into `$PB_TAG`. The export receipt binds the tag. If +you change `RELEASE_TAG`, run `--step export` again before you freeze. + +## What the gates enforce + +Each gate refuses with `SystemExit` before anything is written outside the +stage. Test names are in `tests/test_finish_gpt61sol.py` (F), +`tests/test_freeze_gpt61sol.py` (Z) and `tests/test_run_audit_claude.py` (R). +Tests marked (local) need release 20260929's audit or its grounding on this +machine and skip anywhere else; see the note under the table. + +| Gate | Where | Pinned by | +|---|---|---| +| The run is complete, with `completed == total > 0` as integers and no `stopped_reason`. The run state names `gpt-6.1-sol`. `predictions.csv` is present. | `discover_new_models` | F `test_an_incomplete_or_stopped_run_is_refused`, `test_the_wrong_model_in_a_run_state_is_refused`, `test_a_missing_run_artifact_is_refused` | +| GPT-6.1 Sol's staged inputs are its pinned run. `--step pin-inputs` writes the sha256 of the finished run's `predictions.csv` and `run_state.json` to `docs/gpt61sol/input_pins.json`; export and the freeze read that file as committed at `HEAD`, refuse staged copies that differ, re-check the prepare-time hashes in `stage.json` and `model-provenance.json`, and compare every column of GPT-6.1 Sol's bundle rows with its run file, so its cost, tokens and latency are the run's. | `pin_inputs`, `committed_input_pins`, `verify_new_model_inputs`, `new_model_row_differences`, `export`, `freeze_gpt61sol.main` | F `test_pin_inputs_writes_the_finished_runs_file_hashes`, `test_the_input_pins_are_read_as_committed_at_head`, `test_export_writes_no_receipt_when_the_inputs_are_not_the_pinned_run`, `test_the_committed_input_pins_are_the_finished_runs` (local), `test_the_stages_inputs_are_the_pinned_run` (local); Z `test_the_freeze_refuses_new_model_usage_edited_in_the_bundle`, `test_the_freeze_refuses_a_new_model_input_edited_with_the_bundle`, `test_the_freeze_rechecks_the_prepare_time_hashes` | +| The five reference files equal their 20260929 pins. `prepare` checks the committed copies first, then the staged copies. `export` checks both again. The freeze checks the staged copies, the committed copies and the manifest pins, and after it writes the snapshot it checks the frozen copies. | `verify_reference_pins`, `resolve_base`, `prepare_inputs`, `export`, `verify_references` | F `test_a_reference_file_that_misses_its_pin_is_refused`, `test_resolve_base_checks_the_committed_pins_before_anything_else`, `test_export_refuses_a_*_reference_off_its_pin`, `test_the_committed_references_match_their_pins`; Z `test_any_reference_revision_is_refused`, `test_the_freeze_refuses_a_revised_staged_reference_before_mutation` | +| The base is 45 models, 1,984 outputs, 56 exclusions and 1,928 scored outputs, with the pinned payload and predictions. | `resolve_base`, `base_payload_from_commit` | F `test_the_git_base_must_rewrap_to_the_20260929_asset`, `test_the_base_commit_holds_release_20260929`, `test_the_committed_references_match_their_pins` | +| No incumbent drift. For all 45 incumbents, the exported entry serialized by `json.dumps` must equal the base entry byte for byte, key order included. A missing incumbent counts as drift. No drift is tolerated. | `incumbent_drift`, `export` | F `test_the_no_drift_gate_refuses_any_change_to_any_incumbent` and `test_incumbent_drift_is_exactly_the_changed_rows` (both Hypothesis) | +| The export roster is exactly the 45 incumbents plus GPT-6.1 Sol. | `export` | F `test_the_export_roster_must_be_the_incumbents_plus_the_addition` | +| After the freeze, a re-export reads the 20260929 payload from `BASE_COMMIT` and checks it against the base sha256. Any other pointer is refused. A missing commit is named, and the error says to fetch full history. | `resolve_live_base`, `base_commit_blob` | F `test_a_re_export_after_the_freeze_reads_20260929_from_git`, `test_base_commit_blob_names_*`, `test_any_other_pointer_is_refused` | +| The fold keeps every incumbent row. It adds exactly 1,984 rows for the new model, with the reference key set. | `fold_board`, via `prepare_inputs` | F `test_the_fold_keeps_incumbent_rows_and_adds_1984_rows_for_the_addition` (Hypothesis; see its note on column types) | +| The addition's treatment fingerprint matches the registry. | `validate_treatment` (reused from `freeze_adds0928.py`) | Z `test_the_registered_treatment_matches_the_run_and_a_drift_is_refused` | +| Every judged verdict is bound to its own bytes: its sidecar carries the verdict's sha256. A case whose prompt is the seed's carries its seed verdict over and must keep the seed's verdict bytes; any failure there is refused, not re-judged. Every other verdict must record the sha256 of the prompt it judged; the runner records it, and `scripts/stamp_gpt61sol_prompt_bindings.py` stamped the stage's first 134 re-judges only where each judge's own transcript shows `prompt.md`'s exact text. `prepare` binds the seed in `stage.json`, and `--step bind-seed` binds it for a stage prepared before that; the seed must match `docs/gpt61sol/seed_digest.csv` (each judged case's prompt and verdict sha256), whose bytes `SEED_DIGEST_SHA256` pins. Every read of the seed (`load_seed`, and so judge, triage, the freeze and `rejudged_cases`) re-derives kept, changed and added from the stage's prompts and refuses unless `prompt-changes.json` says the same, so a kept prompt that drifts or a case moved between the lists stops every step. When a prompt changes, `prepare_audit` drops its verdict. Invalid verdicts are set aside in `rejected-verdicts/`, never deleted. | `prepare_cases`, `bind_seed`, `load_seed`, `validate_verdicts`, `set_aside` | F `test_every_verdict_is_bound_to_its_own_bytes`, `test_a_carried_over_verdict_must_keep_the_seeds_bytes`, `test_an_edited_carried_over_verdict_is_refused_after_prepare`, `test_unchanged_incumbent_case_keeps_its_existing_judge`, `test_a_verdict_stays_bound_to_its_prompt_sha256`, `test_prepare_refuses_a_seed_the_committed_digest_does_not_record`, `test_load_seed_refuses_a_stage_without_a_binding_or_with_another`, `test_bind_seed_*`, `test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over`, `test_the_committed_seed_digest_is_the_pinned_one`, `test_the_committed_seed_digest_matches_the_real_seed` (local); `tests/test_stamp_gpt61sol_prompt_bindings.py` | +| Only a case that GPT-6.1 Sol joins may change or appear. An incumbent-only prompt that differs from the seed's is refused, and so is a seed case that disappears. `prompt-changes.json` lists the kept, changed and added cases. The manifest's claim is re-scored on every read of the seed: with `wrong_prediction_rows` over the staged predictions and references, a changed or added case must list GPT-6.1 Sol among its wrong models exactly when its prediction is wrong, and any other case it gets wrong must be parse-failure-only, so a kept case needs its prediction right. | `check_prompt_changes`, `verify_reopened_by_predictions` | F `test_an_incumbent_prompt_that_changes_is_refused`, `test_the_manifest_cannot_reopen_a_case_the_predictions_do_not`, `test_a_household_only_the_new_model_misses_becomes_a_new_case`, `test_check_prompt_changes_names_new_incumbent_only_cases`, `test_every_seed_prompt_rerenders_from_the_committed_snapshot` (slow) | +| The judge covers every wrong model in a case, up to all 46. A verdict that names GPT-6.1 Sol needs hash-bound Opus 5.5 provenance and its prompt's sha256. | `validate_verdicts` | F `test_the_judge_must_cover_all_46_models_in_a_case`, `test_new_model_verdict_requires_bound_opus55_provenance` | +| Each judge sees only its prompt and bills only the lane. It runs from a fresh empty directory outside any git repository, with every built-in tool removed (`--tools ""`) and the file, search, web and shell tools also denied by name, no MCP servers, skills, CLAUDE.md files or user settings (`--setting-sources project,local`), and an allowlisted environment (no API key, base URL, provider switch or keychain override reaches any claude call). Every judge runs at one explicit effort, `AUDIT_EFFORT` (default `xhigh`, the effort every turn of the stage's 17 first hardened re-judges recorded), by `--effort` and by environment; the caller's `CLAUDE_CODE_EFFORT_LEVEL` never reaches it. Its transcript is kept beside the verdict, and a verdict whose transcript shows any tool call but the structured answer, an unlisted context attachment, a session context that is not empty (an account e-mail, an organization or git status), a turn at another effort, an advisor model, or a working directory inside a git repository is rejected. The runner refuses to start unless `CLAUDE_CONFIG_DIR` names a directory other than the desktop login's `~/.claude` (by file identity) and `claude auth status`, run the same way, reports a first-party subscription login: the lane's token when one is set (then `AUDIT_ACCOUNT` must name the account, which a token login does not report), or a home login that is not the desktop's account. The one exception is Max's explicit opt-in of 2026-09-30, `JUDGE_ALLOW_DESKTOP_LOGIN=1`: the judges then run on the desktop's own claude.ai login with `CLAUDE_CONFIG_DIR` unset (the CLI finds that login only under its default directory), with no lane token and no `AUDIT_ACCOUNT`, and every sidecar declares `desktop login (JUDGE_ALLOW_DESKTOP_LOGIN, Max 2026-09-30)`; the rest of the isolation is unchanged. A judge the API refuses is logged with the CLI's error. A login that cannot judge stops the run (no further judge starts and the runner exits 1): the API refusing the login (HTTP 401, 403 or 429), or a transcript showing the login putting account context in the judge's context. Claude Code 2.1.284 adds the account e-mail (`session_context.userEmail`) and a `credential_org` record for any claude.ai login, the desktop's included, so on that version only a token login yields a clean verdict; the 2026-09-30 desktop trial's transcript shows both, and its login was refused 403 `oauth_not_allowed_for_organization`. Each sidecar records the login, the declared account, the effort level and the isolation. | `scripts/run_audit_claude.sh` | R `test_each_judge_runs_isolated_on_the_lanes_login`, `test_the_sidecar_binds_the_prompt_and_records_the_login`, `test_the_runner_refuses_anything_but_the_lanes_own_login`, `test_every_judge_runs_at_one_explicit_effort`, `test_the_desktop_opt_in_runs_on_the_desktop_login_and_records_it`, `test_the_desktop_opt_in_allows_the_desktops_own_login_only`, `test_a_verdict_from_a_judge_that_called_a_tool_is_rejected`, `test_a_transcript_showing_more_than_the_prompt_is_rejected`, `test_a_login_that_cannot_judge_stops_the_run`, `test_a_login_that_puts_account_context_in_the_judge_stops_the_run`, `test_a_verdict_that_is_not_the_judges_one_accepted_answer_is_rejected`, `test_an_event_of_an_unlisted_type_is_rejected`, `test_account_data_anywhere_in_a_transcript_rejects_it_and_stops_the_run`, `test_a_scratch_directory_inside_a_git_repository_is_refused` | +| The published judge provenance, `docs/gpt61sol/judge_provenance.json`, describes the staged new verdicts. It lists every case GPT-6.1 Sol re-opened, each once, with the staged verdict's and prompt's sha256. Its `isolated` flag must match the sidecar: only the hardened runner writes `judge_isolation`. An isolated verdict's `claude.transcript.jsonl` must pass the runner's transcript checks, ported to Python with the source cited. Every call must be StructuredOutput, and exactly one may be accepted; any other is one the schema refused. No attachment may fall outside the runner's list, which rules out a skill listing and `credential_org`. The session context must be empty, the working directory outside git, every turn at the sidecar's effort, and no advisor model. The one accepted call's input must equal the staged `verdict.json` (in the runner, the verdict it writes), and every event must be of a type the stage's 60 isolated transcripts carry (`queue-operation`, `user`, `attachment`, `atis-latch`, `last-prompt`, `assistant`, `cost-state`). Outside the judged prompt's own text and the assistant's turns, no key or string may hold an e-mail address and no key may name an e-mail, account, credential, organization or git status, at any depth; the runner treats such account data as the login's and stops the run. Export refuses a disagreeing record before it writes anything. The receipt binds the record's sha256 and every staged transcript. | `verify_judge_provenance`, `transcript_problems`, `export` | F `test_a_verdict_whose_isolation_disagrees_with_the_record_is_refused`, `test_an_isolated_verdict_whose_transcript_fails_the_runners_checks_is_refused`, `test_the_record_must_list_each_staged_new_verdict_once`, `test_an_answer_the_schema_refused_and_the_judge_gave_again_passes`, `test_the_runners_attachment_allowlist_is_the_gates`, `test_the_runners_event_types_are_the_gates`, `test_the_runners_account_patterns_are_the_gates`, `test_an_address_in_the_prompt_or_the_judges_own_words_passes`, `test_every_isolated_transcript_in_the_stage_passes_the_gate` (local), `test_export_writes_no_receipt_when_the_record_disagrees`; R `test_the_runner_and_the_driver_judge_transcripts_alike`, `test_the_runner_and_the_driver_read_account_data_alike` (Hypothesis); `tests/test_judge_provenance.py` `test_no_verdict_or_provenance_is_recorded_from_a_transcript_the_runner_rejects` | +| Every field of the published judge provenance is checked. Each entry's `judge_effort`, `judge_model_reported`, `judged_at_utc` and `judge_account_declared` must be its sidecar's (an e-mail address withheld as ``), its group may say `isolated: ` only if it is isolated, `counts` must be the tally of the groups, an isolated entry's sidecar must record a token login (`judge_auth.method` `oauth_token`), and the record and its entries carry only their listed keys and no e-mail address anywhere. | `verify_judge_provenance`, `withhold_addresses` | F `test_each_entrys_fields_must_be_its_sidecars`, `test_the_record_withholds_the_sidecars_address_and_names_none`, `test_the_counts_must_be_the_tally_of_the_entries_groups`, `test_an_isolated_verdict_needs_a_token_login`, `test_the_record_and_its_entries_carry_only_checked_keys`, `test_only_an_isolated_entrys_group_says_isolated`, `test_the_committed_provenance_record_is_public_and_tallied`, `test_the_record_and_its_isolation_note_are_the_sidecars` (local) | +| The grounding is the one the 20260929 audit used. | `prepare_cases` (`GROUNDING_SHA256`) | F `test_prepare_refuses_a_grounding_other_than_the_pinned_one`, `test_the_pinned_grounding_is_the_one_the_20260929_stage_used` (local) | +| The staged payload is what export builds from the bound bundle. Before it changes anything, `--dry-run` included, the freeze copies the bundle files the receipt binds into a scratch directory, checking each against its receipt hash, and runs export's own build (`build_payload`: `export_full_run`, Fable 5's carried usage, the roster, drift and schema gates) on the copy. The rebuilt bytes must equal the staged payload's; a difference is refused with the first differing paths. The copy keeps `export_full_run`'s writes (`data.json`, `us/data.json`, `us/analysis/`) out of the stage. On the real stage the rebuild adds about 53 s. | `rebuild_payload`, `build_payload`, `payload_text` | Z `test_the_freeze_refuses_a_payload_edited_after_export`, `test_the_rebuild_reads_a_scratch_copy_of_the_bound_bundle`, `test_the_rebuild_reads_only_the_bytes_the_receipt_binds`, `test_payload_differences_are_empty_exactly_when_the_bytes_match` (Hypothesis); F `test_the_freeze_rebuilds_exactly_what_export_wrote` | +| The freeze refuses a receipt that does not name release 20260929 as its base (tag and payload sha256), or does not bind the payload, the tag, 46 models, the references, the adjudications, the run state and predictions, `model-provenance.json`, `prompt-changes.json`, `stage.json` and, when the stage has them, the wording amendments. It refuses evidence that changed after export. | `verify_receipt` | Z `test_freeze_refuses_changed_or_unbound_evidence`, `test_the_receipt_must_bind_*`, `test_wording_amendments_present_in_the_stage_must_be_bound`, `test_the_default_tag_is_the_driver_release_tag` | +| Adjudications change only where GPT-6.1 Sol re-opened a case. The baseline is release 20260929's record read from git at `BASE_COMMIT`, never the working-tree copy the freeze overwrites. Every committed entry keeps every field byte for byte, key order and entry order included, except that a case in `prompt-changes.json`'s changed or added lists may rewrite its judge fields (`JUDGE_FIELDS` in `scripts/restate_gpt61sol_adjudications.py`, the one definition) and its reasoning exactly as a listed wording amendment says. Rewritten judge fields must be the restate script's: the case's current bound Opus 5.5 verdict as judge, dated by its sidecar, with 20260929's `judge_previous` plus one item, the verdict 20260929's entry names (its seed verdict's judge, classes, flag and day, as the sha256-bound sidecar records them). The file's bytes must be exactly its parsed content in the committed form, with 20260929's note, schema and date conventions. A new entry may decide only a re-opened case; none may be dropped; the scoring exclusions stay the 56. Triage checks this in memory before it writes, and the freeze checks it again. The committed record excludes exactly the 56 and keeps every seed verdict's class. | `verify_adjudication_changes`, `stage_adjudications`, `triage`, `verify_adjudication_record` | F `test_triage_lets_a_rejudged_case_restate_its_judge_fields`, `test_triage_refuses_any_other_change_to_a_recorded_decision`, `test_triage_refuses_a_judge_rewrite_of_an_incumbent_only_case`, `test_the_committed_adjudications_exclude_exactly_the_scoring_exclusions`, `test_every_committed_adjudication_decides_a_case_the_seed_judged`, `test_the_committed_adjudications_keep_the_seed_judge_verdicts`, `test_each_20260929_entry_names_its_seed_verdict_as_its_sidecar_records_it` (local); Z `test_judge_fields_written_by_hand_are_refused`, `test_a_rejudged_case_may_be_restated`, `test_a_rewrite_of_an_incumbent_only_case_is_refused`, `test_triage_may_add_a_decision_that_keeps_the_output_scored`, `test_a_dropped_decision_is_refused`, `test_a_new_exclusion_is_refused`, `test_the_freeze_refuses_a_dropped_adjudication_before_mutation`, `test_the_freeze_baseline_is_release_20260929_in_git_not_the_working_tree` | +| Published wording changes only as `/wording-amendments.json` lists: case id, field, old text, new text and reason. The case must be re-opened. The field must be an entry's `reasoning`, the case note (`case_annotation`) or one model's row annotation (`annotation`, which names the `model`), so no amendment can touch a class, an exclusion or a score. The old text must occur exactly once. Triage applies the list in order (a later amendment may rewrite words an earlier one wrote, and the freeze chains them the same way); every case note must still carry its exact adjudication sentence. The freeze checks each case-note and row amendment is in the staged CSVs and commits the list beside the record. | `load_amendments`, `stage_adjudications`, `amend_annotations`, `verify_annotation_amendments`, `freeze_amendments` | Z `test_a_listed_wording_amendment_is_allowed_and_nothing_else`, `test_load_amendments_*`, `test_an_amendment_must_find_its_old_text_exactly_once`; F `test_triage_applies_exactly_the_listed_wording_amendments`, `test_triage_refuses_an_amendment_it_cannot_apply_exactly` | +| The staged row annotations and case notes are what triage builds, in every column. The freeze rebuilds both from the verdicts, the staged adjudications and the listed amendments, and each staged CSV must be those frames' bytes as triage writes them, so a failure class, flag or count edited on a case no adjudication decides is refused. | `verify_annotation_amendments`, `annotation_csv_text`, `triage` | Z `test_the_freeze_refuses_a_failure_class_no_judge_gave`, `test_the_freeze_refuses_any_column_triage_did_not_write`, `test_the_freeze_refuses_columns_or_rows_triage_did_not_write` | +| `RELEASE_TAG` is the only place the new tag is spelled out, in the driver's scripts, tests and this note. | `finish_gpt61sol.py` | F `test_the_release_tag_is_named_in_one_place` | + +What the stage's amendments correct. The judge's prompt presents each +output's frozen reference as correct, but an excluded output keeps its frozen +value, which can rest on an engine projection that a publication convention +replaces, or on an engine defect. The app frames an excluded output's row with +"This audit note compares the answer with the frozen reference, which carries +the engine defect described above" (or "which assumes one reading of the +unlisted input described above"), and it does not display case notes. The +331 amendments in `wording-amendments.json` for the GPT-6.1 Sol stage change, +in re-opened cases only: +- text that calls an answer an error when a convention the caveat does not + name makes it right (Idaho's held $4,811 zero bracket, SNAP's FY2026 + schedule held for October to December, California's 2025 amounts, the IRS + 2025 sales tax tables), and text that gives the engine's projection as the + rule or the correct figure; +- text that states an engine defect as the law or as a reading of an input + (Idaho's premium subtraction, New Jersey's worker contributions, the IRA + active-participant phase-out, the heat-and-eat allowance behind 100 SNAP), + and rows whose answer is the exclusion's corrected value; +- every row of the scored 118 state credits that states New York's repealed + household-gross-income rule, and decision reasoning a re-judge made stale + (008 state credits, 056 SNAP, 067 SSI). +They leave to the caveat the rows that compare an answer with the frozen +reference in an engine-defect or unlisted-input case, including rows whose +answer takes the other reading of an unlisted input. Each amendment was +checked against the record by an independent reviewer. + +Checks that run only on this machine, because they read release 20260929's +audit (`adds0928-stage2/.../adds0928-v3/audit`) or the main clone's grounding: +`test_the_pinned_grounding_is_the_one_the_20260929_stage_used`, +`test_the_committed_adjudications_keep_the_seed_judge_verdicts`, +`test_the_committed_seed_digest_matches_the_real_seed` and the slow +`test_every_seed_prompt_rerenders_from_the_committed_snapshot`. They skip +elsewhere, so CI cannot fail them. What stands in for the seed everywhere +else is the committed digest: `test_the_committed_seed_digest_is_the_pinned_one` +and `test_every_committed_adjudication_decides_a_case_the_seed_judged` run +anywhere, and `prepare`, `bind-seed` and every later step check the seed and +the stage's binding against it. + +The exporter still needs the Fable 5 usage carry-over. Fable 5 ran through the +Anthropic batch adapter, and its committed rows carry no cost, token or latency +fields, so `export_full_run` reports `costUsd` $0 and omits the rest. The live +payload has $54.11, 3,850,174 tokens and 97.68 s. The driver copies those four +values from the base, and the drift gate then checks every field and the key +order. Ox Alpha's `costUsd` of $0.0 comes out of the exporter unchanged, so it +needs no special handling. The drift gate refuses it if it goes missing or +changes sign. + +## Audit seed and grounding + +The seed is the 20260929 stage's audit: +`/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/results/local/adds0928-v3/audit`. +It has 986 cases, 674 of them judged with prompts. It is exactly the audit +that release 20260929's export receipt binds. The stage's `release-ready.json` +(payload `a5cb9989…`) binds 2,024 audit files, and every one matches. The +stage's annotations equal the committed ones. `stage.json` does not record the +grounding. `finish_adds0928.py` takes it from `--grounding`, and the stage-2 +commands passed `$PB_AUDIT/grounding.csv`, which is the main clone's +`results/local/unified_audit/grounding.csv` (sha256 `b1e4a9bc…b55c`). + +I checked this read-only. Every grounded seed prompt carries that file's text: +184 of 184. `grounding.pre-r33b.csv` differs on `scenario_045` SNAP. +`docs/gpt61sol/seed_digest.csv` records the seed: the case id and the sha256 +of the prompt and of the verdict for each of its 674 judged cases. After the +merge of #182 (`d616e67c`), I rendered the prompts again from the committed +snapshot with that grounding. All 674 seed prompts came out byte-identical, all +986 case ids matched, and `cases.jsonl` was byte-identical. The slow test +`test_every_seed_prompt_rerenders_from_the_committed_snapshot` repeats this +check. + +#182's review excluded `scenario_023` `head_medicaid_eligible` as an audit +exclusion. It is listed in `reference_audit/2026-09-28/final_actions.json`; +its reference value did not move. The review also rewrote the case's +adjudication, row annotations and case note. None of them changes its prompt: + +- A prompt renders the reference value, the reference explanation, the + question, the grounding, and each wrong model's answer and reasoning. +- Row annotations reach only the manifest's `recorded_failure_sources`. +- Case notes and adjudications are not read at all. + +So the case's prompt is still `339d113a…`, the `prompt_sha256` that its Opus +5.5 verdict is bound to. The verdict carries over unless GPT-6.1 Sol answers +the case wrong. In that case the prompt changes and the case is re-judged like +any other case the model joins. If the re-judge changes the class, restate it +in the case's adjudication record, which keeps `excluded_from_scoring` (see +below). So `prepare` re-judges exactly the cases GPT-6.1 Sol joins or opens, +and nothing else. + +## Commands + +Run these from this checkout, one after another. The source paths are read-only. + +```bash +export OPENBLAS_NUM_THREADS=1 PYTHONPATH="$PWD" PYTHONDONTWRITEBYTECODE=1 +PB_PY=/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/.venv/bin/python +PB_RUNS=/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609 +PB_SEED=/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/results/local/adds0928-v3/audit +PB_GROUNDING=/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/grounding.csv +PB_STAGE="$PWD/results/local/gpt61sol-v1" +PB_RUN=us_full_run_20260612_policyengine_4_16_1_populace +PB_TAG=$("$PB_PY" -c 'import sys; sys.path.insert(0, "scripts"); import finish_gpt61sol as d; print(d.RELEASE_TAG)') + +# The run is $PB_RUNS/gpt61sol/run: it must show completed == total == 100, +# stopped_reason null, and a combined predictions.csv. Pin its files, and +# commit docs/gpt61sol/input_pins.json before export: +"$PB_PY" scripts/finish_gpt61sol.py --step pin-inputs --runs-root "$PB_RUNS" +"$PB_PY" scripts/finish_gpt61sol.py --step prepare \ + --runs-root "$PB_RUNS" --stage-dir "$PB_STAGE" \ + --audit-seed "$PB_SEED" --grounding "$PB_GROUNDING" + +# A stage prepared before prepare bound the seed (this one) binds it once: +"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step bind-seed \ + --audit-seed "$PB_SEED" + +# The judges bill the lane's own login only (see scripts/run_audit_claude.sh). +# A keychain-token lane points CLAUDE_CONFIG_DIR at an empty directory kept for +# the lane and passes its token; a home lane points it at its home. +export CLAUDE_CONFIG_DIR= CLAUDE_CODE_OAUTH_TOKEN= \ + AUDIT_ACCOUNT= +# Or, by Max's opt-in of 2026-09-30 only, on the desktop's own claude.ai login +# (CLAUDE_CONFIG_DIR, CLAUDE_CODE_OAUTH_TOKEN and AUDIT_ACCOUNT all unset): +# export JUDGE_ALLOW_DESKTOP_LOGIN=1 +"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step judge + +# Restate the adjudications of re-judged cases (writes only after checking). +"$PB_PY" scripts/restate_gpt61sol_adjudications.py --stage-dir "$PB_STAGE" \ + --audit-seed "$PB_SEED" + +"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step triage +``` + +If triage stops, investigate `$PB_STAGE/reference-flags.csv` and +`unresolved-rows.csv`. A new decision may only decide a case GPT-6.1 Sol +re-opened: record it in +`$PB_STAGE/publish/$PB_RUN/annotations/us_adjudications.json`, keeping the +judge's exact class, then run `triage` again. A re-judged case's record is +restated by the restate script, never by hand. Published wording a re-judge +made wrong (a case note, a row annotation or a decision's reasoning that the +record contradicts) is corrected only through `$PB_STAGE/wording-amendments.json`, +which triage applies. An exclusion that moves has no path in this release: +stop instead. + +```bash +"$PB_PY" scripts/finish_gpt61sol.py --stage-dir "$PB_STAGE" --step export +"$PB_PY" scripts/freeze_gpt61sol.py --stage-dir "$PB_STAGE" --dry-run +"$PB_PY" scripts/freeze_gpt61sol.py --stage-dir "$PB_STAGE" +"$PB_PY" scripts/sensitivity_by_variable.py +"$PB_PY" scripts/rescore_sensitivity_summaries.py --release "$PB_TAG" +# Update the note, paper prose, roster pins (46 models) and tests, then: +"$PB_PY" paper/render_paper.py +"$PB_PY" scripts/freeze_snapshot.py --rendered-only +``` + +`export` writes `data-board46.json` and a hash-bound `release-ready.json`. The +freeze is a local build and uploads nothing. It writes the pointer, the version +label, the snapshot, the annotations, the serving configuration (the new row's +evidence is `adds202609/gpt61sol`) and `$PB_STAGE/predictions.csv.gz`. The +release assets are that file and the staged payload, renamed to +`dashboard-data.json`. Before uploading, check that the tag is still free. + +Optional rehearsal while the run is in progress. It works on a scratch copy and +is marked PARTIAL; no receipt is ever written for it: + +```bash +"$PB_PY" scripts/snapshot_gpt61sol.py --runs-root "$PB_RUNS" \ + --out-dir "$PWD/results/local/gpt61sol-partial-copy" +# In the copy only, set total = completed in gpt61sol/run/run_state.json, then: +"$PB_PY" scripts/finish_gpt61sol.py --runs-root "$PWD/results/local/gpt61sol-partial-copy" \ + --stage-dir "$PWD/results/local/gpt61sol-partial-stage" --early --partial +``` diff --git a/docs/gpt61sol/input_pins.json b/docs/gpt61sol/input_pins.json new file mode 100644 index 00000000..54a360f6 --- /dev/null +++ b/docs/gpt61sol/input_pins.json @@ -0,0 +1,12 @@ +{ + "inputs": { + "gpt61sol": { + "run": "adds202609/gpt61sol/run", + "sha256": { + "predictions.csv": "e1e7662d5afa5a886d1b9b39fd0488b9d11941ebf10ddc88d821b24f3861d2aa", + "run_state.json": "403a46f419c12985fcb925843d972e2e030222fd2c8e336184d920aff5146f1a" + } + } + }, + "note": "The sha256 of GPT-6.1 Sol's supervised run files as the run finished, written by scripts/finish_gpt61sol.py --step pin-inputs from the run directory. prepare copies them into the stage's inputs//; export and the freeze refuse a staged copy that is not these bytes, and read this file as committed at HEAD." +} diff --git a/docs/gpt61sol/judge_provenance.json b/docs/gpt61sol/judge_provenance.json new file mode 100644 index 00000000..56dda377 --- /dev/null +++ b/docs/gpt61sol/judge_provenance.json @@ -0,0 +1,1753 @@ +{ + "note": "Provenance of the 134 new Opus 5.5 judge verdicts in the GPT-6.1 Sol stage (cases GPT-6.1 Sol joined). 'isolated' verdicts (60) ran through scripts/run_audit_claude.sh from an empty directory outside the repo, with no tools and a token login. 43 of them also ran with an allowlisted environment and without user settings; the first 17 (subfleet lane claude-10) ran on the first hardened runner (commit 33493f80), before it did either. Each of their transcripts has an empty session context, every turn records effort xhigh, and the only tool calls are StructuredOutput verdicts: one per transcript, except us__scenario_031__head_medicaid_eligible, where the schema refused the judge's first answer and the judge answered again. 'unhardened' verdicts (74) ran on 2026-09-30 through the earlier runner from inside the repo. Each of their transcripts shows exactly one tool call (the StructuredOutput verdict) and no file read, search or shell call, and every turn records effort xhigh. But each one's context carried the login's account e-mail, the repo's git status, a skill listing, a deferred-tools record and an ultra-effort attachment, and one also carried a credential_org record. Counts are by group. Scores do not depend on judge verdicts.", + "counts": { + "isolated: setup-token 1 (trial)": 1, + "isolated: setup-token 2": 42, + "unhardened: in-repo runner": 74, + "isolated: subfleet lane claude-10": 17 + }, + "verdicts": [ + { + "case_id": "us__scenario_000__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 1 (trial)", + "isolated": true, + "judge_account_declared": "desktop account 1d7d2285 via claude setup-token (Max 2026-09-30)", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:01:12.435387+00:00", + "verdict_sha256": "843a19da6dca70f2b9128f9a0c53bb557e281ebedb0a8ca5647c54d2f84950d2", + "prompt_sha256": "667c9c1561a5fc3f0550f4935b2dd825ef6344f7e96e3cd7aaa4a38b604bbb68" + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:18:05.608650+00:00", + "verdict_sha256": "bc4c7f9530a23599c78db9c68c70d94a38576dd4863fcbc498a1718019e05da6", + "prompt_sha256": "ccb734da3c4f8b8880a82808698e692359935925c1851d95648b6e8d8b0907e5" + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:19:16.725652+00:00", + "verdict_sha256": "84cd048f4c9bf573b2af150d703b43681a5016b5c7d8ddc157335364490bd51b", + "prompt_sha256": "7aa358111f92f9bdb00a251f74905eed494e06b4e569436f3d2d6ae34fb24895" + }, + { + "case_id": "us__scenario_005__payroll_tax", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:17:01.394718+00:00", + "verdict_sha256": "97fe9314c0547c080ea3a24bd4f6ef3c7e26dc484a7cf3be1216f512387705bd", + "prompt_sha256": "ad4ec8ed35a8691cc6df8a956ff8a92892075cad3d1ae0f9b3bde06c3942462b" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:14:28.147889+00:00", + "verdict_sha256": "428c7dcd0356d5b08b0d42c5014cf50b7e113998eb446c9249b1f9592d556cb0", + "prompt_sha256": "962c40e2d29e31772f51cb70a90db6d09a027e209f781a70d77bc84d3ede161e" + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:21:40.504964+00:00", + "verdict_sha256": "a8ee7e56b6b6ce93754d898ee40286cf513e4bd3518abb39b3c2e4545150a1cc", + "prompt_sha256": "769ce0c49e2269b1cf68e955ef21745daba7e8f9a7a2a6b87ca31d5330050b39" + }, + { + "case_id": "us__scenario_008__payroll_tax", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:20:36.470908+00:00", + "verdict_sha256": "b2c839276378cd99aa7ae9088634d410de0956cfad24d0adf679fff6fdeddc37", + "prompt_sha256": "0a4085dd94512b12b7f114551cc5979c87a8ef12a13e699a715b5c6aee11e111" + }, + { + "case_id": "us__scenario_008__state_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:20:54.635222+00:00", + "verdict_sha256": "445d95638a10c09500d4b5dadfc2d835f373b0954b5ebf8fc82ee49f4cf53d29", + "prompt_sha256": "367335bf65391125f25e867c5bfb9735aef5537673944694d3e139655338c4f0" + }, + { + "case_id": "us__scenario_012__snap", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:21:01.069333+00:00", + "verdict_sha256": "e3296ea51701bbe735d6b45cf7945e74c747917095ed3195cafdf0fdc845e28a", + "prompt_sha256": "48f650789fbd5ac179195bb1d19992ad755fe513a1f6e563503cb65ac1864117" + }, + { + "case_id": "us__scenario_013__snap", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:23:02.357769+00:00", + "verdict_sha256": "52e7febb6cc3645de417610233309a9844b501a851088dbcd10892fa827218e9", + "prompt_sha256": "877a255b81bf6ccfc13dd8fa884f8ec405acb9fdfabedca84b432ba557a12623" + }, + { + "case_id": "us__scenario_014__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:23:39.747622+00:00", + "verdict_sha256": "35893721042191dbbe02b9c066d5c0cbd8a8d8b28ffed051bd8e346b35713db8", + "prompt_sha256": "6a26c76b208e8052fa8e69c3128ae4b3550da81c346a063662cfa4dc7d733084" + }, + { + "case_id": "us__scenario_015__head_medicare_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:21:54.894007+00:00", + "verdict_sha256": "64f9bb5d5d53143e09d031c5050319e5011937c6679d0f4ea3acfcc482483c37", + "prompt_sha256": "5febd3eba3ef6502f1a5c9d44669b319d529e83e3979c3a1b6a053904ef43953" + }, + { + "case_id": "us__scenario_018__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:23:04.537826+00:00", + "verdict_sha256": "6c58cd221a4d473a5c77b366f3e6ea97df16ed45572ccc5a25b6adbe256c8008", + "prompt_sha256": "14d1178cd746f0e92aeba05ff9b50c7c5eaf4c706c8bf39f1fc7ec9963ba1022" + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:25:09.358331+00:00", + "verdict_sha256": "9c178d5126f9d8df1a74ed4ee666b51223b268dbdb4eaee6fbe918e559ed41ca", + "prompt_sha256": "485af70bc42609e385f31605477b47cd0be67368eea69d0b261652a42cb36535" + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:25:44.597220+00:00", + "verdict_sha256": "60720f64eb01fba3f686a353daa954df2f1495bd1e57364ca04288941d8613f6", + "prompt_sha256": "b480d10c33435b168bb0f830b76eb3197a3d9838a47559ca3bafbc83e7962c03" + }, + { + "case_id": "us__scenario_022__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:28:02.337910+00:00", + "verdict_sha256": "94989a808fdcd96c8048a081fdc3e97b7ba9b69dc11f92c8b20f760ccc5c70a1", + "prompt_sha256": "5dfc64291c367e7604afa9b27c5c092d15a762ae68685f8b03ec56c0d6be6ea0" + }, + { + "case_id": "us__scenario_023__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:25:17.539114+00:00", + "verdict_sha256": "29605b43f6061ec55982a4aadd99bfc2b6f1a098e2323ca2aa5def74afa4d564", + "prompt_sha256": "24fe86668754f4da309b7adc53da6c4245feff9fa021c5fd707385dfd503402f" + }, + { + "case_id": "us__scenario_023__federal_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:24:37.793808+00:00", + "verdict_sha256": "8e9052d3617bfd55244b74954371185d36d30b8be607f8ce5e7a2d442af300da", + "prompt_sha256": "fd3a3004e6fc4808b6cda239581f0631439e72164c16b8e3655de63ba34c621d" + }, + { + "case_id": "us__scenario_023__head_medicare_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:25:57.915316+00:00", + "verdict_sha256": "fea3f5ab50a0314400d016ef46321a908942e85831e8f6ac53af10427e2202a5", + "prompt_sha256": "327370c1f121b297f87d80f70a5c68b886989d329ac6e15e53089e57c29f56e6" + }, + { + "case_id": "us__scenario_023__payroll_tax", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:26:47.327098+00:00", + "verdict_sha256": "99843b1f8680e16165a913cc1e5e566cee105f883a33a7aaa0cd9b2f9660bbd9", + "prompt_sha256": "46ed4b323d96a057768cf17be38c9db67c43be779a3b6537e64e1903ec7995f4" + }, + { + "case_id": "us__scenario_023__snap", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:27:22.077929+00:00", + "verdict_sha256": "2324dd2325f39e80116ed2283974cbb7887d23de86e23933d84b4ebcc7cad5c9", + "prompt_sha256": "e39a7acb2018cda34ae17e1ba7fd1f5459cdba50daff1014a5fe3dcc640895ef" + }, + { + "case_id": "us__scenario_023__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:27:11.775018+00:00", + "verdict_sha256": "961e2741c24ff5afb00508075290d35aa94c8d292cf88c41939918fcbbbecbcf", + "prompt_sha256": "e5e6e9149de3528bc60f40ff578df43ea80ab5eae13dba2b1f4769c08d1922f5" + }, + { + "case_id": "us__scenario_023__state_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:28:35.975110+00:00", + "verdict_sha256": "ae3637ef365efd857b5aa2cca12b42b7d0e0845d91ea3dcda25971189477ff6f", + "prompt_sha256": "959f531b17d5f4440bd5effcadcb4a569373157d2000edc79e71c6fd85426a38" + }, + { + "case_id": "us__scenario_025__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:29:15.128778+00:00", + "verdict_sha256": "01bf35913d188cb3e5793cb4de5a001dcf05020ccb54b1950e63c9f69d4af10a", + "prompt_sha256": "d265cccd89eeebc5eb02cbb20588d3e04a6412c13eeec59b04ed2a492a68de65" + }, + { + "case_id": "us__scenario_026__child1_medicaid_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:28:46.104374+00:00", + "verdict_sha256": "62ffafa30630a49c589b92e569409371ae833578ad2693994d92ae1506a31a89", + "prompt_sha256": "518cffa19c82cdf94ffdf4eccd2948c70f1184e9d19ef6b7b28a4fae26bf6f01" + }, + { + "case_id": "us__scenario_026__child2_medicaid_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:28:39.936504+00:00", + "verdict_sha256": "a948872af44bc921d9705a5662a7b809d0b6c151cb81bee00d0dd7f11c9dd988", + "prompt_sha256": "453cc47e30d5b1e8d320a0836f10c5eeb966c88393020ab07dbf418dc80248f9" + }, + { + "case_id": "us__scenario_026__child3_medicaid_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:30:39.138291+00:00", + "verdict_sha256": "73b56b6bcd4afb524dd24a2086bfdb6e3df9f2e3121be63dbc2958dce1cccc24", + "prompt_sha256": "4adf06c7f1cb9a99f7ebca495c522ae38c860575ed9f5eb5d7c808ddaa18b5a4" + }, + { + "case_id": "us__scenario_026__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:30:29.220878+00:00", + "verdict_sha256": "9e6fcec5272e99544a79f998933b80bd960b6d680e6332dbf344a9151ae7d5c6", + "prompt_sha256": "d58084474a2d9e5297ff0735b0c6458b60a95e388802e162194190bb2927b532" + }, + { + "case_id": "us__scenario_027__snap", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:30:16.502358+00:00", + "verdict_sha256": "62b9e0f949906d230b582dc26a226b53bbc005e32dfe1409426b67b8a4c70a4d", + "prompt_sha256": "4b903b6b13741ee1e0177427599aa624cc20f392dea2ecd3c8406bb4de5d8e1a" + }, + { + "case_id": "us__scenario_027__spouse_medicare_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:29:38.894497+00:00", + "verdict_sha256": "3e66bee12c86c8451430c567ca95287c46426828099c2135f26abf34687ef54e", + "prompt_sha256": "c06019305cb9cd4a3d5ad92e6215a593f85be077abd8f012ce94f53859d7d98c" + }, + { + "case_id": "us__scenario_030__snap", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:32:03.440790+00:00", + "verdict_sha256": "afd68825646bb9876480bdd2a8b839170fa4ba06edfaa5767109f04fce262526", + "prompt_sha256": "0159b43cea1f9301d40d0c0954b928df8514c7146a400045b39fdaacb2584141" + }, + { + "case_id": "us__scenario_031__head_medicaid_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:32:51.732284+00:00", + "verdict_sha256": "1f6c60c1b7ccabc78e9414dfdddb64c36126994b7d6c9c11ab25af6ef62ca8aa", + "prompt_sha256": "58a22a3dbe7f4eb77a334b18303ad06f606b56d0c950c9d95012e7d6beddd4a1" + }, + { + "case_id": "us__scenario_031__snap", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:07:44.896315+00:00", + "verdict_sha256": "e0c8e68d739dc227dacc6c3051e6a167438374f2633c05d548d4d2d852c72d3c", + "prompt_sha256": "85c386606550649357eaa46008fd65efb08112f951ba98d3ad9e015b4595d103" + }, + { + "case_id": "us__scenario_032__state_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:32:19.070841+00:00", + "verdict_sha256": "8ab8ecbf5fa1b9e9bf30e014b791c91aeddf84c69f92c6b45c731afe1b850779", + "prompt_sha256": "276942ff73cbfe993c9d3075cd70d4589c993ee9cdec5a23df11bcbcefd90834" + }, + { + "case_id": "us__scenario_037__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:31:58.791554+00:00", + "verdict_sha256": "5cf06d77b2cd49155d333591deefa9cb733ea90a29e08801cc593c612573cbc5", + "prompt_sha256": "eade90c260ddbafa55d637e84401727d5f55495a5bc0d70345054a6174df9d7b" + }, + { + "case_id": "us__scenario_037__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:38:37.777673+00:00", + "verdict_sha256": "05df22552cbaec34e23c96c470f33a73ae555ecf8d8b3fe83dde7245f42fc266", + "prompt_sha256": "5e3820d87ef9d8ed2ba56a9c9cffd2fb663daaa43fffaebc549403e226bfa3a8" + }, + { + "case_id": "us__scenario_037__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:34:10.849742+00:00", + "verdict_sha256": "cc707541918bfc2e9ae206fac7f916f76766bc126bf78a5067cd734f26bc93c9", + "prompt_sha256": "9ec405c76471df6deb04a0cef2abf61cf0de6d5bda680b55eff800ec509f6060" + }, + { + "case_id": "us__scenario_039__federal_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:29:46.707226+00:00", + "verdict_sha256": "cbd779439ea98532f630cd19a27de4153c4467e8edb6e77be62424d99f89c382", + "prompt_sha256": "4297b3dd110bc534fb92caee458c592f08b1a3481ae1febbf052789184a365f2" + }, + { + "case_id": "us__scenario_039__head_medicare_eligible", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:33:25.254931+00:00", + "verdict_sha256": "8bf9565cb90291228576c507c78fa9a60a7ec0b8d502f722ad4125921cd92438", + "prompt_sha256": "8b63512df2fc4fed6256e1d19e014d32dccceca49140467f08b195bef7b0fc27" + }, + { + "case_id": "us__scenario_039__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:29:58.025623+00:00", + "verdict_sha256": "d772221a5641374ca43afc675d0c599787d9115e3af3b9ec240515db1c4fcd4b", + "prompt_sha256": "cdbcd91ba66c9a9542bd1054d6fc797d28f9e221afd5a5a1ec985d791008d5f2" + }, + { + "case_id": "us__scenario_042__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:35:00.540617+00:00", + "verdict_sha256": "5ef8c3dd18c32b2da2939c0b8adaffc7a39d1dacedfc8727afc819ac2e9f9c5f", + "prompt_sha256": "a51577e4d07b6eedb77cefb1df2ea6a4047556c8057c0e5347d3e0f557dd0d43" + }, + { + "case_id": "us__scenario_042__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:25:48.824308+00:00", + "verdict_sha256": "dd9a07f6d1e65919d351b2baa63496ea48826e3c7d2c51c7afccd26bd014f394", + "prompt_sha256": "8a2c936bdf4d7850821344d0ec918a18d25f1970528159c30bb5af5bed829550" + }, + { + "case_id": "us__scenario_043__state_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:36:09.089063+00:00", + "verdict_sha256": "9d5c27489b9e710baa64047e6e5a800b75d990dd31e3afd8371b4cd4019407f4", + "prompt_sha256": "cdb1a300dcd209c9ee88a6bef4cf3ddb37eab4be67195d473f2c647f112691fb" + }, + { + "case_id": "us__scenario_045__state_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:36:09.967882+00:00", + "verdict_sha256": "5f218bc5117869d4c83bd16047b6ad15aa9eac7e8269f3d3be28eb119f339ccd", + "prompt_sha256": "eeb1e04043a5e1daf3a146b0d6b8a3c40985c9a2e3dd1149868cd83cff56ce0d" + }, + { + "case_id": "us__scenario_046__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:37:37.876239+00:00", + "verdict_sha256": "36fafab6492613ecf0438980d595eb3a8134b963d73ea3e9e0fae758fe436dc7", + "prompt_sha256": "5377afd0814d93a12b4a85e38e9e9761f3474f7768a2630abb938e4197218035" + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:29:15.877978+00:00", + "verdict_sha256": "c8053637803c354196b59c198f9c7465ff5e2faac5c91abba0973ea0efb31d9a", + "prompt_sha256": "6cb7971270c84244e6dc7e20fb719d955d77a337e244422f4bd2c51c53599df4" + }, + { + "case_id": "us__scenario_049__payroll_tax", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:36:16.624819+00:00", + "verdict_sha256": "9c86329ca6d4b2787f19018dd1b0a2fde8e58ed8110b8af5a1785cec6de88c59", + "prompt_sha256": "91440d005dc2223bbfc3df9d0b4c7601791a83163d0503b3b1a6c57d8395fbf9" + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:38:44.231453+00:00", + "verdict_sha256": "715e6d6c6407b9c70d44623843ddad47965f0b09ac762fef4b38625dadb32aaf", + "prompt_sha256": "ab85f8c09287fcbb5c6704610371403232882bac8e183b9f42204d548816c603" + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:41:43.920908+00:00", + "verdict_sha256": "0d50886c3b0a7472bc9f8a061a3d3baa59a41e0b8db829e05f1bed30b05b7eb4", + "prompt_sha256": "439d7b875a09ca597c3d0f552918defeb0597cb94b973d0910e7e080832959d6" + }, + { + "case_id": "us__scenario_052__payroll_tax", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:38:44.992157+00:00", + "verdict_sha256": "77fac0893496daa2e734842817331f11700291b562d4ea0f565da265f6a61a74", + "prompt_sha256": "a0eecad0a09852a8686aa09dcd226cf0cf0bfdd0fee23dbcb635d1c0c2e60fd8" + }, + { + "case_id": "us__scenario_053__federal_income_tax_before_refundable_credits", + "group": "isolated: setup-token 2", + "isolated": true, + "judge_account_declared": "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30", + "judge_effort": "xhigh", + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T17:39:00.249174+00:00", + "verdict_sha256": "9db6f4b18cc6877553027a58287e7d59bf18c2831f0e1ac4517210d0f54fd2c8", + "prompt_sha256": "383a20693200c05b252edc0e75d3b07d90f7f71ffc619b9969cb97e6802a3d8a" + }, + { + "case_id": "us__scenario_053__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:50:27.584578+00:00", + "verdict_sha256": "84918d4565f6b4220ad0826a53567cc64bd17f0d5a230e0076502ddcd8763a60", + "prompt_sha256": "fc7bd38198ce59065233a244527aac93996100d5086cfabbc0fbe3da3be52a30" + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:52:54.935531+00:00", + "verdict_sha256": "0c2fb9d8cdde6e224354a0a291a8eea4d90c7d336cd588037f4a6bac5ce746af", + "prompt_sha256": "74b0aeda0cdf50542788aa3df554b9c242599c8563860a2727b5e290eec0e2a5" + }, + { + "case_id": "us__scenario_056__snap", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:28:11.286346+00:00", + "verdict_sha256": "9f2946cf1c5f742b98ce344ade9f5019b5dd1028071132f2899e43cf95ae98be", + "prompt_sha256": "336a19e45b5b0cc2a9658f56da6c5e6ebfd91c4b611e372b12b0c89d2cac338e" + }, + { + "case_id": "us__scenario_056__state_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:22:39.346110+00:00", + "verdict_sha256": "8fc8100a6f962be910fae4d4e2cb0c3427c8175c403b54179865d92e38f2665b", + "prompt_sha256": "0a94b3e11e11dba191502287da894e3fe2fab7ab908a30b10deab10f82585905" + }, + { + "case_id": "us__scenario_057__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:53:28.243459+00:00", + "verdict_sha256": "2c9c1fafca0415c6c08052268e58cef0eedf2c7cd7d846d4fc48807102231f83", + "prompt_sha256": "31640356d7793022ccf13f6639322e4ff8e96dd40caa05901f69fa24c3222eb3" + }, + { + "case_id": "us__scenario_057__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:56:31.582195+00:00", + "verdict_sha256": "3ce5834bee87f370256da271126c8a39ebcfa0ac97bb9354f483f3c25193b7e9", + "prompt_sha256": "cece0091ebe154f8d85418525dbc25f1359d9412723c15d62d0bf34e30c61fde" + }, + { + "case_id": "us__scenario_057__spouse_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:54:00.352375+00:00", + "verdict_sha256": "3f4931cb8fc81f78b74eabba5762d66fa9e2afd0eed48c20b254ae9ed8a0ebbf", + "prompt_sha256": "9594fa0ae8fef7633cd0f0a5e8d487dec07647568ee2f8fe045c42f3db365437" + }, + { + "case_id": "us__scenario_057__ssi", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:22:44.356774+00:00", + "verdict_sha256": "e2bc31014415ca8925f93cb858288424e40bfdf6847965d0f5a6c4a3794807ee", + "prompt_sha256": "c37ba2d66db96161a362d19ead4580ef801569e15c53eda6919f7028d63465fa" + }, + { + "case_id": "us__scenario_062__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:56:54.414799+00:00", + "verdict_sha256": "919ad24833f5031ac9f4638a2ce5067f2740d429c683394f833c517ca2c5c4f1", + "prompt_sha256": "91cf3ed639be434bffe3e1be25380e1881d468eec906d512f27c12fff34214ec" + }, + { + "case_id": "us__scenario_064__child1_medicaid_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:56:32.976328+00:00", + "verdict_sha256": "6ee633cf42cc5af8031cf2044791e2d91795893c5aad3536c619bcc121f5c8d9", + "prompt_sha256": "a163306ffa832c6c306c97077ab18cf3edccda61d7921ff1de274ef695f64570" + }, + { + "case_id": "us__scenario_064__dependent1_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:56:54.317680+00:00", + "verdict_sha256": "79b260b9c9748248bdfb551055ad8497b4fc7cb1d97da9bc50d31193b56e010d", + "prompt_sha256": "64c4756c41aaf49f4ca5af5aa8b8ebc05fe73a1c73ed4d1ce6094f14dcdf8259" + }, + { + "case_id": "us__scenario_064__dependent2_medicaid_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:57:36.616873+00:00", + "verdict_sha256": "e933bc244c77b2293a1138231cdfb903b45b479b17b43995681953fcd89ec44f", + "prompt_sha256": "c482b5c61b5624c64f9fc25b2049ed0b99feb1a31917e2b5ee8e4ff894e80a47" + }, + { + "case_id": "us__scenario_064__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:03:51.165688+00:00", + "verdict_sha256": "682f09574e3a06d77672db1f024ab0f850ecbd32a8f55768da84f12de1ab5a51", + "prompt_sha256": "40dce5f8b40e7c3b880192fc9da46140e40d8bbb5711c17ddd9e489d4d8795ae" + }, + { + "case_id": "us__scenario_064__ssi", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T03:58:52.259229+00:00", + "verdict_sha256": "947269714de4553457a34b370a8eefec212a129a1c6404e483d9db21e9bff78a", + "prompt_sha256": "7446e2aa6190aaa1c04a025f644d524acb3e398ea5d5068a82b5d33ed3633efd" + }, + { + "case_id": "us__scenario_064__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:08:49.804510+00:00", + "verdict_sha256": "4d23f3ff02192426d17ba1608fe5b069d17d701a3224ad345589a74d7a332f58", + "prompt_sha256": "1a0a330a5edae5c2016796cd1c39a64cc44e5111fb4f991aa720aac8b699356c" + }, + { + "case_id": "us__scenario_067__ssi", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:00:36.953387+00:00", + "verdict_sha256": "d8ead7f6d91041aca7013a1313ce3350fe4afada33ee2d5d6fcf9b08ac8e24d4", + "prompt_sha256": "0659da5aef8036e3e1b9b1ffe87b8b89e96bdab43ece0fdda2eab1e7adae123b" + }, + { + "case_id": "us__scenario_068__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:01:49.870751+00:00", + "verdict_sha256": "b98a8b36a76a11b130792409457a67122358a16810199127e35e668c56db2034", + "prompt_sha256": "31e9a96d7be18b5f6a5c5b061e25aa03f4414b5d3002c4b689b0abd72f59f2a1" + }, + { + "case_id": "us__scenario_073__head_medicaid_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:01:39.774936+00:00", + "verdict_sha256": "48965a2fe7636d7972f4aa0061cefc5d3304f2c4e667e4d1155636292ef8a70b", + "prompt_sha256": "0ad62652d3553a704ea065d65b8fc242d0a255be297af0d390c6c87beb56fd64" + }, + { + "case_id": "us__scenario_074__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:02:36.778369+00:00", + "verdict_sha256": "f68e05f6a97001dbe04f873c9686524831c455ea6783211e6ceea307db622f92", + "prompt_sha256": "c6c14f58861e7c66368bf32460134eb74908c6db6f8fbcef2753bdc64313bc84" + }, + { + "case_id": "us__scenario_075__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:02:23.644694+00:00", + "verdict_sha256": "d091d4ced680f4c21d01f489eb8fea087e509897a6da4f3018ec802c457620ec", + "prompt_sha256": "b772cbd89b76a4e8145082b542fcb3869161d4df299801bc9782283c9459fbd2" + }, + { + "case_id": "us__scenario_076__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:07:44.896307+00:00", + "verdict_sha256": "269a36cd25c325732cb3a4ca4c10cec03e3bfbfcee23fe9dd9af874c506ad7be", + "prompt_sha256": "520bd8a94466d819b93cec4a5202294d74065920a55205ba00606d8474d70c7c" + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:04:01.174842+00:00", + "verdict_sha256": "b92854c4962b5a229f835f737ea26446381a2370c7e2072971fa6ec567d1938d", + "prompt_sha256": "82b4cac7df74d71917b79419fc995566722e1367091e3d8580f3813fe6062463" + }, + { + "case_id": "us__scenario_078__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T05:07:48.069372+00:00", + "verdict_sha256": "86ec921ee90c4541a3614da6cedee28084de18b5df277653cebe96d0ae91b7ba", + "prompt_sha256": "027d17c5393e04e9b7572ec0b2a2ce6b323c0c0231183d201e7ed660b3e2803d" + }, + { + "case_id": "us__scenario_079__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:04:54.687528+00:00", + "verdict_sha256": "48dea412783c086b1c96d15783f2da65ebcbc3d3f7d70df722d829b8ab7e7201", + "prompt_sha256": "eb612da71ac1d4c6e92ad2e300b37508511f890db7fce9d273fd60bb5039694b" + }, + { + "case_id": "us__scenario_079__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:07:53.015341+00:00", + "verdict_sha256": "76215775a1915ba9bded091f443631ee8ea3157d52e3deb3c60652c025c13f94", + "prompt_sha256": "c58b8b26c6834515b9363d10bdd197b857950766624607072549a83cc6bb27ae" + }, + { + "case_id": "us__scenario_079__spouse_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:05:45.438275+00:00", + "verdict_sha256": "2e385f74d441774fba06673f0649355e114dca399983ec8c6ffb4533a438702d", + "prompt_sha256": "c57f8a3099906ba8ac283017b9a470c2c522d126a621c500daf75936be7d4c07" + }, + { + "case_id": "us__scenario_080__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:07:22.669588+00:00", + "verdict_sha256": "45d0e5474cf178353ea7eb3e651bd83a439010d74a6c697563cc95c49ceb80e1", + "prompt_sha256": "6d04d061d9494f396e801b0553e2428f9fa6307b8ea52b6cdb2b01a2a84408c3" + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:09:19.928030+00:00", + "verdict_sha256": "6b046acbaf2b36276da1ea1f7fdb5b3cabe075ae69e156df9594f7c63f44cdb7", + "prompt_sha256": "b3572286e87613afc4f07f82cfebaa8c56b0d75003beab0af1c8040e9a3e2fe0" + }, + { + "case_id": "us__scenario_081__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:09:28.320992+00:00", + "verdict_sha256": "a82fcb8c5a610207f9b8cb83f44dc710d1fbd66110680bde9376360aa129387c", + "prompt_sha256": "5c559666340a49f8fe770bf1ea6a7b0086edd6969d0e9cc5621c6ac0236d9d38" + }, + { + "case_id": "us__scenario_082__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:11:39.652784+00:00", + "verdict_sha256": "8eb2255b38c4bea8860787ac0fcc0737f0056e521b9c6ada4e3fd1679590b272", + "prompt_sha256": "4ccc8cf90b753bd19b0e943dd1a7714495e11ae9227308e67391f1582eb77e85" + }, + { + "case_id": "us__scenario_082__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:13:32.293060+00:00", + "verdict_sha256": "a7c16a8b6527e720aaa55759d09b0b8486f5dc6d06d84d438c352203b8ef11f3", + "prompt_sha256": "cfb5b6bd3dcfc5389b5bf51d61e08f8bcd65c72ff28a6d7c0f2a1043bd7a1fc0" + }, + { + "case_id": "us__scenario_082__state_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:11:28.215055+00:00", + "verdict_sha256": "55e8fc1cc9a2db2b5042ec2b0b663f30ae7a037c7438f6cb41a34f115e414fa4", + "prompt_sha256": "80ff2aa18f348cffcd657ca5760eee9de02f55440adafe8ff3d8022244de5503" + }, + { + "case_id": "us__scenario_084__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:10:37.998086+00:00", + "verdict_sha256": "1afeb86061c1a9855bdc8e6cedb8ca5e9c6d837e7aee86ae8284cdf2ee37a5e2", + "prompt_sha256": "0c7acf46270a753352b70c1dc435a62a4be5ecc6b1f0617f5adcd3a9f5129287" + }, + { + "case_id": "us__scenario_085__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:12:07.283598+00:00", + "verdict_sha256": "c3df674a0af1d45e867c59650e0d7d8182206f1b316eabff57df900cdd1655e7", + "prompt_sha256": "2e0f74a999fbf5fb8f13cf8a7690164e2a41f0be7ed3650993edd8e3d0aaff22" + }, + { + "case_id": "us__scenario_086__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:14:27.361940+00:00", + "verdict_sha256": "cec83e770def7b775ce87be715428c1cdea9b52fe41ad724f3b012d72e633035", + "prompt_sha256": "f88dd8b482f9545fbd469106db10b5a4f76a6a9dffc6183d3d9fcb85c18c58ca" + }, + { + "case_id": "us__scenario_089__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:14:32.308542+00:00", + "verdict_sha256": "4e54f6c83fd85a9bd710918616eadf12224858cf88f405e7d12784c23b5b070b", + "prompt_sha256": "3d68cae1e41bad08afc8ac7a818f8d33cfcd68f193ed67d64154b220670ffc7d" + }, + { + "case_id": "us__scenario_089__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:13:18.848674+00:00", + "verdict_sha256": "b4e53f11d2a23c11f94689168db3628f6f96706e436b5fe72c7477da6723d049", + "prompt_sha256": "96842922b67d3e023cf25d54b5dfe48e3bac8da826b839370a74a7d0f8633488" + }, + { + "case_id": "us__scenario_089__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:16:56.176504+00:00", + "verdict_sha256": "e4a22e22d0f8e7dba38df25dfdb9739520789b288ef944f3714b17d5fe93aa0b", + "prompt_sha256": "3a18ad686b3cdfac6fe768b12d6c6216451c2c8ca078cfd962e2ee7824c4d487" + }, + { + "case_id": "us__scenario_091__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:16:05.435581+00:00", + "verdict_sha256": "f49caa51aee27d372c9d7d1680b911ac2cefaad32961876d32bd8663510f0a73", + "prompt_sha256": "c8013a1179a4cd1b06c41e5e3ec8d6eff244c9f34a763501cf4d865504e040f3" + }, + { + "case_id": "us__scenario_091__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:15:21.977859+00:00", + "verdict_sha256": "14b1e5791e33c8a7a168e5ec27dd8eb1cb6c52ec2661a8170cfe5f14a65a4dd7", + "prompt_sha256": "264502af818f1fef1c23b60840d544b0d784f247b7893139e0a5db39ac7efab9" + }, + { + "case_id": "us__scenario_091__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:23:31.551552+00:00", + "verdict_sha256": "4d9fd3e14368e2a435a9e767cb358384fc714643a718194645dfa8b15762327d", + "prompt_sha256": "05c6fd8b19d81199582a58d36e922ac5471c88ab2bb9f2e69398b43d322ccb7e" + }, + { + "case_id": "us__scenario_093__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:18:21.314371+00:00", + "verdict_sha256": "792814c5eb17b3da4cec7ec4e82053c4d9ec1073fb1a814b140f16291cfd3751", + "prompt_sha256": "bc3c5d38046448b394deeb35250bd58d02e65a46ba25bb30e05af722ecda5f69" + }, + { + "case_id": "us__scenario_093__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:17:07.405499+00:00", + "verdict_sha256": "d2bd8568cbd17219a008634f12ffba7c7d79d883da16b2456143199f0968a7a3", + "prompt_sha256": "39be37b0e8114e372e82fa73047de8e9f0dcad8d78bea28d5a3d9c9653704df2" + }, + { + "case_id": "us__scenario_093__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:22:03.974843+00:00", + "verdict_sha256": "f28e688b40b6611d5f47152422c2391b77b0b12123a4a34a0ce7d51c3dfbdb17", + "prompt_sha256": "1570f713f4aa199ef0d82eb713683f6c1155b0f95d554b63166d93a2dcf19407" + }, + { + "case_id": "us__scenario_095__state_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:18:20.378945+00:00", + "verdict_sha256": "23f674e32065e89a52cb52c82dc54bf8572cb2e40beb971381867c777ab66142", + "prompt_sha256": "8558bf722f09756d755dc89f429284b0363eeb7a824e4de5a377f89a4fcef70e" + }, + { + "case_id": "us__scenario_099__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:22:59.108122+00:00", + "verdict_sha256": "1d383125725f09931153816a158373de61fc439c9020b14a896cd472a8d48633", + "prompt_sha256": "526fb90150219b1c37f07fbce5d8b88ea30d8b116d61128e10fb025cff7dbedd" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:22:31.520883+00:00", + "verdict_sha256": "92a6d2bb227353c4a55e3a7e170a3530dd337ccfc02ad5564b3e68a3830ac7fa", + "prompt_sha256": "d91ddc4f604f213ce12f792e806c1bd6be0e1c8f741598742d5d24ac120e7c3d" + }, + { + "case_id": "us__scenario_100__federal_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:23:27.013388+00:00", + "verdict_sha256": "0eb7d54533c420226f4e566a5929fe8d3343e2337b6250ef2a43f0816ef34314", + "prompt_sha256": "459976995dcb2ace4864948cce8f6977b3abbb170fca88e4fee8d8d66ea9d1a1" + }, + { + "case_id": "us__scenario_100__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:25:31.340341+00:00", + "verdict_sha256": "5e8ab5b7201c1cde5aea494bf7de631b5c3adc9f51a2f05edefa8928a8bc432a", + "prompt_sha256": "c0581bf082b97c424029f114efd9402558395b300865a62f6a3131eb522d35b4" + }, + { + "case_id": "us__scenario_100__state_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:24:35.838998+00:00", + "verdict_sha256": "f8cece27dd6e6a74968f1d88d1beb57d3fd053cdcab140888d8830c30cf17e9a", + "prompt_sha256": "767c9110f091d5588efe7af75dc865582582354d52c4e5a4e3608d9cd074cfdb" + }, + { + "case_id": "us__scenario_100__tanf", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T05:11:28.191472+00:00", + "verdict_sha256": "10f0ff48da798ac579a6bd5cd6b286c24a7f90cc37b817fd590915bf7d72a6ab", + "prompt_sha256": "7331bc795e7e0fa009ee994da8712bdc20ca699f86061326833aa6bc68366701" + }, + { + "case_id": "us__scenario_102__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:25:04.593959+00:00", + "verdict_sha256": "74362c35647c123d319e9452e0b6d24b81283a326531a0107b14f13925687c71", + "prompt_sha256": "cb7a04b3be14992e066da0f914e0947c2c77ec5297a3b459d9f512806b7c83b0" + }, + { + "case_id": "us__scenario_102__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:25:02.172067+00:00", + "verdict_sha256": "e3c7848bcb73d4b112379903f96efaeb0a86c33329271f1d002b575905cd1680", + "prompt_sha256": "e035d850b7f1bb7e1613eb438e8df66b7425a12cd160b2a8f7c8093a49417457" + }, + { + "case_id": "us__scenario_102__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:26:12.549105+00:00", + "verdict_sha256": "d3e9fa8d9dd81ca3fcfa7f20953b40f933a7f25e09c12db278bac12da40946fa", + "prompt_sha256": "f0aa4e93cce162fd3a3993f2ecbe082c0104e98030b6f271f0cb07fe5faa6eb6" + }, + { + "case_id": "us__scenario_102__spouse_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:25:57.577227+00:00", + "verdict_sha256": "56555b70da3ddbd5bf3f844126541b7c4ec63e69b3415549dc4e9ac73da019ad", + "prompt_sha256": "f46b896f82b5523fe3003eee2b68291323798151d9818d1b614c72404193c311" + }, + { + "case_id": "us__scenario_102__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:26:57.097457+00:00", + "verdict_sha256": "87c1721247774cbb9fa7ba5f08fc32e0a0d5b7edb99b371448a4e9cb311dbeeb", + "prompt_sha256": "190bffb5f3ffaa9767e94370704f5fc69540b7dac0c34e2f42b9d8272bafb2ec" + }, + { + "case_id": "us__scenario_108__state_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:28:32.012622+00:00", + "verdict_sha256": "ee330c536da72a7eccb9bb4f0982411c3aee2cb179155d7a6f9534cc4461cb2c", + "prompt_sha256": "47116e66b9b1b6aaddad882ae7465a3199afb29f76e5f8f3f000a6651df02aae" + }, + { + "case_id": "us__scenario_109__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:28:11.622059+00:00", + "verdict_sha256": "6e7f3cd32d2d91a2567ba2389b3d6dffd6fceae2d3ee379f0342906fa1bc5659", + "prompt_sha256": "3398a1c92ff3a807f55043816f7179feddf58e6b9af5ab452e5f997d8262b922" + }, + { + "case_id": "us__scenario_110__federal_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:41:05.624766+00:00", + "verdict_sha256": "caff7a4cdaa3625bd69b1412f2c8cb001b83e618ed417ee92e726ab1be56a0c0", + "prompt_sha256": "05988f00e685a69ba84798515a67edf1145b7837bf3284af99214bf886abbc35" + }, + { + "case_id": "us__scenario_110__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:28:31.285518+00:00", + "verdict_sha256": "aca7268503636d6e8e25022cac73e4003be0399ce9ff4ee5a00ee973dcdaf10a", + "prompt_sha256": "9a3fba015b3d3d4baa3e800fedab961ff73d097bb6fd79d69e270408f8d73360" + }, + { + "case_id": "us__scenario_110__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:33:49.322653+00:00", + "verdict_sha256": "f88cd19eab0cca287e3c367694b0a6b5e627eb95b3529b7b59d4ad846bf0cdab", + "prompt_sha256": "e6aa5b6f8a2c99a36fdaf6317f1f0088bf2cbbc244d88cd736f2cdfe90a417f2" + }, + { + "case_id": "us__scenario_111__spouse_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:28:57.856448+00:00", + "verdict_sha256": "b6c11aaa57d23f567b259433f8a4b138419cdf7161c6d5d879b457aeee823d04", + "prompt_sha256": "1b4acc243558ec551f071bc68f0a9e2db86c905b08c58e1405bd372e454ac526" + }, + { + "case_id": "us__scenario_112__snap", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:09:39.310213+00:00", + "verdict_sha256": "0ac51f3915858b8697cd370dbd262a4710ab260632f587af8324d0de452032a5", + "prompt_sha256": "120fbc66d889f0d7fff81e1ae7f7ae9cc067ef54ac77317d5f138d795103d4b6" + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:34:26.960453+00:00", + "verdict_sha256": "ba069da56eb141ae8fa08c0acd7b72db4018c9d59f2dc525fe3e2eae87e2f6b6", + "prompt_sha256": "7a85ca29cb8f93840e8923248d3a3952e963ed522f6e8fba1558bb4f45867780" + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:35:28.792780+00:00", + "verdict_sha256": "4f07d313983947636f0243f0d1c4dee09517e8f783112c64b898557ff55ecec4", + "prompt_sha256": "0ec84f3fb2cdfbe024118f9e95566f1db017eed02b82701334b85b0c8e29fa5c" + }, + { + "case_id": "us__scenario_115__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:33:56.989778+00:00", + "verdict_sha256": "2de376c13fa48682cfeedc791b67fe9589e72e5eaec867434196950c4d04caa9", + "prompt_sha256": "94b4a8cceaeb333aab9d4cc4b578e4f2f5a2e8a201d73159eb92adce77a7660c" + }, + { + "case_id": "us__scenario_116__spouse_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:34:11.947210+00:00", + "verdict_sha256": "5d953ccfb4800970e84f8a0c3ac761057cddb400bfe906d65761c0600b70d6c1", + "prompt_sha256": "0efd2eec8758157d1f455e443b0f698c82bc22bce2698af842aedd4040c08641" + }, + { + "case_id": "us__scenario_117__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:39:03.702464+00:00", + "verdict_sha256": "9ea96545c3e7299ad6b0cbe5e66c861129a3506dfc8b2d02b8691ca1d15c79be", + "prompt_sha256": "4a6361205046f5e398ac79536ded8426233a080b9e59d0b37df42b91014e2228" + }, + { + "case_id": "us__scenario_117__state_income_tax_before_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:23:06.664824+00:00", + "verdict_sha256": "e03da95c4a5b3c6ebdb4e925536b66e6d71825da755e938fdc2c548fd19572ac", + "prompt_sha256": "e84c1f2b0574b8a69dbf50a8e3af3c45d3505f8b0025262fba91e0cd4ee684a7" + }, + { + "case_id": "us__scenario_118__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:39:40.563882+00:00", + "verdict_sha256": "d66258f2f82c7aa795382a0309da577f83721196b65f3c894fcbdb47eea3de9a", + "prompt_sha256": "d49c1b2caaa556cdb03e5bc0c4bba73d0764b0002dc6f81c41b8b5cfc6a78c3b" + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:07:16.072829+00:00", + "verdict_sha256": "1fa0ae81eafc8fe63ee68ee99f8b98428f4805b51499f500dc38b68bcd10c231", + "prompt_sha256": "79e17fe7e5c8890456963999a6499527189a638093411719e7acd7460f20f891" + }, + { + "case_id": "us__scenario_119__federal_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:42:42.126750+00:00", + "verdict_sha256": "26d605e9ff17a841134304da4101f9fe35f1eb3e2a5592edba481181428c621e", + "prompt_sha256": "3645ecc66367d998aed76a858e68707eb8d741bf5271f9098cac9f13dc0543ac" + }, + { + "case_id": "us__scenario_119__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:40:10.171438+00:00", + "verdict_sha256": "e5294797ad9fd67690c33e56af234abe913eba73bb9abb5701f49a53467ac8b0", + "prompt_sha256": "2e3e6e070681e314886ab9258066fffb00046304cf84cf51d7c7498a5412ef6f" + }, + { + "case_id": "us__scenario_119__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:44:56.454505+00:00", + "verdict_sha256": "70f9a942a452de66b585596ad26475889eb5bddb6e18da5e030b8e5eec449777", + "prompt_sha256": "ff172164067e0a7652f30b1bf2e9a56f6a0b5670b023d574f0d4682f179eb2cd" + }, + { + "case_id": "us__scenario_119__state_refundable_credits", + "group": "isolated: subfleet lane claude-10", + "isolated": true, + "judge_account_declared": "claude: (subfleet lane claude-10)", + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T09:20:56.021558+00:00", + "verdict_sha256": "20f73fb637134300b1c7d3a74ecec6ad9a4e2544163308c856e4d6c153c20603", + "prompt_sha256": "c9054d4b0d62e0ff8c80ef82f83f04616f35a2225e69fb51135c74de939ed838" + }, + { + "case_id": "us__scenario_120__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:46:08.992744+00:00", + "verdict_sha256": "e12ff6ac781031ad2a409b596a002f99f37952dc964294083ccadb226859ffa5", + "prompt_sha256": "ea6a0aee8c1c2128c2964ba760fdc2677b5f9eb2b24e92bd02ae513f8eaaf135" + }, + { + "case_id": "us__scenario_120__payroll_tax", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:43:19.935426+00:00", + "verdict_sha256": "4fdaace099309e80618dff547161684de344bfdbf11876f3709197ee21d7f70e", + "prompt_sha256": "22f4bd0104c2e38801c02e15a04ae201774016675f385cef9d6d202fe0e532a6" + }, + { + "case_id": "us__scenario_120__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:48:09.639963+00:00", + "verdict_sha256": "ba4d1efeadeb0b01116b1baf300f52d2e954239604677f89f201e94dead6a148", + "prompt_sha256": "a042b5e45360fd78baf307cc54b94baacb2e3cb6484b6915e03e4626aef13ad0" + }, + { + "case_id": "us__scenario_121__head_medicare_eligible", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:44:21.659370+00:00", + "verdict_sha256": "8cef3435d06dc6ae0532f2b09b4cdb9a08690a1039a1bcb4495493fd262b055e", + "prompt_sha256": "30e015f3c46d6e5083eb214a17bde851abdeb42c68874f9786caef816b0bc938" + }, + { + "case_id": "us__scenario_121__snap", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:46:14.560727+00:00", + "verdict_sha256": "c27267ef789ef6c54f0a9673c990e2555ccd908b694d96d06d125f1ab6da2aed", + "prompt_sha256": "03512911eb78f3468bfaada9c89697f192ab69aad5e3e2b0dc0501a6de854b51" + }, + { + "case_id": "us__scenario_122__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:47:55.318735+00:00", + "verdict_sha256": "445f3932b860e766e8b5085e2ec2d67635384770efeeb9888768b69517e86f29", + "prompt_sha256": "a72141faa7a7768a517b740055c76e94e7c354c933985d22b451327d8efa6276" + }, + { + "case_id": "us__scenario_123__federal_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:49:19.500524+00:00", + "verdict_sha256": "dfd9984399068d514a134fdfbd1dc5a7513e5b9e3211f0178fc10e72ea178f9d", + "prompt_sha256": "e77fcb1a75fe68eb875554162b028ae98119d40495f36fe018834c489d35db55" + }, + { + "case_id": "us__scenario_123__state_income_tax_before_refundable_credits", + "group": "unhardened: in-repo runner", + "isolated": false, + "judge_account_declared": null, + "judge_effort": null, + "judge_model_reported": [ + "claude-opus-5-5" + ], + "judged_at_utc": "2026-09-30T04:47:42.669692+00:00", + "verdict_sha256": "df02301c7b23696cd23979446c4e87c8caf9d0d722a0ad491a135191b2010398", + "prompt_sha256": "8d451456e9eb8a8a108d3182cd1eae9881a9166a1172b52fb5419f3ea49ef9f3" + } + ] +} diff --git a/docs/gpt61sol/judge_verdicts_20260930.json b/docs/gpt61sol/judge_verdicts_20260930.json new file mode 100644 index 00000000..a4a9bca3 --- /dev/null +++ b/docs/gpt61sol/judge_verdicts_20260930.json @@ -0,0 +1,439 @@ +{ + "note": "The verdicts release dashboard-data-20260930's restated adjudication entries name: for each decision on a case GPT-6.1 Sol re-opened, the case's verdict in the exported stage (results/local/gpt61sol-v1), read by scripts/date_gpt61sol_judge_verdicts.py from verdict.json and its sha256-bound verdict.meta.json. The verdict each restatement replaced is release 20260929's entry (git show d616e67c33b6:annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json).", + "release": "dashboard-data-20260930", + "base_commit": "d616e67c33b6f80dabf5cb7329f069f9a1de069d", + "cases": { + "us__scenario_003__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T17:18:05.608650+00:00", + "verdict_sha256": "bc4c7f9530a23599c78db9c68c70d94a38576dd4863fcbc498a1718019e05da6" + }, + "us__scenario_005__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:19:16.725652+00:00", + "verdict_sha256": "84cd048f4c9bf573b2af150d703b43681a5016b5c7d8ddc157335364490bd51b" + }, + "us__scenario_005__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T03:14:28.147889+00:00", + "verdict_sha256": "428c7dcd0356d5b08b0d42c5014cf50b7e113998eb446c9249b1f9592d556cb0" + }, + "us__scenario_007__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:21:40.504964+00:00", + "verdict_sha256": "a8ee7e56b6b6ce93754d898ee40286cf513e4bd3518abb39b3c2e4545150a1cc" + }, + "us__scenario_008__payroll_tax": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:20:36.470908+00:00", + "verdict_sha256": "b2c839276378cd99aa7ae9088634d410de0956cfad24d0adf679fff6fdeddc37" + }, + "us__scenario_008__state_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:20:54.635222+00:00", + "verdict_sha256": "445d95638a10c09500d4b5dadfc2d835f373b0954b5ebf8fc82ee49f4cf53d29" + }, + "us__scenario_020__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:25:09.358331+00:00", + "verdict_sha256": "9c178d5126f9d8df1a74ed4ee666b51223b268dbdb4eaee6fbe918e559ed41ca" + }, + "us__scenario_022__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:28:02.337910+00:00", + "verdict_sha256": "94989a808fdcd96c8048a081fdc3e97b7ba9b69dc11f92c8b20f760ccc5c70a1" + }, + "us__scenario_023__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:27:22.077929+00:00", + "verdict_sha256": "2324dd2325f39e80116ed2283974cbb7887d23de86e23933d84b4ebcc7cad5c9" + }, + "us__scenario_030__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:32:03.440790+00:00", + "verdict_sha256": "afd68825646bb9876480bdd2a8b839170fa4ba06edfaa5767109f04fce262526" + }, + "us__scenario_039__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:29:46.707226+00:00", + "verdict_sha256": "cbd779439ea98532f630cd19a27de4153c4467e8edb6e77be62424d99f89c382" + }, + "us__scenario_039__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:29:58.025623+00:00", + "verdict_sha256": "d772221a5641374ca43afc675d0c599787d9115e3af3b9ec240515db1c4fcd4b" + }, + "us__scenario_042__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:35:00.540617+00:00", + "verdict_sha256": "5ef8c3dd18c32b2da2939c0b8adaffc7a39d1dacedfc8727afc819ac2e9f9c5f" + }, + "us__scenario_042__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:25:48.824308+00:00", + "verdict_sha256": "dd9a07f6d1e65919d351b2baa63496ea48826e3c7d2c51c7afccd26bd014f394" + }, + "us__scenario_049__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:29:15.877978+00:00", + "verdict_sha256": "c8053637803c354196b59c198f9c7465ff5e2faac5c91abba0973ea0efb31d9a" + }, + "us__scenario_051__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T17:38:44.231453+00:00", + "verdict_sha256": "715e6d6c6407b9c70d44623843ddad47965f0b09ac762fef4b38625dadb32aaf" + }, + "us__scenario_052__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T17:41:43.920908+00:00", + "verdict_sha256": "0d50886c3b0a7472bc9f8a061a3d3baa59a41e0b8db829e05f1bed30b05b7eb4" + }, + "us__scenario_053__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T03:52:54.935531+00:00", + "verdict_sha256": "0c2fb9d8cdde6e224354a0a291a8eea4d90c7d336cd588037f4a6bac5ce746af" + }, + "us__scenario_056__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:28:11.286346+00:00", + "verdict_sha256": "9f2946cf1c5f742b98ce344ade9f5019b5dd1028071132f2899e43cf95ae98be" + }, + "us__scenario_056__state_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:22:39.346110+00:00", + "verdict_sha256": "8fc8100a6f962be910fae4d4e2cb0c3427c8175c403b54179865d92e38f2665b" + }, + "us__scenario_057__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T03:56:31.582195+00:00", + "verdict_sha256": "3ce5834bee87f370256da271126c8a39ebcfa0ac97bb9354f483f3c25193b7e9" + }, + "us__scenario_057__ssi": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:22:44.356774+00:00", + "verdict_sha256": "e2bc31014415ca8925f93cb858288424e40bfdf6847965d0f5a6c4a3794807ee" + }, + "us__scenario_062__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T03:56:54.414799+00:00", + "verdict_sha256": "919ad24833f5031ac9f4638a2ce5067f2740d429c683394f833c517ca2c5c4f1" + }, + "us__scenario_064__dependent2_medicaid_eligible": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T03:57:36.616873+00:00", + "verdict_sha256": "e933bc244c77b2293a1138231cdfb903b45b479b17b43995681953fcd89ec44f" + }, + "us__scenario_064__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T04:03:51.165688+00:00", + "verdict_sha256": "682f09574e3a06d77672db1f024ab0f850ecbd32a8f55768da84f12de1ab5a51" + }, + "us__scenario_064__ssi": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T03:58:52.259229+00:00", + "verdict_sha256": "947269714de4553457a34b370a8eefec212a129a1c6404e483d9db21e9bff78a" + }, + "us__scenario_064__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:08:49.804510+00:00", + "verdict_sha256": "4d23f3ff02192426d17ba1608fe5b069d17d701a3224ad345589a74d7a332f58" + }, + "us__scenario_067__ssi": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_data_issue_fixed", + "case_failure_subtype": "age_disability", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T04:00:36.953387+00:00", + "verdict_sha256": "d8ead7f6d91041aca7013a1313ce3350fe4afada33ee2d5d6fcf9b08ac8e24d4" + }, + "us__scenario_074__head_medicare_eligible": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:02:36.778369+00:00", + "verdict_sha256": "f68e05f6a97001dbe04f873c9686524831c455ea6783211e6ceea307db622f92" + }, + "us__scenario_076__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:07:44.896307+00:00", + "verdict_sha256": "269a36cd25c325732cb3a4ca4c10cec03e3bfbfcee23fe9dd9af874c506ad7be" + }, + "us__scenario_078__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T05:07:48.069372+00:00", + "verdict_sha256": "86ec921ee90c4541a3614da6cedee28084de18b5df277653cebe96d0ae91b7ba" + }, + "us__scenario_079__head_medicare_eligible": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:04:54.687528+00:00", + "verdict_sha256": "48dea412783c086b1c96d15783f2da65ebcbc3d3f7d70df722d829b8ab7e7201" + }, + "us__scenario_080__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:07:22.669588+00:00", + "verdict_sha256": "45d0e5474cf178353ea7eb3e651bd83a439010d74a6c697563cc95c49ceb80e1" + }, + "us__scenario_081__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:09:28.320992+00:00", + "verdict_sha256": "a82fcb8c5a610207f9b8cb83f44dc710d1fbd66110680bde9376360aa129387c" + }, + "us__scenario_082__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:11:39.652784+00:00", + "verdict_sha256": "8eb2255b38c4bea8860787ac0fcc0737f0056e521b9c6ada4e3fd1679590b272" + }, + "us__scenario_084__head_medicare_eligible": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:10:37.998086+00:00", + "verdict_sha256": "1afeb86061c1a9855bdc8e6cedb8ca5e9c6d837e7aee86ae8284cdf2ee37a5e2" + }, + "us__scenario_091__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:23:31.551552+00:00", + "verdict_sha256": "4d9fd3e14368e2a435a9e767cb358384fc714643a718194645dfa8b15762327d" + }, + "us__scenario_099__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:22:59.108122+00:00", + "verdict_sha256": "1d383125725f09931153816a158373de61fc439c9020b14a896cd472a8d48633" + }, + "us__scenario_099__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:22:31.520883+00:00", + "verdict_sha256": "92a6d2bb227353c4a55e3a7e170a3530dd337ccfc02ad5564b3e68a3830ac7fa" + }, + "us__scenario_100__federal_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:23:27.013388+00:00", + "verdict_sha256": "0eb7d54533c420226f4e566a5929fe8d3343e2337b6250ef2a43f0816ef34314" + }, + "us__scenario_100__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:25:31.340341+00:00", + "verdict_sha256": "5e8ab5b7201c1cde5aea494bf7de631b5c3adc9f51a2f05edefa8928a8bc432a" + }, + "us__scenario_100__state_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:24:35.838998+00:00", + "verdict_sha256": "f8cece27dd6e6a74968f1d88d1beb57d3fd053cdcab140888d8830c30cf17e9a" + }, + "us__scenario_108__state_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:28:32.012622+00:00", + "verdict_sha256": "ee330c536da72a7eccb9bb4f0982411c3aee2cb179155d7a6f9534cc4461cb2c" + }, + "us__scenario_109__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:28:11.622059+00:00", + "verdict_sha256": "6e7f3cd32d2d91a2567ba2389b3d6dffd6fceae2d3ee379f0342906fa1bc5659" + }, + "us__scenario_110__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:41:05.624766+00:00", + "verdict_sha256": "caff7a4cdaa3625bd69b1412f2c8cb001b83e618ed417ee92e726ab1be56a0c0" + }, + "us__scenario_110__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:33:49.322653+00:00", + "verdict_sha256": "f88cd19eab0cca287e3c367694b0a6b5e627eb95b3529b7b59d4ad846bf0cdab" + }, + "us__scenario_112__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T09:09:39.310213+00:00", + "verdict_sha256": "0ac51f3915858b8697cd370dbd262a4710ab260632f587af8324d0de452032a5" + }, + "us__scenario_117__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:39:03.702464+00:00", + "verdict_sha256": "9ea96545c3e7299ad6b0cbe5e66c861129a3506dfc8b2d02b8691ca1d15c79be" + }, + "us__scenario_118__snap": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:39:40.563882+00:00", + "verdict_sha256": "d66258f2f82c7aa795382a0309da577f83721196b65f3c894fcbdb47eea3de9a" + }, + "us__scenario_118__state_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T09:07:16.072829+00:00", + "verdict_sha256": "1fa0ae81eafc8fe63ee68ee99f8b98428f4805b51499f500dc38b68bcd10c231" + }, + "us__scenario_119__federal_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:42:42.126750+00:00", + "verdict_sha256": "26d605e9ff17a841134304da4101f9fe35f1eb3e2a5592edba481181428c621e" + }, + "us__scenario_119__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-30T04:44:56.454505+00:00", + "verdict_sha256": "70f9a942a452de66b585596ad26475889eb5bddb6e18da5e030b8e5eec449777" + }, + "us__scenario_120__federal_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:46:08.992744+00:00", + "verdict_sha256": "e12ff6ac781031ad2a409b596a002f99f37952dc964294083ccadb226859ffa5" + }, + "us__scenario_120__state_income_tax_before_refundable_credits": { + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-30T04:48:09.639963+00:00", + "verdict_sha256": "ba4d1efeadeb0b01116b1baf300f52d2e954239604677f89f201e94dead6a148" + } + } +} diff --git a/docs/gpt61sol/seed_digest.csv b/docs/gpt61sol/seed_digest.csv new file mode 100644 index 00000000..e6c4ff8f --- /dev/null +++ b/docs/gpt61sol/seed_digest.csv @@ -0,0 +1,675 @@ +case_id,prompt_sha256,verdict_sha256 +us__scenario_000__federal_income_tax_before_refundable_credits,bea8f0ddc4154bae09f3f6ceb06c89821d9e511907f48cd1694a91a1fe7bdba9,929c756ea051e5dd356db3d10a9333d3d25f317fea14b643deb678eaebd5c06a +us__scenario_000__head_medicaid_eligible,8089c5b7a0466e36ef70a540a36e0f4d2ef3e03016761d5af347bdf3413c3169,23da81291d2f22a85339bb188697a3ce2973dabd3bcdcc2c9542da0e89f44968 +us__scenario_000__payroll_tax,ad3ec2acf8cf3f51cc1b1163b7235de86faf8155ae40414008d923bd19fd4f68,2976fd7ab6bd851cfb4280f8e4bf6237fc50c8fc6bc9c0bb5fa1bef7ea5081ec +us__scenario_001__federal_income_tax_before_refundable_credits,8e481d2423d8a00d78610db517ca72ebe146b6cdda3bede219de8a605f1537ad,26332b1907accd4a10afa346bde68e6f17f61627dbcb7a66de117a3f3e78fe98 +us__scenario_001__head_medicaid_eligible,5b02a3162ff896f56d86345134b7eaedf86a92df2dc4385cfd12c4a8eea72072,01d80f20bff3223c3e6e45fc067fde2a8dacd0735cae5e387bdc5cf010128119 +us__scenario_001__payroll_tax,9823daa04787348f26f579dc51f57398cfde0e8ac182b9b7b4039777536b61ea,73dcfb1b3d84dd5872bc6df2284173bd61036e6a87f68967029a4c7c8bf798bc +us__scenario_001__snap,194a3313e5d135c9a819556944a0f5e62e7bffc7237f6028662fbd2ce2cc0593,3c28eb95c4f073cec7b6b557eb113f8e38f378e353d2767e41482c606a729613 +us__scenario_001__spouse_medicaid_eligible,0e4e93d6427a4d287d6a94d667ee53a271844c4040c62a1d59280cece0a87d53,881bc970e58f5c834264dd8cd1aef813e3b88f4288ac360c4530a2a60fced54b +us__scenario_001__spouse_medicare_eligible,0656f728338046a43a0d834c0f4142b45f0c2769fbd28ebb16e748773a99e1c5,cc4bf2ef991e1d2df6eb2dc0832bea10d4e98313a634a03c95650062870c9b1c +us__scenario_001__state_income_tax_before_refundable_credits,ec40241c45e8bfa855423975eb9895d04402eef36695e482c368d92d75b075e7,76e16fef6cedb66eaddee61f05552391deed824c858df115640cb34b9fa08e91 +us__scenario_002__federal_income_tax_before_refundable_credits,3879086195ef09bc9fbbb4224f27c4b266087792861bfe60dae6d4ef829be7f9,7b4c64608ce0152998d61f834b2f56e051899f304f1e234c6bc8d67f31d3a53c +us__scenario_002__payroll_tax,8e44f4b6852ea58a9cdbd8f3c0dfde061fec4277853883bcad021cfb08f86393,6941c2013a9ff6a340bbdb01caa5d540ebbd16131ec51ff8c5c1a4191e37e078 +us__scenario_002__spouse_medicaid_eligible,f31870852cd979c20818b8eb95109f28f97bc5884f730d8bb088eb1632540437,0614261cf2a4ae6b08880ad85cba80f2c8422e1d427e790c0af98547fabb430a +us__scenario_003__federal_income_tax_before_refundable_credits,6fd04e9cc5c9f25689d7dd1788dbe8ba1c153a1090587099e28f39624869e864,b701c74f04ece7bac39677f594e480cfc1e51235d59027f442253bd6f261a9dd +us__scenario_003__head_medicare_eligible,3d383853ed8d2408b42b2d8f860e8ef13d837147fd4aeb7a79a2bd2767d65876,963de2b07086fa8200ffc7044fae0f2814b6dcf4416ece7f6251ee761b1093e0 +us__scenario_003__payroll_tax,7bbbfaf9883624c09846b24f41b05c655297f7909657a7b6338b59eff910c970,cc8373e601995d8be14e9a3bde16c2eb9782dd7b5b4be7b5b56ffd550e4c64f6 +us__scenario_003__state_income_tax_before_refundable_credits,887a8e3537b86044d115cf9871eee3fc94fe0d657c12db4fb74d54ef84754d4b,dadce1fd2cf0db15a840b85aec6bd3eae1f9ae0f90dde245e3fb8f352cbf8e6d +us__scenario_004__federal_income_tax_before_refundable_credits,9e983ac85d363eb014b702bb595f1d7c7244ea1e86e64cfb065f486ce116e9ea,594e83cb33e224a96cf4164e8d41c5683f2bc0c1c0e9f44d9e5bed2700b3c35d +us__scenario_004__head_medicaid_eligible,2675aa52eb07eafe4f531ab52c6d097e3d9bad454cd1576d5ff7299fd6efd176,a3bf397cbcc6f02c37862addb0c34de649b1b18ca8ab2c862ce302a9411ecbe9 +us__scenario_004__head_medicare_eligible,c4741b2164e29719d5b32819b5973680169dc4b65a5eec64ac5fddb8df059e27,4c97ce28067f3cef1da48d7d4b1f429c3dae32de6913fb5a27429609897aa84a +us__scenario_004__payroll_tax,6564a474d536aec4c4493b871c44efd8d7cb1bbbb8d8b40dd8e8b20e95bc28f4,3a5985011267f36bcb9d0587e7b81a3c5849723e80eba716a88038e2fe5b02f3 +us__scenario_004__spouse_medicaid_eligible,dea74d788c0be58164bc9e89db06eafc48b2ef87a41487949db5004b53aed483,f67c8c87218256a0f55d4ff186e8c9e3e00ed87c5486c0cdcc84ba779e18f4dd +us__scenario_004__state_income_tax_before_refundable_credits,79e3c8505e4bf3b830175a7a82e1f86c3a30860d1139f78cbfdd7525c16dce73,ac22060d7debccb45bf71e79e45df9e54a606222c64945eb1abfb097b3696605 +us__scenario_004__state_refundable_credits,e65faf7c6f5dc4487aa71a232f55f12658775f0b65e6046a8721abbb20228a35,b1b6c3aea2d345d1780d1fa45fe90a705323880794e00ea121d3951b013d12d7 +us__scenario_005__federal_income_tax_before_refundable_credits,75c44721b37517db2146fb9f0f110dfb2ea2d8f5ae9b9dab3594d91bb091338c,df93ae6729a4c8a7b1faa94b346776e631f437a9363eb56eb7439bb430a56443 +us__scenario_005__payroll_tax,dced8b65d729cb17937f8afcbc1b0a2ea451a99aaabcf0c167b57d5f7dd84e15,a08f99e87de20588f1b2f34d0e8a9084c54db3239292aa484d26b63204825509 +us__scenario_005__state_income_tax_before_refundable_credits,65d24ecc674928da959daae9615fe4f855aa1e23b6f67417eb96ddc7b7090a99,6a72ba07aaef729002b373234d135afd7e1fc7a592a42589e5791f0d1420b80b +us__scenario_007__federal_income_tax_before_refundable_credits,ae7d198f02cbdfbddc6122792fcc78c8780281ba2185cc43e4e5487a88af9451,ae4f45e4c06169ac8062ba546969e25a0e2e25569b56143c4811d70c95867032 +us__scenario_007__head_medicare_eligible,d6f3575f5ba18e234272f55c6dd9f63f19fb7eebf4e13c0647b368d8ca4518f4,3f881ff75efd3f35dfa847af2c94cb597a6fff67eaf70f139a3eddd9a9faaf24 +us__scenario_007__payroll_tax,582c724c86cdf328d26aeed0be2f7982255366d6f05bdc22aead1ff6f68a1565,e9746dc99fcb16bd82cfdc51b35bd027e9cca84a5e27da88fa8b68811c8e20c5 +us__scenario_007__state_income_tax_before_refundable_credits,0d0c94ac7a55cc2775f9088760f1aef686c42cb1f7050911bf530d3d0d0b105a,4c86652fbfe23e755f537782717de45e07b8617da10bb31967771456d9ca65bd +us__scenario_007__state_refundable_credits,f499e91961a2fe504defeb2aacf35971dd388808aba35c9581d52c6a54be5d3c,ef3ef8ea2b037bbf844e47e9020d32e33e9996fd247c00d7e38f2997300bdffa +us__scenario_008__child1_chip_eligible,a93978d5d293cfe92935b099a148d277e0c77be2ae086cb1d20efc27cf3ab30e,507b44d055f2623db2791e3a0b97195fa6353e019d482fbf7b26ff17a055d83e +us__scenario_008__child1_medicaid_eligible,546e03b9768c3db88d457ace694fee63a5d0e8d71e8fc46cf8871a619fa28362,063cbc6c13a76d76c39c448a742b7ba7f58a012d0fffa5a2feff6a5a8055bae7 +us__scenario_008__child1_wic_eligible,3de7603916921d46a583fb60f793984c2a22a885565ee7b9995d7a51c4710dab,5b9e51ff86cec1b37a9a4a1e1f2ca5d7a21b0ac1f4e5fd88ee6b06f789c8e4f2 +us__scenario_008__child2_chip_eligible,a32841e960466ea34fdc40a542b6aca12c991e6f47eed26748eb0953060a92fd,e36890a740d48f7e0f7ee115c17ecb461eaeec02547a792bec190b04885b00b1 +us__scenario_008__child2_medicaid_eligible,a9d876d93eb0a849ecf0c811c87af8ddbc6c1d819e1734eb538ba278d050e4d8,ed47b0ccb8d73d15468005c0e4119960a42faeaf74395c1f270b8b2f3d2b8906 +us__scenario_008__child2_wic_eligible,7a64b827ebe7754de190f98fcb49b3f5a8a86f4f658d603158d798fa9da84953,be6d060fec347cdf4302a2a366d0c89c94715a71a130da165e20b4a1dca9b73b +us__scenario_008__child3_chip_eligible,86f3d921a868bd221ad58746c9bf9f784b67aa30a3135a5a15a42964e4aff436,da787274666b5a020e7c15795d7beef35c29ea2eca43548681504610c455f36d +us__scenario_008__child3_medicaid_eligible,e4cdde65ea61f1e456b8a3b6ef4a8854d2f0b5ae3204f96fe002cca5c9ad3308,3c186a055604dda9bef0aec3a755bdd5eddb762baf246c3e5333430655580971 +us__scenario_008__child3_wic_eligible,91e0ae66cdd124b41e272ae84249cf23d4c6e226d1d7c068e598c551854c8dc0,6d77d109d3db2b657e9487065c82d1dc7b9ef1f30fa36c1ce019188a8e6ab6cd +us__scenario_008__child4_chip_eligible,f72543fe12a4a787dbd68130c3f0300a412f304360666158e5b69c3dcd3acb97,6848564d2585fef5fe7c09011161ffa2b111b7232b44ee054e6b0233157c5783 +us__scenario_008__child4_head_start_eligible,d05799d4149aee75cd136aabe88031f8729e9373db8d95128c207fb33302d947,4919b713b0fc80cb5c1f20bf7e75fcaf3116a0ebf57160ef7c5c6a892db8d5e2 +us__scenario_008__child4_medicaid_eligible,0d7638d57eef642f995ebc42128d99a3d526c2fb5b62de7e6c36d7b4a4078568,9707bca5e6b5e62d675fbbdbeaee6f545bc98d1d8a2bf5ad11d68dd7bdd10096 +us__scenario_008__child4_wic_eligible,81e93b029af16ea8b29b314c86ee7b821b49526c8258fe30a443ac42f71c49e2,326da5c1151f6af665504bb58aaeb5479ca5eeeca760d9a5af3aa713fa304726 +us__scenario_008__child5_chip_eligible,937edc18fe50c08bf1d28180cc7ec4f40e48de815ec078e311d3bcebace8d8aa,2fe026fd635afdf82c4519a1fad61ff70318938c1556806dfe9a605a37eede21 +us__scenario_008__child5_early_head_start_eligible,9f3b2d8d993601c7bbd21fbfefc51f64802eff6e2d6ff21f25d62cce81904de9,97a7d39317ff16aab8399cfe5826cf591941446ea1cf32959b18ce7b6983b23e +us__scenario_008__child5_head_start_eligible,575909b03776d949906e51d3c4f1d60f5b9c50ea316a7eda58ba6a04736a33bf,3132e5bed190bf98ed937baf394957bdea0d024e61ab73e51d222776495357d8 +us__scenario_008__child5_medicaid_eligible,540ef0b5902d898040af3718b670763aae644f2b1ba6bd11c9fe2ec0fc805f28,8868cb9513f817d2d8319f1222377466c24625cb8b7cfa8fba1a48586e437f12 +us__scenario_008__child5_wic_eligible,3793bf4f76035343b5f6455926a085331224243c83afc023002f34349b7a9c7a,cf03b90d080cf9199b8c535d03f855a3b95778a1085ab6e8a9681b00fec6684e +us__scenario_008__child6_chip_eligible,6469dd1346fe7dcd5974bdadba4a8eb9317f6b59de0380ec7a6c58a3128914bb,f4def16caae5f96f71aad03016bf6714159193c57a8d8118793c6ca471b92e61 +us__scenario_008__child6_early_head_start_eligible,b8cf0090ac647bf2b64c32d327415b9282dd118091be384cd510a4e08127af37,81d424fff9979f86a65e7899662b85a2a281ec99e30ac89a6f0a6f5f5a945e25 +us__scenario_008__child6_head_start_eligible,c58dab1204d004755cbd5409a689fdeb19f54eace9683eeebfb351393d0e4faa,e10cdc003427fc03bc1710101cb9cb9f49ee34573816e2e308a8e3ffc45560f4 +us__scenario_008__child6_medicaid_eligible,dc0b70b2df531cb004115d50f027a45358df298499f674d74e02c5b93d70a189,214ab2db89a15a0b69f5ae2f2cf058eca9352833485400cd712cc8c02b7c04b8 +us__scenario_008__federal_refundable_credits,4c6e54d2aa385237512ffaef1614191922632db480db1a5c548dbc99eac05a24,c3aa5b8ed5396186aa25ba8c6acde571d684f35a128375bea0a6a17bf0e91020 +us__scenario_008__free_school_meals_eligible,62f64561aadb71fd2cfda7695d216c0625e7ab435dbdff4afd0f12d0c353c18a,92a815206251bb5249e49a462eaf21ff1e7fb3e1fbaae01a2b0841def2be9ce5 +us__scenario_008__head_medicaid_eligible,e7a663ed314f9e402e824eee6ebc20ae9d3f102bab57545d2c4568c03b65fd26,b8643c04ae58048b8d04654ac041fec217fa3e2283947e100481345d640169e5 +us__scenario_008__head_wic_eligible,92a59dacf6ec33fb005b24c9ab503994ce3de96e9a18ee76da0fcbddd80dd2cb,abbf4b1492cca1ee52c3966140730e888cd04342abbf4a450b4eb922e0487ee8 +us__scenario_008__payroll_tax,2ef60c6d5c7ac4688f70ad8493687cca575fb44681c4d95d83305bd8fb3a3efc,439ddfb2413ecd119f97fb1da66faa6a0fe2f432238f1d428d870e68f4b783a3 +us__scenario_008__reduced_price_school_meals_eligible,52c1c6698450f26671ddf91fa3d5e857c24126574d979402db2f7e2ad1003d14,5f7a30df6cf4fd64e856bcb5599c16260ccdc2938b2eb70aea40e24ecf26cdfd +us__scenario_008__self_employment_tax,9f4e19d631dde986e607c4d58bc589636aea04f388687cfd5cd00989ddc7a5a7,5b669021e8ce7cf60f9bc250336a693d97335cb229db0940c3b6746c2c6f1317 +us__scenario_008__snap,25bc5d2714e1ee2df11ffeaf2a70499955a4dd80f029b920f1ef6c7fd9840758,940009b70fb3756de36e8235e75247b79b3bba7eb235ec55a70ff00929615e5c +us__scenario_008__spouse_medicaid_eligible,cbd76a85e0b395b002e0b988ecd19d3ad38d77b61f5c0df5bd30395b67450e2b,19d57f0fee5b98fbd5dc3642ca577b3459247c8b6e59164b8f05edbbb86b1c96 +us__scenario_008__spouse_medicare_eligible,632f38d7eefb215099c3582020a75ee22ce5df9fa329beca5da6d25e466ebf9a,56784f5ffca023f279575926bf1fe365c6d6544dd1a60487622e021954c5ae58 +us__scenario_008__spouse_wic_eligible,bbf88b0edad4103bcda7ffbdce9c4859b9ba0aa763555a5567232c68ef0a8d1e,0b5455a24e0ada992eebb21a33887b7a38f5d0001c7a424fdb7a54ca9a0b70a2 +us__scenario_008__state_income_tax_before_refundable_credits,bd12f82c481544cd55366a10b31020434dba112c66036aa48dfcdd8acf9c0848,a8019a1a48879df8d9dc9e7b45f685bd38e40aba112a9b250e9980706eee7038 +us__scenario_008__state_refundable_credits,aeb6ac3cebf17ce972691e18d264e9657130f7b2dd53947f3054bf38a55b0981,dbeadb80ba65622d9bb1ed2ff69529705926124f35e893d4223b9b35ecf2c72f +us__scenario_008__tanf,64cd17572bacb38973967f48da745366486b8d8f5bae613b2c8a9650e3ef1b72,c91c68be0bc16580d986730f2486ead82472c8e4e98ea1c38630818502f9e26b +us__scenario_009__federal_income_tax_before_refundable_credits,c5937b4eff7a44ec14588200548abd350f892db43f81b53405c037db1c340a78,2a0b953a00569dc225a1d912e1403bbf0878fb0a3754ee9bfd6fcea06b1e4fb7 +us__scenario_009__head_medicare_eligible,405b568c55ff1c1e31824d4b948ff80fb00ab44d4d02bfa7bc9285abe703f8e9,58a46ae33fd1e902d2c8ce0f93f92f0cba6bb5a0b6061f650cd369afbe2b52c0 +us__scenario_009__payroll_tax,1f3919bdf51bef4cb853f4dce02d98f9b0c4352c0f4f6ee76dd734874e904c2d,93d539ce4cd8ac53c946906c761024e33ad5a3efefc92e0c733f6ab46fde2a99 +us__scenario_009__spouse_medicare_eligible,25c502992c8932a7eb2112bd9d8caa000a82743a864544c26f036ff7e39dac40,76a2cffee43533b871baa2bb6353e2322c5e64a16f255e21bb79c2f226faa4d0 +us__scenario_009__state_income_tax_before_refundable_credits,a5d03d9720496496b60ae0fa7902a5ad97c1e8d7b429cfc2dba1140345b5609d,5aca2be5756bda2475f6dd57a58925122f424596024dd6fe2047a254cf0c74fe +us__scenario_012__child1_chip_eligible,084057af958a4122860d69dd7a951bb0fa21ef72cb2eac1334af0436eee0445f,6559dfd679755953b1f54d17d541830275545647f4a3dfe62650d1f7ea9fa637 +us__scenario_012__child1_medicaid_eligible,ba610b0ddf90239d0506340cfb77196e3a36e761ee9e58727543ccfa3131fca0,61ae0f73c61c3aaa837136b616be17427bf2aed3a29231f1f814771f5033f2d0 +us__scenario_012__child1_wic_eligible,c92b228c812ff8a80bbcaa7e1aeeb00e01f7e3d9cfc0f9fd704bf0bc0ce30151,30db9b9fc4aab1a5b44ca7204368ea86ffe019ecd68123b1e6403d0277093b57 +us__scenario_012__federal_refundable_credits,5e71531c0c942e6870f1152fd37656bdbbdec3d94eb993da7c8276d0a4bf18d9,3b3ef440303b4536320b363f37b3c4447dbb39cd9709f4de9d46b6801eb387fc +us__scenario_012__free_school_meals_eligible,cbbc6345ac9b2b342390f2b4ad67f5f05b6d44521cccf3fa0eb236cbdf72a995,15519c0204fc12014361755deca6b28cdb67e7877080f30fff1667df0ffbcbb6 +us__scenario_012__head_medicaid_eligible,977259a2fc27fa7804e31e8cd2a0d1f7fc5f8bdf9db3b81e92e5b80bdc6839a4,246f3b374a462a06c56a51215437e56d2488d088f1c94388960eab2c4bb934df +us__scenario_012__head_wic_eligible,bcf0075d7a75af15a600f90b50001c63f2d3b1b9998e4c7cd199b837c9051fdd,a67b5f97188d9462b41ed691b75bc934ad1ae9c9aa2d1e05895e9c8cdd8374ed +us__scenario_012__reduced_price_school_meals_eligible,57e51af61f9eedf0319eb1c6e6e5765ed9eb8c48d9e295b64e4ed1fd6f1c47be,47d52e1d7e6d98130880554c72148d759b8c81e87cc4222fab9a06e9c8e65532 +us__scenario_012__snap,bb8f80a284d7278aa336c0ddeef3c5dfb42f138122a8bba2248a190a48808663,c1a8d628efc81acdbdcd0672ca85218fbfb54e6c3193d13b5de202a4d937d56c +us__scenario_012__spouse_chip_eligible,ca2e2b03a2a77572e92a11562373838658ce4c42745d2f6b0574f6854c4195f8,cc3202be5f93f24570bab75f89f47c84f238bad90eb11ab86edfdfc40e1550d9 +us__scenario_012__spouse_medicaid_eligible,381bc83ff39fe00845ef98bbc58821e897b2916469e4f310dd86e093d6d93f69,766adafd08df7490d7469ea1ca4e8bdd712f707a5982508777c47c7a29a0e02c +us__scenario_012__spouse_wic_eligible,26c85250889e2c33332d350fb331cac26d5213d63ea806e88dbf0d4b5e30ce13,90ae25f3122f8f48fcace59e58f4d08acf154763165613ced5520f1725cb6039 +us__scenario_012__state_income_tax_before_refundable_credits,2bb19b10949c5f34f308c3296d315be57bda75ae184c5a56bc9efdcb5758327a,f946db2691209ec8a6ecfb5e5c92f763adfd9cc1b1a04651036bb7ec20eb77e0 +us__scenario_013__federal_income_tax_before_refundable_credits,49b5758bc0ba32cf35934c72284f487c90039703c9fb9c78c65e6ff6c0b2e59f,9f8c1f874fb0670067b450356b5b6040463d98ee3ccb9c6c4cb1465fb71c3bce +us__scenario_013__head_medicaid_eligible,46f55f7fd62870bc3503f9b6fe081e3be1ca4c1cf9a3d60d1d2e8c678aae8aa5,5a252dfc9466fd48248f6e8fee88e41918945798efcd6f781d7d9473e5f529ef +us__scenario_013__snap,1ec3ace92c2c8c84e500dcb1d7817f7cb5a32d03da7f12793aa2fa1b05b6560d,c8ca780ca57dbd632a31792ebed6e1df8d2c0aafbf30eef5844e09a2ac7252b9 +us__scenario_013__state_income_tax_before_refundable_credits,2b20bd6903659831c0640288c3a9df49286aebfa24220f5d3df6e5b86a73b539,df10fe12419c9927b1d23757b7fda20bb6debd2ef892504ea1aa3ffd08f0652c +us__scenario_013__state_refundable_credits,255ca99d6e7f1d7650bb9283eac1ff87b231dbb3b28530004f6c94c45b7ecc38,395cc101c368b10226f9c650a938608c6cf12d7c9b0f4a8ba14cf2a74bf2a652 +us__scenario_014__federal_income_tax_before_refundable_credits,f5d13275e21b97e6556e558666c6eff4b14b45143321da369c355475a37092aa,f4781501411253ad902d58fc6a22a8680defb8d94a83f909fde9d481e81ae297 +us__scenario_014__federal_refundable_credits,77d73d79de6d24641da717e264acaa4bdefe11ad22f59daf6d9b1ac6015162f3,ad32821e5adc7fcf36c1d1587bf10514d6dbd6195cf48a5945605a22db528f64 +us__scenario_014__head_medicaid_eligible,313027647d0287cb666cd7e527d232801b09a99e676b451760ae3008bb6c28a4,5ea332fe1cf93c010ce785127c00d0eccd326b79d0e15b1d7bed20f7905e9432 +us__scenario_014__payroll_tax,69a26f3b5f4c84b500f982be577d094ebeb46eff9c18e7d54d7be54790e29f5b,d659eefae4b3e438c7134dc28d00ca2e5148e39b25a59af7dc25189fd9ca785a +us__scenario_014__state_income_tax_before_refundable_credits,e20dff973b01b047e9e18ad5795c09050045777f00c4bfc4ef8a00d6de79b400,8868e3926c04e86aff3f296af24a33d8b7f7853c5ddeb31c41a49bb20bcd31ab +us__scenario_015__federal_income_tax_before_refundable_credits,4e4fdb40b8a3155b266f63ed0d99b7c51a04150c028fd2cef2ba9f0d6a6bdd06,79e42186c82e3feb6befd89ec090d3bb29ec13a79af4e44a9bd504b5db1c4983 +us__scenario_015__federal_refundable_credits,9762bfd026cc4e732c792f3b5f0715c2b19cc73284c034b388d311fc3d2b24cd,20abcd0302b072f8fb64b41f73a46de45b9ae1c0e56f131e142f11a7a78037b2 +us__scenario_015__head_chip_eligible,d7ceb52a5d6378b9178c11b36e22f119ed4622ee673b23c0e55ec64114fdf9b3,b2bfa22e6876c72ccaebc38f904d41cbee831b190c61438ab9a8fa8f2803ba43 +us__scenario_015__head_medicaid_eligible,16802b03cfffc8cd49b6414abad9bb1000f2797a76ffe6e6bb67b8c0473cd134,5a389e3f4e8f93e1caabc888cded222f52155a41df43eebdcf41f7b087a04d34 +us__scenario_015__head_medicare_eligible,c7d078e705060fc9a8794d9d7ef89e5d928dce79b7ac500587e935cab88577d1,fb5fc80bc1fbd5c22e28a390ed2f619c91ff85c1e42c2c785e3385031d21fa19 +us__scenario_015__payroll_tax,6a77bb32e0efe4f1fe6c73ab04dcc5a94066b37b2a5453a0fb604b8eb260ee26,ae846d93a2d37143ebb4bdd6ada238f55427876a921e626fccafcdb72a37cdf6 +us__scenario_015__state_income_tax_before_refundable_credits,37c1f9b5c7ee8f9dc8250bedd8296a1d8d76e299235fb6eb4bdfa86c4e17aec1,1b92d5310bbe3563809c2c0fa680ef7706e8a927bec11ada1562aa8e5200be4d +us__scenario_016__federal_income_tax_before_refundable_credits,0fc24268d6629a55279229096707458ae759f7da54d6ac37dbc00e4df915bbe2,d12b9af1b9423654b1eb2f16e1378a37ca6a937e8d13c2fc970bdfa0560e6df8 +us__scenario_016__federal_refundable_credits,74c125d608580696a0127c7ed959c60eeb94c73e5124cd773124b191b308745f,753066bf7ef3c17ab8edb6d9371ac9031683926746432636369e9874feb389ee +us__scenario_016__head_medicaid_eligible,34f8bfcf0c92652d3d292df745e1155d766e01eb782a35ed6599aa7362aad3a2,b87a14633b6c862b2fb1ec6a7294353d6528aff896cd47c55746bf27d893459d +us__scenario_016__payroll_tax,a1a09a78f901c8a7ddd61c5e85d56187e7b9ef17e1ea31d093a8d7452d903c31,6de254b0730aaba0aeca8fe7055a17dd0d255c0897a658d03251fc983053d506 +us__scenario_018__federal_income_tax_before_refundable_credits,8195d0c0ada006573bd5fbe15fdb1219bae3a6881340699d9ea586ef0701ecef,725680d89160d57b412219dbc5da326d8bfe33a044c30f818d9472618c353433 +us__scenario_018__payroll_tax,6a3ea93b3489138dd01b6be096119b9a3f0602c388f5b3d927ca33a356e96de5,dd07f311882f306f06fcec6cd99e9358edce470f5b1f41e470f9c071b53a2047 +us__scenario_018__state_income_tax_before_refundable_credits,ee474aa17cf2b72bf5df82baaf7c4a338999411210addb760969be41b91b2ac9,44b1c4119353df4bfeaccb8b066e23d9f16904f15a9fbd52b373d63b1b36196c +us__scenario_020__federal_income_tax_before_refundable_credits,3563344877478032aa12dbb39ad396b65fa5b245e7e5445a80216a9d6be2e2c3,3083467d9c44e31ba649d3b96b2b10d90fe8acc9dc0ed8beade020a516b077fd +us__scenario_020__payroll_tax,f3366d48e05a89d198c6a931766af662a36245c1c688a83ab26019af95a19371,95295e8e51c7bcab0f7f19c37eb9fdbe504d6dc13e1b2646e17b7683eb7890f5 +us__scenario_020__state_income_tax_before_refundable_credits,6428e2ca34286ce588df84f3c842d5e27e9d214be9ecf690ddfeec7eaf97344f,dd784726d5743c8a7967fb027d6088c061f2389ce762e0ff493c967ee3aaf445 +us__scenario_021__head_medicaid_eligible,5cd32520de73b23fe84a1ba52d95619d48577448e2dc2a52f8fddf992e02bcd8,a7579832fd243cc634b16175ceee8c925c39a09867d89457ec3d00ccf93d57b8 +us__scenario_021__payroll_tax,e3ae3bede8198ce8a7d1360392a0835e22bf9283e8ef0a31e6c3ce7395c868bb,7ae9f9a54cea4a2f6d92063344bd6132c8d7feae82cedd64a14875b2ca0b1518 +us__scenario_021__snap,835e04d088d82c85644fdaf4b0324ccda77fd258e69759f3ca916570d6feefd6,981799807d4c70f1c827aa4f94292b61a3cf6f1d7a4142ea19557d9557aee098 +us__scenario_021__state_income_tax_before_refundable_credits,5916cd9b6ae77aa81185ad1717dd28bb20835e4c216482dc5a05a661581e9c0d,d117348f288ef0fe5daf6fdf3e0eb4f2b620a83dff20859874171a8c3c4f4674 +us__scenario_022__federal_income_tax_before_refundable_credits,843d0ac13ba6f62896143f5a8b7e223b6b49e1ac97f295a9bd1a7628c6e9dcf8,4c35f8615120225d10679203e46a920da7921ec413092d31d14e8f9eea259514 +us__scenario_022__head_medicaid_eligible,35a31d563430c21f8372ffa8490df7cf4d85bc4cbbad8175355aeefc62960bc4,c5beefb6218d89d0f2221698b4cdf2eb07f7237aa5204620eff8e71a43cd9e9b +us__scenario_022__payroll_tax,8e74f195331a30b9b84330efae8482a2bcc6ab9834de380390c485f8bd0a84da,2139ffdb06d808e00873f5df1863543df45bed7e50287a42ca26c7aabd28b12f +us__scenario_022__state_income_tax_before_refundable_credits,17f01c83baa690fe2532c9e60a1b969da3ac90c7c757bd8532f2d25c96e6c25e,feca3f8988dea0e271e35d3d6bc440557bc26e349992c189c7cc8f8aa249661e +us__scenario_023__federal_income_tax_before_refundable_credits,35f9760fb75ea4df3fd7e9a9712a7f44a73a241543a8265ca2bbbc437cda9dbf,da98a95a88850cb542a6812e43c6e8f862839ce8bf722c6565e834479a66e548 +us__scenario_023__federal_refundable_credits,93dec804b2346ed07305df6e2f81abe8f8501a97ed4afbe7248450960b5bf290,46caf9ba018c50b7cfdc47c1e2e9b9e4642d80b63e9947a403965c2f2f0c2fa2 +us__scenario_023__free_school_meals_eligible,28e17e05d39de7b8957cd7237eadab9663714555febee49423c59fc3acd48a48,ed682d23132b5d3cab55b174133edd278e9e69116ded256002ff75b0fe508bbf +us__scenario_023__head_chip_eligible,7e5a7ae0577899811db282d1194d037bdcc6d1f3bfbc787d590df0977de38bbc,7e2ead8b1a61e568e5df9a5520e0cbfb554fe62d32db0e0548f5ebc4e0095edc +us__scenario_023__head_medicaid_eligible,339d113a4162a995e6e9f271ec2c6656ec2dca8c6d33f18225bc9f8b6dbc81b9,5094b4eda857c36ce42cf847b76eee7ea7f082c61eb623fabbd7c2e324b28fce +us__scenario_023__head_medicare_eligible,16ad669cafe48bc9f23d906fd06972cfb67c553eeb94552a2f425589472617aa,fc769ca316484bf1f7696838c982df29fb622509bd0490018618d6f7442d9763 +us__scenario_023__payroll_tax,b69e620e8b377d7f107e1516d77b9832b47bac52b019a127d4523c7955cc555d,398bac36887a84864d2d9ad6f0f32eb2a08276cdc5177f9b5d00650d515c7fee +us__scenario_023__snap,19f19942036a8964dc148500f746dea77ecd4c2f7bf0c4338384e2ebf4f2e687,41a054abc7a10195a12df267168716cc1245878221aa7f702c779f0b0e343317 +us__scenario_023__ssi,18155fe58781b257180794ae8d40ea3f566eda49c11ef9a9139128b21b6b0e61,c238e5f6f464eb241c29970e3ed89d9b5f0364be2889675d2365cd62e51d3c21 +us__scenario_023__state_income_tax_before_refundable_credits,2fc67c0c78a5c66279d62fb2898301184404a180f2a31233218249abc366566d,075ff4ae79b1474e6a581c2a6a56a4590e2c1c74ab9fbe03ad8a38836983c8af +us__scenario_023__state_refundable_credits,b07d78041c066bf18ac0dcdd5198697913e1d85a0ed3ecdf076a14520237b731,2e40bc3d3bd9f73d854041be67b8592bd52fc252a3f95be0425c77b67d1f4839 +us__scenario_025__federal_income_tax_before_refundable_credits,c8b40222f42254d8625417cb28057568ceabf08ff18c8faff2be8b8c2bb5bb2c,d03fbceaa10b0e45b6fc667634cdf57d7b4224b848dd873a4e399dd19a6b13d6 +us__scenario_025__head_medicare_eligible,0cf47f2696153fcb224785652ba40de7ae34723671fe6f6e45582e5b80cf3aa7,9614ddd0f70cd0a6f0a6a38561f78ad12d4e07e9f99e0b8039ba2a12dd1440d9 +us__scenario_025__payroll_tax,bb0ca83f76a0e03a92c58a47c4c598ccfbeedc86ae889287ece6123466943ad4,b5ad297fb2ab2419cb15bab678129e12fd348619bf88fdff45946db79547014e +us__scenario_025__state_income_tax_before_refundable_credits,e245d0f6e7963c09a9a45cde128d9f200708583ffa559c76ba97e54567c56142,2a8cf5587ce64cf712f29278922b2f944106176dc14c21e6362cff555ff465bb +us__scenario_026__child1_chip_eligible,61624d00a6f374c5e4ab4411533191b0be374729ccc3588f011f733ac172c20e,07c907416362ff6b3844b9f1a27e7d24e7b3234982e0d1014d4d68dfca39c406 +us__scenario_026__child1_medicaid_eligible,9da4658cab22f95dd4dbf82e718d85eaf983cca079cd8c1ffa0dccafad674eff,0b12aa8dbeed69fe7c73e9b2337c031a2d58aa75756449476e9ac2ec5f485333 +us__scenario_026__child2_chip_eligible,16a8827d04f3b31e2402671cc1f1d083168910f998c11b3372186a663f999d19,a7678031d55bb7f2bf139402c1151f1afb4adce191b7191de1a4df7ba7985c9e +us__scenario_026__child2_medicaid_eligible,ec5964541b4826feb8f88a304c4a6f75cbdd9c0e3bdd9ad62b3fe6bf0a306721,82e7300deb46df31b3235f3cad190ed83869fefbdbef1f8d5dc32bccf661d8c2 +us__scenario_026__child3_chip_eligible,4f5a3b1236a67a6675c7ab52fe4b27a847a94eb3fdaf03bd33ffad15e93420a3,145e9ab5c5be0e265479b0ccf2990184ed0731ffa4f290aa112b2b2322818896 +us__scenario_026__child3_medicaid_eligible,db1e037039332b60992bfffec1a684b46da69fef6f9c36bce5e8307e031f7c74,b1308852afe2bf3884775371e8f3a8ea02445721eec8a68cf5536253e51eeb9c +us__scenario_026__federal_income_tax_before_refundable_credits,2b36eae70a9ab4da2716dcdc3a22bb24a343bae2575978ebd672a95b695d0112,3081c5d9442dee3e8b955def7774ad41eb8ad6eca89debe383b2aed5d84d77fe +us__scenario_026__federal_refundable_credits,0af3536007a3e09e93eff958912cfb987c16b55177f2b49abd6c29d3202c4c41,cc9ff8a3cff796449219c37ad5872cea0df510b378e2b9101b01f7a4d67a868a +us__scenario_026__free_school_meals_eligible,7c755197149b28e687495c92cef81b8f223082d332c904c632de508f68d22e71,8cde748d4d95ccffd6c193f669506e64d922dd48f907f07629001f555505f678 +us__scenario_026__payroll_tax,f2a9d808a10ff56058a67a72fc2ed217be4273a3b6149a74ec1ca25c447206f4,6cde021a41902a75f515360010f1d740a5ff3b9ff79823423fcc765593eed952 +us__scenario_026__reduced_price_school_meals_eligible,c56c708f436584e21e84cdb3b1043ff74a4184f34c68b9ca0024e6bd52d30a1a,e0a3d4d035ec7a4b5126ac804367ea8ebb66ab08381156cfaf2a9487ea0f593f +us__scenario_026__state_income_tax_before_refundable_credits,91cbc17ad7eacd0a7e97d7ac32e9b4186a2bd06c93035f635e53dbf7e8eeaa4d,8a83fd6fad10fc368a6f2636bb7b4c1aaa17c28a21960d899962d88e6c395ccf +us__scenario_027__federal_income_tax_before_refundable_credits,9817e0642225ae620461b92156c7076ef9e34471e6cdbc206ea52009af0a8435,db58c0cbad389a3c99a75ddb7c79ef84d846a0b8fe204c91255b722cebebc176 +us__scenario_027__head_medicaid_eligible,2e7c67fa66042ecb5b36c3232312a1c290ede91dfb39945e73b12b56dcee1e5d,e7c94316f511169fdaf10cabc0944af4ee5f5f7458eeff17a6fbf3bd26850065 +us__scenario_027__snap,50587b8b024179bd68ef7a5cb76b3df478c56ec17a725274847104521faec464,5db7ae5f8224342cf52532dce25cbd2c15fc1f013f5386735d12d9b2c9f0b39c +us__scenario_027__spouse_medicaid_eligible,808a92359b1dfd1875c2e1810c3cddf0b6b1e45bba51c8aaddb2eface709ae5c,4d2e892083fda18b2988dadc997ac40ac7f86f69a3a02a34e55e19da49ec1aea +us__scenario_027__spouse_medicare_eligible,0e29b1a4e1b22f46df2e52e66ce67d491126987fdee925faa19a1d9555a2a892,7fc2f7c054b0ca145a17746c643d6e24c34735cc902aea61c3af8821896dac9b +us__scenario_028__child1_chip_eligible,50f8a30784d924b330782356ecf6a545918adfebe7c26368fcfb69a966b2ac24,76fb1d2fbf42022473baa74923030bbed8e998ac468477fbaf502d9c797cdd32 +us__scenario_028__child1_medicaid_eligible,6d46f58fb551089e1e69eadb0554bb373a26cfb45a9d162b313d0ef9d698cf53,46f8fb273c2e5a7121faa7cc6c7400e7bde1242ec6eb4d9cbb9da3589b7d2041 +us__scenario_028__child1_wic_eligible,e2b0b738fdd154b649d1b5980dc348ff4eabfc0d5b109e9daf9d684bf87b5215,329224ec25191668408cfd9575664dea9f80ab4af46b27e880d39ef7cda515f3 +us__scenario_028__child2_chip_eligible,f7f16fe0d177efabc4a6f71014e9eb5215521d974a37ec84c3b3eb0e94fa4931,93a750461570fe30dfe86e834cd07269e6531714fcf57580ed62eec8bd74e70c +us__scenario_028__child2_medicaid_eligible,fa7bb7fd106751f713da0e507623ea8ee79a20709a013031a2f91ba1cd9015f7,99bae0ce53f2a4fb40095b51ee496bbe2d80c703eb2bdf57ce056c74f497043f +us__scenario_028__child2_wic_eligible,09a3a7f761bed7a37c567b54ee4ceb4f00feb5bcebf268cae5f4a60d076c0c5a,a9f4f2e7d7bd97e90e850739c911d83596c705a2a0ff275946980389a7d071eb +us__scenario_028__child3_chip_eligible,9680c066b3949302154b09c36b72eaa76d09cda283b79d6e4c6c48062098edb0,573d972c8d5cc1bc9e181e8a4f8456a3aa54f3e73f9e2bfc5a4afd7c25721c07 +us__scenario_028__child3_medicaid_eligible,9fa59b53f5be06fe828c3d8740721e7b0b4a6a5e85be5e420e12e2ca4d05b3f5,da114a927b181d2d5d2917db3d1c2bdfa3e34e8a97a4515ecf255a9c08a144d2 +us__scenario_028__child3_wic_eligible,420fc84607dad573fd38591ab7217a520c7081b6dcf5ac8055e921a3e464a9c3,1da539fbf580be5f1c443fcf9ec8699c3d7b707251b9632d398f4c9898ee2ae0 +us__scenario_028__federal_income_tax_before_refundable_credits,0e001e99a2ff674540414e5daee51af4af754c9855d8363f658860583d163543,ad8eb09ceee92e824102bd17a00de1d58015c8a045afee38d1cffa9c9709c414 +us__scenario_028__federal_refundable_credits,0faa96272c758e86eef2bfdca37ed126e38463a0a8777c9167f5c13a5b8912b5,5b61c0b57128832d8fcf50309f51cfdfa2e1e87ec7ff1aa36ca4442a6fca6437 +us__scenario_028__payroll_tax,ceadcd70bcef0a9b757bdc0d34b05eb3beb769a35a4a77890e7cf7da14e40bba,60237e8a33a09712b77bd9c9589123064463af32470d3bb59d337c004da13a1b +us__scenario_028__reduced_price_school_meals_eligible,4c69cabf3d14a66d5904a0bd3ec08f4a4357569559d13943d7be050718cd2a89,f20465304f63c88ce64d3f446732b1eb1007efa991fa3333086e13a42e09038d +us__scenario_028__state_income_tax_before_refundable_credits,840d599862cabffe0c71c424f7bb0de04b52e01db8204ec853c5d786ec42e236,be39eeca78c5de686dc9306224c77d63523842d622ecdb045cc8dbf180335dab +us__scenario_028__state_refundable_credits,dedb05f013f4d0bc3b8e4e44e1db641427086d49e71d6f05b61774f995da06bd,8237894159cc249372c961d2d04f64352128db6c99f330232b918c024382b4b5 +us__scenario_029__federal_income_tax_before_refundable_credits,1f08c254ddea02aee96773f0748ec7f6feadd8392023c2969b58da5300136503,d82f764c3c2ea4ac352b0005b98def82fc050ba5ed00b271dffebcfd163c2b9d +us__scenario_029__head_medicaid_eligible,c0b3872906161eb7715112880328d93a2a5229f11580f3ddb90bbfb6c34a565b,f4dd803e1a9f9ff433c253fb87c9f1133b5842bd611969d5aa1d08156d5ae448 +us__scenario_029__payroll_tax,7d9121c1eceb105fe8b84d4e37f679bf79588c26caf5883e0ad874346bc70104,41fbde72fbf3451e040beb627c841360f5d8162890827f2bc263f7c10a130c3c +us__scenario_029__snap,094f6578b1a61a88a638c8f79cfc0e728376a2887c0dcb1c765c7949feb89e1e,3b61c72afb3c066cd65360378d23594967b39d799bb8bbefef60fcf3a486e568 +us__scenario_029__state_income_tax_before_refundable_credits,fb9fb74ec04b344c173d0a6d3379ea4c755b98276afec70a306bc2e679d9ef4d,c09d8ffbab567b7ced18ee17d2dd5ede24168af7bd0329ecf1e4610175787d20 +us__scenario_029__state_refundable_credits,f3648d7030c87d14b5abc909ef38ff99c5fcede6801ebcdf9c0b1da9d00f4521,127407175b39b91f2841641216826abb888b420130b0ce6057bc02189ce80df4 +us__scenario_030__federal_income_tax_before_refundable_credits,d1adeadd8022d2178d52801d265c4694e1db01eb310abedcd35ee5ffda8ed14e,c02b8fb860b1d77992d687b09041c0bb3b27f69a6c533a05ca952f055651a11e +us__scenario_030__federal_refundable_credits,57f61618e950470c1bf3f02f2ef7783d63f3407420223b6e400666257488adcf,8d9890d58f1002307af01c5417c7782021d840e477d0970642ffd776bc66e65c +us__scenario_030__head_chip_eligible,d16e5feb5fa75542196c3d0bb16efc8812b4c1acfca92e45a0249e2353dcb393,1b466d914250e38efdd7da8d464c353025eace78d75e58440e230b268f7bd781 +us__scenario_030__head_medicaid_eligible,bc56bfa6975337ccf8120ccf0a1c249e8f89a582c239c513d3ca298249aa7c86,1463b28767dd41d43f4891c49be6b7621eafe4d2c07e025d5343987e28a281d7 +us__scenario_030__head_wic_eligible,14cb1029cd2ce2d5aa69ff7505367972ec63e395a1f1890663a8e320bb540d81,90bef848418206359576f3201edc9b39e4ef763fa3334762ece5757d2f6e4c0b +us__scenario_030__payroll_tax,9c2735bb834a6858cd4b17440a48c9012b04cd50f1b89162a6219a8d2658b086,ad985121c6ae0ea09931a7f8a60593d0e9f1e164e787019973e63e03669bb08f +us__scenario_030__snap,7fb50683ec5e4742b2cfaf72c841bada695349ead565c53e32097474952951c8,1a48069de387a5858b2c09f397b611be0eac9b3d3a352185f3e15d7730ce4f83 +us__scenario_030__tanf,fffc09d7657841ad6f1841b8c1720097020d7083b948eb9c8195413a1049f563,44812c88732db5b044421fca8bd9656d514d68381abb2eb0e2e511b6093c0591 +us__scenario_031__federal_income_tax_before_refundable_credits,819c34040c100bb9e43c8fd19dd3dbdfe8743863f39cf0306254c3710782a023,fd09daa78d449c1d07b0e97ebba123b7a30908b070311bda9523c736a272ee9e +us__scenario_031__head_medicaid_eligible,156b2491821ce8eccac9c2e32232f070c73934ed1b0ec457656603d90b3125d1,da67ffb243ed2624b0a01e3be7f4d1ab52189c48ea165bac99f285c360cee48e +us__scenario_031__snap,ddc30c6afe6af45f97c8f614843699d7782dcb4554354d5ef6230eefe04a2296,5e29a9f7892b39914ac9727a5df26c05b3f0b2b4310e05ed6e42d1559413547d +us__scenario_031__state_income_tax_before_refundable_credits,51b74fe0c64842577a891273b3d025e8d84e253e0678dd7f08733a4341bc976d,4d02402567c86e270bc6cbf1cef3fa5e1ea7c8ad77a76c401bd9f252994a91fd +us__scenario_032__child1_chip_eligible,5c14e66cefef3f6899746e2086bffda086d13977f6323f0f092f5204b24a3704,3b70a6b84dbc7a2a4e4ce03aef678af39664298ed35043130d8b632c58e0d927 +us__scenario_032__child1_head_start_eligible,21cb0f5e9e96dc9d8069a03ff69ec795b77f72c36c0260a6a37d93071be225cd,92734c2fa8c206a054399d07852c1c5865232718994bc18fc43b28e3abd20876 +us__scenario_032__child1_medicaid_eligible,a74733aed6b43492d152961fe2904aeef9fc38af34bb7b8bec5a6b8b7c339915,ac20666aaf2c33d4dcf072ee5fe9c312b032f76158aff8620b09762c07a8cc4d +us__scenario_032__child1_medicare_eligible,c22019abadd0cb5caf269da903768c4638e5dae7ab960295d6f10390973e46e4,5d3f8a17bccb2dc1081f32f8c9fa073d861706d17f0b8452b6e91ecdc64e06e9 +us__scenario_032__child1_wic_eligible,e3483185e388f5b5bb16af71b189fa562126761ff6305cc4191c5da306ed1190,5a9f0c9a3160a6841b40baca3ff92e059c3df415ca722c52beeed68ea33a1dcd +us__scenario_032__federal_income_tax_before_refundable_credits,22ec163d64a9f3aaaa77fadcde8592cf7410b2cb8ca0b41667ac11d929d71949,5d0402c111321e64e389cf23362a2c19e26c3d5178cc77d2f9f7da2c10f0c33e +us__scenario_032__federal_refundable_credits,0960e16795dce7cf2a281c04f1df56efd913e2c6a651052e9fc34cb3d41ff3b9,f17ef9b429e6c7f559eb2d523a7ecb6bc3b8a9eed76c8e1a7c4cf448d51768b3 +us__scenario_032__free_school_meals_eligible,a81379b2b87be0428f63c0d26c12665c1945a1bf0293f0d3c360d647e9bae947,da2079f33390650c9cfef0e9109d47866245c3097e955c286ffb6dcdce3c9c5a +us__scenario_032__head_medicaid_eligible,6b7f9308589ea2db980b50e3a9854e58c9613eb7a40250ddcee8b5a344a443c8,a0a7690508989a3cf0ea096f9f456cb1da61b4fac2c53bbf1bb1ea08f81ff339 +us__scenario_032__head_wic_eligible,149fb9de84050071b2b2a1d6057e66ef6b4d475c623b4128cadb26111a5ee512,01482406a486e823f19f8bea2bc62200ef8bbf3af2289f1c308fd3f6cd15e795 +us__scenario_032__payroll_tax,73db05d3115b30af9c15893506342899d586d580b98162ebabea032f915abebb,354482e612feabf4d21b5c8d14b96f6d0d4fb34895d8cd83f183e8caaa20a862 +us__scenario_032__reduced_price_school_meals_eligible,b690845477795d811fdeb1ca474367ed8914299e0501a412f25834da6d9af995,70dc16761420eff9defcd16604a73e761ae794da8c3153e61fff8ad4b6ce11c2 +us__scenario_032__snap,5f2ef8a4eb8fe3eee12e1bbe5530b767d51c04e0afc9f030ba962a67f3414e8c,f455c5f34030ed330632516ff5fdc41cfd09d0207c2ee92b971217caa5924590 +us__scenario_032__spouse_chip_eligible,abd4decc8dd5e0255658590c6f58de80d8158845961e2b955797ea69ff077721,531600fd17e2aed79c421c6a66e381983113078c17418dedfcae5ab2f9500e5b +us__scenario_032__spouse_medicaid_eligible,082628f87e0ffee1293496fbe2bf19a82ed6840bcafa9f0e997f761e243fb817,935ac1049c972dea3bbbcd8569074b935d45419337516f35b9d07edf88abb605 +us__scenario_032__spouse_wic_eligible,4525707f1622925908a5965a705b916bfbc11621159f00d27bb31306ea43ddcd,1fe71fdecdb36728746993cd362e22470b474fa18db8b82c01086d6a136befe0 +us__scenario_032__state_income_tax_before_refundable_credits,989e22da5d9a64d354a1e002e8c0098942b9f698221da16e1d14f30f50411339,05d6027cf5bda1af1f65d5be53f09188c4a14c74658d27b980c46af46d4a5f03 +us__scenario_032__state_refundable_credits,ba9855e7f40177607844083c92afa7cd990ce06a6dc78651c7864e49606b71cb,e817e194224811d7ce89da66e515e713b7256c9f2794c23d306383956eac9fdb +us__scenario_033__federal_income_tax_before_refundable_credits,fb35e73bbcb63399eca0ddb9808aadda958f828c5d4ca84202d2813dcc133fa2,5c1dbb482df86fbc3ae59ee58962e84a115a0640ddae8cd622fd4f0ccfb066a8 +us__scenario_033__payroll_tax,6b6980b0d668dadf5b1b79326498f7565b533312be8688cdebba90c75b2173d3,9af5039d7abdc4929420e509f2007c3f6061670a7513f8bb2683c129ee686b6a +us__scenario_033__self_employment_tax,4ef0f946d4cacf90e1ec2ce6e320106fa25505c7888647178b11772ebba4cff0,c3a6d369dc2ee778f0ddaf3347abcda4fd97b6b67bc38f6a389de2dd9459c5ed +us__scenario_036__federal_income_tax_before_refundable_credits,0ef9ff8529cc26f31bf37237f6d6f23f7ff5c9a7332f098c2dfc09d3832416b3,f22ea1f8ada9191769b8bb449dbb2308257f0f198914552477a669bd0e71863a +us__scenario_036__head_medicaid_eligible,e39b2db67aee20fed42c4d7c3892adffae0ce958931ccfb352d161e1c12f3129,4725f5b400463224b7bb95e59f46ffa390fd1aef49fc61567b7340663d72c08c +us__scenario_036__payroll_tax,11cbc8cf6911b85ef615e214170e417f47d2eea81498bbc7d15439de6c8a2c52,1c8845157fc48a774b09a022e8f09cf93403ba5a62e87f9a6c068d3374019243 +us__scenario_036__state_income_tax_before_refundable_credits,44eb047ec49f9f9670ac92d54e9f04773608d47a0bd1b5af8cb6dcaa879f4096,18fcf4ea207be1ae6722c98560c4f540e0ebcf81c43812974530d96ac205ade9 +us__scenario_037__federal_income_tax_before_refundable_credits,531ac9cdc17f3ea09e0ac72b0b55241fe415fc25ca52c38b8b161761268f8a7d,d9b63841fb4714b7b128e2c435a34949398e48acd99a0f8efaab97182c9e95d7 +us__scenario_037__federal_refundable_credits,1cdaf4d05fea8cca4227be65912a106921bf9aa19cdc3ccab6636ee1428fd089,3dcd4902d9e873ac3b22fe3cca0cad8c2a5c2c66a9c58e6c641a3c98f2f6a178 +us__scenario_037__head_medicaid_eligible,9dee63f6548a1da2aef8b2dbede00a813c136e6f217230f2e78a871716f94d89,9c93dedda1f19c53a46ba9c8c49aef99255f98165bf87cc6c4088c6f72c1427b +us__scenario_037__payroll_tax,c4449a92bc0fe4dc8d9c8fbd60350e1bc3ba2f04d1787e39a64d7cb96e286bb5,2cf84443e8ebfe08f8ec38caf28764f0624b7d8ef1522fcaaadaceb1df5a6131 +us__scenario_037__state_income_tax_before_refundable_credits,c2d3f7b5f374ae84b5e65925a4cee9f4ec990f5416edde533f08bcb0ea4a6900,3a9ddc2f0f4e04b862e3a28b00357b50ba467d0a6ba0c3e2b9373d0bed29fc23 +us__scenario_038__child1_chip_eligible,fb6db1a24c42ecc7eb89bcad872488a45a9b36872b3348dfd193358fa3db8e49,d283bed7f956c0eb83370b0c0ff023cbc0ee01205a1bc3d044ca5e8d8eb4cd3c +us__scenario_038__child1_head_start_eligible,4680754501a9295699acac70c8e2ba4d19b49e5e6ef9a6037494316e144f238c,94fc0d1d9f2ff635e9828b985e2b42257b9ff1705e4492a960ad77aaa9b13672 +us__scenario_038__child1_medicaid_eligible,a0fb7cab630a01a8802977ec0ca2833b2ac81dff313c3423c23f1dbbc36abc03,e1881b7c2399a917f552c4f68bb915d42d0bd605793ecbd0e0ff15af911ca492 +us__scenario_038__child1_wic_eligible,47019b4b8f2d5906ce077cd3814afabc7eac7ebf6802fa29fa20ea701a427f3c,9dfa7ccb8ec1ed65d40ef4a0f78c3cfd4f8e9eee5c8ea345f072ebac8d0e1055 +us__scenario_038__child2_chip_eligible,a21cab0d51ec6f6e81a896d97df7391166739f2056ee49a44ada793f4affc402,e7efdd3bf3e45ac3594254b8a49b1723a79c651739c7283d0c02ac19740bd860 +us__scenario_038__child2_head_start_eligible,bd9f2415f944741d6d17f7996c266a4b981329af0a03e80f646bdbe3f656ed61,2f20dc7270a97f493343287bdde878e4f4d79599308a62740dd76aca7ea9d48f +us__scenario_038__child2_medicaid_eligible,5e50d924df8cfacaa48b506b2647eef3cdda67affbac39115f4e218e6462adb7,bac9e1a1215239ac920601f8afe636dbf7f2f9ccd97886b2b8782b66acc80150 +us__scenario_038__child2_wic_eligible,f51f3cc2a3a957055cdff2ffc37e0134c94eebf4ce7c341afcd3b25c509e054c,18eb10faf6d3616c639deec21e8248d6d7ba5b43c8c1d67158e25f5f3f668eff +us__scenario_038__federal_income_tax_before_refundable_credits,f2898a25fc1d100cde86b30d04c5fd492953e55fcb60eb1a11342917a6e8b7d8,9c27f05cabd6370d201ecd98a538d4140ab5c592a569964b9056b9fbff30d9a0 +us__scenario_038__federal_refundable_credits,fd1b2ef6f0c7f37b840e63ed3e0f9517750a5552d0b567023320a4f37b4f8b19,699d84212cce162ae1d04ab11b60d5e81ffac59fc9fbe86d4156cf006d5fb306 +us__scenario_038__free_school_meals_eligible,e741e90cfb79dc6be87dd576c5479651f3e328374b04e6e1504319b8d59c2871,519f655a6343dab038ddbac5deeae53ced931f7c52b56274aa002e94e9b07b0f +us__scenario_038__head_medicaid_eligible,a31611a47448d514d62cde0bbfa9917677efc5b93691aa3946ee6a4bf0fd3e08,f3b856158cfdbcde1e44a070d4e776f71400462588500ba209110606296c2c49 +us__scenario_038__head_wic_eligible,b36b1d98223d742f75c7429ef91c7b8bdc0830d30e2b23a475b8348bf7439011,ba0f763475147eec949cac234b0075cd796ffa3de5ba9526fd735fc967b77448 +us__scenario_038__payroll_tax,41a36ad7edafe1305d2f6e400d68c37d02ef575de524585953741c19f825bd1a,964767e8eb52b0ca25e7baa56a9366b087ade120fb6e98fa1611f988a1d7a751 +us__scenario_038__reduced_price_school_meals_eligible,d7c8234536ba690ecb2bbb98c6d43fdd6d98c449e4a3b5e64e82049f5678c92b,b3339fc4f1c4ae9c12e31ae368f1c163395cc574646ccd92f5b449e68d3a577a +us__scenario_038__self_employment_tax,f20393a7b6f4323d77f1d146de4f1d15abe7b96585bb93e4acb2a2846def7028,27784a4a540b8743d30727453096186eee03fd313c1355771c447239e68d0d5f +us__scenario_038__snap,a3d7a2992ce4c5bcef60b2b80b9583fb2b74fbe60031e664abeb252f8a98ecb9,d3eb1d8fbbfd5df32733fede72185d8f88ae917851c3ace6868f5ab8d13cf9df +us__scenario_038__spouse_chip_eligible,6d1c5ca1572b421a39f1c5f2bb5a5b30a1dccbdd125e5324cd98aa18438eca7a,7fecd6364b45019bbb174d6bab9c8d7275648455f69e46f2c9bc471ad0700a8d +us__scenario_038__spouse_medicaid_eligible,82e888d5a679a410c60fc568c94bcd74652ea758703e4ea56debb6ea2171f67f,aa2789f5f8884ea945895b166aef41802f0cd5315a2e6b624868b3c73b2c2dc9 +us__scenario_038__spouse_wic_eligible,f77125d8b776513c4ae41e8492e5396411ed7ce0107882c107de765dfe1c71b8,d096f6ba749f5266c735fcc1f6d875dcff025e7a6af86a30ffb853ead5009f23 +us__scenario_038__state_income_tax_before_refundable_credits,49a9c2a7f26080a42cbdb384402a5f86ba4c2217499b06d0b150749c56bc736e,a1b13c01246a2e19d4595269d8c667ef1ebeab01fa7856946498835013346e80 +us__scenario_038__state_refundable_credits,0a0c363c9ffe603893231c199f42efe26782fc506d9902c34e7f62dab95469ad,39caba087491166a27fa2eae02e588525ee6b5ffacfe43ec91101d71420f8353 +us__scenario_039__federal_income_tax_before_refundable_credits,0d5de450dd4d383e1bcb41dd9e8ece9df19b15cb7c158b579d6dfbb72a89d6d5,ee6d3d2d302cdabf1814936915dd0616df025a0ced2b94f7cb6f33f3538679d1 +us__scenario_039__head_medicaid_eligible,0ba5e6c2ac7901139bb50233de7b47528377eac1a26b19a36b035ce6dad1c782,a80165745287307440e6bb2b173dc8dfedaa0c99852b2b6418a9d377cc77877a +us__scenario_039__head_medicare_eligible,7d928f4d000b2c00c6255ac3a01567dba382c4a9ac0c4f30664bdda6077b61d4,03fa1e5a73a017799487221c1ed39b922003ce65168272c04adbc5c635553f8c +us__scenario_039__payroll_tax,754dcfd7934899568d3f5a36bcc16455f391f704edbef9221da71be0d6c7f3f7,076258cd4169dcad780515e462c6bf2f95945ee23c587f8937813aaa8e68e09c +us__scenario_039__state_income_tax_before_refundable_credits,938f810d84e0b2393c73084d8cfb40c6ebe0799abae1e0164fefbbcb3743205a,3aad1cbdc88205b1569d16dfa2077fc1c3665cc5e455613b100bfccad6ea6f72 +us__scenario_040__federal_income_tax_before_refundable_credits,3f672d71c4ee36a64503c28943bbf8241d48698219981a65c99cfbee03b7c5aa,a5ff6471b366f730d62a442214379373964d25390ca21784e79cd82e2234fd12 +us__scenario_040__head_medicaid_eligible,36f71562790ddb5e7fe49883aeaab41245d92b1a023bda582aed544727fbe383,631f4f7cbea809810139f9de12bf5f4c3827963accc309cfaa0bbe9040332acf +us__scenario_040__payroll_tax,2b5d394a7a6c8b1ffffc3bc46b16099fba6b0009725550275d16b3c069ed21f3,8643dce0ee5dd5d6b3844620cbdca3afe7df39a73d3cf97930eeedcb482a0a26 +us__scenario_040__snap,5475c73155e3f6a912436a1fa2e06479725006da6f96de2b7b7e9c16db95c417,4e5ceb875f7b99bbf1e5222e67793919789490d5953930572f5fd1731554e7c3 +us__scenario_040__spouse_medicaid_eligible,67aacfd380b1d4b23388288f07e1ec3e466eeb0d40585861ed0eadcfe68c4987,b0a5af20fee94df77d32da08b3684b2d02e9b9fe1a99de2e2d0e1a4d8ce31a50 +us__scenario_040__spouse_medicare_eligible,3727bf190e86ecf635349b9bc4a8aa0a40578256aee6f2e4f22365abecbae464,1e6fdb7aff3bd6bee9af83655aaea163be4142ae063fb42bec30fbd283c04cfb +us__scenario_040__state_income_tax_before_refundable_credits,3aaffb2fa7c401919c52a7b99274443023dd680128f04d71f02ac961792e5960,1fbec781f81396fc6bdc866abe0c1dad865ded5cbf82e37f647988bc421b48d7 +us__scenario_040__state_refundable_credits,4c204814eb3dcdfe0106056fe29502b5cb1e7b1b6f02fbbcc4fe45876eb91ac6,55ad15c86b01c75b543c0e855b174477801ab2922a1f1af1163a9cbab7a1c991 +us__scenario_042__federal_income_tax_before_refundable_credits,3515d0212844ebcb780bdbe8f86301d1bea403d550a7d97381b0feb5c7933a40,5170813b5e766e78fb652be695e0f32a0093105e1fb1c410eb710bc109d0055c +us__scenario_042__payroll_tax,9f75e99227eab776a2637597c8ee73180586c94b24685c374ae42f11f2d85ed6,d830248982203453fd8aaa5af5bce90e8bc968e0f688b7eec37af83dfc211015 +us__scenario_042__state_income_tax_before_refundable_credits,5549fea005de54bcf3f3337ee10e7fe7be4d79be83ceec84ff3a823ec9617817,bd0484d9b12ea42cbe6d91e30b3877e5f8375e84d2491e5ef58d8d97060b223d +us__scenario_042__state_refundable_credits,62d14d3fc72f3b01dfdd520e2ee8ef32a6f25d5709585a976b3112cecf45cf6c,ba18c56640dfd549e76139605da2191d1a898b4f206df529506773b004cd548c +us__scenario_043__federal_refundable_credits,f28af07658f67f40423abac96b08c60a486ea8f6ce0eae0f58760ef6fe97ddb9,d9d1527e1797ba8cd94d148932bdc59352649e17a5521f34d240db5e69a2e93c +us__scenario_043__head_medicaid_eligible,e1ccc99d8a8cdac22f6bbbdfe1cc4576fcb80ed7c1dffbfd4b1ebfbfebbe073f,7b68064124c7e670231a88f2a2bb80c773c443f4e42c9209b74fc3577f1a2788 +us__scenario_043__payroll_tax,e3ebc8bd7806caf62e96b5642d17387c4404da6f65a0b640ecfabae2a50b2d6a,3baf4b01b10a2d8e12915859767e1f9e366c0f1d59fe6cc7071ce3952431794e +us__scenario_043__snap,0e59e5c9e0292b7afd79b61d8e67a6e1a5f79ab276196c0bb70561a583fa1b09,1574c02572d427990b211e2e3374fa6d2d2037d33f0645e84b3867a231bafcbb +us__scenario_043__ssi,3f2fb2e727eb13cb3dd6329b0cc4f1e9d1fb0e6e941a37be95f4dd26ef98015c,6eb8e40d43ed869b65016f02f060870333528d5a4a440d0ec199f405e2438e77 +us__scenario_043__state_refundable_credits,3ea9fe5599a39237590b50f5dc8ccd1eda1b7763510985e7fda9b249a6930182,5c3ee50b4551d5cedcaa158f8b45923a0a8235a7b635e9fd6ceee77ae4a2b601 +us__scenario_044__federal_income_tax_before_refundable_credits,83c4fdee16e08f063240f90069b0c36b1941bc538f3197869cb0ac4010be2a32,1ad9cf7ce57e2e67729195a6061dac993c28b7da063f7c73fdb21219307a0a69 +us__scenario_044__payroll_tax,adeacc729ca9a98af1b6b4d238c66b8197c435ad37cbc852d374f7577359e6d1,519822c2d72101c61ddadaa2e362a77e396c2614a2e7cf4dfc45885b6154343a +us__scenario_044__spouse_medicare_eligible,327e24a5a80ae7a43a36ab7e0b5d45a0db4b687d96855e3423dbcc032c290c47,6c79e76ee912dc80f51a343cdc636f6637935304584e8747012a52bb5ac6d35e +us__scenario_044__state_income_tax_before_refundable_credits,4a0b0d92a574ff8b6420904f60cb9cb2b6e3c275459d94b7350837ab2c38f27d,38a30ebdd7cf229e4e9cf7ba2e7e9401ae546ee4fc18ba127c62ebe91f03796c +us__scenario_044__state_refundable_credits,1d5b7992aaf84ffa48fb15346233f2510c7b300d4feb2bd033e323d2fe331215,153e9e572bd287d680ad073d03707f9eaf8b609bdea363ae0613edaa0d1b7ae4 +us__scenario_045__federal_income_tax_before_refundable_credits,50894b78a469df0dbff7d649c2abfd92d49f3d1ed427fe94ddae59d5e3574d45,f68938cf74800ce3ceb5eba37662cda34d23c217658f4632942b31977af5a5c0 +us__scenario_045__federal_refundable_credits,13546ea920e4a4cf6b9c4968bf1c96122e78675bab5de11cb21b0c8719fc7c18,120817a62c28915719bb23dc4b3dcb6eab05ef11398fc2001b6df00182b46d65 +us__scenario_045__head_medicaid_eligible,6abf1498f4b6157c34f1a080d8bc781c053f3a2eda99a386f129864de26dfe9d,d76318a7a85f55f8ef1fe56dc4ff16276c47675acfe70fbad48cb6066a67114a +us__scenario_045__payroll_tax,721ad7334fe5f9a80a965997b81e9927953f095ca9fd0fa1ba16455e8b142e84,cf2d7c881b5bd95fe6c7ab897f9bd8eefcf6d5c19f4b51bfc1c66a11688d2b21 +us__scenario_045__snap,b2cbc9c82294274eaf513f55413f8551012d51b914779b579611bd5c8677cfcc,aaa14a511fa37c104fb0a09aa23d8168b5d3bb817264c468eda53fb17d6ef348 +us__scenario_045__state_income_tax_before_refundable_credits,b13fcc727f86bda9f84aa1234efb2bdab45b2de2adbf1884dd2b1a9d4cef9699,c753fe546eaf6cf53b11661a08393329aa9e41de77f20e2180df8b6494d28e7f +us__scenario_045__state_refundable_credits,167589993d5364898b9932384b71e5c67ad919dcdb59a615e706fe25e9de6b3f,f8b11bcfcc584d70a3dd6e21b5f2a36e86f4a7d3e223712c9312ad714eb80347 +us__scenario_046__child1_chip_eligible,8ff89b3d57248ea77c5a07f8663ba37dfa3a42d33108f6d723d07991b69d52db,00a659cd188d3cfcfe57724417b48d020a2c501b7db935da822a790fa45ad8b0 +us__scenario_046__child1_medicaid_eligible,6e2400d102ad8a21791349117e82e0015f27da5e5ef9db010e4e4d9791b4cb3c,5e66e6c3456e42b97dbdaee66c0b31cc008f24eb9482aef8bec2d381db24b0e9 +us__scenario_046__child2_chip_eligible,cc7a7faedfff2ff24e739d9f42a0f73729fb23a043bc4253ae3502bf455f4541,a0978af88b17f0d2d5ea14929ea6fd2122fd1cd3f4a1ef2925fee7a5f5d57994 +us__scenario_046__child2_medicaid_eligible,632e3a92212e2e4a7ab0c31ba2fb09457805de753d7f6440508da00ba6d80011,d283b472cf9db20e924b4e9c096904e1f21415439992f817528fc6b59e9c379e +us__scenario_046__federal_income_tax_before_refundable_credits,db460915c7f17e303056f3851fdabef6a22430fa88f7c20fb3554a9672922320,5509c4beef818a4de894abc090e8cf7b2b6f31f2fdeb73b981b8a773ca4337a6 +us__scenario_046__federal_refundable_credits,522ab6d7784b46476362327048f87ed64145f4c840d9c64616303ce9014b06c7,7b8d903134a9966d7dfdd8803256224030620b63744f2c30b4dd0f4da1e3f490 +us__scenario_046__payroll_tax,979eb484a434b1b1da0222be9101aab630a0ef81849309fcbf5d911e16ee5747,8b54d03ae5a1df19dfb369197ed467a413c577e6f83618a5973d375365e6c9d3 +us__scenario_046__state_income_tax_before_refundable_credits,52789989bf05e6a9916c70041c44b8d59ad9d793f5c56c7e59983b48c99e2caa,180d91ae2d90064d82fcc3db93c639d5539534339040db3db01f1e995a1f7b63 +us__scenario_048__head_medicaid_eligible,af465bb558082e0762f364dc5ecb6e1078dc2d557900287625d73e148ee36773,d713f6a918118bea7d8db980d5d68170f53e3dbd891a4f4cb74c6e871e20c2e0 +us__scenario_048__head_medicare_eligible,1808e6d7862c7bf5cb34277d5ba9b52806f952d38672a674db04916198f7acca,28260c0de1c264e251b4576866a099d027eece45892cc40e0762bebf17fd4a42 +us__scenario_048__payroll_tax,7dad34f15f6d68e9c48cf260c82b21c1a81ae7810bf5e856d0609e1dde2798e4,3adbd4abc558ae22052586f981c63487a68d2d68f27c738c891d7189e50494ce +us__scenario_048__snap,1b55c8442a76c3dbbdf04129ce0247c265a73e1d7533a417fb082c4e7208378a,fa56edf7537532b4a8f2feb5b023066633921ad3d831ca39ffd362b9b3e58e8b +us__scenario_048__state_income_tax_before_refundable_credits,6b35fe6a04e8d1f85c3f85634977b1ad63ba701f23a118459656bf1ed4de99cb,1b7f8f71e11e12cfbcce38e1c085489dce9f98eda8892e85dc3188f2fa13b581 +us__scenario_049__federal_income_tax_before_refundable_credits,a630030a3455dc69ffa6b773c1bcbdf099071ca209790f1dc43a251c896a55bd,64ca2961109ec9568db4ef3e5f43263a74f73da37fab979016d549432d4ba923 +us__scenario_049__head_medicare_eligible,508a5aade6a873bc35ba9fd69f2459eb4de25e5014864cd1599e87b2131e08db,14574d1382c0d1236a4a507153300b4241958d91c0807e6ed6bea507cc5d1a82 +us__scenario_049__payroll_tax,cc58a80aadc63a97f3ebaa60a9cbde240b45708e5a51d6e914a19d2c9433a43a,f1a189f6b64b8f5b08d015e55d54478b880da64bcb5c474f3ffa825322130392 +us__scenario_049__spouse_medicare_eligible,5f2683eb144a866907725559634fc32bdef2dce3f8edc03b49e30a3442390fd8,a0b3db99aebc9aa0a46a192d8a414b15f6e134bf77efec7995267ba8e940afb7 +us__scenario_049__state_income_tax_before_refundable_credits,d0b334d42c5224a0836a87e42e8c74009fba48689e78bedbbb9de1aa532cead3,57c95a2b4ee4823d60e17f31d5c45b71c2f2ed2ca2d82f3f4de6124998f587a8 +us__scenario_051__federal_income_tax_before_refundable_credits,85e0e1f46086461574de76e350467695734d1900e7d58798bbd8efd173de3c27,6ebe19ce21aed104cd20e90091385ee819eaa08d287ba590ca3e38cb42ae4ddc +us__scenario_051__federal_refundable_credits,6e6bc4e61c06b828850b4d0dfa5489c47c5486bdd1496c350a9c1ca6eb430116,ad7a510e71955355b19f8a00ccbe064d642c767b2cb5853f638c2f1108562e4c +us__scenario_051__head_medicaid_eligible,0586fb51dce2e1ed38e98e3c028de7d7bb759db9dee6c082f0d4d7a62b438b2d,4256be67fb88ced26e6fe68b169c1e1a553f3ec35e90dc5ec5e020c17cea7354 +us__scenario_051__payroll_tax,8b3810f6ea11cd7dd4221809c58cbf4b2cc3c42305eb315380a3995f285ea209,52c20d04d1d7cbf60ee335ad28822e6fe7c5e9d1fd1b7cc33af2205c6da5729b +us__scenario_051__state_income_tax_before_refundable_credits,af842dbc18b84c079818087b7d6f257ddf6d573df540defef3b80f05f231c59a,debe0ac227595d88e7b2ec74b9995e165182c239dd5c124d073a1e722bb6c2a5 +us__scenario_052__federal_income_tax_before_refundable_credits,91d163e1a3d9007c19431511ae6f891858e732ee9f4fe573dd2bbc37d2e020ff,998abf3ea2f093a513fc7988dc2b132edbc0dcc6945611d97054d95b534304ac +us__scenario_052__head_medicare_eligible,164e17b4745fa0ef4e25039a0ef8068732ae3a546dcec246c84a61cfdcb805a2,af58ec2c96a3a746b0ad4c6eb8e1eea22489c2a579459b5bdedc9d9f0d33c51a +us__scenario_052__payroll_tax,98e5289e9887c19c13429ebe7511d82fcd1fe7945eb85a9606ea0ea46be547d7,279719aec1fccc89c890c85c5a7dc907ded888da7bb1de5b7052430751c52a37 +us__scenario_052__spouse_medicare_eligible,c12f7c907b26eac5511570a2cc0a6cd9b2f0f932296d848ee2cf68d4ace562ab,f3ddd1bcf7f2506f75121b24b4cd69805bedf69457e3c44e019602e8cdea1a3a +us__scenario_053__federal_income_tax_before_refundable_credits,f83e7065274174600e47181f527f27b60468402eb26f8f85a6708e5d66f957bd,c1d24984921cf9970c19b925c15ef39127ce4d56493dab56ea75624bce2fab06 +us__scenario_053__federal_refundable_credits,ffede74685464280f003d47044a5f15147bd05f69c22e68ec99884f9d0f50c1f,15b94adadeaba3a4b146f4bc23a477f06389d1ff11e121d55988f5bc68c58b07 +us__scenario_053__payroll_tax,a055146a31b782cfaa1cf4892d9341e842dfa41f748994d9db4f220923805d27,8930286bbfdc1cfefc8df6576d255c63da8ccea61791374fd42eeea4cca05181 +us__scenario_053__state_income_tax_before_refundable_credits,2514bf1cf49cb2a69eefa8c934ed4ea4fe31eeeabb3e6d6941a5f5c8b5a63ea3,2f345eedc5d57116fd91a647f50bc8469fabb997154c833e6a1b12f7e3520014 +us__scenario_053__state_refundable_credits,90982adec7967aee24fbfde8b6c1ca0442c191a8ed13064490831e82638ddbe9,2724b39a99a342c4f29198e14e37169d322a1fb39e60dde9980ad38bb1fa63e6 +us__scenario_054__child1_chip_eligible,e910c6e99a7c2b82e93f415f33fa07f1b72493756e0d177a06d11fe1d0711709,99f36d276b92f49d6638f422500ed312ca85a1bd09334cbb7f6ca5c860a29725 +us__scenario_054__child1_medicaid_eligible,755ce7ce7b429ebecb898e41db45889a81e3fb532a488ae436543a438760ca64,780d9c63ac2430244647c0a3cf8b0e2e139686b2bd49b642032d3be324242914 +us__scenario_054__child1_wic_eligible,ebf420a222a27cf59b7d6378d04dd6e26f220d9d6ddcccd70d769784c844d633,59c6a581add091a9d9c395526ccf381352213cad6bbcfea5627009b0261c2a5b +us__scenario_054__child2_chip_eligible,4e56930e77963c77c11b1dcde0db4f7c0aa6b1a720b0c4e35df3c327a737549c,cd86aad7f8dd4e17624e9a99f337ab94b2e989fad77b6e326391522b8cfe87df +us__scenario_054__child2_early_head_start_eligible,1b6908a66c8df12a48e990792d1f70c867072608b6af72f925d40708412a61aa,50bb5636990a7bb880353c2079577a1ad0289d911fffbc0c92b8d978e284fa8d +us__scenario_054__child2_head_start_eligible,e54f05a192cad6896ef57e563eb120fe3ef6bf407dcab7add1ce96689bcfc85f,7b7c98f5e43441e6ca18573bf030a073ac6ed5fdb517d82c19f3af50295e0492 +us__scenario_054__child2_medicaid_eligible,7915be52b82cf6a0feb8293209271869ad1dec7e718a990c8080a1e336e9b420,ce9abe300df96a44969ae510aabd67fc18cf2bea6d1aef25e64c50655563e9d4 +us__scenario_054__child2_wic_eligible,73fa3d6dae841552259f48b922a155d40d7797027efa5aec2048a95a64e63bb0,2a20c57db837283d7745cac1c1f876366f51cbb179728799ffe66ab8ffed0caf +us__scenario_054__federal_refundable_credits,0698dda2cd618c7bd1665c5f8fba5f39cfe565e52498d5a4ca5b8d641e3c2e3d,beb3643182158791f5818ef6da20370b4ded61ea670ed96fd3fe162c727b4610 +us__scenario_054__free_school_meals_eligible,272b41f318402b8f8281fc7015c8f20af1afdfa47bae9f705720de7fe161304e,0fb737c11bdb437e593ac92ede33b104338a0ef2b37b50e47f07e5466823a570 +us__scenario_054__head_medicaid_eligible,17e5abc53ec3bbc718a912a326e94a39589d7e7dfaf5355521d8bf4fc07a51c6,55f985ddff08fc45b85169f79248661951ef21fe907254e639dd16ce8936adf8 +us__scenario_054__head_wic_eligible,25f02c541066aa31f6a2c2037bdadab41d7899d90f3855bbf52084b7b844013c,0ce5f6996b56c8e716934976320d228185dc7e19e367495186f072dca8858e9a +us__scenario_054__payroll_tax,456d183aff785f6755222c483364079db7049ba9738400347935948cb4f6891f,564378153453428d1a8a02c2ba3dbfcb340f1fd1f99a201b9d6e8b2d6d26142b +us__scenario_054__reduced_price_school_meals_eligible,f12a56a4660c97ba4d7f5df29798df060b2955d9ecb829a32e6e5811d890e88f,b27fc7c3d4106e3bc6c14f3d348dad0cd19c0b7814774a1a0e0c1798bc92833d +us__scenario_054__self_employment_tax,a8acdb6ccb643a3820e3b3ca22a76b301690f22cf7f556c70010957061b182f7,34584872334d8dd119b96788585fca3af74b4e8e9cdf2795d187dd6059c877f6 +us__scenario_054__snap,2500a2fb20a83e0666dd3cd6cb9e791e07dbf4e7ce1c387412acf3061de2b8e8,f9af944a45870ba5749f0a34410d74d771a958a727ee67fa4c73a8193e9c74dd +us__scenario_054__state_income_tax_before_refundable_credits,a3e88e18ad91a4baf3c47c93e7b6222ccecc8e62c5e2b3da93f46935d1b83b1f,3baade3fa32e68dc8facf405f96aaeb7335fdecc08574f2fac53b3de85571fe0 +us__scenario_055__federal_income_tax_before_refundable_credits,f18892c3259f7d032f26e3e0253fcd5525adde7110b71f839ad23889d187c29f,cf0a53f92114cf43b729b1dd2a81edb23f65ef69fde6cc3eec2bc53372f1aa3a +us__scenario_055__payroll_tax,d679db023a368822952b62e560f1d87d125577422fe1cd55dd09aa809cd09f46,3e3d36d2e8c525f216592da5e54c991e39f7014827b0c3e1e564fcb01207793d +us__scenario_056__federal_income_tax_before_refundable_credits,869fed4267fd38e002f36fa8c77664485d37446ce7f17227f88b651d7e6077a3,ba7abd54e65ac9d1269f5a042a2f76e33045f3746893f3f3164135cd2e0780f5 +us__scenario_056__head_medicaid_eligible,8cea184c97d14c48ab06ae60786e2e218504e4532b3d05dc5daaf77617ac7f0a,8804b0dc69774740c7590ef027a716e1d0dbda44601e402b6a7cb4ec618b8993 +us__scenario_056__payroll_tax,6531a2cf3210005ac2ae383bdcae048f99d03047c5e6de4736637171d9a3dd00,5cbddaf39f01de2b5d6a0cc5d7eae0b40221abafc0709b317e06282cf8bf6a91 +us__scenario_056__self_employment_tax,7901956ba0a795ba0c00a0c60afdc4e588328f7098b18e4cb3f85a81d327e81d,d2ed557c1f4ce1a2beead21aef264a49b77c71f1b7a5e2d8c98546a890fe7727 +us__scenario_056__snap,2fca2c84af589153b2a5ff63cd3279965c88586ceaaa7164d7e43e7a3890d4dd,cd7f02516e974876d855fbea87b896eaee5c5c75813f3f6bfd8a835edbf31d44 +us__scenario_056__state_income_tax_before_refundable_credits,828b862f6768e81a9585f9ae508db46ebd09bc4c7eb4bb9dadcafd573f2647f0,2be4296e7aa80df26a2403455fb0ec1d3a9ae7fbf8c87715f4cef8fc89ab7466 +us__scenario_056__state_refundable_credits,9ccd5181a1e922a0cc5013a76cb501cb397e9285dde2d534a390fcd9a5eecdac,c6c23b679da533c4f4ecb38d3bba82ddd46b8578dc10a04185740c12dbfae43d +us__scenario_056__tanf,e29d915d8be36465f330cd19283eeb444e984e459bb631b80b28ede890585dae,c9259662a30b84fbc876aff7e14b7cdecd76b47016a909caeaf9f45390d70b5c +us__scenario_057__federal_refundable_credits,3f9283c0d7c688fd650e5de4e59758b9e74fb8ba86b0d3e3ac2414381c65662f,857cc27db2a6ed2681e8352479a993caa1a94b503134f7ac33988e1776467d2c +us__scenario_057__head_medicaid_eligible,232324e443d96dc96c2f845225aba9036508fd7fe90471703643878a3a619c6f,919a4a1120ec0f87f90ac23e44903af2adf9b48201830dc2982060891ee68d62 +us__scenario_057__head_medicare_eligible,96bb4fe44b511474bc69fc78af6ff88de9fdb9836ae4aea160a23de88902a4da,817389ea8f2d6eba79d047872f57268d96b0efa1a3a89e9b31f894e2fd766c11 +us__scenario_057__self_employment_tax,5ce219cef2a55c537084fda62318edad4d9ceebd1b2907131db70215cf6e2687,024e7c4c5998e7c761d8a235dba4191a7ded0b45843ce21ae92e9cf14d14fa4d +us__scenario_057__snap,769afc6849c53e1ce72a7bcac4d69e1c2580d7faddf032776241e31d99dd1a59,29cc6f8b354d55aa186eb9ba4f822446e05d6d5152ae1848ee3a0457e649223b +us__scenario_057__spouse_chip_eligible,40ece4779468b0368b2a671ee61fb8fcd292b92b38b0771797127becac9b3190,d24757f59ecaae276e455383add3bb6de3af7cc5d25ac40f3bba805006598ff7 +us__scenario_057__spouse_medicaid_eligible,c9d80fe640b7c9227e9c5eae6586229bb8c65c2f5ddb7bc41dbd4032fda1ea96,d1835cf06bf4d537ab87d2cf05b494e2fe3ec69cd7f16dd80bad05c1eea7a219 +us__scenario_057__spouse_medicare_eligible,119e2bf51d00f2ec1e2542ca44f4eeca389148c57b7d4330da70bad45eb7beb8,763f6b5016485ec69ba34af5be24f0e1b38bf2d636218d48a3c5f7f1365cf3d0 +us__scenario_057__spouse_wic_eligible,cff4ad62140e6c2b165cc678d22144e5df113f38535884b399283ec3ce395179,5a5d2c00c386a9da7842d62f48676ef7c5912bbc094c7c689ac6784da04b60e3 +us__scenario_057__ssi,93e479c621cec041d3c4c158f8d57747b505e9d019378c1eba4de321cbab9bd8,a03c8f46a67a2683c3661c174cfcb49326a718ae46edb527813af10030059b38 +us__scenario_057__state_income_tax_before_refundable_credits,aa152b05672b14cddbbedfd10c01edc052e23b7948665f0d2fb02b6e66c345a3,6155014a283e4a283cb844becf06bdc5e67668f64d129b48d24cb8106ff54376 +us__scenario_057__state_refundable_credits,8553f329d63d966d787afc1164a2197a49ed584d159325e811c620b85d28f4dd,b945291b4e0b66373f8e9ffe70a3c99455059b626669c47d721dabc8b1c7f8e3 +us__scenario_059__federal_income_tax_before_refundable_credits,1d439d6a4477451fcba34f7427a23275a3de177fbb434c8bb2169ada2af69e4a,9dc6032d8cad01227baf3b1f015650b7d0357e076fa6d7e40794694e92a9140b +us__scenario_059__payroll_tax,419775e1c08e04d6bf1f077f1982d6845dc0f391ee5a7309abad54ceb7d88d07,7693f0de9b470f8ceb671723561f7c1f75062244cb024eb40be97ab1601dfca9 +us__scenario_060__federal_refundable_credits,ab74a66d3dfd2ec829f5e47e906bbc0387fac68fc4c2940cd1bef51d60c079c2,f2c90191bd630870458cefc9d4cb7ed54f90af5e95152059365a924ef85b9a09 +us__scenario_060__head_medicaid_eligible,afe47b5a9a0f5298686c27bad084339c0cbaafdebb1b0103723605e8ec1c34f8,f9757bef2887b783351fb7ca4737cf76aacb2f4d638d70f60943d01c09dcec0f +us__scenario_060__payroll_tax,99b393efce61001a1b7ef18e1934ba1f186a93770060701b9932e454cef62ff3,e2900a1d724e071400b61eca7563b1635d6a2e532542fcf75e9dad0b52f51f1a +us__scenario_060__snap,72fae2f9e5738454676e15a0b6f23b9d5778ea7fb2dd254f1ffecbaa371925af,494e0586a67324fea0722cf480096c813695e55f29aa949df670e8a5d1d1b4e8 +us__scenario_062__federal_income_tax_before_refundable_credits,05f0ae128288aa617a3555109f7ba21d3dc82fda3938ab88938d4748cbeddc5e,d81a5a5dc1eca4ba55cdb3cdaba0a372a40a339a0e79efb01d663ec67aa49a9d +us__scenario_062__head_medicaid_eligible,174f8369cc2f5c91b7ceef7d3a98275d257d9c56be745e0a39843c7c46732b84,97a6ce548ebd03cf6c3424c54c57971602590f8731110058b7f1ceac80be1f8f +us__scenario_062__self_employment_tax,f9890892974cf662e6002cca7eedc3c2b616f76346ee872214beb93f686b071a,8f686ccfcc253f315944cfd2ce35c0f7972dea79a025ba6837f7f05defd695df +us__scenario_062__ssi,e1462635938189f55b8c9d75d8a5219a5631fb0cfaa7b49f6759e777ec44cf1f,b7b62aa2f60ca77d8300fff19b817d86faf1192700b6945bb9551d44b2e6a5ea +us__scenario_064__child1_chip_eligible,5d365c4e2795b1793b190832385354135429d96c17f03000668bc6e648d6acc1,0c8b99b0d8e1c42d4ce88430902504b12cb0b0e51d35bb6888a891ab55821bcc +us__scenario_064__child1_medicaid_eligible,330f20345f568f8242837f54e2cfcc004f0e70833eb55b3f730aebd095ce8190,a343a7fbf71e698cd0150f5fff049c8290f3052ac8a8b630d5955c9abc4e21ec +us__scenario_064__child1_wic_eligible,1f471bc60507ac15134edfdf89f0b723fa478de4d363a66baae1d77af86a8bd0,40c1117e868d81937384ac0ac878ad690fa6e03d9ed17d0aadfb1ef2737e7b82 +us__scenario_064__dependent1_medicaid_eligible,1bd4f46e6c2a106618311db8e984d510c65f906925fb4b6e90efcacd4ae2ddb7,8173e4b5fd7befcff8b654e1e66a1560b53474b93424a19e3b3542350d92acb9 +us__scenario_064__dependent1_medicare_eligible,e43fa50ba14d70838c85a0beefaf210eb088a2878d4f68d218c401c0ae7d757d,7b8a49351ec89a25963939ec856a84c57b9fb7e7907a5e2acc01d58f6262f3cf +us__scenario_064__dependent2_chip_eligible,bc53b95a138fa2c27903cd3b5a3507c9d4a3c97eee1c17172fa5c27c31d01d06,868fc9a16564705f61ad6a51476a5c558fd92ba01e7e3697038d1eb5dbb50575 +us__scenario_064__dependent2_medicaid_eligible,e5581f7e66a002891937d6c0fccd7cec9928de4805cbfbc0d24fd5d6e6cc164c,6a4dc6506ad751fa451ad0d43ff52203a6f501dddaf5d2f3e00500ca6e4ecb91 +us__scenario_064__federal_income_tax_before_refundable_credits,11ebacb1b12b330018b6c739936434acfacb6efe534b29c3abc824975ebeb225,8dd1c443f4d6094c8ecf3ce6022f3b297c51c7366f526f0b4887e66ded2d7151 +us__scenario_064__federal_refundable_credits,198803cbb4e457ff11805fc6fb99ed0efc12cacafacdb48b9d7d5f0a6c2372a9,25387f1dcb0642d4ac5e11ecdb8af787451c7d12734afc3630988376e4bcced4 +us__scenario_064__free_school_meals_eligible,e8e503f32843f5a4cd62293655c0eae89d7ac85538d77b9ce9b944efc647985a,c4257c9d0d823e5d54dd3988a6cf87a6bafe76ca4091c94375cea5ef418a0c10 +us__scenario_064__payroll_tax,9664b997329e18ca93b1ba3af26f86fec41b844c7dc8d16075ba48280aca76e9,fb482bae6bf94222c33c0ff46c1d89a64d7ee131c3b535a730ea71c5a1640d64 +us__scenario_064__self_employment_tax,eec323b1b8ed51e273b9ef58b1a0c588494a76e0d177d944f27402970eb18515,2d0bc850240854ac857104807b4e3d8175335aba56da72ba8dad041af6d2696d +us__scenario_064__ssi,e2b84c4c8e9991f6c28d7eb60033f7148544130f305bbb82f963cb9e0ff544d7,0dc3089d4ec15ca23b7689e8b351e75b4ab7dbabf8c6290b5a8b63e59969f55e +us__scenario_064__state_income_tax_before_refundable_credits,c319626cc67a3df07eb24217380c540769577c241b3b6607a4ef6b96bc605e97,806778b3960b2cb8e5aa46de8a5ccf1ab4191f09a7eecd45f6a31d96ee0fd4fa +us__scenario_064__state_refundable_credits,2d31533eb9728b969695198c8016eb43d7226cbf317011fd5caaee4922e84664,c7dc836f5ddbe8a8ec015472b276a4d8b1c9e4dc1908f8ad1f16a8a62ef66b89 +us__scenario_066__federal_refundable_credits,e364143a6ece2c379e1697803158e39a5850ee4a5c7dd167469df5aa80e2771f,c166cde90b9104849e3db334e1b225dde77d694c7c6c4ff3b13b4fb0d95754dc +us__scenario_066__head_medicaid_eligible,124d6c83eb99fc5208213dd28aedefb00dedbbd488a2c40b42929e468b79bc08,47b8aea10e6dcf42767d89b767b766704c4b4e17526a6af1bcfaab32e285221b +us__scenario_066__snap,8209611ff60c7e4de6ed8a12dbc30f95470b0cce960cc5b36939b845de30f268,0b3f12a3795ea34ac276c9245317ee7337787caeb44b664c08c40edcd49ee827 +us__scenario_066__state_refundable_credits,632b4145c87de99d13c55f1d244bd1a059209a8ddd6a62c2f150baa59c9ba009,7cc7d739abfb98e2e195d3208a5aa91b35ce71e7debd3211e2385558d70e6a47 +us__scenario_067__dependent1_medicaid_eligible,9cbc76d33e89a9d62f135f2b95e8613f4f9d085c4bd2aba9fac6ee7f8c317dab,fb827611e784dfba133a4174709eae2f660a3dde77563313e9821e2d574097be +us__scenario_067__dependent1_medicare_eligible,6c82d350debd4d9e97a14bff9422dc0c2324f7fbdbd6512bc6c54988080c97ee,1c65d7b163e8a3d03d75a45ff8f15dfe10e2a2a7c55ea9c0fbe9816123bb787d +us__scenario_067__federal_income_tax_before_refundable_credits,f0a03d3dd9dbbbbcf6f06228895b7d1bc566df396749241cb9340fd5c20a14a3,14b3c650086550d7d336129a91d9a09a761db827c6ec633e3c846f81bc1e248c +us__scenario_067__federal_refundable_credits,d029f05fad8543b059707f9b3a0b2eb2673f9ffc122d33c166eb5c02d89fa187,8a04a288d5a4c5fc9367a8f6abb9bccc8f388402e23ff7177c6ac8e55bd20ee4 +us__scenario_067__head_medicare_eligible,b52a10fe439c3a0d9fea4523fa7e5d5f5eba94fef9878311a24196a2b36c480e,e420b9cabece63aa44e206a9f4148ec01522c8a821a49654efbbb8d30c62bcbe +us__scenario_067__payroll_tax,4ef064e3a112773d5fc87abe5dd119a7b656c53b06d65f1f2d00473d8427ac6e,7167f712406b9172e2c02c5df3cc7dd4a6700aa24696050df175698622a13898 +us__scenario_067__ssi,af6c500acb21de1097ef5ab80b018fae1f23cd05ea3be2e07b71dd679975a511,2158e5fc2980876918a076ee51fada9d91f4c49cf98203b43368d9f090a6b515 +us__scenario_067__state_income_tax_before_refundable_credits,f4a4705adc49d7054404172d79f66be704eb931f7e7985742f9c401dbd40cb58,dbc043fd81f0818df89db234c343b41259ad40307d957943a777a905ac7744bb +us__scenario_068__federal_income_tax_before_refundable_credits,e5997ffb253c642c9a28efaed746084fb888031409fbe66e36faf12f2bd34082,8eafb0abb87a757cdac37994fa7bf0e6f7c4d72e90f328064331934f84482573 +us__scenario_068__federal_refundable_credits,6300187f606a96161e337ed0da3623b89e0040ed068c4898352bc2061d8d93ce,ea4b061ed07ac0f610687dbe9767a5d9e626851ab1757771493c60a7e358de4f +us__scenario_068__head_medicaid_eligible,19f5a72cdf5d984fc842871eb8607c4b549466018bb36b6f04b7d6be921e3e16,8f59d588f88ebaf855be9b388ad5aaa5e9b2b3e52b202227033bf540ebe0ecae +us__scenario_068__payroll_tax,a43ca39d6de4cb7807845586da8bd87f1a96badcfc83fc6bae9dded21622cac3,765d5220e322360d9d68a2b808cb2d098a013cb6b7db45c43123b9e007fb84bb +us__scenario_068__state_income_tax_before_refundable_credits,6c2f11d412ce65ffd08a34c8b5b53ed1fc9934f3cc735c9a59a364b10bc89f35,0a6cf23e039d6d4e40f8ff2035c65f2371e838ef862c19b31f292a95ce8e234e +us__scenario_068__state_refundable_credits,4a19c8a306292486a5a2ffb5ffa7d23db6eaf519425ba6ebe18a427bf7d76adf,a90a721260c18f62ea2de2c672d2320ab7aed3bba2a75e3a74e53f15e394abed +us__scenario_070__federal_income_tax_before_refundable_credits,17a232f0cf32d934e6de2898a97172c8747e736c8ae7d6f8eb3c5404a1b068b9,8206795d51cdf0d5ecd424f695ebe24f95a9ef148e137af6f61b4c9531189023 +us__scenario_070__federal_refundable_credits,6c984702dc99967234e1448f7c37a55d07dacf5cba470a6ddb36c9020b4350ad,c15a83a6353cba54edff881cf585fc3752a941320ad36c3ac3b387cb95f0119b +us__scenario_070__head_medicaid_eligible,3cb246086d3f2633f7559ded9b7adae70eb773fcd67223e846bb08181a8f068d,42998370c8e6bc126140806ba78054929b2828c1fe2f92e7e41bdca17c300c74 +us__scenario_070__payroll_tax,d3830d6de8663afb49d463f3951e804b12685f2abeb76d570f9443622d758f59,1a49a2cc9cc655655f1cd3f580552f4939080350e7b000214a43ce7654506c3b +us__scenario_070__state_income_tax_before_refundable_credits,d5b22ab156b4f4154fd44abc4be81d5d1bf0612fef4a4103324b85ff550c1c86,6fa410c1064cd206b438b9513472c83a3b1d2abcd30b93cf97005638b145f5ee +us__scenario_070__state_refundable_credits,d6a8f5765042bf78f21616b0c4b87a51827ae0a769a0b39c7448507e2a35608c,678dd72b80d6ec456f5fd815d0e0ead8894860be0022e5a087d5122a45330d94 +us__scenario_071__head_medicaid_eligible,ba99ebca759d39acd30547f4bb2f06d2161374720399b22be919a2983c7a928a,dc1212f95319023a03f94eae3467b623a7495aaf08bdd995ac6ec7d359391cea +us__scenario_071__payroll_tax,123f3d610b3d0ae82b3f4df3a9f5bfb960671a64c47f6bbe6396c7f3202adec5,6616462501479e935116425369020023f1a3e8c39d4d6fd3f53cf00064ff2551 +us__scenario_071__state_refundable_credits,6840e2782fdd59bd46ce7d5aac8b2bcbf2d550da8bc1e91078ea2b4a85d65b60,9ba83cfa83c3e12acc3018f44d1d6ccec02368a9981a68b3afb7b3d1d9dc4b54 +us__scenario_072__head_medicare_eligible,8303ac810b4482adb25fbe1fdfbe1d0154f91d4d3cd660af81dc728b5a352818,619695951d21cfb3b3ce679f6d4057976f9277eb3ceff6fb811f83b7df8b8929 +us__scenario_072__payroll_tax,42e1e96ebaa613af49f2faf9686b75e3164179d2571f3aa394bb03455c8384a7,c61b259094aa5a9a9380f1a0bcd27f06227930baf8f1ad4f19a9ff74387248f1 +us__scenario_072__snap,99d0d803c8e9c2a40c60c14a032da91da14afda5941c513890221b41135c3847,28cab303bf314c742a14f4dcaa304f582f5d2d8cb88e4f5e34be97ec0cd35d48 +us__scenario_072__spouse_medicare_eligible,00c27b71fe5cb873698511e6cf4d986e5e6b3e25d067b5b633628f441adc6cd1,31386bfac7c095691dbcb3ec771403a326a679b4ff0908eb0803760e4e85c31e +us__scenario_072__state_income_tax_before_refundable_credits,4eb78404c03e372ba9c32dbd8c41b4e306a1d07a65050e5bba6f56fd6b1823ae,0c694480a2a62dfd8aea7f709e1feaa39a5593bcf261b9d316f85f66effbcf4c +us__scenario_073__head_medicaid_eligible,1d15f0a2cd18cc26d835eb79c4a47540be4cbf54f7e6155bcda5b7b054cf2fde,0515d34a252bec804783063879bbbfb9ef9dab77c29f369ea051072e068f81b1 +us__scenario_073__head_medicare_eligible,0a1fb04f38e096a055cb7c73ba1455602ba851cd6c3c7db370118253f8167cd5,9e871421c786a853c005cbc79fe0284def24692131427b59955eea5516a6b8c4 +us__scenario_073__payroll_tax,62a8774e28a1d2403c47602a40f50751c068f3063a0df96772719ad687c9b7b2,dfd6807014296d6db200eef2d57f1e2a6ec3e63eb936c6b28f8e1d6e649793a5 +us__scenario_073__snap,36c5d358ecb8cbcfa832562a5700a6c0cf3ba1b9ea75ead99c990a8dc0bce5c2,c465fe7cd2b1ac93b46f53412d80373dcd4bad2efca7d061a86ece2dec8061cd +us__scenario_074__head_medicaid_eligible,84befb4c8867cc4b47c5aa511c00b1e3cf709262c4ccc6c93b66d26d0f8371c4,f358889317dbe6c6c2d77f868aa2033578f7592fefcc40cb64537bcdb567394b +us__scenario_074__head_medicare_eligible,661eaed74f649afdb3d70007460f142aa80411db8afa5cb51adfbca3f797f320,b181ddfa64457c117d3a1797c688674737a6f2fca0298c3763a1d2d10845ea97 +us__scenario_074__payroll_tax,470de108439fb9d94ca606a71cfbfa8ebe2e807907f172934333e6c93cd4ecdd,3ee8e6be5541322047c977c78a792ac03938004e431df721d89d37526fdb2441 +us__scenario_075__federal_income_tax_before_refundable_credits,b92c9f195f318666a6a49cdb60657e4500a4e5e21b804b073b366f9ac3e39c9e,caed73db9ab9e1e99898b927b95a4ed5aef6735121063944db18383901f863eb +us__scenario_075__head_medicaid_eligible,3301d5f151624e892a7d200692c2f9b3286e339f68a0f52b4a14ac573d770521,9285a211798d2f119a35c8179164fb0db08cafe2d72ca3f7d195eaca3f367ed0 +us__scenario_075__head_medicare_eligible,55b0c12ec6b5d0fcfc00963fb78d0628c870e0d7e0922260f2ea4c582c5ed2bb,abfb03885f46cdc3611dca71138fae66ecdff7cf01a213f7ef5a889a6eba09ca +us__scenario_075__payroll_tax,0a4cf42598637457633767cd80a96f894015be79362b187fd6537752f7b387a7,ebf4dcc8e62100166bd922d47f650942942a2799d5d1ec3d5e278e5d659d9278 +us__scenario_076__child2_wic_eligible,dcb324e790cbf06a106c3cdb595c777872c2eca48a3a4ba1f57a0b5348db4ca0,d47eed169beac46c7c0f8ad0537e54e5233515739c6aa11849ec639935e0f93b +us__scenario_076__federal_income_tax_before_refundable_credits,69549e322bb9d4db30f8fd590200d12fa5267b763ea32138a21e016c541a345e,9531eecc0f39155791219b7123a8700e55f4364e2c570b25783f85d9e6304a8e +us__scenario_076__federal_refundable_credits,4125a480c2c2b3f05ef21d1b4b30065f70997a7b9419fc144dbc332500f21fa1,1057142da7c9a28cef711e0e5e206c0188da6869e7c415f99eb7868c0c0958d9 +us__scenario_076__payroll_tax,cf731d2ccb49b1614772ace7fc7a47afcb1fbbd9df341c5e0f06cf1ef31d47e9,2a309f139ac532c2147fe024d1485aeb3f3181f2253a489e275bf4ca086018f2 +us__scenario_076__self_employment_tax,2d150397ae7fe5ef0a663ea7529317a421fab3c9d43adfece5b2f9dcb71a9fb4,56cb0733e2e10bebf17bcc2e51ca2ab7dd0b5aee26eecc95fdf5f74a0d625228 +us__scenario_076__state_income_tax_before_refundable_credits,bf21bc6166bd52fca529bb490a4a031feffeba471f003c7d2210c7fd4d48f7f1,0ac03bfb2f41f7be5111665a1fc71dc29420482194f38a7f60131ecd70f6de57 +us__scenario_076__state_refundable_credits,37e166f66e3e244114112f07c12cbae121ad76c94cce00be2515e177d40dfae6,b7fc957b5576a27f903baa39180eab7c7d5ea6ddd532f6a06b267dbde86ab693 +us__scenario_077__federal_income_tax_before_refundable_credits,191fbd441f79c23d157d93512e7183710809a78d1ff8be06b37673e9a2cf3dcd,0507dba0069f170a79a4e2e5e6202ffd8986c7ef46b18eb86966b9e3c19c6442 +us__scenario_077__federal_refundable_credits,b8f45460d30c69ec4f3b5c14ff1acdc762b6d82fe9f553621d345bb5c5859153,be84ef35a55e2793d5e8a790ce803caa6e270c26003936463169e76f0c1a29f3 +us__scenario_077__head_medicaid_eligible,d72d46c9a98b4b56dc463aaa6553382adc2442bb3a184d3d91222a38c5749899,7daef34f5bd68e1748ad3a0ed17772bce6c440b4ea37f93ba8897de2d10d7e8b +us__scenario_077__payroll_tax,338a29146830eee07f27635e2e63eb6c6a9785b752b595b4cf6d402cb616868f,4d836d7a89aa1405b0da7a4b7e03057d1d94c70f8189e1727b1c84d5a5975967 +us__scenario_077__snap,42cd315c4574a478c9f5ad080d0c1020c3900f0706793796c24be4bd13691110,a9c625950a6df86d4d71c926ed932aa92f4a86f3d272012e9242f57ce95998d5 +us__scenario_077__state_income_tax_before_refundable_credits,904ae6362757937cc84c7a3a794b5f4facf4476ab0dce2c7e5342d7cf0ea62b6,80cee6d82755e4e79c7dcc89e1d4580b4f7ff256a120877ef6203697f65fbf34 +us__scenario_077__state_refundable_credits,016fd89e08a42ec4a55ad0006c9bc9999c5bd0038ba6f84a328a9b40a47db34a,eb290063dd4670a7b7879f0dd59d665a52599fc1669426a7c830194bea20558f +us__scenario_078__federal_income_tax_before_refundable_credits,b017c6921fe5738a814eb6b768893ceafba2ff0a01f731b361c50eda5b931369,6610e1906bd9548d193559ab8b4db2b9331ce40955cbc21be8e9c97a96dd121d +us__scenario_078__local_income_tax,ae75e1f800c367a941799f18bfa5df88046bad46d93395e67dabb75c8020f187,6ec4769f167ee3cfcfda8e2b42a07aad5103527f4940990caf8849b3cebd50d6 +us__scenario_078__payroll_tax,5e2ad2cc8a3a504e76ce068fa40edb19f4b9f2bd56180658e1edc351b6521845,d288b3157c5b4b9baf116871390ad03692a24b157bc225bb3d48fa9c8e69b29a +us__scenario_078__state_income_tax_before_refundable_credits,d18b08c450c3215502c5c1c7f82aa490837fbea7b274a5c48548267b7cc2f780,5834d0cd99c13e06a872d67afdde3c429781faf384ddd206ae15dd58b212c90e +us__scenario_079__head_medicaid_eligible,54607808aec56f1a6d052748f5f44a5f63ebc6ba602de245a30258d5f9375819,d13733174e7a5e03ca4af5124d5a084a74817c97abf9aecb24fcd9c83b2da0a9 +us__scenario_079__head_medicare_eligible,3afb0b4c177c2f79945e033493ea5626fa4ec96846251f44f8e7b319e107f767,851ff4a1182ec096dc4d65fe3b64cd357fcd4c565f98d88e6d2cde138959bf20 +us__scenario_079__snap,52cd35008c350cc1bdf469df07dfe9d12083106bb0a16b5f58cfad643aa21498,b8d0bcc8668541fc943b843679813924d469eb0a0b90893c7ff27a879237724e +us__scenario_079__spouse_medicaid_eligible,c380546d28efeda3bdc439bdf5eb9697e0f1f6329ddd31e18a908af6af7e862e,36397b770011d07056985f6ca8492d56f84bdcbbf024c37a9fd78151576edd92 +us__scenario_079__spouse_medicare_eligible,cec86393c35b4f546c728338780141823369051820cfe5f2d1e76f8e21a765c3,405488af7f686fab047a5ae60ede65fcd4c709b555e376cb11461301192eb84e +us__scenario_079__ssi,1c9c70b5645425d697448c47895f553728c7d6a6ac21fd151e7b9a90e10a3dea,630216cdc7fffaa68090bd1b70e9aa6791f63c3f93e5bb4fc16da615446e3048 +us__scenario_079__state_refundable_credits,22e6eeedeb11dd9faecca7542d30c4600cfc7bf0d4ad5a1db1dd3c6dd0f5df62,3fb107075977b8e14e76897ddcd76cf0920dfeb2a4dad674249e7cacfc039c6b +us__scenario_080__head_chip_eligible,b9e243b6e9c037068d659a827d4efaf40abd5d121f00858acc06f269b5ae1f5f,2bebc812bdcd3afb3421d9cfcd80f85c031a54519c059a9ea5b62a5bd832d008 +us__scenario_080__head_medicaid_eligible,16d1b05f6af3005da239eeb250450ddf94b48ec2bcbfd41f3619bfe35a634db1,fd63fe651f2a41a394659cb102c39e1df15f02821352fb72391743d4f9c031b7 +us__scenario_080__head_medicare_eligible,492bfbaefcfc3cf4d7f171d47776f7c047bc20c284c26e9d66b6300db4a7ab44,42129dfd841242f5cc66cf5d734ce1a8b079879a1bf9d95136305c06f3308fc3 +us__scenario_080__snap,7ad7d538f14c953861eff29192df96f9feb7eb367af18a01d270efdd301fb0f3,222f30b266dcfe3ce6e7c46e65a885ad170343b2205d3b7d29f59c39ffba5d72 +us__scenario_080__ssi,43ec21461fff18629819bf6759eb8e9c588bd39df5f8995dfce3ebcdc481c8f2,84d00efc18d7c7b1daa09fe96695cb46edb33843d7af0e65f0d3d7d97b80940f +us__scenario_080__state_income_tax_before_refundable_credits,26ea6c8424e9afaac05bf877993a8cf76a402aaf6992178465395a8abf77d26b,65e8795b8cbd17ed44252f37465d7cdb50c6ecf219a48c336691cdb1fd8c706c +us__scenario_080__state_refundable_credits,95beccf4253f75c7438dbf00dfc205c887dc689e56002fee1d0b4c18105a4972,ff2fb3e2da9cb07e82d437f6d9a4ca29ddd983170e1876ad3a7ebc3d68e86b9d +us__scenario_081__federal_income_tax_before_refundable_credits,85efbcdeaa0c4816969bb7aea290472564631d48dc28923a5d5a583ea7cf76d5,6aaf7b3344d97eb1c063b918c6a5a1458661e704bf0334d0828ce2a17a3304c7 +us__scenario_081__payroll_tax,1ed3c1e942399f545efac0f9785aa4bd67bf22498ce01336a1c4eaa89bd9d819,b29fd367d87b47db08504f5116ad0dfa315fb8b570074126b2689b4bef9fc54b +us__scenario_081__state_income_tax_before_refundable_credits,87f221ffbc6e4080aa38e4a6f4eb53b415f5041d10afa25859ae6c662cc68733,793d50e48f8598b85f5c1c3c0eeddc1b1fa01ff5a329d9931a079b85bd5d1876 +us__scenario_082__child1_chip_eligible,80b33e55fbf3e28c423c7aaba899db4f361fd63fe4174366cf3a0af84b2559b5,731f2b677eaf50a70fe8968a3ac828a4fa48185dd863e74bb6c5a87316f0b99b +us__scenario_082__child1_early_head_start_eligible,d241170df6fc5ec2029d90f9fc36e3c0efe8f52334fac0783f87e6d89f63f7f9,6f50e20abc79a7c20c8e238114f31f4ad62d176cb1dbad5bf67ab443dba826e3 +us__scenario_082__child1_medicaid_eligible,d4c90f759a1e2b96dc2246cc21623d0d55b418986ae37fbafb34d2f65d2ef1f5,f2f7943ec6b3367fe7205f7f9eefd68110bc23da84128c82960ec570b09de8ce +us__scenario_082__child1_wic_eligible,3be2990225770847807aa5a221c2e4b8aae867b804e2e3eb93b39d925badd0f0,165c8776674e8d5807c4aa4537e26810fef06866a5ad4578193e9103628271a2 +us__scenario_082__federal_income_tax_before_refundable_credits,4fb657fcef719b83e000b20a0106f3ec33d6f7a13ac5d61c3115d064137e224e,81aa2a674a4188d65d1bc87fbf11be80f77db5fbd028db3dcacadf9950f7bc00 +us__scenario_082__federal_refundable_credits,c14b0ec60660e88c83f681ac79565b9e931def8d84dcea0590a0c99e47740a03,da5db67eeced7d31f30ad4a4775297e571fac1755b5118b03629a4c3b4d143c7 +us__scenario_082__local_income_tax,c5b87f8b4749b02085941e27d09aee571394a5de297b9a126eab8dbb3a5e3696,e7790e074408c946cbace6724e8bdfe01086d00180dd1199765c7918b9b798e4 +us__scenario_082__payroll_tax,51557f6b625f9f70bebc0ec9af38223ebfef4704181afea8235669074a842606,b7a8964530c183bb40079292b65272d9387a758efdd583b53e239096e06858fb +us__scenario_082__state_income_tax_before_refundable_credits,3c899069db098d04c6bbd3afcf26c02f68a2f4583f9bb849e1499dc0b80498c8,a155239ccdede0b06275a0bb6351b77067a88efd4cbd0b32c6ceb53937e70dce +us__scenario_082__state_refundable_credits,8ea39c75524817fcdad1887a9b48f9578dfd2e1cfed2ab56bf3ad21cbf6c8edd,7e8430a540e56db4ab5484c7046b98d944b8151c7c1f073247c05b525de7046b +us__scenario_083__federal_income_tax_before_refundable_credits,ebfe45d55e2ac1aa9edd1ce1c1311b2972a11be334240e20605b367d4e5091b6,73e0da8cdd71ae02011f35c379191fd015f3652f82048893e7afa40e3a00a078 +us__scenario_083__federal_refundable_credits,608f65e5e547b0b5ec853ea1d43d57eaa0d895ed98f84d8ea6cf9b72a1e887f6,117ed52ffaad02a72621d18e1c940a874549d1cab93f06eb676a74db9adc648e +us__scenario_083__head_medicaid_eligible,8358e5331be3f07cb279f791570254a115496649a01a60a36c1c6df4f46be714,1e04ead6d9649b46349582885ceb3c20cd5f4592c4fa4a238d9919a7cbf2f64d +us__scenario_083__head_wic_eligible,4435607c931ecb6c4360844bc887a7bc38e9dc9afa7f19a74a94457d3914352e,9c87962fd37c542e9a6272d0dae6388a94115e560ba1bb116bd347e05a97844f +us__scenario_083__payroll_tax,fa01e9d20210c5424321ca7882e05c25bc0fe860101e43d5ca179fd19236b77e,c80c2cfdd4be89b14502a5c9a739b9308bf5eeff37df32135f5770aa926bf052 +us__scenario_083__self_employment_tax,7135171a2c6cac98702a69c4b2942f96153cd6c2123cd81e8e6bf3f958f3a085,7dc7d9549fa10e089409843468037006efc72d517049285383f48a1bbbcb74ec +us__scenario_083__snap,847f77b72b10efe0c5b2f6273e5130dbd0e727511e215cdc07419ab55319acf6,cb1223419824e574d2938c3247d031169ff0e1902a6483b496d0975ddca9a23f +us__scenario_084__head_medicaid_eligible,0e5e90dc483e8be74b2d68446065a96b9c585838afcec38845f7c8bb57d65534,88e6a367b44a1e25fc365bec18aae59c25f3b077635fdb2212ff131be3f4b284 +us__scenario_084__head_medicare_eligible,8e9781877e07d8ba2688818834beffaf982095ef3005bd3d71aa06ee65c69e77,1aae4222e16575101c4da0ed1e8fc764050c1fe4bbbb9ed9ee589ff2da6071ff +us__scenario_084__snap,31df3f430c2ff4676f627b356e2ffa8ea2a50cdd37d5f593e9daf1fcc401545a,cbe27bd9c1a4f5aea880cf52058afdf402cac8536dcdfb776d6c752ea881e73c +us__scenario_085__federal_income_tax_before_refundable_credits,e444cdf8dfd4276c27691ed083fc9c2f849aa678be1af0c1f506c0b0647452aa,2feca4723fdab959a645df2ea2720b71e1b2705879cca813b0fdf28e7c67f37e +us__scenario_085__payroll_tax,d3d31aa47604969bc9eb8a80b68fd9482a43146ff55ecfc0c187b1f33f582406,10b9eaad094fe3eb3b1402f485db0de6c5fee485ca11738722c4ad76decd18f5 +us__scenario_085__state_income_tax_before_refundable_credits,d618c72891fdea1b9638b2ed9e35a072bf6bb270b75cafdf02e9e29f08ebc53f,05015cd8e25e30562174814fb1650261f575788e331b61aadd93d1573e4b969e +us__scenario_086__child1_chip_eligible,132b2353232480d388f6b4fe9b18719217ce2ee58fca95c1bab94eb72f38081a,8b42d7488bda25472edc168c6cc71eb0ee1b7dd3616892e5033eaed742a301df +us__scenario_086__child1_medicaid_eligible,b93c423803d89654ef4205ecbb9450f810b164b7f0e2a494159edc8b3f4f25a5,852e7130b43982e057e00b4f4a0a49d178faff72c066725543f7e6ba5ceb86e7 +us__scenario_086__federal_income_tax_before_refundable_credits,9ff321da9b79e7ac831e84f6f1010325f8db33634035306b85e960a7507538a6,1c97c59a94e665898894b9350df34122ce8a9fdee7e32b4610da97aaf42a14f1 +us__scenario_086__federal_refundable_credits,278696e8621e02462272b88b837c08e4a6dd4ec2518ad57360573d0fd9435d1e,101ad0a457764903753416f9aefa0295e8fb6de8a583407675a3e7e20f504de3 +us__scenario_086__payroll_tax,daa030ce5a2659241a349cc5010205281928f2367e846c278432ab7a30c1a2ff,5a4c6d0128dd3f73b6f231f93181ad95820781102c12c9e76da2ab171dce6b90 +us__scenario_086__state_income_tax_before_refundable_credits,006ca7c6d1c0b41449f1baa1737233a35147e15be93ee9be92d153687582655d,04a67b2968be871acefeab993db92c06de7818803966a8553a35ff7ad587d169 +us__scenario_086__state_refundable_credits,ea1145fdbb6d74270c3ae2ea376d9aa3a345c1c12c125bdf8b139a6e06b0da0c,40da83d0513aa388507d9c5a60bfaac4e222f1fc2b8d970984683217b43dcc7f +us__scenario_088__federal_income_tax_before_refundable_credits,e1bdf03a8438ad2b82d2d5dffdb81acec8c63e18ea0927ad3710a23ca68bb1d8,80265259c57739dbba4b287e55eb8edc58985af7d438b4dfec98641d640d9039 +us__scenario_088__federal_refundable_credits,1b4b87155061a5567d6f438eb8c9507034b9ba0eca8b8e7386a2abb45ac5f8e6,548a6ac1712f13b8de12e2f882aef60f22c960556f3106fd4da39821f47ab74f +us__scenario_088__head_medicaid_eligible,f262f7ff9be7bddb5a00cdee6dbe87c0704c5956d0b93b5b6cd97b8400109992,c94f5e5cac556c5a4826900a6b3793a7939a53ce6fe38819f5b9d8deefc45bb2 +us__scenario_088__head_medicare_eligible,21ae3c74efb67aeeb20856ab39ece2e8545fd6802a9a4db7d8cf2e37f5da9099,7d444b630e33f0cca14a5d33e062b05801ccf72f83ae3b8c015299b54e311d77 +us__scenario_088__payroll_tax,ee728e53172e6c8d1a81d8fe48b4d77018ff88f0d75ebee0921134a06b3276ee,c1636310cee9bbeadd30d5d11afa023f948a25a9330c35f9d86b4cd470e41f5c +us__scenario_088__snap,b513ef27aa643ae954feb25faecaba19236c48352e713e0a023956d1115c0f8c,c2749c18b4c65d93ba5e259c14d9d1ebb9f1d32fa02606585f5407983de1c37c +us__scenario_089__federal_income_tax_before_refundable_credits,8c4663531f1f95ca10d30d828f5e7b875eef226500f03693ed32dc0c99578737,a97ba8e60ea5bf11d25909bbd6be4472e063ec72c4f75d132ed52e00f6e19774 +us__scenario_089__federal_refundable_credits,76b8ffbf111908e61b6585203ae5d2b6ad3a88576729bb525d54a1dc3da1e0f8,abffbc6622c3590de80b2b669cc6b0cd2bb82bc6b5ebfcf9d4db194102637e4c +us__scenario_089__payroll_tax,acb6c51a690ffc0ad71d0de1739fe53077ee70df7c474a1062d0291b18940e96,921b11870b75bb6fd06a2445f0378e6bc1610476b988c83b3c71fba027eda68e +us__scenario_089__self_employment_tax,af4f031ef8c78d06d632aaeb63644df33d0e815f5abe75208bc1f678698b3fd9,6b3a3325e09e82d44ef5f19d476e92276b8f27a0b8bcebdbcf7756b1f4f0a377 +us__scenario_089__state_income_tax_before_refundable_credits,910938b1a93ace7ca25982276c459353edbe356f220c010a2f9204868e36084b,dc6a6a6587e066ea46ef82e0c497248ebe2e044e9a21a628afbf640763ebab3d +us__scenario_090__federal_refundable_credits,883b9d8dbff5f97cf405381d19b5aa6857773f235705f7ed3b6e633181928a9d,0b986c5eb2a684246d25fa0f7b066d3f6664a4b46d150c4cce4899a4a13c3705 +us__scenario_090__head_medicaid_eligible,975518354410f94fda216542642828d9172e924dc5dd5312a6292e316ce5cc95,b2bf8589c249f839f0d978d9f85d4d37bf37455c466db33bcf5547446b7d52c1 +us__scenario_090__head_wic_eligible,de7963217884179690525723fd5e09cabefd68b1fed7287ade0df00331ed0912,cf121c9a8d4a8cad7f923bcb84e57a761878545910ab9745e14f92cdc6f9a9e5 +us__scenario_090__payroll_tax,0241e654bff01cfc8123680d878f92de0b5c8045919cc6c7d34d7972598b7a28,ec28e561136033f45a11d8f085a74170568bb6c6bd2a1b8d525eb152a308972f +us__scenario_090__snap,8b4c8cd1fbf6e67e564afb530189a2db3a9ffa4a064ff14ac5f513cb0190daee,e2ee14fe34e800822456143b8d32fa78b83adbe276ad36e517527d0fe4abf4ec +us__scenario_090__state_income_tax_before_refundable_credits,9bf19f37bb7c98744890933cf4026b4d9ce5c0b25597dff3b06dc154f5761920,2080cd651c58a1a1cc934a4b1a1c0bc864d7648f4b787ffae81f9d0c759a9e61 +us__scenario_090__state_refundable_credits,c69dcf0865eb11ac1db95892523febda5700e6f4cf89c009ebdd24facf4c27cf,b7830c44d0e70c649f4120439b44056b0b8aa160cb07335687e64e09754491f8 +us__scenario_090__tanf,0d5de46899b23929efe04fdc55ccaefaa9ac6fed2a89f738f3cf646a16fc1654,4497551dc0871de67850eb45ec3df301fb19656905b16327354b507d2418f302 +us__scenario_091__federal_income_tax_before_refundable_credits,012e43d131b1cabc9c8ba06e6d6517f23deca6238c80badad7b335356a85468d,8b1b13f9923e229fac336e3ce80fab0808b720bc688e8560877fd0ad0b7e70fc +us__scenario_091__federal_refundable_credits,8df1f4ea06a8e5e061a9360214c3eb0d2aa9cc87b58724a5166427fce19f53c4,b93387945c5ced8c23931f580276818d481bc9c6fe3ff01595fe3a1e41976a0b +us__scenario_091__head_medicaid_eligible,09a593f97e8e0f55da018a2c9dc3bf5d71862ebadc6a1f10d828feac711e3733,bc1816dbce66b03d332116782e6b3b849819fc83821bd7f0e41c1a1afaeabc6d +us__scenario_091__payroll_tax,169c9bb322ccc517893699d6e8378cdbdf41346f0f2732c8bc18314006b3b54f,e78f7cd27868bfb017034d223abc39d20441154b5ade46e390565a27401eea31 +us__scenario_091__state_income_tax_before_refundable_credits,c1ac8f52b319698d0e313ec4cdd1221703181d4542de8f82f52555f610c41e70,4016d2798ec41c954b76f1e21d2ef4e8833826bad8798daa17b5ed43eaa3c31c +us__scenario_092__federal_income_tax_before_refundable_credits,9eb3008ac41ee85eb6a6e7109170d6762b486b8f3b1a0eab767ad1df5004fc21,a6a8d17eda6ba980a1db0841174d0f345b686bb5b24c454c5655dddccd776189 +us__scenario_092__head_medicaid_eligible,d0656f0e15f47f9043415bc39d9b5b1eef75d2cb375a0463806980e068e0192c,ee9ba0e26fa0b67c5c7796ad5a338e1ec2b0fb7d8409ce1b14bd4cb41d28df08 +us__scenario_092__payroll_tax,05ed96e35072eada4a402d80d6455b04caea25572462171004960e696c717e2b,e53420483a6035291619ac59fa234908e0101dc75891ff043783c9f8ea2ffe9b +us__scenario_092__state_income_tax_before_refundable_credits,a7150d425e823dc20533bead348bf37380d391e8fa22d3137d70132cd18c359f,838df2ec9b676e9d0ae5359830a848e99f91fd70245f608f0f9d7523b4d04617 +us__scenario_093__dependent1_medicaid_eligible,eb1c9057f0e3cb56fdf807100bb1a683550fbbdb421c0f5f236da96c28302a3c,f99246c546c6d8c8d52f3ae8c0f9072ce6f1f3dcbcfbecd58f1094abf9a61b2d +us__scenario_093__dependent1_medicare_eligible,4c98f5d5e065d92ff76f3323c7b589f75b3e80876e179c3544ee96764806944d,64ec5368210534eb7ad9108a5fc76fa26d75feafefcbf0c1042c5c17264934b9 +us__scenario_093__federal_income_tax_before_refundable_credits,530d4ae5f36dfd6069ab0f2029b35f284bdf22e0066f556af446f912fa2b606e,ffe6813fab9da2b5199078d0d73d28c980aeae66f175c24387eb046a72790d66 +us__scenario_093__payroll_tax,362396ea7bfffae44b4c6d1583fb68a2ea4a8d408be65d05c26656c5e8a2e094,f444d5a8a73f7021680e0794eaf9507c907d0c009bf639eef25083354cf5be00 +us__scenario_093__state_income_tax_before_refundable_credits,6dde48e773ee560f5298988112b727fe2c84a397bfd6cbab244e3d72dc1ddfe0,ff59b438bd0144216e3ce19fce53ff841e0488b4d0826ae1707d2547b75fb011 +us__scenario_095__federal_income_tax_before_refundable_credits,ccb5c9665099f1d79a8912ce862a72df12f8fb00805a0e82f82f719b93303b14,91791c2297010efe95154511008a2e3d4dda5025fabad17253f3d8297d93d551 +us__scenario_095__federal_refundable_credits,b8bccb7e5a85fd9b6635acf822bbab9961528b69d361dc678efd92e392d2f2e7,c07bd65803460c28987589426954d25cf63a87e43c627082b58df374b144fec2 +us__scenario_095__head_medicaid_eligible,6cd67e0f6cb5665b551e0b9b5c3f83076d9acae10dd5de188266ed30298a516a,257446f7cfffdd8ae8897c54584ea6b445118e09798fd32e98a0cba5dd350e5b +us__scenario_095__payroll_tax,1c474b2f24426553c37f93431a143ee7c8c107d0fe1476f2ae5f690da317c3ce,ced8b7c0a5b863ce0ca9aafb5590cee7a4a677a2848f806dde38ea6d1ad5ceb8 +us__scenario_095__self_employment_tax,20143b1cec485e2263a4bb9acf4a43db0c6519de11bee4d125188e136af5d62c,f57d9dcc03cfdca601a222450c567e38c14377ff8761d59e34c36ebbf2650cc3 +us__scenario_095__snap,098ae23c19657b353b7a0989d70d3ea33b4c9f3721ac59087b78263f336416af,aa7021baa80f939bf546fd2535dd0ae16620d3aab855360654c5db84764581bb +us__scenario_095__state_income_tax_before_refundable_credits,a87ade1b4e8e9ee4bc49463bad701d16e0fc1b8771d00b034e2876123016cf7f,4c892d51b8c3cc174ae824a90a11422666543dad45514cf32c14d40acc1476ba +us__scenario_095__state_refundable_credits,7e7a8d24842d99f54f9a6d1c363ab32189dbbd1198525c68eeeb65ba9f75a58f,7108480f987033e0853c9bc7e9de697493710bd08e271a4e92679c6168157e60 +us__scenario_098__federal_income_tax_before_refundable_credits,a38f4b835a8615fee9e27cfc0ca40aef6c0147b04557ffcc0c4ff32c64eaa212,ec0835e4644076760e0c9e99a307ca4f8278766322d505d908502bf615fd5b97 +us__scenario_098__head_medicaid_eligible,8cd262afd7a4a5b417aefbef9b8e5e0faa2c6cbe9421dbd6c4e682bf64320080,5dc5b7fe103930e61b86379bb10b504b34a78b5c76b8d16a0dd27a520c54ffe2 +us__scenario_098__snap,b70c95dc088b00f8350eb378731243909cdac4379b57e884ff1cbbc6a565e262,d07af42cc921dd613def0a4b91761a0f0e00f600f021c6896ffd9ee03f75d6a2 +us__scenario_098__state_income_tax_before_refundable_credits,3cbd357144aec83f90dc1093c44612e75515b8e3e4501df85ea99fb576480e04,1ee09ff45f612781bd76ae55b794d277525bae38f990b10edf80c9b6633ca8d0 +us__scenario_099__child2_chip_eligible,b0f31b2f454547fe5d09dc6e639c8c4e63989fb675bf88bda01f713c545a3c52,2ea73dcd67efb98b2e7d6f0f267882c81aa878e35b41a9bdc9b7a6bb04358715 +us__scenario_099__child2_early_head_start_eligible,2fa95a87c099ff8ac433c5b2b8d7624ee6144ea3d47a7590ae159eb9a79e77b1,7b908adc23d068598ecbe1c7ef1229e4109ffcfd64159586f34c4890a30acfb8 +us__scenario_099__child2_head_start_eligible,86da19ad621cf5e66d38598fe8e04c7d92e12a7d18fd096c15dbd08065d4dbdc,ceb659b95e5d3b715ef5b5a5a454e87ea1ec5201fb99d0a1078c40d4e33054b4 +us__scenario_099__child2_wic_eligible,fcbed40dc9ba79bd8a1424465919adcd4dc22bb89308647182b676f05ffcbc10,3f3394172da33877788a16cbb8deaec82da485bee529b664be1e9c2780deab4b +us__scenario_099__federal_income_tax_before_refundable_credits,794be385a99f97fd30de85271eb6ab17bedf78a2c8106bc7fdf974fbd52e62e4,34a0352852925ca3f56e90224c39dd4abb35d043f44bf8311506c0edb4b30cfb +us__scenario_099__federal_refundable_credits,168e1138247a6005842f1b2525b8411c148b6527f5c4f41f68a4c82918aeb4fc,de41a2f2039a46d03bf52dbfa99e68a28039d42c1a273e93e5ba5f4da587195d +us__scenario_099__free_school_meals_eligible,f004dfcfee69aeec9e7bcfcae2bb3660594df2c890f59fbaeb115a4b347fb675,bb1e171903416c3e880b35d9745e0a0d3c4bd6191caf6985a75fc7e4e9f2e3c8 +us__scenario_099__payroll_tax,3a7f05528b9a4b49ad84e8839c8096f979316aacfc06f0e774542ba0250e589d,5d5964c8ed0a3873aa02f72a0282d61e301fec582e3fc10d24f4dc8673969e4d +us__scenario_099__state_income_tax_before_refundable_credits,64ebb147711a8eeb6fc8b81873c8aff6b5d33d124dd4ad2fdf8e7a852aed09c9,099ab7858bdf8764fb01c3cf1eec279c1efa5bc9c6ebea8150941e7c1072ca8d +us__scenario_100__child1_chip_eligible,7ffdbc7ca5d08f46ca46ef66182bc29a8c46cd19a72a87925e2eaa4c36d40514,92bb560355412c7e0ebc905e8a42b418aad216365b267c7272ce94e55ac5f87e +us__scenario_100__child1_head_start_eligible,3a45c04a38aa6d578d3f9bcc7372a85cee7651e5498a49bf590595fc4ce95ed7,e461745e43a775ad6610aa7b32d3f9744820f64ac1c7c608fe6293779755a99e +us__scenario_100__child1_medicaid_eligible,f4d72f7e9ec7a2a3c872bcd432b325779ba1e11df4929c9e5c21c7b880ed190c,1d1a4add8b5311c807c8f9ff80c58ae1dd96477788525eedd426296fd953f21a +us__scenario_100__child1_wic_eligible,b93165ac4af96aacdac7ab4614ddbe807465a52a9cffe523de29dd09f7e6cc24,7dc92815ef7b7b2f828b9e4083f18a1a2bbf182180e6ab46847f6c909bcdd828 +us__scenario_100__child2_chip_eligible,fde0a7af991c6e5e3740c523584c06a4a43ffeaed5c9b6febd2bad6d9a910429,b09ad5050f3a921a53a2e3185ff5a6d41efcadbb366b85b3b026ccba41247a7a +us__scenario_100__child2_early_head_start_eligible,c293f7bd6ec6d82d1698e3fea3cbc42d8e812b9003cedaa382769b50631a4924,37412541482c0bb137e2db632405a91b0b55bc15b4f2d40b00dcde3d3b285d72 +us__scenario_100__child2_head_start_eligible,6fec1a2f819664628af449323dabf4d3164be30ac606a39c9ef58fa9436970a3,d12046b335644c4a08ae44c001a1759f6910d7f63ef2ace53e15ff26574840c6 +us__scenario_100__child2_medicaid_eligible,fe030897de6850c05bf4ebedf681a49e2c56fa9d5935b651d2702155c36599be,231902793dbcb27611ef0d54fbe06ded22d5c10211053898d714c0dfcd97a12f +us__scenario_100__child2_wic_eligible,15aac302048c0071ef24dea934b0db74ad55fb9d1f0d15a37db249bcb57a3efc,dbd041e195efadfce756c8f63904babf943637aa01ec27b88b307b5a9486f990 +us__scenario_100__federal_refundable_credits,6fc8b14d084d1cf09e25a95401db7db739ad61f90adba7da633c98194931bbc5,d69995cecd526b9b9c5e882690a2e687f09bdf76ef3d380640c71c3424132127 +us__scenario_100__free_school_meals_eligible,7228854ed21c25c78f6154b660ba69b627b06c3a83ee5f214927e83d16d897ab,346b3cfc914a2538ead9b42441edb562c301cf7f59d04eb601c3825a759fb31c +us__scenario_100__head_medicaid_eligible,8923cf8df4c4e6e63caa569ac41c858a1d8a7100eaeeef75443d45a05bc3bf4f,96081aa4f900774be344f8f0354ba1040eee81fb73a457623caec09828f39819 +us__scenario_100__head_medicare_eligible,ad947d335e7a019c438dd97f875e7b9e58b25aa08455f97573282a67e2dce0ee,0ae21bb28543c90f5963805ea426fe4027fa5b581dbabbc47360e3592591e49d +us__scenario_100__head_wic_eligible,cf153d90fd04487b2e320b91ddd524f4f43a7d4486f8e9d5b7d4288f2cc16b3a,a419bd2dbaa28c22cfbf65bb7e9660ee4fca6d7157e5c153f24f71de0416eaab +us__scenario_100__payroll_tax,f864eaf0d9dfe4da779598c661c52b7491350ec4a6528e4730514d81bacadf61,2ffede13456c8f1cbd057431c0494a09c92dc28c6391cc1c6d40e249bd2b6a90 +us__scenario_100__reduced_price_school_meals_eligible,f48e0f5e3e1be80d313f5e14147df9a8518ebbade9afefaccccf8a085743b7e2,52a1034f6edfb84b8ec945fbaf13dfbbca8de2b289716d76054ae88a909c9364 +us__scenario_100__snap,01016cf16bc7e3b1d8e6b12a0c04b9de8f55822f86c54be24dec24c209ed676a,7ce99fb4948a9c14cce7e3aac16f38f215049336a7ed1959f4d9742b2454045a +us__scenario_100__ssi,09ed7e493c8ac49b0075113658c5d540283544489f4c4a4cc90936015b747c2e,ab0f911ffe235e01b310584be8ba7b9d9e5d3d5d5a63bce6328ec0f604057f76 +us__scenario_100__state_refundable_credits,c4f9e34bee0210e50311859b615bcbf1f23b14cd0a0189033756bdc7198ad4b5,97dc0c0930067a602e1243249df3d13062e54b642406fb48e069482e87f46811 +us__scenario_100__tanf,c2d180482c531aa1ce661fa29f87d2b76ff6dbe2b7b9e416b3e10bef7088e49d,40a7f703fb1ee9d596dd3e8dd9c747b1b8c2e54c6712d228a6924c66caed9847 +us__scenario_101__federal_income_tax_before_refundable_credits,2ade35296a93669322eac38ade7d98c3f4accdf7a68d2d6befb1fac8c4d4160d,02cdc31c2e97504e5d2cb812be0142f385fccb2232652ce3ce71384d529293a6 +us__scenario_101__payroll_tax,a2f08013c87417df310e5bba5659fdf8e70c3c889de68f52f01823ef05d01e09,b762fe92671cfd200d27ebe71cfa91c9f15aa388f38047b4886a727cd01681e8 +us__scenario_101__state_income_tax_before_refundable_credits,4a51ce741f9c4f5e11debc391104c9bdd78ba23a379912d224cfdd8232835533,2c03e3495e68b19fb648a5c21134b23f0f35401773765bad122d24d7f9d3c2a6 +us__scenario_102__federal_income_tax_before_refundable_credits,6580951ded1e5b0a9bcdfebe9f7ab678960c76423ee0ac62c5ba6968dc426224,c80561779b0558c8b7fa005879d05066f3310aae764f62dc5969f83097624ca5 +us__scenario_102__federal_refundable_credits,1e601d03532140264686b284f460d2116091b602a6b80a4ec30d7f3799324997,ae74d7f50b0d6de89a090d1dfdeff35fc4953aa940e1e26481f55c4698c71f54 +us__scenario_102__head_medicaid_eligible,d5d3ef6446b0a066d66bc5785940e16828da8116caa2149a1d1927dad6a94729,751f23df5475cd27a58eb4a61a8afa5c9cbd25d9c10b6dd5145119c1a4c11c51 +us__scenario_102__head_medicare_eligible,9c8252197127659c9777920538fb95f25310821a95711e0cddac4fbdc0ae782e,3260f0c19ea1a55544d355cd60c7166364cb6c61477857c3dd2f1fefefd1ed0a +us__scenario_102__payroll_tax,a523f7e9decd50e805727522f8afed3b0e75ba96ac5be21f2f52f8ca76b958a3,14a08ba286f5452be98361565152a26d67156a02f16ec470f0f292178b49f739 +us__scenario_102__self_employment_tax,8657d250c36b165d52ef2648ecf991d3aa4517cae167f3a8f673b5cbc1b6e573,c20e61a529c60cfd66a8ed543ec7b930133dc678aa78d283cc6d43b1eb7adccd +us__scenario_102__spouse_medicare_eligible,8466fd8d30fc19df5ec6f5f81c4688dc1b4ceaaf48ea48ddb99d2724ef95c23b,bacbcf969d7063d6dd766e66a24a145c9e2afcb766770fe24d38d21f02a502e2 +us__scenario_102__state_income_tax_before_refundable_credits,8288a9db0e39c2f85200d2635109ee82c79b6e973a45b433929fce435bb2fa6a,44bf1aab7663fd23209b6366611eab79d2f52f2f0df756767c30119553912042 +us__scenario_104__head_medicaid_eligible,2be536aa9e33fdc74fb76b8cc8ecb299bc6c6c4b0ec941b37283379e4c02507d,df565d71c1643211ff3ff2db8de166d96e7816be78baf39c115fb8696c9bd35d +us__scenario_104__snap,ddd90e125c60e639aab3fb91979a33aac4d6de88fda09ffef7bf32d89e599952,918f3d22f7bf8d30f940c35441a4d60dae9940efea8118f52f0f5f7e7f83285e +us__scenario_104__state_refundable_credits,e1ccbfae16f82ecc76808d1bdc19607ec277f01a04140777f2853653d7624c9b,48a4b28bcb0b63f57af2e47109f0cda96823db8e281ebbcf46675e98d8a0f605 +us__scenario_107__federal_income_tax_before_refundable_credits,ae86382325d449aa954d9ac405e3b35ed09a4f64375a3dbb9b29fa82e4b702f7,f392af050b61548f936e52455de3aeefd5f32310a0f1478d8d6d6381b054ddef +us__scenario_107__federal_refundable_credits,8ce30c4292c1d550874c94f936ec81c91e7bda6501b4425baf686768ce12bc97,f4bebed066e53a96dc3758b7efd30b421663fb3a56019d93ca6b674097b6b0bb +us__scenario_107__head_medicaid_eligible,d64333207d87b423bad3bae0ef46495e35857778fa0a6c60a1636ec8f557f280,22e27326f52dcbf196f98ca50d3ea3febe7691655708a4c561ab7dc331e6510d +us__scenario_107__payroll_tax,492359a5328a9233be292b09edf7a0011778a0aa21744aa6abb5a771d4cd32ba,45a6e6efced759812f979e51794ee4447950fa7d3a99b59b2234081e1a202d8d +us__scenario_107__snap,a8c76325a7f15ddd29f9480f8ae9a9ed424bb6c0b9be982a75296f8a78654c73,c29da5cb840d5ef24525b4df767de20d35334c4bc5e7a8e0b1c0fe17a2ae5ad3 +us__scenario_107__state_income_tax_before_refundable_credits,3ca54642b83a48e0a9c810433c9d185b443b9d4a1ad29176d771dd79d55a52ae,e4ed77a3fbefcda2d7931eb41de980c5e87f9b61f40a9b0ef77da25bf775342e +us__scenario_108__federal_income_tax_before_refundable_credits,f055f23f31453e1f51c7f34882dd643ddb36ad83b714f8543067fd2c4c9d7b0a,99801a44d1aabbd23aa38a7d1b29d799f4260f88f37cad7feee5baeb7a62f45c +us__scenario_108__federal_refundable_credits,1c413f0dc2dc3bb0f8b33e042c2805e0966026e9c417ee210620d33856d14446,481c7d2928c83010123aee24ffd4f4135a74a91a3ccbced7ae43aa06a42e3d81 +us__scenario_108__head_medicaid_eligible,6a483668c681e02232875b8de5c6c95917e4364a3abf41e8ecf6e7bf3448530e,206ea10d52f62a23e88779c7dc4d54dcfc037270f3d47f06760b2c1b4b7bff6c +us__scenario_108__payroll_tax,02809809d38b37addbfd7ed5ce7ae0b06b3077448f4daae0cdbbbf5e99b00654,0de4a4b0f4a4eedeb8cfc470c02dd520a9eb33accf45c34f99ad5b22f669fb74 +us__scenario_108__snap,0d0150c16a3d7c669900d5095308d611cb11d98299427e92ca3249857c93a6fc,0a76cc9f1894dfc3f49249aed4ad5964e62a9224590ad90b1cf19cf8cd607483 +us__scenario_108__state_income_tax_before_refundable_credits,e9c0825df31e9d945d395d03ab9ebcf9afe01dfe6324c28d93a91bf7e5144071,55578f41ddae1c0e2d0efa06856197efe8c704de227a90f7d757aa77a64222f2 +us__scenario_108__state_refundable_credits,8113a3936c27ca53dd2a9662586ac4177243b87469fcec6ad948d86aefb8bde8,bf289fe7f42a0a8c562da2a581fd0b6e871f4277c54ea2e0c58711eadf25da5a +us__scenario_109__child1_chip_eligible,95b07ff668b9575e4309589318f19b58020cce8d40172ba5de626fa839c9caff,4179653b3eb573f877ae4cd6dac9362b5baee7f96f5901830f8a367ffdc062de +us__scenario_109__child1_medicaid_eligible,c806fea0f6a8b994fdf9205258f536acadec10c25daa12f6ca105c8e4034bcf6,4bc3938fd7c76188713048b6a01cf5ec34245afb71c38abf6bc332264fbcc4bf +us__scenario_109__child2_chip_eligible,7e7873d309632b91ea07dcdba15c95c84e30759c00fe7be071247b983c2f9e8d,8437126d76f27b635bb201c035b1ebf73a8df5352962e49c1237d49317570180 +us__scenario_109__child2_head_start_eligible,f46b048bee44d197f028cae7290c62e42ccf3e5e5e7468331607ec3c136d4119,ff4e4d4fac95ae515a1ded072a26a93678a4fee677e318184fa805fe031d4f66 +us__scenario_109__child2_medicaid_eligible,5f41323210a69526bcef644dc26eaa0f5d4d3cab8aa7dd7adc42d0cf28e34f55,0deb8c46388d3597980907374bd27e0b487b907c9945a868f31c08202fa75d8e +us__scenario_109__child2_wic_eligible,ff0267c6ef81551f3877368a43b5218cfa245e1194a95dff0e69ebf70d8ccb77,78f286838cd944b3e37925f586cc2f2f5f68861204f3e5dc8be6594c03a3bebd +us__scenario_109__child3_chip_eligible,dde97cf5286042bf287f0a20fb76fdead5508c0e9a84b29c104e9bf8ed5b1fdc,b30f8127d5631222cae718b98ada52886c0a400050b5ba1cf7e3bdf1a186010e +us__scenario_109__child3_early_head_start_eligible,2d40b21ecfddd39defe7add5b274ca5d541c7d4ba5600128b0631eabf2eff666,26176a8ad6de724f4d8c5c47e4911c55b6b4e8daca6cb6cbc98057751f501741 +us__scenario_109__child3_head_start_eligible,9bf66fdc97e155bf72a12f9b4de76f3e3ce2d76e39610e67bc6c10702f25e6da,3652c93d43605f903e1e3ae9388150d47a97f2672cd617fadc556865b92668ae +us__scenario_109__child3_medicaid_eligible,ca2be8657ff9de56b93cb7cb0a523e8fb5c22bea9e253d94661966a0a6bde1aa,f0923debc63db7e5edf9b40de1a35df8937ce0e99d3ecc27fb60c6ef3b8c8b51 +us__scenario_109__child3_wic_eligible,af474fc81edb2a4f723848479c0f3a499f588f0081e83740d215ba59a19cc3ad,9ccf3a7e8a096d6450d84dbc2fda4136ebdf18be87515b08b2f0f23e73a134ff +us__scenario_109__federal_income_tax_before_refundable_credits,4d8d528b04dbf78859d3da734e6c564f21c7ddfb57d6cc1cbf6c8c792771db53,be4af2f46c406bcb08c621ba20433a734d9ecdffb1a47b5a2973219c401cd9f5 +us__scenario_109__federal_refundable_credits,03620d981768afde52a1fc57d2b9b27fc864ae34026e5414b678eec8782bd6b1,c66a0a413ba04771dec8ca4612e7b87dca4b7dbc24748ec02fc58f340932c813 +us__scenario_109__free_school_meals_eligible,de534fa64d3370e65f6d7b5d407bdfaf4489cf1e71de949d28ef815da2073b76,b92f7666fda454f815d74ccfb926e9e5fac2e7ce754fe1ce9de630d979ddf3a3 +us__scenario_109__head_medicaid_eligible,e28e69e317185feff2783d9cdd3ee1a87cd2afd9d26cec04e2be619b20e454f9,4efd6c417a20ab25a4bfe8d33b538e37264cba3c8fee877377c66e04a410b8c2 +us__scenario_109__head_wic_eligible,2137dfc79fb6f44c4be90a4c29879317924f7d86111820852a78f38605790804,e9f69a0cc76f50db8b403dd46f905776815a156a2bbe7c53a50003ed22591cbe +us__scenario_109__reduced_price_school_meals_eligible,31bf934a56c29f7f00d7c7c42f59c897b5a175a61da73e7ae5fe75fc6ecc7bcf,6d9b2b7eaaf2e304e0ad51c254f0a181099931d157140a5d55596d84ce84fa40 +us__scenario_109__self_employment_tax,0133f3dee663890285aa94fd9005cd18c50c20487feb9ea9490901e54c34cd07,38c60acdc7d7daa6dde74f177afa29c26a7fd504d2bebe77f40a0527f20ea4e0 +us__scenario_109__snap,f900b822fc0e2c464d5e81df5c16fe166769f67ec94d0945a5eca83bb3f8d523,4f37a6168f6ddd64e13273e4d5b51887a583b3cc4b2e263ba23be3c489200871 +us__scenario_109__spouse_medicaid_eligible,f21941a8c827445dc9ffd1d9fec946c43e531df7ecb97925148b006aa396d9aa,f1e03b0784dd96fdb9eedac1f34427f2adde5f2082fd660972d62c88f59cee88 +us__scenario_109__spouse_wic_eligible,325d38a414903be1464707ece33fe114afa63a8e00c8b52f57c34034f5924398,c5b9e14bbc394fd2f65c3b4ddf1e3d8f3c0135764e4f9125c0fe5b8a07742499 +us__scenario_110__federal_income_tax_before_refundable_credits,f2476fd121814705cdfbd0f21067654431b92a5e69f0dc7193bcaf10fb90990d,2d0acb6d999128a685eb312c9b1ba2cb1c6977144841e85d910d71e48f077ea4 +us__scenario_110__payroll_tax,7fcb1d9c16aeba7fd3c624e8cbd0f6f84cba771040079c214da447d5fc392bbc,2d37c52041393b2a57b9e9cc57919066ead20391ede088db48481d4d949b309a +us__scenario_110__state_income_tax_before_refundable_credits,1b27ba47123113217da6e66268e3f61a8ba2a2f0d80fa28c8497262c7d901daf,fdac8632534960ae57af1b45e93e1c279d50f2b9b62ed59cfb884ebab404ceef +us__scenario_111__federal_income_tax_before_refundable_credits,43d0ef06a041e5007925f42ac571a195b241e9637d19d0b8558604819be914b7,492337b58b95c3ed7954c8ca550c3374f2bca1fbdcc4e373d055964e51f0c9a8 +us__scenario_111__head_medicaid_eligible,796e50469600b975ca6c882bdc31e97b7d134a5910a5e610296f35e08b8f3fa7,c85f7a15f9b4bdafe60197c9d4b89d568071bdbf0b7f2d8e0fca83d95c1fcfe4 +us__scenario_111__payroll_tax,3beb0ace746fb3a63ee03e1205ad309dedd3bbd43062ea75a433d7f111dfb361,513c770f6a6cc764c39ac5c53c079f27692b9fd3f138e0bf9692c319d16e5751 +us__scenario_111__snap,b64d6497d900bdfaa8402f2f8666d20127f340800b74e293d0e51f83d6d38fb0,95a092d05068cee4aadef05c2d697174c94c9384bde66fb42058ea747074461c +us__scenario_111__spouse_medicaid_eligible,85746ff35f782bcdd6e8caa011c15afbd7a7e521b456787d321f5b0df4be7616,d7993641cb2c7b15c57440bb948e5d76090993f247d2084297f121acb3c40ebb +us__scenario_111__spouse_medicare_eligible,a9af5931b56c6a189ce1907b11d91aecd6745e5b2fd46709881234be52b329d4,cbc919a52588cd1e68c08d7910da7abafadb982400d82e2a92c9a28f1e5eab37 +us__scenario_112__federal_income_tax_before_refundable_credits,4e724060206eff175e4504eb2caa1b272013bf1c7fca80397d6217b9f3288157,0ecc7c6bee1489e81a8eae1183d2d9b50b06f4b4da652a5873489d38a9639c95 +us__scenario_112__federal_refundable_credits,6fc1d4fb578c17ab817542e899a18492b0a02518ce31e1e522007e80eb37108d,df1c15a914ab07ed70b700910b46bfbc8dc19fd978671da23d43ec10d3ed02e0 +us__scenario_112__head_medicaid_eligible,239be8c8c005b939b31e41b63d3d9c094c6a301e60a93092e3ac7af65a2ff275,cf005069f379d417e3274be065641b60c14779d6e1817a078b5769f98c136096 +us__scenario_112__head_wic_eligible,c3991638780039d33fe9f6e3c34d50da701f0c919f0f67998e62b5b53d596411,a67037d1bf6467d0560a3fa3d2cf93a456e9ab93be0965f511a08e78413a4989 +us__scenario_112__payroll_tax,60d96684620fee48910f39733de20767413c14547b1142fb9f6c559244bc7172,62ff4ac380676683da71168ce441119a931c332f23a94ad2fe38deec48b691ef +us__scenario_112__self_employment_tax,92f4fde0e650ff25da9e097ad237c91b7c893d29962e071cabe9ebcab71ff218,2fffa4ac34206dd5c70328cefe0d4856b80f86cf7cd71b7f441abfe294c8f3bd +us__scenario_112__snap,7b2426aef957515c82a39e7c7a6834badb9c3b1d312719c5cdc0b56576010dd3,df3b27ef8a1b854dbae0c5f8e2132e226640c8a7a8aabf6b2ac719c64d2b2019 +us__scenario_114__federal_income_tax_before_refundable_credits,f98d755ccb7393880d2d4beb211eae3e31ac8664566b4eefe9ce90256bafb98d,f3f1b2562ad3026b07f05a558df7364954c1c2c01f14890bf38cd9fbfda34cd6 +us__scenario_114__federal_refundable_credits,4893ee696228cb54f450e9dc6c27be2f269016891f0fb00d235926b36107118d,67a85e18b6df494d0bea9ea6322784dde7da0013dd82731b1ec49d2b48906f66 +us__scenario_114__payroll_tax,728ce80c383ebea347b59f6b25ac22a735cea327de9586aae7a1f47047870fb6,61c5b2d82967d3af7df146d7ea4412e0007c2937109a90087177aea3b7f9f9d4 +us__scenario_114__self_employment_tax,976e0494715515829e2ad02f305cc25016077051b55baec3eee1377956c3e8a4,b28a5d2fb6758654353e8be605f64d0a843182a0a08e65b3595e4fe63be5e5c1 +us__scenario_114__state_income_tax_before_refundable_credits,4ca2b638b013598dba97ef8bb40320a3e9465a60b029e7d47f23c2da1ff70f53,59cf75d07bdc5f09f4c08ddb648f22421d131c0d9dfa29f5fc3e421f8768791e +us__scenario_115__head_medicaid_eligible,ecce7759f91e86e416e3790ebd94fd8398dd43380b496474d0517ff5ac21990d,02f548c330509d4f280a34d9e2f7b2b18dfc6b767dce5a51e0e17865eebdfbce +us__scenario_115__snap,634c417adf074bdd41cf1d4360f098e38645492ca5e486e05ffd1764bf24f9ec,3ede57fcbd96772eda588d55904edef6c1fcd3fd4c276c5e86df23367026ceb2 +us__scenario_115__state_income_tax_before_refundable_credits,2e824d2a44dedbe009e151ddf4f1a60197a1607b343bf957573f2cb65a320e5f,a0a1a09cf60749ea9f15fe63d23c48935c9753ddd9bf6e642f7cbf670aa5e3cf +us__scenario_116__snap,d875ad46611cd5214dfaa9c96d4a3a8821334ab2c97a93394607f17b279f7d8f,6971485bb9b884f5db2b4c85b8031d6f02b12efab642ee87a46ddd72a24b436b +us__scenario_116__spouse_medicaid_eligible,14ede8f372d1d4ccc74d4a617692f663b62de35a102b7767aaf6cf4a10a8c7d2,53e47fbabb0aaaf5ad4aad7042e5a579fa726331c69993a8275635d5e6ca3f2e +us__scenario_116__spouse_medicare_eligible,27d1a59bc663a6554964dcd29381234b41d06cd9126096ab39c321387179a7bb,f3eeb61c01244bd97c131c7e51f5433c8d04751db4d8c89728f88f415892c739 +us__scenario_117__child3_early_head_start_eligible,0926d12e2f0241120078e26167ad9cf017333dcb4b8955a2d5a09bf892c007b2,fb8a3a1900d19278385a97eb392b00206f4d61e82f090c754abb697565cb20a1 +us__scenario_117__child3_head_start_eligible,44703423e85c5569457e69aa31ef80b2659065d6f27fe639e083fbebf58fe5ab,8c0858c6f18e6a21d8b2e443a088a19d25ab0b0c09c0aa10aa25a40a0271335f +us__scenario_117__child3_wic_eligible,31ff794afa1cb3ddb82ead5b6dece0a9b998b02ff644f1214d827f7b6a90d20b,4c82af05c92393e1707d1699f64d3d02619c4a506d76d10ba2ab4e9d4f4165f1 +us__scenario_117__federal_income_tax_before_refundable_credits,6123f6df5fd412461878c98a36401c51bf811da1fd58ca7e1f0ae15ce913a102,6fe8a0fcffaba12526ecae2c0f791fac636675aa1e1ff426f5864aab90797642 +us__scenario_117__federal_refundable_credits,b0f1d7553d95d493cb21d918e67ffda978fea56f3db7a5f7c6a48ea50230606b,e7d2a782ec4c5074b9650ede908b9a02eb5cb838fddf78745e839b539a6b312c +us__scenario_117__head_wic_eligible,d65f67cf6634c591eee1aa2dca4f3abb6eab181efbfa9683cd69121f74d78ba3,999b453c4ea5381c95feea026b861503c86c28346fea8abcefb8e3f65a6bc3cd +us__scenario_117__payroll_tax,5723db3827329d4163c1cd858383de1c8f4dc2b7338a45037a2a28ef07b22551,d861a931661c0a076d8ebcb5f3b478231c9e42f94123c61f4135c8dcd41c1bc1 +us__scenario_117__spouse_wic_eligible,3de7fecf4f8871a549d78944fc9fa629faad351400955f029091211d2cef52a5,d3518137770e1e43fafb36317e5a4dd1224f90d663bc6e7f03944820d34a09f9 +us__scenario_117__state_income_tax_before_refundable_credits,f2a8b35b0763263cc63247d16622a70b2c5bf6ea46696f5d8dc091ab6ec63915,0d89ac5445eb49c89f75197d3dbd76f8035bc2e351eff44764a490663efb0ad0 +us__scenario_118__head_medicaid_eligible,c73817e22dd1584f2bbe2a1f09bca6927748893b5f37a7bede5423cd1acf6f0d,bfb90fee56f8afa66629351f04798ed6818d754c832047dd445170445285a949 +us__scenario_118__payroll_tax,ac99d20fe15a935dd5b988b6dcac640b33c3845851e44786a260727e14f52c85,d6aaec3987c5199e4297fc2d723d03dd240caba24a4f9581a1fbe2c170d6f0fd +us__scenario_118__snap,80cacf91274241d04ba829366a619f8950c4748b5ea0e72aab145a4c5f5c7559,da8e108a324a75da56462cc5359bab742464d566c038a552458d47d949109c8b +us__scenario_118__ssi,519f876cbcaf7c9f8c420b205d6b9688cc92e190e7d2667b4de0113cf33960ca,b9ba12382a09c64ed7b38e1b7847dafd98cade0601e969a273b9c4a974244689 +us__scenario_118__state_refundable_credits,255a4d0aacb8f2a9f47c81cda2f3a1512dd03912046179cf20f43403b94447e5,b7f923c1a512c7b94a2273e21d65d636f59069fb5a4aaf54e4fa0ffd54587487 +us__scenario_119__child1_chip_eligible,380eef556ace7ef7f6bc5d713e5843dd735ad306514c576ff1057355704ad129,17ecdeb200e5875fce56e5e12f275c3330e9e62b756e4953dc2ed5069928886f +us__scenario_119__child1_medicaid_eligible,f668ff1e7ae90feca959db659a3ae7454694f0086f0cf96c21aa48fbcd3c37e8,d513b32caf26de5ce33d49f27064fd8be3795cf85bab12eb5a7c2ed57c7d4ecf +us__scenario_119__child2_chip_eligible,a9b1e54d772c0f72bd23e442ebcb3e4b2c22424dd8e4b30ffdef6359a3ef7d26,d3145c69dc8724c0cbfca3788c092cc91871ff062710a5159c697e1e49979ab8 +us__scenario_119__child2_medicaid_eligible,7cb313e5e8fb5d9873baafe18f7267f69f0fa26462027805052b82945c94b108,1186a23be3c19588241bfe8570c02e655f77fcb020fb966268ed6272dacfb313 +us__scenario_119__federal_income_tax_before_refundable_credits,68f748ccbc0a2fc3a151aa72a49c5df3513a500d7daab36adb6fc6f2384abf06,483b63d5796504255a16be834b8d108b39917f05126887cdeb0a6d581eb8e86a +us__scenario_119__federal_refundable_credits,92eb0357c1853eaa2eda73b562a3678e5daea6d386f51c5b9d4dbb143f35c866,a4c92a0da6ff70ba83547bf62db038219b3939471d447938ea84079c7e58143a +us__scenario_119__payroll_tax,cb769fe3086fc959370b9a216a9eae64b73dca1b6cd01b03cd2e3ce043fddab0,7b305387fa263aa1b4a3d182fa86088c6f70a422780c05ceefab428bcae8bc2f +us__scenario_119__reduced_price_school_meals_eligible,71c358e735d0e5b84b1fce88b1b2c6e2fd2a4a7aae846d4ffd6292a0bda5f688,5ef0c626bbd708c1ff5dae3ac9d239f8441037088dc638f482ce246c6ff6e556 +us__scenario_119__state_income_tax_before_refundable_credits,c48e6209867d555f94796191785c4ba858a759ba6682bb318e1ba6de770b7b83,1128d26b6d3f4197b0ddf1f94e77f8f284fb313494b2e9fdd8ebea6a949453b3 +us__scenario_119__state_refundable_credits,22e99c624a5f6c6599bd81f3d2d0d67a9473749458d993376345f8c28ba965e7,868eda6e2844aebab57fb1846c43df17264851f5b97c9d3b4aaf7075ce13015a +us__scenario_120__federal_income_tax_before_refundable_credits,012d13595fe64ba4cc4cdac71bb0f0fdb2051f8748a168c00565a136e64902ec,918306009e82bcc7ae5fd17b491f8a6c64aa8530cbe7af9c2670514f3e8225cd +us__scenario_120__head_medicare_eligible,38423457f0d2f776538ce275d1e8d4614066e17bd7c163adcaf72765980cc522,eeb633fc78773bcd0efed8dd71893b20fa0213a5605078c0a2c01eaa9bb128c8 +us__scenario_120__payroll_tax,af64c38301ebd57c7ada7c7d4b70208c0e83e1d6189950e7a4dcf507964987d0,6538f4744110d18408406d4048a7825188f3f5f7915bb5a4ea0594b55943b133 +us__scenario_120__state_income_tax_before_refundable_credits,d64e69903ede752777ef8919518466d85e9ee12ef2a1a2da6a1163e58ed593e9,40cf423750666c61604f52b14f3ee8bc5ab0dd09b6afbf207b0d9c18b215929e +us__scenario_121__federal_income_tax_before_refundable_credits,61bfca59d67cc2550a0886929d585cd61b021edbf52e10b02a116c6eab77cf70,ec63cc2614dabc2731ae4cd4c2aaf51ff02a2bdfe46e944b673d4d3db7425b36 +us__scenario_121__federal_refundable_credits,21245c794e7489d82d006c47eb91924e7adf2641c548f97a077a37fd96152dfd,172a7cc3c7dc2766ed07f83ffd2fd74afa82fa4903585a1973982dc6fef8e94c +us__scenario_121__head_medicaid_eligible,f1d2f12bbacc579e157d86e1c3ba16e03e842336293472f8d256c129adb3b4b9,72fbf4afa7aaf9bc3d5898710bac6ea7e3d7a5effb3fa7270abecb173b859f31 +us__scenario_121__head_medicare_eligible,4ad89400a3b2df1e8aef7ca57ba6a4f9cd615197682d876927b147872aedd4bf,435f32119d5e958a39d1a4fdd24825de98d3f749cc3aecc6b89815ab452c3d59 +us__scenario_121__payroll_tax,640422209d50a59709a5d256d27a53027530bcc184a59d09704b2abe58ad7d09,66b09cf08016c1dc3370ec43fede9a05b96ebc5e009ae2756e067fcf1aa6e9db +us__scenario_121__snap,f111e1a6556abc797106c9b94f0803cf5ddc9b86506f50d6ad27abda84a2d0de,c258f442e8185f21a2a664e06e5e7869bcb7a1620fa39ae4633bba3a03cb03b5 +us__scenario_121__state_income_tax_before_refundable_credits,ce60fe57a06785eb16d777ea41b1cd3c9a76e1982937a20d627a5a68c819c61f,a8b657c3e9398e752ec8ab3cf4d875ba203398c56ba07803b1a6b0ffc7533375 +us__scenario_122__federal_income_tax_before_refundable_credits,1651e5d2391184bd11a1c8daa6b35992e02f055052b9585172fac9142b3c3be1,8dbaf3a0688ebc840977f814d334aa59e4ddc6a4469af9be8e002f3fc321ea85 +us__scenario_122__head_medicaid_eligible,9cb1915eea284a648754090f09defe9de9174b415c189f7fbe36e396b2cc153f,f33e1094f352c7a871ac9bddd720e1a9cf13c42735d4cbcc766bb5775599be76 +us__scenario_122__payroll_tax,6194402baac22b4f36529d9b2ff9da2c38282f5954eec05586484ffb93af6ba9,8b7cfde758ff2f97f0bc27f7eda3f2326f9a04ff7ca3d277ef9ff936dbed67b3 +us__scenario_122__state_income_tax_before_refundable_credits,d2308ad10853c665e416837d89378c501e11d8cb290f5b700f55b595cb19582e,e4876cca5459dcbfd3b41a89d7f035579e4ec7b4d82eda5bb0348845112cf4e0 +us__scenario_122__state_refundable_credits,b032c5c37245f6c6c08e03b129af3c18af4f5095260eb2315c74fae180986175,d04072605a758bf7afdef0d7bd603251a3594b41f84b4d7c4d34118c1787cf0b +us__scenario_123__child1_chip_eligible,396ea9294f25df4757accd3fa501d5116ab243cfd339fa977f462450b459067d,655d730efd8a49a87d1448fd3a55ff88b9872ab3bd5b72cf5450350f7bc8ad23 +us__scenario_123__federal_income_tax_before_refundable_credits,2d1f922cc7bf87e76432cd80337e1344076b206e42435d2c5a12b7e06a6fbd76,0a87767581bd3f0f1355051b8222338cbca68b62a4171df9381e1a62cb3f0ebb +us__scenario_123__federal_refundable_credits,1b4e0478dd57381352d13443a3bb844989a26c0843514ebb45d2a142d0653232,a9abcef52c7be874497616bf1ce520817ca82b28aadba529c84c2bffd47fa435 +us__scenario_123__head_medicaid_eligible,bcff1a25f1f72e5213b3c6611ef16ca31b2f8bf6f89a2fa9376e171df6866671,746a18bf8fd8d6650ea14f83587282a4119c5a7a4e053619f18a86b11b82f84c +us__scenario_123__local_income_tax,73d046ea1cd007f5d7371adfc9df36cc8ef4abb512879b2d82ec9d6885d75bc3,a5aa6a1188d8fda5e8d7872e1724e0403e7114749df22a94769458369a2b5e6d +us__scenario_123__payroll_tax,bdfa2aab60f42aaa803c2a2bf0deb2a4afab3f5edab46391e592ca7908d8c298,5cb7217c4b2d00506c5af57448942416f137f84d588a7401303702815464328b +us__scenario_123__spouse_medicaid_eligible,0ea7b3c8cc4c093da15503cb7cfaf83494a83b12184a598bd2138df85b630449,40d4250a45815d85b6550450c375aa9390128d15b3c74eddad4687365e5ab3cf +us__scenario_123__state_income_tax_before_refundable_credits,5e8b8a612737185ee515dd1fc60f75531effd3a0f63b2b298ee2c865e7ddcc9f,97d4f13045ec4d3349b46475397beea403ec3a464b27e67d651dd4a732282bfd diff --git a/docs/paper.md b/docs/paper.md index f9c3404f..ddb58cc6 100644 --- a/docs/paper.md +++ b/docs/paper.md @@ -23,6 +23,10 @@ It builds against: - Web: [`app/public/paper/web/`](https://policybench.org/paper/web/) - Both rendered artefacts are sha256-pinned in `paper/snapshot/20260501/manifest.json` under `rendered_paper_artifacts`. +- The /paper page takes its snapshot date, response window and manuscript + `?v=` keys from `app/src/paperSnapshot.json`, which + `scripts/freeze_snapshot.py` writes from the manifest (including with + `--rendered-only`). ## What to cite @@ -41,25 +45,38 @@ The manifest at `paper/snapshot/20260501/manifest.json` lists: source-run dashboard exports, snapshot CSV hashes, and per-run compact artefacts (`runs//`) including `predictions.csv.gz` with raw provider responses wherever the transport - exposed them (blank for Claude Fable 5's batch-served rows and 64 Kimi K3 - parse failures) + exposed them (blank for Claude Fable 5's batch-served rows, 78 GLM-5.3 rows + and 64 Kimi K3 parse failures; the paper's reproducibility note computes the + tally from the frozen predictions) - `published_dashboard_artifact` and `live_dashboard_artifact`, with the pinned release URL, byte count, and sha256 for each dashboard payload - `rendered_paper_artifacts`, with the rendered PDF and web bundle hashes -- `reference_output_refresh`, with the PolicyEngine and PolicyEngine US - versions plus the certified US populace dataset's build id, URI, and sha256 +- `reference_output_refresh`, with the policyengine-us version that computes + each scored reference (2.15.17 in this snapshot) and the policyengine.py + version it records for provenance. Its dataset fields name the reference + runtime's default dataset (build populace-us-2024-spm-20260915, from the + policyengine.py 6.1.2 bundle), which reference computation does not read: + PolicyBench computes each scored reference with `policyengine_us.Simulation` + from the household's own listed inputs, the nine publication conventions, + the Maryland output-scope adapter and the scenario builder's stated-hours + alias, as the reference sidecar's `engine_upgrade` revision pins them + (`fix_modules`, `builder`) +- `household_dataset`, with the build the households were drawn from + (populace-us-2024-5da5a95-20260611), its URI and sha256, as the run's + `scenarios.csv.meta.json` records them - `population_weight_artifact`, with the committed scoring-weight path and sha256 - `audit_annotation_artifacts`, with the row and case audit file hashes. The - frozen annotations cover 8,783 scored rows whose legacy threshold score is - below 1: 8,780 of 8,780 exact-match misses and three exact hits. Another 1,605 - scored rows have bounded score below 100 but were outside that selection and are not - annotated. A judge_provenance block tallies which judge model produced each - case verdict. + frozen annotations cover 7,860 scored rows whose legacy threshold score is + below 1: 7,856 of 7,856 exact-match misses and 4 exact hits. Another 2,107 + scored rows have bounded score below 100 but were outside that selection and + are not annotated. A judge_provenance block tallies which judge model produced + each case verdict. - `reference_exclusions`, with the outputs removed from scoring for every model - because their reference depends on an input the data never carried (eleven in - this snapshot; 1,973 of 1,984 outputs per model are scored), the file's hash, - and the tally by unlisted input. + because their reference rests on a reference-engine defect not fixed upstream + or depends on an input the prompt never states (56 in this snapshot; 1,928 of + 1,984 outputs per model are scored), the file's hash, and the tallies by + unlisted input and by engine-defect root cause. - `reproducibility_notes`, covering model-alias instability and what is not retained locally (LiteLLM cache, since it is a generated request cache) diff --git a/docs/prompt_contract_v2.md b/docs/prompt_contract_v2.md new file mode 100644 index 00000000..596d36ef --- /dev/null +++ b/docs/prompt_contract_v2.md @@ -0,0 +1,462 @@ +# US household prompt contract v2 + +`policybench.prompt_contract_v2` adds an **opt-in, unactivated household-fact +contract** for the prompt slice of [issue 165](https://github.com/PolicyEngine/policybench/issues/165), +including the October 5 audit decisions **d963, d972, and d974**. +It renders existing `Scenario` objects without importing an engine, discovering +inputs, calculating outcomes, or calling a model. No existing evaluator imports +this module. Proposed v2 output wording lives in this v2-only module; +`benchmark_specs.json` remains byte-for-byte unchanged. The published v1 +wording, spec hash, schemas, references, board, and cost path remain unchanged. + +This slice does not complete issue 165 or create a v2 benchmark board. In +particular, proposed definitions are not a requested-output list. It supplies +no answer schema, marital-unit +mapping, data provenance certification, or reference validation. Its text and +diagnostics are for contract review. Do not feed this partial contract into the +v1 evaluator or treat its output as a certified evaluation request. + +## Identity and example + +The version is `2.1.0`. `contract_identity()` returns: + +```text +policybench-us-household-prompt/2.1.0:sha256:57a244e80b09866b07e6c7b01fc1530610eb6f326b091f8b00c3e8fb7b25b2df +``` + +The SHA-256 covers the **exact UTF-8 source bytes of the module**, a newline, +and the canonical JSON of the proposed output definitions returned by +`v2_output_definitions()`. Those definitions are literal constants in the +v2 module, scoped to version 2.1.0; the renderer does not load the published +`benchmark_specs.json`. A source installation is required; bytecode-only +distributions are unsupported. +This identity does not cover a model runtime, dataset, requested outputs, +or the caller's per-person facts. There are no mutable v1 helper or +runtime-registry dependencies. Record the rendered text separately for each +household. The pinned +[`scenario_074.txt`](../tests/fixtures/prompt_contract_v2/scenario_074.txt) checks +both the source identity and the full rendering. Changes require an explicit +review of the identity and golden text; do not automatically bless a new golden. + +For an existing US `scenario`: + +```python +from policybench.prompt_contract_v2 import render_household_contract + +contract = render_household_contract( + scenario, + policyengine_us_version="2.15.17", # Explicit review context, not a runtime call. +) +print(contract.text) +print(contract.unknown_facts) +print(contract.unsupported_inputs) +``` + +The caller must supply an exact `x.y.z` model version. The renderer labels this +version as unverified; it never reads an installed package version or asserts +that the declared conventions match that version. `source_dataset` is likewise +a label, not evidence of observation or imputation. `metadata` is not interpreted +as provenance, dates, or engine unit mappings. + +## Supported values and unknown provenance + +All input lines retain the exact input name in brackets. Facts sort by name +within each person/entity; people retain the scenario's order. Values do not +round to dollars or whole hours. Built-in Python types corresponding to JSON +scalars are supported; NumPy scalars, enums, arrays, period dictionaries, NaN, +and infinities are not silently converted. + +| Inputs | Contract interpretation | +| --- | --- | +| `is_disabled` | General survey characteristic; establishes no program-specific disability gate. | +| `is_blind` | Blindness indicator; establishes no separate program-specific disability finding. | +| `meets_ssi_disability_criteria` | SSI disability criterion before the substantial-gainful-activity test. The earnings test remains separate. | +| `is_usda_disabled` | Supplied SNAP receipt-based disability status; this renderer does not calculate it from receipts. | +| `is_permanently_and_totally_disabled` | Supplied IRC 152/22 disability fact. | +| `is_incapable_of_self_care` | Supplied IRC 21 self-care fact. | +| `social_security_disability` | Supplied SSDI income, separate from entitlement history. | +| `months_receiving_social_security_disability` | Nonnegative integer months, measured as of January 1 of the scenario year under the declared convention. `24` prints as `24 months`; `24.0`, `"24"`, and `True` raise `ContractInputError`. | + +The six disability booleans accept only `True`, `False`, or `None`. All eight +fields above print for every person, including when absent. Omitted or null +values are **unknown**, overriding the generic unlisted-input default. The +renderer never uses general disability or SSDI income to fill another field. + +Optional provenance uses exact person and field names: + +```python +from policybench.prompt_contract_v2 import FactProvenance + +# Only use this structure when the named source actually supports the value. +provenance = { + "head": { + "meets_ssi_disability_criteria": FactProvenance( + "imputed", "the actual build and field-level imputation record" + ), + }, +} +``` + +`observed` and `imputed` require a non-empty, single-line source and a non-null +supplied value. The single-line boundary also rejects Unicode line separators, +including NEXT LINE (`U+0085`), in sources, scenario IDs, and dataset labels. +`unknown` may name a source documenting missingness. Missing +annotations default to `unknown`, never `observed`. A supplied `False` without +provenance prints `no (supplied value; provenance: unknown; not an observed +fact)`; absent values print `unknown (not supplied; provenance: unknown)`. +Unknown persons, misspelled fields, unsupported provenance kinds, and annotations +for unsupported fields raise an error. The renderer displays sources without +fetching or validating them. Synthetic annotations in tests demonstrate syntax +only; they are not claims about the frozen dataset. + +`unknown_facts` lists sorted paths with an unknown value **or** unknown +provenance, for example `person.head.meets_ssi_disability_criteria`. +`unsupported_inputs` lists sorted paths whose meaning/units are unsupported. +Neither tuple is a readiness certificate, even if empty. + +Other supported person facts are: + +- Required `age` (nonnegative integer years) and `employment_income` (finite + annual wages). Repeating either in `Person.inputs` raises an error. +- Finite monetary inputs: `employer_sponsored_insurance_premiums`, + `pre_tax_health_insurance_premiums`, + `health_insurance_premiums_without_medicare_part_b`, + `health_insurance_premiums`, `other_health_insurance_premiums`, + `medicare_part_b_premium`, + `medical_expense_health_insurance_premiums`, `financial_assistance`, + `social_security_retirement`, `social_security_dependents`, + `social_security_survivors`, `veterans_benefits`, `ssi_reported`, + `disability_benefits`, `self_employment_income`, `bank_account_assets`, + `stock_assets`, `pre_subsidy_rent`, `real_estate_taxes`, + `home_mortgage_interest`, and `tip_income`. Signed monetary inputs retain their + sign; premiums and financial assistance must be nonnegative. Every premium + label identifies the payer and tax treatment, as described below. Tips remain + included in stated wages. +- Boolean `is_tax_unit_head`, `is_tax_unit_spouse`, + `is_unmarried_partner_of_household_head`, `has_esi`, + `takes_up_medicare_if_eligible`, `is_surviving_spouse`, + `dependent_child_lives_in_home`, and + `state_paid_leave_employee_share_withheld`. These are supplied facts; + no couple or engine unit mapping is inferred. +- Single-line text `financial_assistance_source` and integer + `spouse_death_year`, which cannot be later than the scenario year. They make + the deciding facts explicit rather than treating a loaded label as a legal + conclusion. +- Tax-unit nonnegative annual dollars `state_withheld_income_tax`, + `state_sales_tax`, and `local_sales_tax`. The names of the engine inputs + are retained; the renderer does not substitute state tax liability. +- Nonnegative finite `hourly_wage` in dollars/hour, and the weekly-hours fields + below. Optional supported inputs may be null, which means unknown. + +Known fields on the wrong entity raise an error. Other scalar inputs print as +exact JSON with `unsupported input; meaning and units not interpreted`, including +strings and zeros. Nothing is filtered using v1's excluded-field list. For +example, an unrecognized integer status prints its integer value without a +guessed currency, enum label, or boolean interpretation. Non-scalar or non-finite +unsupported values raise an error. Reviewers must resolve these marked inputs +before any future evaluation. + +Only US state/DC codes, integer calendar years, and the existing lowercase +`single`, `joint`, and `head_of_household` filing statuses are accepted. Person +names must be unique lowercase identifiers. UK scenarios, unknown filing status, +and malformed structural inputs are rejected. + +## Frozen assumptions + +The preamble follows the proposed language in issue 165 as a **declared v2 +convention**, not a verified description of the published reference engine. + +- Filing and full take-up apply to eligible requested benefits and eligible + TANF/MOE noncash benefits used for SNAP categorical eligibility. Computed + receipt may feed downstream calculations; the exception does not establish + eligibility or historical entitlement. Social Security, SSDI, and veterans + payments/histories remain supplied inputs only. +- The existing `scenarios.DEFAULT_TAKEUP_INPUTS` vocabulary is accepted on its + specified entities only when exactly `True`, apart from the explicit Medicare + exception below. False, null, numeric, and string overrides of the remaining + take-up inputs conflict with the preamble and raise an error. Additional + take-up names are unsupported and remain explicitly marked. +- `hours_worked_last_week`, `weekly_hours_worked`, and + `weekly_hours_worked_before_lsr` accept finite numbers from + 0 to 168 hours/week. Each supplied name is printed without aliasing. If more + than one is supplied, their values must agree; null and a numeric value + conflict. When none is supplied, each person's text states the **0 hours/week + contract assumption**. Null is unknown, not zero. Hours never come from wages divided + by an hourly rate. The convention explicitly names the swept engine reader + `weekly_hours_worked_before_lsr`. A supplied `hours_worked_last_week` gets an + explicit same-value alias to that reader, following the existing scenario + adapter. Supplying `weekly_hours_worked` alone explicitly renders the + before-response reader `weekly_hours_worked_before_lsr` as unknown and adds + its path to `unknown_facts`. The supplied field includes behavioral-response + hours in the engine; the renderer cannot establish the base input from it. + This fixed-hours slice requires + agreement among supplied names and has no separate behavioral-response + schema. This renderer does not create or modify any engine input. +- Facts remain constant throughout the year, with no income volatility or + status changes. Medicare eligibility and SSDI duration use January 1. This + slice accepts no separate duration date/start-date schema; any such extra input + remains marked unsupported. The caller must resolve the reference date before + a future evaluation. +- Generic unlisted numerics, including integers, default to zero, and unlisted + booleans to false, with the explicit unknown, filing/take-up, and named + calculation-convention exceptions. + SSI means federal SSI only; a supplement needs a separately requested output. + +## October 5 calculation conventions + +These additions state reference conventions where the frozen households lack +observed facts. They do not certify that a household actually paid, enrolled, +withheld, or received the named amount. The renderer accepts supplied values +where supported and never runs a simulation to fill them. + +**State income tax paid for SALT (d963; [PR 191](https://github.com/PolicyEngine/policybench/pull/191)).** +For each supplied tax unit, state the annual input +`state_withheld_income_tax`, or follow the declared model's per-state AGI-based +withholding estimate and treat that estimate as paid during the year. This is +distinct from final state income tax liability. A supplied tax-unit amount +overrides the estimate. The audited policyengine-us 2.15.17 federal SALT reader +chooses the larger of `state_withheld_income_tax + local_income_tax` and +`state_sales_tax + local_sales_tax`, then adds real estate taxes and applies +the deduction cap. For year 2026 and declared policyengine-us version 2.15.17, +an absent `state_sales_tax` follows the reference's IRS optional +sales-tax-table convention: use the 2025 table data pinned by +[`r19_irs_sales_tax_2025.json`](../reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json), +with the reference's state, engine income-bracket, and tax-unit-size lookup +(size clipped to 1–6). The reference holds all 5,814 published 2025 table cells +unchanged for 2026 through +[`latest_c_irs_sales_tax_2025.py`](../reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py); +the raw engine's later-year uprating is not the reference convention. This is a +declared reference convention for the scenario year, not a claim that this is +a subsequently published table for that year. `local_sales_tax` defaults to +zero in CT, DC, IN, KY, MA, MD, ME, MI, NJ, and RI, and 20% of that state-table +amount elsewhere, unless a tax-unit amount is supplied. This follows the +audited [local-sales-tax reader](https://github.com/PolicyEngine/policyengine-us/blob/79be99f67132c4e5215b19bcf0108222fb67d989/policyengine_us/variables/gov/local/tax/sales/local_sales_tax.py) +as a declared proxy rather than an observed local rate. Thus +both computed sales-tax inputs are named exceptions to the generic zero rule. +For another scenario year or declared model version, an absent state sales-tax +amount is rendered as unknown; this slice has no verified sales-table +convention for that context. A local amount derived as 20% is unknown when its +state base is unknown. Supplied state/local amounts and the declared +zero-local-jurisdiction convention remain supported. +Montana's separate person-level `mt_withheld_income_tax` +reader requires its own stated input or convention before that reference can +be scored; the tax-unit convention does not cover it. + +**Medicare enrollment and premiums (d974; [PR 193](https://github.com/PolicyEngine/policybench/pull/193)).** +Every person's text states `takes_up_medicare_if_eligible`: enrollment includes +Part B and is assumed if eligible unless a supplied boolean overrides it. +A supplied null is unknown. Eligibility still depends on the separate age, +SSDI, and entitlement-duration facts; this convention does not supply an +unknown duration. The annual enrollee payment `medicare_part_b_premium` is +employee after-tax spending. When absent, it follows the declared model's +standard premium plus IRMAA, net of Medicare Savings Program support, and is +paid only while enrolled. For year 2026 with declared version 2.15.17, the +rendering additionally states the $202.90 monthly / $2,434.80 annual standard +premium before IRMAA and support. The rendering labels unlisted two-year-prior +IRMAA MAGI as $0 under a declared convention because no prior-year income is +supplied; this is not an observed amount. + +The audited +[gross Part B premium reader](https://github.com/PolicyEngine/policyengine-us/blob/79be99f67132c4e5215b19bcf0108222fb67d989/policyengine_us/variables/gov/hhs/medicare/eligibility/part_b/gross_medicare_part_b_premium.py) +uses tax-unit `medicare_irmaa_magi_two_years_prior`. +Its [lagged-MAGI formula](https://github.com/PolicyEngine/policyengine-us/blob/79be99f67132c4e5215b19bcf0108222fb67d989/policyengine_us/variables/gov/hhs/medicare/eligibility/medicare_irmaa_magi_two_years_prior.py) +adds adjusted gross income and tax-exempt interest from two years before the +benefit year, rather than current income. Missing a direct lagged-MAGI override +does not universally mean zero: prior-year facts can produce it. The +[October 5 Medicare audit](https://github.com/PolicyEngine/policybench/blob/65ca4af9d1395d61d06f9d106bf134b81c6b52dd/reference_audit/2026-10-05-medicare-part-b/README.md) +finds these frozen 2026 fixtures supply no 2024 income, which is why their +lagged input is zero. + +The medical-expense reader `medical_expense_health_insurance_premiums` uses +a nonzero direct `health_insurance_premiums` total instead of the component +sum. Otherwise it uses `health_insurance_premiums_without_medicare_part_b` +(zero when absent) plus the enrolled person's net Part B premium. Supplying +zero in `health_insurance_premiums` still invokes that component sum; it does +not suppress Medicare spending. A direct medical-expense input overrides the +aggregate. These rules avoid counting an all-in premium and its components +twice. + +**Who pays a premium.** `employer_sponsored_insurance_premiums` means +employer-paid premiums excluded from the stated wages, rather than a deduction +from those wages. `pre_tax_health_insurance_premiums` means employee pre-tax +payroll withholding and reduces income-tax and FICA wages under the declared +model. Direct out-of-pocket, non-Medicare, Medicare, and medical-expense +premium labels mean employee after-tax spending under this convention and +do not reduce FICA wages. `other_health_insurance_premiums` is likewise +employee after-tax spending; it is not added directly to the medical-expense +aggregate described above. + +**Employer withholding choice (d972; [PR 194](https://github.com/PolicyEngine/policybench/pull/194)).** +For each person in MN Paid Leave, CO FAMLI, MA PFML, NY PFL/DBL, DE Paid Leave, +ME PFML, VT child-care contribution, or WA PFML, the prompt states whether +the employer withholds the employee share. The declared convention is full +employee-share withholding unless +`state_paid_leave_employee_share_withheld` supplies another choice. `False` +means the employer pays that share and the payroll output excludes it; null +means the choice is unknown. Supplying this choice in a state outside the +listed convention raises an error. Amounts follow the declared model's +employee-share parameters, including its 52-weeks-per-year annualization of +NY DBL. This states a reference convention rather than claiming every model +parameter is the legal maximum; PR 194 records a Massachusetts cap ambiguity +and classifies WA PFML as mixed/unclear. Other programs classified as mandatory +in that audit retain the employee-side payroll scope. + +**Loaded household labels.** When `is_surviving_spouse` is true, state the +spouse's death year and whether a dependent child lives in the home. Missing +death year uses the synthetic convention of the previous calendar year; +for a joint filer or someone with a living tax-unit spouse, that date refers +to a prior spouse and the currently listed spouse remains alive. +Listed children are assumed dependent children living in the home. These are +declared assumptions, not observations of the frozen data. Apply the dated +filing rules; the label alone establishes no qualifying-surviving-spouse +status. Scenario 000 has no listed child, so its convention explicitly says +no dependent child lives in the home. + +`financial_assistance` names cash gifts from friends or relatives outside the +household when no explicit `financial_assistance_source` is supplied, and +states that this is SNAP unearned cash income. This makes scenario 030's +source and countability explicit; it does not invent an observed donor. + +## Proposed v2 output scope + +`v2_output_definitions()` returns proposed definitions scoped to this v2 +module and contract version. Their base wording comes from the output +definitions in `benchmark_specs.json`, with the explicit extensions below. +The renderer displays them as proposals, not an output request. The published +spec file remains byte-for-byte unchanged, preserving the v1 spec hash and +resume metadata as well as its wording. No v1 evaluator imports the v2 module. + +The v2 payroll definition covers all listed people's employee-side payroll +tax, including dependent wages, mandatory state contributions, and optional +employee shares that the employer chooses to withhold under the stated +convention. Employer-paid shares and employer taxes remain outside that +output. Thus the optional pass-through convention has an explicit counterpart +in the output definition. + +The federal income tax, federal refundable credits, state income tax, state +refundable credits, and local income tax definitions all state the same +household tax scope: do not add a dependent's separate income tax return; +retain dependent-related provisions on the specified tax-unit return. The +local definition separately retains explicitly applicable local wage or +earnings taxes on listed people. Each output retains its own jurisdiction and +credit boundary. In particular, +scenario 123's 16-year-old's $45,000 of wages count in payroll, without adding +a separate dependent federal/state return. The convention does not ignore +dependent inputs that affect the parent's return, or discard wage taxes that +the separate local-income-tax output reads directly from a dependent. + +## Required-facts report and release hook + +Every input a scored reference reads whose change by a plausible amount moves +that reference by **more than $1** must be stated or covered by a named, +explicit convention. A generic unlisted-zero rule does not cover an engine's +computed estimate. Unknown values, unknown required provenance, and raw +unsupported labels remain gaps even when their names appear in the text. + +The report reuses the comparison and exclusion logic from +[PR 196](https://github.com/PolicyEngine/policybench/pull/196), vendored as +`policybench.unlisted_input_sweep`. The helper +`policybench.prompt_contract_v2_required_facts` replays actual recorded +policyengine-us 2.15.17 baselines and perturbations across all **100 frozen +households and 1,984 outputs**. The evidence fixture names its source hashes, +model version, and run provenance. It is an offline replay, not a fresh engine +run or a discovery proof; absent readings are not fabricated as no-ops. The +recorded sweep predates PR 196's later Part B `not_enrolled` reading, which a +fresh release sweep must include. + +`tests/test_prompt_contract_v2_required_facts.py` runs that logic and prints +remaining unstated, individually moving inputs in +`test_legacy_required_fact_report_lists_gaps_without_gating`. It reports +legacy gaps without requiring zero gaps. Named convention coverage requires +the actual convention marker in the relevant entity or affected person's +section of the rendered text, with supplied nulls taking precedence as +unknown. Another person's convention cannot cover that person's missing +input; membership in a global registry alone is insufficient. Compound +readings can change multiple inputs and override new +v2 conventions, so they still need a fresh run under the proposed reference +builder even when every original input is now covered. + +The current replay has 70 moving rows. Its remaining individually moving inputs +are `county` (2 outputs), `meets_ssi_disability_criteria` (4), +`months_receiving_social_security_disability` (5), +`first_home_mortgage_origination_year` and +`second_home_mortgage_origination_year` (one shared output). All 12 distinct +affected outputs are already excluded from legacy scoring; none is a scored +residual. These counts describe +outputs, not newly discovered people or proven legal effects. The stated hours +convention now covers `weekly_hours_worked_before_lsr`, and the sales-tax +convention covers `local_sales_tax`. Four original compound-reading rows +change multiple inputs, including now-stated conventions; the replay does +not assign their whole movement to any one input. A fresh sweep under v2 +conventions must validate those rows. + +Before activation, adapt the reference builder to the stated conventions and +run the complete sweep through `unlisted_input_sweep.add_arguments(parser)` +and `unlisted_input_sweep.run(args)`, including its registered engine-estimate +inputs and plausible readings. Extend that registry for other reference-read +inputs rather than treating it as an exhaustive discovery mechanism. In +particular, register `state_sales_tax`, `mt_withheld_income_tax`, and +`medicare_irmaa_magi_two_years_prior`, with plausible readings of their own: +the existing `local_sales_tax` estimate cannot detect a state-table mismatch +or cover Montana's separate withholding reader or lagged MAGI. Also register +the employer-withholding choice and support it in the reference adapter; +#194's separately recorded payroll reading currently supplements that gap +in the replay. The checked-in `latest_final.py` fix reconstructs +legacy references and must be adapted to v2 conventions before that release +run. The sweep CLI is deliberately not registered in the published v1 path. +Resolve or exclude every scored move, validate baseline agreement, +and record the new evidence. Merely rendering all households or replaying +legacy evidence is insufficient to activate a board. + +## Evidence and remaining gates + +Tests read all 100 existing public households from +[`scenarios.csv`](../paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/scenarios.csv). +The fixture's SHA-256 is +`71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a`. +Scenario 074 supplies SSDI income but no duration; scenario 008 supplies a +general disability flag. They demonstrate rendering/missingness, not new +eligibility findings. Tests retain the original fixture and assert no mutation. +Additional cases deliberately vary fixture inputs to test integer types, +program-specific labels, explicit unknowns, conflicting inputs, provenance, +SALT payment overrides, Medicare enrollment/premium aggregation, premium +payers, optional payroll withholding, dependent-return scope, and the deciding +facts behind loaded labels. V1 prompt tests check that its frozen text and +payload remain unchanged. + +```bash +uv run pytest tests/test_prompt_contract_v2.py tests/test_prompt_contract_v2_required_facts.py tests/test_eval_no_tools.py tests/test_spec.py -q +uv run pytest tests/test_prompt_contract_v2_required_facts.py::test_legacy_required_fact_report_lists_gaps_without_gating -s -q +uv run ruff check . +uv run ruff format --check . +``` + +Issue 165 still requires suitable Microcosm data, real per-fact provenance, +duration/date evidence, marital-unit identifiers, resolution of the engine's +working-student disability gate, and an explicit decision on IRC 152/22 data +coverage. Before activating any later board, retain the standing requirement +to recompute references under every unlisted-input reading and exclude outputs +that move, using `reference_exclusions.json`. Those computations, model runs, +reference regeneration, data migration, board publication, and changes to cost +ledgers/schemas/reporting are outside this slice. Coordinator review and all +existing approval conditions remain required; this document grants none. + +## Change record + +2026-09-07: added the unactivated v2 household-contract module, dedicated tests, +and pinned fixture rendering. Host finding PB-CONTRACT-001 was fixed by rejecting +all recognized Unicode line separators and updating the source identity; fixture +fact text did not change. No published v1 file or board artifact changed. + +2026-10-06: extended the draft contract to 2.1.0 for October 5 decisions d963, +d972, and d974. Added named SALT, Medicare, and employer-withholding conventions; +premium payer/tax labels; dependent-return scope; spouse-death/child and cash +assistance source facts; and the required-facts rule with PR 196's report-only +legacy sweep replay. Proposed definitions are contained in the v2 module, +and their canonical JSON now contributes to the contract identity. Independent +review caught that adding keys to the shared spec would change the v1 spec +hash and resume metadata; the final change leaves that file byte-for-byte +unchanged. Updated the golden rendering and +rendered all 100 frozen fixtures. PR 173 remains a draft; benchmark activation is +Max's decision. No published v1 prompt, reference, exclusion, board, or payload +was changed. diff --git a/docs/runbook.md b/docs/runbook.md index fbc53812..aaac2903 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -197,6 +197,13 @@ minimax-m3 qwen-3.7-max ``` +DeepSeek retired V4 Flash on 2026-09-10, when it released V4.1 Flash +([news](https://api-docs.deepseek.com/news/news260910)). The +`deepseek-v4-flash` name now routes to V4.1 Flash, and `deepseek-v4-pro` had +already moved to the August V4 Pro release, so neither row can be re-run under +its name. V4.1 Flash is the `deepseek-v4.1-flash` row, which calls +DeepSeek's current name `deepseek-flash`. + OpenAI made [GPT-5.6 generally available](https://openai.com/index/gpt-5-6/) across ChatGPT, Codex, and the API on 2026-07-09, with a global rollout over 24 hours. Because these models are new to the PolicyBench harness, run the serving @@ -231,6 +238,18 @@ those errors remain incomplete and should be retried or rerun. Each evaluation CSV also has a `.spend.jsonl` call ledger. It records initial, failed, and repair calls separately; the supervisor uses this sidecar for its disk spend total and falls back to the legacy CSV total when no ledger exists. +A single-country supervised run (`policybench run`) writes +`policyengine_provenance.json` to its run directory at start, from a one-off +Python process. The file holds the PolicyEngine bundle provenance that every +scenario sidecar records, plus a fingerprint of the installed PolicyEngine +packages. Workers read it through `POLICYBENCH_POLICYENGINE_PROVENANCE` +instead of computing it, and the supervisor checks each worker's +sidecar against the copy it read back. A worker whose environment no longer +matches the fingerprint computes the provenance itself, as every worker did +before; `run_state.json` counts those workers in +`policyengine_provenance_recomputed`. When no file was written (a +mixed-country manifest, or a failed write), `run_state.json` shows +`policyengine_provenance: null` and every worker computes it. ## 4b. Batch Mode (Anthropic, OpenAI, Gemini) diff --git a/notes/data/bbce_asset_households_20260922.csv b/notes/data/bbce_asset_households_20260922.csv new file mode 100644 index 00000000..485762a1 --- /dev/null +++ b/notes/data/bbce_asset_households_20260922.csv @@ -0,0 +1,169 @@ +model,scenario_id,state,reference,scored,prediction,mentions_categorical_eligibility,mentions_assets +claude-fable-5,scenario_008,NJ,15108.0,True,8244.0,True,True +claude-fable-5,scenario_054,NC,6060.0,True,6480.0,True,False +claude-fable-5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-fable-5.1,scenario_008,NJ,15108.0,True,15128.64,False,False +claude-fable-5.1,scenario_054,NC,6060.0,True,6068.4,False,False +claude-fable-5.1,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5.1,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-haiku-4.5,scenario_008,NJ,15108.0,True,6996.0,False,False +claude-haiku-4.5,scenario_054,NC,6060.0,True,0.0,False,True +claude-haiku-4.5,scenario_066,VA,3576.0,True,0.0,False,True +claude-haiku-4.5,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-4.7,scenario_008,NJ,15108.0,True,13620.0,False,False +claude-opus-4.7,scenario_054,NC,6060.0,True,7152.0,False,False +claude-opus-4.7,scenario_066,VA,3576.0,True,3300.0,True,True +claude-opus-4.7,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-opus-4.8,scenario_008,NJ,15108.0,True,11352.0,False,False +claude-opus-4.8,scenario_054,NC,6060.0,True,8484.0,True,True +claude-opus-4.8,scenario_066,VA,3576.0,True,2496.0,False,False +claude-opus-4.8,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-5,scenario_008,NJ,15108.0,True,12500.0,False,False +claude-opus-5,scenario_054,NC,6060.0,True,7000.0,False,False +claude-opus-5,scenario_066,VA,3576.0,True,3396.0,False,False +claude-opus-5,scenario_080,PA,3596.039794921875,False,2622.0,False,False +claude-opus-5.5,scenario_008,NJ,15108.0,True,15117.84,True,True +claude-opus-5.5,scenario_054,NC,6060.0,True,6068.4,False,False +claude-opus-5.5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-opus-5.5,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-sonnet-4.6,scenario_008,NJ,15108.0,True,11940.0,False,False +claude-sonnet-4.6,scenario_054,NC,6060.0,True,5532.0,False,False +claude-sonnet-4.6,scenario_066,VA,3576.0,True,0.0,True,True +claude-sonnet-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +claude-sonnet-5,scenario_008,NJ,15108.0,True,11500.0,False,False +claude-sonnet-5,scenario_054,NC,6060.0,True,4200.0,False,False +claude-sonnet-5,scenario_066,VA,3576.0,True,0.0,False,False +claude-sonnet-5,scenario_080,PA,3596.039794921875,False,0.0,False,True +deepseek-v4-flash-0731,scenario_008,NJ,15108.0,True,14700.0,False,False +deepseek-v4-flash-0731,scenario_054,NC,6060.0,True,6489.0,False,False +deepseek-v4-flash-0731,scenario_066,VA,3576.0,True,3528.0,False,False +deepseek-v4-flash-0731,scenario_080,PA,3596.039794921875,False,2987.0,False,False +deepseek-v4-pro,scenario_008,NJ,15108.0,True,16453.44,False,False +deepseek-v4-pro,scenario_054,NC,6060.0,True,5988.0,False,False +deepseek-v4-pro,scenario_066,VA,3576.0,True,876.0,False,False +deepseek-v4-pro,scenario_080,PA,3596.039794921875,False,2868.0,True,True +deepseek-v4-pro-0813,scenario_008,NJ,15108.0,True,14650.0,False,False +deepseek-v4-pro-0813,scenario_054,NC,6060.0,True,6052.8,False,False +deepseek-v4-pro-0813,scenario_066,VA,3576.0,True,3504.0,True,True +deepseek-v4-pro-0813,scenario_080,PA,3596.039794921875,False,2988.0,False,False +gemini-3-flash-preview,scenario_008,NJ,15108.0,True,15300.0,False,False +gemini-3-flash-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3-flash-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3-flash-preview,scenario_080,PA,3596.039794921875,False,3492.0,False,False +gemini-3.1-flash-lite-preview,scenario_008,NJ,15108.0,True,16400.0,False,False +gemini-3.1-flash-lite-preview,scenario_054,NC,6060.0,True,3720.0,False,False +gemini-3.1-flash-lite-preview,scenario_066,VA,3576.0,True,0.0,False,True +gemini-3.1-flash-lite-preview,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.1-pro-preview,scenario_008,NJ,15108.0,True,15684.0,False,False +gemini-3.1-pro-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3.1-pro-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.1-pro-preview,scenario_080,PA,3596.039794921875,False,3504.0,False,False +gemini-3.5-flash,scenario_008,NJ,15108.0,True,16797.0,True,True +gemini-3.5-flash,scenario_054,NC,6060.0,True,5573.0,False,False +gemini-3.5-flash,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.5-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.5-flash-lite,scenario_008,NJ,15108.0,True,14500.0,False,False +gemini-3.5-flash-lite,scenario_054,NC,6060.0,True,0.0,False,False +gemini-3.5-flash-lite,scenario_066,VA,3576.0,True,3504.0,False,True +gemini-3.5-flash-lite,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.6-flash,scenario_008,NJ,15108.0,True,14676.0,True,False +gemini-3.6-flash,scenario_054,NC,6060.0,True,7548.0,False,False +gemini-3.6-flash,scenario_066,VA,3576.0,True,3504.0,True,True +gemini-3.6-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.7-flash,scenario_008,NJ,15108.0,True,16848.0,False,False +gemini-3.7-flash,scenario_054,NC,6060.0,True,5820.0,False,False +gemini-3.7-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.7-flash,scenario_080,PA,3596.039794921875,False,3576.0,True,False +gemini-3.8-flash,scenario_008,NJ,15108.0,True,15800.0,False,False +gemini-3.8-flash,scenario_054,NC,6060.0,True,5844.0,True,False +gemini-3.8-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.8-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,False +glm-5.2,scenario_008,NJ,15108.0,True,13608.9,False,False +glm-5.2,scenario_054,NC,6060.0,True,6317.0,False,False +glm-5.2,scenario_066,VA,3576.0,True,3600.0,True,True +glm-5.2,scenario_080,PA,3596.039794921875,False,,False,False +glm-5.3,scenario_008,NJ,15108.0,True,14894.0,False,False +glm-5.3,scenario_054,NC,6060.0,True,6440.0,False,False +glm-5.3,scenario_066,VA,3576.0,True,3564.0,True,True +glm-5.3,scenario_080,PA,3596.039794921875,False,2880.0,True,True +gpt-5.4-mini,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-mini,scenario_054,NC,6060.0,True,8868.0,False,True +gpt-5.4-mini,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.4-mini,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.4-nano,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-nano,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.4-nano,scenario_066,VA,3576.0,True,0.0,False,False +gpt-5.4-nano,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.5,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-5.5,scenario_054,NC,6060.0,True,6900.0,False,False +gpt-5.5,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.5,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-5.6-luna,scenario_008,NJ,15108.0,True,13668.0,False,False +gpt-5.6-luna,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.6-luna,scenario_066,VA,3576.0,True,3576.0,False,True +gpt-5.6-luna,scenario_080,PA,3596.039794921875,False,2966.4,False,False +gpt-5.6-sol,scenario_008,NJ,15108.0,True,15144.0,False,False +gpt-5.6-sol,scenario_054,NC,6060.0,True,6072.0,True,True +gpt-5.6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.6-sol,scenario_080,PA,3596.039794921875,False,3240.0,False,False +gpt-5.6-terra,scenario_008,NJ,15108.0,True,15408.2,False,False +gpt-5.6-terra,scenario_054,NC,6060.0,True,6072.0,False,False +gpt-5.6-terra,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.6-terra,scenario_080,PA,3596.039794921875,False,0.0,False,True +gpt-6-astra,scenario_008,NJ,15108.0,True,15117.9,True,True +gpt-6-astra,scenario_054,NC,6060.0,True,6068.4,True,True +gpt-6-astra,scenario_066,VA,3576.0,True,3576.0,True,True +gpt-6-astra,scenario_080,PA,3596.039794921875,False,3576.0,True,True +gpt-6-luna,scenario_008,NJ,15108.0,True,15458.0,False,False +gpt-6-luna,scenario_054,NC,6060.0,True,6068.0,False,False +gpt-6-luna,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-luna,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-6-sol,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-6-sol,scenario_054,NC,6060.0,True,6060.0,False,False +gpt-6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-sol,scenario_080,PA,3596.039794921875,False,2964.0,False,False +grok-4.3,scenario_008,NJ,15108.0,True,0.0,False,False +grok-4.3,scenario_054,NC,6060.0,True,0.0,False,True +grok-4.3,scenario_066,VA,3576.0,True,0.0,False,True +grok-4.3,scenario_080,PA,3596.039794921875,False,0.0,False,False +grok-4.5,scenario_008,NJ,15108.0,True,15600.0,True,True +grok-4.5,scenario_054,NC,6060.0,True,5801.0,True,True +grok-4.5,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.5,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-4.6,scenario_008,NJ,15108.0,True,14724.0,True,True +grok-4.6,scenario_054,NC,6060.0,True,5846.0,True,True +grok-4.6,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-build-0.1,scenario_008,NJ,15108.0,True,0.0,False,True +grok-build-0.1,scenario_054,NC,6060.0,True,6370.0,True,True +grok-build-0.1,scenario_066,VA,3576.0,True,3504.0,True,True +grok-build-0.1,scenario_080,PA,3596.039794921875,False,0.0,False,True +inkling,scenario_008,NJ,15108.0,True,15306.0,True,False +inkling,scenario_054,NC,6060.0,True,0.0,False,True +inkling,scenario_066,VA,3576.0,True,3600.0,False,False +inkling,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k2.6,scenario_008,NJ,15108.0,True,,False,False +kimi-k2.6,scenario_054,NC,6060.0,True,0.0,False,True +kimi-k2.6,scenario_066,VA,3576.0,True,0.0,True,True +kimi-k2.6,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k3,scenario_008,NJ,15108.0,True,15117.9,False,False +kimi-k3,scenario_054,NC,6060.0,True,6036.0,True,False +kimi-k3,scenario_066,VA,3576.0,True,3576.0,True,True +kimi-k3,scenario_080,PA,3596.039794921875,False,3504.0,True,True +minimax-m3,scenario_008,NJ,15108.0,True,0.0,False,True +minimax-m3,scenario_054,NC,6060.0,True,0.0,False,False +minimax-m3,scenario_066,VA,3576.0,True,0.0,False,True +minimax-m3,scenario_080,PA,3596.039794921875,False,0.0,False,True +ox-alpha,scenario_008,NJ,15108.0,True,15183.0,True,True +ox-alpha,scenario_054,NC,6060.0,True,6068.4,False,False +ox-alpha,scenario_066,VA,3576.0,True,3576.0,True,True +ox-alpha,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen-3.7-max,scenario_008,NJ,15108.0,True,14784.0,False,False +qwen-3.7-max,scenario_054,NC,6060.0,True,11208.0,False,False +qwen-3.7-max,scenario_066,VA,3576.0,True,2632.0,False,False +qwen-3.7-max,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen3.8-max,scenario_008,NJ,15108.0,True,0.0,False,False +qwen3.8-max,scenario_054,NC,6060.0,True,0.0,False,False +qwen3.8-max,scenario_066,VA,3576.0,True,1788.0,False,False +qwen3.8-max,scenario_080,PA,3596.039794921875,False,0.0,False,True diff --git a/notes/data/bbce_asset_households_20260922.csv.meta.json b/notes/data/bbce_asset_households_20260922.csv.meta.json new file mode 100644 index 00000000..3c6eab03 --- /dev/null +++ b/notes/data/bbce_asset_households_20260922.csv.meta.json @@ -0,0 +1,19 @@ +{ + "release": "dashboard-data-20260922b", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "run_payload_sha256": "3d7c4ef5632c923e36f308b20248592a7c3f5683dede47d14f6eb9a3ece00c5e", + "release_payload_sha256": "b1c4ee340a01328f75158bb4ba40b49030a979d2b03badb2c01d57c16554d3b4", + "households": [ + "scenario_008", + "scenario_054", + "scenario_066", + "scenario_080" + ], + "mention_patterns": { + "mentions_categorical_eligibility": "broad-based|\\bbbce\\b|categorical(?:ly)?[- ]eligib", + "mentions_assets": "asset|resource" + }, + "rows": 168, + "generated_at_utc": "2026-09-24T05:28:42.927883+00:00", + "script": "scripts/bbce_household_rows.py" +} diff --git a/notes/data/bbce_asset_households_20260929.csv b/notes/data/bbce_asset_households_20260929.csv new file mode 100644 index 00000000..9ee2fb11 --- /dev/null +++ b/notes/data/bbce_asset_households_20260929.csv @@ -0,0 +1,181 @@ +model,scenario_id,state,reference,scored,prediction,mentions_categorical_eligibility,mentions_assets +claude-fable-5,scenario_008,NJ,15108.0,True,8244.0,True,True +claude-fable-5,scenario_054,NC,6060.0,True,6480.0,True,False +claude-fable-5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-fable-5.1,scenario_008,NJ,15108.0,True,15128.64,False,False +claude-fable-5.1,scenario_054,NC,6060.0,True,6068.4,False,False +claude-fable-5.1,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5.1,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-haiku-4.5,scenario_008,NJ,15108.0,True,6996.0,False,False +claude-haiku-4.5,scenario_054,NC,6060.0,True,0.0,False,True +claude-haiku-4.5,scenario_066,VA,3576.0,True,0.0,False,True +claude-haiku-4.5,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-4.7,scenario_008,NJ,15108.0,True,13620.0,False,False +claude-opus-4.7,scenario_054,NC,6060.0,True,7152.0,False,False +claude-opus-4.7,scenario_066,VA,3576.0,True,3300.0,True,True +claude-opus-4.7,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-opus-4.8,scenario_008,NJ,15108.0,True,11352.0,False,False +claude-opus-4.8,scenario_054,NC,6060.0,True,8484.0,True,True +claude-opus-4.8,scenario_066,VA,3576.0,True,2496.0,False,False +claude-opus-4.8,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-5,scenario_008,NJ,15108.0,True,12500.0,False,False +claude-opus-5,scenario_054,NC,6060.0,True,7000.0,False,False +claude-opus-5,scenario_066,VA,3576.0,True,3396.0,False,False +claude-opus-5,scenario_080,PA,3596.039794921875,False,2622.0,False,False +claude-opus-5.5,scenario_008,NJ,15108.0,True,15117.84,True,True +claude-opus-5.5,scenario_054,NC,6060.0,True,6068.4,False,False +claude-opus-5.5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-opus-5.5,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-sonnet-4.6,scenario_008,NJ,15108.0,True,11940.0,False,False +claude-sonnet-4.6,scenario_054,NC,6060.0,True,5532.0,False,False +claude-sonnet-4.6,scenario_066,VA,3576.0,True,0.0,True,True +claude-sonnet-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +claude-sonnet-5,scenario_008,NJ,15108.0,True,11500.0,False,False +claude-sonnet-5,scenario_054,NC,6060.0,True,4200.0,False,False +claude-sonnet-5,scenario_066,VA,3576.0,True,0.0,False,False +claude-sonnet-5,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-sonnet-5.5,scenario_008,NJ,15108.0,True,15120.0,False,False +claude-sonnet-5.5,scenario_054,NC,6060.0,True,6068.0,False,False +claude-sonnet-5.5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-sonnet-5.5,scenario_080,PA,3596.039794921875,False,2964.0,True,False +deepseek-v4-flash-0731,scenario_008,NJ,15108.0,True,14700.0,False,False +deepseek-v4-flash-0731,scenario_054,NC,6060.0,True,6489.0,False,False +deepseek-v4-flash-0731,scenario_066,VA,3576.0,True,3528.0,False,False +deepseek-v4-flash-0731,scenario_080,PA,3596.039794921875,False,2987.0,False,False +deepseek-v4-pro,scenario_008,NJ,15108.0,True,16453.44,False,False +deepseek-v4-pro,scenario_054,NC,6060.0,True,5988.0,False,False +deepseek-v4-pro,scenario_066,VA,3576.0,True,876.0,False,False +deepseek-v4-pro,scenario_080,PA,3596.039794921875,False,2868.0,True,True +deepseek-v4-pro-0813,scenario_008,NJ,15108.0,True,14650.0,False,False +deepseek-v4-pro-0813,scenario_054,NC,6060.0,True,6052.8,False,False +deepseek-v4-pro-0813,scenario_066,VA,3576.0,True,3504.0,True,True +deepseek-v4-pro-0813,scenario_080,PA,3596.039794921875,False,2988.0,False,False +deepseek-v4.1-flash,scenario_008,NJ,15108.0,True,15289.44,False,False +deepseek-v4.1-flash,scenario_054,NC,6060.0,True,6092.0,False,False +deepseek-v4.1-flash,scenario_066,VA,3576.0,True,3576.0,False,False +deepseek-v4.1-flash,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3-flash-preview,scenario_008,NJ,15108.0,True,15300.0,False,False +gemini-3-flash-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3-flash-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3-flash-preview,scenario_080,PA,3596.039794921875,False,3492.0,False,False +gemini-3.1-flash-lite-preview,scenario_008,NJ,15108.0,True,16400.0,False,False +gemini-3.1-flash-lite-preview,scenario_054,NC,6060.0,True,3720.0,False,False +gemini-3.1-flash-lite-preview,scenario_066,VA,3576.0,True,0.0,False,True +gemini-3.1-flash-lite-preview,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.1-pro-preview,scenario_008,NJ,15108.0,True,15684.0,False,False +gemini-3.1-pro-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3.1-pro-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.1-pro-preview,scenario_080,PA,3596.039794921875,False,3504.0,False,False +gemini-3.5-flash,scenario_008,NJ,15108.0,True,16797.0,True,True +gemini-3.5-flash,scenario_054,NC,6060.0,True,5573.0,False,False +gemini-3.5-flash,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.5-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.5-flash-lite,scenario_008,NJ,15108.0,True,14500.0,False,False +gemini-3.5-flash-lite,scenario_054,NC,6060.0,True,0.0,False,False +gemini-3.5-flash-lite,scenario_066,VA,3576.0,True,3504.0,False,True +gemini-3.5-flash-lite,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.6-flash,scenario_008,NJ,15108.0,True,14676.0,True,False +gemini-3.6-flash,scenario_054,NC,6060.0,True,7548.0,False,False +gemini-3.6-flash,scenario_066,VA,3576.0,True,3504.0,True,True +gemini-3.6-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.7-flash,scenario_008,NJ,15108.0,True,16848.0,False,False +gemini-3.7-flash,scenario_054,NC,6060.0,True,5820.0,False,False +gemini-3.7-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.7-flash,scenario_080,PA,3596.039794921875,False,3576.0,True,False +gemini-3.8-flash,scenario_008,NJ,15108.0,True,15800.0,False,False +gemini-3.8-flash,scenario_054,NC,6060.0,True,5844.0,True,False +gemini-3.8-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.8-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,False +glm-5.2,scenario_008,NJ,15108.0,True,13608.9,False,False +glm-5.2,scenario_054,NC,6060.0,True,6317.0,False,False +glm-5.2,scenario_066,VA,3576.0,True,3600.0,True,True +glm-5.2,scenario_080,PA,3596.039794921875,False,,False,False +glm-5.3,scenario_008,NJ,15108.0,True,14894.0,False,False +glm-5.3,scenario_054,NC,6060.0,True,6440.0,False,False +glm-5.3,scenario_066,VA,3576.0,True,3564.0,True,True +glm-5.3,scenario_080,PA,3596.039794921875,False,2880.0,True,True +gpt-5.4-mini,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-mini,scenario_054,NC,6060.0,True,8868.0,False,True +gpt-5.4-mini,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.4-mini,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.4-nano,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-nano,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.4-nano,scenario_066,VA,3576.0,True,0.0,False,False +gpt-5.4-nano,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.5,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-5.5,scenario_054,NC,6060.0,True,6900.0,False,False +gpt-5.5,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.5,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-5.6-luna,scenario_008,NJ,15108.0,True,13668.0,False,False +gpt-5.6-luna,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.6-luna,scenario_066,VA,3576.0,True,3576.0,False,True +gpt-5.6-luna,scenario_080,PA,3596.039794921875,False,2966.4,False,False +gpt-5.6-sol,scenario_008,NJ,15108.0,True,15144.0,False,False +gpt-5.6-sol,scenario_054,NC,6060.0,True,6072.0,True,True +gpt-5.6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.6-sol,scenario_080,PA,3596.039794921875,False,3240.0,False,False +gpt-5.6-terra,scenario_008,NJ,15108.0,True,15408.2,False,False +gpt-5.6-terra,scenario_054,NC,6060.0,True,6072.0,False,False +gpt-5.6-terra,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.6-terra,scenario_080,PA,3596.039794921875,False,0.0,False,True +gpt-6-astra,scenario_008,NJ,15108.0,True,15117.9,True,True +gpt-6-astra,scenario_054,NC,6060.0,True,6068.4,True,True +gpt-6-astra,scenario_066,VA,3576.0,True,3576.0,True,True +gpt-6-astra,scenario_080,PA,3596.039794921875,False,3576.0,True,True +gpt-6-luna,scenario_008,NJ,15108.0,True,15458.0,False,False +gpt-6-luna,scenario_054,NC,6060.0,True,6068.0,False,False +gpt-6-luna,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-luna,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-6-sol,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-6-sol,scenario_054,NC,6060.0,True,6060.0,False,False +gpt-6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-sol,scenario_080,PA,3596.039794921875,False,2964.0,False,False +grok-4.3,scenario_008,NJ,15108.0,True,0.0,False,False +grok-4.3,scenario_054,NC,6060.0,True,0.0,False,True +grok-4.3,scenario_066,VA,3576.0,True,0.0,False,True +grok-4.3,scenario_080,PA,3596.039794921875,False,0.0,False,False +grok-4.5,scenario_008,NJ,15108.0,True,15600.0,True,True +grok-4.5,scenario_054,NC,6060.0,True,5801.0,True,True +grok-4.5,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.5,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-4.6,scenario_008,NJ,15108.0,True,14724.0,True,True +grok-4.6,scenario_054,NC,6060.0,True,5846.0,True,True +grok-4.6,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-4.7,scenario_008,NJ,15108.0,True,14693.1,True,True +grok-4.7,scenario_054,NC,6060.0,True,5846.4,True,True +grok-4.7,scenario_066,VA,3576.0,True,3576.0,True,True +grok-4.7,scenario_080,PA,3596.039794921875,False,3576.0,True,True +grok-build-0.1,scenario_008,NJ,15108.0,True,0.0,False,True +grok-build-0.1,scenario_054,NC,6060.0,True,6370.0,True,True +grok-build-0.1,scenario_066,VA,3576.0,True,3504.0,True,True +grok-build-0.1,scenario_080,PA,3596.039794921875,False,0.0,False,True +inkling,scenario_008,NJ,15108.0,True,15306.0,True,False +inkling,scenario_054,NC,6060.0,True,0.0,False,True +inkling,scenario_066,VA,3576.0,True,3600.0,False,False +inkling,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k2.6,scenario_008,NJ,15108.0,True,,False,False +kimi-k2.6,scenario_054,NC,6060.0,True,0.0,False,True +kimi-k2.6,scenario_066,VA,3576.0,True,0.0,True,True +kimi-k2.6,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k3,scenario_008,NJ,15108.0,True,15117.9,False,False +kimi-k3,scenario_054,NC,6060.0,True,6036.0,True,False +kimi-k3,scenario_066,VA,3576.0,True,3576.0,True,True +kimi-k3,scenario_080,PA,3596.039794921875,False,3504.0,True,True +minimax-m3,scenario_008,NJ,15108.0,True,0.0,False,True +minimax-m3,scenario_054,NC,6060.0,True,0.0,False,False +minimax-m3,scenario_066,VA,3576.0,True,0.0,False,True +minimax-m3,scenario_080,PA,3596.039794921875,False,0.0,False,True +ox-alpha,scenario_008,NJ,15108.0,True,15183.0,True,True +ox-alpha,scenario_054,NC,6060.0,True,6068.4,False,False +ox-alpha,scenario_066,VA,3576.0,True,3576.0,True,True +ox-alpha,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen-3.7-max,scenario_008,NJ,15108.0,True,14784.0,False,False +qwen-3.7-max,scenario_054,NC,6060.0,True,11208.0,False,False +qwen-3.7-max,scenario_066,VA,3576.0,True,2632.0,False,False +qwen-3.7-max,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen3.8-max,scenario_008,NJ,15108.0,True,0.0,False,False +qwen3.8-max,scenario_054,NC,6060.0,True,0.0,False,False +qwen3.8-max,scenario_066,VA,3576.0,True,1788.0,False,False +qwen3.8-max,scenario_080,PA,3596.039794921875,False,0.0,False,True diff --git a/notes/data/bbce_asset_households_20260929.csv.meta.json b/notes/data/bbce_asset_households_20260929.csv.meta.json new file mode 100644 index 00000000..78ce2008 --- /dev/null +++ b/notes/data/bbce_asset_households_20260929.csv.meta.json @@ -0,0 +1,19 @@ +{ + "release": "dashboard-data-20260929", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "run_payload_sha256": "9b807f4ab9a95d4019047a5a0ab2a6da8de472b9ba4a6626898d33e0dc381a1f", + "release_payload_sha256": "a5cb9989d78cb18d040fec2f1f5d0775df15b9b917ae99d883d8701b7fa480a7", + "households": [ + "scenario_008", + "scenario_054", + "scenario_066", + "scenario_080" + ], + "mention_patterns": { + "mentions_categorical_eligibility": "broad-based|\\bbbce\\b|categorical(?:ly)?[- ]eligib", + "mentions_assets": "asset|resource" + }, + "rows": 180, + "generated_at_utc": "2026-09-30T00:52:03.102661+00:00", + "script": "scripts/bbce_households_20260929.py" +} diff --git a/notes/data/bbce_asset_households_20260930.csv b/notes/data/bbce_asset_households_20260930.csv new file mode 100644 index 00000000..656d3c43 --- /dev/null +++ b/notes/data/bbce_asset_households_20260930.csv @@ -0,0 +1,185 @@ +model,scenario_id,state,reference,scored,prediction,mentions_categorical_eligibility,mentions_assets +claude-fable-5,scenario_008,NJ,15108.0,True,8244.0,True,True +claude-fable-5,scenario_054,NC,6060.0,True,6480.0,True,False +claude-fable-5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-fable-5.1,scenario_008,NJ,15108.0,True,15128.64,False,False +claude-fable-5.1,scenario_054,NC,6060.0,True,6068.4,False,False +claude-fable-5.1,scenario_066,VA,3576.0,True,3576.0,True,True +claude-fable-5.1,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-haiku-4.5,scenario_008,NJ,15108.0,True,6996.0,False,False +claude-haiku-4.5,scenario_054,NC,6060.0,True,0.0,False,True +claude-haiku-4.5,scenario_066,VA,3576.0,True,0.0,False,True +claude-haiku-4.5,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-4.7,scenario_008,NJ,15108.0,True,13620.0,False,False +claude-opus-4.7,scenario_054,NC,6060.0,True,7152.0,False,False +claude-opus-4.7,scenario_066,VA,3576.0,True,3300.0,True,True +claude-opus-4.7,scenario_080,PA,3596.039794921875,False,3516.0,True,True +claude-opus-4.8,scenario_008,NJ,15108.0,True,11352.0,False,False +claude-opus-4.8,scenario_054,NC,6060.0,True,8484.0,True,True +claude-opus-4.8,scenario_066,VA,3576.0,True,2496.0,False,False +claude-opus-4.8,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-opus-5,scenario_008,NJ,15108.0,True,12500.0,False,False +claude-opus-5,scenario_054,NC,6060.0,True,7000.0,False,False +claude-opus-5,scenario_066,VA,3576.0,True,3396.0,False,False +claude-opus-5,scenario_080,PA,3596.039794921875,False,2622.0,False,False +claude-opus-5.5,scenario_008,NJ,15108.0,True,15117.84,True,True +claude-opus-5.5,scenario_054,NC,6060.0,True,6068.4,False,False +claude-opus-5.5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-opus-5.5,scenario_080,PA,3596.039794921875,False,3576.0,True,True +claude-sonnet-4.6,scenario_008,NJ,15108.0,True,11940.0,False,False +claude-sonnet-4.6,scenario_054,NC,6060.0,True,5532.0,False,False +claude-sonnet-4.6,scenario_066,VA,3576.0,True,0.0,True,True +claude-sonnet-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +claude-sonnet-5,scenario_008,NJ,15108.0,True,11500.0,False,False +claude-sonnet-5,scenario_054,NC,6060.0,True,4200.0,False,False +claude-sonnet-5,scenario_066,VA,3576.0,True,0.0,False,False +claude-sonnet-5,scenario_080,PA,3596.039794921875,False,0.0,False,True +claude-sonnet-5.5,scenario_008,NJ,15108.0,True,15120.0,False,False +claude-sonnet-5.5,scenario_054,NC,6060.0,True,6068.0,False,False +claude-sonnet-5.5,scenario_066,VA,3576.0,True,3576.0,True,True +claude-sonnet-5.5,scenario_080,PA,3596.039794921875,False,2964.0,True,False +deepseek-v4-flash-0731,scenario_008,NJ,15108.0,True,14700.0,False,False +deepseek-v4-flash-0731,scenario_054,NC,6060.0,True,6489.0,False,False +deepseek-v4-flash-0731,scenario_066,VA,3576.0,True,3528.0,False,False +deepseek-v4-flash-0731,scenario_080,PA,3596.039794921875,False,2987.0,False,False +deepseek-v4-pro,scenario_008,NJ,15108.0,True,16453.44,False,False +deepseek-v4-pro,scenario_054,NC,6060.0,True,5988.0,False,False +deepseek-v4-pro,scenario_066,VA,3576.0,True,876.0,False,False +deepseek-v4-pro,scenario_080,PA,3596.039794921875,False,2868.0,True,True +deepseek-v4-pro-0813,scenario_008,NJ,15108.0,True,14650.0,False,False +deepseek-v4-pro-0813,scenario_054,NC,6060.0,True,6052.8,False,False +deepseek-v4-pro-0813,scenario_066,VA,3576.0,True,3504.0,True,True +deepseek-v4-pro-0813,scenario_080,PA,3596.039794921875,False,2988.0,False,False +deepseek-v4.1-flash,scenario_008,NJ,15108.0,True,15289.44,False,False +deepseek-v4.1-flash,scenario_054,NC,6060.0,True,6092.0,False,False +deepseek-v4.1-flash,scenario_066,VA,3576.0,True,3576.0,False,False +deepseek-v4.1-flash,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3-flash-preview,scenario_008,NJ,15108.0,True,15300.0,False,False +gemini-3-flash-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3-flash-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3-flash-preview,scenario_080,PA,3596.039794921875,False,3492.0,False,False +gemini-3.1-flash-lite-preview,scenario_008,NJ,15108.0,True,16400.0,False,False +gemini-3.1-flash-lite-preview,scenario_054,NC,6060.0,True,3720.0,False,False +gemini-3.1-flash-lite-preview,scenario_066,VA,3576.0,True,0.0,False,True +gemini-3.1-flash-lite-preview,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.1-pro-preview,scenario_008,NJ,15108.0,True,15684.0,False,False +gemini-3.1-pro-preview,scenario_054,NC,6060.0,True,5800.8,False,False +gemini-3.1-pro-preview,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.1-pro-preview,scenario_080,PA,3596.039794921875,False,3504.0,False,False +gemini-3.5-flash,scenario_008,NJ,15108.0,True,16797.0,True,True +gemini-3.5-flash,scenario_054,NC,6060.0,True,5573.0,False,False +gemini-3.5-flash,scenario_066,VA,3576.0,True,3492.0,True,True +gemini-3.5-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.5-flash-lite,scenario_008,NJ,15108.0,True,14500.0,False,False +gemini-3.5-flash-lite,scenario_054,NC,6060.0,True,0.0,False,False +gemini-3.5-flash-lite,scenario_066,VA,3576.0,True,3504.0,False,True +gemini-3.5-flash-lite,scenario_080,PA,3596.039794921875,False,0.0,False,True +gemini-3.6-flash,scenario_008,NJ,15108.0,True,14676.0,True,False +gemini-3.6-flash,scenario_054,NC,6060.0,True,7548.0,False,False +gemini-3.6-flash,scenario_066,VA,3576.0,True,3504.0,True,True +gemini-3.6-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,True +gemini-3.7-flash,scenario_008,NJ,15108.0,True,16848.0,False,False +gemini-3.7-flash,scenario_054,NC,6060.0,True,5820.0,False,False +gemini-3.7-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.7-flash,scenario_080,PA,3596.039794921875,False,3576.0,True,False +gemini-3.8-flash,scenario_008,NJ,15108.0,True,15800.0,False,False +gemini-3.8-flash,scenario_054,NC,6060.0,True,5844.0,True,False +gemini-3.8-flash,scenario_066,VA,3576.0,True,3504.0,False,False +gemini-3.8-flash,scenario_080,PA,3596.039794921875,False,3504.0,True,False +glm-5.2,scenario_008,NJ,15108.0,True,13608.9,False,False +glm-5.2,scenario_054,NC,6060.0,True,6317.0,False,False +glm-5.2,scenario_066,VA,3576.0,True,3600.0,True,True +glm-5.2,scenario_080,PA,3596.039794921875,False,,False,False +glm-5.3,scenario_008,NJ,15108.0,True,14894.0,False,False +glm-5.3,scenario_054,NC,6060.0,True,6440.0,False,False +glm-5.3,scenario_066,VA,3576.0,True,3564.0,True,True +glm-5.3,scenario_080,PA,3596.039794921875,False,2880.0,True,True +gpt-5.4-mini,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-mini,scenario_054,NC,6060.0,True,8868.0,False,True +gpt-5.4-mini,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.4-mini,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.4-nano,scenario_008,NJ,15108.0,True,0.0,False,False +gpt-5.4-nano,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.4-nano,scenario_066,VA,3576.0,True,0.0,False,False +gpt-5.4-nano,scenario_080,PA,3596.039794921875,False,0.0,False,False +gpt-5.5,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-5.5,scenario_054,NC,6060.0,True,6900.0,False,False +gpt-5.5,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.5,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-5.6-luna,scenario_008,NJ,15108.0,True,13668.0,False,False +gpt-5.6-luna,scenario_054,NC,6060.0,True,0.0,False,True +gpt-5.6-luna,scenario_066,VA,3576.0,True,3576.0,False,True +gpt-5.6-luna,scenario_080,PA,3596.039794921875,False,2966.4,False,False +gpt-5.6-sol,scenario_008,NJ,15108.0,True,15144.0,False,False +gpt-5.6-sol,scenario_054,NC,6060.0,True,6072.0,True,True +gpt-5.6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-5.6-sol,scenario_080,PA,3596.039794921875,False,3240.0,False,False +gpt-5.6-terra,scenario_008,NJ,15108.0,True,15408.2,False,False +gpt-5.6-terra,scenario_054,NC,6060.0,True,6072.0,False,False +gpt-5.6-terra,scenario_066,VA,3576.0,True,0.0,False,True +gpt-5.6-terra,scenario_080,PA,3596.039794921875,False,0.0,False,True +gpt-6-astra,scenario_008,NJ,15108.0,True,15117.9,True,True +gpt-6-astra,scenario_054,NC,6060.0,True,6068.4,True,True +gpt-6-astra,scenario_066,VA,3576.0,True,3576.0,True,True +gpt-6-astra,scenario_080,PA,3596.039794921875,False,3576.0,True,True +gpt-6-luna,scenario_008,NJ,15108.0,True,15458.0,False,False +gpt-6-luna,scenario_054,NC,6060.0,True,6068.0,False,False +gpt-6-luna,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-luna,scenario_080,PA,3596.039794921875,False,3228.0,False,False +gpt-6-sol,scenario_008,NJ,15108.0,True,15120.0,False,False +gpt-6-sol,scenario_054,NC,6060.0,True,6060.0,False,False +gpt-6-sol,scenario_066,VA,3576.0,True,3576.0,False,False +gpt-6-sol,scenario_080,PA,3596.039794921875,False,2964.0,False,False +gpt-6.1-sol,scenario_008,NJ,15108.0,True,15108.0,True,True +gpt-6.1-sol,scenario_054,NC,6060.0,True,6060.0,True,True +gpt-6.1-sol,scenario_066,VA,3576.0,True,3576.0,True,True +gpt-6.1-sol,scenario_080,PA,3596.039794921875,False,2966.4,True,True +grok-4.3,scenario_008,NJ,15108.0,True,0.0,False,False +grok-4.3,scenario_054,NC,6060.0,True,0.0,False,True +grok-4.3,scenario_066,VA,3576.0,True,0.0,False,True +grok-4.3,scenario_080,PA,3596.039794921875,False,0.0,False,False +grok-4.5,scenario_008,NJ,15108.0,True,15600.0,True,True +grok-4.5,scenario_054,NC,6060.0,True,5801.0,True,True +grok-4.5,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.5,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-4.6,scenario_008,NJ,15108.0,True,14724.0,True,True +grok-4.6,scenario_054,NC,6060.0,True,5846.0,True,True +grok-4.6,scenario_066,VA,3576.0,True,3504.0,True,True +grok-4.6,scenario_080,PA,3596.039794921875,False,3504.0,True,True +grok-4.7,scenario_008,NJ,15108.0,True,14693.1,True,True +grok-4.7,scenario_054,NC,6060.0,True,5846.4,True,True +grok-4.7,scenario_066,VA,3576.0,True,3576.0,True,True +grok-4.7,scenario_080,PA,3596.039794921875,False,3576.0,True,True +grok-build-0.1,scenario_008,NJ,15108.0,True,0.0,False,True +grok-build-0.1,scenario_054,NC,6060.0,True,6370.0,True,True +grok-build-0.1,scenario_066,VA,3576.0,True,3504.0,True,True +grok-build-0.1,scenario_080,PA,3596.039794921875,False,0.0,False,True +inkling,scenario_008,NJ,15108.0,True,15306.0,True,False +inkling,scenario_054,NC,6060.0,True,0.0,False,True +inkling,scenario_066,VA,3576.0,True,3600.0,False,False +inkling,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k2.6,scenario_008,NJ,15108.0,True,,False,False +kimi-k2.6,scenario_054,NC,6060.0,True,0.0,False,True +kimi-k2.6,scenario_066,VA,3576.0,True,0.0,True,True +kimi-k2.6,scenario_080,PA,3596.039794921875,False,0.0,False,True +kimi-k3,scenario_008,NJ,15108.0,True,15117.9,False,False +kimi-k3,scenario_054,NC,6060.0,True,6036.0,True,False +kimi-k3,scenario_066,VA,3576.0,True,3576.0,True,True +kimi-k3,scenario_080,PA,3596.039794921875,False,3504.0,True,True +minimax-m3,scenario_008,NJ,15108.0,True,0.0,False,True +minimax-m3,scenario_054,NC,6060.0,True,0.0,False,False +minimax-m3,scenario_066,VA,3576.0,True,0.0,False,True +minimax-m3,scenario_080,PA,3596.039794921875,False,0.0,False,True +ox-alpha,scenario_008,NJ,15108.0,True,15183.0,True,True +ox-alpha,scenario_054,NC,6060.0,True,6068.4,False,False +ox-alpha,scenario_066,VA,3576.0,True,3576.0,True,True +ox-alpha,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen-3.7-max,scenario_008,NJ,15108.0,True,14784.0,False,False +qwen-3.7-max,scenario_054,NC,6060.0,True,11208.0,False,False +qwen-3.7-max,scenario_066,VA,3576.0,True,2632.0,False,False +qwen-3.7-max,scenario_080,PA,3596.039794921875,False,0.0,False,True +qwen3.8-max,scenario_008,NJ,15108.0,True,0.0,False,False +qwen3.8-max,scenario_054,NC,6060.0,True,0.0,False,False +qwen3.8-max,scenario_066,VA,3576.0,True,1788.0,False,False +qwen3.8-max,scenario_080,PA,3596.039794921875,False,0.0,False,True diff --git a/notes/data/bbce_asset_households_20260930.csv.meta.json b/notes/data/bbce_asset_households_20260930.csv.meta.json new file mode 100644 index 00000000..efe44862 --- /dev/null +++ b/notes/data/bbce_asset_households_20260930.csv.meta.json @@ -0,0 +1,21 @@ +{ + "release": "dashboard-data-20260930", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "run_payload_sha256": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "release_payload_sha256": "d1cae7456cf91ab6fa04644a4d5d570e359522386a7c52f9645d5923bfc11258", + "pathways": "notes/data/snap_pathways_20260930.csv", + "pathways_sha256": "6e8c4887fea0ef79ee4dfe54e4082a4a9e0f6e01f2e46cff0c8ca4b426b147d2", + "households": [ + "scenario_008", + "scenario_054", + "scenario_066", + "scenario_080" + ], + "mention_patterns": { + "mentions_categorical_eligibility": "broad-based|\\bbbce\\b|categorical(?:ly)?[- ]eligib|expanded categorical", + "mentions_assets": "asset|resource" + }, + "rows": 184, + "generated_at_utc": "2026-10-05T12:24:00.650136+00:00", + "script": "scripts/bbce_households_20260930.py" +} diff --git a/notes/data/bbce_households_20260922.csv b/notes/data/bbce_households_20260922.csv new file mode 100644 index 00000000..7f45b454 --- /dev/null +++ b/notes/data/bbce_households_20260922.csv @@ -0,0 +1,169 @@ +model,scenario_id,state,reference,prediction,within_1_dollar,mentions_categorical_eligibility,mentions_net_income_limit +claude-fable-5,scenario_027,CT,288.0,0.0,False,True,True +claude-fable-5,scenario_030,TX,288.0,0.0,False,False,False +claude-fable-5,scenario_073,MI,288.0,0.0,False,True,True +claude-fable-5,scenario_108,WI,288.0,0.0,False,False,True +claude-fable-5.1,scenario_027,CT,288.0,0.0,False,False,True +claude-fable-5.1,scenario_030,TX,288.0,0.0,False,False,True +claude-fable-5.1,scenario_073,MI,288.0,0.0,False,False,True +claude-fable-5.1,scenario_108,WI,288.0,0.0,False,False,True +claude-haiku-4.5,scenario_027,CT,288.0,0.0,False,False,False +claude-haiku-4.5,scenario_030,TX,288.0,0.0,False,False,False +claude-haiku-4.5,scenario_073,MI,288.0,0.0,False,False,False +claude-haiku-4.5,scenario_108,WI,288.0,0.0,False,False,False +claude-opus-4.7,scenario_027,CT,288.0,0.0,False,False,False +claude-opus-4.7,scenario_030,TX,288.0,0.0,False,True,True +claude-opus-4.7,scenario_073,MI,288.0,3576.0,False,True,True +claude-opus-4.7,scenario_108,WI,288.0,2227.0,False,False,False +claude-opus-4.8,scenario_027,CT,288.0,0.0,False,False,True +claude-opus-4.8,scenario_030,TX,288.0,2298.0,False,False,False +claude-opus-4.8,scenario_073,MI,288.0,0.0,False,False,True +claude-opus-4.8,scenario_108,WI,288.0,2376.0,False,False,True +claude-opus-5,scenario_027,CT,288.0,0.0,False,False,True +claude-opus-5,scenario_030,TX,288.0,1608.0,False,False,False +claude-opus-5,scenario_073,MI,288.0,2568.0,False,True,False +claude-opus-5,scenario_108,WI,288.0,276.0,False,False,True +claude-opus-5.5,scenario_027,CT,288.0,0.0,False,False,True +claude-opus-5.5,scenario_030,TX,288.0,1208.0,False,False,False +claude-opus-5.5,scenario_073,MI,288.0,0.0,False,False,True +claude-opus-5.5,scenario_108,WI,288.0,288.0,True,True,False +claude-sonnet-4.6,scenario_027,CT,288.0,0.0,False,False,True +claude-sonnet-4.6,scenario_030,TX,288.0,0.0,False,True,True +claude-sonnet-4.6,scenario_073,MI,288.0,0.0,False,False,True +claude-sonnet-4.6,scenario_108,WI,288.0,0.0,False,True,True +claude-sonnet-5,scenario_027,CT,288.0,0.0,False,False,False +claude-sonnet-5,scenario_030,TX,288.0,4200.0,False,False,False +claude-sonnet-5,scenario_073,MI,288.0,0.0,False,False,True +claude-sonnet-5,scenario_108,WI,288.0,1968.0,False,False,False +deepseek-v4-flash-0731,scenario_027,CT,288.0,0.0,False,False,False +deepseek-v4-flash-0731,scenario_030,TX,288.0,0.0,False,False,False +deepseek-v4-flash-0731,scenario_073,MI,288.0,300.0,False,False,True +deepseek-v4-flash-0731,scenario_108,WI,288.0,0.0,False,False,True +deepseek-v4-pro,scenario_027,CT,288.0,0.0,False,False,True +deepseek-v4-pro,scenario_030,TX,288.0,0.0,False,False,False +deepseek-v4-pro,scenario_073,MI,288.0,0.0,False,False,False +deepseek-v4-pro,scenario_108,WI,288.0,0.0,False,False,False +deepseek-v4-pro-0813,scenario_027,CT,288.0,0.0,False,False,True +deepseek-v4-pro-0813,scenario_030,TX,288.0,0.0,False,False,False +deepseek-v4-pro-0813,scenario_073,MI,288.0,0.0,False,False,True +deepseek-v4-pro-0813,scenario_108,WI,288.0,0.0,False,False,False +gemini-3-flash-preview,scenario_027,CT,288.0,0.0,False,True,False +gemini-3-flash-preview,scenario_030,TX,288.0,0.0,False,False,False +gemini-3-flash-preview,scenario_073,MI,288.0,276.0,False,True,False +gemini-3-flash-preview,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.1-flash-lite-preview,scenario_027,CT,288.0,0.0,False,False,False +gemini-3.1-flash-lite-preview,scenario_030,TX,288.0,2604.0,False,False,False +gemini-3.1-flash-lite-preview,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.1-flash-lite-preview,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.1-pro-preview,scenario_027,CT,288.0,0.0,False,True,False +gemini-3.1-pro-preview,scenario_030,TX,288.0,0.0,False,False,False +gemini-3.1-pro-preview,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.1-pro-preview,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.5-flash,scenario_027,CT,288.0,276.0,False,True,False +gemini-3.5-flash,scenario_030,TX,288.0,0.0,False,False,False +gemini-3.5-flash,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.5-flash,scenario_108,WI,288.0,276.0,False,True,False +gemini-3.5-flash-lite,scenario_027,CT,288.0,0.0,False,False,False +gemini-3.5-flash-lite,scenario_030,TX,288.0,0.0,False,False,False +gemini-3.5-flash-lite,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.5-flash-lite,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.6-flash,scenario_027,CT,288.0,276.0,False,True,False +gemini-3.6-flash,scenario_030,TX,288.0,0.0,False,False,True +gemini-3.6-flash,scenario_073,MI,288.0,276.0,False,True,False +gemini-3.6-flash,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.7-flash,scenario_027,CT,288.0,276.0,False,True,False +gemini-3.7-flash,scenario_030,TX,288.0,0.0,False,False,False +gemini-3.7-flash,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.7-flash,scenario_108,WI,288.0,0.0,False,False,False +gemini-3.8-flash,scenario_027,CT,288.0,0.0,False,False,False +gemini-3.8-flash,scenario_030,TX,288.0,0.0,False,False,False +gemini-3.8-flash,scenario_073,MI,288.0,0.0,False,False,False +gemini-3.8-flash,scenario_108,WI,288.0,0.0,False,False,False +glm-5.2,scenario_027,CT,288.0,,False,False,False +glm-5.2,scenario_030,TX,288.0,2873.0,False,False,False +glm-5.2,scenario_073,MI,288.0,0.0,False,False,True +glm-5.2,scenario_108,WI,288.0,0.0,False,False,False +glm-5.3,scenario_027,CT,288.0,0.0,False,False,False +glm-5.3,scenario_030,TX,288.0,0.0,False,False,True +glm-5.3,scenario_073,MI,288.0,0.0,False,False,True +glm-5.3,scenario_108,WI,288.0,0.0,False,False,True +gpt-5.4-mini,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.4-mini,scenario_030,TX,288.0,2116.0,False,False,False +gpt-5.4-mini,scenario_073,MI,288.0,0.0,False,False,False +gpt-5.4-mini,scenario_108,WI,288.0,1172.0,False,False,False +gpt-5.4-nano,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.4-nano,scenario_030,TX,288.0,0.0,False,False,False +gpt-5.4-nano,scenario_073,MI,288.0,0.0,False,False,False +gpt-5.4-nano,scenario_108,WI,288.0,0.0,False,False,False +gpt-5.5,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.5,scenario_030,TX,288.0,0.0,False,False,False +gpt-5.5,scenario_073,MI,288.0,288.0,True,False,True +gpt-5.5,scenario_108,WI,288.0,0.0,False,False,False +gpt-5.6-luna,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.6-luna,scenario_030,TX,288.0,0.0,False,False,False +gpt-5.6-luna,scenario_073,MI,288.0,0.0,False,False,False +gpt-5.6-luna,scenario_108,WI,288.0,1236.0,False,False,False +gpt-5.6-sol,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.6-sol,scenario_030,TX,288.0,0.0,False,False,False +gpt-5.6-sol,scenario_073,MI,288.0,0.0,False,False,False +gpt-5.6-sol,scenario_108,WI,288.0,0.0,False,False,False +gpt-5.6-terra,scenario_027,CT,288.0,0.0,False,False,False +gpt-5.6-terra,scenario_030,TX,288.0,0.0,False,False,False +gpt-5.6-terra,scenario_073,MI,288.0,0.0,False,False,False +gpt-5.6-terra,scenario_108,WI,288.0,0.0,False,False,True +gpt-6-astra,scenario_027,CT,288.0,288.0,True,True,False +gpt-6-astra,scenario_030,TX,288.0,0.0,False,False,False +gpt-6-astra,scenario_073,MI,288.0,288.0,True,True,False +gpt-6-astra,scenario_108,WI,288.0,288.0,True,True,False +gpt-6-luna,scenario_027,CT,288.0,0.0,False,False,False +gpt-6-luna,scenario_030,TX,288.0,0.0,False,False,False +gpt-6-luna,scenario_073,MI,288.0,0.0,False,False,False +gpt-6-luna,scenario_108,WI,288.0,0.0,False,False,True +gpt-6-sol,scenario_027,CT,288.0,0.0,False,False,True +gpt-6-sol,scenario_030,TX,288.0,1208.4,False,False,False +gpt-6-sol,scenario_073,MI,288.0,0.0,False,False,False +gpt-6-sol,scenario_108,WI,288.0,0.0,False,False,False +grok-4.3,scenario_027,CT,288.0,0.0,False,False,False +grok-4.3,scenario_030,TX,288.0,0.0,False,False,False +grok-4.3,scenario_073,MI,288.0,0.0,False,False,True +grok-4.3,scenario_108,WI,288.0,0.0,False,False,False +grok-4.5,scenario_027,CT,288.0,0.0,False,False,False +grok-4.5,scenario_030,TX,288.0,0.0,False,False,False +grok-4.5,scenario_073,MI,288.0,0.0,False,False,True +grok-4.5,scenario_108,WI,288.0,0.0,False,False,True +grok-4.6,scenario_027,CT,288.0,0.0,False,False,False +grok-4.6,scenario_030,TX,288.0,0.0,False,True,False +grok-4.6,scenario_073,MI,288.0,0.0,False,False,False +grok-4.6,scenario_108,WI,288.0,0.0,False,False,True +grok-build-0.1,scenario_027,CT,288.0,0.0,False,False,False +grok-build-0.1,scenario_030,TX,288.0,0.0,False,False,False +grok-build-0.1,scenario_073,MI,288.0,0.0,False,False,False +grok-build-0.1,scenario_108,WI,288.0,0.0,False,False,True +inkling,scenario_027,CT,288.0,0.0,False,False,True +inkling,scenario_030,TX,288.0,0.0,False,False,False +inkling,scenario_073,MI,288.0,276.0,False,False,False +inkling,scenario_108,WI,288.0,0.0,False,False,True +kimi-k2.6,scenario_027,CT,288.0,0.0,False,False,True +kimi-k2.6,scenario_030,TX,288.0,,False,False,False +kimi-k2.6,scenario_073,MI,288.0,0.0,False,False,False +kimi-k2.6,scenario_108,WI,288.0,0.0,False,False,True +kimi-k3,scenario_027,CT,288.0,0.0,False,False,True +kimi-k3,scenario_030,TX,288.0,0.0,False,False,False +kimi-k3,scenario_073,MI,288.0,0.0,False,False,True +kimi-k3,scenario_108,WI,288.0,0.0,False,False,False +minimax-m3,scenario_027,CT,288.0,0.0,False,False,False +minimax-m3,scenario_030,TX,288.0,0.0,False,False,False +minimax-m3,scenario_073,MI,288.0,0.0,False,False,False +minimax-m3,scenario_108,WI,288.0,0.0,False,False,True +ox-alpha,scenario_027,CT,288.0,0.0,False,False,True +ox-alpha,scenario_030,TX,288.0,0.0,False,False,False +ox-alpha,scenario_073,MI,288.0,0.0,False,True,True +ox-alpha,scenario_108,WI,288.0,0.0,False,False,False +qwen-3.7-max,scenario_027,CT,288.0,0.0,False,False,True +qwen-3.7-max,scenario_030,TX,288.0,2576.0,False,False,False +qwen-3.7-max,scenario_073,MI,288.0,0.0,False,False,False +qwen-3.7-max,scenario_108,WI,288.0,0.0,False,False,False +qwen3.8-max,scenario_027,CT,288.0,5160.0,False,False,False +qwen3.8-max,scenario_030,TX,288.0,3576.0,False,False,False +qwen3.8-max,scenario_073,MI,288.0,0.0,False,False,False +qwen3.8-max,scenario_108,WI,288.0,0.0,False,False,False diff --git a/notes/data/bbce_households_20260922.csv.meta.json b/notes/data/bbce_households_20260922.csv.meta.json new file mode 100644 index 00000000..da0fc442 --- /dev/null +++ b/notes/data/bbce_households_20260922.csv.meta.json @@ -0,0 +1,19 @@ +{ + "release": "dashboard-data-20260922b", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "run_payload_sha256": "3d7c4ef5632c923e36f308b20248592a7c3f5683dede47d14f6eb9a3ece00c5e", + "release_payload_sha256": "b1c4ee340a01328f75158bb4ba40b49030a979d2b03badb2c01d57c16554d3b4", + "households": [ + "scenario_027", + "scenario_030", + "scenario_073", + "scenario_108" + ], + "mention_patterns": { + "mentions_categorical_eligibility": "broad-based|\\bbbce\\b|categorical(?:ly)?[- ]eligib", + "mentions_net_income_limit": "\\bnet[- ](?:income[- ])?(?:limit|test|screen|threshold|ceiling|rule)|(? 0) | person(\"receives_ssi\", period)\n all_members_receive_ssi = spm_unit.all(receives_ssi)\n return (\n all_members_receive_ssi\n | (add(spm_unit, period, spm_level_programs) > 0)\n | spm_unit(\"receives_tanf\", period)\n )\n", + "is_tanf_non_cash_eligible": " def formula(spm_unit, period, parameters):\n gross = spm_unit(\"meets_tanf_non_cash_gross_income_test\", period)\n net = spm_unit(\"meets_tanf_non_cash_net_income_test\", period)\n asset = spm_unit(\"meets_tanf_non_cash_asset_test\", period)\n return gross & net & asset\n", + "meets_tanf_non_cash_gross_income_test": " def formula(spm_unit, period, parameters):\n # Use the gross-test income concept so states electing full\n # counting of certain ineligible aliens' income under the gross\n # income test (7 CFR 273.11(c)(3)(i)) apply it to this categorical\n # eligibility screen as well.\n gross_income = spm_unit(\"snap_gross_test_income\", period)\n limit = spm_unit(\"tanf_non_cash_gross_income_limit\", period)\n return gross_income <= limit\n", + "meets_tanf_non_cash_net_income_test": " def formula(spm_unit, period, parameters):\n # Determine if the net income limit applies to the household.\n applies = parameters(period).gov.hhs.tanf.non_cash.income_limit.net_applies\n state = spm_unit.household(\"state_code_str\", period)\n # Varies depending on if the household has elderly and disabled people.\n hheod = spm_unit(\"is_tanf_non_cash_hheod\", period)\n net_limit_applies = where(\n hheod, applies.hheod[state], applies.non_hheod[state]\n ).astype(bool)\n net_income = spm_unit(\"snap_net_income\", period)\n net_limit = parameters(period).gov.usda.snap.income.limit.net\n # The state's BBCE net test uses the federal SNAP net standard: the\n # FNS table built per 7 CFR 273.9(a)(3)(ii), including the separately\n # rounded-up increment above eight persons. It is compared against\n # the whole-dollar rounded net income; a raw ratio would deny\n # households exactly at the published standard.\n limit = snap_monthly_income_standard(spm_unit, period, parameters, net_limit)\n # Either the net limit doesn't apply or they pass it.\n return ~net_limit_applies | (net_income <= limit)\n", + "meets_tanf_non_cash_asset_test": " def formula(spm_unit, period, parameters):\n assets = spm_unit(\"snap_assets\", period)\n state = spm_unit.household(\"state_code_str\", period)\n limits = parameters(period).gov.hhs.tanf.non_cash\n vehicle_value = spm_unit.household(\"household_vehicles_value\", period)\n vehicles_owned = spm_unit.household(\"household_vehicles_owned\", period)\n # Preserve the historical one-vehicle assumption when vehicle value is\n # provided without a vehicle count.\n vehicle_count = max_(vehicles_owned, vehicle_value > 0)\n vehicle_exemption = where(\n vehicle_count > 0,\n limits.tx_vehicle_exemption\n + max_(vehicle_count - 1, 0) * limits.tx_additional_vehicle_exemption,\n 0,\n )\n tx_vehicle_value = max_(vehicle_value - vehicle_exemption, 0)\n assets = assets + where(state == \"TX\", tx_vehicle_value, 0)\n asset_limit = limits.asset_limit[state]\n return assets <= asset_limit\n", + "tanf_non_cash_gross_income_limit": " def formula(spm_unit, period, parameters):\n state = spm_unit.household(\"state_code_str\", period.this_year)\n limits = parameters(period).gov.hhs.tanf.non_cash.income_limit\n gross_limit = limits.gross[state]\n hheod = spm_unit(\"is_tanf_non_cash_hheod\", period)\n gross_limit = where(hheod, limits.gross_hheod[state], gross_limit)\n\n ny = state == \"NY\"\n has_dependent_care = spm_unit(\"snap_dependent_care_deduction\", period) > 0\n has_earned_income = spm_unit(\"snap_earned_income\", period) > 0\n ny_gross_limit = where(\n has_dependent_care | hheod,\n limits.ny.dependent_care,\n where(\n has_earned_income,\n limits.ny.earned_income,\n limits.gross.NY,\n ),\n )\n gross_limit = where(ny, ny_gross_limit, gross_limit)\n\n # The standard is the state's percentage of the poverty guideline\n # vintage its BBCE schedule selects (fpg_year_start_month). The\n # rounding each state applies when publishing its chart is a\n # feature of that chart, not of the regulation: categorically\n # eligible households are exempt from the 7 CFR 273.9 income\n # standards, so the rounding 273.9(a)(3) prescribes for the\n # federal tests does not reach this limit. Return the exact\n # computation rather than assert an unlegislated convention.\n # Charts round in state-specific directions: Washington and\n # Maine publish figures equal to the exact value, while states\n # that round up publish a figure up to about $2 above it, so a\n # household between the two can score ineligible here slightly\n # before the published chart would say so.\n fpg = spm_unit(\"tanf_non_cash_fpg\", period)\n return gross_limit * fpg\n", + "is_tanf_non_cash_hheod": " def formula(spm_unit, period, parameters):\n state = spm_unit.household(\"state_code_str\", period)\n bbce = parameters(period).gov.hhs.tanf.non_cash\n requires_all = bbce.requires_all_for_hheod[state]\n # Excluded members are not household members under the 7 CFR 271.2\n # definition, so they do not confer elderly or disabled status.\n return where(\n requires_all,\n spm_unit(\"has_all_usda_elderly_disabled\", period),\n spm_unit(\"has_snap_elderly_disabled_member\", period),\n )\n", + "is_snap_eligible": " def formula(spm_unit, period, parameters):\n net = spm_unit(\"meets_snap_net_income_test\", period)\n gross = spm_unit(\"meets_snap_gross_income_test\", period)\n asset = spm_unit(\"meets_snap_asset_test\", period)\n normal_eligibility = net & gross & asset\n # Categorical eligibility (SSI, TANF, and BBCE TANF) overrides tests.\n categorical_eligibility = spm_unit(\"meets_snap_categorical_eligibility\", period)\n person = spm_unit.members\n # At least one member must satisfy the student (7 USC 2015(e)),\n # immigration (7 USC 2015(f)), and work-requirement (7 CFR 273.7,\n # 273.24) rules simultaneously. Testing these conditions with\n # separate any() reductions would incorrectly pass a unit where\n # different members satisfy different rules with no single member\n # satisfying all of them; is_snap_excluded_member already encodes\n # the per-person disjunction of the three filters.\n eligible_member_present = spm_unit.any(\n ~person(\"is_snap_excluded_member\", period)\n )\n return (normal_eligibility | categorical_eligibility) & eligible_member_present\n", + "snap_normal_allotment": " def formula(spm_unit, period, parameters):\n # Federal SNAP rules are defined in U.S.C Title 7, Chapter 51, which\n # also defines state powers to modify the rules.\n expected_contribution = spm_unit(\"snap_expected_contribution\", period)\n max_allotment = spm_unit(\"snap_max_allotment\", period)\n normal_allotment = max_allotment - expected_contribution\n min_allotment = spm_unit(\"snap_min_allotment\", period)\n return max_(min_allotment, normal_allotment)\n", + "meets_snap_gross_income_test": " def formula(spm_unit, period, parameters):\n p = parameters(period).gov.usda.snap.income.limit\n # The gross test uses state-specific income counting for certain\n # ineligible aliens (7 CFR 273.11(c)(3)(i)).\n income = spm_unit(\"snap_gross_test_income\", period)\n # 7 CFR 273.9(a)(3)(i): the monthly standard is 130% of the poverty\n # guideline divided by 12, rounded up to the next whole dollar, with\n # a separately rounded per-person increment for large households.\n limit = snap_monthly_income_standard(spm_unit, period, parameters, p.gross)\n # Households with an elderly or disabled unit member are exempt\n # from the gross income test; excluded members are not household\n # members under the 7 CFR 271.2 definition, so they do not confer\n # this status.\n has_elderly_disabled = spm_unit(\"has_snap_elderly_disabled_member\", period)\n return has_elderly_disabled | (income <= limit)\n" + }, + "excluded_snap_rows": { + "note": "Rows whose snap_scored is False show PolicyEngine's computation, which the release's exclusion record sets aside for every model (reference_exclusions.json)", + "households": { + "scenario_023": "reference_depends_on_unlisted_input (meets_ssi_disability_criteria)", + "scenario_056": "reference_depends_on_unlisted_input (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)", + "scenario_057": "reference_depends_on_unlisted_input (meets_ssi_disability_criteria)", + "scenario_080": "reference_engine_defect (r30_snap_heat_and_eat_sua); upstream: to be filed", + "scenario_100": "reference_depends_on_unlisted_input (meets_ssi_disability_criteria)", + "scenario_112": "reference_depends_on_unlisted_input (weekly_hours_worked_before_lsr)", + "scenario_118": "reference_depends_on_unlisted_input (whether the listed home mortgage interest is on the home the SNAP household occupies)" + } + }, + "employer_premiums_paid_by_household": { + "engine_documentation": "Annual employer-paid health insurance premiums. CBO treats this as part of household market income.", + "reading": "employer_sponsored_insurance_premiums moved into health_insurance_premiums_without_medicare_part_b, for households with a member the engine treats as elderly or disabled", + "households": { + "scenario_000": { + "monthly_net_income": "8921 8921 8921 8921 8921 8921 8921 8921 8921 8921 8921 8921", + "monthly_net_income_premiums_paid": "8221 8221 8221 8221 8221 8221 8221 8221 8221 8221 8221 8221", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_003": { + "monthly_net_income": "17512 17512 17512 17512 17512 17512 17512 17512 17512 17512 17512 17512", + "monthly_net_income_premiums_paid": "16145 16145 16145 16145 16145 16145 16145 16145 16145 16145 16145 16145", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_004": { + "monthly_net_income": "4174 4174 4174 4174 4174 4174 4174 4174 4174 4174 4174 4174", + "monthly_net_income_premiums_paid": "4174 4174 4174 4174 4174 4174 4174 4174 4174 4174 4174 4174", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_007": { + "monthly_net_income": "4211 4211 4211 4211 4211 4211 4211 4211 4211 4211 4211 4211", + "monthly_net_income_premiums_paid": "3685 3685 3685 3685 3685 3685 3685 3685 3685 3685 3685 3685", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_025": { + "monthly_net_income": "6083 6083 6083 6083 6083 6083 6083 6083 6083 6083 6083 6083", + "monthly_net_income_premiums_paid": "4315 4315 4315 4315 4315 4315 4315 4315 4315 4315 4315 4315", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_039": { + "monthly_net_income": "4821 4821 4821 4821 4821 4821 4821 4821 4821 4821 4821 4821", + "monthly_net_income_premiums_paid": "4521 4521 4521 4521 4521 4521 4521 4521 4521 4521 4521 4521", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_043": { + "monthly_net_income": "0 0 0 0 0 0 0 0 0 0 0 0", + "monthly_net_income_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0", + "net_income_test_months_premiums_paid": 12, + "pathway_by_month_premiums_paid": "ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary", + "monthly_snap_premiums_paid": "298 298 298 298 298 298 298 298 298 298 298 298" + }, + "scenario_073": { + "monthly_net_income": "1755 1755 1755 1755 1755 1755 1755 1755 1755 1755 1755 1755", + "monthly_net_income_premiums_paid": "1228 1228 1228 1228 1228 1228 1228 1228 1228 1228 1228 1228", + "net_income_test_months_premiums_paid": 12, + "pathway_by_month_premiums_paid": "ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary", + "monthly_snap_premiums_paid": "24 24 24 24 24 24 24 24 24 24 24 24" + }, + "scenario_084": { + "monthly_net_income": "2120 2120 2120 2120 2120 2120 2120 2120 2120 2120 2120 2120", + "monthly_net_income_premiums_paid": "1721 1721 1721 1721 1721 1721 1721 1721 1721 1721 1721 1721", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + }, + "scenario_104": { + "monthly_net_income": "2307 2307 2307 2307 2307 2307 2307 2307 2307 2307 2307 2307", + "monthly_net_income_premiums_paid": "1258 1258 1258 1258 1258 1258 1258 1258 1258 1258 1258 1258", + "net_income_test_months_premiums_paid": 12, + "pathway_by_month_premiums_paid": "ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary ordinary", + "monthly_snap_premiums_paid": "24 24 24 24 24 24 24 24 24 24 24 24" + }, + "scenario_120": { + "monthly_net_income": "17617 17617 17617 17617 17617 17617 17617 17617 17617 17617 17617 17617", + "monthly_net_income_premiums_paid": "16998 16998 16998 16998 16998 16998 16998 16998 16998 16998 16998 16998", + "net_income_test_months_premiums_paid": 0, + "pathway_by_month_premiums_paid": "ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible ineligible", + "monthly_snap_premiums_paid": "0 0 0 0 0 0 0 0 0 0 0 0" + } + } + }, + "generated_at_utc": "2026-10-05T12:54:30.665476+00:00", + "script": "scripts/snap_pathways_20260930.py" +} diff --git a/paper/index.qmd b/paper/index.qmd index d7194d7c..d1966f22 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -29,6 +29,7 @@ from pathlib import Path import numpy as np import pandas as pd +from IPython.display import Markdown from policybench.config import SEED from policybench.manuscript_views import ( country_scores_from_rows, @@ -241,7 +242,7 @@ def load_snapshot_ground_truth() -> dict[str, pd.DataFrame]: """The scored reference: the frozen CSV minus the outputs its ``reference_exclusions.json`` removes from scoring for every model, so every manuscript scorer (bootstrap intervals, weighting comparison, - simple baselines) uses the same 1,973 outputs as the published board.""" + simple baselines) uses the same scored outputs as the published board.""" from policybench.reference_exclusions import scored_reference_for return { @@ -731,9 +732,11 @@ def fmt_latency_seconds(seconds) -> str: def fmt_cost_per_household(usd) -> str: + """A dollar amount in Markdown: the escaped sign keeps Pandoc from reading + two amounts in one sentence as inline math. Every caller emits Markdown + (a pipe table, or an inline expression wrapped in ``Markdown``).""" if usd is None or pd.isna(usd): return "—" - # Escape for Quarto so the paired dollar signs are not read as inline math. return f"\\${usd:.3f}" @@ -827,7 +830,9 @@ def cost_latency_summary() -> str: ) -cost_latency_text = cost_latency_summary() +# Markdown, so Quarto renders the escaped dollar signs rather than printing +# the string's repr (which doubled each backslash). +cost_latency_text = Markdown(cost_latency_summary()) federal_state_joint = federal_state_joint_accuracy() weighting_sensitivity_table = weighting_sensitivity() deviation_audit_summary = deviation_audit_summary_table() @@ -853,8 +858,8 @@ snapshot_provenance = pd.DataFrame( ["Scoring code SHA-256 prefix", file_hash_prefix("policybench/analysis.py")], ["US run label", US_RUN_LABEL], ["Household sample seed", str(SEED)], - ["PolicyEngine.py", r.policyengine_version], - ["PolicyEngine-US", f"policyengine-us {r.policyengine_us_version}"], + ["Reference engine", f"policyengine-us {r.policyengine_us_version}"], + ["PolicyEngine.py (recorded for provenance)", r.policyengine_version], ["US dataset", f"{r.dataset_id} ({r.dataset_build_id})"], ["Households", f"{r.n_households_fmt} US"], ["Models", r.n_models_fmt], @@ -881,6 +886,11 @@ reference_audit = pd.DataFrame( ["US state and local income tax", "Reviewed state-rule and bracket suspects; most applied wrong schedules or missed conformity elections and forgiveness programs. One genuine defect: New Jersey's filing-threshold floor was bypassed on the benchmarked path (fixed, policyengine-us #8845)."], ["US Head Start eligibility", "Two flags reflected a genuine age-window defect (fixed, policyengine-us #8846); references regenerated with the fix."], ["US payroll and overtime", "Reviewed during development; upstream data fixes and prompt clarifications were applied before the frozen snapshot."], + ["US federal income tax (September 2026 wave)", "Engine defects confirmed: the traditional IRA deduction ignores the compensation limit and the active-participant phase-out (IRC 219(b), 219(g)); estate income never enters gross income; elective deferrals count as EITC and refundable-CTC earned income. Affected outputs excluded. Capital gain distributions reported without Schedule D never reached AGI, a defect fixed upstream after the freeze; the references it alone moves are regenerated with the fix. The taxability of employment disability benefits and whether the output includes the net investment income tax excluded as unlisted inputs. The educator-expense flag dissolved against the $350 2026 cap."], + ["US state income tax and credits (September 2026 wave)", "Engine defects confirmed in New Jersey worker contributions, Wisconsin's elected retirement exclusion, Wisconsin's capital gain exclusion (which leaves out capital gain distributions), Idaho's health-premium subtraction, California itemized-deduction conformity, and Massachusetts dividends taxed without the short-term capital loss offset; affected outputs excluded. The CalEITC's missing second lookup at adjusted gross income and New York's renter cap were fixed upstream after the freeze; the affected references are regenerated with the fixes. Wisconsin homestead income turned on the unlisted type of survivor benefits (excluded as an unlisted input). The New Jersey EITC, New York real property tax credit and Idaho child tax credit flags dissolved against the statute (Idaho's credit expired after 2025)."], + ["US SNAP, SSI and CHIP (September 2026 wave)", "Whether listed mortgage interest is on the occupied home, unlisted hours worked, survivor-benefit taxability, and whether an adult tax dependent is the filers' child (three Medicaid outputs) excluded as unlisted inputs. SNAP October-December references regenerated with the FY2026 figures, the last published before the freeze. SNAP references regenerated for every month of 2026 with the engine's SNAP rounding defects fixed as policyengine-us fixed them after the freeze (cents in the allotment, an unrounded minimum benefit, floored net income, unrounded income standards). One Michigan SNAP reference regenerated with the fix for the SNAP child support option, whose engine values carried the opposite meaning (Michigan deducts child support from net income, and the engine excluded it from gross income; fixed upstream in policyengine-us #9586 on 2026-09-24). The heat-and-eat utility allowance granted without an elderly or disabled member, which P.L. 119-21 ended, excluded as an engine defect. The CHIP flag dissolved: employer coverage bars CHIP."], + ["US projected 2026 parameters (September 2026 wave)", "No judge flag: a recomputation with every projected 2026 value held found references resting on price-index projections. Wisconsin, Minnesota, Michigan and Missouri amounts and Maryland's withholding allowance had been published before the freeze; California's indexed amounts, the IRS sales tax tables, Idaho's threshold and Maryland's return deduction had not. Scored references regenerated with the published or last-published amounts; Wisconsin's reach only the corrected values of Wisconsin outputs excluded for engine defects."], + [f"US references on policyengine-us {r.policyengine_us_version} (September 2026 upgrade)", f"Once PolicyBench had ported the conventions, an investigator traced every output that still moved between policyengine-us {r.previous_policyengine_us_version} and {r.policyengine_us_version} to its upstream change and settled it against primary sources and their publication dates. An independent reviewer tried to refute each settlement. The upgrade changes {r.engine_upgrade_scored_change_count} scored references, for law published before the freeze that {r.previous_policyengine_us_version} did not encode or misapplied: New Jersey's 2026-2028 child tax credit schedule (P.L.2026, c.26), Arizona's 200 percent SNAP categorical eligibility limit from March 2026, child support received counted as income for school meals (7 CFR 245.6(a)(5)(ii)), and New York's Empire State child credit phase-out rounding. The investigation found, and PolicyBench excluded, {r.engine_upgrade_new_exclusion_count} federal income tax outputs that depend on an unlisted input: {r.policyengine_us_version} counts a listed state and local tax refund as income, which is right only if the refunded tax reduced federal tax in the prior year (26 U.S.C. 111(a)). PolicyBench re-reviewed the {r.engine_upgrade_rechecked_count} excluded outputs that move; all stay excluded. The re-review of a California household's excluded SNAP output flagged its head's Medicaid eligibility, which turns on the same unlisted input, SSI's disability criterion, through the Working Disabled Program; PolicyBench excluded it."], ], columns=["Discrepancy class", "Review outcome"], ) @@ -957,7 +967,7 @@ conclusion_snapshot_summary = country_top_sentence(us_top_model) ## Abstract {.unnumbered} -PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine `{python} r.dataset_label` microdataset, scored through policyengine.py `{python} r.policyengine_version`. It evaluates `{python} r.n_models_fmt` frontier models on `{python} r.n_households_fmt` households across `{python} r.n_output_groups_fmt` output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: `{python} r.zero_share_pct_fmt`% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of `{python} r.always_zero_exact_fmt`% weighted exact by `{python} r.top_exact_margin_fmt` points (Section 3.1 details the construction). The manuscript snapshot is a `{python} r.n_households_fmt`-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. `{python} abstract_snapshot_summary` Within model families, newer is not always stronger: Claude Opus 4.8 scores `{python} r.opus48_score_fmt`% exact, below the `{python} r.opus47_score_fmt`% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That universe contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not selected or annotated. Reference-suspect flags in the selected universe were adjudicated against primary sources and engine traces, and the frozen release has no known reference-computation defects. The `{python} r.parse_contract_failure_pct_fmt`% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at . +PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine `{python} r.dataset_label` microdataset and scoring answers against reference outputs from policyengine-us `{python} r.policyengine_us_version`. It evaluates `{python} r.n_models_fmt` frontier models on `{python} r.n_households_fmt` households across `{python} r.n_output_groups_fmt` output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: `{python} r.zero_share_pct_fmt`% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of `{python} r.always_zero_exact_fmt`% weighted exact by `{python} r.top_exact_margin_fmt` points (Section 3.1 details the construction). The manuscript snapshot is a `{python} r.n_households_fmt`-household public preview, not a protected held-out leaderboard: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. `{python} abstract_snapshot_summary` Within model families, newer is not always stronger: Claude Opus 4.8 scores `{python} r.opus48_score_fmt`% exact, below the `{python} r.opus47_score_fmt`% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That universe contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on `{python} r.reference_rebuilt_date` with policyengine-us `{python} r.policyengine_us_version`, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the `{python} r.engine_defect_exclusion_count` outputs whose references rest on engine defects the audits found and upstream has not fixed, and the `{python} r.unlisted_input_exclusion_count` whose references depend on an input the prompt does not state. The `{python} r.parse_contract_failure_pct_fmt`% of cells with a missing or unparsable answer remain scored as misses rather than dropped. The live benchmark is available at . ## Introduction @@ -1013,7 +1023,7 @@ The benchmark requires each model response to include numeric answers and one ex Because explanations are required, the canonical task measures policy estimation under a public-facing structured-response contract, not isolated arithmetic accuracy. Prompt fairness is part of the benchmark contract. The current release uses one prompt template per country, with no model-specific tuning. Models receive the same household facts and requested outputs. They receive no web or external tool access. The prompt sets unlisted numeric inputs to `0`, unlisted boolean or status facts to false, and household characteristics as constant over the tax-benefit year. Provider-specific differences are limited to structured-output transport, request shape, and the reasoning setup in @tbl-model-runs. The harness sends no reasoning-control or sampling parameters. It leaves reasoning effort, temperature, and related decoding controls at provider defaults. The frozen GPT-5.5 wave uses provider-default reasoning effort. -The forced `tool_choice` suppresses Claude's extended thinking in this snapshot ([`sensitivity/claude-thinking-2026-08.md`](https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md)). A labeled sensitivity with `tool_choice: "auto"` is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under `tool_choice: "auto"` scores 1.2 points above that row. The next board version moves every model to `tool_choice: "auto"` so each provider's default reasoning posture can engage. +The forced `tool_choice` suppresses Claude's extended thinking in this snapshot ([`sensitivity/claude-thinking-2026-08.md`](https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md)). A labeled sensitivity with `tool_choice: "auto"` is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under `tool_choice: "auto"` scores `{python} r.fable51_auto_uplift_fmt` points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to `tool_choice: "auto"` so each provider's default reasoning posture can engage. Two dimensions on which requests were not identical across models are request shape and answer transport (@tbl-model-runs). `{python} n_chunked_models` of the `{python} r.n_models_fmt` models could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable rather than accommodated. @@ -1031,7 +1041,7 @@ md_table(snapshot_provenance) ```{python} model_runs_caption = ( - "Model run configuration in the frozen manuscript snapshot. Every model answers through the transport its model card records, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (`tool_choice` set to the answer tool) or a JSON object, the latter where the provider rejects a forced tool or where the model card selects JSON. " + "Model run configuration in the frozen manuscript snapshot. Every model answers through its row's transport, with no external tools, provider-default temperature and sampling, no reasoning-control parameters, and the same household facts and requested outputs. Transport is the answer contract: a forced answer-schema tool call (`tool_choice` set to the answer tool) or a JSON object, the latter where the model card or its family default selects JSON, for most such rows because the provider rejects a forced tool call. " + r.serving_evidence_caption + " Rows with an output count answered the template over subsets of the requested outputs, an accommodation that predates the canonical whole-scenario rule described in the text. Released is the first public availability (paid tiers count; trusted-tester previews do not), compiled from vendor announcements and contemporaneous press; grok-build-0.1's date rests on secondary trackers. Leaderboard artifact keys match the final segment of the provider id (qwen-3.7-max adds a hyphen). A dagger marks open-weight models; Kimi K3's weights, announced at its API launch [@willison2026kimik3], shipped on Hugging Face on July 27, 2026 under a custom license [@techtimes2026kimik3weights]. GLM-5.3-Flash (preview) was queried as the OpenRouter stealth listing Ox Alpha (artifact key ox-alpha); Z.ai identified the listing as GLM-5.3-Flash after its run [@openrouter2026oxalpha], and its Released date is its August 20, 2026 OpenRouter listing date." ) @@ -1044,6 +1054,10 @@ md_table(model_runs, caption=model_runs_caption, label="tbl-model-runs") The US benchmark is built from the certified PolicyEngine populace dataset (`{python} r.dataset_id`) using PolicyEngine US. The dataset contains `{python} r.populace_people_fmt` people in `{python} r.populace_households_fmt` households, with household, tax-unit, SPM-unit, family, marital-unit, and person records for PolicyEngine calculations. The sampled households are filtered to keep a single-tax-unit, single-family, single-Supplemental Poverty Measure (SPM)-unit structure with at least one adult and a supported filing status. Of the `{python} r.populace_households_fmt` households, `{python} r.populace_eligible_households_fmt` (`{python} r.populace_eligible_pct_fmt`%) pass the filter and form the eligible draw (this filter-pass rate is coincidentally close to the `{python} r.zero_share_pct_fmt`% zero-output share discussed under the benchmark design; the two figures are unrelated). The `{python} r.populace_excluded_pct_fmt`% excluded by the filter include multi-tax-unit households (e.g., adult roommates), multi-family households, multi-SPM-unit households, and households whose head reports a filing status outside the supported set. These excluded compositions are exactly the kind of cases where federal/state credit allocations and benefit-unit rules become hardest, so the eligible draw is a tractable subset rather than the full distribution of US households. Prompts include nonzero promptable raw inputs across relevant entities rather than a hand-curated summary, so the models see many of the same facts the simulator receives. Filing status is not stated in the prompt; the reference computation infers it from tax-unit role flags. Models therefore see the same household facts that drive the reference filing-status assignment, but they do not receive that assignment as a label. +### Disability in the household facts {#sec-disability} + +The household facts carry one disability fact, while the programs apply several. The certified microdata marks a person as disabled when they report any of the six disability-difficulty items in the Current Population Survey, and the prompt passes that flag through as `is disabled`: `{python} r.disabled_person_count` of the `{python} r.benchmark_person_count` people in the benchmark households carry it. The law defines disability program by program, and policyengine-us follows it with a separate input for each determination. SSI requires its own disability criterion and no substantial gainful activity (42 U.S.C. 1382c(a)(3)). SNAP counts a member as disabled through receipt-based routes, among them receiving SSI or Social Security disability benefits and a qualifying veterans' status (7 U.S.C. 2012(j)); policyengine-us approximates the SSI route with SSI disability status, which does not require receiving SSI. Medicare's disability route before age 65 requires 24 months of entitlement to Social Security disability benefits (42 U.S.C. 426(b)), the route policyengine-us models. The tax code uses permanent and total disability for dependents (26 U.S.C. 152), retirement on disability for the credit for the elderly and disabled (26 U.S.C. 22), and incapacity for self-care for the child and dependent care credit (26 U.S.C. 21). The general flag enters none of these determinations; in the reference engine it reaches mainly state provisions, and in SNAP only the work-requirement and student rules. One of those state provisions is California's Working Disabled Program, a Medi-Cal pathway for people with disabilities who work: the program uses SSI's definition of disability (42 CFR 435.540(a)), and policyengine-us tests the general flag. No benchmark person carries any of the program-specific inputs. The certified build never sets SSI's disability criterion, and the months of Social Security disability receipt are integer-typed, which the prompt builder does not expose. The references of a person the prompt calls disabled therefore take the non-disabled path through each of these determinations unless another listed fact establishes it: listed Social Security disability income, for example, makes a person disabled for SNAP. A careful reader of `is disabled` could take either path, so wherever the other reading moves a reference, the output is excluded from scoring for every model (@sec-reference-credibility): SSI and SNAP outputs under SSI's disability criterion, the SNAP output whose engine defect turns on whether its head is a disabled member, and one California Medicaid output, which the other reading moves once the engine's Working Disabled Program test reads SSI's definition. Separately, the prompt lists Social Security disability income but not how long it has been received, so the Medicare eligibility of the five household heads with that income, which turns on 24 months of it, is excluded whether or not the prompt calls them disabled. Where the other reading moves nothing, the output stays scored. The next scenario refresh defines the prompt's disability fact and states the program-specific determinations as separate facts (PolicyEngine/policybench#165, PolicyEngine/microcosm#876). + The current US release requests `{python} r.n_output_groups_fmt` scored output groups spanning federal income tax, refundable credits, payroll and self-employment tax, state and local income tax, Supplemental Nutrition Assistance Program (SNAP), Supplemental Security Income (SSI), Temporary Assistance for Needy Families (TANF), school-meal eligibility, and person-level coverage eligibility for the Special Supplemental Nutrition Program for Women, Infants, and Children (WIC), Medicaid, the Children's Health Insurance Program (CHIP), Medicare, Head Start, and Early Head Start; several of these categories span two scored groups each (federal and state refundable credits, payroll and self-employment tax, state and local income tax, free and reduced-price school meals), which is how the named categories reconcile to the `{python} r.n_output_groups_fmt` output groups. The source run also requested the Affordable Care Act (ACA) Premium Tax Credit (PTC), but explanation audits showed that the prompt could be misleading when households lacked plan-specific Marketplace information. We therefore preserve those raw responses but exclude PTC from the canonical scored leaderboard until the prompt contract is revised. The output scope is intentionally narrower than the full PolicyEngine model. @tbl-scope-rationale summarizes the inclusion rule. The benchmark asks for WIC eligibility rather than a WIC dollar amount; WIC dollar values are used only as impact-weight proxies for coverage flags, not as requested model outputs. @@ -1054,19 +1068,27 @@ The output scope is intentionally narrower than the full PolicyEngine model. @tb md_table(scope_rationale) ``` -### Reference-output credibility +### Reference-output credibility {#sec-reference-credibility} PolicyBench treats PolicyEngine outputs as benchmark reference outputs, not as administrative records. The reference source is nevertheless stronger than an ad hoc answer key: PolicyEngine is open source, used for household calculators and reform analysis, and externally checked in specific domains. No. 10 Downing Street's data science team adapted PolicyEngine's open-source microsimulation model for experimental policy simulation, with validation against external projections before use [@woodruff2026no10; @policyengine2026downing]. In the US, @policyengine2024statetax reports matching the National Bureau of Economic Research (NBER) TAXSIM-35 model [@feenberg1993taxsim] to the cent on the vast majority of cases for the 2021 tax year across hundreds of thousands of tax units per state, with state-specific differences documented in the integration tests. We do not restate that comparison as a single percentage because the published source uses qualitative phrasing rather than a headline accuracy number. PolicyEngine has also signed a memorandum of understanding [@policyengine2025atlantafed] with the Federal Reserve Bank of Atlanta for future validation work against its Policy Rules Database [-@atlantafed2026prd]. The Atlanta Fed sources are a caveat rather than evidence of completed validation for this benchmark: they document planned collaboration and the comparison source, not finished checks of the frozen PolicyBench outputs. Taken together, these sources support using PolicyEngine as a transparent reference implementation with partial external validation, but they do not validate every benchmark output. -This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and three audit waves, the model-assisted review covered `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That selected universe contains `{python} r.annotated_exact_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine's computation traces. Reviewers raised 35 reference-suspect flags in total. Thirty-one dissolved under adjudication, with the reviewer's hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot's reference outputs are generated with the fixed engine, policyengine-us `{python} r.policyengine_us_version`. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. The asymmetry of this record is itself evidence about the reference: even among the cells the reviewing models found most suspicious, 31 of 35 flags (89%) resolved to model-side errors, set against thousands of confirmed model errors overall and four reference defects. @tbl-reference-review summarizes the reviewed discrepancy classes and outcomes; no unresolved reference-computation defect is known in the frozen snapshot. +This does not make PolicyEngine infallible, so we treated the reference outputs as claims to be attacked rather than assumptions. Across development and each later audit, the model-assisted review covered `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule`. That selected universe contains `{python} r.annotated_exact_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits; `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100 were not annotated. Reviewers adjudicated each reference-suspect flag by hand against primary sources, including statutes, regulations, and published program parameters, and against the engine's computation traces. Before the September 22 wave, reviewers had raised 35 reference-suspect flags. Thirty-one dissolved under adjudication, with the reviewer's hypothesis failing against a primary source (stale thresholds, missed program mergers, missed federal-conformity elections, misread eligibility-income definitions) or against the engine trace. Four were genuine: two Head Start eligibility flags and one New Jersey state income tax cell reflected policyengine-us defects (an age-window bound and a filing-threshold floor the benchmarked path bypassed), and one SNAP cell rested on a scenario whose encoded inputs did not match the disability status its prompt implied. The engine defects were fixed upstream (policyengine-us #8845 and #8846, plus #8844, found in the same review but affecting no benchmark cell), and this snapshot's scored reference outputs come from the fixed engine, policyengine-us `{python} r.policyengine_us_version`. The mis-encoded scenario is disclosed under Limitations pending the next scenario refresh. @tbl-reference-review summarizes the reviewed discrepancy classes and outcomes. + +The audit wave run with the September 22 additions used a stronger judge, Claude Opus 5.5, and it found what the earlier waves had not. The judge flagged `{python} r.audit_flagged_case_count` references. Adjudication against primary sources and sandbox runs of the reference engine affirmed `{python} r.audit_flagged_count("affirmed")` of them, traced `{python} r.audit_flagged_count("unlisted_input")` to inputs the prompt never lists, replaced `{python} r.audit_flagged_count("regenerated")` with a regenerated reference, and confirmed `{python} r.audit_flagged_count("engine_defect")` as defects in policyengine-us `{python} r.previous_policyengine_us_version`, the version behind the references then. The judge's stated hypothesis was not always the defect: a Wisconsin flag blamed a missing retirement exclusion that the engine does apply, on a path the benchmarked output skips, and a Montana flag cited a child tax credit that was never enacted, while the reference was wrong for another reason. The judge also sees only the cases a new model joins, so its flags sample each defect rather than bound it. Each root cause's rule was therefore implemented as a sandbox fix on the same engine version and every reference recomputed under it. Of those root causes, `{python} r.upstream_fixed_root_cause_count` had been fixed in policyengine-us after the references were frozen (`{python} r.upstream_fix_prs_fmt`): capital gain distributions, New York's renter cap, the CalEITC's lookup at adjusted gross income, and the SNAP rounding and SNAP child support option described below. For the September 22 references, PolicyBench applied each fix on that engine version to the scored references it moves, `{python} r.regenerated_by_upstream_fix_count` in all, and policyengine-us `{python} r.policyengine_us_version` contains all `{python} r.upstream_fixed_root_cause_count` fixes. An output one of them moves that an unfixed cause also moves stays excluded. The capital gain fix as first built also extended Wisconsin's 30 percent capital gain exclusion (Wis. Stat. 71.05(6)(b)9) to the distributions. The upstream fix does not, so that part is recorded as its own defect, not fixed upstream, measured on top of the upstream fix and Wisconsin's published amounts, and the two outputs it moves are excluded (one of them already excluded for another Wisconsin defect). Every output a defect not yet fixed upstream moves by more than a dollar is excluded from scoring for every model: `{python} r.engine_defect_exclusion_count` outputs in `{python} r.excluded_output_households_by_reason["reference_engine_defect"]` households are recorded as engine-defect exclusions across `{python} r.engine_defect_root_cause_count` root causes, and `{python} r.engine_defect_unflagged_count` of them had never been flagged (an output such a defect moves that was already excluded for an unlisted input keeps that record). They stay excluded until references built on a fixed engine replace them. A second model, GPT-6 Astra, then re-derived the sandbox fixes the first pass built from primary sources and reran their sweeps. It narrowed several fixes to the benchmark year and filers they reach, reclassified two root causes as unlisted inputs (the type of a household's survivor benefits, and whether listed mortgage interest is on the home the household occupies), and found that a California credit move came from a separate defect: the engine never looks the CalEITC up a second time at adjusted gross income, as the credit worksheet requires. + +The same wave tested a failure mode no judge flagged. policyengine-us fills a parameter's values past its last encoded year by projecting it with a price index, so a 2026 reference can rest on a forecast rather than on the amount a government published. Every reference was recomputed with each projected 2026 value held at its last encoded value, and each projected amount behind a moved output was then sourced. Where the government published the 2026 amount before the freeze, the reference takes it: Minnesota's, Michigan's and Missouri's brackets, deductions and exemptions, and Maryland's withholding allowance, which reaches federal tax through the state and local tax deduction. Wisconsin had published its 2026 brackets and deductions too, but every Wisconsin output they move is excluded for an engine defect, so they enter only those outputs' corrected values. Where no 2026 amount published before the freeze was found, the reference takes the last amount published: California's indexed amounts, whose 2026 factor rests on June 2026 prices; the IRS optional sales tax tables, whose 2026 edition had not been found by September 22; Idaho's zero-rate threshold; Maryland's return deduction; and SNAP's FY2027 figures, published on 2026-08-21. Illinois's exemption and Minnesota's child credit projections matched the published amounts. These corrections change parameter values and no rule, so the affected references were regenerated with the same engine version rather than excluded: `{python} r.regenerated_by_convention_count` references, most of them SNAP. With the upstream fixes, the September 22 release regenerated `{python} r.regenerated_reference_count` references in `{python} r.regenerated_reference_household_count` households, `{python} r.regenerated_snap_reference_count` of them SNAP. The reference sidecar records each regenerated value with the fix that produced it, and the fixes, the recomputation of every reference under each, and the verification reports are committed with the snapshot (`reference_audit/2026-09-22`). + +The SNAP convention as first built also corrected the engine's SNAP arithmetic, and an independent review of the snapshot separated those corrections from the parameter holds. They are engine defects, each fixed in policyengine-us after the freeze (#9162 and #9318), so the SNAP references apply the fixes: the engine kept cents in the allotment, where 7 CFR 273.10(e)(2)(ii)(A) requires a whole-dollar allotment; it returned a minimum benefit of $23.84 a month, where 7 CFR 273.10(e)(2)(ii)(C) rounds 8 percent of the one-person maximum to the nearest dollar ($24); it floored net income, which 7 CFR 273.10(e)(1)(ii) requires a state to round to the nearest dollar at each step or to calculate by its TANF procedure, which may keep cents; and it compared income with unrounded income standards. Given the engine's household size, eligibility and floored net income, the Axiom rules engine's encoding of 7 U.S.C. 2017(a) reproduces the whole-dollar allotment of all 19 households the first SNAP convention module changed, in each month from January to September (171 of 171 monthly comparisons), and the federal minimum benefit; it did not test New Jersey's state minimum, and it covers the allotment rounding, not the net income rounding the published references also apply. Recomputing SNAP with net income's cents kept, rather than rounded to the nearest dollar as the upstream fix does, moves no scored reference by more than the dollar tolerance. One SNAP defect is not fixed upstream: the engine still grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 section 10103 ended in July 2025, and the scored output it moves is excluded. That household's prompt calls its head disabled but states none of the routes by which 7 U.S.C. 2012(j) defines a disabled member, such as receiving SSI or Social Security disability benefits; read the other way, the allowance would stand, and so would the reference the publication conventions and upstream SNAP fixes give ($3,576, which the output would carry if scored), so the output is excluded on either reading. One more SNAP defect was found after the September 22 release and fixed upstream on 2026-09-24: the engine's values for the SNAP child support option carried the opposite meaning from the one its formulas read. A state may exclude legally obligated child support paid from gross income (7 CFR 273.9(c)(17)); a state that does not must deduct it from net income (273.9(d)(5)). For 2026 the engine excluded it in 37 jurisdictions that USDA's SNAP State Options Report lists as deducting it, Michigan among them, and deducted it in 9 of the 14 it lists as excluding it. Recomputed under the fix (policyengine-us #9586), one reference moves: a one-person Michigan household that pays child support stayed within the 200 percent gross limit of Michigan's broad-based categorical eligibility only because the engine excluded the payments, so the engine granted it the $24 monthly minimum; counting the payments, as Michigan does, leaves it ineligible, and the regenerated reference is $0. + +PolicyBench rebuilt the references on `{python} r.reference_rebuilt_date` with policyengine-us `{python} r.policyengine_us_version`, the newest release when it began sweeping the references that day (uploaded `{python} r.reference_engine_uploaded_utc` UTC), in place of `{python} r.previous_policyengine_us_version`. It computes each scored reference with `policyengine_us.Simulation`. The snapshot also records policyengine.py `{python} r.policyengine_version` for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside `{python} r.policyengine_us_version`. PolicyBench ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA's FY2027 figures as published values. policyengine-us `{python} r.policyengine_us_version` counts Maryland county income tax in its state income tax. PolicyBench's state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP's work rules read, so the scenario builder also passes each prompt's stated usual weekly hours under that input's name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes `{python} r.engine_upgrade_scored_change_count` scored references (@tbl-reference-review). Two take law published before the freeze that policyengine-us `{python} r.previous_policyengine_us_version` did not encode: New Jersey's child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona's broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York's Empire State child credit phase-out. The investigation found, and PolicyBench excluded, `{python} r.engine_upgrade_new_exclusion_count` federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household's excluded SNAP output also flagged a scored output the move did not change: the head's Medicaid eligibility. The head's income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California's Working Disabled Program requires SSI's definition of disability, the unlisted input that already excludes the household's SNAP. policyengine-us `{python} r.policyengine_us_version` tests the general disability flag there and gives 1 under either reading; with the program's test reading SSI's definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another `{python} r.engine_upgrade_within_tolerance_count` references move by less than the dollar tolerance. The `{python} r.excluded_output_count` excluded outputs keep the values they were decided on (`{python} r.excluded_outputs_on_previous_engine_count` computed with policyengine-us `{python} r.previous_policyengine_us_version`, `{python} r.excluded_outputs_on_reference_engine_count` with `{python} r.policyengine_us_version`), and PolicyBench re-reviewed the `{python} r.engine_upgrade_rechecked_count` of them that move on `{python} r.policyengine_us_version`; all stay excluded. policyengine-us `{python} r.publication_check_policyengine_us_version`, the newest release when PolicyBench checked PyPI on `{python} r.publication_check_pypi_read_date` at `{python} r.publication_check_pypi_read_utc` UTC, gives the same value as `{python} r.policyengine_us_version` for all `{python} r.total_outputs_per_model_fmt` outputs under the same conventions and adapter. The reference sidecar lists every change, and `reference_audit/2026-09-28` holds the recomputation of every output, the investigations and the reviews. The audit tooling itself required hardening, which we report because it changes how much the annotations can be trusted. The derivation narratives displayed alongside reference values are generated from engine computation traces. After earlier model-written narratives were found to misattribute eligibility pathways, mechanical validators now enforce that each narrative states the engine's actual mechanism — for example, that a child ineligible for CHIP because they qualify for Medicaid is never described as failing a CHIP income test. The failure-audit judge reads these grounded narratives together with engine facts and must return a per-model diagnosis naming the specific rule or computation the model missed. It is barred from re-adjudicating the reference in prose: hedged or reference-adjudicating verdicts are mechanically rejected and re-judged, and genuine reference doubt is routed to a structured flag that triggers the manual adjudication described above. -After freezing the snapshot and completing response-contract repairs, we annotated the `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule` and their scenario-output cases. @tbl-deviation-audit reports the final classifications for that legacy-threshold audit universe. It contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits. The `{python} r.unannotated_below_full_bounded_score_count_fmt` other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: `llm_error` (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or `parse_contract_failure` (a missing or unparsable answer, classified deterministically). A third class, `prompt_ambiguity`, is reserved for outputs the snapshot does not score: `{python} r.excluded_output_phrase` in `{python} r.excluded_output_households_phrase`, whose reference depends on an engine input the certified household data never carried (the SSI disability criterion, and months of SSDI receipt) and the prompt therefore never listed. For each, the reference was recomputed with the engine version that produced it under the reading a careful reader could take of the stated facts, and it moved; the output is removed from scoring for every model, so no model gains or loses from it, and of the `{python} r.excluded_output_annotation_row_count_fmt` annotated rows on those outputs, `{python} r.prompt_ambiguity_row_count_fmt` carry `prompt_ambiguity` as description rather than as a scored class and the rest are answers that never parsed. Every model is therefore scored on `{python} r.scored_outputs_per_model_fmt` of its `{python} r.total_outputs_per_model_fmt` requested outputs. The exclusion record (`reference_exclusions.json`) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records `{python} r.audit_adjudicated_case_phrase` of developer adjudication, kept beside the annotations with the judge's verdict and the reasoning. Two judge models produced the verdicts, and both are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions, and Claude Opus 5, called through the Claude Code CLI, judged every case a September 2026 addition joined (`{python} r.audit_opus_judged_case_count_fmt` of `{python} r.audit_case_count_fmt` cases, each recorded in a per-case provenance sidecar in the audit tree). Judge verdicts classify misses after scoring and change no score. The two unlisted inputs behind the exclusions are the SSI disability criterion (`meets_ssi_disability_criteria`), false for every person in the certified June 2026 microdata build, and months of SSDI receipt (`months_receiving_social_security_disability`), never carried by the data and integer-typed, which the prompt builder does not expose. The prompt lists a general `is disabled` fact, drawn from the six CPS disability-difficulty items, and never a program-specific determination. The exclusion set was derived mechanically rather than from which models missed: every reference was recomputed under each alternative reading and every output that moved was excluded, which is why it includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). +After freezing the snapshot and completing response-contract repairs, we annotated the `{python} r.audit_annotated_row_count_fmt` `{python} r.audit_selection_rule` and their scenario-output cases. @tbl-deviation-audit reports the final classifications for that legacy-threshold audit universe. It contains `{python} r.annotated_exact_miss_count_fmt` of the snapshot's `{python} r.exact_match_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits. The `{python} r.unannotated_below_full_bounded_score_count_fmt` other rows with bounded score below 100 fall outside the selection and have no audit annotation. Each annotated row on a scored output resolves to one of two final classes: `llm_error` (a substantive miscalculation, with a per-model diagnosis naming the rule or computation the model missed) or `parse_contract_failure` (a missing or unparsable answer, classified deterministically). Two further classes describe outputs the snapshot does not score, and each such output is removed from scoring for every model, so no model gains or loses from it. `prompt_ambiguity` marks `{python} r.unlisted_input_exclusion_count` outputs whose reference depends on an input the prompt never lists, so a careful reader could take the stated facts either way; for each, the reference was recomputed under the other reading with the engine version that produced it, and it moved (the California Medicaid output once the engine's Working Disabled Program test read SSI's definition, as the program does). `reference_engine_defect` marks the `{python} r.engine_defect_exclusion_count` outputs where that engine misapplies the law on stated facts. Together they remove `{python} r.excluded_output_phrase` in `{python} r.excluded_output_households_phrase`, and of the `{python} r.excluded_output_annotation_row_count_fmt` annotated rows on those outputs, `{python} f"{r.excluded_descriptive_row_count:,}"` carry one of these classes as description rather than as a scored class and the rest are answers that never parsed. A scored reference also follows from law published before the references were frozen on 2026-07-03. USDA published the FY2027 SNAP cost-of-living figures on 2026-08-21, after the freeze and after most rows were answered, so grading any model on them would score it against law it could not have seen; SNAP's October to December months therefore hold the FY2026 figures, and the publication conventions described above apply the same rule to every projected amount. Every model is therefore scored on `{python} r.scored_outputs_per_model_fmt` of its `{python} r.total_outputs_per_model_fmt` requested outputs. The exclusion record (`reference_exclusions.json`) travels with the frozen run and the manifest pins it. No scored row remains classified as a prompt ambiguity, unresolved reference issue, or needs-review item, and zero annotated rows carry a standing reference-suspect flag. The audit pipeline is model-assisted, with developer adjudication of each reference-suspect or prompt-ambiguity flag rather than independent external validation; this snapshot records `{python} r.audit_adjudicated_case_phrase` of developer adjudication, kept beside the annotations with the judge's verdict and the reasoning. Three judge models produced the verdicts, and all are board rows: GPT-5.6 Sol, called through the Codex CLI, judged the cases whose verdicts predate the September 2026 additions; Claude Opus 5, called through the Claude Code CLI, judged the `{python} r.audit_opus_judged_case_count_fmt` cases the September 5 additions joined that no later judge re-judged; and Claude Opus 5.5 judged the cases the September 22, September 29 and September 30 additions joined or a reference revision changed (`{python} r.audit_opus55_judged_case_count_fmt` of `{python} r.audit_case_count_fmt` cases), its own row's cases included. Each verdict is recorded in a per-case provenance sidecar in the audit tree. Judge verdicts classify misses after scoring and change no score, and every reference flag a judge raised is settled by the developer adjudication above. The unlisted inputs behind the unlisted-input exclusions are the SSI disability criterion (`meets_ssi_disability_criteria`), false for every person in the certified June 2026 microdata build; months of SSDI receipt (`months_receiving_social_security_disability`), never carried by the data and integer-typed, which the prompt builder does not expose; weekly hours worked (`weekly_hours_worked_before_lsr`), which policyengine-us `{python} r.previous_policyengine_us_version` defaulted to 40 where the prompt treats unlisted numbers as zero; whether survivor benefits other than Social Security are a pension or annuity, which the prompt does not say and which decides both their taxability and whether they count toward Wisconsin's homestead credit income; who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable; whether listed home mortgage interest is on the home the household occupies, which decides whether SNAP counts it as a shelter cost; whether an adult tax dependent is the claiming filers' child, which decides whose income counts in that dependent's Medicaid household; whether the tax behind a listed state and local tax refund reduced federal tax in the year the household paid it, which decides how much of the refund is income; and whether listed interest comes from a Massachusetts bank deposit, which moves only an output already recorded as an engine-defect exclusion. One definition was also open: the output definition of federal income tax before refundable credits does not say whether it includes the net investment income tax, which the reference adds and Form 1040 reports after its line for tax after nonrefundable credits, so the outputs that tax moves are excluded as well. The prompt lists a general `is disabled` fact and never a program-specific determination (@sec-disability). Every exclusion but the California Medicaid output was derived mechanically rather than from which models missed: each is an output that a sweep of every reference, under an alternative reading or an engine defect's sandbox fix, moved on policyengine-us `{python} r.previous_policyengine_us_version` or, for the state and local tax refund reading, on `{python} r.policyengine_us_version`. On `{python} r.previous_policyengine_us_version`, PolicyBench excluded every output an alternative reading of the September 22 audit moved and every output a defect not fixed upstream moved by more than a dollar, which is why the set includes three SNAP outputs that move through the elderly-or-disabled SNAP rules (in two of those households SSI itself stays at zero; in the third an aged member already receives SSI and the disabled member would add a second benefit). On `{python} r.policyengine_us_version`, PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of those sweeps (the IRA deduction limit fix, the net investment income tax definition, and the readings for mortgage residence and 40 unlisted weekly hours) and ran a new one for the state and local tax refund reading. Beyond the one-dollar exact-match tolerance, the new sweep moves `{python} r.rerun_sweep_new_excluded_count_word` previously scored federal income tax outputs, and PolicyBench excluded them. Set against the same `{python} r.policyengine_us_version` calculation without its fix or reading, no sweep moves a scored output by more than the tolerance, and `{python} r.rerun_sweep_scored_within_tolerance_count_word` scored outputs move by less. PolicyBench did not re-run the other sweeps on `{python} r.policyengine_us_version`. On `{python} r.policyengine_us_version`, the California Medicaid output stays at 1 under either reading of SSI's disability criterion, because the engine's Working Disabled Program test reads the general flag in its place. PolicyBench found that output when it re-reviewed the household's excluded SNAP output, and excluded it because the program itself requires SSI's definition of disability. ```{python} #| label: tbl-reference-review -#| tbl-cap: "Development discrepancy review before the frozen snapshot." +#| tbl-cap: "Discrepancy review across the audit waves." md_table(reference_audit) ``` @@ -1100,11 +1122,11 @@ Accuracy is one axis; the price and speed of a no-tools answer are another. @tbl ```{python} #| label: tbl-us-cost-latency -#| tbl-cap: "Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: provider-reported where the provider returns one, otherwise reconstructed at the configured list price at request time. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household's summed request time. A dash marks usage or wall-clock latency that was not measured per call." -cost_latency +#| tbl-cap: "Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household's summed request time. A dash marks usage or wall-clock latency that was not measured per call." +md_table(cost_latency) ``` -Cost preserves each call's recorded total, including provider-reported charges where available and request-time reconstructions otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5's usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. +Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5's usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. ### Simple baselines @@ -1232,7 +1254,9 @@ PolicyBench is not a substitute for a production tax-and-benefit calculator. Sev - the public scenario explorer exposes the current test set and reference outputs, so open-set leakage is a prominent limitation rather than a minor implementation detail - the `{python} r.n_households_fmt`-household manuscript snapshot should be treated as a preview until larger frozen runs are published - the annotations cover the `{python} r.audit_annotated_row_count_fmt`-row legacy-threshold audit universe, including `{python} r.annotated_exact_miss_count_fmt` exact-match misses and `{python} r.annotated_exact_hit_count_fmt` exact hits, but not `{python} r.unannotated_below_full_bounded_score_count_fmt` additional rows with bounded score below 100; the review is developer-adjudicated and not an independent validation of each reference value -- one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (SSI disability status and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address; affected cells score models against the encoded inputs +- one frozen scenario encodes household inputs that do not match its prompt narrative (a South Carolina household whose prompt implies a qualifying disability the inputs omit), and two input-semantics conventions (disability, which the prompt states as one general fact where each program applies its own determination, and adult tax-dependent relationships) are under-specified in ways a scheduled scenario refresh will address (@sec-disability); outputs whose reference moves under the other reading of either convention are excluded, while the mis-encoded scenario's cells score models against its encoded inputs +- the reference engine carries known defects: `{python} r.engine_defect_exclusion_count` outputs, recorded as engine-defect exclusions, rest on `{python} r.engine_defect_root_cause_count` root causes that the audit found in policyengine-us `{python} r.previous_policyengine_us_version` and that `{python} r.policyengine_us_version` does not fix, and are excluded until a reference version built on a fixed engine restores them; the judge that finds defects reviews only the cases a new model joins, the engine-upgrade review covered only the outputs the upgrade moved, and on `{python} r.policyengine_us_version` PolicyBench re-ran `{python} r.rerun_sweep_september_22_count_word` of the September 22 alternative-reading and defect sweeps rather than all of them, so defects that no reviewed case, move or re-run sweep exposes can remain +- the references were frozen on 2026-07-03 for tax year 2026, so amounts published later reach the benchmark only through a stated convention: where policyengine-us projected a 2026 amount, the reference takes the amount published before the freeze or, failing that, the last one published (SNAP's FY2026 figures rather than USDA's FY2027 figures of 2026-08-21; California's 2025 indexed amounts; the IRS's 2025 sales tax tables), so a model that knew later law is scored against the earlier law, and the projection audit sourced only the amounts whose projection moved an output; law published before the freeze reaches a reference only once policyengine-us encodes it, and policyengine-us encoded New Jersey's 2026 child tax credit schedule and Arizona's SNAP categorical eligibility limit only after `{python} r.previous_policyengine_us_version` - benchmark success should not be interpreted as policy-advice readiness The current paper is therefore an evaluation of model performance under a specific structured-output benchmark, not a general certification of tax or benefit competence. diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json index eaca160a..97b18dde 100644 --- a/paper/snapshot/20260501/manifest.json +++ b/paper/snapshot/20260501/manifest.json @@ -2,63 +2,99 @@ "audit_annotation_artifacts": { "developer_adjudications": { "by_judge_verdict": { - "llm_error": 10, - "prompt_ambiguity": 1 + "llm_error": 59, + "reference_data_issue_fixed": 1, + "reference_engine_defect": 3, + "reference_model_issue_fixed": 6 }, - "cases": 11, + "by_reference_verdict": { + "affirmed": 5, + "engine_defect": 28, + "regenerated": 6, + "unlisted_input": 20 + }, + "cases": 69, "file": "us_adjudications.json", - "note": "Judge verdicts outside the final classes (llm_error, parse_contract_failure) are resolved by a recorded developer adjudication that keeps the judge's verdict and reasoning beside the adjudicated class; applied to the bundle before export so the published payload and the frozen annotations agree." + "judge_flagged_by_reference_verdict": { + "affirmed": 5, + "engine_defect": 20, + "regenerated": 6, + "unlisted_input": 8 + }, + "note": "Judge verdicts outside the final classes (llm_error, parse_contract_failure) are resolved by a recorded developer adjudication that keeps the judge's verdict and reasoning beside the adjudicated class; applied to the bundle before export so the published payload and the frozen annotations agree. Every case the judge flagged reference-suspect carries a reference verdict (affirmed, regenerated, engine_defect, unlisted_input, later_law) that clears the flag; judge_reference_suspect records which entries the judge flagged." }, "files": { - "us_adjudications.json": "73dca1c2cef9d61bc7ab14965974569e5dc3fca10934b87aafaf4cb0674f9fc6", - "us_audit_row_annotations.csv": "735b75d5a6599c8375a97f000b93c1904d6f27588480300bee0ab22430b893ef", - "us_case_notes.csv": "0c62ab95ea62c01a0c4044b8346b9280e151b1498e92bf38e84d4afee338fa4d", - "us_case_reference_explanations.csv": "a3fc7504dcf3d8abe11e98c2228c786a5171a825640b980835e1e210d3afa198" + "us_adjudications.json": "42d6a90fe402b6dc8a37422999ea05cd0e481a5ead690ee4045f649d682270c6", + "us_audit_row_annotations.csv": "3b27bfa643a5725935e90bea39f2584dc507572418a8e1474e981b841078d3ad", + "us_case_notes.csv": "2bef75c043ec08338038276e018f300ed86fcb937c44ee26d0daadb58d7f2c08", + "us_case_reference_explanations.csv": "6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4" }, "judge_provenance": { "by_judge": { "claude-opus-5": { - "cases": 350, + "cases": 116, "judged_on_utc": [ "2026-09-05" ], "runner": "Claude Code CLI (scripts/run_audit_claude.sh)" }, + "claude-opus-5-5": { + "cases": 260, + "judged_on_utc": [ + "2026-09-22", + "2026-09-23", + "2026-09-24", + "2026-09-29", + "2026-09-30" + ], + "runner": "Claude Code CLI (scripts/run_audit_claude.sh) and Claude Code Workflow subagents (results/local/adds202609/judge_stages.py)" + }, "gpt-5.6-sol": { - "cases": 318, - "judged_on_utc": [], + "cases": 298, + "judged_on_utc": [ + "2026-08-18", + "2026-08-22", + "2026-09-02" + ], "runner": "Codex CLI (scripts/run_audit_codex.sh)" } }, - "cases_judged": 668, - "note": "Judge model per case: a verdict.meta.json sidecar (Claude Code runner) or the codex.log model header (Codex runner) in the audit tree. Verdicts classify misses after scoring and change no score. Both judge models are also board rows." + "cases_judged": 674, + "note": "Judge model per case: the verdict.meta.json sidecar bound to the case's verdict by sha256 (either runner), else, for a case with no sidecar, the codex.log model header when the log was written alongside the verdict (Codex runner before it wrote sidecars). Verdicts classify misses after scoring and change no score. Every judge model is also a board row." }, - "note": "Model-assisted, developer-adjudicated row and case audit annotations for the 9,076 prediction rows selected because their legacy threshold score was below 1. This audit universe is not identical to the exact-match or bounded-score miss sets. The annotations were produced under the decisive-diagnosis contract (per-model diagnoses grounded in engine facts; hedged verdicts mechanically rejected and re-judged). Row-level failure_source values are llm_error for substantive misses and parse_contract_failure for missing or unparseable answers; zero rows are reference-suspect. Case notes are stored as us_case_notes.csv with case_failure_sources / case_failure_subtypes columns. Reference narratives the judge and dashboard display are frozen as us_case_reference_explanations.csv. Judge provenance per case is tallied under judge_provenance.", + "note": "Model-assisted, developer-adjudicated row and case audit annotations for the 9,971 prediction rows selected because their legacy threshold score was below 1. This audit universe is not identical to the exact-match or bounded-score miss sets. The annotations were produced under the decisive-diagnosis contract (per-model diagnoses grounded in engine facts; hedged verdicts mechanically rejected and re-judged). Row-level failure_source values are llm_error for substantive misses and parse_contract_failure for missing or unparseable answers; zero rows are reference-suspect. Case notes are stored as us_case_notes.csv with case_failure_sources / case_failure_subtypes columns. Reference narratives the judge and dashboard display are frozen as us_case_reference_explanations.csv. Judge provenance per case is tallied under judge_provenance.", "path": "annotations/us_full_run_20260612_policyengine_4_16_1_populace" }, "committed_snapshot_artifacts": { - "model_serving_config.json": "5e56b2bfc6df00fd9b086bac71073980c513d91cbf2e313afce92a75ac904550", - "us_impact_summary_by_model.csv": "540f5e57aef1f065560e9c6cd53527e0e2ab9832f74f1581187e4a098725de69", - "us_reference_outputs.csv": "b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368", + "model_serving_config.json": "019e0b9f2987102b49f82b238ca30568cf403c6931444742f4a975a471d1da15", + "us_impact_summary_by_model.csv": "e6e034adee408cc798897bc0825ace9740055ab20e528ce842bd06e2c81e0922", + "us_reference_outputs.csv": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", "us_scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a" }, - "description": "The 2026-09-05 scored manuscript snapshot reports the household-impact-weighted exact-match rate as its headline metric.", + "description": "The 2026-09-30 scored manuscript snapshot reports the household-impact-weighted exact-match rate as its headline metric.", "files": [ { "path": "runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz", - "sha256": "2cce2598dbf9f33511f31d944221a7fc27f69b487a56c59e2c17e77d4ba5ad1f" + "sha256": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" } ], + "household_dataset": { + "policyengine_us_data_artifact_sha256": "f32c2e5e9098bc6540724fdd5debf963af495da4c29b3a7a63fb53c2a4bb5a34", + "policyengine_us_data_build_id": "populace-us-2024-5da5a95-20260611", + "policyengine_us_dataset": "populace_us_2024", + "policyengine_us_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-5da5a95-20260611", + "source": "runs/us_full_run_20260612_policyengine_4_16_1_populace/scenarios.csv.meta.json" + }, "live_dashboard_artifact": { "asset": "dashboard-data.json", - "bytes": 109225250, + "bytes": 126109700, "derivation": "At freeze time the live artifact equals the frozen published_dashboard_artifact: the combined export of the source-run data.json listed under source_run_artifacts. Annotation-class republishes may advance this entry ahead of the frozen pin without changing any score.", - "sha256": "838bb3757db372fc473daf717616c1faea9254ecadfd1581d6217b1c796890a6", - "tag": "dashboard-data-20260905c", - "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260905c/dashboard-data.json" + "sha256": "d1cae7456cf91ab6fa04644a4d5d570e359522386a7c52f9645d5923bfc11258", + "tag": "dashboard-data-20260930", + "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260930/dashboard-data.json" }, "live_dashboard_note": "The live dashboard payload is a published release asset; the committed pointer app/src/data.artifact.json must reference the artifact pinned under live_dashboard_artifact. The separate published_dashboard_artifact freezes the combined export of the source run data.json files listed under source_run_artifacts. A later publication may advance the live entry without changing the frozen pin.", - "model_response_date": "2026-06-12 to 2026-09-05", + "model_response_date": "2026-06-12 to 2026-09-30", "policy_period": { "us": "tax year 2026" }, @@ -69,43 +105,68 @@ }, "published_dashboard_artifact": { "asset": "dashboard-data.json", - "bytes": 109225250, - "sha256": "838bb3757db372fc473daf717616c1faea9254ecadfd1581d6217b1c796890a6", - "tag": "dashboard-data-20260905c", - "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260905c/dashboard-data.json" + "bytes": 126109700, + "sha256": "d1cae7456cf91ab6fa04644a4d5d570e359522386a7c52f9645d5923bfc11258", + "tag": "dashboard-data-20260930", + "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260930/dashboard-data.json" }, "reference_exclusions": { + "by_engine_defect_root_cause": { + "r01_ira_compensation": 3, + "r02_ira_219g": 7, + "r02_ira_219g+r03_estate_income": 2, + "r02_ira_219g+r11_ca_itemized_conformity": 2, + "r03_estate_income": 2, + "r05_nj_worker_ui": 1, + "r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions": 1, + "r07_idaho_health_premiums": 2, + "r08_eitc_earned_income_deferrals": 4, + "r11_ca_itemized_conformity": 1, + "r22_ma_part_a_loss_offset": 1, + "r30_snap_heat_and_eat_sua": 1, + "r32_wi_capital_gain_distributions": 1 + }, "by_unlisted_input": { - "meets_ssi_disability_criteria": 6, - "months_receiving_social_security_disability": 5 + "meets_ssi_disability_criteria": 7, + "months_receiving_social_security_disability": 5, + "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits": 1, + "taxability of survivor_benefits": 1, + "the relationship of an adult tax dependent to the filers who claim them": 3, + "weekly_hours_worked_before_lsr": 1, + "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies": 1, + "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say": 1, + "whether the listed home mortgage interest is on the home the SNAP household occupies": 1, + "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)": 3, + "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable": 4 }, "file": "reference_exclusions.json", - "note": "Outputs whose reference depends on an engine input the certified household data never carried (so the prompt never listed it) are removed from scoring for every model, symmetrically; their rows stay in the payload with scored=false. Each entry records the alternative reading and the reference under both readings, recomputed with the engine version that produced the references.", - "outputs": 11, - "scored_outputs_per_model": 1973 + "note": "Outputs are removed from scoring for every model, symmetrically, when the reference depends on an input or definition the prompt never states (a careful reader could take the stated facts either way), or when the engine that produced the reference misapplies the law on facts the prompt states. Their rows stay in the payload with scored=false. An unlisted-input entry records the alternative reading and the reference under both readings; an engine-defect entry records the root cause, the law, the upstream issue, and the corrected value computed with the same engine version and a sandbox fix.", + "outputs": 56, + "scored_outputs_per_model": 1928 }, "reference_output_refresh": { - "date": "2026-07-03", + "date": "2026-09-29", "generated_at_utc": "2026-07-03T04:21:31.815629+00:00", - "policyengine_us_data_artifact_sha256": "f32c2e5e9098bc6540724fdd5debf963af495da4c29b3a7a63fb53c2a4bb5a34", - "policyengine_us_data_build_id": "populace-us-2024-5da5a95-20260611", + "policyengine_us_data_artifact_sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "policyengine_us_data_build_id": "populace-us-2024-spm-20260915", "policyengine_us_dataset": "populace_us_2024", - "policyengine_us_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-5da5a95-20260611", - "policyengine_us_version": "1.755.4", - "policyengine_version": "4.16.1", - "reference_csv_sha256": "b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368", + "policyengine_us_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "policyengine_us_version": "2.15.17", + "policyengine_version": "6.1.2", + "reference_csv_sha256": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + "regenerated_at_utc": "2026-09-29T15:04:45.724738+00:00", "row_count": 1984, - "snapshot_date": "2026-09-05" + "snapshot_date": "2026-09-30" }, "rendered_paper_artifacts": { "pdf": { "path": "app/public/paper/policybench.pdf", - "sha256": "f74679bd046f7e6a78b98e88c6f7c0aeb7d1016380a0fa38900e8727f8e03990" + "sha256": "87f78f83e929fd2755945cb1283cb2cddffc4a4f677c9072a8773a95969512cf" }, "web": { "files": { - "figures/positive_zero_scatter.png": "213ab1b526869250e379054718991fc4abbb2dd6ebc82a5ea300a226d5948407", - "index.html": "ea5a0152c3fdeb9669f7fb7130f6d9fb4e1d837fccef1556514ee5460e5a1bed", + "figures/positive_zero_scatter.png": "ee8a07e71a18f7756062b74a995ce9eeae045a379ffb3842c107ea501aec246f", + "index.html": "bad4b3c0eb58047b4578b4e4ff3f702826096ceeac36fca6ed9b0b985cedfe9a", "pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8", "policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9", "site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9", @@ -127,11 +188,11 @@ }, "reproducibility_notes": [ "The top-level scenario, reference-output, and impact-summary CSVs are byte-identical to the corresponding compact source-run artifacts copied under paper/snapshot/20260501/runs/.", - "Model responses were collected in waves between June 12 and September 1, 2026, as models were added to the board; each model's full 100-household run is a single consistent wave. Reference outputs were generated with policyengine.py 4.16.1 and policyengine-us 1.755.4 against the certified PolicyEngine US populace dataset (populace-us-2024-5da5a95-20260611, populace_us_2024).", + "Model responses were collected in waves between June 12 and September 30, 2026, as models were added to the board; each model's full 100-household run is a single consistent wave. PolicyBench computes each scored reference output with policyengine_us.Simulation from policyengine-us 2.15.17, using the household's own listed inputs, the nine publication conventions, the Maryland output-scope adapter and the scenario builder's stated-hours alias, as the reference sidecar's engine_upgrade revision pins them (fix_modules, builder); policyengine.py 6.1.2 is recorded for provenance only. The households were drawn from the certified PolicyEngine US populace dataset (populace-us-2024-5da5a95-20260611, populace_us_2024), as scenarios.csv.meta.json records (household_dataset); the reference_output_refresh dataset fields name the reference runtime's default dataset, which reference computation does not read.", "Canonical prediction files include parser recovery. Later waves ran under the resumable supervised runner, which retries failed or timed-out scenarios in bounded rounds; every model's canonical file covers all 100 households.", "Raw provider responses are retained in the compressed source-run predictions.csv.gz file wherever the transport exposed them; rows served through the Anthropic batch adapter (Claude Fable 5) and 64 Kimi K3 parse failures carry no raw payload. The separate LiteLLM cache remains local-only because it is a generated request cache, not the canonical snapshot artifact.", "The frozen scenarios.csv source_dataset column carries a stale enhanced_cps_2024 label from the pre-#77 scenario generator; the run metadata (scenarios.csv.meta.json) records the populace_us_2024 build actually loaded.", - "Model APIs and upstream model aliases may change after the recorded 2026-06-12 to 2026-09-05 response window, so exact reruns can diverge even with the committed household inputs, reference outputs, parsed dashboard export, and analysis summaries." + "Model APIs and upstream model aliases may change after the recorded 2026-06-12 to 2026-09-30 response window, so exact reruns can diverge even with the committed household inputs, reference outputs, parsed dashboard export, and analysis summaries." ], "response_retry_artifacts": { "files": {}, @@ -148,27 +209,27 @@ "households": { "us": 100 }, - "models": 39, + "models": 46, "output_groups": { "us": 18 } }, - "snapshot_date": "2026-09-05", + "snapshot_date": "2026-09-30", "source_run_artifacts": { "note": "Compact copies of run outputs used to verify this snapshot. The run data.json retains parsed scenario predictions, explanations, summaries, heatmaps, and PolicyEngine runtime metadata used by the dashboard. predictions.csv.gz is a deterministic gzip of the run's raw provider responses.", "us_full_run_20260612_policyengine_4_16_1_populace": { "files": { - "analysis/impact_summary_by_model.csv": "540f5e57aef1f065560e9c6cd53527e0e2ab9832f74f1581187e4a098725de69", - "analysis/metrics.csv": "145a5de3f7b1b6f8ba33928df2342200526b4a72a8bbc619737c0a59be392b91", - "analysis/report.md": "7beb4652c002c053f53f4dcaa32a78e395fce76d5bf1606e2900b8c6223621e5", - "analysis/summary_by_model.csv": "fb3310cbd37aeb8486609714d4fd4309266ad31b9c979aa7cbdd1daba5f82438", - "analysis/summary_by_variable.csv": "dd4962fca660e02b1db85f07ea8d5827e533b7ed6b0db87fd268b4c493481658", - "analysis/usage_summary.csv": "458eae34cb84cf3c11c52437612f12d659baa960857dc93a379e602422448641", - "data.json.gz": "2cce2598dbf9f33511f31d944221a7fc27f69b487a56c59e2c17e77d4ba5ad1f", - "predictions.csv.gz": "8c91a14e922689358f34fb47be5c9061e30edc4987ad770c1ae3de53a87e15d6", - "reference_exclusions.json": "3482ce7cd7461e32ab17ab2c8473613cb313be178485139be8194f3bfcd8172e", - "reference_outputs.csv": "b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368", - "reference_outputs.csv.meta.json": "4ea7911857fabf72fc0f74feab90566e9f6d98cab4958e0c1401043e58c94210", + "analysis/impact_summary_by_model.csv": "e6e034adee408cc798897bc0825ace9740055ab20e528ce842bd06e2c81e0922", + "analysis/metrics.csv": "8b97a753a0775cd79ec45399ddaa7b93bcf4ffd4acd4ccd076e10994ecbfd8b1", + "analysis/report.md": "85e9456b34b32c08f980360639e2b8657c0d7e1b6aff536d536ee3912523d61f", + "analysis/summary_by_model.csv": "fe929fc5d4c01f74c4a199a68701e92001a64eee84b83131eaa22ef1d4ffb851", + "analysis/summary_by_variable.csv": "93eaedd458859fdc0f36b63b97c737615348628216aead83023e30f901629399", + "analysis/usage_summary.csv": "6d2f6338af5eb9db19dbfb0f1ba37363387b3280b85ae64feaa8dba2e2ccf9db", + "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "predictions.csv.gz": "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + "reference_exclusions.json": "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + "reference_outputs.csv": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + "reference_outputs.csv.meta.json": "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b", "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", "scenarios.csv.meta.json": "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" }, diff --git a/paper/snapshot/20260501/model_serving_config.json b/paper/snapshot/20260501/model_serving_config.json index 8edd3088..99dcbe8c 100644 --- a/paper/snapshot/20260501/model_serving_config.json +++ b/paper/snapshot/20260501/model_serving_config.json @@ -13,7 +13,7 @@ }, "evidence_summary": { "registry": 29, - "run_state": 10 + "run_state": 17 }, "models": { "claude-fable-5": { @@ -171,6 +171,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "claude-opus-5.5": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "a32cbcf7da538881617ce1ed524024c8488e9f962b27939307b60928031065f4", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "77ce7395dafec31dcbf1d8b14941aa51750046a44ee501b0d414f6e89f6d023c", + "source_rows_sha256": "a32cbcf7da538881617ce1ed524024c8488e9f962b27939307b60928031065f4", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "claude-opus-5-5", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 600, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "claude-opus-5-5", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 600, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "claude-sonnet-4.6": { "answer_contract": "tool", "evidence": { @@ -213,6 +270,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "claude-sonnet-5.5": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "1c14b56193ec04e905fa39896b5199b0546e327dc3a18aeb1da51642de422402", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/sonnet55", + "source_predictions_sha256": "b747a404243ddaa3d509ab662f5564b613ae481c4e8baf5b83767583f0d3f602", + "source_rows_sha256": "1c14b56193ec04e905fa39896b5199b0546e327dc3a18aeb1da51642de422402", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "claude-sonnet-5-5", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 600, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "claude-sonnet-5-5", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 600, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "deepseek-v4-flash-0731": { "answer_contract": "tool", "evidence": { @@ -336,6 +450,63 @@ "shared_completion_budget_tokens": 98304, "tool_choice": "forced" }, + "deepseek-v4.1-flash": { + "answer_contract": "json", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "8ab3c0e72c8daa6613223fb0242d7b88e8340daf1e44859bb4cac3ca57503323", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/dsflash41", + "source_predictions_sha256": "0504729cc50c336db24eca623e9c53ffb2f51ffe41202a960e764a21ec8d18d4", + "source_rows_sha256": "8ab3c0e72c8daa6613223fb0242d7b88e8340daf1e44859bb4cac3ca57503323", + "treatment_fingerprint": { + "answer_contract": "json", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "deepseek/deepseek-flash", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": null + } + }, + "provider_id": "deepseek/deepseek-flash", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": null + }, "gemini-3-flash-preview": { "answer_contract": "json", "evidence": { @@ -841,6 +1012,177 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "gpt-6-luna": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "66deb97c4aac19e7e85f0128841d87bd7523394fbd9b439ffee45d7df40d5433", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "02537c603f784519510655092cce346eedf55776049fd68720503c1cfaa34eca", + "source_rows_sha256": "66deb97c4aac19e7e85f0128841d87bd7523394fbd9b439ffee45d7df40d5433", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6-luna", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6-luna", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, + "gpt-6-sol": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "33491207380157db2dbf606b4e13177daf9ea6867b167f51d8e8d44f56682353", + "kind": "run_state", + "rows_match": true, + "run": "adds202609", + "source_predictions_sha256": "67699eed8dc234e9801ee7a5d537fafd7e133f59425527c97ab9def6fe5de85e", + "source_rows_sha256": "33491207380157db2dbf606b4e13177daf9ea6867b167f51d8e8d44f56682353", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6-sol", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6-sol", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, + "gpt-6.1-sol": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "6d20d23add25271aebf0ee05cce7075817d2ff5286eeea868ea7834238b5c992", + "kind": "run_state", + "rows_match": true, + "run": "adds202609/gpt61sol", + "source_predictions_sha256": "e1e7662d5afa5a886d1b9b39fd0488b9d11941ebf10ddc88d821b24f3861d2aa", + "source_rows_sha256": "6d20d23add25271aebf0ee05cce7075817d2ff5286eeea868ea7834238b5c992", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "gpt-6.1-sol", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 300, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "gpt-6.1-sol", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 300, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, "grok-4.3": { "answer_contract": "tool", "evidence": { @@ -932,6 +1274,63 @@ "shared_completion_budget_tokens": 16384, "tool_choice": "forced" }, + "grok-4.7": { + "answer_contract": "tool", + "evidence": { + "compared_columns": [ + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_resolved_model", + "prompt_tokens", + "completion_tokens", + "error" + ], + "fields": [ + "answer_contract", + "chunk_size", + "completion_budget_ceiling", + "fingerprint_version", + "initial_completion_budget_tokens", + "max_repair_rounds", + "model_id", + "prompt_contract_version", + "request_timeout_seconds", + "thinking", + "tool_choice_mode" + ], + "frozen_rows_sha256": "f225f0d3e8c7aae04ffd220cbbb1e9fe9ce48c8da350fbb16e5679104c27e866", + "kind": "run_state", + "rows_match": true, + "run": "adds0928/grok47", + "source_predictions_sha256": "f8142eadad320820a97b469da2007dc052fcc041090475ee6620ab743cbd2748", + "source_rows_sha256": "f225f0d3e8c7aae04ffd220cbbb1e9fe9ce48c8da350fbb16e5679104c27e866", + "treatment_fingerprint": { + "answer_contract": "tool", + "chunk_size": null, + "completion_budget_ceiling": 128000, + "fingerprint_version": 3, + "initial_completion_budget_tokens": 16384, + "max_repair_rounds": 2, + "model_id": "xai/grok-4.7", + "prompt_contract_version": "2026-08-09-v2-scoring-contract", + "request_timeout_seconds": 1800, + "thinking": { + "mode": "provider_default" + }, + "tool_choice_mode": "forced" + } + }, + "provider_id": "xai/grok-4.7", + "reasoning_setup": "provider default; 16,384-token shared budget", + "registry_derived": [], + "request_shape": "whole scenario", + "request_timeout_seconds": 1800, + "shared_completion_budget_tokens": 16384, + "tool_choice": "forced" + }, "grok-build-0.1": { "answer_contract": "tool", "evidence": { @@ -1129,7 +1528,7 @@ "tool_choice": null } }, - "registry_commit": "d1d841e99284757e8274ee5bffe1d26472fe73fe", + "registry_commit": "cddd01812ad6d6311502ef997f93634f70c28fda", "sources": [ "policybench.config.MODELS", "policybench.model_cards", diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv index 5259a8ca..d34b2b5b 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/impact_summary_by_model.csv @@ -1,40 +1,47 @@ model,mean_impact_score,mean_household_score,mean_household_coverage,households,total_variables,parsed_variables,floor_share -gpt-6-astra,0.9496437469868486,0.9715149854884928,1.0,100,1973,1973,0.3 -claude-fable-5.1,0.938558478818549,0.9723387729440702,1.0,100,1973,1973,0.3 -gpt-5.6-sol,0.9367561386850141,0.9696920448728172,1.0,100,1973,1973,0.3 -inkling,0.9364586213430499,0.962243744202623,1.0,100,1973,1973,0.3 -gpt-5.5,0.9192437189687681,0.9593886640786914,1.0,100,1973,1973,0.3 -kimi-k3,0.9146618878255316,0.9406982797753375,0.97,100,1973,1910,0.3 -gpt-5.6-terra,0.9146364805309777,0.9569218477579463,1.0,100,1973,1973,0.3 -gemini-3.8-flash,0.9092371191481496,0.9652723307891868,1.0,100,1973,1973,0.3 -ox-alpha,0.9088756571628427,0.954790817286395,1.0,100,1973,1973,0.3 -grok-4.6,0.9067629363079034,0.9628776223415428,1.0,100,1973,1973,0.3 -gemini-3.6-flash,0.9065446578039231,0.9628270206868003,1.0,100,1973,1973,0.3 -claude-opus-4.7,0.8938486818299418,0.9283232013207957,1.0,100,1973,1973,0.3 -gemini-3-flash-preview,0.8938155904300443,0.9522189245402665,1.0,100,1973,1973,0.3 -claude-fable-5,0.8926079230715791,0.9466651773258209,1.0,100,1973,1973,0.3 -claude-sonnet-4.6,0.8920563224013588,0.9371230932434323,1.0,100,1973,1973,0.3 -gpt-5.6-luna,0.891518703158717,0.9499694816710595,1.0,100,1973,1973,0.3 -deepseek-v4-flash-0731,0.891464792542364,0.9449788777185419,1.0,100,1973,1973,0.3 -deepseek-v4-pro-0813,0.8912300678385735,0.9563902384151355,1.0,100,1973,1973,0.3 -grok-4.5,0.8870289536651306,0.9531819926162508,1.0,100,1973,1973,0.3 -gemini-3.7-flash,0.8817914276293224,0.9544800401080973,1.0,100,1973,1973,0.3 -gemini-3.1-pro-preview,0.8810846627992194,0.9533512172611641,1.0,100,1973,1973,0.3 -gemini-3.5-flash,0.8776770775707564,0.9491763139769515,1.0,100,1973,1973,0.3 -claude-opus-5,0.8750821644772023,0.947890502076229,1.0,100,1973,1973,0.3 -glm-5.3,0.8704894715328001,0.9184145585473523,0.97,100,1973,1896,0.3 -grok-build-0.1,0.8702102624393789,0.937939104852214,1.0,100,1973,1973,0.3 -deepseek-v4-pro,0.8690474475624503,0.9439243184233473,1.0,100,1973,1973,0.3 -claude-opus-4.8,0.8551010884409126,0.9099994486195605,1.0,100,1973,1973,0.3 -gemini-3.1-flash-lite-preview,0.8332526192159111,0.9279862856738541,1.0,100,1973,1973,0.3 -claude-sonnet-5,0.8323133044741627,0.905770284778892,1.0,100,1973,1973,0.3 -grok-4.3,0.8195264993290182,0.9316608219320247,1.0,100,1973,1973,0.3 -qwen-3.7-max,0.8187989458142549,0.9139856473453446,1.0,100,1973,1973,0.3 -glm-5.2,0.8071387853647619,0.8863723652540183,0.9419665696145959,100,1973,1834,0.3 -qwen3.8-max,0.7864757794964761,0.8943579441272115,1.0,100,1973,1973,0.3 -gemini-3.5-flash-lite,0.7862088975121634,0.9286285233712686,1.0,100,1973,1973,0.3 -claude-haiku-4.5,0.7744794545746602,0.8978781106226874,1.0,100,1973,1973,0.3 -gpt-5.4-mini,0.7141033302649024,0.888519622370442,1.0,100,1973,1973,0.3 -minimax-m3,0.7037588163656016,0.8982717732275121,1.0,100,1973,1973,0.3 -gpt-5.4-nano,0.6509176804344221,0.8779672475285321,1.0,100,1973,1973,0.3 -kimi-k2.6,0.5624316010699739,0.7623994474586452,0.783860742165048,100,1973,1551,0.3 +gpt-6-astra,0.9674395301873611,0.978257782717636,1.0,100,1928,1928,0.3 +gpt-6-sol,0.9662824005912722,0.9823887231923742,1.0,100,1928,1928,0.3 +gpt-6-luna,0.9658376159373622,0.9773386229254812,1.0,100,1928,1928,0.3 +claude-opus-5.5,0.9608671065690544,0.9823954326352231,1.0,100,1928,1928,0.3 +gpt-6.1-sol,0.9586945574651387,0.9752733926090379,1.0,100,1928,1928,0.3 +gpt-5.6-sol,0.9581475169972634,0.9762335155147531,1.0,100,1928,1928,0.3 +claude-sonnet-5.5,0.9580535920873353,0.9730704556439665,1.0,100,1928,1928,0.3 +claude-fable-5.1,0.9555813124174095,0.9779033840407823,1.0,100,1928,1928,0.3 +grok-4.7,0.9531030169930793,0.9758195364701391,1.0,100,1928,1928,0.3 +inkling,0.95278940419871,0.9708083462322512,1.0,100,1928,1928,0.3 +gpt-5.5,0.9444535880318884,0.9659348038436416,1.0,100,1928,1928,0.3 +deepseek-v4.1-flash,0.9437394502936982,0.9674174501414599,1.0,100,1928,1928,0.3 +gpt-5.6-terra,0.9423390142779876,0.9651008738219248,1.0,100,1928,1928,0.3 +gemini-3.8-flash,0.9395394036327269,0.9753939876195191,1.0,100,1928,1928,0.3 +grok-4.6,0.935512292170297,0.9706379118326355,1.0,100,1928,1928,0.3 +gemini-3.6-flash,0.9340484471155557,0.9707033409108929,1.0,100,1928,1928,0.3 +ox-alpha,0.9329622083651122,0.9639201283365196,1.0,100,1928,1928,0.3 +kimi-k3,0.9312389646048008,0.9468695137686975,0.97,100,1928,1870,0.3 +grok-4.5,0.9248152464910463,0.9634117776809533,1.0,100,1928,1928,0.3 +deepseek-v4-pro-0813,0.9245585396764721,0.966361343016232,1.0,100,1928,1928,0.3 +gpt-5.6-luna,0.9221866400950565,0.9594239447421039,1.0,100,1928,1928,0.3 +deepseek-v4-flash-0731,0.9187089252487097,0.9546492938053259,1.0,100,1928,1928,0.3 +gemini-3-flash-preview,0.9170555704087302,0.9608133948736268,1.0,100,1928,1928,0.3 +claude-sonnet-4.6,0.9147170663321406,0.9450085166651927,1.0,100,1928,1928,0.3 +claude-opus-4.7,0.9138626894667351,0.9333394692464237,1.0,100,1928,1928,0.3 +claude-fable-5,0.9118097137275345,0.9538040486732181,1.0,100,1928,1928,0.3 +claude-opus-5,0.9093403745672698,0.9565726626978903,1.0,100,1928,1928,0.3 +gemini-3.7-flash,0.9068894551658637,0.9637007836479945,1.0,100,1928,1928,0.3 +grok-build-0.1,0.9035771268657874,0.9490575162370398,1.0,100,1928,1928,0.3 +gemini-3.1-pro-preview,0.8996459348257337,0.9613824023677643,1.0,100,1928,1928,0.3 +deepseek-v4-pro,0.8939788090152366,0.9531021316162993,1.0,100,1928,1928,0.3 +gemini-3.5-flash,0.892262363142274,0.9566317304583035,1.0,100,1928,1928,0.3 +glm-5.3,0.8870442747598982,0.9269952358769811,0.97,100,1928,1857,0.3 +claude-opus-4.8,0.8805442215879067,0.9191835487991205,1.0,100,1928,1928,0.3 +claude-sonnet-5,0.8639047411962291,0.9156874253444743,1.0,100,1928,1928,0.3 +gemini-3.1-flash-lite-preview,0.8621157646423028,0.9399938349552724,1.0,100,1928,1928,0.3 +qwen-3.7-max,0.8542922488394893,0.9254172874391244,1.0,100,1928,1928,0.3 +grok-4.3,0.8535754470989819,0.94337639429603,1.0,100,1928,1928,0.3 +glm-5.2,0.8468008249398641,0.8985638347071989,0.9430477571052661,100,1928,1795,0.3 +qwen3.8-max,0.8163971792504239,0.906206350579505,1.0,100,1928,1928,0.3 +gemini-3.5-flash-lite,0.8140072027414716,0.9406660914684142,1.0,100,1928,1928,0.3 +claude-haiku-4.5,0.7958926936620644,0.9088133283583146,1.0,100,1928,1928,0.3 +gpt-5.4-mini,0.7565923451264394,0.9024786162568723,1.0,100,1928,1928,0.3 +minimax-m3,0.7364968916060559,0.915496790900971,1.0,100,1928,1928,0.3 +gpt-5.4-nano,0.6872355186399137,0.8911936267908885,1.0,100,1928,1928,0.3 +kimi-k2.6,0.5816850782720341,0.775441245816979,0.7944015327506267,100,1928,1538,0.3 diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv index a78f14ba..3389ea4f 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/metrics.csv @@ -1,703 +1,829 @@ model,variable,n,n_parsed,coverage,mae,mape,accuracy,exact,within_1pct,within_5pct,within_10pct,threshold_score,score -claude-fable-5,federal_income_tax_before_refundable_credits,100,100,1.0,1107.0905798339843,0.4359408366437689,,0.52,0.54,0.59,0.67,0.58,0.8566783765175026 -claude-fable-5,federal_refundable_credits,100,100,1.0,113.0726400756836,0.29239471145564455,,0.86,0.89,0.91,0.91,0.8925000000000001,0.9343124685197405 +claude-fable-5,federal_income_tax_before_refundable_credits,82,82,1.0,609.0313333650914,0.4794336645106638,,0.5853658536585366,0.5975609756097561,0.6463414634146342,0.7073170731707317,0.6341463414634145,0.8653258079822098 +claude-fable-5,federal_refundable_credits,98,98,1.0,73.61601506447306,0.2161314886582061,,0.8775510204081632,0.9081632653061225,0.9285714285714286,0.9285714285714286,0.9107142857142858,0.945128098211834 claude-fable-5,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-fable-5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-fable-5,payroll_tax,100,100,1.0,236.97827380371092,0.026463223886278572,,0.82,0.83,0.9,0.95,0.875,0.9830635367127816 +claude-fable-5,payroll_tax,99,99,1.0,239.20946858723957,0.026765186750973252,,0.8282828282828283,0.8282828282828283,0.898989898989899,0.9494949494949495,0.8762626262626262,0.9829676084311989 claude-fable-5,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 claude-fable-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-fable-5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -claude-fable-5,person_medicaid_eligible,177,177,1.0,0.062146892655367235,,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328 +claude-fable-5,person_medicaid_eligible,173,173,1.0,0.05202312138728324,,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167 claude-fable-5,person_medicare_eligible,172,172,1.0,0.0872093023255814,,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186 claude-fable-5,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -claude-fable-5,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-fable-5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-fable-5,self_employment_tax,100,100,1.0,34.48554270019532,0.0007099896483805804,,0.97,0.99,0.99,0.99,0.9850000000000001,0.9899432008281295 -claude-fable-5,snap,97,97,1.0,273.4027300962468,0.5514286891942455,,0.7938144329896907,0.8041237113402062,0.8247422680412371,0.845360824742268,0.8170103092783506,0.8724300235432765 +claude-fable-5,snap,93,93,1.0,262.7204301075269,0.5753977027817445,,0.8387096774193549,0.8387096774193549,0.8494623655913979,0.8709677419354839,0.8494623655913979,0.8873099985358851 claude-fable-5,ssi,97,97,1.0,123.63505154639175,0.19494688610780198,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.979381443298969,0.9742268041237113,0.9856711982245815 -claude-fable-5,state_income_tax_before_refundable_credits,100,100,1.0,340.23372068309783,0.494791624417577,,0.55,0.56,0.63,0.65,0.5975,0.7988182327457908 -claude-fable-5,state_refundable_credits,100,100,1.0,70.35355062961578,0.6636808710470935,,0.8,0.8,0.81,0.82,0.8075,0.8611670168093117 +claude-fable-5,state_income_tax_before_refundable_credits,86,86,1.0,219.88635216091956,0.40420537130478595,,0.627906976744186,0.6395348837209303,0.6976744186046512,0.7209302325581395,0.6715116279069767,0.8296060324047224 +claude-fable-5,state_refundable_credits,98,98,1.0,67.9308757246757,0.7076828863448931,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8469387755102041,0.8341836734693878,0.8825146752432662 claude-fable-5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-fable-5.1,federal_income_tax_before_refundable_credits,100,100,1.0,369.13481489257805,0.1787233318445481,,0.69,0.79,0.88,0.89,0.8125,0.9347594008001984 -claude-fable-5.1,federal_refundable_credits,100,100,1.0,25.493024011230464,0.06757803362637425,,0.97,0.97,0.97,0.98,0.9725,0.9912148556285714 +claude-fable-5.1,federal_income_tax_before_refundable_credits,82,82,1.0,300.2469381192835,0.15374949222604853,,0.7926829268292683,0.8292682926829268,0.9024390243902439,0.9024390243902439,0.8567073170731707,0.9390642445389619 +claude-fable-5.1,federal_refundable_credits,98,98,1.0,1.9756582703882253,0.005376964556959538,,0.9897959183673469,0.9897959183673469,0.9897959183673469,1.0,0.9923469387755102,0.9993964631619738 claude-fable-5.1,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 claude-fable-5.1,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-fable-5.1,payroll_tax,100,100,1.0,111.24153728027345,0.014732745065501165,,0.88,0.9,0.9,0.96,0.91,0.9905710431580792 +claude-fable-5.1,payroll_tax,99,99,1.0,111.228219376381,0.014140493178512596,,0.8888888888888888,0.9090909090909091,0.9090909090909091,0.9595959595959596,0.9166666666666667,0.9910015043409465 claude-fable-5.1,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786 claude-fable-5.1,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-fable-5.1,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -claude-fable-5.1,person_medicaid_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 +claude-fable-5.1,person_medicaid_eligible,173,173,1.0,0.028901734104046242,,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537 claude-fable-5.1,person_medicare_eligible,172,172,1.0,0.03488372093023256,,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675 claude-fable-5.1,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -claude-fable-5.1,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-fable-5.1,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-fable-5.1,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 -claude-fable-5.1,snap,97,97,1.0,87.86410594192978,0.4390610758890219,,0.8144329896907216,0.865979381443299,0.8969072164948454,0.8969072164948454,0.8685567010309279,0.9127418732978002 +claude-fable-5.1,snap,93,93,1.0,58.35526881720429,0.3985246550949543,,0.8817204301075269,0.9247311827956989,0.9247311827956989,0.9247311827956989,0.9139784946236559,0.9335395643415655 claude-fable-5.1,ssi,97,97,1.0,102.61855670103093,0.0,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846 -claude-fable-5.1,state_income_tax_before_refundable_credits,100,100,1.0,115.05700294628143,0.15578375171470787,,0.63,0.7,0.8,0.86,0.7474999999999999,0.9216365930274049 -claude-fable-5.1,state_refundable_credits,100,100,1.0,27.19493817653656,0.40366842904734973,,0.89,0.9,0.9,0.9,0.8975,0.9263769983211091 +claude-fable-5.1,state_income_tax_before_refundable_credits,86,86,1.0,70.51427605717682,0.10463357054826823,,0.7325581395348837,0.7906976744186046,0.872093023255814,0.9069767441860465,0.8255813953488371,0.9402441032971158 +claude-fable-5.1,state_refundable_credits,98,98,1.0,22.977769108013234,0.35719178258518913,,0.9183673469387755,0.9183673469387755,0.9183673469387755,0.9285714285714286,0.9209183673469388,0.9457682117915495 claude-fable-5.1,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-haiku-4.5,federal_income_tax_before_refundable_credits,100,100,1.0,3262.2004235839845,0.5990951144909413,,0.46,0.48,0.55,0.59,0.52,0.7111017774207682 -claude-haiku-4.5,federal_refundable_credits,100,100,1.0,697.1598025512695,0.6292503670690871,,0.75,0.75,0.75,0.75,0.75,0.7981974522810187 +claude-haiku-4.5,federal_income_tax_before_refundable_credits,82,82,1.0,1173.3247546684452,0.6174508141046803,,0.524390243902439,0.5365853658536586,0.5975609756097561,0.6219512195121951,0.5701219512195123,0.7427641153355573 +claude-haiku-4.5,federal_refundable_credits,98,98,1.0,678.9802872015506,0.6425826245005123,,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.7653061224489796,0.8054243992907588 claude-haiku-4.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 claude-haiku-4.5,local_income_tax,100,100,1.0,35.45,,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-haiku-4.5,payroll_tax,100,100,1.0,926.5703920410156,0.14209327711792302,,0.65,0.71,0.75,0.84,0.7374999999999999,0.9052997894183282 +claude-haiku-4.5,payroll_tax,99,99,1.0,934.8121130741002,0.14353671231957388,,0.6565656565656566,0.7171717171717171,0.7575757575757576,0.8383838383838383,0.7424242424242424,0.9048599575883508 claude-haiku-4.5,person_chip_eligible,177,177,1.0,0.096045197740113,,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887 claude-haiku-4.5,person_early_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579 claude-haiku-4.5,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -claude-haiku-4.5,person_medicaid_eligible,177,177,1.0,0.1751412429378531,,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468 +claude-haiku-4.5,person_medicaid_eligible,173,173,1.0,0.17341040462427745,,0.8265895953757225,0.8265895953757225,0.8265895953757225,0.8265895953757225,0.8265895953757225,0.8265895953757225,0.8265895953757225 claude-haiku-4.5,person_medicare_eligible,172,172,1.0,0.023255813953488372,,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116 claude-haiku-4.5,person_wic_eligible,177,177,1.0,0.07344632768361582,,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842 -claude-haiku-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.08,,0.92,0.92,0.92,0.92,0.92,0.92,0.92 +claude-haiku-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.07,,0.93,0.93,0.93,0.93,0.93,0.93,0.93 claude-haiku-4.5,self_employment_tax,100,100,1.0,74.96613400878907,0.0007522057448829237,,0.91,0.94,0.94,0.94,0.9325,0.9299398235404093 -claude-haiku-4.5,snap,97,97,1.0,527.8658963232925,1.4476443622146922,,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8367607144569837 +claude-haiku-4.5,snap,93,93,1.0,465.3333333333333,0.9093570110130051,,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8621328909336659 claude-haiku-4.5,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -claude-haiku-4.5,state_income_tax_before_refundable_credits,100,100,1.0,859.6294401166917,0.6486025135644299,,0.55,0.56,0.6,0.62,0.5825,0.7097905201014636 -claude-haiku-4.5,state_refundable_credits,100,100,1.0,122.28137747287751,1.0,,0.79,0.79,0.79,0.79,0.79,0.79 +claude-haiku-4.5,state_income_tax_before_refundable_credits,86,86,1.0,449.8351398607742,0.6790645695255925,,0.627906976744186,0.6395348837209303,0.6627906976744186,0.686046511627907,0.6540697674418604,0.749965383809873 +claude-haiku-4.5,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449 claude-haiku-4.5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-opus-4.7,federal_income_tax_before_refundable_credits,100,100,1.0,1338.909669189453,0.6004019224064938,,0.54,0.56,0.6,0.67,0.5925,0.855614284964672 -claude-opus-4.7,federal_refundable_credits,100,100,1.0,183.2322543334961,0.3124009796052566,,0.81,0.82,0.83,0.85,0.8275,0.8893878726513167 +claude-opus-4.7,federal_income_tax_before_refundable_credits,82,82,1.0,788.6295433974848,0.632492412109637,,0.6097560975609756,0.6219512195121951,0.6463414634146342,0.6829268292682927,0.6402439024390244,0.8658825813535689 +claude-opus-4.7,federal_refundable_credits,98,98,1.0,162.337958822445,0.2924270186642666,,0.826530612244898,0.8367346938775511,0.8469387755102041,0.8673469387755102,0.8443877551020409,0.8957479877009497 claude-opus-4.7,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-opus-4.7,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-opus-4.7,payroll_tax,100,100,1.0,118.31507087402343,0.014668878322522134,,0.83,0.85,0.93,0.96,0.8925,0.9906119178735858 +claude-opus-4.7,payroll_tax,99,99,1.0,118.3732028044113,0.014075612677708502,,0.8383838383838383,0.8585858585858586,0.9393939393939394,0.9595959595959596,0.898989898989899,0.9910427919323673 claude-opus-4.7,person_chip_eligible,177,177,1.0,0.11864406779661017,,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898,0.8813559322033898 claude-opus-4.7,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-opus-4.7,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579 -claude-opus-4.7,person_medicaid_eligible,177,177,1.0,0.13559322033898305,,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017 +claude-opus-4.7,person_medicaid_eligible,173,173,1.0,0.1329479768786127,,0.8670520231213873,0.8670520231213873,0.8670520231213873,0.8670520231213873,0.8670520231213873,0.8670520231213873,0.8670520231213873 claude-opus-4.7,person_medicare_eligible,172,172,1.0,0.0755813953488372,,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628 claude-opus-4.7,person_wic_eligible,177,177,1.0,0.04519774011299435,,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056 -claude-opus-4.7,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-opus-4.7,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-opus-4.7,self_employment_tax,100,100,1.0,0.022542163085938684,0.00020048201810881564,,1.0,1.0,1.0,1.0,1.0,0.9999839614385513 -claude-opus-4.7,snap,97,97,1.0,460.6576292686856,1.9769836007985588,,0.7525773195876289,0.7525773195876289,0.7628865979381443,0.7731958762886598,0.7603092783505154,0.8248522589304347 +claude-opus-4.7,snap,93,93,1.0,465.18279569892474,1.8454323762656035,,0.7741935483870968,0.7741935483870968,0.7741935483870968,0.7956989247311828,0.7795698924731183,0.8296632759580936 claude-opus-4.7,ssi,97,97,1.0,166.9278350515464,0.3306139867067448,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9828739384184175 -claude-opus-4.7,state_income_tax_before_refundable_credits,100,100,1.0,270.230998075676,0.31548613149464094,,0.57,0.61,0.66,0.78,0.655,0.8525820612536021 -claude-opus-4.7,state_refundable_credits,100,100,1.0,84.90355490207672,3.5477149665465686,,0.75,0.75,0.76,0.78,0.76,0.81385518651811 +claude-opus-4.7,state_income_tax_before_refundable_credits,86,86,1.0,198.069614674546,0.2672699416986895,,0.6511627906976745,0.686046511627907,0.7325581395348837,0.8255813953488372,0.7238372093023256,0.8835104854539455 +claude-opus-4.7,state_refundable_credits,98,98,1.0,77.61455355371747,4.132937401880446,,0.7653061224489796,0.7653061224489796,0.7755102040816326,0.7959183673469388,0.7755102040816326,0.8147258452245381 claude-opus-4.7,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-opus-4.8,federal_income_tax_before_refundable_credits,100,100,1.0,2158.5744494628907,0.7858144899840176,,0.5,0.54,0.6,0.67,0.5775,0.781902781789626 -claude-opus-4.8,federal_refundable_credits,100,100,1.0,228.6542025756836,0.42503457090311547,,0.82,0.82,0.84,0.84,0.83,0.8847455057825951 +claude-opus-4.8,federal_income_tax_before_refundable_credits,82,82,1.0,919.4929080125762,0.8511261519272454,,0.573170731707317,0.6097560975609756,0.6585365853658537,0.7195121951219512,0.6402439024390243,0.8179498270666544 +claude-opus-4.8,federal_refundable_credits,98,98,1.0,198.6052529471261,0.3940920199472762,,0.8367346938775511,0.8367346938775511,0.8571428571428571,0.8571428571428571,0.8469387755102041,0.8945406916385711 claude-opus-4.8,free_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97 claude-opus-4.8,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-opus-4.8,payroll_tax,100,100,1.0,312.49507946777345,0.03238448435666907,,0.81,0.84,0.86,0.92,0.8574999999999999,0.9792739300117319 +claude-opus-4.8,payroll_tax,99,99,1.0,314.514625626381,0.032072418807635554,,0.8181818181818182,0.8484848484848485,0.8686868686868687,0.9191919191919192,0.8636363636363636,0.9795902789405955 claude-opus-4.8,person_chip_eligible,177,177,1.0,0.12429378531073447,,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656 claude-opus-4.8,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 claude-opus-4.8,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579 -claude-opus-4.8,person_medicaid_eligible,177,177,1.0,0.22033898305084745,,0.7796610169491526,0.7796610169491526,0.7796610169491526,0.7796610169491526,0.7796610169491526,0.7796610169491526,0.7796610169491526 +claude-opus-4.8,person_medicaid_eligible,173,173,1.0,0.2138728323699422,,0.7861271676300579,0.7861271676300579,0.7861271676300579,0.7861271676300579,0.7861271676300579,0.7861271676300579,0.7861271676300579 claude-opus-4.8,person_medicare_eligible,172,172,1.0,0.029069767441860465,,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395 claude-opus-4.8,person_wic_eligible,177,177,1.0,0.07909604519774012,,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598 -claude-opus-4.8,reduced_price_school_meals_eligible,100,100,1.0,0.06,,0.94,0.94,0.94,0.94,0.94,0.94,0.94 +claude-opus-4.8,reduced_price_school_meals_eligible,100,100,1.0,0.05,,0.95,0.95,0.95,0.95,0.95,0.95,0.95 claude-opus-4.8,self_employment_tax,100,100,1.0,0.022667846679689775,0.0001990587122224831,,1.0,1.0,1.0,1.0,1.0,0.9999840753030221 -claude-opus-4.8,snap,97,97,1.0,583.2749577158505,1.7020654696106645,,0.7319587628865979,0.7319587628865979,0.7319587628865979,0.7422680412371134,0.7345360824742267,0.7971210096948417 +claude-opus-4.8,snap,93,93,1.0,533.0860215053764,1.5275875662679301,,0.7634408602150538,0.7634408602150538,0.7634408602150538,0.7741935483870968,0.7661290322580645,0.8211884763998233 claude-opus-4.8,ssi,97,97,1.0,131.2164948453608,0.6589103291713961,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.9864142200170846 -claude-opus-4.8,state_income_tax_before_refundable_credits,100,100,1.0,341.23064289989475,0.3348153009790936,,0.57,0.57,0.62,0.65,0.6024999999999999,0.8360586677195816 -claude-opus-4.8,state_refundable_credits,100,100,1.0,58.89138693332672,1.1705316441328408,,0.78,0.79,0.79,0.79,0.7875000000000001,0.8056388931797368 +claude-opus-4.8,state_income_tax_before_refundable_credits,86,86,1.0,240.04871799712961,0.284835865809115,,0.6511627906976745,0.6511627906976745,0.6976744186046512,0.7325581395348837,0.6831395348837209,0.8657549305317039 +claude-opus-4.8,state_refundable_credits,98,98,1.0,53.63291890767156,1.455666318564968,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8086734693877551,0.831563420488021 claude-opus-4.8,tanf,100,100,1.0,10.75041015625,0.17725445767969758,,0.99,0.99,0.99,0.99,0.99,0.998227455423203 -claude-opus-5,federal_income_tax_before_refundable_credits,100,100,1.0,1935.9647790527345,0.552756771029521,,0.5,0.5,0.57,0.63,0.5499999999999999,0.8154361739017841 -claude-opus-5,federal_refundable_credits,100,100,1.0,204.18189178466798,0.17073388143809629,,0.83,0.84,0.86,0.87,0.85,0.9278045954130475 +claude-opus-5,federal_income_tax_before_refundable_credits,82,82,1.0,666.7105921303354,0.5309223465039903,,0.5609756097560976,0.5609756097560976,0.6219512195121951,0.6829268292682927,0.6067073170731708,0.840076924328095 +claude-opus-5,federal_refundable_credits,98,98,1.0,195.23770048180404,0.16099216947942677,,0.8469387755102041,0.8571428571428571,0.8775510204081632,0.8877551020408163,0.8673469387755102,0.9309090422012888 claude-opus-5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 claude-opus-5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-opus-5,payroll_tax,100,100,1.0,139.3476744873047,0.016010063094513534,,0.85,0.9,0.91,0.96,0.905,0.9897535596195113 +claude-opus-5,payroll_tax,99,99,1.0,139.61623675722063,0.015436618248772748,,0.8585858585858586,0.9090909090909091,0.9191919191919192,0.9595959595959596,0.9116161616161615,0.9901766974780536 claude-opus-5,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272 claude-opus-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-opus-5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -claude-opus-5,person_medicaid_eligible,177,177,1.0,0.06779661016949153,,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084 +claude-opus-5,person_medicaid_eligible,173,173,1.0,0.05202312138728324,,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167 claude-opus-5,person_medicare_eligible,172,172,1.0,0.03488372093023256,,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675 claude-opus-5,person_wic_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 -claude-opus-5,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-opus-5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-opus-5,self_employment_tax,100,100,1.0,54.23308532714844,0.0003572365859157118,,0.98,0.98,0.98,0.98,0.98,0.9799714210731267 -claude-opus-5,snap,97,97,1.0,297.8025430895619,1.6517755403544798,,0.7525773195876289,0.7525773195876289,0.7628865979381443,0.7938144329896907,0.7654639175257731,0.8443993585969686 +claude-opus-5,snap,93,93,1.0,246.79569892473117,1.2430147066149062,,0.7849462365591398,0.7849462365591398,0.7956989247311828,0.8279569892473119,0.7983870967741935,0.8638796646667334 claude-opus-5,ssi,97,97,1.0,129.7319587628866,0.2856273584768493,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.983801497763364 -claude-opus-5,state_income_tax_before_refundable_credits,100,100,1.0,273.163437138176,0.36304565307762865,,0.59,0.61,0.69,0.78,0.6675,0.8794444459429391 -claude-opus-5,state_refundable_credits,100,100,1.0,66.2835454416275,1.3179521547015922,,0.79,0.8,0.81,0.81,0.8025000000000001,0.8216511001442447 +claude-opus-5,state_income_tax_before_refundable_credits,86,86,1.0,132.5505235589937,0.2597295307946852,,0.6744186046511628,0.6744186046511628,0.7558139534883721,0.8372093023255814,0.7354651162790699,0.9093966753041797 +claude-opus-5,state_refundable_credits,98,98,1.0,62.69297532159455,1.4088503474920302,,0.8163265306122449,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8290816326530612,0.841661214370422 claude-opus-5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-sonnet-4.6,federal_income_tax_before_refundable_credits,100,100,1.0,954.2971545410156,0.42346375804849146,,0.48,0.5,0.67,0.73,0.595,0.8649431003976104 -claude-sonnet-4.6,federal_refundable_credits,100,100,1.0,132.5344808959961,0.1815531082763526,,0.89,0.9,0.91,0.93,0.9075000000000001,0.9463980959240742 +claude-opus-5.5,federal_income_tax_before_refundable_credits,82,82,1.0,305.7924935689786,0.16241021606028166,,0.8292682926829268,0.8658536585365854,0.9024390243902439,0.9024390243902439,0.875,0.9354732127052555 +claude-opus-5.5,federal_refundable_credits,98,98,1.0,0.003099128567447277,2.48862942154661e-05,,1.0,1.0,1.0,1.0,1.0,0.999997206640445 +claude-opus-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-opus-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,payroll_tax,99,99,1.0,23.89883875374844,0.0047634325182585304,,0.9292929292929293,0.9494949494949495,0.9494949494949495,1.0,0.9570707070707071,0.9969687247611082 +claude-opus-5.5,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 +claude-opus-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,person_medicaid_eligible,173,173,1.0,0.023121387283236993,,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763 +claude-opus-5.5,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +claude-opus-5.5,self_employment_tax,100,100,1.0,0.0015256591796855901,1.8684773766991133e-05,,1.0,1.0,1.0,1.0,1.0,0.9999985052180986 +claude-opus-5.5,snap,93,93,1.0,85.5294623655914,0.5025349792380832,,0.8924731182795699,0.9139784946236559,0.9139784946236559,0.9139784946236559,0.9086021505376344,0.9318439754231114 +claude-opus-5.5,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 +claude-opus-5.5,state_income_tax_before_refundable_credits,86,86,1.0,66.77772406090139,0.12146610240211328,,0.8023255813953488,0.8372093023255814,0.8837209302325582,0.9186046511627907,0.8604651162790697,0.9460001968364722 +claude-opus-5.5,state_refundable_credits,98,98,1.0,15.400220080589763,0.28047062896970437,,0.8979591836734694,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9056122448979591,0.9484849865157685 +claude-opus-5.5,tanf,100,100,1.0,0.55041015625,0.009075249439740776,,0.99,1.0,1.0,1.0,0.9975,0.9999092475056025 +claude-sonnet-4.6,federal_income_tax_before_refundable_credits,82,82,1.0,410.1034650104802,0.43777181793870706,,0.5487804878048781,0.573170731707317,0.7439024390243902,0.7926829268292683,0.6646341463414634,0.8950328317002604 +claude-sonnet-4.6,federal_refundable_credits,98,98,1.0,122.44227164132255,0.1746156152687253,,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9387755102040817,0.9158163265306123,0.9497880431841226 claude-sonnet-4.6,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 claude-sonnet-4.6,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -claude-sonnet-4.6,payroll_tax,100,100,1.0,49.73063181152339,0.0073000592476852024,,0.83,0.87,0.97,1.0,0.9175,0.9953279620814814 +claude-sonnet-4.6,payroll_tax,99,99,1.0,49.095991630168825,0.006589828220731302,,0.8383838383838383,0.8787878787878788,0.9797979797979798,1.0,0.9242424242424242,0.9958064729504437 claude-sonnet-4.6,person_chip_eligible,177,177,1.0,0.10734463276836158,,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384 claude-sonnet-4.6,person_early_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 claude-sonnet-4.6,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -claude-sonnet-4.6,person_medicaid_eligible,177,177,1.0,0.12994350282485875,,0.8700564971751412,0.8700564971751412,0.8700564971751412,0.8700564971751412,0.8700564971751412,0.8700564971751412,0.8700564971751412 +claude-sonnet-4.6,person_medicaid_eligible,173,173,1.0,0.12716763005780346,,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965 claude-sonnet-4.6,person_medicare_eligible,172,172,1.0,0.05813953488372093,,0.9418604651162791,0.9418604651162791,0.9418604651162791,0.9418604651162791,0.9418604651162791,0.9418604651162791,0.9418604651162791 claude-sonnet-4.6,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -claude-sonnet-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-sonnet-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-sonnet-4.6,self_employment_tax,100,100,1.0,34.44671838378906,0.00010697989103262154,,0.98,0.99,0.99,0.99,0.9875,0.9899914416087173 -claude-sonnet-4.6,snap,97,97,1.0,272.3532375060406,0.587390989950604,,0.7835051546391752,0.7835051546391752,0.8144329896907216,0.8350515463917526,0.8041237113402062,0.8563444264755804 +claude-sonnet-4.6,snap,93,93,1.0,257.6774193548387,0.5978746457112838,,0.8172043010752689,0.8172043010752689,0.8387096774193549,0.8602150537634409,0.8333333333333334,0.8744471957173431 claude-sonnet-4.6,ssi,97,97,1.0,10.68041237113402,0.05230402529537605,,0.979381443298969,0.979381443298969,0.9896907216494846,1.0,0.9871134020618557,0.9989215664887551 -claude-sonnet-4.6,state_income_tax_before_refundable_credits,100,100,1.0,238.7952242963791,0.3155090875319906,,0.54,0.56,0.65,0.7,0.6125,0.84267891769434 -claude-sonnet-4.6,state_refundable_credits,100,100,1.0,66.1113774728775,0.9188051758008948,,0.79,0.79,0.79,0.79,0.79,0.8170509130818121 +claude-sonnet-4.6,state_income_tax_before_refundable_credits,86,86,1.0,194.42291114075238,0.31717611179789446,,0.627906976744186,0.6511627906976745,0.7093023255813954,0.7558139534883721,0.686046511627907,0.8730296195273088 +claude-sonnet-4.6,state_refundable_credits,98,98,1.0,62.84603654608435,0.9347312353332404,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8385187526938945 claude-sonnet-4.6,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-sonnet-5,federal_income_tax_before_refundable_credits,100,100,1.0,2741.590760498047,0.9787422693898803,,0.44,0.47,0.48,0.53,0.48,0.7196465466852675 -claude-sonnet-5,federal_refundable_credits,100,100,1.0,314.9303207397461,0.8457814730261796,,0.78,0.78,0.8,0.8,0.79,0.8407440333345684 +claude-sonnet-5,federal_income_tax_before_refundable_credits,82,82,1.0,1235.6977032917302,1.0669475767508045,,0.5,0.5121951219512195,0.524390243902439,0.573170731707317,0.5274390243902439,0.7570716795537921 +claude-sonnet-5,federal_refundable_credits,98,98,1.0,257.36857106734294,0.8007227080206903,,0.7959183673469388,0.7959183673469388,0.8163265306122449,0.8163265306122449,0.8061224489795918,0.8579020748311923 claude-sonnet-5,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 claude-sonnet-5,local_income_tax,100,100,1.0,38.309,,,0.99,0.99,0.99,0.99,0.99,0.99 -claude-sonnet-5,payroll_tax,100,100,1.0,211.2534711669922,0.02402879423927846,,0.76,0.86,0.9,0.94,0.865,0.9846215716868619 +claude-sonnet-5,payroll_tax,99,99,1.0,212.25037481751104,0.023584098688381593,,0.7676767676767676,0.8686868686868687,0.9090909090909091,0.9393939393939394,0.8712121212121212,0.9849919371983026 claude-sonnet-5,person_chip_eligible,177,177,1.0,0.0847457627118644,,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356,0.9152542372881356 claude-sonnet-5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-sonnet-5,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 -claude-sonnet-5,person_medicaid_eligible,177,177,1.0,0.2542372881355932,,0.7457627118644068,0.7457627118644068,0.7457627118644068,0.7457627118644068,0.7457627118644068,0.7457627118644068,0.7457627118644068 +claude-sonnet-5,person_medicaid_eligible,173,173,1.0,0.26011560693641617,,0.7398843930635838,0.7398843930635838,0.7398843930635838,0.7398843930635838,0.7398843930635838,0.7398843930635838,0.7398843930635838 claude-sonnet-5,person_medicare_eligible,172,172,1.0,0.011627906976744186,,0.9883720930232558,0.9883720930232558,0.9883720930232558,0.9883720930232558,0.9883720930232558,0.9883720930232558,0.9883720930232558 claude-sonnet-5,person_wic_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 -claude-sonnet-5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +claude-sonnet-5,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 claude-sonnet-5,self_employment_tax,100,100,1.0,82.20191687011719,0.06053951590470468,,0.96,0.96,0.96,0.96,0.96,0.9651568387276237 -claude-sonnet-5,snap,97,97,1.0,681.8384808412533,1.7425776597179794,,0.711340206185567,0.711340206185567,0.711340206185567,0.7319587628865979,0.7164948453608248,0.7609941569471296 +claude-sonnet-5,snap,93,93,1.0,650.2150537634409,2.0402639067599613,,0.7419354838709677,0.7419354838709677,0.7419354838709677,0.7526881720430108,0.7446236559139785,0.7837982352557759 claude-sonnet-5,ssi,97,97,1.0,141.5257731958763,0.6684457728437233,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.9862176129310573 -claude-sonnet-5,state_income_tax_before_refundable_credits,100,100,1.0,500.994878153801,0.49334180112056275,,0.54,0.56,0.59,0.64,0.5825,0.7977633130883103 -claude-sonnet-5,state_refundable_credits,100,100,1.0,95.61138693332673,3.837933222308551,,0.78,0.78,0.78,0.79,0.7825,0.7956129706836254 +claude-sonnet-5,state_income_tax_before_refundable_credits,86,86,1.0,308.57630662585416,0.36401092716495426,,0.6162790697674418,0.627906976744186,0.6511627906976745,0.686046511627907,0.6453488372093024,0.8265078161052485 +claude-sonnet-5,state_refundable_credits,98,98,1.0,82.61251074440625,4.366205870595451,,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8137755102040817,0.8220544240805049 claude-sonnet-5,tanf,100,100,1.0,21.40958984375,0.35300469300628384,,0.99,0.99,0.99,0.99,0.99,0.9964699530699371 -deepseek-v4-flash-0731,federal_income_tax_before_refundable_credits,100,100,1.0,1773.5454823730468,0.7355911138114194,,0.48,0.49,0.63,0.63,0.5575,0.8043744588698144 -deepseek-v4-flash-0731,federal_refundable_credits,100,100,1.0,300.20168245849607,0.19765432671325733,,0.81,0.84,0.88,0.88,0.8524999999999999,0.8943049375272765 +claude-sonnet-5.5,federal_income_tax_before_refundable_credits,82,82,1.0,152.49378632336126,0.1267096119922374,,0.8048780487804879,0.8292682926829268,0.8902439024390244,0.926829268292683,0.8628048780487805,0.9474618681983406 +claude-sonnet-5.5,federal_refundable_credits,98,98,1.0,6.057284994319991,0.0020666673613282675,,0.9795918367346939,0.9795918367346939,0.9897959183673469,0.9897959183673469,0.9846938775510204,0.9895639455002591 +claude-sonnet-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +claude-sonnet-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +claude-sonnet-5.5,payroll_tax,99,99,1.0,26.00660045276989,0.006205183231822571,,0.9292929292929293,0.9494949494949495,0.9494949494949495,0.98989898989899,0.9545454545454546,0.9960512470342947 +claude-sonnet-5.5,person_chip_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 +claude-sonnet-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-sonnet-5.5,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 +claude-sonnet-5.5,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 +claude-sonnet-5.5,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-sonnet-5.5,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +claude-sonnet-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +claude-sonnet-5.5,self_employment_tax,100,100,1.0,0.002925219726561318,2.0997990722224957e-05,,1.0,1.0,1.0,1.0,1.0,0.9899983201607424 +claude-sonnet-5.5,snap,93,93,1.0,132.10752688172042,0.5926619058930392,,0.8602150537634409,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.8924731182795699,0.9102035545906004 +claude-sonnet-5.5,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 +claude-sonnet-5.5,state_income_tax_before_refundable_credits,86,86,1.0,74.50006805220315,0.2587764044844357,,0.7209302325581395,0.7441860465116279,0.8488372093023255,0.8604651162790697,0.7936046511627906,0.9198460450664387 +claude-sonnet-5.5,state_refundable_credits,98,98,1.0,80.80052383636942,1.105303908488078,,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.7959183673469388,0.848005399455642 +claude-sonnet-5.5,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 +deepseek-v4-flash-0731,federal_income_tax_before_refundable_credits,82,82,1.0,770.0108560403963,0.7426262693114861,,0.5609756097560976,0.573170731707317,0.7317073170731707,0.7317073170731707,0.649390243902439,0.8509498375409471 +deepseek-v4-flash-0731,federal_refundable_credits,98,98,1.0,275.2400379040776,0.1366504809061787,,0.826530612244898,0.8571428571428571,0.8979591836734694,0.8979591836734694,0.8698979591836735,0.9030290276533881 deepseek-v4-flash-0731,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 deepseek-v4-flash-0731,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -deepseek-v4-flash-0731,payroll_tax,100,100,1.0,267.29993348144535,0.042788115563960005,,0.79,0.81,0.86,0.92,0.845,0.9726156060390657 +deepseek-v4-flash-0731,payroll_tax,99,99,1.0,268.86094281190816,0.0426397191701152,,0.797979797979798,0.8181818181818182,0.8686868686868687,0.9191919191919192,0.8510101010101011,0.9728656332553812 deepseek-v4-flash-0731,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272 deepseek-v4-flash-0731,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 deepseek-v4-flash-0731,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -deepseek-v4-flash-0731,person_medicaid_eligible,177,177,1.0,0.04519774011299435,,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056 +deepseek-v4-flash-0731,person_medicaid_eligible,173,173,1.0,0.04046242774566474,,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352 deepseek-v4-flash-0731,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 deepseek-v4-flash-0731,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -deepseek-v4-flash-0731,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +deepseek-v4-flash-0731,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 deepseek-v4-flash-0731,self_employment_tax,100,100,1.0,34.44071120605469,0.00019954144430183585,,0.99,0.99,0.99,0.99,0.99,0.9799840366844559 -deepseek-v4-flash-0731,snap,97,97,1.0,157.73047932103736,0.4042595089763843,,0.7938144329896907,0.8144329896907216,0.8556701030927835,0.865979381443299,0.8324742268041238,0.8982225602824894 +deepseek-v4-flash-0731,snap,93,93,1.0,137.18494623655914,0.3493761853590002,,0.8279569892473119,0.8387096774193549,0.8924731182795699,0.9032258064516129,0.8655913978494624,0.9189044041971289 deepseek-v4-flash-0731,ssi,97,97,1.0,0.9896907216494846,0.004759119120710396,,0.979381443298969,1.0,1.0,1.0,0.9948453608247423,0.9999018738325628 -deepseek-v4-flash-0731,state_income_tax_before_refundable_credits,100,100,1.0,331.88509519481653,0.4481698901874545,,0.46,0.5,0.54,0.59,0.5225,0.7194503807473847 -deepseek-v4-flash-0731,state_refundable_credits,100,100,1.0,82.94313817653656,0.7593095554535045,,0.78,0.78,0.79,0.79,0.785,0.8207727760818253 +deepseek-v4-flash-0731,state_income_tax_before_refundable_credits,86,86,1.0,269.01068571379017,0.43273931716104264,,0.5232558139534884,0.5581395348837209,0.5813953488372093,0.6162790697674418,0.5697674418604652,0.7295171117546718 +deepseek-v4-flash-0731,state_refundable_credits,98,98,1.0,82.26940176107445,0.8221637663529808,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8403452641152473 deepseek-v4-flash-0731,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -deepseek-v4-pro,federal_income_tax_before_refundable_credits,100,100,1.0,2167.578434326172,0.5732526927375406,,0.41,0.42,0.54,0.6,0.49250000000000005,0.7422565685050497 -deepseek-v4-pro,federal_refundable_credits,100,100,1.0,145.1672808959961,0.37224340208580753,,0.85,0.86,0.88,0.9,0.8724999999999999,0.921608357728845 +deepseek-v4-pro,federal_income_tax_before_refundable_credits,82,82,1.0,765.3589424542683,0.47986377209757336,,0.4878048780487805,0.5,0.6219512195121951,0.6829268292682927,0.5731707317073171,0.7933837505103175 +deepseek-v4-pro,federal_refundable_credits,98,98,1.0,108.02676143724092,0.3154062599117203,,0.8673469387755102,0.8775510204081632,0.8979591836734694,0.9183673469387755,0.8903061224489796,0.933985011642562 deepseek-v4-pro,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 deepseek-v4-pro,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -deepseek-v4-pro,payroll_tax,100,100,1.0,207.7551757568359,0.09060634658302444,,0.68,0.73,0.84,0.89,0.785,0.9510409656937536 +deepseek-v4-pro,payroll_tax,99,99,1.0,209.23108652442392,0.09159215170393993,,0.6868686868686869,0.7373737373737373,0.8383838383838383,0.8888888888888888,0.7878787878787878,0.9508343150842496 deepseek-v4-pro,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272 deepseek-v4-pro,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 deepseek-v4-pro,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -deepseek-v4-pro,person_medicaid_eligible,177,177,1.0,0.062146892655367235,,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328 +deepseek-v4-pro,person_medicaid_eligible,173,173,1.0,0.06358381502890173,,0.9364161849710982,0.9364161849710982,0.9364161849710982,0.9364161849710982,0.9364161849710982,0.9364161849710982,0.9364161849710982 deepseek-v4-pro,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 deepseek-v4-pro,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -deepseek-v4-pro,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +deepseek-v4-pro,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 deepseek-v4-pro,self_employment_tax,100,100,1.0,34.42659978027344,1.906980687814305e-05,,0.99,0.99,0.99,0.99,0.99,0.9899984744154497 -deepseek-v4-pro,snap,97,97,1.0,165.30432461138852,0.5647300178265945,,0.8144329896907216,0.8144329896907216,0.845360824742268,0.865979381443299,0.8350515463917526,0.8907174195561638 +deepseek-v4-pro,snap,93,93,1.0,142.6658064516129,0.5539193918409281,,0.8494623655913979,0.8494623655913979,0.8817204301075269,0.9032258064516129,0.8709677419354839,0.9118177194200853 deepseek-v4-pro,ssi,97,97,1.0,7.051546391752577,0.03606961075014709,,0.979381443298969,0.9896907216494846,0.9896907216494846,1.0,0.9896907216494846,0.999256296685564 -deepseek-v4-pro,state_income_tax_before_refundable_credits,100,100,1.0,351.20057278270724,0.4975103396570636,,0.5,0.5,0.6,0.63,0.5575,0.787046097355654 -deepseek-v4-pro,state_refundable_credits,100,100,1.0,81.50188266086577,0.7324457113683307,,0.79,0.79,0.8,0.8,0.7949999999999999,0.8350177335929553 +deepseek-v4-pro,state_income_tax_before_refundable_credits,86,86,1.0,251.78893897278363,0.3804795997178542,,0.5697674418604651,0.5697674418604651,0.6511627906976745,0.686046511627907,0.6191860465116279,0.7982536836055245 +deepseek-v4-pro,state_refundable_credits,98,98,1.0,77.93138592875732,0.784702024008234,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8609095024121691 deepseek-v4-pro,tanf,100,100,1.0,5.62958984375,0.09282156496446835,,0.99,0.99,0.99,1.0,0.9924999999999999,0.9990717843503554 -deepseek-v4-pro-0813,federal_income_tax_before_refundable_credits,100,100,1.0,1801.3791659667966,0.6927568436056936,,0.42,0.42,0.47,0.51,0.455,0.7394834310931655 -deepseek-v4-pro-0813,federal_refundable_credits,100,100,1.0,120.10827010498048,0.34734770126069275,,0.89,0.91,0.91,0.93,0.91,0.9548447988361098 +deepseek-v4-pro-0813,federal_income_tax_before_refundable_credits,82,82,1.0,901.4561799733232,0.6728676462222739,,0.5,0.5,0.5487804878048781,0.5609756097560976,0.5274390243902439,0.782773902448932 +deepseek-v4-pro-0813,federal_refundable_credits,98,98,1.0,83.4375667525311,0.28892672511029677,,0.9081632653061225,0.9285714285714286,0.9285714285714286,0.9489795918367347,0.9285714285714286,0.9675694492223136 deepseek-v4-pro-0813,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 deepseek-v4-pro-0813,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -deepseek-v4-pro-0813,payroll_tax,100,100,1.0,162.6636300537109,0.04631167459343155,,0.79,0.84,0.92,0.95,0.875,0.9802382971843574 +deepseek-v4-pro-0813,payroll_tax,99,99,1.0,163.88710116299717,0.04674190856032396,,0.797979797979798,0.8484848484848485,0.9191919191919192,0.9494949494949495,0.8787878787878788,0.9802326934658074 deepseek-v4-pro-0813,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 deepseek-v4-pro-0813,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 deepseek-v4-pro-0813,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -deepseek-v4-pro-0813,person_medicaid_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 +deepseek-v4-pro-0813,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 deepseek-v4-pro-0813,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 deepseek-v4-pro-0813,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -deepseek-v4-pro-0813,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +deepseek-v4-pro-0813,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 deepseek-v4-pro-0813,self_employment_tax,100,100,1.0,0.0045822998046855905,2.7091941169829102e-05,,1.0,1.0,1.0,1.0,1.0,0.9999978326447063 -deepseek-v4-pro-0813,snap,97,97,1.0,122.06987913780605,0.4472708281853013,,0.7938144329896907,0.7938144329896907,0.8350515463917526,0.865979381443299,0.8221649484536082,0.8906844940293801 +deepseek-v4-pro-0813,snap,93,93,1.0,90.04215053763441,0.412657687654936,,0.8279569892473119,0.8387096774193549,0.9032258064516129,0.9032258064516129,0.8682795698924731,0.9100586028009229 deepseek-v4-pro-0813,ssi,97,97,1.0,1.6082474226804124,0.0076697449040421615,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9998418609298135 -deepseek-v4-pro-0813,state_income_tax_before_refundable_credits,100,100,1.0,217.25262865184789,0.2945134331314276,,0.55,0.61,0.67,0.73,0.64,0.8493176276154557 -deepseek-v4-pro-0813,state_refundable_credits,100,100,1.0,45.811915522727965,0.6675734575673242,,0.8,0.8,0.81,0.81,0.805,0.849809573910862 +deepseek-v4-pro-0813,state_income_tax_before_refundable_credits,86,86,1.0,141.20354245434254,0.2371757692888958,,0.627906976744186,0.6976744186046512,0.7325581395348837,0.7906976744186046,0.7122093023255814,0.8675037243184849 +deepseek-v4-pro-0813,state_refundable_credits,98,98,1.0,40.30240190307462,0.6197609478856888,,0.826530612244898,0.826530612244898,0.826530612244898,0.8367346938775511,0.8290816326530612,0.8759622748781388 deepseek-v4-pro-0813,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3-flash-preview,federal_income_tax_before_refundable_credits,100,100,1.0,1457.411917138672,0.6258631725885073,,0.46,0.47,0.5,0.55,0.495,0.7640719583410751 -gemini-3-flash-preview,federal_refundable_credits,100,100,1.0,115.79088494873045,0.2010748166663469,,0.84,0.85,0.87,0.92,0.87,0.9338602738333748 +deepseek-v4.1-flash,federal_income_tax_before_refundable_credits,82,82,1.0,380.22547682450454,0.227581994538357,,0.6341463414634146,0.6585365853658537,0.7317073170731707,0.7560975609756098,0.6951219512195121,0.8766772952701026 +deepseek-v4.1-flash,federal_refundable_credits,98,98,1.0,97.08988818110251,0.1129619008852461,,0.8979591836734694,0.9387755102040817,0.9591836734693877,0.9591836734693877,0.9387755102040817,0.977116521329207 +deepseek-v4.1-flash,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +deepseek-v4.1-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +deepseek-v4.1-flash,payroll_tax,99,99,1.0,38.342086662523684,0.005261477684698107,,0.8888888888888888,0.9292929292929293,0.9797979797979798,0.98989898989899,0.946969696969697,0.9966517869279194 +deepseek-v4.1-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786 +deepseek-v4.1-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +deepseek-v4.1-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +deepseek-v4.1-flash,person_medicaid_eligible,173,173,1.0,0.04046242774566474,,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352 +deepseek-v4.1-flash,person_medicare_eligible,172,172,1.0,0.046511627906976744,,0.9534883720930233,0.9534883720930233,0.9534883720930233,0.9534883720930233,0.9534883720930233,0.9534883720930233,0.9534883720930233 +deepseek-v4.1-flash,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +deepseek-v4.1-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +deepseek-v4.1-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 +deepseek-v4.1-flash,snap,93,93,1.0,171.45505376344084,0.5203532952046377,,0.8494623655913979,0.8924731182795699,0.9032258064516129,0.9032258064516129,0.8870967741935484,0.9078885966190541 +deepseek-v4.1-flash,ssi,97,97,1.0,11.268041237113403,0.057990420209935334,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9988043212327848 +deepseek-v4.1-flash,state_income_tax_before_refundable_credits,86,86,1.0,90.22480290412902,0.12250724686018281,,0.6511627906976745,0.6744186046511628,0.7674418604651163,0.8488372093023255,0.7354651162790697,0.8874974720255174 +deepseek-v4.1-flash,state_refundable_credits,98,98,1.0,46.36871282052021,0.5478642572780168,,0.8367346938775511,0.8367346938775511,0.8469387755102041,0.8571428571428571,0.8443877551020409,0.8891677894795479 +deepseek-v4.1-flash,tanf,100,100,1.0,18.224789843750003,0.300493208457008,,0.99,0.99,0.99,0.99,0.99,0.99699506791543 +gemini-3-flash-preview,federal_income_tax_before_refundable_credits,82,82,1.0,738.2186471036586,0.5734728739033849,,0.5365853658536586,0.5487804878048781,0.5853658536585366,0.5975609756097561,0.5670731707317074,0.7893089153594549 +gemini-3-flash-preview,federal_refundable_credits,98,98,1.0,115.1862306555923,0.2283897327994623,,0.8571428571428571,0.8673469387755102,0.8775510204081632,0.9183673469387755,0.8801020408163265,0.9335480912163869 gemini-3-flash-preview,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3-flash-preview,local_income_tax,100,100,1.0,82.772,,,0.98,0.98,0.98,0.98,0.98,0.98 -gemini-3-flash-preview,payroll_tax,100,100,1.0,222.3663746337891,0.08215317056808995,,0.65,0.67,0.78,0.84,0.735,0.9474219708364224 +gemini-3-flash-preview,payroll_tax,99,99,1.0,224.35755023378317,0.08327194678978311,,0.6565656565656566,0.6767676767676768,0.7777777777777778,0.8383838383838383,0.7373737373737373,0.9470087611337743 gemini-3-flash-preview,person_chip_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 gemini-3-flash-preview,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3-flash-preview,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3-flash-preview,person_medicaid_eligible,177,177,1.0,0.04519774011299435,,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056 +gemini-3-flash-preview,person_medicaid_eligible,173,173,1.0,0.04046242774566474,,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352,0.9595375722543352 gemini-3-flash-preview,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 gemini-3-flash-preview,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -gemini-3-flash-preview,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +gemini-3-flash-preview,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3-flash-preview,self_employment_tax,100,100,1.0,34.431631323242186,8.50016838390725e-05,,0.99,0.99,0.99,0.99,0.99,0.9899931998652929 -gemini-3-flash-preview,snap,97,97,1.0,119.55454730790915,0.36394091060748246,,0.7835051546391752,0.7938144329896907,0.845360824742268,0.865979381443299,0.8221649484536082,0.8949794280378639 +gemini-3-flash-preview,snap,93,93,1.0,111.59784946236557,0.35513631357379194,,0.8172043010752689,0.8172043010752689,0.8709677419354839,0.8817204301075269,0.8467741935483871,0.9073465368122656 gemini-3-flash-preview,ssi,97,97,1.0,19.79381443298969,0.09693297442642909,,0.979381443298969,0.979381443298969,0.979381443298969,0.9896907216494846,0.9819587628865979,0.9980013819705891 -gemini-3-flash-preview,state_income_tax_before_refundable_credits,100,100,1.0,243.09986564159394,0.36927536726522503,,0.55,0.57,0.63,0.7,0.6125,0.8412470445445244 -gemini-3-flash-preview,state_refundable_credits,100,100,1.0,33.783902326393125,0.4776376463862694,,0.8,0.8,0.8,0.82,0.805,0.8896960942588834 +gemini-3-flash-preview,state_income_tax_before_refundable_credits,86,86,1.0,141.87490356955416,0.27109317436894176,,0.627906976744186,0.6395348837209303,0.686046511627907,0.7558139534883721,0.6773255813953488,0.8712130396772643 +gemini-3-flash-preview,state_refundable_credits,98,98,1.0,29.953633193385848,0.40578631341772303,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8571428571428571,0.8367346938775511,0.9152637383518468 gemini-3-flash-preview,tanf,100,100,1.0,16.45958984375,0.27138831253689305,,0.99,0.99,0.99,0.99,0.99,0.9972861168746312 -gemini-3.1-flash-lite-preview,federal_income_tax_before_refundable_credits,100,100,1.0,2345.803333349609,0.8235333237814103,,0.4,0.42,0.51,0.52,0.4625,0.6752314525252495 -gemini-3.1-flash-lite-preview,federal_refundable_credits,100,100,1.0,308.64797149658204,0.5880391376555931,,0.83,0.83,0.83,0.86,0.8374999999999999,0.8873484144159725 +gemini-3.1-flash-lite-preview,federal_income_tax_before_refundable_credits,82,82,1.0,1307.2033155487804,0.9141110639686388,,0.4634146341463415,0.47560975609756095,0.5487804878048781,0.5609756097560976,0.5121951219512195,0.7011653771585691 +gemini-3.1-flash-lite-preview,federal_refundable_credits,98,98,1.0,272.9479859021245,0.5649095930538068,,0.8469387755102041,0.8469387755102041,0.8469387755102041,0.8673469387755102,0.8520408163265306,0.8957754538408992 gemini-3.1-flash-lite-preview,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.1-flash-lite-preview,local_income_tax,100,100,1.0,31.5,,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3.1-flash-lite-preview,payroll_tax,100,100,1.0,330.8882183837891,0.034699962970139525,,0.76,0.84,0.85,0.93,0.8450000000000001,0.9777920236991108 +gemini-3.1-flash-lite-preview,payroll_tax,99,99,1.0,333.0915336223564,0.03442318320178963,,0.7676767676767676,0.8484848484848485,0.8585858585858586,0.9292929292929293,0.851010101010101,0.9780943379624975 gemini-3.1-flash-lite-preview,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.1-flash-lite-preview,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 gemini-3.1-flash-lite-preview,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -gemini-3.1-flash-lite-preview,person_medicaid_eligible,177,177,1.0,0.096045197740113,,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887 +gemini-3.1-flash-lite-preview,person_medicaid_eligible,173,173,1.0,0.08670520231213873,,0.9132947976878613,0.9132947976878613,0.9132947976878613,0.9132947976878613,0.9132947976878613,0.9132947976878613,0.9132947976878613 gemini-3.1-flash-lite-preview,person_medicare_eligible,172,172,1.0,0.03488372093023256,,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675 gemini-3.1-flash-lite-preview,person_wic_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 -gemini-3.1-flash-lite-preview,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gemini-3.1-flash-lite-preview,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.1-flash-lite-preview,self_employment_tax,100,100,1.0,20.57040251464844,0.0024292504799131066,,0.96,0.98,0.99,0.99,0.98,0.979805659961607 -gemini-3.1-flash-lite-preview,snap,97,97,1.0,317.876346548808,1.1753161674131307,,0.8144329896907216,0.8144329896907216,0.8144329896907216,0.8350515463917526,0.8195876288659795,0.8564046356737899 +gemini-3.1-flash-lite-preview,snap,93,93,1.0,248.08602150537635,1.2318771910502693,,0.8494623655913979,0.8494623655913979,0.8494623655913979,0.8709677419354839,0.8548387096774194,0.8927662707850879 gemini-3.1-flash-lite-preview,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -gemini-3.1-flash-lite-preview,state_income_tax_before_refundable_credits,100,100,1.0,577.9122522748947,0.505345241708035,,0.46,0.48,0.51,0.56,0.5025,0.7241366484999304 -gemini-3.1-flash-lite-preview,state_refundable_credits,100,100,1.0,118.9813774728775,0.9758427583177776,,0.79,0.79,0.79,0.79,0.79,0.7950730207532666 +gemini-3.1-flash-lite-preview,state_income_tax_before_refundable_credits,86,86,1.0,382.12637509878294,0.3835454982223732,,0.5348837209302325,0.5581395348837209,0.5813953488372093,0.5930232558139535,0.5668604651162791,0.7541088840295558 +gemini-3.1-flash-lite-preview,state_refundable_credits,98,98,1.0,114.71338348485986,0.9725137431284357,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8213750267723281 gemini-3.1-flash-lite-preview,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3.1-pro-preview,federal_income_tax_before_refundable_credits,100,100,1.0,1457.905817138672,0.5264694044761647,,0.46,0.47,0.51,0.53,0.4925,0.7727715114808862 -gemini-3.1-pro-preview,federal_refundable_credits,100,100,1.0,126.6988808959961,0.34491661056316497,,0.85,0.88,0.88,0.9,0.8775,0.9251608406267886 +gemini-3.1-pro-preview,federal_income_tax_before_refundable_credits,82,82,1.0,712.8772008384146,0.419506726408391,,0.524390243902439,0.5365853658536586,0.5487804878048781,0.573170731707317,0.5457317073170732,0.7902522567405342 +gemini-3.1-pro-preview,federal_refundable_credits,98,98,1.0,92.27329204948582,0.29286136212849573,,0.8673469387755102,0.8979591836734694,0.8979591836734694,0.9183673469387755,0.8954081632653061,0.9365155613937403 gemini-3.1-pro-preview,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.1-pro-preview,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.1-pro-preview,payroll_tax,100,100,1.0,237.56112907714842,0.09268460175709882,,0.66,0.67,0.76,0.82,0.7274999999999999,0.9406818548754567 +gemini-3.1-pro-preview,payroll_tax,99,99,1.0,239.70578704525727,0.09397054355322071,,0.6666666666666666,0.6767676767676768,0.7575757575757576,0.8181818181818182,0.7297979797979799,0.9402005631934051 gemini-3.1-pro-preview,person_chip_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272 gemini-3.1-pro-preview,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.1-pro-preview,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 -gemini-3.1-pro-preview,person_medicaid_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 +gemini-3.1-pro-preview,person_medicaid_eligible,173,173,1.0,0.028901734104046242,,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537 gemini-3.1-pro-preview,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 gemini-3.1-pro-preview,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -gemini-3.1-pro-preview,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gemini-3.1-pro-preview,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.1-pro-preview,self_employment_tax,100,100,1.0,0.011282299804687455,0.000145894883346029,,1.0,1.0,1.0,1.0,1.0,0.9999883284093323 -gemini-3.1-pro-preview,snap,97,97,1.0,146.00474866100194,0.5172227691602627,,0.8041237113402062,0.8041237113402062,0.845360824742268,0.865979381443299,0.8298969072164949,0.8887341538585107 +gemini-3.1-pro-preview,snap,93,93,1.0,120.24946236559141,0.4976533739500507,,0.8387096774193549,0.8387096774193549,0.8924731182795699,0.8924731182795699,0.8655913978494624,0.908930173533864 gemini-3.1-pro-preview,ssi,97,97,1.0,110.84536082474227,0.06619001968689023,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.9896907216494846,0.9768041237113402,0.9883259789755281 -gemini-3.1-pro-preview,state_income_tax_before_refundable_credits,100,100,1.0,238.86674607372285,0.35107355638127524,,0.57,0.58,0.66,0.71,0.63,0.8572228803372687 -gemini-3.1-pro-preview,state_refundable_credits,100,100,1.0,59.179375519752504,0.5768080999105047,,0.81,0.81,0.81,0.82,0.8125,0.858870299018794 +gemini-3.1-pro-preview,state_income_tax_before_refundable_credits,86,86,1.0,177.27644822741664,0.27135958799885185,,0.6511627906976745,0.6627906976744186,0.7209302325581395,0.7790697674418605,0.7034883720930233,0.8813624702458875 +gemini-3.1-pro-preview,state_refundable_credits,98,98,1.0,54.76942216620154,0.5418215330691546,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8800735959668899 gemini-3.1-pro-preview,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3.5-flash,federal_income_tax_before_refundable_credits,100,100,1.0,1744.903372314453,0.6379029127839916,,0.44,0.45,0.48,0.5,0.4675,0.7474100410381019 -gemini-3.5-flash,federal_refundable_credits,100,100,1.0,145.3862808959961,0.31459301539932955,,0.84,0.85,0.85,0.9,0.86,0.9191029079980872 +gemini-3.5-flash,federal_income_tax_before_refundable_credits,82,82,1.0,963.3471233088795,0.5831620630839038,,0.5121951219512195,0.524390243902439,0.5487804878048781,0.5609756097560976,0.5365853658536586,0.7687022603287419 +gemini-3.5-flash,federal_refundable_credits,98,98,1.0,108.31145531479194,0.2475640087124831,,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.9183673469387755,0.8775510204081632,0.9313958765730888 gemini-3.5-flash,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.5-flash,local_income_tax,100,100,1.0,28.1812,,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3.5-flash,payroll_tax,100,100,1.0,286.73518815917964,0.11224518401095761,,0.61,0.63,0.69,0.79,0.6799999999999999,0.9281630822329872 +gemini-3.5-flash,payroll_tax,99,99,1.0,289.5179679361979,0.11394436060679412,,0.6161616161616161,0.6262626262626263,0.6868686868686869,0.7878787878787878,0.6792929292929293,0.927489952341131 gemini-3.5-flash,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 gemini-3.5-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.5-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.5-flash,person_medicaid_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 +gemini-3.5-flash,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 gemini-3.5-flash,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 gemini-3.5-flash,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.5-flash,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +gemini-3.5-flash,reduced_price_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97 gemini-3.5-flash,self_employment_tax,100,100,1.0,0.008882299804683953,0.00014162740073649118,,1.0,1.0,1.0,1.0,1.0,0.9999886698079411 -gemini-3.5-flash,snap,97,97,1.0,117.48956865133698,0.40597392531810983,,0.8041237113402062,0.8144329896907216,0.8762886597938144,0.8865979381443299,0.845360824742268,0.9193002833001903 +gemini-3.5-flash,snap,93,93,1.0,86.02150537634408,0.275273051861285,,0.8387096774193549,0.8494623655913979,0.9032258064516129,0.9139784946236559,0.8763440860215054,0.9400155949011109 gemini-3.5-flash,ssi,97,97,1.0,5.072164948453608,0.025894900504143357,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9994660845256877 -gemini-3.5-flash,state_income_tax_before_refundable_credits,100,100,1.0,268.0647614545822,0.4202758291359997,,0.53,0.56,0.6,0.68,0.5925,0.8165025794906317 -gemini-3.5-flash,state_refundable_credits,100,100,1.0,46.81258302707672,0.635664365829951,,0.78,0.79,0.79,0.79,0.7875000000000001,0.8565104831757102 +gemini-3.5-flash,state_income_tax_before_refundable_credits,86,86,1.0,216.40433272051257,0.3966010803927464,,0.6046511627906976,0.627906976744186,0.6627906976744186,0.7093023255813954,0.6511627906976744,0.8326373748796952 +gemini-3.5-flash,state_refundable_credits,98,98,1.0,41.819855690197066,0.6144782376127053,,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8137755102040817,0.876932568601748 gemini-3.5-flash,tanf,100,100,1.0,7.66958984375,0.1264574066124597,,0.99,0.99,0.99,0.99,0.99,0.9987354259338754 -gemini-3.5-flash-lite,federal_income_tax_before_refundable_credits,100,100,1.0,1927.909012451172,0.6282653275915364,,0.49,0.51,0.55,0.59,0.535,0.7488370469901828 -gemini-3.5-flash-lite,federal_refundable_credits,100,100,1.0,353.81008087158204,0.7377125317494377,,0.86,0.86,0.86,0.86,0.86,0.8940973708725731 +gemini-3.5-flash-lite,federal_income_tax_before_refundable_credits,82,82,1.0,1090.8342999714177,0.6006407896027405,,0.5487804878048781,0.5609756097560976,0.5975609756097561,0.6341463414634146,0.5853658536585367,0.7719096528072235 +gemini-3.5-flash-lite,federal_refundable_credits,98,98,1.0,302.53982263681837,0.6900239011584266,,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.9123442559924215 gemini-3.5-flash-lite,free_school_meals_eligible,100,100,1.0,0.04,,0.96,0.96,0.96,0.96,0.96,0.96,0.96 gemini-3.5-flash-lite,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.5-flash-lite,payroll_tax,100,100,1.0,524.5180274658203,0.06456644251050425,,0.78,0.82,0.85,0.91,0.8400000000000001,0.9586774767932772 +gemini-3.5-flash-lite,payroll_tax,99,99,1.0,528.6792195638021,0.06476520169407127,,0.7878787878787878,0.8282828282828283,0.8585858585858586,0.9090909090909091,0.845959595959596,0.9587857807401364 gemini-3.5-flash-lite,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 gemini-3.5-flash-lite,person_early_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 gemini-3.5-flash-lite,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -gemini-3.5-flash-lite,person_medicaid_eligible,177,177,1.0,0.13559322033898305,,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017 +gemini-3.5-flash-lite,person_medicaid_eligible,173,173,1.0,0.12716763005780346,,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965,0.8728323699421965 gemini-3.5-flash-lite,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.5-flash-lite,person_wic_eligible,177,177,1.0,0.01694915254237288,,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272,0.9830508474576272 -gemini-3.5-flash-lite,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gemini-3.5-flash-lite,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.5-flash-lite,self_employment_tax,100,100,1.0,48.990022631835934,0.33281306928834004,,0.95,0.97,0.97,0.97,0.9649999999999999,0.9633749544569329 -gemini-3.5-flash-lite,snap,97,97,1.0,424.3748275914143,0.8695162587965176,,0.8247422680412371,0.8247422680412371,0.845360824742268,0.845360824742268,0.8350515463917526,0.8476105525820535 +gemini-3.5-flash-lite,snap,93,93,1.0,351.2043010752688,0.8279607700662017,,0.8602150537634409,0.8602150537634409,0.8817204301075269,0.8817204301075269,0.8709677419354839,0.884263548270316 gemini-3.5-flash-lite,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -gemini-3.5-flash-lite,state_income_tax_before_refundable_credits,100,100,1.0,620.3028699018479,0.782688533016045,,0.53,0.53,0.55,0.6,0.5525,0.745577530354175 -gemini-3.5-flash-lite,state_refundable_credits,100,100,1.0,104.28137747287751,0.9762432706146689,,0.79,0.79,0.79,0.79,0.79,0.7949889131709196 +gemini-3.5-flash-lite,state_income_tax_before_refundable_credits,86,86,1.0,430.98314403955334,0.5208521464526006,,0.6162790697674418,0.6162790697674418,0.627906976744186,0.6627906976744186,0.6308139534883721,0.78658574709952 +gemini-3.5-flash-lite,state_refundable_credits,98,98,1.0,99.71338348485986,0.9733517660885691,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8212211041878138 gemini-3.5-flash-lite,tanf,100,100,1.0,118.64958984375,1.0,,0.98,0.98,0.98,0.98,0.98,0.98 -gemini-3.6-flash,federal_income_tax_before_refundable_credits,100,100,1.0,1296.2101244628907,0.3713388970275841,,0.49,0.5,0.6,0.69,0.57,0.8539755339413407 -gemini-3.6-flash,federal_refundable_credits,100,100,1.0,67.81066072998047,0.1492465457705398,,0.88,0.9,0.94,0.95,0.9175,0.9705979490498298 +gemini-3.6-flash,federal_income_tax_before_refundable_credits,82,82,1.0,561.1786092320883,0.3344431907464499,,0.5609756097560976,0.573170731707317,0.6585365853658537,0.7073170731707317,0.625,0.8726988996671031 +gemini-3.6-flash,federal_refundable_credits,98,98,1.0,61.924495961714754,0.15370798512544603,,0.8979591836734694,0.9183673469387755,0.9489795918367347,0.9591836734693877,0.9311224489795918,0.9725429812614295 gemini-3.6-flash,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.6-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.6-flash,payroll_tax,100,100,1.0,204.29153137207027,0.05105107658256172,,0.7,0.71,0.81,0.87,0.7725,0.9673273109871606 +gemini-3.6-flash,payroll_tax,99,99,1.0,206.09791057548134,0.05167455413982679,,0.7070707070707071,0.7171717171717171,0.8080808080808081,0.8686868686868687,0.7752525252525253,0.9671161928201103 gemini-3.6-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786 gemini-3.6-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.6-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.6-flash,person_medicaid_eligible,177,177,1.0,0.05084745762711865,,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814 +gemini-3.6-flash,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 gemini-3.6-flash,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.6-flash,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.6-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gemini-3.6-flash,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.6-flash,self_employment_tax,100,100,1.0,0.0027822998046872272,2.518602137197596e-05,,1.0,1.0,1.0,1.0,1.0,0.9999979851182903 -gemini-3.6-flash,snap,97,97,1.0,112.16961214360502,0.32905795366404433,,0.8144329896907216,0.8144329896907216,0.8762886597938144,0.8865979381443299,0.8479381443298969,0.9320207710073325 +gemini-3.6-flash,snap,93,93,1.0,99.29032258064517,0.29846023082180606,,0.8494623655913979,0.8494623655913979,0.9032258064516129,0.9032258064516129,0.8763440860215054,0.9475270645087798 gemini-3.6-flash,ssi,97,97,1.0,1.731958762886598,0.008747871130837852,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9998196315230755 -gemini-3.6-flash,state_income_tax_before_refundable_credits,100,100,1.0,196.83616813182832,0.990985118437224,,0.55,0.58,0.7,0.73,0.6399999999999999,0.851575593964569 -gemini-3.6-flash,state_refundable_credits,100,100,1.0,44.53802339572907,0.638005636245991,,0.8,0.8,0.81,0.82,0.8075,0.8560188163883419 +gemini-3.6-flash,state_income_tax_before_refundable_credits,86,86,1.0,155.2569035922649,0.7124883043575841,,0.627906976744186,0.6627906976744186,0.7558139534883721,0.7674418604651163,0.7034883720930233,0.8684608765006566 +gemini-3.6-flash,state_refundable_credits,98,98,1.0,43.578893688941484,0.6382126164966077,,0.8163265306122449,0.8163265306122449,0.826530612244898,0.826530612244898,0.8214285714285714,0.8725731928883781 gemini-3.6-flash,tanf,100,100,1.0,7.664789843750004,0.12637826345564096,,0.99,0.99,0.99,0.99,0.99,0.9987362173654435 -gemini-3.7-flash,federal_income_tax_before_refundable_credits,100,100,1.0,1971.5559195800784,0.6197064240135476,,0.47,0.48,0.54,0.59,0.52,0.7779639590392045 -gemini-3.7-flash,federal_refundable_credits,100,100,1.0,156.2984808959961,0.3677584599089761,,0.84,0.86,0.89,0.9,0.8724999999999999,0.9121914002118331 +gemini-3.7-flash,federal_income_tax_before_refundable_credits,82,82,1.0,790.6302397341844,0.5885153990209967,,0.5487804878048781,0.5609756097560976,0.6097560975609756,0.6463414634146342,0.5914634146341464,0.8114168263574779 +gemini-3.7-flash,federal_refundable_credits,98,98,1.0,116.61798592703684,0.3015668492484629,,0.8571428571428571,0.8775510204081632,0.9081632653061225,0.9183673469387755,0.8903061224489796,0.9253343332476215 gemini-3.7-flash,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.7-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.7-flash,payroll_tax,100,100,1.0,208.76437541503907,0.08502004357608353,,0.71,0.71,0.8,0.85,0.7675,0.9455871721113066 +gemini-3.7-flash,payroll_tax,99,99,1.0,210.6028035481771,0.08617317007296617,,0.7171717171717171,0.7171717171717171,0.797979797979798,0.8484848484848485,0.7702020202020202,0.9451625281353851 gemini-3.7-flash,person_chip_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786 gemini-3.7-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.7-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.7-flash,person_medicaid_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 +gemini-3.7-flash,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 gemini-3.7-flash,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.7-flash,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -gemini-3.7-flash,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +gemini-3.7-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.7-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 -gemini-3.7-flash,snap,97,97,1.0,192.98285855951997,0.584383079145046,,0.8041237113402062,0.8144329896907216,0.8556701030927835,0.865979381443299,0.8350515463917526,0.8913613895597357 +gemini-3.7-flash,snap,93,93,1.0,153.13978494623655,0.4787112113664552,,0.8387096774193549,0.8387096774193549,0.8817204301075269,0.8817204301075269,0.8602150537634409,0.9155966841502522 gemini-3.7-flash,ssi,97,97,1.0,11.257731958762887,0.0577729290234779,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9988088055871448 -gemini-3.7-flash,state_income_tax_before_refundable_credits,100,100,1.0,266.05897878856655,0.4342434487380769,,0.6,0.63,0.7,0.74,0.6675,0.8557870233218727 -gemini-3.7-flash,state_refundable_credits,100,100,1.0,37.9141581735611,0.5436745568230077,,0.81,0.8,0.82,0.83,0.8150000000000001,0.8858283430671685 +gemini-3.7-flash,state_income_tax_before_refundable_credits,86,86,1.0,166.44406632334687,0.31171639221099096,,0.686046511627907,0.7209302325581395,0.7790697674418605,0.8023255813953488,0.747093023255814,0.8812850441330728 +gemini-3.7-flash,state_refundable_credits,98,98,1.0,33.076565023344386,0.5133737012636046,,0.8367346938775511,0.826530612244898,0.8469387755102041,0.8571428571428571,0.8418367346938777,0.9062100282421046 gemini-3.7-flash,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gemini-3.8-flash,federal_income_tax_before_refundable_credits,100,100,1.0,1066.2643183105467,0.40304718013237045,,0.5,0.5,0.64,0.74,0.595,0.867569422304299 -gemini-3.8-flash,federal_refundable_credits,100,100,1.0,92.72586072998048,0.19133566681474312,,0.89,0.91,0.93,0.95,0.9199999999999999,0.9751263633140834 +gemini-3.8-flash,federal_income_tax_before_refundable_credits,82,82,1.0,546.486308117378,0.3695996554636807,,0.573170731707317,0.573170731707317,0.7073170731707317,0.7682926829268293,0.6554878048780488,0.8890823842789214 +gemini-3.8-flash,federal_refundable_credits,98,98,1.0,80.45643473722498,0.1819363933843249,,0.9081632653061225,0.9285714285714286,0.9387755102040817,0.9591836734693877,0.9336734693877551,0.9795785680895146 gemini-3.8-flash,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gemini-3.8-flash,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.8-flash,payroll_tax,100,100,1.0,138.82892712402344,0.020032415295262843,,0.81,0.84,0.91,0.96,0.88,0.9871792542110317 +gemini-3.8-flash,payroll_tax,99,99,1.0,139.0942697236032,0.019524285157953042,,0.8181818181818182,0.8484848484848485,0.9191919191919192,0.9595959595959596,0.8863636363636365,0.9875754548994844 gemini-3.8-flash,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028 gemini-3.8-flash,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.8-flash,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.8-flash,person_medicaid_eligible,177,177,1.0,0.04519774011299435,,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056,0.9548022598870056 +gemini-3.8-flash,person_medicaid_eligible,173,173,1.0,0.028901734104046242,,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537 gemini-3.8-flash,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gemini-3.8-flash,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gemini-3.8-flash,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +gemini-3.8-flash,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 gemini-3.8-flash,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 -gemini-3.8-flash,snap,97,97,1.0,83.11071475314111,0.47140414187817326,,0.8144329896907216,0.8247422680412371,0.8865979381443299,0.8969072164948454,0.8556701030927835,0.9070735009079491 +gemini-3.8-flash,snap,93,93,1.0,52.98924731182796,0.41794495554072547,,0.8494623655913979,0.8494623655913979,0.9247311827956989,0.9247311827956989,0.8870967741935484,0.9308248986878556 gemini-3.8-flash,ssi,97,97,1.0,8.412371134020619,0.040561924526802184,,0.979381443298969,0.979381443298969,1.0,1.0,0.9896907216494846,0.9991636716592412 -gemini-3.8-flash,state_income_tax_before_refundable_credits,100,100,1.0,249.2778580244064,0.37059693651776393,,0.62,0.65,0.73,0.76,0.69,0.8551390581856023 -gemini-3.8-flash,state_refundable_credits,100,100,1.0,38.56078001003265,0.583582746222936,,0.83,0.82,0.83,0.84,0.83,0.8774476232931835 +gemini-3.8-flash,state_income_tax_before_refundable_credits,86,86,1.0,166.90487376168718,0.18881248970926562,,0.6976744186046512,0.7325581395348837,0.8023255813953488,0.8255813953488372,0.7645348837209303,0.8876235501014191 +gemini-3.8-flash,state_refundable_credits,98,98,1.0,32.65154933909981,0.4995806944963879,,0.8571428571428571,0.8469387755102041,0.8673469387755102,0.8673469387755102,0.8596938775510203,0.9082402806027043 gemini-3.8-flash,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -glm-5.2,federal_income_tax_before_refundable_credits,100,91,0.91,1900.192681737208,0.4824531885311725,,0.43,0.43,0.5700000000000001,0.6400000000000001,0.5175000000000001,0.765244589153494 -glm-5.2,federal_refundable_credits,100,90,0.9,184.33857675509984,0.4766750040159229,,0.79,0.79,0.7999999999999999,0.7999999999999999,0.7949999999999999,0.8270992496385667 +glm-5.2,federal_income_tax_before_refundable_credits,82,76,0.926829268292683,569.0805494449013,0.36285345175152156,,0.4878048780487805,0.4878048780487805,0.6463414634146342,0.7073170731707318,0.5823170731707318,0.8107754400075127 +glm-5.2,federal_refundable_credits,98,88,0.8979591836734694,155.9154522566362,0.4696846776647093,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8337980332280309 glm-5.2,free_school_meals_eligible,100,95,0.95,0.021052631578947368,,0.93,0.93,0.93,0.93,0.93,0.93,0.93 glm-5.2,local_income_tax,100,95,0.95,37.4,,,0.94,0.94,0.94,0.94,0.94,0.94 -glm-5.2,payroll_tax,100,90,0.9,159.53915186903214,0.047224253978130015,,0.73,0.7999999999999999,0.85,0.8600000000000001,0.81,0.8834321866964008 +glm-5.2,payroll_tax,99,90,0.9090909090909091,159.53915186903214,0.047224253978130015,,0.7373737373737373,0.808080808080808,0.8585858585858586,0.8686868686868687,0.8181818181818181,0.8923557441377785 glm-5.2,person_chip_eligible,177,165,0.9322033898305084,0.0,,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084 glm-5.2,person_early_head_start_eligible,38,34,0.8947368421052632,0.0,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632 glm-5.2,person_head_start_eligible,38,34,0.8947368421052632,0.0,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632 -glm-5.2,person_medicaid_eligible,177,165,0.9322033898305084,0.07878787878787878,,0.8587570621468926,0.8587570621468926,0.8587570621468926,0.8587570621468926,0.8587570621468926,0.8587570621468926,0.8587570621468926 +glm-5.2,person_medicaid_eligible,173,161,0.930635838150289,0.062111801242236024,,0.8728323699421966,0.8728323699421966,0.8728323699421966,0.8728323699421966,0.8728323699421966,0.8728323699421966,0.8728323699421966 glm-5.2,person_medicare_eligible,172,160,0.9302325581395349,0.01875,,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186 glm-5.2,person_wic_eligible,177,165,0.9322033898305084,0.006060606060606061,,0.9265536723163841,0.9265536723163841,0.9265536723163841,0.9265536723163841,0.9265536723163841,0.9265536723163841,0.9265536723163841 glm-5.2,reduced_price_school_meals_eligible,100,95,0.95,0.010526315789473684,,0.94,0.94,0.94,0.94,0.94,0.94,0.94 glm-5.2,self_employment_tax,100,92,0.92,40.462241555918816,3.352336279141471e-05,,0.91,0.91,0.91,0.91,0.91,0.8999976533646046 -glm-5.2,snap,97,92,0.9484536082474226,173.76680756942085,1.0528899810023542,,0.7835051546391752,0.8041237113402061,0.8247422680412371,0.8350515463917526,0.8118556701030928,0.8582084661955846 +glm-5.2,snap,93,89,0.956989247311828,160.2334831460674,1.1696998867207566,,0.8172043010752689,0.8279569892473119,0.8602150537634409,0.881720430107527,0.8467741935483871,0.8828924267797433 glm-5.2,ssi,97,92,0.9484536082474226,42.84804347826087,0.1165670367207515,,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9278350515463917,0.9369426078688582 -glm-5.2,state_income_tax_before_refundable_credits,100,92,0.92,678.1509128651413,1.6432847495716034,,0.45,0.46,0.53,0.56,0.5,0.6631731209123422 -glm-5.2,state_refundable_credits,100,92,0.92,56.99146796454554,0.9182678644949198,,0.7200000000000001,0.7200000000000001,0.7200000000000001,0.7200000000000001,0.7200000000000001,0.7351917843909145 +glm-5.2,state_income_tax_before_refundable_credits,86,79,0.9186046511627907,418.3952914291092,1.3217642863698869,,0.5232558139534884,0.5348837209302325,0.6046511627906976,0.6395348837209303,0.5755813953488372,0.7134769848393566 +glm-5.2,state_refundable_credits,98,90,0.9183673469387755,45.22765429963006,0.9019214373939038,,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7448979591836735,0.7603997799907289 glm-5.2,tanf,100,95,0.95,63.841673519736844,1.0,,0.94,0.94,0.94,0.94,0.94,0.94 -glm-5.3,federal_income_tax_before_refundable_credits,100,97,0.97,947.9669899222777,0.3830765319601031,,0.48,0.5299999999999999,0.6499999999999999,0.6900000000000001,0.5874999999999999,0.7967055570128564 -glm-5.3,federal_refundable_credits,100,97,0.97,279.65414046140063,0.39675342905903016,,0.81,0.82,0.85,0.86,0.835,0.8777187526999692 +glm-5.3,federal_income_tax_before_refundable_credits,82,80,0.975609756097561,593.4866197509766,0.446117869051389,,0.5609756097560975,0.6097560975609756,0.6951219512195121,0.7317073170731707,0.649390243902439,0.8209921667839896 +glm-5.3,federal_refundable_credits,98,96,0.9795918367346939,253.52654483795166,0.34476172707133146,,0.826530612244898,0.836734693877551,0.8673469387755102,0.8775510204081632,0.8520408163265306,0.895311606335925 glm-5.3,free_school_meals_eligible,100,97,0.97,0.020618556701030927,,0.95,0.95,0.95,0.95,0.95,0.95,0.95 glm-5.3,local_income_tax,100,97,0.97,0.0,,,0.97,0.97,0.97,0.97,0.97,0.97 -glm-5.3,payroll_tax,100,97,0.97,204.5856839209488,0.02969181028576974,,0.81,0.84,0.87,0.9199999999999999,0.86,0.9418879957256805 +glm-5.3,payroll_tax,99,96,0.9696969696969697,205.5442846934001,0.02931926355810288,,0.8181818181818182,0.8484848484848485,0.8787878787878788,0.9191919191919192,0.8661616161616161,0.9418267089546851 glm-5.3,person_chip_eligible,177,168,0.9491525423728814,0.017857142857142856,,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084,0.9322033898305084 glm-5.3,person_early_head_start_eligible,38,35,0.9210526315789473,0.0,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473 glm-5.3,person_head_start_eligible,38,35,0.9210526315789473,0.0,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473 -glm-5.3,person_medicaid_eligible,177,168,0.9491525423728814,0.07142857142857142,,0.8813559322033899,0.8813559322033899,0.8813559322033899,0.8813559322033899,0.8813559322033899,0.8813559322033899,0.8813559322033899 +glm-5.3,person_medicaid_eligible,173,166,0.9595375722543352,0.06626506024096386,,0.8959537572254335,0.8959537572254335,0.8959537572254335,0.8959537572254335,0.8959537572254335,0.8959537572254335,0.8959537572254335 glm-5.3,person_medicare_eligible,172,163,0.9476744186046512,0.0,,0.9476744186046512,0.9476744186046512,0.9476744186046512,0.9476744186046512,0.9476744186046512,0.9476744186046512,0.9476744186046512 glm-5.3,person_wic_eligible,177,168,0.9491525423728814,0.0,,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814 -glm-5.3,reduced_price_school_meals_eligible,100,97,0.97,0.010309278350515464,,0.96,0.96,0.96,0.96,0.96,0.96,0.96 +glm-5.3,reduced_price_school_meals_eligible,100,97,0.97,0.020618556701030927,,0.95,0.95,0.95,0.95,0.95,0.95,0.95 glm-5.3,self_employment_tax,100,97,0.97,0.00884773175740759,0.00010957748686821732,,0.97,0.97,0.97,0.97,0.97,0.9699912338010506 -glm-5.3,snap,97,94,0.9690721649484536,143.74262253781583,0.5270643997911939,,0.7731958762886598,0.7835051546391752,0.8247422680412371,0.8350515463917525,0.8041237113402062,0.8592678931412379 +glm-5.3,snap,93,90,0.967741935483871,116.08888888888889,0.49502346382348883,,0.8064516129032259,0.8387096774193549,0.8602150537634409,0.8709677419354839,0.8440860215053764,0.8789404585705626 glm-5.3,ssi,97,95,0.979381443298969,102.6842105263158,0.0005675368898978433,,0.9587628865979382,0.9690721649484535,0.9690721649484535,0.9690721649484535,0.9664948453608246,0.9690604631569093 -glm-5.3,state_income_tax_before_refundable_credits,100,97,0.97,329.61908964943643,1.1135879064451264,,0.49,0.54,0.6,0.6,0.5575,0.7572924564599712 -glm-5.3,state_refundable_credits,100,97,0.97,72.40760562641104,1.145340393633365,,0.76,0.76,0.76,0.76,0.76,0.8054039188499995 +glm-5.3,state_income_tax_before_refundable_credits,86,85,0.9883720930232558,223.48080443415924,0.7640520113690127,,0.569767441860465,0.627906976744186,0.6744186046511628,0.6744186046511628,0.6366279069767441,0.8057820801044986 +glm-5.3,state_refundable_credits,98,95,0.9693877551020408,67.42412189463565,1.176670574544347,,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.8298857836589686 glm-5.3,tanf,100,97,0.97,235.72122664304123,1.0,,0.94,0.94,0.94,0.94,0.94,0.94 -gpt-5.4-mini,federal_income_tax_before_refundable_credits,100,100,1.0,4372.801055908203,0.7194043783123879,,0.5,0.5,0.52,0.53,0.5125,0.6346858984100537 -gpt-5.4-mini,federal_refundable_credits,100,100,1.0,544.7896267700195,0.7755116887811467,,0.81,0.81,0.81,0.81,0.81,0.8391834804584509 +gpt-5.4-mini,federal_income_tax_before_refundable_credits,82,82,1.0,1605.057772008384,0.7593133141206919,,0.5609756097560976,0.5609756097560976,0.5609756097560976,0.573170731707317,0.5640243902439025,0.6607725282914204 +gpt-5.4-mini,federal_refundable_credits,98,98,1.0,514.8456610854792,0.7891013249460958,,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.826530612244898,0.8502029125060504 gpt-5.4-mini,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.4-mini,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.4-mini,payroll_tax,100,100,1.0,1047.5937669726563,0.2665907620114608,,0.54,0.65,0.66,0.74,0.6475,0.819381912312665 +gpt-5.4-mini,payroll_tax,99,99,1.0,1038.6377445154671,0.25662650891971256,,0.5454545454545454,0.6565656565656566,0.6666666666666666,0.7474747474747475,0.654040404040404,0.8265912114955364 gpt-5.4-mini,person_chip_eligible,177,177,1.0,0.07909604519774012,,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598 gpt-5.4-mini,person_early_head_start_eligible,38,38,1.0,0.07894736842105263,,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473,0.9210526315789473 gpt-5.4-mini,person_head_start_eligible,38,38,1.0,0.15789473684210525,,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947,0.8421052631578947 -gpt-5.4-mini,person_medicaid_eligible,177,177,1.0,0.1638418079096045,,0.8361581920903954,0.8361581920903954,0.8361581920903954,0.8361581920903954,0.8361581920903954,0.8361581920903954,0.8361581920903954 +gpt-5.4-mini,person_medicaid_eligible,173,173,1.0,0.15606936416184972,,0.8439306358381503,0.8439306358381503,0.8439306358381503,0.8439306358381503,0.8439306358381503,0.8439306358381503,0.8439306358381503 gpt-5.4-mini,person_medicare_eligible,172,172,1.0,0.029069767441860465,,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395,0.9709302325581395 gpt-5.4-mini,person_wic_eligible,177,177,1.0,0.13559322033898305,,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017,0.864406779661017 -gpt-5.4-mini,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-5.4-mini,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.4-mini,self_employment_tax,100,100,1.0,73.5364168701172,0.08842207254988142,,0.9,0.95,0.95,0.95,0.9375,0.9629262341960094 -gpt-5.4-mini,snap,97,97,1.0,608.9285460796552,1.2941930876626702,,0.7628865979381443,0.7628865979381443,0.7628865979381443,0.7628865979381443,0.7628865979381443,0.7879170893800542 +gpt-5.4-mini,snap,93,93,1.0,546.5591397849462,1.3854698063203785,,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.7956989247311828,0.8215651525215243 gpt-5.4-mini,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -gpt-5.4-mini,state_income_tax_before_refundable_credits,100,100,1.0,896.7456386518478,0.7016337314125992,,0.56,0.56,0.58,0.59,0.5725,0.6882974954925822 -gpt-5.4-mini,state_refundable_credits,100,100,1.0,122.28137747287751,1.0,,0.79,0.79,0.79,0.79,0.79,0.79 +gpt-5.4-mini,state_income_tax_before_refundable_credits,86,86,1.0,498.3730254894079,0.6869272295473228,,0.6395348837209303,0.6395348837209303,0.6627906976744186,0.6744186046511628,0.6540697674418604,0.7487463152741897 +gpt-5.4-mini,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449 gpt-5.4-mini,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gpt-5.4-nano,federal_income_tax_before_refundable_credits,100,100,1.0,4591.384639892578,0.9634468146831402,,0.47,0.48,0.48,0.48,0.4775,0.6117267633314158 -gpt-5.4-nano,federal_refundable_credits,100,100,1.0,676.220080871582,0.9948789980355367,,0.86,0.86,0.86,0.86,0.86,0.8606657302553802 +gpt-5.4-nano,federal_income_tax_before_refundable_credits,82,82,1.0,2425.040467797256,1.1033600815326783,,0.524390243902439,0.5365853658536586,0.5365853658536586,0.5365853658536586,0.5335365853658537,0.6329741544612347 +gpt-5.4-nano,federal_refundable_credits,98,98,1.0,633.4683940653898,1.0,,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632,0.8775510204081632 gpt-5.4-nano,free_school_meals_eligible,100,100,1.0,0.08,,0.92,0.92,0.92,0.92,0.92,0.92,0.92 gpt-5.4-nano,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.4-nano,payroll_tax,100,100,1.0,1723.165034301758,0.25091920588190963,,0.17,0.22,0.36,0.46,0.3025,0.6291785927671683 +gpt-5.4-nano,payroll_tax,99,99,1.0,1734.1287214182844,0.2502213763215863,,0.1717171717171717,0.2222222222222222,0.36363636363636365,0.46464646464646464,0.3055555555555556,0.6284115327767588 gpt-5.4-nano,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.4-nano,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 gpt-5.4-nano,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 -gpt-5.4-nano,person_medicaid_eligible,177,177,1.0,0.288135593220339,,0.711864406779661,0.711864406779661,0.711864406779661,0.711864406779661,0.711864406779661,0.711864406779661,0.711864406779661 +gpt-5.4-nano,person_medicaid_eligible,173,173,1.0,0.2774566473988439,,0.7225433526011561,0.7225433526011561,0.7225433526011561,0.7225433526011561,0.7225433526011561,0.7225433526011561,0.7225433526011561 gpt-5.4-nano,person_medicare_eligible,172,172,1.0,0.06976744186046512,,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349 gpt-5.4-nano,person_wic_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 -gpt-5.4-nano,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-5.4-nano,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.4-nano,self_employment_tax,100,100,1.0,44.312894702148434,0.37297677917394395,,0.91,0.92,0.92,0.93,0.92,0.9601618576660846 -gpt-5.4-nano,snap,97,97,1.0,624.9315286223422,1.0,,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8247422680412371 +gpt-5.4-nano,snap,93,93,1.0,560.3870967741935,1.0,,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8602150537634409 gpt-5.4-nano,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -gpt-5.4-nano,state_income_tax_before_refundable_credits,100,100,1.0,873.5715611982346,0.9386845311331268,,0.52,0.52,0.54,0.58,0.54,0.6451274801110108 -gpt-5.4-nano,state_refundable_credits,100,100,1.0,122.28137747287751,1.0,,0.79,0.79,0.79,0.79,0.79,0.79 +gpt-5.4-nano,state_income_tax_before_refundable_credits,86,86,1.0,701.888582711996,0.8939009376941695,,0.5930232558139535,0.5930232558139535,0.6162790697674418,0.6395348837209303,0.6104651162790697,0.687253460923772 +gpt-5.4-nano,state_refundable_credits,98,98,1.0,118.08073042363537,1.0,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449 gpt-5.4-nano,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gpt-5.5,federal_income_tax_before_refundable_credits,100,100,1.0,663.7474453613281,0.23646089720337637,,0.61,0.65,0.7,0.77,0.6825,0.8778906666509588 -gpt-5.5,federal_refundable_credits,100,100,1.0,75.4260189819336,0.16225672756668486,,0.88,0.9,0.92,0.95,0.9125000000000001,0.9589066254163311 +gpt-5.5,federal_income_tax_before_refundable_credits,82,82,1.0,460.2016968368902,0.2042749984536805,,0.6951219512195121,0.6951219512195121,0.7317073170731707,0.7926829268292683,0.728658536585366,0.8825611258062455 +gpt-5.5,federal_refundable_credits,98,98,1.0,63.1065284573302,0.1484962879061112,,0.8877551020408163,0.9081632653061225,0.9285714285714286,0.9591836734693877,0.9209183673469388,0.9629238860513549 gpt-5.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.5,payroll_tax,100,100,1.0,62.2160158056641,0.008959655704553232,,0.88,0.9,0.96,0.98,0.93,0.994265820349086 +gpt-5.5,payroll_tax,99,99,1.0,61.430016063959954,0.008074158685088458,,0.8888888888888888,0.9090909090909091,0.9696969696969697,0.9797979797979798,0.9368686868686869,0.99486189901858 gpt-5.5,person_chip_eligible,177,177,1.0,0.05084745762711865,,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814 gpt-5.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.5,person_medicaid_eligible,177,177,1.0,0.05084745762711865,,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814,0.9491525423728814 +gpt-5.5,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 gpt-5.5,person_medicare_eligible,172,172,1.0,0.0755813953488372,,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628,0.9244186046511628 gpt-5.5,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-5.5,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.5,self_employment_tax,100,100,1.0,0.018062299804685152,0.00017368058028906577,,1.0,1.0,1.0,1.0,1.0,0.9999861055535768 -gpt-5.5,snap,97,97,1.0,149.19128292123068,0.38820715753633045,,0.7835051546391752,0.8247422680412371,0.8350515463917526,0.8350515463917526,0.8195876288659795,0.8804173022874473 +gpt-5.5,snap,93,93,1.0,136.08602150537635,0.3620869173659745,,0.8494623655913979,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8575268817204301,0.8956222588628208 gpt-5.5,ssi,97,97,1.0,117.52577319587628,0.0019214346712211785,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9896511044397687 -gpt-5.5,state_income_tax_before_refundable_credits,100,100,1.0,156.68154079003142,0.8624744704815537,,0.6,0.67,0.8,0.85,0.7300000000000001,0.9019979535049452 -gpt-5.5,state_refundable_credits,100,100,1.0,33.97207747287751,0.4242878724163192,,0.83,0.84,0.85,0.85,0.8425,0.8908995467925729 +gpt-5.5,state_income_tax_before_refundable_credits,86,86,1.0,68.22549683354401,0.5870162536956741,,0.6976744186046512,0.7325581395348837,0.8604651162790697,0.9069767441860465,0.7994186046511627,0.9307323769355182 +gpt-5.5,state_refundable_credits,98,98,1.0,32.649730281635215,0.38087954249699574,,0.8571428571428571,0.8571428571428571,0.8673469387755102,0.8775510204081632,0.8647959183673469,0.9096343697454499 gpt-5.5,tanf,100,100,1.0,8.08958984375,0.13338243283410497,,0.99,0.99,0.99,0.99,0.99,0.9986661756716589 -gpt-5.6-luna,federal_income_tax_before_refundable_credits,100,100,1.0,960.1580074511717,0.3133888361485226,,0.65,0.7,0.77,0.83,0.7375,0.8745185393810757 -gpt-5.6-luna,federal_refundable_credits,100,100,1.0,85.88539041748047,0.10117906173643951,,0.9,0.9,0.93,0.94,0.9175,0.9568467219742628 +gpt-5.6-luna,federal_income_tax_before_refundable_credits,82,82,1.0,755.4387691501524,0.31965167854071824,,0.7804878048780488,0.8170731707317073,0.8292682926829268,0.8780487804878049,0.8262195121951219,0.9175401023866179 +gpt-5.6-luna,federal_refundable_credits,98,98,1.0,62.033617454061705,0.03964802280153711,,0.9081632653061225,0.9081632653061225,0.9387755102040817,0.9489795918367347,0.9260204081632654,0.9649374668283988 gpt-5.6-luna,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.6-luna,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-luna,payroll_tax,100,100,1.0,152.90353111816407,0.05730062156720264,,0.89,0.91,0.93,0.97,0.925,0.9755487296826246 +gpt-5.6-luna,payroll_tax,99,99,1.0,153.02679920789933,0.05717752417505218,,0.898989898989899,0.9191919191919192,0.9393939393939394,0.9696969696969697,0.9318181818181819,0.9759588760155063 gpt-5.6-luna,person_chip_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 gpt-5.6-luna,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-luna,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-luna,person_medicaid_eligible,177,177,1.0,0.062146892655367235,,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328 +gpt-5.6-luna,person_medicaid_eligible,173,173,1.0,0.046242774566473986,,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526 gpt-5.6-luna,person_medicare_eligible,172,172,1.0,0.023255813953488372,,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116 gpt-5.6-luna,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-luna,reduced_price_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97 +gpt-5.6-luna,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.6-luna,self_employment_tax,100,100,1.0,34.44594216308594,0.00011566184949856606,,0.98,0.99,0.99,0.99,0.9875,0.98999074705204 -gpt-5.6-luna,snap,97,97,1.0,225.07698705299612,0.736201006460053,,0.8041237113402062,0.8350515463917526,0.8556701030927835,0.8556701030927835,0.8376288659793815,0.8812316919806541 +gpt-5.6-luna,snap,93,93,1.0,203.15354838709675,0.7818908704351123,,0.8602150537634409,0.8817204301075269,0.8924731182795699,0.9032258064516129,0.8844086021505376,0.9019856033181286 gpt-5.6-luna,ssi,97,97,1.0,98.22680412371135,0.0022701475595913734,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.9896439144833074 -gpt-5.6-luna,state_income_tax_before_refundable_credits,100,100,1.0,307.1513317963791,0.44744876759470964,,0.53,0.56,0.63,0.68,0.6000000000000001,0.7889717805737867 -gpt-5.6-luna,state_refundable_credits,100,100,1.0,58.96350780735016,0.7344619864153722,,0.79,0.79,0.79,0.79,0.79,0.8357629828527717 +gpt-5.6-luna,state_income_tax_before_refundable_credits,86,86,1.0,241.7301871899893,0.3986934222755633,,0.6046511627906976,0.627906976744186,0.6976744186046512,0.7441860465116279,0.6686046511627907,0.8075868053527013 +gpt-5.6-luna,state_refundable_credits,98,98,1.0,56.63304424149649,0.7593327685591696,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8507225538518426 gpt-5.6-luna,tanf,100,100,1.0,17.50958984375,0.28870087809100625,,0.99,0.99,0.99,0.99,0.99,0.9971129912190899 -gpt-5.6-sol,federal_income_tax_before_refundable_credits,100,100,1.0,419.03692540039066,0.15109023196850765,,0.75,0.84,0.87,0.88,0.835,0.9331172697913235 -gpt-5.6-sol,federal_refundable_credits,100,100,1.0,6.372603190917964,0.005525108834395669,,0.94,0.96,0.99,0.99,0.97,0.9892817358515285 +gpt-5.6-sol,federal_income_tax_before_refundable_credits,82,82,1.0,289.7567120331555,0.1317059224111787,,0.8902439024390244,0.9146341463414634,0.9146341463414634,0.9146341463414634,0.9085365853658537,0.9482042612914688 +gpt-5.6-sol,federal_refundable_credits,98,98,1.0,5.7052631412428205,0.004040567642417285,,0.9489795918367347,0.9693877551020408,0.9897959183673469,0.9897959183673469,0.9744897959183674,0.9893423852646266 gpt-5.6-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-sol,payroll_tax,100,100,1.0,30.7759612841797,0.011740654184803221,,0.94,0.94,0.97,0.99,0.96,0.9924859813217257 +gpt-5.6-sol,payroll_tax,99,99,1.0,29.557334729127223,0.010815706048970792,,0.9494949494949495,0.9494949494949495,0.9797979797979798,0.98989898989899,0.9671717171717172,0.9931172779688368 gpt-5.6-sol,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 gpt-5.6-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-sol,person_medicaid_eligible,177,177,1.0,0.03389830508474576,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 +gpt-5.6-sol,person_medicaid_eligible,173,173,1.0,0.028901734104046242,,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537,0.9710982658959537 gpt-5.6-sol,person_medicare_eligible,172,172,1.0,0.03488372093023256,,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675,0.9651162790697675 gpt-5.6-sol,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-sol,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-5.6-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-sol,self_employment_tax,100,100,1.0,0.0045822998046855905,2.7091941169829102e-05,,1.0,1.0,1.0,1.0,1.0,0.9999978326447063 -gpt-5.6-sol,snap,97,97,1.0,135.486387272471,0.45283323866736314,,0.8041237113402062,0.845360824742268,0.865979381443299,0.8762886597938144,0.8479381443298969,0.9000189169345859 +gpt-5.6-sol,snap,93,93,1.0,106.06451612903226,0.4119675302124047,,0.8817204301075269,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.8978494623655914,0.920907764593965 gpt-5.6-sol,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-sol,state_income_tax_before_refundable_credits,100,100,1.0,132.8726677202072,0.20288428524384547,,0.64,0.68,0.79,0.83,0.7350000000000001,0.8913895832597691 -gpt-5.6-sol,state_refundable_credits,100,100,1.0,69.50326858669281,0.5532356128201533,,0.84,0.84,0.84,0.86,0.845,0.874181207964992 +gpt-5.6-sol,state_income_tax_before_refundable_credits,86,86,1.0,87.55527773323058,0.15683604873102971,,0.7209302325581395,0.7325581395348837,0.8372093023255814,0.8604651162790697,0.7877906976744187,0.9104060295124315 +gpt-5.6-sol,state_refundable_credits,98,98,1.0,60.30785531954862,0.4599184508904545,,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8673469387755102,0.8951170192242023 gpt-5.6-sol,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565 -gpt-5.6-terra,federal_income_tax_before_refundable_credits,100,100,1.0,1074.9350924316407,0.3088931281150671,,0.63,0.69,0.76,0.82,0.725,0.90511961970652 -gpt-5.6-terra,federal_refundable_credits,100,100,1.0,53.59075584716797,0.14219126966729162,,0.9,0.91,0.94,0.95,0.925,0.9615151349432521 +gpt-5.6-terra,federal_income_tax_before_refundable_credits,82,82,1.0,393.1045152915396,0.33231758194447947,,0.7317073170731707,0.7804878048780488,0.8292682926829268,0.8658536585365854,0.8018292682926829,0.925591168182202 +gpt-5.6-terra,federal_refundable_credits,98,98,1.0,50.46908281598772,0.1548855545572883,,0.9081632653061225,0.9183673469387755,0.9489795918367347,0.9591836734693877,0.9336734693877551,0.9622067234680595 gpt-5.6-terra,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.6-terra,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-terra,payroll_tax,100,100,1.0,53.222668491210946,0.02058588162550147,,0.83,0.85,0.9,0.95,0.8825000000000001,0.986825035759679 +gpt-5.6-terra,payroll_tax,99,99,1.0,52.85340261501736,0.020253724710156254,,0.8383838383838383,0.8585858585858586,0.898989898989899,0.9494949494949495,0.8863636363636364,0.987111266093537 gpt-5.6-terra,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-terra,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-terra,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-5.6-terra,person_medicaid_eligible,177,177,1.0,0.0903954802259887,,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112 +gpt-5.6-terra,person_medicaid_eligible,173,173,1.0,0.07514450867052024,,0.9248554913294798,0.9248554913294798,0.9248554913294798,0.9248554913294798,0.9248554913294798,0.9248554913294798,0.9248554913294798 gpt-5.6-terra,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-5.6-terra,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 -gpt-5.6-terra,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-5.6-terra,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 gpt-5.6-terra,self_employment_tax,100,100,1.0,34.42530623324218,2.7180447370748754e-05,,0.99,0.99,0.99,0.99,0.99,0.9799978255642102 -gpt-5.6-terra,snap,97,97,1.0,160.87368793290915,0.5529390201954794,,0.8144329896907216,0.8350515463917526,0.8762886597938144,0.8762886597938144,0.8505154639175257,0.8927838830585242 +gpt-5.6-terra,snap,93,93,1.0,99.8731182795699,0.47646152716612583,,0.8709677419354839,0.9032258064516129,0.9139784946236559,0.9139784946236559,0.9005376344086021,0.9226451628692512 gpt-5.6-terra,ssi,97,97,1.0,99.3917525773196,0.014579785218063085,,0.9690721649484536,0.979381443298969,0.9896907216494846,0.9896907216494846,0.981958762886598,0.9893901075212771 -gpt-5.6-terra,state_income_tax_before_refundable_credits,100,100,1.0,202.7277924834061,0.322648005868746,,0.58,0.62,0.7,0.78,0.6699999999999999,0.8430177334410659 -gpt-5.6-terra,state_refundable_credits,100,100,1.0,62.20922144260407,0.8459406177806118,,0.78,0.78,0.78,0.79,0.7825,0.8227131569232958 +gpt-5.6-terra,state_income_tax_before_refundable_credits,86,86,1.0,144.56421646814573,0.2700036449340835,,0.6744186046511628,0.7093023255813954,0.7558139534883721,0.8255813953488372,0.741279069767442,0.8635962696249612 +gpt-5.6-terra,state_refundable_credits,98,98,1.0,51.78011618575272,0.7670587174238916,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8489075825139791 gpt-5.6-terra,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -gpt-6-astra,federal_income_tax_before_refundable_credits,100,100,1.0,445.59483911132804,0.1670381653653854,,0.73,0.77,0.83,0.86,0.7975,0.9380607728313914 -gpt-6-astra,federal_refundable_credits,100,100,1.0,46.80006444091796,0.12567619576549108,,0.98,0.98,0.98,0.98,0.98,0.9836620945504861 +gpt-6-astra,federal_income_tax_before_refundable_credits,82,82,1.0,209.01094559832316,0.12053699118384797,,0.8414634146341463,0.8536585365853658,0.8658536585365854,0.8780487804878049,0.8597560975609756,0.9500212475579167 +gpt-6-astra,federal_refundable_credits,98,98,1.0,0.0019240149673053412,7.631848793559927e-06,,1.0,1.0,1.0,1.0,1.0,0.9999991433639108 gpt-6-astra,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-6-astra,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-6-astra,payroll_tax,100,100,1.0,83.03050144042965,0.01882152985779105,,0.86,0.88,0.92,0.94,0.9,0.9879542208910137 +gpt-6-astra,payroll_tax,99,99,1.0,82.718688422309,0.018284344877775503,,0.8686868686868687,0.8888888888888888,0.9292929292929293,0.9393939393939394,0.9065656565656566,0.9883645078050518 gpt-6-astra,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-6-astra,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-6-astra,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-6-astra,person_medicaid_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 +gpt-6-astra,person_medicaid_eligible,173,173,1.0,0.005780346820809248,,0.9942196531791907,0.9942196531791907,0.9942196531791907,0.9942196531791907,0.9942196531791907,0.9942196531791907,0.9942196531791907 gpt-6-astra,person_medicare_eligible,172,172,1.0,0.0872093023255814,,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186 gpt-6-astra,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-6-astra,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-6-astra,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 gpt-6-astra,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 -gpt-6-astra,snap,97,97,1.0,89.05339461179126,0.2771119468823597,,0.8350515463917526,0.8865979381443299,0.9175257731958762,0.9175257731958762,0.8891752577319587,0.9308154319896895 +gpt-6-astra,snap,93,93,1.0,51.7632258064516,0.16769533166098569,,0.9032258064516129,0.946236559139785,0.946236559139785,0.946236559139785,0.935483870967742,0.955053340735561 gpt-6-astra,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 -gpt-6-astra,state_income_tax_before_refundable_credits,100,100,1.0,142.20327131786348,0.18964060028996352,,0.65,0.73,0.79,0.82,0.7474999999999999,0.9170781481400451 -gpt-6-astra,state_refundable_credits,100,100,1.0,51.4205423293686,2.296483544727249,,0.84,0.84,0.85,0.86,0.8474999999999999,0.888791087186225 +gpt-6-astra,state_income_tax_before_refundable_credits,86,86,1.0,72.17286344816517,0.12099113003574123,,0.7674418604651163,0.813953488372093,0.8488372093023255,0.8837209302325582,0.8284883720930232,0.946165884871253 +gpt-6-astra,state_refundable_credits,98,98,1.0,47.45212313434056,2.5319099387418778,,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8775510204081632,0.8724489795918366,0.9135804516381729 gpt-6-astra,tanf,100,100,1.0,0.01208984375,0.00019933924996272452,,0.99,1.0,1.0,1.0,0.9975,0.9999980066075004 -grok-4.3,federal_income_tax_before_refundable_credits,100,100,1.0,2359.705213623047,0.8980419888892918,,0.49,0.52,0.58,0.65,0.5599999999999999,0.7471445877930081 -grok-4.3,federal_refundable_credits,100,100,1.0,582.970080871582,0.8825208275191467,,0.84,0.84,0.84,0.84,0.84,0.8552722924225108 +gpt-6-luna,federal_income_tax_before_refundable_credits,82,82,1.0,144.78360530201982,0.09474387615971218,,0.8048780487804879,0.8658536585365854,0.8902439024390244,0.9146341463414634,0.8689024390243902,0.9485208318362169 +gpt-6-luna,federal_refundable_credits,98,98,1.0,18.44153642926897,0.05039417377182949,,0.9489795918367347,0.9693877551020408,0.9897959183673469,0.9897959183673469,0.9744897959183674,0.9943435111072436 +gpt-6-luna,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +gpt-6-luna,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-luna,payroll_tax,99,99,1.0,16.019333989307132,0.002640622920514413,,0.9191919191919192,0.9494949494949495,0.9797979797979798,1.0,0.9621212121212122,0.9983196035960363 +gpt-6-luna,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 +gpt-6-luna,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-luna,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-luna,person_medicaid_eligible,173,173,1.0,0.023121387283236993,,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763 +gpt-6-luna,person_medicare_eligible,172,172,1.0,0.01744186046511628,,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837 +gpt-6-luna,person_wic_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 +gpt-6-luna,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +gpt-6-luna,self_employment_tax,100,100,1.0,34.46423132324219,0.00014640257678745792,,0.99,0.99,0.99,0.99,0.99,0.989988287793857 +gpt-6-luna,snap,93,93,1.0,104.26451612903226,0.47748056751353485,,0.8602150537634409,0.8924731182795699,0.9139784946236559,0.9139784946236559,0.8951612903225806,0.9134335298805345 +gpt-6-luna,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-luna,state_income_tax_before_refundable_credits,86,86,1.0,65.08056791593863,0.08459187449731541,,0.7325581395348837,0.7441860465116279,0.8604651162790697,0.8953488372093024,0.8081395348837208,0.923979578663727 +gpt-6-luna,state_refundable_credits,98,98,1.0,43.718861553231065,0.4563971920076426,,0.8673469387755102,0.8673469387755102,0.8877551020408163,0.8877551020408163,0.8775510204081631,0.9065499925781576 +gpt-6-luna,tanf,100,100,1.0,7.66958984375,0.1264574066124597,,0.99,0.99,0.99,0.99,0.99,0.9987354259338754 +gpt-6-sol,federal_income_tax_before_refundable_credits,82,82,1.0,112.26243547541917,0.06552706016832348,,0.8780487804878049,0.9146341463414634,0.9146341463414634,0.9390243902439024,0.9115853658536586,0.9728302433448415 +gpt-6-sol,federal_refundable_credits,98,98,1.0,11.101151414520874,0.022654343243450975,,0.9489795918367347,0.9591836734693877,0.9897959183673469,0.9897959183673469,0.9719387755102041,0.9974571655543064 +gpt-6-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,payroll_tax,99,99,1.0,1.2952200353140917,0.0008703413599367797,,0.98989898989899,0.98989898989899,0.98989898989899,1.0,0.9924242424242424,0.9994461464073129 +gpt-6-sol,person_chip_eligible,177,177,1.0,0.005649717514124294,,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758,0.9943502824858758 +gpt-6-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,person_medicaid_eligible,173,173,1.0,0.023121387283236993,,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763 +gpt-6-sol,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6-sol,self_employment_tax,100,100,1.0,0.005250299804688438,3.994699523699791e-05,,1.0,1.0,1.0,1.0,1.0,0.9999968042403811 +gpt-6-sol,snap,93,93,1.0,115.83225806451613,0.5855172715081373,,0.8924731182795699,0.9032258064516129,0.9032258064516129,0.9032258064516129,0.9005376344086021,0.9202592344486833 +gpt-6-sol,ssi,97,97,1.0,10.762886597938145,0.05572160546541417,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9988511009182389 +gpt-6-sol,state_income_tax_before_refundable_credits,86,86,1.0,79.60832806026549,0.15098260025229626,,0.7441860465116279,0.7674418604651163,0.8488372093023255,0.8604651162790697,0.8052325581395348,0.9124479301445481 +gpt-6-sol,state_refundable_credits,98,98,1.0,32.428406478920756,0.5121929777284749,,0.8571428571428571,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.8622448979591837,0.8957196571519128 +gpt-6-sol,tanf,100,100,1.0,16.45958984375,0.27138831253689305,,0.99,0.99,0.99,0.99,0.99,0.9972861168746312 +gpt-6.1-sol,federal_income_tax_before_refundable_credits,82,82,1.0,226.08470131478654,0.15652881139744798,,0.8414634146341463,0.8536585365853658,0.8536585365853658,0.8780487804878049,0.8567073170731707,0.9350978099083753 +gpt-6.1-sol,federal_refundable_credits,98,98,1.0,3.978148504763224,7.631848793559927e-06,,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897959183673469,0.9897950617312578 +gpt-6.1-sol,free_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,payroll_tax,99,99,1.0,83.28207725201226,0.024635189894032693,,0.8686868686868687,0.8686868686868687,0.9090909090909091,0.9393939393939394,0.8964646464646465,0.9843230609765246 +gpt-6.1-sol,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 +gpt-6.1-sol,person_medicare_eligible,172,172,1.0,0.08139534883720931,,0.9186046511627907,0.9186046511627907,0.9186046511627907,0.9186046511627907,0.9186046511627907,0.9186046511627907,0.9186046511627907 +gpt-6.1-sol,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,self_employment_tax,100,100,1.0,0.0012822998046863176,1.742575068402932e-05,,1.0,1.0,1.0,1.0,1.0,0.9999986059399453 +gpt-6.1-sol,snap,93,93,1.0,64.50193548387097,0.25887264064273635,,0.9139784946236559,0.9247311827956989,0.9247311827956989,0.9247311827956989,0.9220430107526881,0.942308125501553 +gpt-6.1-sol,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 +gpt-6.1-sol,state_income_tax_before_refundable_credits,86,86,1.0,65.70432357943336,0.1332244926267002,,0.7441860465116279,0.7674418604651163,0.8372093023255814,0.8837209302325582,0.808139534883721,0.9418984328046396 +gpt-6.1-sol,state_refundable_credits,98,98,1.0,21.216245583320152,0.19622238526538657,,0.8979591836734694,0.8979591836734694,0.9081632653061225,0.9183673469387755,0.9056122448979591,0.9435509904614596 +gpt-6.1-sol,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565 +grok-4.3,federal_income_tax_before_refundable_credits,82,82,1.0,1094.8344964748476,0.9139224728808983,,0.5487804878048781,0.5853658536585366,0.6219512195121951,0.6951219512195121,0.6128048780487805,0.7759575619928454 +grok-4.3,federal_refundable_credits,98,98,1.0,536.376557330696,0.8611609779771734,,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.872726829002562 grok-4.3,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 grok-4.3,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.3,payroll_tax,100,100,1.0,385.60862243652343,0.04966788173773992,,0.77,0.84,0.85,0.91,0.8425,0.9682125556878465 +grok-4.3,payroll_tax,99,99,1.0,388.3646690291588,0.04962868798157417,,0.7777777777777778,0.8484848484848485,0.8585858585858586,0.9090909090909091,0.8484848484848485,0.9684181076480892 grok-4.3,person_chip_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 grok-4.3,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 grok-4.3,person_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -grok-4.3,person_medicaid_eligible,177,177,1.0,0.11299435028248588,,0.8870056497175142,0.8870056497175142,0.8870056497175142,0.8870056497175142,0.8870056497175142,0.8870056497175142,0.8870056497175142 +grok-4.3,person_medicaid_eligible,173,173,1.0,0.10404624277456648,,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336,0.8959537572254336 grok-4.3,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 grok-4.3,person_wic_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028 -grok-4.3,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +grok-4.3,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 grok-4.3,self_employment_tax,100,100,1.0,34.579960327148434,0.0007042345874243298,,0.96,0.99,0.99,0.99,0.9824999999999999,0.989943661233006 -grok-4.3,snap,97,97,1.0,637.3026626429607,0.9572373011305837,,0.8041237113402062,0.8041237113402062,0.8041237113402062,0.8041237113402062,0.8041237113402062,0.8116182049564957 +grok-4.3,snap,93,93,1.0,573.2903225806451,0.9432999432999433,,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8387096774193549,0.8466354917967822 grok-4.3,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -grok-4.3,state_income_tax_before_refundable_credits,100,100,1.0,389.89393455028534,0.42768577068825964,,0.57,0.6,0.64,0.68,0.6225,0.8060951186040484 -grok-4.3,state_refundable_credits,100,100,1.0,117.28137747287751,0.9891161442920792,,0.79,0.79,0.79,0.79,0.79,0.7922856096986634 +grok-4.3,state_income_tax_before_refundable_credits,86,86,1.0,236.67906144053438,0.4113355995456235,,0.6511627906976745,0.6627906976744186,0.6976744186046512,0.7325581395348837,0.686046511627907,0.8448829303910615 +grok-4.3,state_refundable_credits,98,98,1.0,112.97868960730884,0.9873021683407591,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8186587854067994 grok-4.3,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -grok-4.5,federal_income_tax_before_refundable_credits,100,100,1.0,2073.836492919922,0.5965107657659735,,0.48,0.49,0.51,0.57,0.5125,0.778117786468514 -grok-4.5,federal_refundable_credits,100,100,1.0,124.96448089599609,0.2675025511122579,,0.87,0.89,0.9,0.92,0.895,0.9452246683554065 +grok-4.5,federal_income_tax_before_refundable_credits,82,82,1.0,759.121903582317,0.49601384688912187,,0.5487804878048781,0.5609756097560976,0.5853658536585366,0.6463414634146342,0.5853658536585367,0.8108803411218974 +grok-4.5,federal_refundable_credits,98,98,1.0,90.67696551887356,0.201694384459498,,0.8877551020408163,0.9081632653061225,0.9183673469387755,0.9387755102040817,0.913265306122449,0.9569526711320971 grok-4.5,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 grok-4.5,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.5,payroll_tax,100,100,1.0,75.08208533691406,0.01857311406298839,,0.85,0.88,0.95,0.98,0.9149999999999999,0.9881132069996874 +grok-4.5,payroll_tax,99,99,1.0,75.18634872652304,0.01839263611034336,,0.8585858585858586,0.8888888888888888,0.9494949494949495,0.9797979797979798,0.9191919191919191,0.9882955952025088 grok-4.5,person_chip_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 grok-4.5,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 grok-4.5,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.5,person_medicaid_eligible,177,177,1.0,0.062146892655367235,,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328 +grok-4.5,person_medicaid_eligible,173,173,1.0,0.046242774566473986,,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526 grok-4.5,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 grok-4.5,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +grok-4.5,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 grok-4.5,self_employment_tax,100,100,1.0,0.016382299804686227,0.00019320155724966045,,1.0,1.0,1.0,1.0,1.0,0.99998454387542 -grok-4.5,snap,97,97,1.0,107.80829549573131,0.5324607438588024,,0.8144329896907216,0.8247422680412371,0.8762886597938144,0.8969072164948454,0.8530927835051546,0.897756896947126 +grok-4.5,snap,93,93,1.0,76.6236559139785,0.49252935709384804,,0.8494623655913979,0.8602150537634409,0.9139784946236559,0.9247311827956989,0.8870967741935484,0.9223451052554452 grok-4.5,ssi,97,97,1.0,11.030927835051546,0.05674460161430561,,0.979381443298969,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9871134020618557,0.998830008214138 -grok-4.5,state_income_tax_before_refundable_credits,100,100,1.0,198.9617003583908,0.25718362130970057,,0.57,0.58,0.65,0.71,0.6275,0.8388045579671989 -grok-4.5,state_refundable_credits,100,100,1.0,63.93360037326813,0.7825811170081218,,0.78,0.78,0.78,0.78,0.78,0.8156579654282945 +grok-4.5,state_income_tax_before_refundable_credits,86,86,1.0,148.42793583869934,0.18286492264238113,,0.6511627906976745,0.6627906976744186,0.7093023255813954,0.7558139534883721,0.6947674418604651,0.8548145618689368 +grok-4.5,state_refundable_credits,98,98,1.0,58.54218236280947,0.7499925769866382,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8418380981044951 grok-4.5,tanf,100,100,1.0,16.44958984375,0.2712234309601872,,0.99,0.99,0.99,0.99,0.99,0.9972877656903981 -grok-4.6,federal_income_tax_before_refundable_credits,100,100,1.0,1671.2186022949218,0.5829681293363206,,0.5,0.51,0.56,0.61,0.545,0.8117850021288782 -grok-4.6,federal_refundable_credits,100,100,1.0,78.66448089599609,0.14570317369007904,,0.9,0.91,0.94,0.96,0.9275,0.9810585874202897 +grok-4.6,federal_income_tax_before_refundable_credits,82,82,1.0,836.3879811356708,0.49840426316522757,,0.5609756097560976,0.5609756097560976,0.6097560975609756,0.6585365853658537,0.5975609756097562,0.8320931532075976 +grok-4.6,federal_refundable_credits,98,98,1.0,61.75859817193479,0.11442975660933365,,0.9183673469387755,0.9285714285714286,0.9489795918367347,0.9693877551020408,0.9413265306122449,0.987155843645891 grok-4.6,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 grok-4.6,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.6,payroll_tax,100,100,1.0,20.861292797851565,0.00379894333358157,,0.88,0.93,0.98,1.0,0.9475,0.9975686762665077 +grok-4.6,payroll_tax,99,99,1.0,20.413023126775574,0.0033804320423386595,,0.8888888888888888,0.9393939393939394,0.9797979797979798,1.0,0.952020202020202,0.9978488159730573 grok-4.6,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 grok-4.6,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 grok-4.6,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.6,person_medicaid_eligible,177,177,1.0,0.03954802259887006,,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113,0.96045197740113 +grok-4.6,person_medicaid_eligible,173,173,1.0,0.03468208092485549,,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445,0.9653179190751445 grok-4.6,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 grok-4.6,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +grok-4.6,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 grok-4.6,self_employment_tax,100,100,1.0,0.015082299804685135,0.00018936798364602316,,1.0,1.0,1.0,1.0,1.0,0.9999848505613084 -grok-4.6,snap,97,97,1.0,110.35359490778028,0.4500326592233965,,0.7938144329896907,0.8041237113402062,0.865979381443299,0.865979381443299,0.8324742268041238,0.8902004617855902 +grok-4.6,snap,93,93,1.0,89.39784946236558,0.4142431666072162,,0.8279569892473119,0.8387096774193549,0.8924731182795699,0.8924731182795699,0.8629032258064516,0.9098369767108193 grok-4.6,ssi,97,97,1.0,3.9587628865979383,0.020130596490479824,,0.979381443298969,0.9896907216494846,1.0,1.0,0.9922680412371134,0.9995849361548356 -grok-4.6,state_income_tax_before_refundable_credits,100,100,1.0,164.2500502729416,0.2415644113942016,,0.58,0.65,0.74,0.78,0.6875,0.8749968566383841 -grok-4.6,state_refundable_credits,100,100,1.0,49.93360037326813,0.5627668160136663,,0.83,0.83,0.83,0.83,0.83,0.88181896863713 +grok-4.6,state_income_tax_before_refundable_credits,86,86,1.0,104.48890995979309,0.16698031585809708,,0.6627906976744186,0.7209302325581395,0.7790697674418605,0.8255813953488372,0.747093023255814,0.8952394247006638 +grok-4.6,state_refundable_credits,98,98,1.0,44.256468077095185,0.49418623535212247,,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.8571428571428571,0.909231099629202 grok-4.6,tanf,100,100,1.0,16.48958984375,0.27188295726701056,,0.99,0.99,0.99,0.99,0.99,0.9972811704273299 -grok-build-0.1,federal_income_tax_before_refundable_credits,100,100,1.0,2068.507178955078,0.502030444443333,,0.4,0.4,0.45,0.47,0.43,0.7094404186324946 -grok-build-0.1,federal_refundable_credits,100,100,1.0,148.1731332397461,0.2787852918316426,,0.83,0.83,0.85,0.88,0.8474999999999999,0.9037579120618864 +grok-4.7,federal_income_tax_before_refundable_credits,82,82,1.0,519.4989628191692,0.35188907590813073,,0.5853658536585366,0.6951219512195121,0.7926829268292683,0.8170731707317073,0.7225609756097561,0.9033515062538545 +grok-4.7,federal_refundable_credits,98,98,1.0,28.61213208256935,0.03835701888150592,,0.9081632653061225,0.9489795918367347,0.9489795918367347,0.9897959183673469,0.9489795918367347,0.9956946203296269 +grok-4.7,free_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97 +grok-4.7,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 +grok-4.7,payroll_tax,99,99,1.0,9.59769047407671,0.0025630788402871774,,0.9292929292929293,0.9494949494949495,0.9797979797979798,1.0,0.9646464646464646,0.9983689498289082 +grok-4.7,person_chip_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 +grok-4.7,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +grok-4.7,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +grok-4.7,person_medicaid_eligible,173,173,1.0,0.023121387283236993,,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763 +grok-4.7,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 +grok-4.7,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +grok-4.7,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +grok-4.7,self_employment_tax,100,100,1.0,0.013281323242185863,9.278463705377946e-05,,1.0,1.0,1.0,1.0,1.0,0.9999925772290357 +grok-4.7,snap,93,93,1.0,81.92580645161291,0.33514802771074387,,0.8602150537634409,0.8709677419354839,0.9247311827956989,0.9247311827956989,0.8951612903225806,0.9316459746210789 +grok-4.7,ssi,97,97,1.0,0.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0 +grok-4.7,state_income_tax_before_refundable_credits,86,86,1.0,90.0947851641899,0.14279469543681114,,0.6976744186046512,0.7325581395348837,0.7790697674418605,0.8372093023255814,0.7616279069767442,0.9036762690336706 +grok-4.7,state_refundable_credits,98,98,1.0,27.02574830755896,0.3738464532377186,,0.8571428571428571,0.8673469387755102,0.8673469387755102,0.8775510204081632,0.8673469387755102,0.921130243282868 +grok-4.7,tanf,100,100,1.0,16.45208984375,0.2712646513543637,,0.99,0.99,0.99,0.99,0.99,0.9972873534864565 +grok-build-0.1,federal_income_tax_before_refundable_credits,82,82,1.0,1032.261435904154,0.4166273840968176,,0.47560975609756095,0.47560975609756095,0.524390243902439,0.5365853658536586,0.5030487804878049,0.7415731579632797 +grok-build-0.1,federal_refundable_credits,98,98,1.0,111.45127993213887,0.20595095265447802,,0.8469387755102041,0.8469387755102041,0.8673469387755102,0.8979591836734694,0.864795918367347,0.9156585665387831 grok-build-0.1,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 grok-build-0.1,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -grok-build-0.1,payroll_tax,100,100,1.0,207.35748454589844,0.045832462478850935,,0.71,0.76,0.8,0.88,0.7875,0.9706672240135354 +grok-build-0.1,payroll_tax,99,99,1.0,208.31301459418404,0.045732389051909156,,0.7171717171717171,0.7676767676767676,0.8080808080808081,0.8787878787878788,0.7929292929292929,0.9708975706033305 grok-build-0.1,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028 grok-build-0.1,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 grok-build-0.1,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-build-0.1,person_medicaid_eligible,177,177,1.0,0.05649717514124294,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 +grok-build-0.1,person_medicaid_eligible,173,173,1.0,0.046242774566473986,,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526 grok-build-0.1,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 grok-build-0.1,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -grok-build-0.1,reduced_price_school_meals_eligible,100,100,1.0,0.03,,0.97,0.97,0.97,0.97,0.97,0.97,0.97 +grok-build-0.1,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 grok-build-0.1,self_employment_tax,100,100,1.0,19.800231323242187,0.00021513468896632062,,0.99,0.99,0.99,0.99,0.99,0.9899827892248827 -grok-build-0.1,snap,97,97,1.0,300.578672428721,0.5683329382274372,,0.8041237113402062,0.8247422680412371,0.845360824742268,0.8556701030927835,0.8324742268041238,0.8797767015477688 +grok-build-0.1,snap,93,93,1.0,253.53763440860214,0.5006941553309233,,0.8387096774193549,0.8602150537634409,0.8709677419354839,0.8817204301075269,0.8629032258064516,0.9085051180720215 grok-build-0.1,ssi,97,97,1.0,106.9278350515464,0.04653972128171252,,0.9690721649484536,0.9690721649484536,0.9896907216494846,0.9896907216494846,0.9793814432989691,0.9887311397673872 -grok-build-0.1,state_income_tax_before_refundable_credits,100,100,1.0,281.92015006542204,0.9365117154521054,,0.47,0.49,0.58,0.63,0.5425,0.7510254304153742 -grok-build-0.1,state_refundable_credits,100,100,1.0,80.5593755197525,1.2140010196895343,,0.78,0.78,0.78,0.78,0.78,0.7976913648125662 +grok-build-0.1,state_income_tax_before_refundable_credits,86,86,1.0,229.0531803341799,0.7021743715658014,,0.5348837209302325,0.5581395348837209,0.6046511627906976,0.6395348837209303,0.5843023255813953,0.7589485960533782 +grok-build-0.1,state_refundable_credits,98,98,1.0,75.52452420701786,1.2333790047333604,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8271666919340446 grok-build-0.1,tanf,100,100,1.0,6.00958984375,0.09908706487929027,,0.99,0.99,0.99,1.0,0.9924999999999999,0.9990091293512071 -inkling,federal_income_tax_before_refundable_credits,100,100,1.0,756.3287039550783,0.26411409588921064,,0.6,0.68,0.72,0.76,0.69,0.887557693753797 -inkling,federal_refundable_credits,100,100,1.0,101.08229339599609,0.14714227799575236,,0.87,0.92,0.92,0.94,0.9125,0.9408715038605522 +inkling,federal_income_tax_before_refundable_credits,82,82,1.0,328.69643090224844,0.17574055583043977,,0.7317073170731707,0.7804878048780488,0.7804878048780488,0.8048780487804879,0.774390243902439,0.9271319646556713 +inkling,federal_refundable_credits,98,98,1.0,81.34820276377152,0.10642112209169272,,0.8877551020408163,0.9285714285714286,0.9285714285714286,0.9489795918367347,0.9234693877551021,0.9472384454795039 inkling,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 inkling,local_income_tax,100,100,1.0,39.24,,,0.99,0.99,0.99,0.99,0.99,0.99 -inkling,payroll_tax,100,100,1.0,83.17338249511718,0.020576686257239987,,0.81,0.86,0.9,0.96,0.8825,0.9868309207953665 +inkling,payroll_tax,99,99,1.0,83.86967938663983,0.02079878961165186,,0.8181818181818182,0.8585858585858586,0.898989898989899,0.9595959595959596,0.8838383838383839,0.9867644066107671 inkling,person_chip_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 inkling,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 inkling,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -inkling,person_medicaid_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028 +inkling,person_medicaid_eligible,173,173,1.0,0.023121387283236993,,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763,0.976878612716763 inkling,person_medicare_eligible,172,172,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 inkling,person_wic_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 -inkling,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +inkling,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 inkling,self_employment_tax,100,100,1.0,34.43888229980468,0.0001841476840988503,,0.99,0.99,0.99,0.99,0.99,0.9899852681852721 -inkling,snap,97,97,1.0,195.15951569547357,0.5256859579948263,,0.8144329896907216,0.865979381443299,0.8865979381443299,0.8865979381443299,0.86340206185567,0.8987998677179129 +inkling,snap,93,93,1.0,149.09677419354838,0.4805822155985559,,0.8602150537634409,0.9032258064516129,0.9247311827956989,0.9247311827956989,0.9032258064516129,0.9238612673536069 inkling,ssi,97,97,1.0,10.68041237113402,0.05529461998292058,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9988599047426203 -inkling,state_income_tax_before_refundable_credits,100,100,1.0,236.6320845014572,0.30778275705884695,,0.53,0.56,0.66,0.71,0.615,0.8285650578295062 -inkling,state_refundable_credits,100,100,1.0,69.85360373020173,0.7777649256962836,,0.8,0.8,0.81,0.81,0.805,0.8438426275790394 +inkling,state_income_tax_before_refundable_credits,86,86,1.0,132.00886482216592,0.2386237703998181,,0.6046511627906976,0.6395348837209303,0.7325581395348837,0.7790697674418605,0.688953488372093,0.8453972637628607 +inkling,state_refundable_credits,98,98,1.0,62.31431975170057,0.7678618120653087,,0.826530612244898,0.826530612244898,0.8367346938775511,0.8367346938775511,0.8316326530612246,0.8707272557191053 inkling,tanf,100,100,1.0,16.44958984375,0.2712234309601872,,0.99,0.99,0.99,0.99,0.99,0.9972877656903981 -kimi-k2.6,federal_income_tax_before_refundable_credits,100,51,0.51,3432.6543930951284,0.20181370937600498,,0.4,0.4,0.44,0.44,0.42,0.4798186290623995 -kimi-k2.6,federal_refundable_credits,100,49,0.49,212.5264614307637,0.431626995340822,,0.45,0.45,0.45,0.45,0.45,0.45841865023295886 +kimi-k2.6,federal_income_tax_before_refundable_credits,82,45,0.5487804878048781,3575.6949115668403,0.12506863265036838,,0.45121951219512196,0.45121951219512196,0.5,0.5,0.47560975609756095,0.525908775261554 +kimi-k2.6,federal_refundable_credits,98,48,0.4897959183673469,174.12708188374836,0.36096984340474547,,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.45918367346938777,0.4648583737385818 kimi-k2.6,free_school_meals_eligible,100,93,0.93,0.021505376344086023,,0.91,0.91,0.91,0.91,0.91,0.91,0.91 kimi-k2.6,local_income_tax,100,87,0.87,0.0,,,0.87,0.87,0.87,0.87,0.87,0.87 -kimi-k2.6,payroll_tax,100,48,0.48,12.856362915039073,0.0035978953827421386,,0.46,0.46,0.47,0.48,0.4675,0.47924444196962407 +kimi-k2.6,payroll_tax,99,48,0.48484848484848486,12.856362915039073,0.0035978953827421386,,0.4646464646464647,0.4646464646464647,0.47474747474747475,0.48484848484848486,0.47222222222222227,0.48408529491881225 kimi-k2.6,person_chip_eligible,177,151,0.8531073446327684,0.019867549668874173,,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955,0.8361581920903955 kimi-k2.6,person_early_head_start_eligible,38,37,0.9736842105263158,0.02702702702702703,,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316,0.9473684210526316 kimi-k2.6,person_head_start_eligible,38,37,0.9736842105263158,0.0,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 -kimi-k2.6,person_medicaid_eligible,177,149,0.8418079096045198,0.020134228187919462,,0.824858757062147,0.824858757062147,0.824858757062147,0.824858757062147,0.824858757062147,0.824858757062147,0.824858757062147 +kimi-k2.6,person_medicaid_eligible,173,148,0.8554913294797688,0.02027027027027027,,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341 kimi-k2.6,person_medicare_eligible,172,160,0.9302325581395349,0.01875,,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186,0.9127906976744186 kimi-k2.6,person_wic_eligible,177,161,0.9096045197740112,0.0,,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112,0.9096045197740112 -kimi-k2.6,reduced_price_school_meals_eligible,100,92,0.92,0.021739130434782608,,0.9,0.9,0.9,0.9,0.9,0.9,0.9 +kimi-k2.6,reduced_price_school_meals_eligible,100,92,0.92,0.010869565217391304,,0.91,0.91,0.91,0.91,0.91,0.91,0.91 kimi-k2.6,self_employment_tax,100,60,0.6,0.0023014322916651508,1.6540086871946673e-05,,0.6,0.6,0.6,0.6,0.6,0.5999996691982625 -kimi-k2.6,snap,97,86,0.8865979381443299,185.4542122774346,0.7363454087226466,,0.7835051546391752,0.7835051546391752,0.7938144329896907,0.8041237113402061,0.7912371134020618,0.8106860403378714 +kimi-k2.6,snap,93,83,0.8924731182795699,139.51807228915663,0.7636339837034091,,0.8064516129032259,0.8064516129032259,0.8172043010752688,0.8279569892473118,0.814516129032258,0.826784173444868 kimi-k2.6,ssi,97,86,0.8865979381443299,0.4186046511627907,0.003842869342442357,,0.8762886597938144,0.8865979381443299,0.8865979381443299,0.8865979381443299,0.884020618556701,0.8865583209346138 -kimi-k2.6,state_income_tax_before_refundable_credits,100,58,0.58,113.08772182826337,0.5217238969281759,,0.45999999999999996,0.45999999999999996,0.4699999999999999,0.51,0.475,0.5125098449080012 -kimi-k2.6,state_refundable_credits,100,57,0.57,25.044675366920337,1.0,,0.4799999999999999,0.4799999999999999,0.4799999999999999,0.4799999999999999,0.4799999999999999,0.4799999999999999 +kimi-k2.6,state_income_tax_before_refundable_credits,86,57,0.6627906976744186,115.05470420134694,0.5209272704784943,,0.5232558139534883,0.5232558139534883,0.5348837209302325,0.5813953488372092,0.5406976744186045,0.5843081077069643 +kimi-k2.6,state_refundable_credits,98,56,0.5714285714285714,10.34587128673281,1.0,,0.5,0.5,0.5,0.5,0.5,0.5 kimi-k2.6,tanf,100,89,0.89,0.0,,,0.89,0.89,0.89,0.89,0.89,0.89 -kimi-k3,federal_income_tax_before_refundable_credits,100,97,0.97,361.51274278149174,0.19645555135470086,,0.71,0.7899999999999999,0.86,0.88,0.8099999999999999,0.9156514221175616 -kimi-k3,federal_refundable_credits,100,97,0.97,33.09750522259585,0.09418415362565433,,0.9,0.9199999999999999,0.94,0.94,0.9249999999999999,0.947756060028665 +kimi-k3,federal_income_tax_before_refundable_credits,82,81,0.9878048780487805,249.9359975405092,0.08855597445197522,,0.8414634146341463,0.8658536585365854,0.8902439024390244,0.9146341463414634,0.8780487804878049,0.9523733158462313 +kimi-k3,federal_refundable_credits,98,95,0.9693877551020408,17.939430497018922,0.06333102758780991,,0.9081632653061225,0.9285714285714285,0.9489795918367347,0.9489795918367347,0.9336734693877551,0.9520750887401437 kimi-k3,free_school_meals_eligible,100,97,0.97,0.010309278350515464,,0.96,0.96,0.96,0.96,0.96,0.96,0.96 kimi-k3,local_income_tax,100,97,0.97,0.0,,,0.97,0.97,0.97,0.97,0.97,0.97 -kimi-k3,payroll_tax,100,97,0.97,10.597338640665296,0.00206759101156522,,0.9199999999999999,0.93,0.96,0.97,0.9450000000000001,0.9687387694829451 +kimi-k3,payroll_tax,99,96,0.9696969696969697,9.193248519897486,0.0009816461506643713,,0.9292929292929294,0.9393939393939394,0.9696969696969697,0.9696969696969697,0.9520202020202021,0.969102032635961 kimi-k3,person_chip_eligible,177,171,0.9661016949152542,0.029239766081871343,,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327,0.9378531073446327 kimi-k3,person_early_head_start_eligible,38,36,0.9473684210526315,0.0,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 kimi-k3,person_head_start_eligible,38,36,0.9473684210526315,0.0,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 -kimi-k3,person_medicaid_eligible,177,171,0.9661016949152542,0.023391812865497075,,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757,0.943502824858757 +kimi-k3,person_medicaid_eligible,173,168,0.9710982658959537,0.023809523809523808,,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167,0.9479768786127167 kimi-k3,person_medicare_eligible,172,166,0.9651162790697675,0.03614457831325301,,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349,0.9302325581395349 kimi-k3,person_wic_eligible,177,171,0.9661016949152542,0.0,,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542,0.9661016949152542 -kimi-k3,reduced_price_school_meals_eligible,100,97,0.97,0.020618556701030927,,0.95,0.95,0.95,0.95,0.95,0.95,0.95 +kimi-k3,reduced_price_school_meals_eligible,100,97,0.97,0.010309278350515464,,0.96,0.96,0.96,0.96,0.96,0.96,0.96 kimi-k3,self_employment_tax,100,97,0.97,0.0033838142316346756,5.9567983600926226e-05,,0.97,0.97,0.97,0.97,0.97,0.969995234561312 -kimi-k3,snap,97,95,0.979381443298969,124.66361693050987,0.5142091044002021,,0.7938144329896907,0.8350515463917525,0.8762886597938144,0.8762886597938144,0.845360824742268,0.8801162909650656 +kimi-k3,snap,93,91,0.978494623655914,85.72285714285714,0.47465267116100845,,0.8602150537634409,0.903225806451613,0.903225806451613,0.903225806451613,0.89247311827957,0.9031032923010517 kimi-k3,ssi,97,94,0.9690721649484536,11.106382978723405,0.05572160546541417,,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9587628865979382,0.9679232658666925 -kimi-k3,state_income_tax_before_refundable_credits,100,97,0.97,145.6476956882673,0.20239182683174278,,0.61,0.64,0.76,0.7699999999999999,0.695,0.8774457200766114 -kimi-k3,state_refundable_credits,100,97,0.97,33.63171838288455,0.6277804376567555,,0.78,0.7899999999999999,0.7899999999999999,0.7899999999999999,0.7875,0.834443912468649 +kimi-k3,state_income_tax_before_refundable_credits,86,85,0.9883720930232558,101.90464587918451,0.12350512485124471,,0.6976744186046512,0.7441860465116279,0.8372093023255813,0.8372093023255813,0.7790697674418603,0.9118412951083013 +kimi-k3,state_refundable_credits,98,95,0.9693877551020408,30.48218292336715,0.5969869954367941,,0.8061224489795918,0.8061224489795918,0.8061224489795918,0.8163265306122448,0.8086734693877551,0.8556247048732093 kimi-k3,tanf,100,97,0.97,16.960917364690722,0.2712646513543637,,0.96,0.96,0.96,0.96,0.96,0.9672873534864564 -minimax-m3,federal_income_tax_before_refundable_credits,100,100,1.0,5291.478026123047,1.2952843420015465,,0.47,0.5,0.55,0.56,0.52,0.5830671657978096 -minimax-m3,federal_refundable_credits,100,100,1.0,475.3115213012695,0.738076800322437,,0.79,0.79,0.8,0.8,0.7949999999999999,0.8240500159580832 +minimax-m3,federal_income_tax_before_refundable_credits,82,82,1.0,2151.7156803782395,1.22609124971039,,0.524390243902439,0.5487804878048781,0.6097560975609756,0.6097560975609756,0.5731707317073171,0.6380769405781693 +minimax-m3,federal_refundable_credits,98,98,1.0,426.5208842997648,0.690454400381062,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.840867363222534 minimax-m3,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 minimax-m3,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -minimax-m3,payroll_tax,100,100,1.0,1941.7401016845702,0.24288977298477785,,0.63,0.69,0.78,0.82,0.7299999999999999,0.844550545289742 +minimax-m3,payroll_tax,99,99,1.0,1960.21464806759,0.24591759337539043,,0.6363636363636364,0.696969696969697,0.7878787878787878,0.8181818181818182,0.7348484848484849,0.8435069860338423 minimax-m3,person_chip_eligible,177,177,1.0,0.022598870056497175,,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028,0.9774011299435028 minimax-m3,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 minimax-m3,person_head_start_eligible,38,38,1.0,0.05263157894736842,,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315,0.9473684210526315 -minimax-m3,person_medicaid_eligible,177,177,1.0,0.15819209039548024,,0.8418079096045198,0.8418079096045198,0.8418079096045198,0.8418079096045198,0.8418079096045198,0.8418079096045198,0.8418079096045198 +minimax-m3,person_medicaid_eligible,173,173,1.0,0.15028901734104047,,0.8497109826589595,0.8497109826589595,0.8497109826589595,0.8497109826589595,0.8497109826589595,0.8497109826589595,0.8497109826589595 minimax-m3,person_medicare_eligible,172,172,1.0,0.01744186046511628,,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837 minimax-m3,person_wic_eligible,177,177,1.0,0.02824858757062147,,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786,0.9717514124293786 -minimax-m3,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +minimax-m3,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 minimax-m3,self_employment_tax,100,100,1.0,52.137589819335936,0.2117598110686142,,0.93,0.95,0.96,0.96,0.95,0.9630592151145109 -minimax-m3,snap,97,97,1.0,494.6027794277545,0.9022816030311082,,0.8247422680412371,0.8247422680412371,0.8247422680412371,0.8350515463917526,0.827319587628866,0.8418681726646511 +minimax-m3,snap,93,93,1.0,426.06451612903226,0.8724417371025018,,0.8602150537634409,0.8602150537634409,0.8602150537634409,0.8709677419354839,0.8629032258064516,0.8780457786845964 minimax-m3,ssi,97,97,1.0,205.56701030927834,1.0,,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969,0.979381443298969 -minimax-m3,state_income_tax_before_refundable_credits,100,100,1.0,1331.8996446943283,1.7030067470855392,,0.52,0.53,0.56,0.56,0.5425,0.6348987409238792 -minimax-m3,state_refundable_credits,100,100,1.0,119.03137747287751,0.9576922096732867,,0.79,0.79,0.79,0.79,0.79,0.7988846359686099 +minimax-m3,state_income_tax_before_refundable_credits,86,86,1.0,598.4353853214619,1.3516083365443936,,0.6046511627906976,0.6162790697674418,0.6511627906976745,0.6511627906976745,0.6308139534883721,0.723576182597974 +minimax-m3,state_refundable_credits,98,98,1.0,114.76440389302311,0.9506409112855011,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.825392485682255 minimax-m3,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -ox-alpha,federal_income_tax_before_refundable_credits,100,100,1.0,654.7970466308593,0.3255936039284364,,0.63,0.7,0.75,0.77,0.7125,0.8577917526608693 -ox-alpha,federal_refundable_credits,100,100,1.0,140.93125838623047,0.16228780620947156,,0.84,0.88,0.88,0.9,0.875,0.9289025851927687 +ox-alpha,federal_income_tax_before_refundable_credits,82,82,1.0,425.23533215034297,0.2662146808853651,,0.7439024390243902,0.7682926829268293,0.7926829268292683,0.8048780487804879,0.7774390243902438,0.8776550715442846 +ox-alpha,federal_refundable_credits,98,98,1.0,109.45626254334742,0.08470054563216628,,0.8571428571428571,0.8979591836734694,0.8979591836734694,0.9183673469387755,0.8928571428571428,0.9394723877351651 ox-alpha,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 ox-alpha,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -ox-alpha,payroll_tax,100,100,1.0,98.31265920410158,0.010648064539592113,,0.87,0.89,0.92,0.98,0.915,0.9931852386946612 +ox-alpha,payroll_tax,99,99,1.0,98.16874657216697,0.00999097645377959,,0.8787878787878788,0.898989898989899,0.9292929292929293,0.9797979797979798,0.9217171717171716,0.9936421058930494 ox-alpha,person_chip_eligible,177,177,1.0,0.07344632768361582,,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842,0.9265536723163842 ox-alpha,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 ox-alpha,person_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -ox-alpha,person_medicaid_eligible,177,177,1.0,0.062146892655367235,,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328,0.9378531073446328 +ox-alpha,person_medicaid_eligible,173,173,1.0,0.046242774566473986,,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526,0.953757225433526 ox-alpha,person_medicare_eligible,172,172,1.0,0.005813953488372093,,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279,0.9941860465116279 ox-alpha,person_wic_eligible,177,177,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 -ox-alpha,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +ox-alpha,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 ox-alpha,self_employment_tax,100,100,1.0,23.392942163085937,0.00010208031325015381,,0.99,0.99,0.99,0.99,0.99,0.9799918335749399 -ox-alpha,snap,97,97,1.0,133.12623605629832,0.5552537227608245,,0.8144329896907216,0.8556701030927835,0.8762886597938144,0.8762886597938144,0.8556701030927835,0.8934140819717129 +ox-alpha,snap,93,93,1.0,71.8537634408602,0.4892323148995972,,0.8709677419354839,0.9139784946236559,0.9139784946236559,0.9139784946236559,0.9032258064516129,0.9224348876641922 ox-alpha,ssi,97,97,1.0,96.5,0.0,,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846,0.9896907216494846 -ox-alpha,state_income_tax_before_refundable_credits,100,100,1.0,169.2861525068283,0.34273331994352496,,0.55,0.59,0.72,0.79,0.6625000000000001,0.8650267294436379 -ox-alpha,state_refundable_credits,100,100,1.0,48.93725552272797,0.7300732121524628,,0.79,0.79,0.8,0.81,0.7975,0.8366846254479828 +ox-alpha,state_income_tax_before_refundable_credits,86,86,1.0,107.43600560898004,0.1751925845483012,,0.6395348837209303,0.6744186046511628,0.7790697674418605,0.8488372093023255,0.7354651162790697,0.8923746798087322 +ox-alpha,state_refundable_credits,98,98,1.0,45.65407537246238,0.717552980967496,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8367346938775511,0.8214285714285714,0.8580004728835212 ox-alpha,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -qwen-3.7-max,federal_income_tax_before_refundable_credits,100,100,1.0,2815.2204682128904,0.5959489934567627,,0.39,0.41,0.48,0.52,0.45,0.6584013347525541 -qwen-3.7-max,federal_refundable_credits,100,100,1.0,460.3751715209961,0.6091805880382328,,0.76,0.77,0.77,0.78,0.77,0.8151359022491655 +qwen-3.7-max,federal_income_tax_before_refundable_credits,82,82,1.0,1274.8403671636813,0.5348992618894934,,0.45121951219512196,0.4634146341463415,0.5365853658536586,0.5853658536585366,0.5091463414634146,0.6988004444287399 +qwen-3.7-max,federal_refundable_credits,98,98,1.0,369.4145165392817,0.40785543409849506,,0.7755102040816326,0.7857142857142857,0.7857142857142857,0.7959183673469388,0.7857142857142856,0.8317713288256792 qwen-3.7-max,free_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 qwen-3.7-max,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -qwen-3.7-max,payroll_tax,100,100,1.0,331.01844699707027,0.04204051908966916,,0.73,0.81,0.87,0.92,0.8325,0.9730940677826118 +qwen-3.7-max,payroll_tax,99,99,1.0,334.2605524236506,0.04263406947771043,,0.7373737373737373,0.8080808080808081,0.8686868686868687,0.9191919191919192,0.8333333333333334,0.9728692285141842 qwen-3.7-max,person_chip_eligible,177,177,1.0,0.096045197740113,,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887,0.903954802259887 qwen-3.7-max,person_early_head_start_eligible,38,38,1.0,0.02631578947368421,,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158,0.9736842105263158 qwen-3.7-max,person_head_start_eligible,38,38,1.0,0.10526315789473684,,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632,0.8947368421052632 -qwen-3.7-max,person_medicaid_eligible,177,177,1.0,0.12429378531073447,,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656,0.8757062146892656 +qwen-3.7-max,person_medicaid_eligible,173,173,1.0,0.11560693641618497,,0.884393063583815,0.884393063583815,0.884393063583815,0.884393063583815,0.884393063583815,0.884393063583815,0.884393063583815 qwen-3.7-max,person_medicare_eligible,172,172,1.0,0.01744186046511628,,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837,0.9825581395348837 qwen-3.7-max,person_wic_eligible,177,177,1.0,0.011299435028248588,,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514,0.9887005649717514 -qwen-3.7-max,reduced_price_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 +qwen-3.7-max,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 qwen-3.7-max,self_employment_tax,100,100,1.0,96.83744475097657,0.2059975897273345,,0.96,0.96,0.96,0.96,0.96,0.9635201928218131 -qwen-3.7-max,snap,97,97,1.0,477.4720622583763,3.0366185148679996,,0.7938144329896907,0.7938144329896907,0.8144329896907216,0.8144329896907216,0.8041237113402062,0.85161665400997 +qwen-3.7-max,snap,93,93,1.0,414.3333333333333,1.2063119918995897,,0.8172043010752689,0.8172043010752689,0.8387096774193549,0.8387096774193549,0.8279569892473119,0.8679149704672419 qwen-3.7-max,ssi,97,97,1.0,161.4020618556701,0.5232690124858116,,0.9690721649484536,0.9690721649484536,0.979381443298969,0.979381443298969,0.9742268041237113,0.9789016698456533 -qwen-3.7-max,state_income_tax_before_refundable_credits,100,100,1.0,598.8367811811447,0.524998113269531,,0.46,0.46,0.49,0.51,0.48000000000000004,0.7140151580038158 -qwen-3.7-max,state_refundable_credits,100,100,1.0,102.47415976047516,0.9164896651497292,,0.78,0.78,0.79,0.79,0.785,0.7985066384122771 +qwen-3.7-max,state_income_tax_before_refundable_credits,86,86,1.0,459.5964441394807,0.4528242520786607,,0.5348837209302325,0.5348837209302325,0.5465116279069767,0.5581395348837209,0.5436046511627907,0.7302575686093454 +qwen-3.7-max,state_refundable_credits,98,98,1.0,98.95411766305261,0.9676512169222556,,0.8061224489795918,0.8061224489795918,0.8163265306122449,0.8163265306122449,0.8112244897959184,0.8250067738900787 qwen-3.7-max,tanf,100,100,1.0,60.64958984375,1.0,,0.99,0.99,0.99,0.99,0.99,0.99 -qwen3.8-max,federal_income_tax_before_refundable_credits,100,100,1.0,2935.3017075683597,1.17134340173351,,0.45,0.45,0.49,0.5,0.47250000000000003,0.6468768341609332 -qwen3.8-max,federal_refundable_credits,100,100,1.0,803.6316957153321,0.6553485788005329,,0.66,0.66,0.67,0.67,0.665,0.7061606467175635 +qwen3.8-max,federal_income_tax_before_refundable_credits,82,82,1.0,1888.948673780488,1.3163388445365902,,0.5,0.5,0.524390243902439,0.5365853658536586,0.5152439024390244,0.6484186688049249 +qwen3.8-max,federal_refundable_credits,98,98,1.0,767.3067765590123,0.6106816981178466,,0.673469387755102,0.673469387755102,0.6836734693877551,0.6836734693877551,0.6785714285714286,0.7185520154782343 qwen3.8-max,free_school_meals_eligible,100,100,1.0,0.02,,0.98,0.98,0.98,0.98,0.98,0.98,0.98 qwen3.8-max,local_income_tax,100,100,1.0,0.0,,,1.0,1.0,1.0,1.0,1.0,1.0 -qwen3.8-max,payroll_tax,100,100,1.0,532.1247353759765,0.11477986648971544,,0.71,0.74,0.8,0.85,0.775,0.9293288908751262 +qwen3.8-max,payroll_tax,99,99,1.0,536.3627629073943,0.11577566414914295,,0.7171717171717171,0.7474747474747475,0.8080808080808081,0.8484848484848485,0.7803030303030303,0.9291407444591758 qwen3.8-max,person_chip_eligible,177,177,1.0,0.07909604519774012,,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598,0.9209039548022598 qwen3.8-max,person_early_head_start_eligible,38,38,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 qwen3.8-max,person_head_start_eligible,38,38,1.0,0.13157894736842105,,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579,0.868421052631579 -qwen3.8-max,person_medicaid_eligible,177,177,1.0,0.1751412429378531,,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468,0.8248587570621468 +qwen3.8-max,person_medicaid_eligible,173,173,1.0,0.16184971098265896,,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341,0.838150289017341 qwen3.8-max,person_medicare_eligible,172,172,1.0,0.023255813953488372,,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116,0.9767441860465116 qwen3.8-max,person_wic_eligible,177,177,1.0,0.10734463276836158,,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384,0.8926553672316384 -qwen3.8-max,reduced_price_school_meals_eligible,100,100,1.0,0.01,,0.99,0.99,0.99,0.99,0.99,0.99,0.99 +qwen3.8-max,reduced_price_school_meals_eligible,100,100,1.0,0.0,,1.0,1.0,1.0,1.0,1.0,1.0,1.0 qwen3.8-max,self_employment_tax,100,100,1.0,85.60206276855467,0.43511756292674586,,0.92,0.92,0.92,0.94,0.925,0.9551905949658603 -qwen3.8-max,snap,97,97,1.0,719.924005567413,2.4676580582197714,,0.7938144329896907,0.7938144329896907,0.7938144329896907,0.7938144329896907,0.7938144329896907,0.8084180751519086 +qwen3.8-max,snap,93,93,1.0,662.9677419354839,2.919012327941218,,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8279569892473119,0.8428620760225537 qwen3.8-max,ssi,97,97,1.0,285.1505154639175,0.8968785471055618,,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9690721649484536,0.9711983804720503 -qwen3.8-max,state_income_tax_before_refundable_credits,100,100,1.0,560.7825783979415,0.5826562880646626,,0.54,0.56,0.61,0.65,0.59,0.7688227302868575 -qwen3.8-max,state_refundable_credits,100,100,1.0,99.2613774728775,0.9350766735407154,,0.79,0.79,0.79,0.79,0.79,0.8036338985564498 +qwen3.8-max,state_income_tax_before_refundable_credits,86,86,1.0,352.56610949649365,0.4701372619468074,,0.6162790697674418,0.6395348837209303,0.6744186046511628,0.7093023255813954,0.6598837209302325,0.8070404676544418 +qwen3.8-max,state_refundable_credits,98,98,1.0,94.59093450526802,0.9250063908137852,,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8163265306122449,0.8301008669933864 qwen3.8-max,tanf,100,100,1.0,93.16958984375,1.0,,0.98,0.98,0.98,0.98,0.98,0.98 diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md index a868f89d..d80bb199 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/report.md @@ -4,15 +4,15 @@ This report is generated by `policybench analyze` from no-tools benchmark result ## Headline -Top model: `gpt-6-astra` with `bounded_score=0.963` and `within_1pct=0.898`. +Top model: `gpt-6-sol` with `bounded_score=0.980` and `within_1pct=0.959`. ## Usage -Recorded-usage cost subtotal (36 of 39 models): $406.873. +Recorded-usage cost subtotal (43 of 46 models): $450.639. -Published model costs total $470.952 (includes release-metadata costs for: claude-fable-5, gemini-3.6-flash, grok-build-0.1). +Published model costs total $514.719 (includes release-metadata costs for: claude-fable-5, gemini-3.6-flash, grok-build-0.1). -Cumulative request-time: `10074.9 min` (summed across concurrent calls; not wall-clock). Run wall-clock: `93270.4 min`. +Cumulative request-time: `11326.1 min` (summed across concurrent calls; not wall-clock). Run wall-clock: `129046.0 min`. | model | total_cost_usd | cost_rows_estimated | cumulative_request_time | wall_clock | total_tokens | reasoning_tokens | parsed_rows | total_rows | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | @@ -22,11 +22,14 @@ Cumulative request-time: `10074.9 min` (summed across concurrent calls; not wall | claude-opus-4.7 | 29.303 | 1984 | 103.4 min | n/a | 4569140.000 | 0.000 | 1984 | 1984 | | claude-opus-4.8 | 25.506 | 1984 | 105.7 min | n/a | 3782202.000 | 0.000 | 1984 | 1984 | | claude-opus-5 | 6.749 | 1984 | 30.1 min | 8.6 min | 746677.000 | 0.000 | 1984 | 1984 | +| claude-opus-5.5 | 6.749 | 0 | 44.8 min | 20.4 min | 504348.000 | 0.000 | 1984 | 1984 | | claude-sonnet-4.6 | 18.582 | 1984 | 220.6 min | n/a | 3674958.000 | 0.000 | 1984 | 1984 | | claude-sonnet-5 | 17.176 | 1984 | 110.4 min | 49.6 min | 4029053.000 | 0.000 | 1984 | 1984 | +| claude-sonnet-5.5 | 3.431 | 0 | 37.0 min | 205.2 min | 509970.000 | 0.000 | 1984 | 1984 | | deepseek-v4-flash-0731 | 0.347 | 0 | 583.5 min | 132.9 min | 2183674.000 | 1540655.000 | 1984 | 1984 | | deepseek-v4-pro | 1.090 | 1984 | 318.4 min | 99.5 min | 1324941.000 | 1071780.000 | 1984 | 1984 | | deepseek-v4-pro-0813 | 19.866 | 0 | 1457.6 min | 309.8 min | 6250616.000 | 5551000.000 | 1984 | 1984 | +| deepseek-v4.1-flash | 2.419 | 0 | 138.5 min | 211.5 min | 2121390.000 | 1885065.000 | 1984 | 1984 | | gemini-3-flash-preview | 4.834 | 1984 | 109.2 min | n/a | 1752695.000 | 1430547.000 | 1984 | 1984 | | gemini-3.1-flash-lite-preview | 0.188 | 1984 | 8.3 min | n/a | 240829.000 | n/a | 1984 | 1984 | | gemini-3.1-pro-preview | 8.468 | 1984 | 80.9 min | n/a | 820951.000 | 565613.000 | 1984 | 1984 | @@ -44,9 +47,13 @@ Cumulative request-time: `10074.9 min` (summed across concurrent calls; not wall | gpt-5.6-sol | 8.952 | 1984 | 63.2 min | 40.8 min | 496378.000 | 155156.000 | 1984 | 1984 | | gpt-5.6-terra | 3.893 | 1984 | 33.7 min | 26.3 min | 457518.000 | 126873.000 | 1984 | 1984 | | gpt-6-astra | 12.640 | 0 | 61.4 min | 307.1 min | 450016.000 | 114690.000 | 1984 | 1984 | +| gpt-6-luna | 0.191 | 0 | 51.2 min | 152.1 min | 576729.000 | 239727.000 | 1984 | 1984 | +| gpt-6-sol | 2.710 | 0 | 40.1 min | 150.3 min | 466450.000 | 137755.000 | 1984 | 1984 | +| gpt-6.1-sol | 2.300 | 0 | 60.7 min | 204.9 min | 424952.000 | 89656.000 | 1984 | 1984 | | grok-4.3 | 1.541 | 1984 | 43.6 min | n/a | 1022867.000 | 149054.000 | 1984 | 1984 | | grok-4.5 | 7.268 | 1968 | 196.7 min | 80.7 min | 1502083.000 | 986107.000 | 1984 | 1984 | | grok-4.6 | 8.701 | 1984 | 386.1 min | 147.2 min | 1722142.000 | 1157238.000 | 1984 | 1984 | +| grok-4.7 | 25.967 | 0 | 879.0 min | 310.9 min | 4691776.000 | 4052657.000 | 1984 | 1984 | | grok-build-0.1 | n/a | 0 | 240.0 min | n/a | 2458922.000 | 1906156.000 | 1984 | 1984 | | inkling | 9.147 | 0 | 498.5 min | 201.3 min | 2567921.000 | 2001295.000 | 1984 | 1984 | | kimi-k2.6 | 39.110 | 0 | 464.7 min | 316.9 min | 11534040.000 | 10225616.000 | 1558 | 1984 | @@ -60,45 +67,52 @@ Cumulative request-time: `10074.9 min` (summed across concurrent calls; not wall | model | mean_score | mean_exact | mean_within_1pct | mean_within_5pct | mean_within_10pct | mean_binary_accuracy | mean_mae | total_n | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| gpt-6-astra | 0.975 | 0.932 | 0.944 | 0.955 | 0.960 | 0.987 | 47.679 | 1973 | -| claude-fable-5.1 | 0.973 | 0.929 | 0.943 | 0.955 | 0.963 | 0.983 | 49.966 | 1973 | -| gpt-5.6-sol | 0.972 | 0.934 | 0.945 | 0.957 | 0.963 | 0.989 | 45.033 | 1973 | -| inkling | 0.961 | 0.907 | 0.921 | 0.933 | 0.942 | 0.992 | 85.728 | 1973 | -| gemini-3.8-flash | 0.964 | 0.907 | 0.911 | 0.934 | 0.946 | 0.987 | 96.552 | 1973 | -| gpt-5.5 | 0.960 | 0.908 | 0.920 | 0.935 | 0.945 | 0.974 | 70.393 | 1973 | -| kimi-k3 | 0.938 | 0.898 | 0.908 | 0.924 | 0.926 | 0.948 | 40.963 | 1973 | -| grok-4.6 | 0.962 | 0.907 | 0.916 | 0.932 | 0.939 | 0.984 | 117.548 | 1973 | -| gemini-3.6-flash | 0.963 | 0.895 | 0.899 | 0.924 | 0.935 | 0.988 | 107.297 | 1973 | -| gpt-5.6-terra | 0.958 | 0.909 | 0.918 | 0.933 | 0.945 | 0.984 | 100.120 | 1973 | -| ox-alpha | 0.954 | 0.905 | 0.917 | 0.930 | 0.940 | 0.979 | 79.228 | 1973 | -| gpt-5.6-luna | 0.951 | 0.908 | 0.916 | 0.928 | 0.937 | 0.979 | 107.805 | 1973 | -| claude-fable-5 | 0.950 | 0.894 | 0.900 | 0.914 | 0.924 | 0.978 | 131.115 | 1973 | -| gemini-3.7-flash | 0.953 | 0.894 | 0.898 | 0.916 | 0.925 | 0.987 | 161.421 | 1973 | -| grok-4.5 | 0.952 | 0.901 | 0.906 | 0.918 | 0.929 | 0.985 | 148.456 | 1973 | -| gemini-3.1-pro-preview | 0.949 | 0.888 | 0.891 | 0.906 | 0.916 | 0.983 | 135.437 | 1973 | -| deepseek-v4-pro-0813 | 0.955 | 0.897 | 0.905 | 0.919 | 0.929 | 0.991 | 140.645 | 1973 | -| deepseek-v4-flash-0731 | 0.943 | 0.887 | 0.894 | 0.912 | 0.919 | 0.986 | 167.211 | 1973 | -| claude-opus-5 | 0.948 | 0.893 | 0.898 | 0.910 | 0.923 | 0.979 | 175.641 | 1973 | -| gemini-3-flash-preview | 0.953 | 0.886 | 0.889 | 0.905 | 0.920 | 0.990 | 130.308 | 1973 | -| gemini-3.5-flash | 0.949 | 0.881 | 0.887 | 0.898 | 0.913 | 0.988 | 147.246 | 1973 | -| claude-sonnet-4.6 | 0.938 | 0.880 | 0.886 | 0.909 | 0.919 | 0.947 | 101.112 | 1973 | -| deepseek-v4-pro | 0.943 | 0.881 | 0.885 | 0.907 | 0.918 | 0.982 | 175.876 | 1973 | -| claude-opus-4.7 | 0.926 | 0.871 | 0.876 | 0.887 | 0.902 | 0.933 | 149.133 | 1973 | -| grok-build-0.1 | 0.936 | 0.878 | 0.883 | 0.897 | 0.908 | 0.983 | 178.887 | 1973 | -| glm-5.3 | 0.908 | 0.857 | 0.866 | 0.881 | 0.888 | 0.933 | 128.695 | 1973 | -| gemini-3.5-flash-lite | 0.921 | 0.880 | 0.885 | 0.891 | 0.899 | 0.958 | 240.486 | 1973 | -| grok-4.3 | 0.927 | 0.886 | 0.894 | 0.901 | 0.910 | 0.968 | 265.212 | 1973 | -| gemini-3.1-flash-lite-preview | 0.924 | 0.875 | 0.883 | 0.891 | 0.902 | 0.973 | 239.923 | 1973 | -| qwen-3.7-max | 0.907 | 0.857 | 0.863 | 0.874 | 0.881 | 0.949 | 283.594 | 1973 | -| glm-5.2 | 0.874 | 0.828 | 0.834 | 0.850 | 0.857 | 0.911 | 185.426 | 1973 | -| claude-opus-4.8 | 0.909 | 0.860 | 0.864 | 0.873 | 0.882 | 0.912 | 212.545 | 1973 | -| claude-haiku-4.5 | 0.890 | 0.848 | 0.855 | 0.863 | 0.872 | 0.922 | 376.276 | 1973 | -| claude-sonnet-5 | 0.909 | 0.858 | 0.867 | 0.872 | 0.882 | 0.940 | 268.341 | 1973 | -| qwen3.8-max | 0.890 | 0.848 | 0.851 | 0.860 | 0.866 | 0.932 | 339.750 | 1973 | -| minimax-m3 | 0.896 | 0.867 | 0.873 | 0.884 | 0.887 | 0.959 | 554.041 | 1973 | -| gpt-5.4-mini | 0.879 | 0.842 | 0.851 | 0.854 | 0.859 | 0.916 | 440.754 | 1973 | -| gpt-5.4-nano | 0.874 | 0.830 | 0.834 | 0.843 | 0.852 | 0.929 | 495.703 | 1973 | -| kimi-k2.6 | 0.760 | 0.749 | 0.750 | 0.754 | 0.757 | 0.902 | 221.232 | 1973 | +| gpt-6-sol | 0.981 | 0.959 | 0.963 | 0.970 | 0.973 | 0.996 | 21.099 | 1928 | +| claude-opus-5.5 | 0.981 | 0.958 | 0.965 | 0.970 | 0.976 | 0.988 | 27.669 | 1928 | +| gpt-6-luna | 0.977 | 0.946 | 0.955 | 0.967 | 0.972 | 0.989 | 24.141 | 1928 | +| claude-sonnet-5.5 | 0.970 | 0.942 | 0.948 | 0.958 | 0.963 | 0.984 | 29.597 | 1928 | +| gpt-6-astra | 0.981 | 0.953 | 0.960 | 0.965 | 0.968 | 0.988 | 25.735 | 1928 | +| gpt-5.6-sol | 0.977 | 0.954 | 0.958 | 0.967 | 0.969 | 0.991 | 33.082 | 1928 | +| grok-4.7 | 0.974 | 0.928 | 0.941 | 0.953 | 0.962 | 0.984 | 42.964 | 1928 | +| claude-fable-5.1 | 0.978 | 0.948 | 0.957 | 0.965 | 0.971 | 0.985 | 40.483 | 1928 | +| inkling | 0.968 | 0.924 | 0.936 | 0.945 | 0.953 | 0.993 | 52.122 | 1928 | +| gemini-3.8-flash | 0.972 | 0.922 | 0.926 | 0.948 | 0.956 | 0.990 | 60.429 | 1928 | +| gpt-6.1-sol | 0.979 | 0.952 | 0.954 | 0.961 | 0.967 | 0.985 | 26.741 | 1928 | +| gemini-3.6-flash | 0.968 | 0.909 | 0.914 | 0.934 | 0.942 | 0.991 | 63.156 | 1928 | +| gpt-5.6-terra | 0.964 | 0.925 | 0.934 | 0.945 | 0.954 | 0.985 | 54.846 | 1928 | +| gpt-5.5 | 0.965 | 0.925 | 0.930 | 0.944 | 0.953 | 0.975 | 52.641 | 1928 | +| deepseek-v4.1-flash | 0.966 | 0.921 | 0.931 | 0.946 | 0.953 | 0.982 | 47.408 | 1928 | +| grok-4.6 | 0.968 | 0.921 | 0.929 | 0.941 | 0.949 | 0.986 | 65.404 | 1928 | +| ox-alpha | 0.960 | 0.923 | 0.932 | 0.941 | 0.950 | 0.979 | 57.695 | 1928 | +| gpt-5.6-luna | 0.958 | 0.926 | 0.933 | 0.941 | 0.949 | 0.982 | 90.130 | 1928 | +| kimi-k3 | 0.945 | 0.917 | 0.925 | 0.934 | 0.936 | 0.950 | 29.075 | 1928 | +| grok-4.5 | 0.960 | 0.915 | 0.921 | 0.932 | 0.941 | 0.988 | 68.676 | 1928 | +| deepseek-v4-pro-0813 | 0.962 | 0.911 | 0.920 | 0.933 | 0.940 | 0.993 | 82.369 | 1928 | +| gemini-3.7-flash | 0.960 | 0.909 | 0.913 | 0.929 | 0.936 | 0.989 | 85.695 | 1928 | +| deepseek-v4-flash-0731 | 0.949 | 0.900 | 0.907 | 0.926 | 0.931 | 0.988 | 105.486 | 1928 | +| claude-opus-5 | 0.955 | 0.908 | 0.912 | 0.922 | 0.935 | 0.982 | 93.798 | 1928 | +| gemini-3.1-pro-preview | 0.955 | 0.901 | 0.904 | 0.916 | 0.926 | 0.985 | 87.154 | 1928 | +| claude-fable-5 | 0.955 | 0.908 | 0.912 | 0.924 | 0.934 | 0.978 | 93.963 | 1928 | +| gemini-3-flash-preview | 0.959 | 0.900 | 0.903 | 0.917 | 0.930 | 0.992 | 84.151 | 1928 | +| gemini-3.5-flash | 0.953 | 0.894 | 0.898 | 0.909 | 0.921 | 0.986 | 97.026 | 1928 | +| deepseek-v4-pro | 0.949 | 0.893 | 0.898 | 0.918 | 0.930 | 0.980 | 89.015 | 1928 | +| grok-build-0.1 | 0.944 | 0.892 | 0.898 | 0.908 | 0.918 | 0.986 | 113.500 | 1928 | +| claude-sonnet-4.6 | 0.944 | 0.894 | 0.900 | 0.920 | 0.930 | 0.949 | 66.821 | 1928 | +| gemini-3.5-flash-lite | 0.930 | 0.894 | 0.898 | 0.904 | 0.911 | 0.960 | 176.527 | 1928 | +| claude-opus-4.7 | 0.929 | 0.883 | 0.888 | 0.897 | 0.909 | 0.935 | 113.241 | 1928 | +| glm-5.3 | 0.916 | 0.871 | 0.881 | 0.893 | 0.899 | 0.933 | 99.894 | 1928 | +| gemini-3.1-flash-lite-preview | 0.933 | 0.889 | 0.897 | 0.903 | 0.911 | 0.976 | 165.370 | 1928 | +| grok-4.3 | 0.936 | 0.899 | 0.907 | 0.912 | 0.921 | 0.971 | 180.197 | 1928 | +| qwen-3.7-max | 0.915 | 0.869 | 0.875 | 0.885 | 0.892 | 0.951 | 181.704 | 1928 | +| glm-5.2 | 0.884 | 0.841 | 0.846 | 0.864 | 0.871 | 0.913 | 94.059 | 1928 | +| claude-opus-4.8 | 0.917 | 0.874 | 0.878 | 0.885 | 0.895 | 0.914 | 133.447 | 1928 | +| claude-haiku-4.5 | 0.898 | 0.861 | 0.867 | 0.874 | 0.881 | 0.923 | 233.201 | 1928 | +| qwen3.8-max | 0.898 | 0.861 | 0.864 | 0.871 | 0.877 | 0.935 | 264.844 | 1928 | +| claude-sonnet-5 | 0.916 | 0.871 | 0.878 | 0.883 | 0.890 | 0.941 | 168.369 | 1928 | +| minimax-m3 | 0.910 | 0.880 | 0.886 | 0.898 | 0.900 | 0.962 | 333.132 | 1928 | +| gpt-5.4-mini | 0.889 | 0.855 | 0.864 | 0.866 | 0.872 | 0.918 | 258.998 | 1928 | +| gpt-5.4-nano | 0.883 | 0.843 | 0.847 | 0.856 | 0.864 | 0.932 | 360.226 | 1928 | +| kimi-k2.6 | 0.771 | 0.760 | 0.761 | 0.765 | 0.769 | 0.905 | 223.785 | 1928 | ## Population household-impact weights (headline) @@ -106,45 +120,52 @@ Households receive equal weight. The score is a weighted average of row scores: | model | bounded_score | amount_accuracy | participation_accuracy | | --- | ---: | ---: | ---: | -| claude-fable-5 | 0.920 | 0.781 | 0.967 | -| claude-fable-5.1 | 0.959 | 0.872 | 0.981 | -| claude-haiku-4.5 | 0.824 | 0.591 | 0.912 | -| claude-opus-4.7 | 0.891 | 0.778 | 0.941 | -| claude-opus-4.8 | 0.836 | 0.717 | 0.924 | -| claude-opus-5 | 0.915 | 0.764 | 0.965 | -| claude-sonnet-4.6 | 0.899 | 0.802 | 0.948 | -| claude-sonnet-5 | 0.812 | 0.694 | 0.924 | -| deepseek-v4-flash-0731 | 0.915 | 0.771 | 0.962 | -| deepseek-v4-pro | 0.898 | 0.735 | 0.964 | -| deepseek-v4-pro-0813 | 0.915 | 0.764 | 0.974 | -| gemini-3-flash-preview | 0.911 | 0.752 | 0.974 | -| gemini-3.1-flash-lite-preview | 0.872 | 0.675 | 0.947 | -| gemini-3.1-pro-preview | 0.917 | 0.724 | 0.971 | -| gemini-3.5-flash | 0.907 | 0.728 | 0.970 | -| gemini-3.5-flash-lite | 0.883 | 0.600 | 0.939 | -| gemini-3.6-flash | 0.938 | 0.817 | 0.978 | -| gemini-3.7-flash | 0.920 | 0.748 | 0.972 | -| gemini-3.8-flash | 0.946 | 0.811 | 0.977 | -| glm-5.2 | 0.845 | 0.665 | 0.894 | -| glm-5.3 | 0.885 | 0.757 | 0.927 | -| gpt-5.4-mini | 0.789 | 0.448 | 0.899 | -| gpt-5.4-nano | 0.738 | 0.447 | 0.892 | -| gpt-5.5 | 0.939 | 0.850 | 0.968 | -| gpt-5.6-luna | 0.927 | 0.796 | 0.964 | -| gpt-5.6-sol | 0.958 | 0.853 | 0.980 | -| gpt-5.6-terra | 0.938 | 0.816 | 0.970 | -| gpt-6-astra | 0.963 | 0.879 | 0.984 | -| grok-4.3 | 0.875 | 0.629 | 0.941 | -| grok-4.5 | 0.918 | 0.752 | 0.971 | -| grok-4.6 | 0.938 | 0.790 | 0.976 | -| grok-build-0.1 | 0.891 | 0.732 | 0.958 | -| inkling | 0.952 | 0.810 | 0.975 | -| kimi-k2.6 | 0.671 | 0.272 | 0.767 | -| kimi-k3 | 0.938 | 0.819 | 0.947 | -| minimax-m3 | 0.793 | 0.468 | 0.908 | -| ox-alpha | 0.933 | 0.790 | 0.964 | -| qwen-3.7-max | 0.852 | 0.665 | 0.934 | -| qwen3.8-max | 0.810 | 0.567 | 0.909 | +| claude-fable-5 | 0.929 | 0.808 | 0.967 | +| claude-fable-5.1 | 0.966 | 0.895 | 0.983 | +| claude-haiku-4.5 | 0.836 | 0.640 | 0.918 | +| claude-opus-4.7 | 0.899 | 0.813 | 0.941 | +| claude-opus-4.8 | 0.849 | 0.766 | 0.928 | +| claude-opus-5 | 0.931 | 0.814 | 0.968 | +| claude-opus-5.5 | 0.976 | 0.885 | 0.987 | +| claude-sonnet-4.6 | 0.909 | 0.848 | 0.951 | +| claude-sonnet-5 | 0.825 | 0.757 | 0.927 | +| claude-sonnet-5.5 | 0.970 | 0.883 | 0.979 | +| deepseek-v4-flash-0731 | 0.931 | 0.824 | 0.965 | +| deepseek-v4-pro | 0.916 | 0.791 | 0.966 | +| deepseek-v4-pro-0813 | 0.935 | 0.823 | 0.977 | +| deepseek-v4.1-flash | 0.951 | 0.859 | 0.975 | +| gemini-3-flash-preview | 0.925 | 0.793 | 0.977 | +| gemini-3.1-flash-lite-preview | 0.891 | 0.734 | 0.952 | +| gemini-3.1-pro-preview | 0.930 | 0.766 | 0.975 | +| gemini-3.5-flash | 0.920 | 0.766 | 0.972 | +| gemini-3.5-flash-lite | 0.900 | 0.658 | 0.945 | +| gemini-3.6-flash | 0.952 | 0.848 | 0.981 | +| gemini-3.7-flash | 0.935 | 0.784 | 0.976 | +| gemini-3.8-flash | 0.963 | 0.844 | 0.982 | +| glm-5.2 | 0.869 | 0.733 | 0.900 | +| glm-5.3 | 0.896 | 0.795 | 0.931 | +| gpt-5.4-mini | 0.813 | 0.504 | 0.908 | +| gpt-5.4-nano | 0.757 | 0.495 | 0.901 | +| gpt-5.5 | 0.951 | 0.879 | 0.970 | +| gpt-5.6-luna | 0.946 | 0.836 | 0.968 | +| gpt-5.6-sol | 0.969 | 0.887 | 0.983 | +| gpt-5.6-terra | 0.952 | 0.866 | 0.973 | +| gpt-6-astra | 0.970 | 0.909 | 0.987 | +| gpt-6-luna | 0.974 | 0.890 | 0.982 | +| gpt-6-sol | 0.980 | 0.905 | 0.987 | +| gpt-6.1-sol | 0.956 | 0.898 | 0.982 | +| grok-4.3 | 0.891 | 0.694 | 0.949 | +| grok-4.5 | 0.937 | 0.811 | 0.976 | +| grok-4.6 | 0.949 | 0.834 | 0.979 | +| grok-4.7 | 0.968 | 0.882 | 0.981 | +| grok-build-0.1 | 0.911 | 0.787 | 0.963 | +| inkling | 0.964 | 0.856 | 0.978 | +| kimi-k2.6 | 0.695 | 0.282 | 0.780 | +| kimi-k3 | 0.946 | 0.854 | 0.951 | +| minimax-m3 | 0.821 | 0.507 | 0.923 | +| ox-alpha | 0.949 | 0.832 | 0.968 | +| qwen-3.7-max | 0.873 | 0.726 | 0.937 | +| qwen3.8-max | 0.830 | 0.606 | 0.915 | ## Summary by variable @@ -153,21 +174,21 @@ Amount variables use the tolerance columns. Binary coverage flags use `mean_accu | variable | metric_type | mean_score | mean_exact | mean_within_1pct | mean_within_5pct | mean_within_10pct | mean_accuracy | mean_mae | total_n | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| federal_income_tax_before_refundable_credits | amount_tolerance | 0.781 | 0.511 | 0.537 | 0.601 | 0.643 | n/a | 1863.451 | 3900 | -| federal_refundable_credits | amount_tolerance | 0.896 | 0.838 | 0.849 | 0.862 | 0.874 | n/a | 229.403 | 3900 | -| free_school_meals_eligible | binary_accuracy | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.017 | 3900 | -| local_income_tax | amount_tolerance | 0.992 | 0.992 | 0.992 | 0.992 | 0.992 | n/a | 7.509 | 3900 | -| payroll_tax | amount_tolerance | 0.941 | 0.753 | 0.790 | 0.843 | 0.893 | n/a | 310.599 | 3900 | -| person_chip_eligible | binary_accuracy | 0.955 | 0.955 | 0.955 | 0.955 | 0.955 | 0.955 | 0.037 | 6903 | -| person_early_head_start_eligible | binary_accuracy | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.013 | 1482 | -| person_head_start_eligible | binary_accuracy | 0.967 | 0.967 | 0.967 | 0.967 | 0.967 | 0.967 | 0.026 | 1482 | -| person_medicaid_eligible | binary_accuracy | 0.905 | 0.905 | 0.905 | 0.905 | 0.905 | 0.905 | 0.087 | 6903 | -| person_medicare_eligible | binary_accuracy | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.023 | 6708 | -| person_wic_eligible | binary_accuracy | 0.976 | 0.976 | 0.976 | 0.976 | 0.976 | 0.976 | 0.018 | 6903 | -| reduced_price_school_meals_eligible | binary_accuracy | 0.977 | 0.977 | 0.977 | 0.977 | 0.977 | 0.977 | 0.018 | 3900 | -| self_employment_tax | amount_tolerance | 0.972 | 0.965 | 0.971 | 0.972 | 0.972 | n/a | 26.341 | 3900 | -| snap | amount_tolerance | 0.867 | 0.796 | 0.809 | 0.835 | 0.844 | n/a | 280.544 | 3783 | -| ssi | amount_tolerance | 0.985 | 0.974 | 0.978 | 0.981 | 0.982 | n/a | 91.280 | 3783 | -| state_income_tax_before_refundable_credits | amount_tolerance | 0.798 | 0.547 | 0.574 | 0.640 | 0.685 | n/a | 369.513 | 3900 | -| state_refundable_credits | amount_tolerance | 0.822 | 0.787 | 0.788 | 0.792 | 0.795 | n/a | 70.408 | 3900 | -| tanf | amount_tolerance | 0.987 | 0.984 | 0.984 | 0.984 | 0.984 | n/a | 47.378 | 3900 | +| federal_income_tax_before_refundable_credits | amount_tolerance | 0.827 | 0.616 | 0.636 | 0.687 | 0.718 | n/a | 826.301 | 3772 | +| federal_refundable_credits | amount_tolerance | 0.919 | 0.868 | 0.880 | 0.893 | 0.904 | n/a | 173.006 | 4508 | +| free_school_meals_eligible | binary_accuracy | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.980 | 0.017 | 4600 | +| local_income_tax | amount_tolerance | 0.993 | 0.993 | 0.993 | 0.993 | 0.993 | n/a | 6.366 | 4600 | +| payroll_tax | amount_tolerance | 0.950 | 0.785 | 0.819 | 0.866 | 0.907 | n/a | 269.028 | 4554 | +| person_chip_eligible | binary_accuracy | 0.958 | 0.958 | 0.958 | 0.958 | 0.958 | 0.958 | 0.036 | 8142 | +| person_early_head_start_eligible | binary_accuracy | 0.983 | 0.983 | 0.983 | 0.983 | 0.983 | 0.983 | 0.011 | 1748 | +| person_head_start_eligible | binary_accuracy | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.971 | 0.023 | 1748 | +| person_medicaid_eligible | binary_accuracy | 0.922 | 0.922 | 0.922 | 0.922 | 0.922 | 0.922 | 0.072 | 7958 | +| person_medicare_eligible | binary_accuracy | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.972 | 0.023 | 7912 | +| person_wic_eligible | binary_accuracy | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.979 | 0.015 | 8142 | +| reduced_price_school_meals_eligible | binary_accuracy | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.985 | 0.011 | 4600 | +| self_employment_tax | amount_tolerance | 0.976 | 0.970 | 0.975 | 0.976 | 0.976 | n/a | 23.083 | 4600 | +| snap | amount_tolerance | 0.895 | 0.842 | 0.855 | 0.874 | 0.880 | n/a | 221.958 | 4278 | +| ssi | amount_tolerance | 0.987 | 0.977 | 0.981 | 0.984 | 0.985 | n/a | 77.869 | 4462 | +| state_income_tax_before_refundable_credits | amount_tolerance | 0.842 | 0.643 | 0.667 | 0.724 | 0.761 | n/a | 214.939 | 3956 | +| state_refundable_credits | amount_tolerance | 0.855 | 0.819 | 0.820 | 0.825 | 0.828 | n/a | 61.134 | 4508 | +| tanf | amount_tolerance | 0.988 | 0.985 | 0.985 | 0.985 | 0.985 | n/a | 43.135 | 4600 | diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv index c6ad90ce..3d68303b 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_model.csv @@ -1,40 +1,47 @@ model,mean_score,mean_exact,mean_within_1pct,mean_within_5pct,mean_threshold_score,mean_mae,mean_mape,mean_within_10pct,mean_accuracy,mean_coverage,total_n,parsed_n,bounded_score,amount_accuracy,participation_accuracy,weighted_exact,weighted_within_1pct,weighted_within_5pct,weighted_within_10pct,weighted_threshold_score,equal_score,aggregate_score -gpt-6-astra,0.974638852683144,0.9323440292528916,0.9435410510169235,0.9547037085197874,0.9475731243273474,47.678824689727406,0.34166541643209847,0.9597037085197874,0.9871426225200368,1.0,1973,1973,0.9627637446116617,0.8792817427431432,0.9842878864673087,0.8798405966173023,0.8980045177911136,0.9214941963687224,0.9325182740578966,0.9079643962087587,0.9712535816688499,0.9536853915818365 -claude-fable-5.1,0.9733310821242773,0.9287273943197003,0.9427021938615101,0.954975962475485,0.9472898227274895,49.9662157059445,0.25106275477090967,0.9627537402532629,0.9828711733018001,1.0,1973,1973,0.9589060707148584,0.8721342880047261,0.9807399898631526,0.8694450508949266,0.8989778548059895,0.9243105580987035,0.9399092381670505,0.9081606754916676,0.9708316652333271,0.950977142345685 -gpt-5.6-sol,0.97153769001177,0.9341134583498322,0.9447377424277246,0.9569943289111151,0.9496031490714816,45.033031503303185,0.18317787500162244,0.9625670665972549,0.9887398173695966,1.0,1973,1973,0.9581724376533884,0.8532115935692468,0.979726305119108,0.8916113620851432,0.9147236563828984,0.9341439501413595,0.9422896882445738,0.9206921642134938,0.9689246180886683,0.9479927802428617 -inkling,0.9610390169483176,0.9066791892364144,0.9212095443337799,0.932910575261615,0.9257885821344671,85.72790925517208,0.26330765550215185,0.9423550197060594,0.9920127118644068,1.0,1973,1973,0.9518907309481883,0.8098143837788206,0.974657881398885,0.8412384274320336,0.8707916016126718,0.8923248877386851,0.9138877059026244,0.8795606556715038,0.9579557374059035,0.9243137752508704 -gemini-3.8-flash,0.963938938313658,0.9070009901566777,0.9114626167317064,0.9338222959986021,0.9245869008095986,96.55219437999308,0.3422864930154151,0.9460617003514086,0.9865254237288136,1.0,1973,1973,0.945990200699823,0.8108464681487306,0.9771920932589965,0.821689176212669,0.8294292398386122,0.8777830447876082,0.9086094411651235,0.8593777255010032,0.9621322706901324,0.9169125831443159 -gpt-5.5,0.9603002772257372,0.9080894604075039,0.9198420377270915,0.9354147754132313,0.9270513733513757,70.39306016572775,0.2464582587771592,0.9448592198576757,0.9740904611746157,1.0,1973,1973,0.9394026362245165,0.8499177930017718,0.9675620881905728,0.841231014316272,0.8594889842827935,0.8886965782335765,0.9100579033121002,0.8748686200361856,0.9593260900597084,0.9211856286402571 -kimi-k3,0.9378769475787444,0.8975002414972817,0.908124525575174,0.9237488096530664,0.9138361521502026,40.96338921090958,0.2182371655204444,0.9259710318752885,0.9478033784204303,0.967033989676012,1973,1910,0.9382712893838939,0.8186816196726526,0.9472883933096807,0.8670466707725133,0.8887865045331198,0.9173764728827781,0.9235685339545969,0.8991945455357521,0.9392914966041508,0.9288970763226462 -grok-4.6,0.9615685729044523,0.9070639265860303,0.9159871797360876,0.9322185657612881,0.9236776150167292,117.54839454281402,0.2532263394147091,0.9394407879835103,0.9842443502824858,1.0,1973,1973,0.9379948364561399,0.7898702643277468,0.9756715661429296,0.8308223054329461,0.8460998843383978,0.8737167086787602,0.8900854944329768,0.8601810982207703,0.960569196267162,0.8965492689772527 -gemini-3.6-flash,0.9628318757859802,0.8947065770995528,0.8991682036745815,0.9237329230331152,0.913256118909404,107.2974859548746,0.2960929498150884,0.9354167718303662,0.9876129943502825,1.0,1973,1973,0.9376480764973395,0.8166120044252899,0.9776989356310187,0.7937882830456495,0.8002247785821344,0.8485978928535536,0.8789200230854358,0.8303827443916932,0.960660321741303,0.9064059260925051 -gpt-5.6-terra,0.9580731832876506,0.9087477753832813,0.9176882106639227,0.9333296768723992,0.9261905016146673,100.11955069338838,0.35642276543534795,0.9449963435390659,0.9842443502824859,1.0,1973,1973,0.9375287427266437,0.8156044936497355,0.9700963000506843,0.8351417190945837,0.8543447318813855,0.8842734971348294,0.9095312410568578,0.8708227972919141,0.9548241719527062,0.923834784965011 -ox-alpha,0.9535155774893723,0.9051509187507141,0.9168863139397173,0.9302540115342192,0.9231363139397174,79.22803041598384,0.3474102010941736,0.9402540115342192,0.9785741032715806,1.0,1973,1973,0.932726511085841,0.7895361341749771,0.9640141915864167,0.8412418881943892,0.8640577736894801,0.8880769753071012,0.9068624669486741,0.875059776034911,0.952029769510382,0.9079204605307925 -gpt-5.6-luna,0.9511292826392229,0.9080113412747541,0.9164134031304242,0.9281144340582593,0.9223856253526466,107.80502182143236,0.29789632971359953,0.937003322947148,0.9788373735382998,1.0,1973,1973,0.9269679520434653,0.795668838659076,0.9635073492143943,0.8464588249546756,0.8632363179076276,0.8866974616742729,0.9087724969626098,0.8762912753747965,0.9466756229828032,0.9034629215586181 -claude-fable-5,0.9497654836876551,0.8942545139119346,0.8998272515980743,0.9137676868787159,0.9079128758821522,131.11544358667777,0.40670631471119895,0.9238020511398842,0.9779618315595848,1.0,1973,1973,0.9199514374300629,0.7807091461974256,0.9665484034465281,0.8042658310774163,0.8124293618310976,0.8399437285240426,0.8660447746663887,0.8306709240247363,0.9442254770992659,0.89546330941077 -gemini-3.7-flash,0.9530044650641442,0.8944477126086423,0.8983709657586993,0.9156619165032586,0.9084010344872218,161.42149010237569,0.4102862629976555,0.9251235430782871,0.9870692090395481,1.0,1973,1973,0.9196552602897557,0.7475296505411132,0.9721236695387735,0.7971220852338265,0.8025749186633044,0.836394450214059,0.8577533970315866,0.8234612127856942,0.9503348849388769,0.871051200696168 -grok-4.5,0.9522509510430108,0.9008085639898723,0.9058429282510407,0.9181510611261839,0.9133858835775013,148.45571198000005,0.3092192385832874,0.928740980942908,0.9850924648535015,1.0,1973,1973,0.9179409734908818,0.7519971540556022,0.9711099847947289,0.8125376381278256,0.8213341926073351,0.8435031548321483,0.8656026867680118,0.8357444180838302,0.9482404498705438,0.8715799202896477 -gemini-3.1-pro-preview,0.9492673966669768,0.8876807315950931,0.8910140649284265,0.9055444200257919,0.9000976846306029,135.43654850234265,0.38616788409096753,0.9161515219731,0.9831321615528771,1.0,1973,1973,0.9172165054855052,0.7243188134790699,0.9711099847947289,0.784538047372326,0.7898843877037368,0.8194102551774681,0.8380638022718107,0.8079741231313353,0.9482526774665649,0.8679005959409433 -deepseek-v4-pro-0813,0.9546225679245823,0.8967880101492942,0.9045829700576561,0.9185577695994659,0.9121345164494087,140.64549559431703,0.38927453057656475,0.9286093159912185,0.9911235382998292,1.0,1973,1973,0.9149100758381041,0.7641651741784975,0.9736441966548404,0.7946433836506062,0.807264353771781,0.8359282549781442,0.8544415897354963,0.8230693955340068,0.9501159497471745,0.8577027713793348 -deepseek-v4-flash-0731,0.9425564481337229,0.8866436284794887,0.8944901347796033,0.9123366410797177,0.9031227235539446,167.21107851891614,0.39919235236344364,0.9190204898769685,0.985798679542767,1.0,1973,1973,0.9149096439787072,0.7709203494418247,0.9619868220983274,0.7967104729569928,0.8066935213343279,0.8488265659200368,0.861905024897103,0.8285338962771153,0.9380862955138014,0.8686072860032904 -claude-opus-5,0.947851846321369,0.8933733647703187,0.8983733647703187,0.909501658012014,0.9060337313224378,175.6405241025849,0.5953620731953996,0.9228865377370999,0.9786338851662069,1.0,1973,1973,0.9147088254429412,0.7636085056271273,0.9650278763304613,0.8033087936951527,0.8130828038187567,0.8348064188192317,0.8619729077100102,0.8282927310107878,0.9430608867470397,0.8855122465600354 -gemini-3-flash-preview,0.9530136855518356,0.8855875260726961,0.8894935970921691,0.90457950774509,0.8999618101505883,130.30815770698328,0.27648348585878696,0.9201866096923982,0.9897111089212981,1.0,1973,1973,0.9105815203008053,0.7515386766759045,0.9736441966548404,0.774619527918973,0.7818886303490086,0.8125897413257619,0.8395531063447892,0.8021627514846332,0.9482251469093863,0.8603866885526945 -gemini-3.5-flash,0.9486621820184014,0.8813469374148437,0.8869368572315677,0.8981682432567681,0.894770476933744,147.24571399310028,0.29768324077729763,0.9126298698317967,0.9875924648535015,1.0,1973,1973,0.9067873900596061,0.7279805967892965,0.9695894576786619,0.7676583294306227,0.7757862921026134,0.7967222645262144,0.8242327066128471,0.7910998981680745,0.9445928853904667,0.8499311310406693 -claude-sonnet-4.6,0.9378684259633473,0.8801589911958904,0.8857145467514459,0.9085610530515604,0.8984006864994414,101.11215843771764,0.38738146489360303,0.9191681549988685,0.9474969054484852,1.0,1973,1973,0.8989903386910217,0.8019190527551915,0.947795235681703,0.7708724064844426,0.7830180741510694,0.8407050331285426,0.862328206445131,0.8142309300522963,0.9382429704496993,0.8911313469784118 -deepseek-v4-pro,0.9427298995672149,0.8809410515175428,0.8854026780925714,0.9071208911509907,0.8979370423537397,175.87568236418977,0.3288554173088728,0.9182835486538544,0.98164056179076,1.0,1973,1973,0.8981520663084136,0.7348908565808743,0.9640141915864167,0.7636972918891604,0.7743020928828357,0.8243721499909666,0.8482256682126246,0.8026493007438968,0.9392659947136874,0.843204592351766 -claude-opus-4.7,0.9257314506157136,0.870836339642791,0.875836339642791,0.8869646328844862,0.8839047817962846,149.1325410595977,0.8998301053220994,0.9019818150150704,0.9329255786292693,1.0,1973,1973,0.8913574199089824,0.7780317664395848,0.941206284845413,0.7784836657611881,0.7884769696884549,0.8133296665198746,0.8407202708922198,0.8052526432154343,0.9292374160530662,0.8844902685729334 -grok-build-0.1,0.936398450618944,0.8782381043112526,0.883272468572421,0.8966745304280911,0.8914969817453878,178.88714227621347,0.4101484214414303,0.9078028236697864,0.983136250164236,1.0,1973,1973,0.8912137393376327,0.7319008320593892,0.9579320831221491,0.7634955418189706,0.7735623463396475,0.7977626286169701,0.8192943340122595,0.7885287126969618,0.9307142464529993,0.8272666742504371 -glm-5.3,0.9083233231676111,0.8569139060586625,0.8658371592087196,0.881461443286612,0.8729505612705754,128.6950350315968,0.5106879539501504,0.8875897365283073,0.9328114432711657,0.9603161620627007,1973,1896,0.8852443865703014,0.7573053265903535,0.9265078560567663,0.7775907833920889,0.7964606924558509,0.8316911885325188,0.8484672082856362,0.8135524681665237,0.9134301537643128,0.8513037961989999 -gemini-3.5-flash-lite,0.9208087984486352,0.8803409330498086,0.8847853774942531,0.8909308528665325,0.8888303374026151,240.48560030311234,0.7102006037296723,0.8992641861998659,0.9577516354445436,1.0,1973,1973,0.8826614214747605,0.6000454006619693,0.9386720729853015,0.7863634564981931,0.7970950083016879,0.8120904636938452,0.8325737596999189,0.8070306720484113,0.9213951823223282,0.8341598114361417 -grok-4.3,0.9270203721289418,0.88555102107031,0.894439909959199,0.90055102107031,0.8976343544036435,265.2117799362681,0.6894415720938362,0.9099954655147546,0.9683016530783009,1.0,1973,1973,0.8747037033156172,0.6286209256753433,0.941206284845413,0.7752108095302402,0.7948914210662249,0.81096659852564,0.8370480281053685,0.8045292143068684,0.9241138124730359,0.8350195675594297 -gemini-3.1-flash-lite-preview,0.924461853676774,0.8754974722413182,0.8832752500190959,0.8910530277968738,0.8878671744177214,239.92285345149352,0.6783562047028888,0.9016429476135978,0.9731425084192546,1.0,1973,1973,0.872182689604189,0.6752955754691964,0.9467815509376584,0.7614313366186152,0.7800637195931291,0.8016277668470069,0.82154689729851,0.7911674300893153,0.92030596188407,0.8043949051545586 -qwen-3.7-max,0.9073629106647348,0.8567904095569728,0.862901520668084,0.8740641781709476,0.8686217383084046,283.59426920040346,0.8282825551205634,0.8807308448376143,0.9486675967609208,1.0,1973,1973,0.8521696232644789,0.6648071931474315,0.9341104916371009,0.7387586428074273,0.7559284104401975,0.7825278209149518,0.8003394452832714,0.769388579861462,0.9085975859238969,0.7867178251473982 -glm-5.2,0.8743926757999719,0.8283954840202388,0.8339854038369627,0.8501308792092421,0.8423316237682341,185.42592970814337,0.6374884001864051,0.8568147280064929,0.9112223132723413,0.9268457571298018,1973,1834,0.8449867111238569,0.6652806513980082,0.8940699442473391,0.7353598614689811,0.7480879986045914,0.7897574394281218,0.80789771801679,0.7702757543796211,0.877911364647671,0.8085896848537153 -claude-opus-4.8,0.9093707345043409,0.8600359382412378,0.8644803826856822,0.8728137160190155,0.8699013448849949,212.54505473453074,0.5874455339477463,0.8822753425940442,0.912413335269589,1.0,1973,1973,0.8360837559951022,0.7165814733534971,0.9244804865686771,0.7305729785780646,0.7434350628570979,0.7625117523941336,0.7884678484128789,0.7562469105605437,0.9101509941018512,0.8292926416799227 -claude-haiku-4.5,0.8897049000755981,0.8482411328990573,0.8549077995657239,0.8632411328990572,0.8594911328990572,376.27588053167256,0.718604204466884,0.8715744662323907,0.9217770851053531,1.0,1973,1973,0.8235704829231111,0.5913921042250149,0.9118094272681196,0.7196935118992598,0.7346694141299938,0.7574925102106169,0.7811655985250583,0.7482552586912321,0.8905865315057568,0.7844775402033506 -claude-sonnet-5,0.9087492936245316,0.8583878854206513,0.8667212187539846,0.8722767743095401,0.8697853653748323,268.3414065530457,1.0004883557285715,0.881755583015153,0.9400325360108983,1.0,1973,1973,0.8117415981610512,0.6935842081705925,0.9239736441966548,0.6989458587658737,0.7223443048662783,0.7333395344761461,0.75377677243003,0.727101617634582,0.903648953336567,0.7932742346216795 -qwen3.8-max,0.890178520497827,0.8481372175395712,0.8509149953173488,0.8598038842062378,0.8563316619840156,339.7497047142416,0.9176509974312462,0.8664705508729045,0.931697914721767,1.0,1973,1973,0.8099181352444372,0.5666008791702518,0.9092752154080082,0.7177933771983871,0.7238557136436904,0.7448922205635194,0.7578396660215462,0.7360952443567857,0.8897490916691819,0.762137367702676 -minimax-m3,0.8963517310059714,0.8665941630239689,0.8732608296906355,0.883816385246191,0.877709569667726,554.0412816362845,0.8945545873519234,0.8871669007101085,0.959321402886404,1.0,1973,1973,0.7928830388302339,0.4681089582765488,0.9082615306639635,0.7257001244234154,0.7423709404236964,0.7693611780522359,0.7781053685681215,0.7538844028668673,0.8894599979530428,0.7171777765672098 -gpt-5.4-mini,0.8787405892998578,0.8421013941714315,0.8509902830603204,0.8537680608380982,0.851545838615876,440.75374843469757,0.7606395245255718,0.8593236163936537,0.9156946317310817,1.0,1973,1973,0.7893089887263386,0.4476707569860188,0.8986315255955398,0.7058600077934797,0.7247798739826367,0.7316423642807699,0.7472868069995411,0.7273922632641069,0.8838634933606965,0.7347176662131385 -gpt-5.4-nano,0.8735880949650299,0.8304291825133043,0.8343180714021933,0.8432069602910821,0.8398736269577489,495.70278420225833,0.8356562587675175,0.8515402936244155,0.9292001967374092,1.0,1973,1973,0.7375782134039863,0.44694528367738545,0.8920425747592499,0.6260653863633667,0.6363789914202844,0.6604173064097992,0.6792976558240081,0.6505398350043646,0.8695088037070893,0.6851425424440358 -kimi-k2.6,0.7600944663790917,0.7491254784784949,0.7496982161646346,0.7536042871841078,0.7523456961188156,221.2318753504815,0.3623709143974633,0.7569548026480253,0.90180809977249,0.7830731460828959,1973,1551,0.6708104738922708,0.2722058930005656,0.7668525088697415,0.6468017887356505,0.647007942581644,0.6576999521754702,0.6622314995095888,0.6534352957505883,0.741128796849098,0.5848920003832458 +gpt-6-sol,0.9814179607937498,0.9589805000560949,0.9634692750345306,0.9702588125549652,0.9663823939526416,21.09912763195815,0.18498827325090708,0.9728209881649759,0.9964036119003299,1.0,1928,1928,0.980299198559808,0.9050395821621691,0.9865145228215768,0.9500349913571872,0.9586029750969454,0.9642213904089867,0.9706811228110646,0.9608851199185459,0.9811661086475718,0.9762659140116118 +claude-opus-5.5,0.9814448136513197,0.958258316413452,0.9651014543762297,0.9702848474536346,0.9673100449390701,27.669246843538268,0.12008490885512936,0.9755955615129643,0.9884163237647366,1.0,1928,1928,0.9760997380035821,0.8847610305356218,0.9870331950207469,0.9369749019238288,0.9488831885276894,0.9574625086459541,0.9687372623411001,0.9530144653596432,0.9824218821638284,0.9652829756190323 +gpt-6-luna,0.9770199089499402,0.9458698495329028,0.9545127814671976,0.9674735618476203,0.9599362717993171,24.1405419381156,0.14365023511775513,0.9718888943495477,0.9890609499636592,1.0,1928,1928,0.9741637983000545,0.8902870586912563,0.9823651452282157,0.9209699160500828,0.9381308317526569,0.9573666125898856,0.9669276653615521,0.9458487564385443,0.9762892388464636,0.9658375040261253 +claude-sonnet-5.5,0.9703463446957241,0.9419961273885191,0.9481549513188592,0.9579233321066063,0.9527302317472928,29.59684454331689,0.3435271866046292,0.9628465161751865,0.9843879780645893,1.0,1928,1928,0.9701245986422122,0.8834868732193254,0.979253112033195,0.9208077168426426,0.9310611423025681,0.9471230799077293,0.9613462720532719,0.9400845527765529,0.9710188357168519,0.9635664323242513 +gpt-6-astra,0.9811217521873845,0.952509735548159,0.9598385979230696,0.9652656515296331,0.9615140737573802,25.734785123180995,0.3288491259277409,0.9684423100286589,0.9883762938567011,1.0,1928,1928,0.9698446045414372,0.9089705409272095,0.9865145228215768,0.9167398399934551,0.926704193695381,0.9391637338255535,0.9453781568309596,0.9319964810863374,0.9790228034968075,0.9649470446555626 +gpt-5.6-sol,0.9766275018846758,0.9540906197305199,0.9584201580909963,0.9670514001109426,0.9621166837338294,33.082150895497456,0.1607306632479988,0.9689045570028593,0.9906143887421841,1.0,1928,1928,0.9685553786104374,0.8873636850190184,0.9828838174273858,0.9357174672125771,0.9426182283296569,0.955545604572212,0.9587353434639426,0.9481541608945971,0.9760835621335312,0.9625081019145191 +grok-4.7,0.9736508321195915,0.9279123256586492,0.9405020442713456,0.9531764373091222,0.9458272584586077,42.963662721226775,0.16843953177851276,0.9617182265953136,0.9843209355108935,1.0,1928,1928,0.9676800515903546,0.8821664403772467,0.9808091286307054,0.883352356001671,0.908559540301881,0.9357550903955741,0.9484097272845778,0.919019178495926,0.9751010633997181,0.9494464709931986 +claude-fable-5.1,0.9781482986920358,0.9478705738728485,0.9566448890743803,0.9652318935490762,0.9602142143821868,40.48275514642431,0.22595937599340182,0.9711095010324422,0.9847457446743875,1.0,1928,1928,0.9661083374877115,0.8946044148481577,0.9828838174273858,0.9082809795026888,0.9224870514513555,0.9390559636414038,0.9506434828354557,0.9301168693577259,0.9770670579048212,0.9595755765726094 +inkling,0.9679351511049066,0.9241283869684912,0.935678127432821,0.9448523915547504,0.9394877701510475,52.122087619837494,0.23519227380274155,0.9532921746481272,0.9931973972110643,1.0,1928,1928,0.9639903404563158,0.8559774726592909,0.9782157676348547,0.8822006716270263,0.9022048592027716,0.916394075900917,0.9353534600182131,0.909038266687232,0.9674936961726915,0.9488438065306128 +gemini-3.8-flash,0.9716992672276967,0.9217597942040326,0.9259481738076948,0.9482326445207081,0.9380578090879934,60.42930149291984,0.30199753600331375,0.9562906238195379,0.989812424479932,1.0,1928,1928,0.9629217728031497,0.8442975314835007,0.9818464730290456,0.8594579492214984,0.8674176419758158,0.9127675751184425,0.9317056158315293,0.8928371955368216,0.972734728451396,0.9413556307999181 +gpt-6.1-sol,0.9787878895026749,0.9516662498186138,0.9542331178228779,0.9609206491504977,0.9584825134820816,26.74093784952796,0.11564146986446056,0.9671100371363366,0.9854903212797419,1.0,1928,1928,0.9558118540969375,0.8981716471310712,0.9823651452282157,0.905950906002756,0.9097537892746472,0.9220411569594783,0.9344346489403966,0.9180451252943195,0.9762099167131537,0.9522130134086377 +gemini-3.6-flash,0.9681412429532106,0.9086751193473108,0.9135583023805487,0.9343464827207113,0.9245540764731586,63.15553317811516,0.25823757803284125,0.9416364014440632,0.9908836664380654,1.0,1928,1928,0.9524952325919379,0.8482295728516276,0.9813278008298755,0.8309499152818556,0.8377568744351451,0.8790786893770673,0.8993339257104228,0.8617798512011227,0.968634418383436,0.9277137230652192 +gpt-5.6-terra,0.9638139933140464,0.92544673083851,0.9341488226960892,0.9445582875592233,0.9394983368834259,54.84621636313941,0.3372875240446065,0.9538395064398804,0.9849007217269194,1.0,1928,1928,0.9521005308051074,0.8659779297334677,0.9730290456431535,0.8769497703300592,0.8924375535140979,0.9133116919368196,0.9325789584167555,0.903819493549433,0.9628327732854907,0.9444338799439029 +gpt-5.5,0.9646404648991201,0.9246842273825712,0.9300484424336888,0.944254589550459,0.9380105257611654,52.64077923622599,0.2029228562987933,0.9530548436779421,0.9748611332623985,1.0,1928,1928,0.9512547300857929,0.8794634658357077,0.9699170124481328,0.8815164989242267,0.8886576150317778,0.915182660691513,0.9306851538766645,0.9040104821310454,0.9656883437448258,0.9373700993926879 +deepseek-v4.1-flash,0.9658652674175693,0.9212507257926723,0.931372195560965,0.945709082844094,0.93776153126703,47.40807539889512,0.21055902520764067,0.9527141208703891,0.9818471695970921,1.0,1928,1928,0.9505626593089697,0.8589675935210785,0.9745850622406639,0.870552377499924,0.8863835381257342,0.9150223230715568,0.9257265838043338,0.8994212056253872,0.9662068932038941,0.9296733787785436 +grok-4.6,0.9676153897191448,0.9207958098267345,0.9285686228014003,0.9414466753329932,0.9349519777565745,65.40430245665854,0.22042523237505246,0.94899680306517,0.9861025929917377,1.0,1928,1928,0.94910144673076,0.8344304283167924,0.9787344398340249,0.8640208736627577,0.8774845427809858,0.9003659610470429,0.9151445377859353,0.8892539788191804,0.9693291751539781,0.9180889964628234 +ox-alpha,0.9598755058341616,0.9228249998408226,0.9318973922993491,0.9407498610213584,0.9364622499463796,57.695123375040794,0.3047762404111062,0.9503767466239889,0.9793121180326922,1.0,1928,1928,0.9486340123268947,0.8323635545464565,0.9683609958506224,0.8849353035198623,0.8973663128671125,0.9141211107760786,0.9300044543189705,0.9066067953705059,0.9609317477776966,0.930757645749188 +gpt-5.6-luna,0.9584490092723846,0.9263787925574842,0.9331486733743601,0.941122535434566,0.9374785774755162,90.13009436971377,0.2941645526985832,0.9492643085356539,0.9820753882994114,1.0,1928,1928,0.9459529089279916,0.8361243267856716,0.9683609958506224,0.8930789108165669,0.9053600129749496,0.9170421014385453,0.9343828154470901,0.912465960169288,0.9566637176129394,0.9324597909831576 +kimi-k3,0.9453459258075978,0.9165886387942346,0.9246120720088958,0.9339523327469365,0.9277568208177936,29.07549217141032,0.1861176960492084,0.9358742397211081,0.9496126351396752,0.9691872786932749,1928,1870,0.9458318717784635,0.854467058832511,0.9512448132780082,0.9043471486856777,0.9148262667645122,0.9303337615499089,0.9346669844422065,0.9210435403605763,0.9460441860032874,0.9420423644201579 +grok-4.5,0.9598818070767913,0.9154385741064495,0.9207494734963861,0.9316092209912941,0.927198165721404,68.67606933804946,0.2744054398126192,0.9409953942914862,0.9883304796146132,1.0,1928,1928,0.9372547899612037,0.8109488737104825,0.9756224066390041,0.8508110483198444,0.8600850651724337,0.8814557170517974,0.9007512942777474,0.8732757812054557,0.9593331470997878,0.9023260230468467 +deepseek-v4-pro-0813,0.962099699376765,0.9112096296052186,0.9201953308359619,0.9329284801040149,0.9261383958850238,82.36930600567031,0.36509194684084745,0.9402201429949,0.9929817810090811,1.0,1928,1928,0.9352207817631968,0.8225994292583274,0.9771784232365145,0.8337167580453175,0.8482718523955461,0.8742991217590282,0.8868234431644124,0.8607777938410761,0.9615896933084268,0.8926578397400001 +gemini-3.7-flash,0.9602906927657445,0.909188166796313,0.9129432891776808,0.9285965816414651,0.9216471880583631,85.69492388836963,0.3708718975508487,0.9358607146179929,0.9889274517487998,1.0,1928,1928,0.9349128429689975,0.7841291502815011,0.9761410788381742,0.8311782882466474,0.8363691536248353,0.8661828353374164,0.8837484809941563,0.8543696895507638,0.9604840204302811,0.9014062245286685 +deepseek-v4-flash-0731,0.9492567743190694,0.9001848591205346,0.9073662120568959,0.9260929508817777,0.9162695412853482,105.48642986058614,0.392350488758424,0.9314341430821848,0.9876405935886833,1.0,1928,1928,0.931391679010326,0.823701250834762,0.9652489626556017,0.832758619160859,0.8418326968295833,0.8838514671885079,0.8946053404560782,0.8632620309087571,0.9484353902516142,0.9007185738602966 +claude-opus-5,0.9547065392942847,0.9075615750408347,0.9115011980280767,0.9222699147927611,0.9190289443461936,93.79797314101958,0.544992257132953,0.9347830895231017,0.9818555712639828,1.0,1928,1928,0.9310434478976686,0.8139376188354033,0.9678423236514523,0.8402988376350788,0.8500690856768351,0.8677312547350386,0.8909939384862093,0.8622732791332904,0.952637650446624,0.911258430877379 +gemini-3.1-pro-preview,0.9553168217701609,0.9007740532254841,0.904359402332536,0.9163158166021889,0.9118559010934888,87.15432176651447,0.35372322685315566,0.9259743322137466,0.9850067329254645,1.0,1928,1928,0.9295579714889485,0.765917379554329,0.9745850622406639,0.8163648613293834,0.8221503833486231,0.8453467776508942,0.8646018765972694,0.8371159747315425,0.9572577873328608,0.8914280291827413 +claude-fable-5,0.9545713913114773,0.907624308747288,0.9117596017376169,0.9244985631162942,0.9194070082375868,93.96338003758437,0.4005859084563832,0.9337455593491482,0.9779773029680953,1.0,1928,1928,0.9287276020256175,0.8076595240711623,0.9673236514522822,0.8362539444385491,0.8404194042865061,0.8670526164239241,0.8892460130189772,0.858242994541989,0.9513869894132212,0.9115431101721931 +gemini-3-flash-preview,0.9589663313888457,0.8996522998146228,0.9026650293014359,0.9170138459067836,0.9122092424648538,84.15074586858799,0.2539507381666943,0.9295057948365736,0.9915530229672144,1.0,1928,1928,0.9248205252928468,0.7927728586814228,0.9766597510373444,0.8095099070665046,0.8161251554501574,0.847519758726799,0.8662149249317327,0.8348424365437984,0.9575693036196992,0.8888230319361261 +gemini-3.5-flash,0.9530733194933845,0.8935733483740117,0.8984079081403217,0.9086275050652919,0.9053853159279461,97.02578452755242,0.26483519297636193,0.9209325021321588,0.9862444928734878,1.0,1928,1928,0.9195749854100529,0.7664274123581386,0.9719917012448133,0.8004238564541742,0.8065165534394115,0.8265249080337206,0.8523510538068777,0.821454092933546,0.9527394460017372,0.8763336211906928 +deepseek-v4-pro,0.9488443394488234,0.8932581031671444,0.8978810771915575,0.9182825760123128,0.9098231275284887,89.01494278960614,0.3038748272001937,0.9298707537429399,0.9802109464940681,1.0,1928,1928,0.9162568626737899,0.7914765292508901,0.9662863070539419,0.7975124065440952,0.8091779413249449,0.854731858125084,0.8793991141377664,0.8352053300329726,0.9494713206767496,0.8801994777368662 +grok-build-0.1,0.9436565089664983,0.8924475969210237,0.8977401658545028,0.9081554749797403,0.9039739954984535,113.49963228871815,0.3611555753648065,0.9175527442385472,0.9856680502360821,1.0,1928,1928,0.911324834497806,0.7865725229098195,0.9631742738589212,0.8036084468856437,0.8144544593251806,0.8345594821451222,0.8521073113533914,0.8261824249273344,0.9429261894430833,0.8614742542624602 +claude-sonnet-4.6,0.9443492800125435,0.893816326552655,0.8998304476578007,0.920491563187698,0.9109868772835988,66.82067026699806,0.39124113993966564,0.9298091717362413,0.9490938895443671,1.0,1928,1928,0.9093225103176983,0.8476235314972707,0.9512448132780082,0.8027617496987095,0.8152873822877525,0.8684827007162907,0.8842639688767113,0.842698950394866,0.9466964076408272,0.9112986879338776 +gemini-3.5-flash-lite,0.9299058422606034,0.8942695037428585,0.898302790540373,0.9038595505113461,0.9017669340666052,176.5266863823748,0.66782307159455,0.9106358914718431,0.9600548342296911,1.0,1928,1928,0.9003121813140013,0.6583192583063302,0.9450207468879668,0.819592849674438,0.8284338141389417,0.8418154889958008,0.8592299535611709,0.8372680265925878,0.934790014690542,0.8642923626322461 +claude-opus-4.7,0.9294155966652754,0.883358613731469,0.8876633328823648,0.8972254504314093,0.8943144944350566,113.24064367430096,0.9461610257801338,0.9090105806949834,0.9345062340618155,1.0,1928,1928,0.8990261973339878,0.8126567879344271,0.9408713692946058,0.809638962030392,0.8170908369740485,0.8400944767428736,0.8562462327242545,0.8307676271178921,0.9349168120322772,0.8989995553258002 +glm-5.3,0.9160488818087755,0.8707485354694499,0.8813037842189458,0.893209236058423,0.8859780615077325,99.89393937320148,0.4729580026438265,0.8986506902841115,0.9333861713989211,0.962633562677778,1928,1857,0.8961270934577402,0.7949526628856647,0.9310165975103735,0.8109010878646017,0.82887622207975,0.8534735655714336,0.8674625194707213,0.8401783487466267,0.9220863606487083,0.8710868252911296 +gemini-3.1-flash-lite-preview,0.9326084731439737,0.8890313500304539,0.8966012954033865,0.9030750485012652,0.8998407361541543,165.36950765373197,0.678201058122803,0.9106552506815111,0.9755600078477515,1.0,1928,1928,0.8914415485236766,0.7335430908291792,0.9522821576763485,0.7973940156545853,0.8148438903762849,0.8300632269033785,0.8474813485700159,0.8224456203760662,0.9336205118095529,0.8396908508711926 +grok-4.3,0.936220341272469,0.8991468276048935,0.9074201800236419,0.9119518520728032,0.9098198932649605,180.1974997561796,0.6852615649570439,0.9207607133585031,0.9706701665167907,1.0,1928,1928,0.8913971012353002,0.6936499214298234,0.9486514522821576,0.8062160266220387,0.8253905982363676,0.8355781152859795,0.8589631293707023,0.8315369673787721,0.9370300256296413,0.8650105677077221 +qwen-3.7-max,0.9148372111324807,0.8694118729203308,0.8745844437142721,0.8849968568041263,0.8801796954345934,181.70446667166084,0.593493647619928,0.8917256082996445,0.9510034528727395,1.0,1928,1928,0.8733527967112642,0.7263852207337363,0.9372406639004149,0.7738613945075842,0.7879437682154989,0.8127196352984511,0.8317596242907331,0.8015711055780669,0.9213910024734056,0.8275536117528435 +glm-5.2,0.8841384713439249,0.8410193335630537,0.8461908705550457,0.8640392718817209,0.8555925441527662,94.059002352998,0.5988609504402723,0.8711207006112445,0.9129817267467543,0.9283912819377359,1928,1795,0.8686613898600047,0.7327937248573134,0.9004149377593361,0.7704754617804436,0.7840294653618636,0.8254327679945089,0.8406122691696918,0.8051374910766269,0.8912899338382545,0.8448881754965842 +claude-opus-4.8,0.917277011591461,0.8741093122108663,0.8778253342195712,0.885375461951748,0.8829952974584289,133.4474580562063,0.5979715763208318,0.8946710814515301,0.9144716041047022,1.0,1928,1928,0.8491925054467029,0.7659158053582666,0.9284232365145229,0.7617545674851087,0.7725613559407958,0.7875534238142073,0.8100663384747758,0.7829839214287219,0.9204903869347357,0.8550425377335356 +claude-haiku-4.5,0.8975967813092348,0.8608492699066819,0.867206441547461,0.874130633998218,0.8708633301218671,233.2007302891389,0.6658604374675829,0.8812669750351074,0.92324343989455,1.0,1928,1928,0.8364631494260415,0.6395117537948684,0.9175311203319502,0.7466036494857531,0.7597216713709039,0.7783789532532154,0.7978305248301713,0.770633699735011,0.9025550691161903,0.8101744104080428 +qwen3.8-max,0.897743259143331,0.8609528171795334,0.8639283085271178,0.871155203364417,0.8682907009322809,264.843794022813,0.9654386997264108,0.8771264746580554,0.9346093562161663,1.0,1928,1928,0.8295047048737395,0.6056235240758913,0.9154564315352697,0.7522482335536425,0.7583105699989459,0.7757329061246871,0.7873445137394236,0.7684090558541747,0.9029942791670642,0.7930790000853262 +claude-sonnet-5,0.9163640300577666,0.8706497527903911,0.8781518200998755,0.8834997722984341,0.8806825005830362,168.36902323615237,1.0826361188594396,0.8904286571434442,0.9405477461607955,1.0,1928,1928,0.8249049611863575,0.7566682623955442,0.9273858921161826,0.7264249589279979,0.7466300183212481,0.7583771649556214,0.7742378744351629,0.7514175041600075,0.9154873154677481,0.8212286727816881 +minimax-m3,0.9096878161865847,0.8799958231031496,0.8864749459634409,0.8979734183650004,0.8911746197613017,333.1320685425183,0.8387682266075392,0.9002542916136158,0.961559287018209,1.0,1928,1928,0.8210743147259882,0.5070351016020668,0.9232365145228216,0.7574649533851503,0.772712752290567,0.799814357124164,0.8053641549589191,0.7838390544397,0.9091157441071778,0.7626378618112046 +gpt-5.4-mini,0.8888801014329085,0.8554017804119598,0.8643523976959104,0.866205554587827,0.8644945316783261,258.9979867129371,0.773984472933787,0.8720183940176072,0.9179161871995511,1.0,1928,1928,0.8129939112383417,0.5044004645969282,0.908195020746888,0.7366669485735765,0.7563642913518649,0.7592267879361472,0.7768933140410081,0.7572878354756492,0.8999141242971496,0.7732990603483123 +gpt-5.4-nano,0.8825864207573021,0.8431602910687306,0.8471991895385234,0.8563475203922806,0.852628434722448,360.2260670847895,0.8467176860802642,0.8638067378902577,0.931785064965096,1.0,1928,1928,0.7566934446405278,0.4950288188898851,0.9009336099585062,0.6533891134939903,0.6637027185509079,0.6891289387798616,0.7082365628740529,0.6786143334247032,0.8845196616478074,0.7119230351678831 +kimi-k2.6,0.770569946963265,0.7599334481720343,0.7605061858581741,0.7650207465834338,0.7635559113295471,223.78534444526923,0.3472571293811342,0.7687632647045466,0.9047195412668892,0.7912510772014619,1928,1538,0.6945103956892189,0.2815131325996502,0.7800829875518672,0.6740417281119927,0.6742572172627803,0.6853579711586619,0.6899989280107147,0.6809139611360373,0.7559141764378726,0.6215421764469465 diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv index 9469672f..5c4a5dad 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/summary_by_variable.csv @@ -1,19 +1,19 @@ variable,mean_score,mean_exact,mean_within_1pct,mean_within_5pct,mean_threshold_score,mean_mae,mean_mape,mean_within_10pct,mean_accuracy,mean_coverage,total_n,parsed_n -federal_income_tax_before_refundable_credits,0.780685900774454,0.5107692307692308,0.5371794871794872,0.6012820512820513,0.5729487179487179,1863.4514815857115,0.5525134098679534,0.6425641025641026,,0.9835897435897436,3900,3836 -federal_refundable_credits,0.8964753036984047,0.8376923076923077,0.8489743589743589,0.8623076923076923,0.8558333333333332,229.40282801295163,0.3674270735368196,0.8743589743589744,,0.9828205128205127,3900,3833 -free_school_meals_eligible,0.9787179487179488,0.9787179487179488,0.9787179487179488,0.9787179487179488,0.9787179487179488,0.016756047255758456,,0.9787179487179488,0.9787179487179488,0.9953846153846154,3900,3882 -local_income_tax,0.9917948717948718,0.9917948717948718,0.9917948717948718,0.9917948717948718,0.9917948717948718,7.509030769230769,,0.9917948717948718,,0.9938461538461538,3900,3876 -payroll_tax,0.9406600856048715,0.7528205128205128,0.7899999999999999,0.8433333333333334,0.8197435897435897,310.59921716639076,0.05810635209761717,0.8928205128205128,,0.9825641025641025,3900,3832 -person_chip_eligible,0.9549471244386498,0.9549471244386498,0.9549471244386498,0.9549471244386498,0.9549471244386498,0.0374985816563228,,0.9549471244386498,0.9549471244386498,0.9923221787628567,6903,6850 -person_early_head_start_eligible,0.9804318488529015,0.9804318488529015,0.9804318488529015,0.9804318488529015,0.9804318488529015,0.012838749680854944,,0.9804318488529015,0.9804318488529015,0.9932523616734142,1482,1472 -person_head_start_eligible,0.9669365721997301,0.9669365721997301,0.9669365721997301,0.9669365721997301,0.9669365721997301,0.026315789473684206,,0.9669365721997301,0.9669365721997301,0.9932523616734142,1482,1472 -person_medicaid_eligible,0.9049688541213965,0.9049688541213965,0.9049688541213965,0.9049688541213965,0.9049688541213965,0.08739568607196385,,0.9049688541213965,0.9049688541213965,0.9920324496595683,6903,6848 -person_medicare_eligible,0.9712283840190816,0.9712283840190816,0.9712283840190816,0.9712283840190816,0.9712283840190816,0.023057076247745903,,0.9712283840190816,0.9712283840190816,0.994186046511628,6708,6669 -person_wic_eligible,0.9756627553237723,0.9756627553237723,0.9756627553237723,0.9756627553237723,0.9756627553237723,0.018118604559282527,,0.9756627553237723,0.9756627553237723,0.9937708242792989,6903,6860 -reduced_price_school_meals_eligible,0.9774358974358973,0.9774358974358973,0.9774358974358973,0.9774358974358973,0.9774358974358973,0.017774186699379555,,0.9774358974358973,0.9774358974358973,0.9951282051282052,3900,3881 -self_employment_tax,0.9723790172000574,0.9653846153846154,0.9710256410256409,0.9715384615384616,0.9700641025641027,26.341312572011727,0.04398065924249991,0.9723076923076923,,0.9861538461538462,3900,3846 -snap,0.8667289077385529,0.796193497224425,0.8094105207507268,0.8347872059212265,0.8211075865715041,280.544215034844,0.865629712803505,0.8440391223896379,,0.9944488501189531,3783,3762 -ssi,0.9847900766510853,0.9740946338884482,0.9777954004758129,0.9812318265926513,0.9788527623579171,91.28000328801029,0.29050705044880887,0.9822891884747554,,0.9944488501189531,3783,3762 -state_income_tax_before_refundable_credits,0.7977006900764958,0.5471794871794872,0.5738461538461538,0.6402564102564102,0.611474358974359,369.5132164550931,0.5399122839524048,0.6846153846153846,,0.9856410256410256,3900,3844 -state_refundable_credits,0.8222507868569306,0.7866666666666666,0.7876923076923078,0.7915384615384615,0.7903205128205129,70.40800511139568,0.9958572918329688,0.7953846153846154,,0.9853846153846154,3900,3843 -tanf,0.9866601196063574,0.9835897435897436,0.9838461538461538,0.9838461538461538,0.9839102564102564,47.3777046761851,0.6848824614527874,0.9843589743589743,,0.9943589743589744,3900,3878 +federal_income_tax_before_refundable_credits,0.8270114441901764,0.616118769883351,0.6362672322375398,0.6871686108165429,0.6643027571580064,826.3006887051604,0.4798283353935361,0.7176564156945918,,0.9878048780487805,3772,3726 +federal_refundable_credits,0.9190875115986977,0.8684560780834072,0.8804347826086957,0.8926353149955635,0.8862577639751553,173.00560977562694,0.28203535783422007,0.9035048802129547,,0.9855811889973382,4508,4443 +free_school_meals_eligible,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.9795652173913044,0.016597518325534345,,0.9795652173913044,0.9795652173913044,0.9960869565217392,4600,4582 +local_income_tax,0.9930434782608696,0.9930434782608696,0.9930434782608696,0.9930434782608696,0.9930434782608696,6.366352173913043,,0.9930434782608696,,0.9947826086956522,4600,4576 +payroll_tax,0.9495457150909299,0.785024154589372,0.8188405797101449,0.8658322353974528,0.8442577953447519,269.0284228879276,0.05014541219683578,0.9073342116820378,,0.9855072463768116,4554,4488 +person_chip_eligible,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.9579955784819455,0.0355996940768357,,0.9579955784819455,0.9579955784819455,0.9934905428641612,8142,8089 +person_early_head_start_eligible,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.9834096109839817,0.010885026903333539,,0.9834096109839817,0.9834096109839817,0.994279176201373,1748,1738 +person_head_start_eligible,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.9713958810068649,0.02288329519450801,,0.9713958810068649,0.9713958810068649,0.994279176201373,1748,1738 +person_medicaid_eligible,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.9218396582055793,0.07223360158487031,,0.9218396582055793,0.9218396582055793,0.9938426740387031,7958,7909 +person_medicare_eligible,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.9723205257836198,0.02283453835564706,,0.9723205257836198,0.9723205257836198,0.9950707785642063,7912,7873 +person_wic_eligible,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.9792434291328912,0.015484245550568322,,0.9792434291328912,0.9792434291328912,0.9947187423237535,8142,8099 +reduced_price_school_meals_eligible,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.9850000000000001,0.01092008078387851,,0.9850000000000001,0.9850000000000001,0.9958695652173913,4600,4581 +self_employment_tax,0.976146812550527,0.9704347826086956,0.9752173913043478,0.9756521739130435,0.9744021739130434,23.08262975507091,0.037295638672444176,0.9763043478260869,,0.9882608695652174,4600,4546 +snap,0.8950815250162902,0.8419822346891072,0.8548387096774193,0.8740065451145395,0.8627279102384292,221.95803343535022,0.7373026265141042,0.8800841514726507,,0.9955586722767649,4278,4259 +ssi,0.9870536611205077,0.9773644105782161,0.9807261317794712,0.9836396234872254,0.9815665620797849,77.86850121885766,0.24877145637345424,0.9845360824742267,,0.9952935903182428,4462,4441 +state_income_tax_before_refundable_credits,0.8420508644142654,0.6430738119312437,0.6665824064711829,0.7237108190091001,0.6986223458038422,214.93925324059367,0.3869237269608064,0.7611223458038422,,0.9903943377148635,3956,3918 +state_refundable_credits,0.854945577117676,0.8194321206743567,0.8198757763975155,0.8245341614906833,0.8230368234250223,61.134148706701154,0.947599145924913,0.8283052351375332,,0.9875776397515529,4508,4452 +tanf,0.9882009832576173,0.9845652173913043,0.9850000000000001,0.9850000000000001,0.9850000000000001,43.13453546934715,0.6283439336657944,0.9854347826086957,,0.9952173913043478,4600,4578 diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv index e4dce8c2..2fc00d90 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/analysis/usage_summary.csv @@ -5,11 +5,14 @@ claude-haiku-4.5,1984,1984,16,,5.54126899999997,5.54126899999997,1984,5.54126899 claude-opus-4.7,1984,1984,0,,29.30251999999998,29.30251999999998,1984,29.30251999999998,6206.862578470866,,,,4246299.0,322841.0,4569140.0,0.0,0.0, claude-opus-4.8,1984,1984,0,,25.505849999999974,25.505849999999974,1984,25.505849999999974,6341.968646365076,,,,3452460.0,329742.0,3782202.0,0.0,0.0, claude-opus-5,1984,1984,0,,6.748584999999974,6.748584999999974,1984,6.748584999999974,1808.0877771629603,516.2362098693848,1784918536.916325,1784919053.152535,595917.0,150760.0,746677.0,0.0,0.0,0.0 +claude-opus-5.5,1984,1984,0,,6.7490399999999795,6.7490399999999795,0,6.7490399999999795,2685.3920743390336,1223.5300071239471,1790095878.102897,1790097101.632904,208620.0,295728.0,504348.0,0.0,0.0,0.0 claude-sonnet-4.6,1984,1984,19,,18.581885999999972,18.581885999999972,1984,18.581885999999972,13238.981947822951,,,,3045207.0,629751.0,3674958.0,0.0,0.0, claude-sonnet-5,1984,1984,0,,17.175698999999973,17.175698999999973,1984,17.175698999999973,6622.604365989508,2976.2204461097717,1782978630.540542,1782981606.760988,3605008.0,424045.0,4029053.0,0.0,0.0, +claude-sonnet-5.5,1984,1984,0,,3.4307399999999766,3.4307399999999766,0,3.4307399999999766,2220.8435553733143,12310.435461044312,1790626110.3269188,1790638420.76238,208620.0,301350.0,509970.0,0.0,0.0,0.0 deepseek-v4-flash-0731,1984,1984,0,0.34530876799997345,0.3466731949999584,0.3466731949999584,0,0.3466731949999584,35008.575926833,7975.363518953323,1788522611.5125,1788530586.876019,403375.0,1780299.0,2183674.0,1540655.0,11008.0,0.0 deepseek-v4-pro,1984,1984,0,,1.090330892999965,1.090330892999965,1984,1.090330892999965,19105.404083625996,5972.14363694191,1783264088.99296,1783270061.136597,139947.0,1184994.0,1324941.0,1071780.0,3456.0, deepseek-v4-pro-0813,1984,1984,0,12.378114560879968,19.865588542799987,19.865588542799987,0,19.865588542799987,87457.32326587,18585.698496103287,1788531693.8939688,1788550279.592465,470783.0,5779833.0,6250616.0,5551000.0,178176.0,0.0 +deepseek-v4.1-flash,1984,1984,0,,2.4189574439999353,2.4189574439999353,0,2.4189574439999353,8310.926795706735,12687.724947929382,1790626141.695307,1790638829.420255,140455.0,1980935.0,2121390.0,1885065.0,1024.0, gemini-3-flash-preview,1984,1984,36,,4.833864999999969,4.833864999999969,1984,4.833864999999969,6549.0484062079995,,,,169688.0,1583007.0,1752695.0,1430547.0,, gemini-3.1-flash-lite-preview,1984,1984,0,,0.18828349999999838,0.18828349999999838,1984,0.18828349999999838,495.3015514161434,,,,138368.0,102461.0,240829.0,,, gemini-3.1-pro-preview,1984,1984,0,,8.467731999999957,8.467731999999957,1984,8.467731999999957,4852.748729715,,,,138368.0,682583.0,820951.0,565613.0,, @@ -27,9 +30,13 @@ gpt-5.6-luna,1984,1984,0,,1.8656761999999611,1.8656761999999611,1984,1.865676199 gpt-5.6-sol,1984,1984,0,,8.952270999999971,8.952270999999971,1984,8.952270999999971,3791.32924903417,2447.224079847336,1783623303.475012,1783625750.699092,248923.0,247455.0,496378.0,155156.0,1917.0,234106.0 gpt-5.6-terra,1984,1984,0,,3.89323549999997,3.89323549999997,1984,3.89323549999997,2023.1675169692608,1580.1203260421753,1783623303.476923,1783624883.597249,248923.0,208595.0,457518.0,126873.0,1917.0,234106.0 gpt-6-astra,1984,1984,0,,12.640101999999974,12.640101999999974,0,12.640101999999974,3684.9014010820247,18423.340574026108,1788556429.716652,1788574853.057226,248923.0,201093.0,450016.0,114690.0,42877.0,192846.0 +gpt-6-luna,1984,1984,0,,0.1905161749999766,0.1905161749999766,0,0.1905161749999766,3073.682229919039,9126.795432806015,1790107796.3817222,1790116923.177155,248923.0,327806.0,576729.0,239727.0,36280.0,199443.0 +gpt-6-sol,1984,1984,0,,2.71040389999997,2.71040389999997,0,2.71040389999997,2403.1760812419234,9015.502176046371,1790107796.383682,1790116811.885858,248923.0,217527.0,466450.0,137755.0,35032.0,200691.0 +gpt-6.1-sol,1984,1984,0,,2.3001227999999774,2.3001227999999774,0,2.3001227999999774,3641.613490711141,12294.100305080414,1790709098.1048489,1790721392.205154,248923.0,176029.0,424952.0,89656.0,32989.0,202734.0 grok-4.3,1984,1984,56,,1.540529199999975,1.540529199999975,1984,1.540529199999975,2617.3619202122964,,,,727886.0,294981.0,1022867.0,149054.0,101696.0, grok-4.5,1984,1984,0,,7.268361999999971,7.268361999999971,1968,7.268361999999971,11801.759922254994,4839.604803085327,1783728925.6229708,1783733765.227774,391592.0,1110491.0,1502083.0,986107.0,78592.0, grok-4.6,1984,1984,0,,8.70121599999997,8.70121599999997,1984,8.70121599999997,23167.131800621988,8831.936488866806,1787386137.732978,1787394969.669467,382325.0,1339817.0,1722142.0,1157238.0,68224.0, +grok-4.7,1984,1984,0,,25.96658799999996,25.96658799999996,0,25.96658799999996,52737.247623663745,18651.518170118332,1790626552.413439,1790645203.9316092,490145.0,4201631.0,4691776.0,4052657.0,148992.0, grok-build-0.1,1984,1984,0,,,,0,,14399.200466045018,,,,430643.0,2028279.0,2458922.0,1906156.0,132096.0, inkling,1984,1984,0,9.14737169999998,,9.14737169999998,0,9.14737169999998,29912.6794615346,12077.893361330032,1785887290.7756228,1785899368.6689842,338205.0,2229716.0,2567921.0,2001295.0,58240.0,0.0 kimi-k2.6,1984,1558,1476,39.10951922799996,,39.10951922799996,0,39.10951922799996,27880.1620396644,19014.219599962234,1783420244.710639,1783439258.930239,875917.0,10658123.0,11534040.0,10225616.0,289443.0, diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz index 06a87a13..9467415d 100644 Binary files a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz and b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/data.json.gz differ diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz index 9edbc2d1..7bfbb864 100644 Binary files a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz and b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz differ diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json index c79e7e68..e9370569 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json @@ -1,7 +1,7 @@ { "schema_version": 1, "rule": "An output is excluded from scoring for every model when its reference depends on an engine input that the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Excluded outputs stay in the published payload, marked scored=false, so every prompt and response remains inspectable.", - "derivation": "All 1,984 references were recomputed with policyengine-us 1.755.4 (the version that produced them; base recompute matched the frozen CSV on every row) under each alternative reading; every output whose value moved is listed. 2026-09-05.", + "derivation": "All 1,984 references were recomputed with policyengine-us 1.755.4 (the version that produced them; base recompute matched the frozen CSV on every row) under each alternative reading; every output whose value moved is listed. 2026-09-05. On 2026-09-29 the references moved to policyengine-us 2.15.17 with the pre-freeze conventions. Every excluded output was recomputed there, and each one that moved was re-reviewed and stays excluded (the reference sidecar's engine_upgrade revision lists them). Records decided before that date keep the values they were decided on; the four records decided that day were computed on 2.15.17. The audit excluded one of them on review of the release. Its reference did not move, so the engine_upgrade revision does not list it (final_actions.json audit_exclusions).", "exclusions": [ { "scenario_id": "scenario_007", @@ -80,12 +80,12 @@ "unlisted_input": "meets_ssi_disability_criteria", "alternative_reading": "The household fact 'is disabled' is read as satisfying SSI's disability criterion (the engine input meets_ssi_disability_criteria, which the certified microdata never sets and the prompt never lists). Recomputed with policyengine-us 1.755.4 with each person in a correctly built marital unit; the run's own situation placed every household member in one marital unit, which deems the whole household's income to the adult and masks the change.", "frozen_value": 461.339722, - "alternative_value": 3596.039795, + "alternative_value": 3576.0, "engine_version": "policyengine-us 1.755.4", "decided_on": "2026-09-05", "decided_by": "developer", "judge_verdict": "llm_error where any row was judged", - "note": "SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules." + "note": "SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-05 record gave 3,596.04 under that reading on the frozen engine; the 2026-09-22 audit recomputes it with every publication convention and upstream fix." }, { "scenario_id": "scenario_057", @@ -94,12 +94,12 @@ "unlisted_input": "meets_ssi_disability_criteria", "alternative_reading": "The household fact 'is disabled' is read as satisfying SSI's disability criterion (the engine input meets_ssi_disability_criteria, which the certified microdata never sets and the prompt never lists). Recomputed with policyengine-us 1.755.4 with each person in a correctly built marital unit; the run's own situation placed every household member in one marital unit, which deems the whole household's income to the adult and masks the change.", "frozen_value": 2669.217041, - "alternative_value": 883.617065, + "alternative_value": 840.0, "engine_version": "policyengine-us 1.755.4", "decided_on": "2026-09-05", "decided_by": "developer", "judge_verdict": "llm_error where any row was judged", - "note": "Moves with the additional SSI counted as household income." + "note": "Moves with the additional SSI counted as household income. The 2026-09-05 record gave 883.62 under that reading on the frozen engine; the 2026-09-22 audit recomputes it with every publication convention and upstream fix." }, { "scenario_id": "scenario_100", @@ -108,12 +108,12 @@ "unlisted_input": "meets_ssi_disability_criteria", "alternative_reading": "The household fact 'is disabled' is read as satisfying SSI's disability criterion (the engine input meets_ssi_disability_criteria, which the certified microdata never sets and the prompt never lists). Recomputed with policyengine-us 1.755.4 with each person in a correctly built marital unit; the run's own situation placed every household member in one marital unit, which deems the whole household's income to the adult and masks the change.", "frozen_value": 8625.889648, - "alternative_value": 8917.490234, + "alternative_value": 8844.0, "engine_version": "policyengine-us 1.755.4", "decided_on": "2026-09-05", "decided_by": "developer", "judge_verdict": "llm_error where any row was judged", - "note": "SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules." + "note": "SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-05 record gave 8,917.49 under that reading on the frozen engine; the 2026-09-22 audit recomputes it with every publication convention and upstream fix. r30_snap_heat_and_eat_sua (engine defect, not fixed upstream) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification." }, { "scenario_id": "scenario_057", @@ -156,6 +156,676 @@ "decided_by": "developer", "judge_verdict": "llm_error (gpt-5.6-sol, 2026-09-01 board)", "note": "Under the alternative reading nine of the twelve models that paid SSI land within 10% of $1,368 (six exactly); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion." + }, + { + "scenario_id": "scenario_002", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "frozen_value": 0.0, + "alternative_value": 1890.300049, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "note": "Corrected value applies r24_disability_benefits_taxable with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_003", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r01_ira_compensation", + "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.", + "frozen_value": 22154.69921875, + "alternative_value": 22400.654297, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return", + "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "upstream": "to be filed", + "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 106505.8984375, + "alternative_value": 107198.335938, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input)." + }, + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g+r11_ca_itemized_conformity", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.", + "frozen_value": 41051.51171875, + "alternative_value": 40920.402344, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 and r11_ca_itemized_conformity_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r07_idaho_health_premiums", + "alternative_reading": "Idaho subtracts health insurance premiums the taxpayer pays that are not otherwise deducted or excluded.", + "frozen_value": 755.7783203125, + "alternative_value": 651.317017, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays", + "law": "Idaho Code 63-3022P; Idaho Form 39R", + "upstream": "to be filed", + "note": "Corrected value applies r07_idaho_health_premiums_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_008", + "variable": "payroll_tax", + "reason_code": "reference_engine_defect", + "root_cause": "r05_nj_worker_ui", + "alternative_reading": "New Jersey workers contribute 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds, up to the 2026 wage base of $44,800, alongside the TDI and FLI contributions.", + "frozen_value": 2162.760009765625, + "alternative_value": 2276.659912, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax", + "law": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates", + "upstream": "to be filed", + "note": "Corrected value applies r05_nj_worker_ui_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.", + "frozen_value": 68056.7109375, + "alternative_value": 68112.09375, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say", + "note": "Corrected value applies r25_niit_excluded with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r11_ca_itemized_conformity", + "alternative_reading": "California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.", + "frozen_value": 2439.650146484375, + "alternative_value": 1968.80542, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions", + "law": "Cal. R&TC 17024.5 (conformity date), 17076", + "upstream": "to be filed", + "note": "Corrected value applies r11_ca_itemized_conformity_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_039", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r03_estate_income", + "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.", + "frozen_value": 1345.510498046875, + "alternative_value": 8596.02832, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income", + "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "upstream": "related: PolicyEngine/policyengine-us#9304", + "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_039", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r03_estate_income", + "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.", + "frozen_value": 514.4984130859375, + "alternative_value": 1975.798218, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income", + "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "upstream": "related: PolicyEngine/policyengine-us#9304", + "note": "Corrected value applies r03_estate_income__qbi_false_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions", + "alternative_reading": "A filer aged 67 or older who elects the retirement income exclusion computes Wisconsin tax on the reduced income and forfeits the credits, so the tax before refundable credits follows the elected path. Wisconsin subtracts 30% of net long-term capital gain, and a capital gain distribution reported without Schedule D is long-term capital gain.", + "frozen_value": 284.74090576171875, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies; PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%", + "law": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16; Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", + "upstream": "to be filed", + "note": "Corrected value applies r06_wi_act15_before_refundable_v2 and r32_wi_capital_gain_distributions together with every publication convention and upstream fix. The output also moves under r04_capital_gain_distributions (engine defect, fixed upstream)." + }, + { + "scenario_id": "scenario_049", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 30543.908203125, + "alternative_value": 30702.589844, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_052", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 104211.40625, + "alternative_value": 104225.257812, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input)." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r07_idaho_health_premiums", + "alternative_reading": "Idaho subtracts health insurance premiums the taxpayer pays that are not otherwise deducted or excluded.", + "frozen_value": 2435.2783203125, + "alternative_value": 2176.057129, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays", + "law": "Idaho Code 63-3022P; Idaho Form 39R", + "upstream": "to be filed", + "note": "Corrected value applies r07_idaho_health_premiums_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "frozen_value": 1140.0, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies", + "note": "Corrected value applies r14_unlisted_weekly_hours_v2_v2 and r15_snap_mortgage_interest_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_062", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72.", + "frozen_value": 0.0, + "alternative_value": 4328.386719, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "taxability of survivor_benefits", + "note": "Corrected value applies r16_survivor_benefits_federal_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_064", + "variable": "dependent1_medicaid_eligible", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", + "frozen_value": 1.0, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "the relationship of an adult tax dependent to the filers who claim them", + "note": "Corrected value applies r067_adult_dependent_relationship with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_064", + "variable": "dependent2_medicaid_eligible", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household.", + "frozen_value": 0.0, + "alternative_value": 1.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "the relationship of an adult tax dependent to the filers who claim them", + "note": "Corrected value applies r067_adult_dependent_nonchild with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_064", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r01_ira_compensation", + "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.", + "frozen_value": 4439.28955078125, + "alternative_value": 4441.455078, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return", + "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "upstream": "to be filed", + "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r01_ira_compensation", + "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.", + "frozen_value": 4605.9970703125, + "alternative_value": 4599.621582, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return", + "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "upstream": "to be filed", + "note": "Corrected value applies r01_ira_compensation_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_067", + "variable": "dependent1_medicaid_eligible", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", + "frozen_value": 1.0, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "the relationship of an adult tax dependent to the filers who claim them", + "note": "Corrected value applies r067_adult_dependent_relationship with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_080", + "variable": "snap", + "reason_code": "reference_engine_defect", + "root_cause": "r30_snap_heat_and_eat_sua", + "alternative_reading": "A LIHEAP payment makes a household eligible for the standard utility allowance only if the household has an elderly or disabled member; other households take the allowance only if they incur heating or cooling costs.", + "frozen_value": 3596.039794921875, + "alternative_value": 3240.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended", + "law": "7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04)", + "upstream": "to be filed", + "note": "Corrected value applies r30_snap_heat_and_eat_sua with every publication convention and upstream fix. The prompt says the head is disabled but states none of the routes 7 U.S.C. 2012(j) lists for a disabled member, such as receipt of SSI or Social Security disability benefits, so the engine and this record treat the household as having no elderly or disabled member (verification/v9_axiom_us-2014-sua.md). Read the other way, the LIHEAP payment would still confer the standard utility allowance, and the value the publication conventions and upstream SNAP fixes give, 3,576.00 (the reference it would carry if scored), would stand; the output is excluded either way." + }, + { + "scenario_id": "scenario_081", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r22_ma_part_a_loss_offset", + "alternative_reading": "Massachusetts applies excess short-term capital losses against Part A interest and dividends, up to $2,000 of combined losses, before taxing Part A income.", + "frozen_value": 8238.40625, + "alternative_value": 8232.900391, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes", + "law": "M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21", + "upstream": "to be filed", + "note": "Corrected value applies ma_part_a_loss_offset with every publication convention and upstream fix. The output also moves under r23_ma_interest_source (unlisted input)." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 9563.052734375, + "alternative_value": 9564.916992, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r32_wi_capital_gain_distributions", + "alternative_reading": "Wisconsin subtracts 30% of net long-term capital gain, and a capital gain distribution reported without Schedule D is long-term capital gain.", + "frozen_value": 843.6614990234375, + "alternative_value": 878.515747, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%", + "law": "Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", + "upstream": "to be filed", + "note": "Corrected value applies r32_wi_capital_gain_distributions with every publication convention and upstream fix. The output also moves under r04_capital_gain_distributions (engine defect, fixed upstream)." + }, + { + "scenario_id": "scenario_099", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 10679.75, + "alternative_value": 10711.484375, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g+r11_ca_itemized_conformity", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.", + "frozen_value": 4493.7431640625, + "alternative_value": 4588.483887, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); Cal. R&TC 17024.5 (conformity date), 17076", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 and r11_ca_itemized_conformity_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_100", + "variable": "federal_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r08_eitc_earned_income_deferrals", + "alternative_reading": "EITC earned income counts wages only if they are includible in gross income, which excludes elective deferrals (W-2 box 1 wages); the refundable child tax credit and state credits built on the EITC follow.", + "frozen_value": 2878.0966796875, + "alternative_value": 822.288269, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit", + "law": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "upstream": "to be filed", + "note": "Corrected value applies r08_eitc_earned_income_deferrals_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_100", + "variable": "state_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r08_eitc_earned_income_deferrals", + "alternative_reading": "EITC earned income counts wages only if they are includible in gross income, which excludes elective deferrals (W-2 box 1 wages); the refundable child tax credit and state credits built on the EITC follow.", + "frozen_value": 473.1777038574219, + "alternative_value": 164.457657, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit", + "law": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "upstream": "to be filed", + "note": "Corrected value applies r08_eitc_earned_income_deferrals_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_107", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "frozen_value": 0.0, + "alternative_value": 211.232422, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "note": "Corrected value applies r24_disability_benefits_taxable with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_108", + "variable": "state_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72.", + "frozen_value": 271.3916015625, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits", + "note": "Corrected value applies r10_wi_homestead_income and r16_survivor_benefits_federal_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_110", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g+r03_estate_income", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.", + "frozen_value": 23897.443359375, + "alternative_value": 25747.773438, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "upstream": "related: PolicyEngine/policyengine-us#9304", + "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_110", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g+r03_estate_income", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing. Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.", + "frozen_value": 4057.47216796875, + "alternative_value": 4236.807617, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out; PolicyEngine leaves estate income out of gross income while counting it as qualified business income", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges); 26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "upstream": "related: PolicyEngine/policyengine-us#9304", + "note": "Corrected value applies r02_ira_219g_v2 and r03_estate_income__qbi_false_v2 together with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.", + "frozen_value": 287.68316650390625, + "alternative_value": 0.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "weekly_hours_worked_before_lsr", + "note": "Corrected value applies r14_unlisted_weekly_hours_v2_v2 with every publication convention and upstream fix. The output also moves under r28_snap_min_allotment_rounding (engine defect, fixed upstream)." + }, + { + "scenario_id": "scenario_118", + "variable": "snap", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "frozen_value": 2903.9404296875, + "alternative_value": 3576.0, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "whether the listed home mortgage interest is on the home the SNAP household occupies", + "note": "Corrected value applies r15_snap_mortgage_interest_v2 with every publication convention and upstream fix. The output also moves under r26_snap_contribution_rounding (engine defect, fixed upstream)." + }, + { + "scenario_id": "scenario_119", + "variable": "federal_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r08_eitc_earned_income_deferrals", + "alternative_reading": "EITC earned income counts wages only if they are includible in gross income, which excludes elective deferrals (W-2 box 1 wages); the refundable child tax credit and state credits built on the EITC follow.", + "frozen_value": 2854.0087890625, + "alternative_value": 3366.07373, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit", + "law": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "upstream": "to be filed", + "note": "Corrected value applies r08_eitc_earned_income_deferrals_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_119", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r08_eitc_earned_income_deferrals", + "alternative_reading": "EITC earned income counts wages only if they are includible in gross income, which excludes elective deferrals (W-2 box 1 wages); the refundable child tax credit and state credits built on the EITC follow.", + "frozen_value": 1859.0943603515625, + "alternative_value": 1756.681396, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit", + "law": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "upstream": "to be filed", + "note": "Corrected value applies r08_eitc_earned_income_deferrals_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 40021.81640625, + "alternative_value": 40028.320312, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix. The output also moves under r25_niit_in_federal_output (unlisted input)." + }, + { + "scenario_id": "scenario_120", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_engine_defect", + "root_cause": "r02_ira_219g", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "frozen_value": 11917.18359375, + "alternative_value": 11919.057617, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "upstream": "to be filed", + "note": "Corrected value applies r02_ira_219g_v2 with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_121", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "frozen_value": 0.0, + "alternative_value": 875.337524, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "note": "Corrected value applies r24_disability_benefits_taxable with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_121", + "variable": "state_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "frozen_value": 0.0, + "alternative_value": 212.233505, + "engine_version": "policyengine-us 1.755.4", + "decided_on": "2026-09-22", + "decided_by": "developer", + "unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "note": "Corrected value applies r24_disability_benefits_taxable with every publication convention and upstream fix." + }, + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 3818.148193, + "alternative_value": 3788.988037, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py)." + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 24164.457031, + "alternative_value": 23772.800781, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py). The frozen value also counts Maryland county income tax, at the rate policyengine-us 2.15.17 assigns an unlisted county, in the federal SALT deduction through the Maryland withholding proxy (c_md_2026); the SALT cap binds at every 2026 county rate." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 24961.339844, + "alternative_value": 24391.796875, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py)." + }, + { + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "reason_code": "reference_depends_on_unlisted_input", + "unlisted_input": "meets_ssi_disability_criteria", + "alternative_reading": "The household fact 'is disabled' is read as not meeting the Social Security definition of disability (the engine input meets_ssi_disability_criteria, which the prompt does not list). The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition (42 CFR 435.540(a)); policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in that program (medicaid_category WORKING_DISABLED_BUY_IN).", + "frozen_value": 1.0, + "alternative_value": 0.0, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "judge_verdict": "llm_error (claude-opus-5-5, 2026-09-29)", + "note": "Flagged in the 2026-09-29 engine upgrade by the investigation of cluster excl_snap_ssi_disability and its independent review, and excluded on review of the release (reference_audit/2026-09-28/final_actions.json, audit_exclusions). Alternative computed on 2.15.17 with the publication conventions and the Working Disabled Program's disability test reading meets_ssi_disability_criteria (reference_audit/2026-09-28/scripts/probe_023_medicaid.py); unmodified, 2.15.17 gives 1 under either reading. The same unlisted input excludes this household's SNAP." } ] } diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv index 8e7b68ee..67d4e5ae 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv @@ -1,5 +1,5 @@ scenario_id,variable,value,impact_weight -scenario_000,federal_income_tax_before_refundable_credits,2883.49365234375, +scenario_000,federal_income_tax_before_refundable_credits,2906.45361328125, scenario_000,federal_refundable_credits,0.0, scenario_000,payroll_tax,0.0, scenario_000,self_employment_tax,0.0, @@ -136,9 +136,9 @@ scenario_008,federal_refundable_credits,12433.611328125, scenario_008,payroll_tax,2162.760009765625, scenario_008,self_employment_tax,565.1820068359375, scenario_008,state_income_tax_before_refundable_credits,264.80999755859375, -scenario_008,state_refundable_credits,5342.39990234375, +scenario_008,state_refundable_credits,5842.39990234375, scenario_008,local_income_tax,0.0, -scenario_008,snap,15246.9052734375, +scenario_008,snap,15108.0, scenario_008,ssi,0.0, scenario_008,tanf,0.0, scenario_008,head_wic_eligible,0.0,0.0 @@ -214,7 +214,7 @@ scenario_012,self_employment_tax,0.0, scenario_012,state_income_tax_before_refundable_credits,0.0, scenario_012,state_refundable_credits,0.0, scenario_012,local_income_tax,0.0, -scenario_012,snap,4952.08935546875, +scenario_012,snap,4884.0, scenario_012,ssi,0.0, scenario_012,tanf,0.0, scenario_012,head_wic_eligible,0.0,0.0 @@ -240,7 +240,7 @@ scenario_013,self_employment_tax,0.0, scenario_013,state_income_tax_before_refundable_credits,0.0, scenario_013,state_refundable_credits,25.0, scenario_013,local_income_tax,0.0, -scenario_013,snap,0.0, +scenario_013,snap,240.0, scenario_013,ssi,0.0, scenario_013,tanf,0.0, scenario_013,head_wic_eligible,0.0,0.0 @@ -349,7 +349,7 @@ scenario_021,head_chip_eligible,0.0,0.0 scenario_021,head_medicare_eligible,1.0,5285.2001953125 scenario_021,free_school_meals_eligible,0.0,0.0 scenario_021,reduced_price_school_meals_eligible,0.0,0.0 -scenario_022,federal_income_tax_before_refundable_credits,11131.3271484375, +scenario_022,federal_income_tax_before_refundable_credits,11113.5703125, scenario_022,federal_refundable_credits,0.0, scenario_022,payroll_tax,4710.83251953125, scenario_022,self_employment_tax,0.0, @@ -369,8 +369,8 @@ scenario_023,federal_income_tax_before_refundable_credits,643.43359375, scenario_023,federal_refundable_credits,0.0, scenario_023,payroll_tax,1561.1168212890625, scenario_023,self_employment_tax,0.0, -scenario_023,state_income_tax_before_refundable_credits,6.7984771728515625, -scenario_023,state_refundable_credits,148.3108673095703, +scenario_023,state_income_tax_before_refundable_credits,12.776718139648438, +scenario_023,state_refundable_credits,95.15400695800781, scenario_023,local_income_tax,0.0, scenario_023,snap,461.3397216796875, scenario_023,ssi,0.0, @@ -446,7 +446,7 @@ scenario_027,self_employment_tax,0.0, scenario_027,state_income_tax_before_refundable_credits,0.0, scenario_027,state_refundable_credits,0.0, scenario_027,local_income_tax,0.0, -scenario_027,snap,287.68316650390625, +scenario_027,snap,288.0, scenario_027,ssi,0.0, scenario_027,tanf,0.0, scenario_027,head_wic_eligible,0.0,0.0 @@ -492,7 +492,7 @@ scenario_028,child1_early_head_start_eligible,0.0,0.0 scenario_028,child2_early_head_start_eligible,0.0,0.0 scenario_028,child3_early_head_start_eligible,0.0,0.0 scenario_028,free_school_meals_eligible,0.0,0.0 -scenario_028,reduced_price_school_meals_eligible,1.0,3009.81884765625 +scenario_028,reduced_price_school_meals_eligible,0.0,3009.81884765625 scenario_029,federal_income_tax_before_refundable_credits,0.0, scenario_029,federal_refundable_credits,0.0, scenario_029,payroll_tax,0.0, @@ -516,7 +516,7 @@ scenario_030,self_employment_tax,0.0, scenario_030,state_income_tax_before_refundable_credits,0.0, scenario_030,state_refundable_credits,0.0, scenario_030,local_income_tax,0.0, -scenario_030,snap,287.68316650390625, +scenario_030,snap,288.0, scenario_030,ssi,0.0, scenario_030,tanf,0.0, scenario_030,head_wic_eligible,1.0,1026.747314453125 @@ -567,7 +567,7 @@ scenario_032,child1_head_start_eligible,0.0,0.0 scenario_032,child1_early_head_start_eligible,0.0,0.0 scenario_032,free_school_meals_eligible,1.0,1130.9622802734375 scenario_032,reduced_price_school_meals_eligible,0.0,0.0 -scenario_033,federal_income_tax_before_refundable_credits,3788.988037109375, +scenario_033,federal_income_tax_before_refundable_credits,3818.148193359375, scenario_033,federal_refundable_credits,0.0, scenario_033,payroll_tax,6043.5, scenario_033,self_employment_tax,0.0, @@ -626,7 +626,7 @@ scenario_038,self_employment_tax,2193.612548828125, scenario_038,state_income_tax_before_refundable_credits,0.0, scenario_038,state_refundable_credits,365.8000183105469, scenario_038,local_income_tax,0.0, -scenario_038,snap,7286.9443359375, +scenario_038,snap,7212.0, scenario_038,ssi,0.0, scenario_038,tanf,0.0, scenario_038,head_wic_eligible,0.0,0.0 @@ -687,7 +687,7 @@ scenario_040,head_medicare_eligible,1.0,5285.2001953125 scenario_040,spouse_medicare_eligible,1.0,5285.2001953125 scenario_040,free_school_meals_eligible,0.0,0.0 scenario_040,reduced_price_school_meals_eligible,0.0,0.0 -scenario_042,federal_income_tax_before_refundable_credits,1979.1583251953125, +scenario_042,federal_income_tax_before_refundable_credits,2361.96484375, scenario_042,federal_refundable_credits,0.0, scenario_042,payroll_tax,0.0, scenario_042,self_employment_tax,0.0, @@ -710,7 +710,7 @@ scenario_043,self_employment_tax,0.0, scenario_043,state_income_tax_before_refundable_credits,0.0, scenario_043,state_refundable_credits,19.0, scenario_043,local_income_tax,0.0, -scenario_043,snap,3596.039794921875, +scenario_043,snap,3576.0, scenario_043,ssi,0.0, scenario_043,tanf,0.0, scenario_043,head_wic_eligible,0.0,0.0 @@ -743,10 +743,10 @@ scenario_045,federal_income_tax_before_refundable_credits,2173.07080078125, scenario_045,federal_refundable_credits,0.0, scenario_045,payroll_tax,2775.08251953125, scenario_045,self_employment_tax,0.0, -scenario_045,state_income_tax_before_refundable_credits,1288.8375244140625, +scenario_045,state_income_tax_before_refundable_credits,1290.9625244140625, scenario_045,state_refundable_credits,760.7887573242188, scenario_045,local_income_tax,0.0, -scenario_045,snap,287.68316650390625, +scenario_045,snap,0.0, scenario_045,ssi,0.0, scenario_045,tanf,0.0, scenario_045,head_wic_eligible,0.0,0.0 @@ -827,7 +827,7 @@ scenario_051,federal_income_tax_before_refundable_credits,2620.0, scenario_051,federal_refundable_credits,0.0, scenario_051,payroll_tax,3060.0, scenario_051,self_employment_tax,0.0, -scenario_051,state_income_tax_before_refundable_credits,814.9500122070312, +scenario_051,state_income_tax_before_refundable_credits,820.3499755859375, scenario_051,state_refundable_credits,0.0, scenario_051,local_income_tax,0.0, scenario_051,snap,0.0, @@ -882,7 +882,7 @@ scenario_054,self_employment_tax,2416.15283203125, scenario_054,state_income_tax_before_refundable_credits,0.0, scenario_054,state_refundable_credits,0.0, scenario_054,local_income_tax,0.0, -scenario_054,snap,6125.68896484375, +scenario_054,snap,6060.0, scenario_054,ssi,0.0, scenario_054,tanf,0.0, scenario_054,head_wic_eligible,0.0,0.0 @@ -1048,7 +1048,7 @@ scenario_066,self_employment_tax,0.0, scenario_066,state_income_tax_before_refundable_credits,0.0, scenario_066,state_refundable_credits,7.955999851226807, scenario_066,local_income_tax,0.0, -scenario_066,snap,3596.039794921875, +scenario_066,snap,3576.0, scenario_066,ssi,0.0, scenario_066,tanf,0.0, scenario_066,head_wic_eligible,0.0,0.0 @@ -1085,7 +1085,7 @@ scenario_068,federal_income_tax_before_refundable_credits,1910.0238037109375, scenario_068,federal_refundable_credits,0.0, scenario_068,payroll_tax,2607.39013671875, scenario_068,self_employment_tax,0.0, -scenario_068,state_income_tax_before_refundable_credits,1252.9677734375, +scenario_068,state_income_tax_before_refundable_credits,1255.3427734375, scenario_068,state_refundable_credits,0.0, scenario_068,local_income_tax,0.0, scenario_068,snap,0.0, @@ -1156,7 +1156,7 @@ scenario_073,self_employment_tax,0.0, scenario_073,state_income_tax_before_refundable_credits,0.0, scenario_073,state_refundable_credits,0.0, scenario_073,local_income_tax,0.0, -scenario_073,snap,287.68316650390625, +scenario_073,snap,288.0, scenario_073,ssi,0.0, scenario_073,tanf,0.0, scenario_073,head_wic_eligible,0.0,0.0 @@ -1201,7 +1201,7 @@ scenario_076,federal_income_tax_before_refundable_credits,19611.80078125, scenario_076,federal_refundable_credits,0.0, scenario_076,payroll_tax,12240.0, scenario_076,self_employment_tax,0.0, -scenario_076,state_income_tax_before_refundable_credits,6806.78662109375, +scenario_076,state_income_tax_before_refundable_credits,6818.34423828125, scenario_076,state_refundable_credits,465.0, scenario_076,local_income_tax,0.0, scenario_076,snap,0.0, @@ -1241,11 +1241,11 @@ scenario_077,head_chip_eligible,0.0,0.0 scenario_077,head_medicare_eligible,0.0,0.0 scenario_077,free_school_meals_eligible,0.0,0.0 scenario_077,reduced_price_school_meals_eligible,0.0,0.0 -scenario_078,federal_income_tax_before_refundable_credits,24772.693359375, +scenario_078,federal_income_tax_before_refundable_credits,24164.45703125, scenario_078,federal_refundable_credits,0.0, scenario_078,payroll_tax,14317.322265625, scenario_078,self_employment_tax,0.0, -scenario_078,state_income_tax_before_refundable_credits,6936.33740234375, +scenario_078,state_income_tax_before_refundable_credits,6936.87646484375, scenario_078,state_refundable_credits,0.0, scenario_078,local_income_tax,0.0, scenario_078,snap,0.0, @@ -1264,7 +1264,7 @@ scenario_079,self_employment_tax,0.0, scenario_079,state_income_tax_before_refundable_credits,0.0, scenario_079,state_refundable_credits,50.0, scenario_079,local_income_tax,0.0, -scenario_079,snap,2428.017333984375, +scenario_079,snap,2376.0, scenario_079,ssi,10572.0, scenario_079,tanf,0.0, scenario_079,head_wic_eligible,0.0,0.0 @@ -1314,7 +1314,7 @@ scenario_082,federal_refundable_credits,0.0, scenario_082,payroll_tax,8108.02734375, scenario_082,self_employment_tax,0.0, scenario_082,state_income_tax_before_refundable_credits,5598.55224609375, -scenario_082,state_refundable_credits,650.5, +scenario_082,state_refundable_credits,667.0, scenario_082,local_income_tax,0.0, scenario_082,snap,0.0, scenario_082,ssi,0.0, @@ -1489,7 +1489,7 @@ scenario_093,federal_income_tax_before_refundable_credits,8628.990234375, scenario_093,federal_refundable_credits,0.0, scenario_093,payroll_tax,12469.5, scenario_093,self_employment_tax,0.0, -scenario_093,state_income_tax_before_refundable_credits,3389.65087890625, +scenario_093,state_income_tax_before_refundable_credits,3388.24560546875, scenario_093,state_refundable_credits,0.0, scenario_093,local_income_tax,0.0, scenario_093,snap,0.0, @@ -1642,7 +1642,7 @@ scenario_104,federal_refundable_credits,0.0, scenario_104,payroll_tax,0.0, scenario_104,self_employment_tax,0.0, scenario_104,state_income_tax_before_refundable_credits,0.0, -scenario_104,state_refundable_credits,375.0, +scenario_104,state_refundable_credits,0.0, scenario_104,local_income_tax,0.0, scenario_104,snap,0.0, scenario_104,ssi,0.0, @@ -1676,7 +1676,7 @@ scenario_108,self_employment_tax,0.0, scenario_108,state_income_tax_before_refundable_credits,0.0, scenario_108,state_refundable_credits,271.3916015625, scenario_108,local_income_tax,0.0, -scenario_108,snap,287.68316650390625, +scenario_108,snap,288.0, scenario_108,ssi,0.0, scenario_108,tanf,0.0, scenario_108,head_wic_eligible,0.0,0.0 @@ -1692,7 +1692,7 @@ scenario_109,self_employment_tax,4238.86474609375, scenario_109,state_income_tax_before_refundable_credits,0.0, scenario_109,state_refundable_credits,0.0, scenario_109,local_income_tax,0.0, -scenario_109,snap,8020.55419921875, +scenario_109,snap,7932.0, scenario_109,ssi,0.0, scenario_109,tanf,0.0, scenario_109,head_wic_eligible,0.0,0.0 @@ -1827,11 +1827,11 @@ scenario_116,head_medicare_eligible,0.0,0.0 scenario_116,spouse_medicare_eligible,0.0,0.0 scenario_116,free_school_meals_eligible,0.0,0.0 scenario_116,reduced_price_school_meals_eligible,0.0,0.0 -scenario_117,federal_income_tax_before_refundable_credits,24391.796875, +scenario_117,federal_income_tax_before_refundable_credits,24961.33984375, scenario_117,federal_refundable_credits,0.0, scenario_117,payroll_tax,15044.787109375, scenario_117,self_employment_tax,0.0, -scenario_117,state_income_tax_before_refundable_credits,7984.05419921875, +scenario_117,state_income_tax_before_refundable_credits,7984.53369140625, scenario_117,state_refundable_credits,0.0, scenario_117,local_income_tax,0.0, scenario_117,snap,0.0, @@ -1945,7 +1945,7 @@ scenario_122,federal_income_tax_before_refundable_credits,4746.66162109375, scenario_122,federal_refundable_credits,0.0, scenario_122,payroll_tax,0.0, scenario_122,self_employment_tax,0.0, -scenario_122,state_income_tax_before_refundable_credits,1028.216552734375, +scenario_122,state_income_tax_before_refundable_credits,1025.54150390625, scenario_122,state_refundable_credits,0.0, scenario_122,local_income_tax,0.0, scenario_122,snap,0.0, diff --git a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json index 1cad33ce..49293df7 100644 --- a/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json +++ b/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv.meta.json @@ -6,29 +6,29 @@ "output": "results/local/v1_1/us/reference_outputs.csv", "policyengine_bundles": { "us": { - "bundle_id": "us-4.16.1", - "bundled_certified_by": "policyengine-bundles", - "bundled_compatibility_basis": "bundle_candidate", - "bundled_model_version": "1.723.0", - "bundled_policyengine_version": "4.16.1", - "certified_by": "installed model package; no matching policyengine.py bundle manifest", - "certified_data_artifact_sha256": "f32c2e5e9098bc6540724fdd5debf963af495da4c29b3a7a63fb53c2a4bb5a34", - "certified_data_build_id": "populace-us-2024-5da5a95-20260611", - "compatibility_basis": "installed_model_package_not_policyengine_py_bundle", + "bundle_id": "us-6.1.2", "country_id": "us", - "data_build_fingerprint": null, - "data_build_model_git_sha": null, - "data_build_model_version": "1.723.0", - "data_package": "populace-data", - "data_version": "0.1.0", - "default_dataset": "populace_us_2024", - "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-5da5a95-20260611", - "model_direct_url": null, - "model_matches_policyengine_bundle": false, + "policyengine_version": "6.1.2", + "bundled_policyengine_version": "6.1.2", "model_package": "policyengine-us", - "model_version": "1.755.4", + "model_version": "2.15.17", + "model_direct_url": null, + "bundled_model_version": "2.2.1", "model_version_source": "installed package", - "policyengine_version": "4.16.1" + "model_matches_policyengine_bundle": false, + "data_package": "microcosm-data", + "data_version": "0.1.0", + "default_dataset": "populace_us_2024", + "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "certified_data_build_id": "populace-us-2024-spm-20260915", + "certified_data_artifact_sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "data_build_model_version": "2.2.1", + "data_build_model_git_sha": null, + "data_build_fingerprint": null, + "compatibility_basis": "installed_model_package_not_policyengine_py_bundle", + "bundled_compatibility_basis": "built_with_model_package", + "certified_by": "installed model package; no matching policyengine.py bundle manifest", + "bundled_certified_by": "policyengine.py bundle certification" } }, "private_fraction": 0.0, @@ -59,5 +59,1103 @@ "seed": 42, "split": "public", "split_seed": 1042, - "task": "reference_outputs" -} \ No newline at end of file + "task": "reference_outputs", + "reference_csv_sha256": "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + "revisions": [ + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_snap_hold_fy2026", + "outputs": "snap", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.", + "basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14)", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r13_hold_fy2026_v3.py", + "fix_module_sha256": "7de931064360e7ff68ab4ca72144f6239f79baffa73ffb49b67caa4e26739d70", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_008", + "variable": "snap", + "frozen": 15246.9052734375, + "regenerated": 15108.0 + }, + { + "scenario_id": "scenario_012", + "variable": "snap", + "frozen": 4952.08935546875, + "regenerated": 4884.0 + }, + { + "scenario_id": "scenario_027", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_030", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_038", + "variable": "snap", + "frozen": 7286.9443359375, + "regenerated": 7212.0 + }, + { + "scenario_id": "scenario_043", + "variable": "snap", + "frozen": 3596.039794921875, + "regenerated": 3576.0 + }, + { + "scenario_id": "scenario_045", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 0.0 + }, + { + "scenario_id": "scenario_054", + "variable": "snap", + "frozen": 6125.68896484375, + "regenerated": 6060.0 + }, + { + "scenario_id": "scenario_066", + "variable": "snap", + "frozen": 3596.039794921875, + "regenerated": 3576.0 + }, + { + "scenario_id": "scenario_073", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_079", + "variable": "snap", + "frozen": 2428.017333984375, + "regenerated": 2376.0 + }, + { + "scenario_id": "scenario_108", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_109", + "variable": "snap", + "frozen": 8020.55419921875, + "regenerated": 7932.0 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_ca_hold_2025", + "outputs": "California income tax and credits", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. California's 2026 indexed income tax amounts hold the 2025 amounts FTB published (October 2025 Tax News; 2025 Form 540 and 3514 booklets), the last published before the 2026-07-03 reference freeze; the 2026 factor rests on June 2026 California CPI (BLS 2026-07-14; DIR 2026-08-12). The CalEITC final phase-out breakpoints take their statutory 2025 values ($252/$636), which policyengine-us 1.755.4 had projected ($257/$649).", + "basis": "Cal. R&TC 17041(h), 17052(o); FTB Tax News, October 2025; FTB 2026 Form 540-ES instructions", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_ca_convention.py", + "fix_module_sha256": "937429d39a8695bc70bdf33ee74c3fc67771f782f82b3d288668ad73b9057e8c", + "applied_together_with": [ + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_022", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 11131.3271484375, + "regenerated": 11113.5703125 + }, + { + "scenario_id": "scenario_023", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 6.7984771728515625, + "regenerated": 12.776718139648438 + }, + { + "scenario_id": "scenario_023", + "variable": "state_refundable_credits", + "frozen": 148.3108673095703, + "regenerated": 95.15400695800781 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_irs_sales_tax_2025", + "outputs": "federal income tax of filers who deduct general sales tax", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. The optional state sales tax tables for 2026 are the 2025 tables the IRS published in the 2025 Instructions for Schedule A (posted 2025-12-18), the last published before the 2026-07-03 reference freeze; no 2026 edition had been published by 2026-09-22. policyengine-us 1.755.4 projected 2026 tables from its 2023 edition with chained CPI, which is not the IRS method.", + "basis": "IRS, 2025 Instructions for Schedule A (Form 1040), Optional State Sales Tax Tables", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_irs_sales_tax_convention.py", + "fix_module_sha256": "202f05cad631b28692edc315b61b5f1a2a5604e66d7ae3df72c66e794b8a3895", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_000", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 2883.49365234375, + "regenerated": 2906.45361328125 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_wi_published_2026", + "outputs": "Wisconsin income tax", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Wisconsin's 2026 standard deduction and tax brackets are the amounts DOR published before the 2026-07-03 reference freeze (2026 Form 1-ES instructions, rev. 1-26; 2026 WT-4A worksheet, rev. 11-25), not policyengine-us 1.755.4's CPI projections.", + "basis": "Wisconsin DOR, 2026 Form 1-ES instructions (R. 1-26), pp. 2-3; 2026 WT-4A worksheet (R. 11-25), p. 2", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_wi_convention.py", + "fix_module_sha256": "d66c3786882542eba0e9da02b197faa3b6459b78fae6f1b5c19c1774dd04a250", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_id_hold_2025", + "outputs": "Idaho income tax", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Idaho's 2026 zero-rate taxable income thresholds hold the 2025 amounts ($4,811 single; $9,622 joint and head of household) the Tax Commission published, the last located before the 2026-07-03 reference freeze; no 2026 publication was found by 2026-09-22. The retirement-benefit caps take their statutory 2026 values; they do not affect any benchmark household.", + "basis": "Idaho Code 63-3024(3), 63-3022A", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_id_convention.py", + "fix_module_sha256": "3449518830d4d2c341365241810d631f1cc3b18f26f5e3d67e64f0f1fd8ddd51", + "applied_together_with": [ + "c_ca_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_076", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 6806.78662109375, + "regenerated": 6818.34423828125 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_mn_published_2026", + "outputs": "Minnesota income tax and credits", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Minnesota's 2026 inflation-adjusted amounts are those the Department of Revenue published on 2025-12-01 (announced 2025-12-16), before the 2026-07-03 reference freeze, not policyengine-us 1.755.4's CPI projections; the 2025 renter's credit schedule takes its published values. The marriage credit cap, first located in an August 2026 draft, holds its 2025 amount.", + "basis": "Minnesota Department of Revenue, 2026 inflation-adjusted amounts (2025-12-01), pp. 1-3, 5", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_mn_convention.py", + "fix_module_sha256": "b3d386f6c74af02f0b22869f5f8b6fc25e4b2a645b71762399626391c7cd42b3", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_122", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 1028.216552734375, + "regenerated": 1025.54150390625 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_md_2026", + "outputs": "Maryland income tax; federal tax through the Maryland withholding proxy", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Maryland's 2026 withholding allowance is the $3,400 the Comptroller published in the 2026 Employer Withholding Guide (revised December 2025); the 2026 flat standard deduction holds the published 2025 amounts ($3,350; $6,700), because no 2026 return amount was located before the 2026-07-03 reference freeze (provisional); the child and dependent care credit caps hold their 2025 amounts.", + "basis": "Md. Tax-Gen. 10-217 (Chapter 604 of 2025); Comptroller Tax Alert rev. 2025-12-22; 2026 Employer Withholding Guide", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_md_convention.py", + "fix_module_sha256": "91afaba40009fbae5cc430841d5882ae04c9f025ede425bd99321e52f6c5fb1d", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 1252.9677734375, + "regenerated": 1255.3427734375 + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 24772.693359375, + "regenerated": 24780.61328125 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_mi_published_2026", + "outputs": "Michigan income tax and credits", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Michigan's 2026 personal exemption ($5,900) and retirement limits are those Treasury published in the 2026 Form 446 (revised February 2026), before the 2026-07-03 reference freeze; amounts with no located 2026 publication (disability exemption, senior investment limits, homestead and home heating tables) hold their published 2025 values, which also replace the engine's projected 2025 entries.", + "basis": "Michigan Treasury Form 446 (2026, rev. February 2026); 2025 MI-1040 and MI-1040CR-7 instructions", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_mi_convention.py", + "fix_module_sha256": "15e16cfaeac13805ce3e9f32f91fc427f3b5fb687b31cc19f18c2604a233e974", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_045", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 1288.8375244140625, + "regenerated": 1290.9625244140625 + } + ] + }, + { + "date": "2026-09-22", + "kind": "convention", + "convention": "c_mo_published_2026", + "outputs": "Missouri income tax", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. Missouri's 2026 income tax brackets are those the Department of Revenue published in its 2026 withholding formula (dated 2025-11-21), before the 2026-07-03 reference freeze, not policyengine-us 1.755.4's CPI projections.", + "basis": "Missouri DOR 2026 Withholding Formula (2025-11-21), annual table p. 2", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r19_mo_convention.py", + "fix_module_sha256": "a277c0790d2692f584d63773f62c215c917b8ebf12722a4d01d15f04f71025fc", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_093", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 3389.65087890625, + "regenerated": 3388.24560546875 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r04_capital_gain_distributions", + "outputs": "federal_income_tax_before_refundable_credits, state_income_tax_before_refundable_credits", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.", + "defect": "PolicyEngine leaves capital gain distributions reported without Schedule D out of gross income", + "basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7", + "upstream": "fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r04_capital_gain_distributions.py", + "fix_module_sha256": "ccbbf3dd0b866975347319bfb6778bcfcf1c6415faa4f1149c4059a9cd9ed929", + "measured_against": "frozen", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_042", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 1979.1583251953125, + "regenerated": 2361.96484375 + }, + { + "scenario_id": "scenario_051", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 814.9500122070312, + "regenerated": 820.3499755859375 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r09_ny_rptc_rent_cap", + "outputs": "state_refundable_credits", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. A renter whose adjusted rent averages more than $450 a month gets no real property tax credit.", + "defect": "PolicyEngine applies New York's $450-a-month renter cap to a quarter of rent instead of the rent itself", + "basis": "N.Y. Tax Law 606(e)(7)(D); Form IT-214", + "upstream": "fixed in PolicyEngine/policyengine-us#9301 (issue #9298) and PolicyEngine/policyengine-us#9313 (the 2025 flat credit tables the sandbox module also applies), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r09_ny_rptc_rent_cap_v2.py", + "fix_module_sha256": "ccdc92a415ad516e00d0ca2f4c8710b4aa3ae332e1fc2ec8e55a69781b9b26b9", + "measured_against": "frozen", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_104", + "variable": "state_refundable_credits", + "frozen": 375.0, + "regenerated": 0.0 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r17_caleitc_agi_comparison", + "outputs": "state_refundable_credits", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. When adjusted gross income exceeds the income at which the CalEITC starts to phase out, the credit is looked up again at AGI and the smaller amount applies.", + "defect": "PolicyEngine phases out the CalEITC on earned income only, without the second lookup at adjusted gross income", + "basis": "Cal. R&TC 17052(a), incorporating IRC 32(a)(2)(B); FTB 3514 instructions, CalEITC worksheet", + "upstream": "fixed in PolicyEngine/policyengine-us#9363 (merged 2026-09-01), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r17_caleitc_agi_comparison.py", + "fix_module_sha256": "170b2766c9460bdb53491f5c375864c894b6b5e6d6b4a1d2f797c330879947a5", + "measured_against": "frozen", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_023", + "variable": "state_refundable_credits", + "frozen": 148.3108673095703, + "regenerated": 95.15400695800781 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r26_snap_contribution_rounding", + "outputs": "snap", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.", + "defect": "PolicyEngine subtracts 30% of net income from the maximum allotment without making the allotment a whole-dollar amount", + "basis": "7 CFR 273.10(e)(2)(ii)(A)", + "upstream": "fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r26_snap_contribution_rounding.py", + "fix_module_sha256": "42e01d6bd04141d145c3df7ff0fad876d4a2ff62b87235cf0ee2e4607a9cf238", + "measured_against": "c_snap_hold_fy2026", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_008", + "variable": "snap", + "frozen": 15246.9052734375, + "regenerated": 15108.0 + }, + { + "scenario_id": "scenario_012", + "variable": "snap", + "frozen": 4952.08935546875, + "regenerated": 4884.0 + }, + { + "scenario_id": "scenario_038", + "variable": "snap", + "frozen": 7286.9443359375, + "regenerated": 7212.0 + }, + { + "scenario_id": "scenario_054", + "variable": "snap", + "frozen": 6125.68896484375, + "regenerated": 6060.0 + }, + { + "scenario_id": "scenario_079", + "variable": "snap", + "frozen": 2428.017333984375, + "regenerated": 2376.0 + }, + { + "scenario_id": "scenario_109", + "variable": "snap", + "frozen": 8020.55419921875, + "regenerated": 7932.0 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r28_snap_min_allotment_rounding", + "outputs": "snap", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).", + "defect": "PolicyEngine sets the SNAP minimum allotment at 8% of the one-person maximum without rounding ($23.84 a month for FY2026)", + "basis": "7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)", + "upstream": "fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r28_snap_min_allotment_rounding.py", + "fix_module_sha256": "07e555626ad1636988dbedcc2f75e312ced8ee68cd270d7296fb0c289dba8357", + "measured_against": "c_snap_hold_fy2026", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_027", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_030", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_045", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 0.0 + }, + { + "scenario_id": "scenario_073", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + }, + { + "scenario_id": "scenario_108", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 288.0 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r27_snap_net_income_rounding", + "outputs": "snap", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. SNAP net income is rounded to the nearest dollar, as policyengine-us#9318 does in every state and California's CalFresh rules require; recomputing with cents kept instead moves no scored reference.", + "defect": "PolicyEngine floors SNAP net income in every state, which matches neither procedure 7 CFR 273.10(e)(1)(ii) allows", + "basis": "7 CFR 273.10(e)(1)(ii); California MPP 63-503.311 (CDSS summary)", + "upstream": "fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r27_snap_net_income_rounding.py", + "fix_module_sha256": "81eb155229603203605cf6daf259e002ecac22d4bedadd3591e2b600ab62682d", + "measured_against": "c_snap_hold_fy2026", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_008", + "variable": "snap", + "frozen": 15246.9052734375, + "regenerated": 15108.0 + }, + { + "scenario_id": "scenario_012", + "variable": "snap", + "frozen": 4952.08935546875, + "regenerated": 4884.0 + }, + { + "scenario_id": "scenario_038", + "variable": "snap", + "frozen": 7286.9443359375, + "regenerated": 7212.0 + } + ] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r31_snap_income_limit_rounding", + "outputs": "none", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. The monthly gross and net income standards are rounded up to the next whole dollar.", + "defect": "PolicyEngine compares SNAP income to the unrounded gross and net income standards", + "basis": "7 CFR 273.9(a)(3)", + "upstream": "fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r31_snap_income_limit_rounding.py", + "fix_module_sha256": "9800f9c2275733d825039041de6e3df2b792cf4a72663f0c2e687be61eb4bf35", + "measured_against": "c_snap_hold_fy2026", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r33_snap_child_support_treatment" + ], + "excluded_outputs_untouched": true, + "changed": [] + }, + { + "date": "2026-09-22", + "kind": "upstream_fix", + "root_cause": "r33_snap_child_support_treatment", + "outputs": "snap", + "rule": "An engine defect fixed in policyengine-us after the reference freeze is regenerated with the fix. A state that does not take the option in 7 CFR 273.9(c)(17) counts legally obligated child support paid to nonhousehold members in gross income and deducts it when computing net income (7 CFR 273.9(d)(5)). USDA's 16th and 17th State Options Reports list Michigan among those states (the 15th lists it among the exclusion states), and Michigan's budget (BEM 556) subtracts child support at line 20, after gross income at line 10.", + "defect": "PolicyEngine reads its SNAP child support flag as the option to exclude child support paid from gross income, but the values carry the opposite meaning: for 2026 it excludes the payments from gross income in 37 jurisdictions that USDA's State Options Report lists as deducting them from net income, Michigan among them, and deducts them in 9 of the 14 it lists as excluding them", + "basis": "7 CFR 273.9(c)(17); 7 CFR 273.9(d)(5); USDA SNAP State Options Report, 16th edition (June 2024), Treatment of Child Support Payments, p. 15, and 17th edition, p. 21 (Michigan: child support income expense deduction); Michigan Bridges Eligibility Manual 556 (11-1-2025), line 20", + "upstream": "fixed in PolicyEngine/policyengine-us#9586 (merged 2026-09-24)", + "engine_version": "policyengine-us 1.755.4", + "fix_module": "r33_snap_child_support_treatment.py", + "fix_module_sha256": "1c05732b502a83ce164c443e88341dc9c48040cb8cb4c454bdf42273c97dffad", + "measured_against": "c_snap_hold_fy2026", + "applied_together_with": [ + "c_ca_hold_2025", + "c_id_hold_2025", + "c_irs_sales_tax_2025", + "c_md_2026", + "c_mi_published_2026", + "c_mn_published_2026", + "c_mo_published_2026", + "c_snap_hold_fy2026", + "c_wi_published_2026", + "r04_capital_gain_distributions", + "r09_ny_rptc_rent_cap", + "r17_caleitc_agi_comparison", + "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding", + "r31_snap_income_limit_rounding" + ], + "excluded_outputs_untouched": true, + "changed": [ + { + "scenario_id": "scenario_045", + "variable": "snap", + "frozen": 287.68316650390625, + "regenerated": 0.0 + } + ] + }, + { + "date": "2026-09-29", + "kind": "engine_upgrade", + "root_cause": "engine_upgrade_policyengine_us_2_15_17", + "outputs": "every scored output", + "rule": "A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze. References come from the newest policyengine-us release when PolicyBench begins the reference sweep; at publication PolicyBench checks that the newest release gives the same values. Max's ruling, 2026-09-28: 'we should be using the latest pe for this always!' The conventions that hold pre-freeze law are re-expressed for the release the sweep uses.", + "engine_version": "policyengine-us 2.15.17", + "previous_engine_version": "policyengine-us 1.755.4", + "policyengine_py": "policyengine 6.1.2, provenance only: its certified US bundle is policyengine-us 2.2.1, and it refuses to import next to a newer model", + "fix_modules": [ + { + "module": "latest_c_ca_hold_2025.py", + "sha256": "f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2" + }, + { + "module": "latest_c_irs_sales_tax_2025.py", + "sha256": "7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e" + }, + { + "module": "latest_c_wi_published_2026.py", + "sha256": "4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce" + }, + { + "module": "latest_c_id_hold_2025.py", + "sha256": "39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5" + }, + { + "module": "latest_c_mn_published_2026.py", + "sha256": "02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522" + }, + { + "module": "latest_c_md_2026.py", + "sha256": "8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76" + }, + { + "module": "latest_c_mi_published_2026.py", + "sha256": "787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e" + }, + { + "module": "latest_c_mo_published_2026.py", + "sha256": "0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916" + }, + { + "module": "latest_c_snap_hold_fy2026.py", + "sha256": "3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd" + }, + { + "module": "latest_conventions.py", + "sha256": "4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb" + }, + { + "module": "latest_md_local_output_scope.py", + "sha256": "53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd" + } + ], + "builder": "policybench.scenarios.Scenario.to_pe_household passes stated usual weekly hours (hours_worked_last_week) to weekly_hours_worked_before_lsr as well; upstream policyengine-us#9261 changed that input's default from 40 to 0", + "excluded_outputs_untouched": true, + "excluded_outputs_rechecked": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "kept_value": 106505.8984375, + "value_on_2_15_17": 107833.15625, + "reason": "r02 is unfixed in 2.15.17. gov.irs.ald.deductions (2026) still lists traditional_ira_contributions; the list and the retirement variables are identical to 1.755.4 apart from uprating attributes (44001a4c24). No 219(g) or active-participant code exists, and upstream #8388 scoped 219(g) out with no follow-up. Both spouses list 401(k) deferrals, so both are active participants under 219(g)(5)(A)(i); ira_219g_magi is $543,076, far above $149,000, so the deductible limit is $0 and the engine's $2,163.84 deduction is wrong. The +$1,327.26 move is #9122 alone: salt_refund_income=0 on 2.15.17 gives exactly 106505.8984375. The alternative on 2.15.17 is 108525.578125 with the refund in AGI; 107198.3359375 applies with both readings or under a hold. Both were reproduced by the harness and by my engine-free calculation to $0.01. It also depends on the unlisted 402(g) excess allocation (106,342.59 to 108,096.20 on latest), so it should stay excluded even after an upstream r02 fix unless that allocation is ruled. NIIT is included (r25), as in the board." + }, + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 41051.51171875, + "value_on_2_15_17": 41267.01171875, + "reason": "Both r11 and r02 are still in 2.15.17. The CA itemized, charity and misc variables and parameters, traditional_ira_contributions and gov/irs/ald/deductions.yaml are identical over 06665727d8..79be99f671, and git grep finds no active_participant or 219(g) logic. The stored, unscored value is 2.15.17 + conventions = 41,267.011719; I reproduced it in the harness and by hand. The +215.50 gap to the board splits as +168.36 from c_ca_hold_2025 (the board kept raw v1.1 for this excluded output) and +47.14 from #9122: 23.14 through the line 29 limitation (248.86 x 9.3%) and 24.00 through the exemption credits (13 to 15 phase-out steps x $6 x 2). With #9122 reverted on the conventions, the value is 41,219.867188 = 1.755.4 + r19. The output also depends on three stated-but-unresolved or unlisted facts: whether the SALT refund was taxable (40,975.25 vs 40,920.40 after r11 + r02), who issued the tax-exempt interest (+833.41 if non-California), and how the over-limit 401(k) deferrals are cut (-491.85). The exclusion should therefore also carry an unlisted-input reason, and alternative_value should record both SALT-refund readings rather than a single 'law-correct' 40,975.25." + }, + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 755.7783203125, + "value_on_2_15_17": 761.5570068359375, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 755.7783203125, identical to v11_1755. latest_conventions gives 761.5570068359375 (+5.7787 = 0.053 x (4920.0313 - 4811)). latest_conventions + r07_v2 gives 651.3170166. The subtractions.yaml and main/single.yaml files are identical between 1.755.4 and 2.15.17, and the premium subtraction is still absent, so the r07 defect is still present. The board row is the raw 755.7783 because the c_id_hold_2025 revision has excluded_outputs_untouched: true. By hand: taxable SS 13,784.125, Idaho TI 19,180, and 0.053 x (19,180 - 4,811) = 761.557. With the premium subtraction, 0.053 x (17,100 - 4,811) = 651.317, which matches the exclusion record's alternative_value. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "kept_value": 68056.7109375, + "value_on_2_15_17": 68334.734375, + "reason": "Confirmed. On 2.15.17, sweep_latest.py gives 68334.734375 both raw and with latest_conventions.py. latest_conventions plus r25_niit_excluded gives 68112.09375, exactly the exclusion record's alternative_value, so the two readings still differ by 222.64 (more than the $1 tolerance). The r25 root cause is the benchmark_specs.json output definition ('federal individual income tax after nonrefundable credits and before refundable credits'), which is unchanged. The engine formula (income_tax_before_refundable_credits.py; git diff 06665727d8..79be99f671 is empty; cmp of the installed files is identical) still adds net_investment_income_tax, and net_investment_income_tax.py has the same sha1 in both engines. The move comes only from #9616 (3f029abf5b, 2026-09-27), which fixed the column-shifted sales tax table. The IRS figure it carries, the 2025 TX size-1 $300k+ cell of 1,595, was published in December 2025, before the freeze. No hold module is needed, because raw and conventions agree." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 2439.650146484375, + "value_on_2_15_17": 2505.8701171875, + "reason": "r11 is still present. ca_itemized_deductions_pre_limitation carries the federal charitable_deduction (0.5% floor from 2026: 14,847.67 instead of 15,393.41) and the federal misc_deduction (0 since 2018). R&TC 17076(c) (Stats. 2025 ch. 231, SB 711 Sec. 11, effective 2025-10-01) and the conformity date in 17024.5(a)(1)(Q) (IRC as of 2025-01-01) mean California allows both. The law-correct value is 1,968.81; I reproduced it by harness and by hand. Federal AGI is 109,149.16 and CA AGI is 91,524.41; itemized deductions are 8,829 + 15,393.41 + 6,167.56 = 30,389.97; CA taxable income 61,134.44 gives tax of 2,274.81, less 306 of exemption credits. That equals the Sept 22 alternative. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 2,505.870117 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. I found no unlisted-input dependence: medical stays under the floor under any reading, and the prompt lists no refund and no tax-exempt interest. This is a pure engine defect, so the output could be un-excluded once r11 is fixed upstream. The value uses the rate schedule, as for every CA output (FTB's tax table for taxable income of $100,000 or less could differ by about a dollar)." + }, + { + "scenario_id": "scenario_023", + "variable": "snap", + "kept_value": 461.3397216796875, + "value_on_2_15_17": 408.0, + "reason": "Reproduced on the harness. latest_conventions gives 408 ($34/mo); raw 2.15.17 gives 462. Hand check: gross $2,120.22 (wages $1,453.55 plus 403(b) $666.67); minus 20% earned ($290.71) and the $209 standard deduction gives $1,620.51; minus the $744 cap (binds with or without the SUA) gives $876.51, rounded to $877; 30% is $263.10, rounded up to $264; $298 - $264 = $34. The unlisted input still decides the output under law. Under reading B, federal SSI is $0 (countable $1,340.94 > $994) and CA SSP is $0 (countable income over the $1,335.81 standard). She has no MAGI route (141.19% FPL > 138%). Her Medi-Cal is a disability pathway (engine: SENIOR_OR_DISABLED), and receiving it makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), both text-verified. 2.15.17 does not model that route, so flipping meets_ssi_disability_criteria alone gives 408 (rv_probe conv_B). Record the alternative as 3,576 with the engine's CA heat-and-eat SUA, and 2,832 if no LIHEAP payment is inferred (a SUA of about $206 suffices, not $384). Reword the record: in 2.15.17 SSI criteria no longer confer SNAP status (#9345); the route is Medi-Cal receipt." + }, + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 284.74090576171875, + "value_on_2_15_17": 464.18280029296875, + "reason": "Both defects are still present in policyengine-us 2.15.17, and I confirmed each directly.\n- diff -rq shows wi_capital_gain_loss_subtraction.py and wi_income_tax_before_refundable_credits.py byte-identical to 1.755.4, and git log 06665727d8..79be99f671 shows no commit touching either file.\n- My probe of 2.15.17 plus latest_conventions shows non_sch_d_capital_gains 3,753 in federal AGI (50,580.71; net_capital_gain 4,681) but wi_capital_gain_loss_subtraction 0.\n- The probe also shows wi_income_tax 0 (the min with the line-16 exclusion path, since wi_retirement_income_exclusion_amount is 24,000 and wi_retirement_income_exclusion_tax is 0) while wi_income_tax_before_refundable_credits is 464.1828.\nHarness runs on 042/091:\n- latest_conventions: 464.1828.\n- Plus r32: 408.698425.\n- Plus r32 and the r06 elected-path port: 0.0.\nAn engine-free recompute on the 2026 1-ES parameters reproduces 464.1827 (standard path, no r32), 408.6984 (with r32) and 0.0 (elected line 16). 71.05(6)(b)54m lets a filer aged 67 or older subtract up to $24,000 of qualified-plan and IRA distributions, and 54m.d forfeits the credits. On the elected path, WI income 8,608.81 is below the 13,960 standard deduction plus 950 exemptions, so tax is 0, and 0 is below the standard path's net 408.70, so the filer elects it. The 0.0 holds under every alternative reading I tried:\n- only the 19,200 IRA counts as line 16 (WI income 14,534.71 or 13,408.81, both below 14,910);\n- the farm loss is double-counted;\n- qualified_bdc_income is added.\nr06 alone suffices; r32 does not change the alternative. #9122 does not apply (salt_refund_income is 0). If the record is refreshed, engine value 464.1828 and alternative 0.0." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 2435.2783203125, + "value_on_2_15_17": 2441.05712890625, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 2435.2783203125, identical to v11_1755. latest_conventions gives 2441.05712890625. latest_conventions + r07_v2 gives 2176.05712890625 (-265.00 = 0.053 x 5,000). id_subtractions is 0 and the r07 defect is still present. By hand: 0.053 x (50,868.68 - 4,811) = 2441.057, and 0.053 x (45,868.68 - 4,811) = 2176.057. The alternative also assumes the $5,000 is not paid through a pre-tax salary reduction. The prompt leaves that unstated (the household has wages and ESI), which gives an additional unlisted-input reason to keep this output excluded. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "kept_value": 1140.0, + "value_on_2_15_17": 95.0, + "reason": "Two inputs are still unlisted: hours, and whether the mortgage (and the property tax) is on the occupied home. At 0 hours the value also depends on the county and on 3-in-36 history. Refresh the record: frozen_value 95, alternative_value 1,908, engine_version 2.15.17, flipped alternative_reading. The 95 is January 2026 only, from the FNS-approved 20-county waiver (letter 2024-12-16, through 2026-01-31; NJ DFD confirms) applied to the engine's Atlantic County fallback. The amount is the NJ state minimum $95 (law signed 2023-02-08; federal formula $54). The alternative of 40 hours with the mortgage counted gives shelter capped at $744, net $461, contribution $139, so 12 x 159 = 1,908. I reproduced it with my own composition (latest_alt_snap_mortgage_residence reform plus the latest_alt_unlisted_hours_40 patch) and by hand. The one-input alternatives are 1,140 (40 hours only) and 159 (mortgage only)." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "kept_value": 2669.217041015625, + "value_on_2_15_17": 2628.0, + "reason": "Reproduced. latest_conventions gives 2,628 ($219/mo); raw gives 2,682. Hand and engine agree. Head SSI is $994 - $391.42 = $602.58/mo; gross is $1,472.42; minus $173.33 (20% of earned) and $209 gives $1,090.08, which rounds to $1,090; 30% is $327; $546 - $327 = $219. Under reading B (rv_probe conv_B, and with the r30 port and the title XIX route), both spouses get SSI at the couple rate: $1,491 - $392.42 = $1,098.58/mo in total ($6,591.50 per person per year). Net is $1,586, 30% is $476, $546 - $476 = $70, and 840 per year, so the recorded alternative of 840 stands on 2.15.17 plus the hold. LA always_standard is false, so there is no SUA and the r30 port cannot move it. The dependence runs through actual SSI receipt, an unlisted-input ambiguity that no engine version resolves." + }, + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 4605.9970703125, + "value_on_2_15_17": 4598.47607421875, + "reason": "Root cause r01 is still present in 2.15.17. traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, above_the_line_deductions.py and gov/irs/ald/deductions.yaml have identical blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. The dependent's $18.03 IRA contribution (stated in the prompt; $0 wages) is still in the joint return's ALD (19,926.22 = 19,800 loss ALD + 108.19 + 18.03). Under 26 U.S.C. 219(a), (b)(1)(B) and (c) (LII, re-fetched this session) it is not deductible there. The review harness gives 2.15.17 + latest_conventions = 4,598.476074, which equals 1.755.4 + r19_wi_convention (re-run this session). Adding r01_v2 gives 4,599.621582, and also reproduces the 064 federal (4,441.455078) and 003 federal (22,400.654297) alternative values exactly; the only other output it moves is scenario_085 federal, by +0.089. Under the board's formula convention the defect moves this output by $1.15, beyond tolerance, so it stays excluded. Caveat: Wis. Stat. 71.05(22)(dp)2 makes DOR's $500-bracket table determinative. On that reading r01's effect is $0.96, and 4,598.476 is $0.64 from a table-law value of about 4,597.83, but the 2026 table was not published before the freeze. Exclusion is the safe call under either reading. Update frozen_value to 4598.476074 and engine_version to 2.15.17. Keep alternative_value at 4599.621582, and note in the exclusion record that it is formula-convention based." + }, + { + "scenario_id": "scenario_080", + "variable": "snap", + "kept_value": 3596.039794921875, + "value_on_2_15_17": 3576.0, + "reason": "The r30 defect is unfixed in 2.15.17. snap_state_using_standard_utility_allowance.py is byte-identical to 1.755.4 and returns p.always_standard[state]. always_standard.yaml's only change since 06665727d8 is metadata (d7fc17abfe), and PA stays true from 2015-10-01. snap_utility_allowance_type still grants the SUA on has_heating_cooling | always_sua. The only upstream fix in progress (draft PR #9610) covers California only. My trace on 2.15.17 + latest_conventions shows always_sua=True, has_heating_cooling_expense=False, is_usda_disabled=False and has_snap_elderly_disabled_member=False. The $857 SUA is capped at $744, net income is 0, and the benefit is $298 x 12 = 3,576. With r30 added it is $270 x 12 = 3,240 (net $92, contribution $28). Under 7 U.S.C. 2014(e)(6)(C)(iv)(I) as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04, before the freeze), the engine's own reading (no elderly or disabled member) should give 3,240, so 3,576 is defect-driven under that reading. With is_usda_disabled forced True, 2.15.17 gives 3,576 with or without r30, and the household stays eligible. The prompt supports both readings. It says 'is disabled' but gives no 2012(j) receipt route. Its $3,600 'financial assistance' is unexplained and could be disability-based general assistance. Its $2,000 'spm unit energy subsidy' could be LIHEAP or a state-law payment. It states no certification timing, which matters under the FNS recertification transition. Model answers split the same way: 15 near the SUA-conferred value (3,492-3,576) against 3,240, 3,228 and 2,964-2,988 on other readings. The output is therefore both defect-affected and indeterminate on the stated facts, and stays excluded. The -$20.04 move against the board (3,596.04 to 3,576) comes from the FY2026 SNAP hold convention: the FY2027 COLA memo is dated 2026-08-21, after the freeze. It does not come from r30. Scoring is unaffected." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "kept_value": 9563.052734375, + "value_on_2_15_17": 9577.90234375, + "reason": "r02 is unfixed in 2.15.17 (same evidence as 005). The HoH head lists a $181 traditional 401(k) deferral with $100,195 of wages, so is an active participant; ira_219g_magi is $117,661, above $91,000, so the limit is $0 but the engine deducts $8.48. The +$14.85 move is #9122 alone (67.50 at 22%); refund=0 on 2.15.17 gives exactly 9563.052734375. The alternative on 2.15.17 is 9579.767578125 with the refund in AGI, or 9564.9169921875 with both readings or under a hold. My engine-free calc reproduces these (standard deduction $24,150 + 170(p) $700; CTC $2,200; CDCC 20% of $3,000). No other unlisted dependence was found. The only non-excluded output r02 moves is 082 state, by $0.59, under the $1 tolerance." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 843.6614990234375, + "value_on_2_15_17": 895.8131103515625, + "reason": "The r32 defect is still present in 2.15.17. The subtraction file is byte-identical to 1.755.4 and no commit in the range touches it. My probe on 2.15.17 plus conventions shows non_sch_d_capital_gains 1,170 in AGI 36,182.56 but wi_capital_gain_loss_subtraction 0.\nThe harness gives 895.81311 with latest_conventions and 878.515747 with r32 added; the engine-free recompute gives 895.8131 and 878.5158. 71.05(6)(b)9 subtracts 30% of long-term gain, IRC 852(b)(3)(B) treats capital gain dividends as long-term, and 2025 SB line 5 applies the 30% exclusion to distributions reported without Schedule WD.\nThere is a second, independent reason to keep this excluded. 2.15.17 does not model the WI medical care insurance subtraction: 71.05(6)(b)42, which covers an employee whose employer pays part of the cost, and 2025 SB line 6, Worksheet 2. The prompt lists $1,800 of other health insurance premiums (repeated as 'health insurance premiums excluding Medicare Part B') without saying whether they were paid pre-tax. The law-correct value is:\n- 878.52 if the premiums were pre-tax;\n- 789.81 if the $1,800 was paid after tax, the reading the prompt's unlisted-boolean-false rule and the engine's own AGI point to;\n- 701.11 if a reader sums both lines.\nThe exclusion record should list root causes r32_wi_capital_gain_distributions plus this unlisted pre-tax-premium ambiguity, and it should label 878.515747 as the r32-only correction, not the unique law-correct value." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "kept_value": 4493.7431640625, + "value_on_2_15_17": 4640.77734375, + "reason": "r11 and r02 are still present. The engine's federal floor removes 797.93 of charity (-63.83). Both spouses defer into a 401(k), which makes them active participants, and MAGI of 159,729.30 exceeds the $149,000 top of the 2026 joint range (IRS Notice 2025-67, posted 2025-11-13), so the $144.26 IRA deduction is fully phased out (+11.54 after r11). Together these give 4,588.48, the Sept 22 alternative. I confirm the new educator-expense defect: FTB 2025 Schedule CA line 11 says 'California law does not conform to federal law regarding educator expenses', but 2.15.17's ca_additions adds only ca_hsa_addition. Adding back $337.50 x 8% = +27.00 gives a fully law-correct 4,615.48 (harness and hand calculation agree). The exclusion reason should cite this third defect. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 4,640.777344 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. Possible extra reading: if both premium lines are summed, the output moves -241.62." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "kept_value": 8625.8896484375, + "value_on_2_15_17": 8556.0, + "reason": "Reproduced. latest_conventions gives 8,556 ($713/mo); raw gives 8,637. Pre-shelter net is $690.73 (earned $492.89 plus the engine's MT TANF of $505.41/mo, minus $98.58 and $209). With the $799 SUA (FNS FY2026 SUA table: MT HCSUA $799), net is $237, 30% rounds up to $72, and $785 - $72 = $713. r30 is confirmed still present in 2.15.17. The engine grants the always_standard SUA with has_heating_cooling_expense = 0 and no elderly-or-disabled member. My r30 port (rv_fix_r30.py, full sweep rv_r30_full.csv) moves only 080 and 100, giving 100 = 6,924: net $691, $208, $785 - $208 = $577. Keep the output excluded under both bases. Add r30_snap_heat_and_eat_sua with the stated-facts corrected value 6,924. Do not drop the unlisted-input basis. Under reading B, the working head qualifies for Montana MWD (ABD 201-6; CMA 001 $15,000 resource limit), and 42 CFR 435.404 lets her select it. Receipt makes her elderly or disabled under 7 CFR 271.2(11), giving 8,844 with the engine's SUA (the existing alternative) or 7,116 without. The mechanism text should change from 'SSI criteria' to 'MWD (title XIX) receipt', which 2.15.17 does not model." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "kept_value": 287.68316650390625, + "value_on_2_15_17": 0.0, + "reason": "Hours are unlisted, and at 0 hours the law also depends on unlisted 3-in-36 history, so no engine version resolves this. Refresh the record: frozen_value 0 (the 2.15.17 plus conventions reference), alternative_value 288, engine_version 2.15.17, and flip the alternative_reading text. At 20 or more hours, TX BBCE (165%; $5,000 assets with one vehicle up to $22,000 excluded) is met, and the net income of about $1,269 (engine) or $1,429 (farm rent counted) leaves only the $24 one-person minimum: 12 x 24 = 288. The text should say the 0 comes from the engine's no-history shortcut (the law gives up to $72), and that the stated wages imply at most 15.7 hours a week at the federal minimum wage." + }, + { + "scenario_id": "scenario_118", + "variable": "snap", + "kept_value": 2903.9404296875, + "value_on_2_15_17": 2868.0, + "reason": "Keep excluded, now for three independent reasons.\n1. The mortgage-residence ambiguity (r15_snap_mortgage_interest), which no engine version resolves. 2.15.17 still counts only mortgage_payments, not mortgage interest, as a SNAP shelter cost.\n2. A second unlisted input, the county. The prompt gives only NY, and New York's FY2026 HCSUA is $877, $988 or $1,062 by region. That gives 2,868, 3,276 or 3,540 on 2.15.17 + conventions.\n3. An unfixed 2.15.17 defect: the NY State Supplement Program ($87 a month for an individual living alone in 2026, per the OTDA chart revised 2025-10-27; pre-freeze law) is missing from SNAP income. With it, the value would be 2,400, 2,796 or 3,072 by region.\nRecord 2,868 as the regenerated engine value, replacing the stale raw-1.755.4 2,903.94. Label it as 2.15.17 + latest_conventions under the engine's default county (Albany, Rest of State SUA) without NY SSP, not as the law-correct stated-facts value. Add the NY SSP omission as an unfixed-defect root cause and the county/SUA region as a second unlisted input. The alternative value stays 3,576 (298 x 12). It is the same across all three regions, with or without SSP, and under every 7 CFR 273.10(e) rounding option. Scoring is unaffected." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "kept_value": 40021.81640625, + "value_on_2_15_17": 40416.66796875, + "reason": "r02 is unfixed in 2.15.17. The single head (76) lists a $617 traditional 401(k) deferral with $165,597 of wages, so is an active participant; ira_219g_magi is $234,414, above $91,000, so the limit is $0 but the engine deducts $28.85. The +$394.85 move is #9122 alone. Taxable income rises $1,645.22, because the engine's ct_withheld_income_tax proxy (CT single rates on federal AGI less $15,000) raises SALT by $113.77 and the 0.5% charitable floor rises $8.75; CT's own tax is unchanged at 11917.18 because CT subtracts salt_refund_income. Refund=0 gives exactly 40021.81640625. The alternative on 2.15.17 is 40423.17578125 with the refund in AGI, or 40028.3203125 with both readings or under a hold. Beyond r02 the output also depends on the engine's missing 469 limit (+24.75) and the unlisted mortgage origination date (-224.64 if grandfathered), so it should not be un-excluded on an r02 fix alone." + } + ], + "changed": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "frozen": 5342.39990234375, + "previous": 5342.39990234375, + "regenerated": 5842.39990234375, + "cause": "nj_ctc_fy2027_budget", + "basis": "New Jersey child tax credit schedule for 2026-2028 (P.L.2026, c.26, approved June 30, 2026), encoded upstream in policyengine-us#8971 (0319635b6e)." + }, + { + "scenario_id": "scenario_013", + "variable": "snap", + "frozen": 0.0, + "previous": 0.0, + "regenerated": 240.0, + "cause": "az_snap_bbce_200", + "basis": "Arizona raised its expanded categorical eligibility gross limit from 185% to 200% of poverty from benefit month 03/2026 (DES CNAP \"What's Changed on 03/23/2026\"); encoded upstream after 1.755.4." + }, + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "frozen": 1.0, + "previous": 1.0, + "regenerated": 0.0, + "cause": "school_meals_child_support_income", + "basis": "Child support received counts as household income for school meals (7 CFR 245.6(a)(5)(ii)); policyengine-us added it to the school-meal income sources after 1.755.4." + }, + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 3788.988037109375, + "previous": 3788.988037109375, + "regenerated": 3818.148193359375, + "cause": "excluded_reference_depends_on_unlisted_input", + "basis": "Newly excluded from scoring on 2026-09-29 (reference_exclusions.json): whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)" + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 24772.693359375, + "previous": 24780.61328125, + "regenerated": 24164.45703125, + "cause": "excluded_reference_depends_on_unlisted_input", + "basis": "Newly excluded from scoring on 2026-09-29 (reference_exclusions.json): whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)" + }, + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "frozen": 650.5, + "previous": 650.5, + "regenerated": 667.0, + "cause": "ny_ctc_phaseout_rounding_9425", + "basis": "New York Empire State child credit phase-out rounding corrected upstream (policyengine-us#9425)." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "frozen": 24391.796875, + "previous": 24391.796875, + "regenerated": 24961.33984375, + "cause": "excluded_reference_depends_on_unlisted_input", + "basis": "Newly excluded from scoring on 2026-09-29 (reference_exclusions.json): whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)" + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 6936.33740234375, + "previous": 6936.33740234375, + "regenerated": 6936.87646484375, + "cause": "engine_upgrade_within_1", + "basis": "Moves by $1 or less on policyengine-us 2.15.17; no score changes at the exact-match tolerance." + }, + { + "scenario_id": "scenario_117", + "variable": "state_income_tax_before_refundable_credits", + "frozen": 7984.05419921875, + "previous": 7984.05419921875, + "regenerated": 7984.53369140625, + "cause": "engine_upgrade_within_1", + "basis": "Moves by $1 or less on policyengine-us 2.15.17; no score changes at the exact-match tolerance." + } + ] + } + ], + "regenerated_at_utc": "2026-09-29T15:04:45.724738+00:00" +} diff --git a/paper/snapshot/20260501/us_impact_summary_by_model.csv b/paper/snapshot/20260501/us_impact_summary_by_model.csv index 5259a8ca..d34b2b5b 100644 --- a/paper/snapshot/20260501/us_impact_summary_by_model.csv +++ b/paper/snapshot/20260501/us_impact_summary_by_model.csv @@ -1,40 +1,47 @@ model,mean_impact_score,mean_household_score,mean_household_coverage,households,total_variables,parsed_variables,floor_share -gpt-6-astra,0.9496437469868486,0.9715149854884928,1.0,100,1973,1973,0.3 -claude-fable-5.1,0.938558478818549,0.9723387729440702,1.0,100,1973,1973,0.3 -gpt-5.6-sol,0.9367561386850141,0.9696920448728172,1.0,100,1973,1973,0.3 -inkling,0.9364586213430499,0.962243744202623,1.0,100,1973,1973,0.3 -gpt-5.5,0.9192437189687681,0.9593886640786914,1.0,100,1973,1973,0.3 -kimi-k3,0.9146618878255316,0.9406982797753375,0.97,100,1973,1910,0.3 -gpt-5.6-terra,0.9146364805309777,0.9569218477579463,1.0,100,1973,1973,0.3 -gemini-3.8-flash,0.9092371191481496,0.9652723307891868,1.0,100,1973,1973,0.3 -ox-alpha,0.9088756571628427,0.954790817286395,1.0,100,1973,1973,0.3 -grok-4.6,0.9067629363079034,0.9628776223415428,1.0,100,1973,1973,0.3 -gemini-3.6-flash,0.9065446578039231,0.9628270206868003,1.0,100,1973,1973,0.3 -claude-opus-4.7,0.8938486818299418,0.9283232013207957,1.0,100,1973,1973,0.3 -gemini-3-flash-preview,0.8938155904300443,0.9522189245402665,1.0,100,1973,1973,0.3 -claude-fable-5,0.8926079230715791,0.9466651773258209,1.0,100,1973,1973,0.3 -claude-sonnet-4.6,0.8920563224013588,0.9371230932434323,1.0,100,1973,1973,0.3 -gpt-5.6-luna,0.891518703158717,0.9499694816710595,1.0,100,1973,1973,0.3 -deepseek-v4-flash-0731,0.891464792542364,0.9449788777185419,1.0,100,1973,1973,0.3 -deepseek-v4-pro-0813,0.8912300678385735,0.9563902384151355,1.0,100,1973,1973,0.3 -grok-4.5,0.8870289536651306,0.9531819926162508,1.0,100,1973,1973,0.3 -gemini-3.7-flash,0.8817914276293224,0.9544800401080973,1.0,100,1973,1973,0.3 -gemini-3.1-pro-preview,0.8810846627992194,0.9533512172611641,1.0,100,1973,1973,0.3 -gemini-3.5-flash,0.8776770775707564,0.9491763139769515,1.0,100,1973,1973,0.3 -claude-opus-5,0.8750821644772023,0.947890502076229,1.0,100,1973,1973,0.3 -glm-5.3,0.8704894715328001,0.9184145585473523,0.97,100,1973,1896,0.3 -grok-build-0.1,0.8702102624393789,0.937939104852214,1.0,100,1973,1973,0.3 -deepseek-v4-pro,0.8690474475624503,0.9439243184233473,1.0,100,1973,1973,0.3 -claude-opus-4.8,0.8551010884409126,0.9099994486195605,1.0,100,1973,1973,0.3 -gemini-3.1-flash-lite-preview,0.8332526192159111,0.9279862856738541,1.0,100,1973,1973,0.3 -claude-sonnet-5,0.8323133044741627,0.905770284778892,1.0,100,1973,1973,0.3 -grok-4.3,0.8195264993290182,0.9316608219320247,1.0,100,1973,1973,0.3 -qwen-3.7-max,0.8187989458142549,0.9139856473453446,1.0,100,1973,1973,0.3 -glm-5.2,0.8071387853647619,0.8863723652540183,0.9419665696145959,100,1973,1834,0.3 -qwen3.8-max,0.7864757794964761,0.8943579441272115,1.0,100,1973,1973,0.3 -gemini-3.5-flash-lite,0.7862088975121634,0.9286285233712686,1.0,100,1973,1973,0.3 -claude-haiku-4.5,0.7744794545746602,0.8978781106226874,1.0,100,1973,1973,0.3 -gpt-5.4-mini,0.7141033302649024,0.888519622370442,1.0,100,1973,1973,0.3 -minimax-m3,0.7037588163656016,0.8982717732275121,1.0,100,1973,1973,0.3 -gpt-5.4-nano,0.6509176804344221,0.8779672475285321,1.0,100,1973,1973,0.3 -kimi-k2.6,0.5624316010699739,0.7623994474586452,0.783860742165048,100,1973,1551,0.3 +gpt-6-astra,0.9674395301873611,0.978257782717636,1.0,100,1928,1928,0.3 +gpt-6-sol,0.9662824005912722,0.9823887231923742,1.0,100,1928,1928,0.3 +gpt-6-luna,0.9658376159373622,0.9773386229254812,1.0,100,1928,1928,0.3 +claude-opus-5.5,0.9608671065690544,0.9823954326352231,1.0,100,1928,1928,0.3 +gpt-6.1-sol,0.9586945574651387,0.9752733926090379,1.0,100,1928,1928,0.3 +gpt-5.6-sol,0.9581475169972634,0.9762335155147531,1.0,100,1928,1928,0.3 +claude-sonnet-5.5,0.9580535920873353,0.9730704556439665,1.0,100,1928,1928,0.3 +claude-fable-5.1,0.9555813124174095,0.9779033840407823,1.0,100,1928,1928,0.3 +grok-4.7,0.9531030169930793,0.9758195364701391,1.0,100,1928,1928,0.3 +inkling,0.95278940419871,0.9708083462322512,1.0,100,1928,1928,0.3 +gpt-5.5,0.9444535880318884,0.9659348038436416,1.0,100,1928,1928,0.3 +deepseek-v4.1-flash,0.9437394502936982,0.9674174501414599,1.0,100,1928,1928,0.3 +gpt-5.6-terra,0.9423390142779876,0.9651008738219248,1.0,100,1928,1928,0.3 +gemini-3.8-flash,0.9395394036327269,0.9753939876195191,1.0,100,1928,1928,0.3 +grok-4.6,0.935512292170297,0.9706379118326355,1.0,100,1928,1928,0.3 +gemini-3.6-flash,0.9340484471155557,0.9707033409108929,1.0,100,1928,1928,0.3 +ox-alpha,0.9329622083651122,0.9639201283365196,1.0,100,1928,1928,0.3 +kimi-k3,0.9312389646048008,0.9468695137686975,0.97,100,1928,1870,0.3 +grok-4.5,0.9248152464910463,0.9634117776809533,1.0,100,1928,1928,0.3 +deepseek-v4-pro-0813,0.9245585396764721,0.966361343016232,1.0,100,1928,1928,0.3 +gpt-5.6-luna,0.9221866400950565,0.9594239447421039,1.0,100,1928,1928,0.3 +deepseek-v4-flash-0731,0.9187089252487097,0.9546492938053259,1.0,100,1928,1928,0.3 +gemini-3-flash-preview,0.9170555704087302,0.9608133948736268,1.0,100,1928,1928,0.3 +claude-sonnet-4.6,0.9147170663321406,0.9450085166651927,1.0,100,1928,1928,0.3 +claude-opus-4.7,0.9138626894667351,0.9333394692464237,1.0,100,1928,1928,0.3 +claude-fable-5,0.9118097137275345,0.9538040486732181,1.0,100,1928,1928,0.3 +claude-opus-5,0.9093403745672698,0.9565726626978903,1.0,100,1928,1928,0.3 +gemini-3.7-flash,0.9068894551658637,0.9637007836479945,1.0,100,1928,1928,0.3 +grok-build-0.1,0.9035771268657874,0.9490575162370398,1.0,100,1928,1928,0.3 +gemini-3.1-pro-preview,0.8996459348257337,0.9613824023677643,1.0,100,1928,1928,0.3 +deepseek-v4-pro,0.8939788090152366,0.9531021316162993,1.0,100,1928,1928,0.3 +gemini-3.5-flash,0.892262363142274,0.9566317304583035,1.0,100,1928,1928,0.3 +glm-5.3,0.8870442747598982,0.9269952358769811,0.97,100,1928,1857,0.3 +claude-opus-4.8,0.8805442215879067,0.9191835487991205,1.0,100,1928,1928,0.3 +claude-sonnet-5,0.8639047411962291,0.9156874253444743,1.0,100,1928,1928,0.3 +gemini-3.1-flash-lite-preview,0.8621157646423028,0.9399938349552724,1.0,100,1928,1928,0.3 +qwen-3.7-max,0.8542922488394893,0.9254172874391244,1.0,100,1928,1928,0.3 +grok-4.3,0.8535754470989819,0.94337639429603,1.0,100,1928,1928,0.3 +glm-5.2,0.8468008249398641,0.8985638347071989,0.9430477571052661,100,1928,1795,0.3 +qwen3.8-max,0.8163971792504239,0.906206350579505,1.0,100,1928,1928,0.3 +gemini-3.5-flash-lite,0.8140072027414716,0.9406660914684142,1.0,100,1928,1928,0.3 +claude-haiku-4.5,0.7958926936620644,0.9088133283583146,1.0,100,1928,1928,0.3 +gpt-5.4-mini,0.7565923451264394,0.9024786162568723,1.0,100,1928,1928,0.3 +minimax-m3,0.7364968916060559,0.915496790900971,1.0,100,1928,1928,0.3 +gpt-5.4-nano,0.6872355186399137,0.8911936267908885,1.0,100,1928,1928,0.3 +kimi-k2.6,0.5816850782720341,0.775441245816979,0.7944015327506267,100,1928,1538,0.3 diff --git a/paper/snapshot/20260501/us_reference_outputs.csv b/paper/snapshot/20260501/us_reference_outputs.csv index 8e7b68ee..67d4e5ae 100644 --- a/paper/snapshot/20260501/us_reference_outputs.csv +++ b/paper/snapshot/20260501/us_reference_outputs.csv @@ -1,5 +1,5 @@ scenario_id,variable,value,impact_weight -scenario_000,federal_income_tax_before_refundable_credits,2883.49365234375, +scenario_000,federal_income_tax_before_refundable_credits,2906.45361328125, scenario_000,federal_refundable_credits,0.0, scenario_000,payroll_tax,0.0, scenario_000,self_employment_tax,0.0, @@ -136,9 +136,9 @@ scenario_008,federal_refundable_credits,12433.611328125, scenario_008,payroll_tax,2162.760009765625, scenario_008,self_employment_tax,565.1820068359375, scenario_008,state_income_tax_before_refundable_credits,264.80999755859375, -scenario_008,state_refundable_credits,5342.39990234375, +scenario_008,state_refundable_credits,5842.39990234375, scenario_008,local_income_tax,0.0, -scenario_008,snap,15246.9052734375, +scenario_008,snap,15108.0, scenario_008,ssi,0.0, scenario_008,tanf,0.0, scenario_008,head_wic_eligible,0.0,0.0 @@ -214,7 +214,7 @@ scenario_012,self_employment_tax,0.0, scenario_012,state_income_tax_before_refundable_credits,0.0, scenario_012,state_refundable_credits,0.0, scenario_012,local_income_tax,0.0, -scenario_012,snap,4952.08935546875, +scenario_012,snap,4884.0, scenario_012,ssi,0.0, scenario_012,tanf,0.0, scenario_012,head_wic_eligible,0.0,0.0 @@ -240,7 +240,7 @@ scenario_013,self_employment_tax,0.0, scenario_013,state_income_tax_before_refundable_credits,0.0, scenario_013,state_refundable_credits,25.0, scenario_013,local_income_tax,0.0, -scenario_013,snap,0.0, +scenario_013,snap,240.0, scenario_013,ssi,0.0, scenario_013,tanf,0.0, scenario_013,head_wic_eligible,0.0,0.0 @@ -349,7 +349,7 @@ scenario_021,head_chip_eligible,0.0,0.0 scenario_021,head_medicare_eligible,1.0,5285.2001953125 scenario_021,free_school_meals_eligible,0.0,0.0 scenario_021,reduced_price_school_meals_eligible,0.0,0.0 -scenario_022,federal_income_tax_before_refundable_credits,11131.3271484375, +scenario_022,federal_income_tax_before_refundable_credits,11113.5703125, scenario_022,federal_refundable_credits,0.0, scenario_022,payroll_tax,4710.83251953125, scenario_022,self_employment_tax,0.0, @@ -369,8 +369,8 @@ scenario_023,federal_income_tax_before_refundable_credits,643.43359375, scenario_023,federal_refundable_credits,0.0, scenario_023,payroll_tax,1561.1168212890625, scenario_023,self_employment_tax,0.0, -scenario_023,state_income_tax_before_refundable_credits,6.7984771728515625, -scenario_023,state_refundable_credits,148.3108673095703, +scenario_023,state_income_tax_before_refundable_credits,12.776718139648438, +scenario_023,state_refundable_credits,95.15400695800781, scenario_023,local_income_tax,0.0, scenario_023,snap,461.3397216796875, scenario_023,ssi,0.0, @@ -446,7 +446,7 @@ scenario_027,self_employment_tax,0.0, scenario_027,state_income_tax_before_refundable_credits,0.0, scenario_027,state_refundable_credits,0.0, scenario_027,local_income_tax,0.0, -scenario_027,snap,287.68316650390625, +scenario_027,snap,288.0, scenario_027,ssi,0.0, scenario_027,tanf,0.0, scenario_027,head_wic_eligible,0.0,0.0 @@ -492,7 +492,7 @@ scenario_028,child1_early_head_start_eligible,0.0,0.0 scenario_028,child2_early_head_start_eligible,0.0,0.0 scenario_028,child3_early_head_start_eligible,0.0,0.0 scenario_028,free_school_meals_eligible,0.0,0.0 -scenario_028,reduced_price_school_meals_eligible,1.0,3009.81884765625 +scenario_028,reduced_price_school_meals_eligible,0.0,3009.81884765625 scenario_029,federal_income_tax_before_refundable_credits,0.0, scenario_029,federal_refundable_credits,0.0, scenario_029,payroll_tax,0.0, @@ -516,7 +516,7 @@ scenario_030,self_employment_tax,0.0, scenario_030,state_income_tax_before_refundable_credits,0.0, scenario_030,state_refundable_credits,0.0, scenario_030,local_income_tax,0.0, -scenario_030,snap,287.68316650390625, +scenario_030,snap,288.0, scenario_030,ssi,0.0, scenario_030,tanf,0.0, scenario_030,head_wic_eligible,1.0,1026.747314453125 @@ -567,7 +567,7 @@ scenario_032,child1_head_start_eligible,0.0,0.0 scenario_032,child1_early_head_start_eligible,0.0,0.0 scenario_032,free_school_meals_eligible,1.0,1130.9622802734375 scenario_032,reduced_price_school_meals_eligible,0.0,0.0 -scenario_033,federal_income_tax_before_refundable_credits,3788.988037109375, +scenario_033,federal_income_tax_before_refundable_credits,3818.148193359375, scenario_033,federal_refundable_credits,0.0, scenario_033,payroll_tax,6043.5, scenario_033,self_employment_tax,0.0, @@ -626,7 +626,7 @@ scenario_038,self_employment_tax,2193.612548828125, scenario_038,state_income_tax_before_refundable_credits,0.0, scenario_038,state_refundable_credits,365.8000183105469, scenario_038,local_income_tax,0.0, -scenario_038,snap,7286.9443359375, +scenario_038,snap,7212.0, scenario_038,ssi,0.0, scenario_038,tanf,0.0, scenario_038,head_wic_eligible,0.0,0.0 @@ -687,7 +687,7 @@ scenario_040,head_medicare_eligible,1.0,5285.2001953125 scenario_040,spouse_medicare_eligible,1.0,5285.2001953125 scenario_040,free_school_meals_eligible,0.0,0.0 scenario_040,reduced_price_school_meals_eligible,0.0,0.0 -scenario_042,federal_income_tax_before_refundable_credits,1979.1583251953125, +scenario_042,federal_income_tax_before_refundable_credits,2361.96484375, scenario_042,federal_refundable_credits,0.0, scenario_042,payroll_tax,0.0, scenario_042,self_employment_tax,0.0, @@ -710,7 +710,7 @@ scenario_043,self_employment_tax,0.0, scenario_043,state_income_tax_before_refundable_credits,0.0, scenario_043,state_refundable_credits,19.0, scenario_043,local_income_tax,0.0, -scenario_043,snap,3596.039794921875, +scenario_043,snap,3576.0, scenario_043,ssi,0.0, scenario_043,tanf,0.0, scenario_043,head_wic_eligible,0.0,0.0 @@ -743,10 +743,10 @@ scenario_045,federal_income_tax_before_refundable_credits,2173.07080078125, scenario_045,federal_refundable_credits,0.0, scenario_045,payroll_tax,2775.08251953125, scenario_045,self_employment_tax,0.0, -scenario_045,state_income_tax_before_refundable_credits,1288.8375244140625, +scenario_045,state_income_tax_before_refundable_credits,1290.9625244140625, scenario_045,state_refundable_credits,760.7887573242188, scenario_045,local_income_tax,0.0, -scenario_045,snap,287.68316650390625, +scenario_045,snap,0.0, scenario_045,ssi,0.0, scenario_045,tanf,0.0, scenario_045,head_wic_eligible,0.0,0.0 @@ -827,7 +827,7 @@ scenario_051,federal_income_tax_before_refundable_credits,2620.0, scenario_051,federal_refundable_credits,0.0, scenario_051,payroll_tax,3060.0, scenario_051,self_employment_tax,0.0, -scenario_051,state_income_tax_before_refundable_credits,814.9500122070312, +scenario_051,state_income_tax_before_refundable_credits,820.3499755859375, scenario_051,state_refundable_credits,0.0, scenario_051,local_income_tax,0.0, scenario_051,snap,0.0, @@ -882,7 +882,7 @@ scenario_054,self_employment_tax,2416.15283203125, scenario_054,state_income_tax_before_refundable_credits,0.0, scenario_054,state_refundable_credits,0.0, scenario_054,local_income_tax,0.0, -scenario_054,snap,6125.68896484375, +scenario_054,snap,6060.0, scenario_054,ssi,0.0, scenario_054,tanf,0.0, scenario_054,head_wic_eligible,0.0,0.0 @@ -1048,7 +1048,7 @@ scenario_066,self_employment_tax,0.0, scenario_066,state_income_tax_before_refundable_credits,0.0, scenario_066,state_refundable_credits,7.955999851226807, scenario_066,local_income_tax,0.0, -scenario_066,snap,3596.039794921875, +scenario_066,snap,3576.0, scenario_066,ssi,0.0, scenario_066,tanf,0.0, scenario_066,head_wic_eligible,0.0,0.0 @@ -1085,7 +1085,7 @@ scenario_068,federal_income_tax_before_refundable_credits,1910.0238037109375, scenario_068,federal_refundable_credits,0.0, scenario_068,payroll_tax,2607.39013671875, scenario_068,self_employment_tax,0.0, -scenario_068,state_income_tax_before_refundable_credits,1252.9677734375, +scenario_068,state_income_tax_before_refundable_credits,1255.3427734375, scenario_068,state_refundable_credits,0.0, scenario_068,local_income_tax,0.0, scenario_068,snap,0.0, @@ -1156,7 +1156,7 @@ scenario_073,self_employment_tax,0.0, scenario_073,state_income_tax_before_refundable_credits,0.0, scenario_073,state_refundable_credits,0.0, scenario_073,local_income_tax,0.0, -scenario_073,snap,287.68316650390625, +scenario_073,snap,288.0, scenario_073,ssi,0.0, scenario_073,tanf,0.0, scenario_073,head_wic_eligible,0.0,0.0 @@ -1201,7 +1201,7 @@ scenario_076,federal_income_tax_before_refundable_credits,19611.80078125, scenario_076,federal_refundable_credits,0.0, scenario_076,payroll_tax,12240.0, scenario_076,self_employment_tax,0.0, -scenario_076,state_income_tax_before_refundable_credits,6806.78662109375, +scenario_076,state_income_tax_before_refundable_credits,6818.34423828125, scenario_076,state_refundable_credits,465.0, scenario_076,local_income_tax,0.0, scenario_076,snap,0.0, @@ -1241,11 +1241,11 @@ scenario_077,head_chip_eligible,0.0,0.0 scenario_077,head_medicare_eligible,0.0,0.0 scenario_077,free_school_meals_eligible,0.0,0.0 scenario_077,reduced_price_school_meals_eligible,0.0,0.0 -scenario_078,federal_income_tax_before_refundable_credits,24772.693359375, +scenario_078,federal_income_tax_before_refundable_credits,24164.45703125, scenario_078,federal_refundable_credits,0.0, scenario_078,payroll_tax,14317.322265625, scenario_078,self_employment_tax,0.0, -scenario_078,state_income_tax_before_refundable_credits,6936.33740234375, +scenario_078,state_income_tax_before_refundable_credits,6936.87646484375, scenario_078,state_refundable_credits,0.0, scenario_078,local_income_tax,0.0, scenario_078,snap,0.0, @@ -1264,7 +1264,7 @@ scenario_079,self_employment_tax,0.0, scenario_079,state_income_tax_before_refundable_credits,0.0, scenario_079,state_refundable_credits,50.0, scenario_079,local_income_tax,0.0, -scenario_079,snap,2428.017333984375, +scenario_079,snap,2376.0, scenario_079,ssi,10572.0, scenario_079,tanf,0.0, scenario_079,head_wic_eligible,0.0,0.0 @@ -1314,7 +1314,7 @@ scenario_082,federal_refundable_credits,0.0, scenario_082,payroll_tax,8108.02734375, scenario_082,self_employment_tax,0.0, scenario_082,state_income_tax_before_refundable_credits,5598.55224609375, -scenario_082,state_refundable_credits,650.5, +scenario_082,state_refundable_credits,667.0, scenario_082,local_income_tax,0.0, scenario_082,snap,0.0, scenario_082,ssi,0.0, @@ -1489,7 +1489,7 @@ scenario_093,federal_income_tax_before_refundable_credits,8628.990234375, scenario_093,federal_refundable_credits,0.0, scenario_093,payroll_tax,12469.5, scenario_093,self_employment_tax,0.0, -scenario_093,state_income_tax_before_refundable_credits,3389.65087890625, +scenario_093,state_income_tax_before_refundable_credits,3388.24560546875, scenario_093,state_refundable_credits,0.0, scenario_093,local_income_tax,0.0, scenario_093,snap,0.0, @@ -1642,7 +1642,7 @@ scenario_104,federal_refundable_credits,0.0, scenario_104,payroll_tax,0.0, scenario_104,self_employment_tax,0.0, scenario_104,state_income_tax_before_refundable_credits,0.0, -scenario_104,state_refundable_credits,375.0, +scenario_104,state_refundable_credits,0.0, scenario_104,local_income_tax,0.0, scenario_104,snap,0.0, scenario_104,ssi,0.0, @@ -1676,7 +1676,7 @@ scenario_108,self_employment_tax,0.0, scenario_108,state_income_tax_before_refundable_credits,0.0, scenario_108,state_refundable_credits,271.3916015625, scenario_108,local_income_tax,0.0, -scenario_108,snap,287.68316650390625, +scenario_108,snap,288.0, scenario_108,ssi,0.0, scenario_108,tanf,0.0, scenario_108,head_wic_eligible,0.0,0.0 @@ -1692,7 +1692,7 @@ scenario_109,self_employment_tax,4238.86474609375, scenario_109,state_income_tax_before_refundable_credits,0.0, scenario_109,state_refundable_credits,0.0, scenario_109,local_income_tax,0.0, -scenario_109,snap,8020.55419921875, +scenario_109,snap,7932.0, scenario_109,ssi,0.0, scenario_109,tanf,0.0, scenario_109,head_wic_eligible,0.0,0.0 @@ -1827,11 +1827,11 @@ scenario_116,head_medicare_eligible,0.0,0.0 scenario_116,spouse_medicare_eligible,0.0,0.0 scenario_116,free_school_meals_eligible,0.0,0.0 scenario_116,reduced_price_school_meals_eligible,0.0,0.0 -scenario_117,federal_income_tax_before_refundable_credits,24391.796875, +scenario_117,federal_income_tax_before_refundable_credits,24961.33984375, scenario_117,federal_refundable_credits,0.0, scenario_117,payroll_tax,15044.787109375, scenario_117,self_employment_tax,0.0, -scenario_117,state_income_tax_before_refundable_credits,7984.05419921875, +scenario_117,state_income_tax_before_refundable_credits,7984.53369140625, scenario_117,state_refundable_credits,0.0, scenario_117,local_income_tax,0.0, scenario_117,snap,0.0, @@ -1945,7 +1945,7 @@ scenario_122,federal_income_tax_before_refundable_credits,4746.66162109375, scenario_122,federal_refundable_credits,0.0, scenario_122,payroll_tax,0.0, scenario_122,self_employment_tax,0.0, -scenario_122,state_income_tax_before_refundable_credits,1028.216552734375, +scenario_122,state_income_tax_before_refundable_credits,1025.54150390625, scenario_122,state_refundable_credits,0.0, scenario_122,local_income_tax,0.0, scenario_122,snap,0.0, diff --git a/policybench/adjudications.py b/policybench/adjudications.py index a0441a1a..28871282 100644 --- a/policybench/adjudications.py +++ b/policybench/adjudications.py @@ -38,9 +38,25 @@ class and the reasoning, so the override stays auditable, and this module ) FINAL_SOURCES = frozenset({"llm_error", "parse_contract_failure"}) # A developer may affirm a judge's (or override a judge's) verdict as prompt -# ambiguity only when the output is also removed from scoring for every model -# (see policybench.reference_exclusions); an ambiguous output is never scored. -AFFIRMABLE_WITH_EXCLUSION = frozenset({"prompt_ambiguity"}) +# ambiguity, or record a reference engine defect, only when the output is also +# removed from scoring for every model (see policybench.reference_exclusions); +# an ambiguous or defective reference is never scored. +AFFIRMABLE_WITH_EXCLUSION = frozenset( + {"prompt_ambiguity", "reference_engine_defect", "reference_later_law"} +) +# The developer's verdict on a case's reference, recorded whenever the case +# carried a reference-suspect flag. Applying it clears the flag: "affirmed" +# keeps the output scored against the reference; "engine_defect" and +# "unlisted_input" go with an exclusion of the matching reason code. +REFERENCE_VERDICTS = { + "affirmed": None, + # The flagged reference was replaced by a regenerated one under a recorded + # convention (the reference sidecar's revisions); the output stays scored. + "regenerated": None, + "engine_defect": "reference_engine_defect", + "unlisted_input": "prompt_ambiguity", + "later_law": "reference_later_law", +} class AdjudicationError(ValueError): @@ -51,7 +67,15 @@ def load_adjudications(path: Path) -> list[dict]: """Read and validate an adjudication file; an absent file means none.""" if not path.exists(): return [] - payload = json.loads(path.read_text()) + return parse_adjudications(json.loads(path.read_text()), path) + + +def parse_adjudications(payload: dict, path: Path | str) -> list[dict]: + """Validate an adjudication record already in memory. + + ``path`` only names the record in error messages, so a record can be + checked before it is written anywhere. + """ entries = payload.get("adjudications") if not isinstance(entries, list): raise AdjudicationError(f"{path}: 'adjudications' must be a list") @@ -75,7 +99,46 @@ def load_adjudications(path: Path) -> list[dict]: elif excluded: raise AdjudicationError( f"{path}: excluded_from_scoring is only valid with " - f"prompt_ambiguity, got {adjudicated!r}" + f"{' or '.join(sorted(AFFIRMABLE_WITH_EXCLUSION))}, got {adjudicated!r}" + ) + verdict = entry.get("reference_verdict") + if verdict is not None: + if verdict not in REFERENCE_VERDICTS: + raise AdjudicationError( + f"{path}: unknown reference_verdict {verdict!r}" + ) + expected = REFERENCE_VERDICTS[verdict] + if expected is None: + if excluded or adjudicated not in FINAL_SOURCES: + raise AdjudicationError( + f"{path}: an {verdict} reference keeps the output scored " + f"with a final class, got {adjudicated!r}, excluded={excluded}" + ) + elif adjudicated != expected: + raise AdjudicationError( + f"{path}: reference_verdict {verdict!r} requires " + f"adjudicated_failure_source {expected!r}, got {adjudicated!r}" + ) + if not entry.get("reference_basis"): + raise AdjudicationError( + f"{path}: reference_verdict needs reference_basis (the law or " + f"evidence the verdict rests on): {entry}" + ) + if "judge_reference_suspect" in entry and not isinstance( + entry["judge_reference_suspect"], bool + ): + raise AdjudicationError( + f"{path}: judge_reference_suspect must be true or false: {entry}" + ) + if entry.get("judge_reference_suspect") and not verdict: + raise AdjudicationError( + f"{path}: a case the judge flagged needs a reference_verdict: {entry}" + ) + if verdict is not None: + pass + elif adjudicated in ("reference_engine_defect", "reference_later_law"): + raise AdjudicationError( + f"{path}: {adjudicated} requires a reference_verdict" ) for field in ("judge_failure_source",): if entry[field] not in FAILURE_SOURCE_VALUES: @@ -99,14 +162,34 @@ def _strip_adjudication_sentence(note: str) -> str: return note if index < 0 else note[:index] +_REFERENCE_VERDICT_PHRASES = { + "affirmed": "Reference affirmed", + "regenerated": "Reference regenerated", + "engine_defect": "Reference is an engine defect; output excluded from scoring", + "unlisted_input": ( + "Reference depends on an unlisted input; output excluded from scoring" + ), + "later_law": ( + "Reference depends on law published after the reference freeze; output " + "excluded from scoring" + ), +} + + def adjudication_sentence(entry: dict) -> str: """The sentence a case note carries once ``entry`` has been applied.""" - return ( + sentence = ( f" Developer adjudication ({entry['adjudicated_on']}): the judge " f"({entry['judge_model']}) returned {entry['judge_failure_source']}; " f"adjudicated {entry['adjudicated_failure_source']} " f"({entry['adjudicated_failure_subtype']}). {entry['reasoning']}" ) + verdict = entry.get("reference_verdict") + if verdict: + sentence += ( + f" {_REFERENCE_VERDICT_PHRASES[verdict]} ({entry['reference_basis']})." + ) + return sentence def _case_mask(frame: pd.DataFrame, entry: dict) -> pd.Series: @@ -152,6 +235,13 @@ def apply_adjudications( .any(axis=1) .sum() ) + if entry.get("reference_verdict"): + # The developer's verdict resolves the judge's reference-suspect + # flag on every row of the case, parse failures included. + if "reference_suspect" in rows.columns: + rows.loc[_case_mask(rows, entry), "reference_suspect"] = False + if "reference_suspect" in cases.columns: + cases.loc[case_mask, "reference_suspect"] = False cases.loc[case_mask, "case_failure_sources"] = entry[ "adjudicated_failure_source" ] @@ -235,6 +325,19 @@ def verify_adjudications_applied( f"{entry['adjudicated_failure_source']!r}/" f"{entry['adjudicated_failure_subtype']!r}" ) + if entry.get("reference_verdict"): + for frame, mask, label in ( + (rows, row_mask, "rows"), + (cases, case_mask, "case note"), + ): + if "reference_suspect" not in frame.columns: + continue + flags = frame.loc[mask, "reference_suspect"].map(_truthy) + if flags.any(): + raise AdjudicationError( + f"{label} of {key} still carry reference_suspect after " + "the reference verdict" + ) sentence = adjudication_sentence(entry).strip() for note in cases.loc[case_mask, "case_annotation"].astype(str): if sentence not in note: @@ -244,6 +347,33 @@ def verify_adjudications_applied( ) +def _truthy(value: object) -> bool: + return str(value).strip().lower() in {"true", "1", "yes"} + + +def unresolved_suspect_cases( + cases: pd.DataFrame, adjudications: list[dict] +) -> list[tuple[str, str, str]]: + """Cases still flagged reference-suspect with no developer reference verdict. + + A frozen snapshot may not carry one: every flag the judge raises is settled + by an adjudication that affirms the reference or excludes the output. + """ + if "reference_suspect" not in cases.columns: + return [] + decided = { + tuple(str(e[c]) for c in CASE_KEY) + for e in adjudications + if e.get("reference_verdict") + } + flagged = cases.loc[cases["reference_suspect"].map(_truthy), CASE_KEY] + return [ + tuple(str(v) for v in row) + for row in flagged.itertuples(index=False) + if tuple(str(v) for v in row) not in decided + ] + + def excluded_case_keys(adjudications: list[dict]) -> set[tuple[str, str]]: """Cases the record removes from scoring (must equal the exclusion record).""" return { diff --git a/policybench/analysis.py b/policybench/analysis.py index e3810fda..0ae3b331 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -17,6 +17,7 @@ normalize_weights, ) from policybench.prompts import make_no_tools_batch_prompt +from policybench.reference_exclusions import exclusion_basis from policybench.scenarios import scenario_from_dict from policybench.spec import ( expand_programs_for_scenario, @@ -2145,6 +2146,33 @@ def build_failure_modes_payload( } +def _reference_exclusion_payload(entry: dict) -> dict: + """Serialize one exclusion record entry for the dashboard payload.""" + item = { + "scenarioId": str(entry["scenario_id"]), + "variable": str(entry["variable"]), + "reasonCode": str(entry["reason_code"]), + "alternativeReading": str(entry["alternative_reading"]), + "frozenValue": float(entry["frozen_value"]), + "alternativeValue": float(entry["alternative_value"]), + "engineVersion": str(entry["engine_version"]), + "decidedOn": str(entry["decided_on"]), + "note": str(entry.get("note", "")), + } + if entry.get("unlisted_input"): + item["unlistedInput"] = str(entry["unlisted_input"]) + for key, field in ( + ("rootCause", "root_cause"), + ("defect", "defect"), + ("law", "law"), + ("upstream", "upstream"), + ("published", "published"), + ): + if entry.get(field): + item[key] = str(entry[field]) + return item + + def build_dashboard_payload( ground_truth: pd.DataFrame, predictions: pd.DataFrame, @@ -2446,7 +2474,7 @@ def build_dashboard_payload( exclusion = excluded_keys.get((str(row["scenario_id"]), str(row["variable"]))) if exclusion is not None: prediction_item["excludedReason"] = str(exclusion["reason_code"]) - prediction_item["excludedInput"] = str(exclusion["unlisted_input"]) + prediction_item["excludedInput"] = exclusion_basis(exclusion) explanation = row.get("explanation") if isinstance(explanation, str) and explanation.strip(): prediction_item["explanation"] = explanation.strip() @@ -2526,18 +2554,7 @@ def _weights_dict(weights: pd.Series) -> dict[str, float]: "policyengineBundles": policyengine_bundles, "scenarios": scenario_payload, "referenceExclusions": [ - { - "scenarioId": str(entry["scenario_id"]), - "variable": str(entry["variable"]), - "reasonCode": str(entry["reason_code"]), - "unlistedInput": str(entry["unlisted_input"]), - "alternativeReading": str(entry["alternative_reading"]), - "frozenValue": float(entry["frozen_value"]), - "alternativeValue": float(entry["alternative_value"]), - "engineVersion": str(entry["engine_version"]), - "decidedOn": str(entry["decided_on"]), - "note": str(entry.get("note", "")), - } + _reference_exclusion_payload(entry) for entry in (reference_exclusions or []) ], "modelStats": model_stats, diff --git a/policybench/annotation_taxonomy.py b/policybench/annotation_taxonomy.py index 58cd172a..69112508 100644 --- a/policybench/annotation_taxonomy.py +++ b/policybench/annotation_taxonomy.py @@ -9,6 +9,12 @@ "prompt_ambiguity", "reference_model_issue_fixed", "reference_data_issue_fixed", + # Developer adjudication only: the reference misapplies the law, so the + # output is excluded from scoring (policybench.reference_exclusions). + "reference_engine_defect", + # Developer adjudication only: the reference depends on law published after + # the reference freeze, so the output is excluded from scoring. + "reference_later_law", "parse_contract_failure", "budget_exhausted_at_ceiling", "needs_review", diff --git a/policybench/config.py b/policybench/config.py index 2edd9aa2..7c52b6b2 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -32,26 +32,33 @@ "gpt-5.6-terra": "gpt-5.6-terra", "gpt-5.6-luna": "gpt-5.6-luna", } -# GPT-6 Astra (API availability 2026-09-04) shares the GPT-5.6 line's +# GPT-6 Astra (API availability 2026-09-04), GPT-6 Sol and GPT-6 Luna (both +# announced and available 2026-09-22) share the GPT-5.6 line's # Responses-API transport and pricing structure (cache reads at a tenth, # cache writes at 1.25x, doubled input and 1.5x output above 272k tokens). GPT_6_MODELS = { "gpt-6-astra": "gpt-6-astra", + "gpt-6-sol": "gpt-6-sol", + "gpt-6.1-sol": "gpt-6.1-sol", + "gpt-6-luna": "gpt-6-luna", } GPT_RESPONSES_MODELS = {**GPT_56_MODELS, **GPT_6_MODELS} MODELS = { "claude-fable-5": "claude-fable-5", "claude-fable-5.1": "claude-fable-5-1", + "claude-opus-5.5": "claude-opus-5-5", "claude-opus-5": "claude-opus-5", "claude-opus-4.8": "claude-opus-4-8", "claude-opus-4.7": "claude-opus-4-7", + "claude-sonnet-5.5": "claude-sonnet-5-5", "claude-sonnet-5": "claude-sonnet-5", "claude-sonnet-4.6": "claude-sonnet-4-6", "claude-haiku-4.5": "claude-haiku-4-5-20251001", "grok-4.3": "xai/grok-4.3", "grok-4.5": "xai/grok-4.5", "grok-4.6": "xai/grok-4.6", + "grok-4.7": "xai/grok-4.7", "grok-build-0.1": "xai/grok-build-0.1", **GPT_6_MODELS, **GPT_56_MODELS, @@ -69,8 +76,26 @@ "ox-alpha": "openrouter/stealth/ox-alpha", "gemini-3-flash-preview": "gemini/gemini-3-flash-preview", "gemini-3.1-flash-lite-preview": "gemini/gemini-3.1-flash-lite-preview", + # DeepSeek retired V4 Flash when it released V4.1 Flash on 2026-09-10 + # (api-docs.deepseek.com/news/news260910). The legacy name + # deepseek-v4-flash now routes to V4.1 Flash: on 2026-09-28 a request + # under that name answered as model "deepseek-flash" with the same system + # fingerprint as a deepseek-flash request. Rows recorded under + # deepseek-v4-flash before the retirement are V4 Flash, and that name can + # no longer re-run them. The same post says deepseek-v4-pro routes to + # V4.1 Flash from 2026-09-14 until V4.1 Pro launches, but on 2026-09-28 a + # deepseek-v4-pro request still answered as "deepseek-v4-pro" with a + # different fingerprint and prompt-token count, and the pricing page + # lists it as DeepSeek-V4-Pro-0813. Either way that name no longer + # reaches the model the deepseek-v4-pro row ran on: the alias had already + # moved to the August release (see the deepseek-v4-pro-0813 model card). "deepseek-v4-pro": "deepseek/deepseek-v4-pro", "deepseek-v4-flash": "deepseek/deepseek-v4-flash", + # deepseek-flash is DeepSeek's current name for V4.1 Flash and is itself + # a moving alias: GET /models maps it to DeepSeek-V4.1-Flash, and + # responses echo "deepseek-flash" with no version string, so each row's + # provider_system_fingerprint is the record of which build answered. + "deepseek-v4.1-flash": "deepseek/deepseek-flash", "deepseek-v4-pro-0813": "openrouter/deepseek/deepseek-v4-pro-0813", "deepseek-v4-flash-0731": "openrouter/deepseek/deepseek-v4-flash-0731", "kimi-k2.6": "openrouter/moonshotai/kimi-k2.6", @@ -84,9 +109,10 @@ } # Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model -# id. The frozen row's cost is its recorded per-call cost: provider-reported -# where the provider returns one, otherwise reconstructed at the configured -# list price at request time. List-price overrides apply at request time, not +# id. The frozen row's cost is its recorded per-call cost: reconstructed from +# token counts at the list price configured at request time, or the +# provider-reported charge where no reconstruction was available (see +# eval_no_tools usage accounting). List-price overrides apply at request time, not # retroactively to recorded costs. Analysis preserves each recorded total. PRICE_OVERRIDES_PER_1M: dict[str, dict[str, float]] = { # GPT-5.6 list prices from OpenAI's general-availability announcement @@ -96,11 +122,34 @@ # writes, $20 / $75 above 272k prompt tokens (OpenAI model page and the # OpenRouter mirror, retrieved 2026-09-04). "gpt-6-astra": {"input": 10.0, "output": 50.0}, + # gpt-6-sol: $2 / $10 per 1M input/output, $0.20 cached input; gpt-6-luna: + # $0.10 / $0.50, $0.01 cached input; both double input and cache and take + # 1.5x output above 272k tokens (developers.openai.com/api/docs/models/ + # gpt-6-sol and /gpt-6-luna, read 2026-09-22, the day OpenAI announced + # both; the Models API lists each id with created 2026-09-14). + "gpt-6-sol": {"input": 2.0, "output": 10.0}, + # gpt-6.1-sol: $2 / $10 per 1M input/output, $0.10 cached input, $2.50 cache + # writes; 2x input and cache and 1.5x output above 272k input tokens + # (developers.openai.com/api/docs/models/gpt-6.1-sol, read 2026-09-29, the + # day OpenAI announced it at DevDay; the Models API lists the id with + # created 2026-09-27T23:47Z). + "gpt-6.1-sol": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + "gpt-6-luna": {"input": 0.10, "output": 0.50}, "gpt-5.6-sol": {"input": 5.0, "output": 30.0}, "gpt-5.6-terra": {"input": 2.5, "output": 15.0}, "gpt-5.6-luna": {"input": 1.0, "output": 6.0}, # grok-build-0.1: $1 / $2 per 1M input/output tokens (https://x.ai/api). "grok-build-0.1": {"input": 1.0, "output": 2.0}, + # grok-4.7: $2 / $6 per 1M input/output tokens with $0.50 cached input + # (docs.x.ai/docs/models/grok-4.7, read 2026-09-28). The live record at + # api.x.ai/v1/language-models/grok-4.7 gives 20000 / 60000 / 5000 in its + # unit of $0.0001 per 1M tokens, identical to grok-4.6's record, which + # litellm's map prices at $2 / $6 with $0.50 cache reads. Doubled rates + # above 200k prompt tokens are not modeled; benchmark prompts stay far + # below that. On the 2026-09-28 reasoning probe these rates reproduced + # xAI's reported charge (cost_in_usd_ticks) exactly. litellm's bundled + # map lacks the id, so the override keeps costs off the remote fetch. + "grok-4.7": {"input": 2.0, "output": 6.0, "cache_read": 0.50}, # kimi-k3: $3 / $15 per 1M input/output tokens (OpenRouter model listing, # 2026-07-17); litellm's price map does not yet cover the id. "kimi-k3": {"input": 3.0, "output": 15.0}, @@ -149,12 +198,50 @@ # map carries the same figures; this fallback keeps the leaderboard priced # if reconstruction is unavailable. Introductory billing ($2 / $10 through # 2026-08-31) is intentionally not used — costs compare at standard rates. + # claude-opus-5-5: $4 / $20 per 1M input/output tokens, $5 five-minute + # cache writes and $0.20 cache reads (0.05x base input; platform.claude.com + # /docs/en/about-claude/pricing, read 2026-09-22). The Models API lists + # the id with created_at 2026-09-21. litellm's map lacked the id at + # onboarding; eval_no_tools registers it locally, as for the Fable line. + "claude-opus-5.5": { + "input": 4.0, + "output": 20.0, + "cache_read": 0.20, + "cache_write": 5.0, + }, "claude-sonnet-5": {"input": 3.0, "output": 15.0}, + # claude-sonnet-5-5: $2 / $10 per 1M input/output tokens, $2.50 + # five-minute cache writes and $0.20 cache reads (platform.claude.com/docs + # /en/about-claude/pricing, read 2026-09-28). No footnote marks the row as + # introductory, so this is the standard rate. (Footnote 3 on the same page + # now makes $2 / $10 Sonnet 5's standard price as well; the claude-sonnet-5 + # entry above keeps $3 / $15 and is not changed here.) The Models API lists + # the id with created_at 2026-09-28. litellm's bundled map lacks the id; + # eval_no_tools registers it locally, as for Opus 5.5. + "claude-sonnet-5.5": { + "input": 2.0, + "output": 10.0, + "cache_read": 0.20, + "cache_write": 2.50, + }, # Open-weight additions, per-1M USD from the OpenRouter live model list # (https://openrouter.ai/api/v1/models, retrieved 2026-07-05). DeepSeek - # runs on its native API at the same list prices. + # runs on its native API at the same list prices. These two rates + # predate the 2026-09-10 V4.1 Flash release (see MODELS) and are left + # unchanged; neither name can re-run its row. "deepseek-v4-pro": {"input": 0.435, "output": 0.87}, "deepseek-v4-flash": {"input": 0.09, "output": 0.18}, + # deepseek-v4.1-flash (model name deepseek-flash): $0.30 per 1M input on + # a cache miss, $0.006 on a cache hit, $1.20 per 1M output + # (api-docs.deepseek.com/quick_start/pricing, read 2026-09-28). These are + # the peak rates; off-peak rates are half, and a request's rate depends + # on the hour it lands (peak is 01:00-04:00 and 06:00-10:00 UTC on + # weekdays other than Chinese public holidays). The row is priced at the + # peak list rate whenever the run happens, as claude-sonnet-5 is priced + # at its standard rather than its introductory rate. DeepSeek lists no + # cache-write charge and its usage reports none. litellm's bundled map + # lacks the id. + "deepseek-v4.1-flash": {"input": 0.30, "output": 1.20, "cache_read": 0.006}, "kimi-k2.6": {"input": 0.66, "output": 3.41}, "glm-5.2": {"input": 0.574, "output": 1.804}, "minimax-m3": {"input": 0.3, "output": 1.2}, diff --git a/policybench/eval_no_tools.py b/policybench/eval_no_tools.py index 231fd837..3472703a 100644 --- a/policybench/eval_no_tools.py +++ b/policybench/eval_no_tools.py @@ -36,7 +36,7 @@ completion_budget_ceiling_for, explanation_chunk_size_for, ) -from policybench.policyengine_runtime import policyengine_bundles_for_countries +from policybench.policyengine_runtime import resolve_policyengine_bundles from policybench.prompts import ( get_variable_description, make_explanation_repair_prompt, @@ -87,12 +87,18 @@ # litellm resolves an unprefixed model's provider (and prices it) through its # model-cost map, whose remote refresh can time out mid-run and whose bundled -# backup lags brand-new models. Register the Claude Fable line locally so -# provider routing and cost reconstruction never depend on the remote fetch. +# backup lags brand-new models. Register the Claude models litellm does not +# yet price locally so provider routing and cost reconstruction never depend +# on the remote fetch. # Prices come from the canonical per-million overrides so registration and # PolicyBench's reconstruction cannot drift apart. -_LOCAL_CLAUDE_FABLE_MODELS = ("claude-fable-5", "claude-fable-5.1") -for _display_id in _LOCAL_CLAUDE_FABLE_MODELS: +_LOCAL_CLAUDE_MODELS = ( + "claude-fable-5", + "claude-fable-5.1", + "claude-opus-5.5", + "claude-sonnet-5.5", +) +for _display_id in _LOCAL_CLAUDE_MODELS: _model_id = MODELS[_display_id] if _model_id in litellm.model_cost: continue @@ -193,13 +199,16 @@ def _env_int(name: str, default: int, env: dict | None = None) -> int: # Claude models whose requests run thinking without us asking: the Fable line # cannot disable it, and Sonnet 5 defaults to adaptive thinking when the # request omits the thinking param (ours do — models run at their API -# defaults). Hard single outputs can think for minutes, so they get a longer -# timeout than the rest of the Claude family. +# defaults); so does Sonnet 5.5 (a thinking block on a request with no thinking +# param at its 2026-09-28 onboarding). Hard single outputs can think for +# minutes, so they get a longer timeout than the rest of the Claude family. THINKING_DEFAULT_CLAUDE_MODELS = ( "claude-fable-5", "claude-fable-5-1", "claude-sonnet-5", "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-5-5", ) THINKING_CLAUDE_REQUEST_TIMEOUT_SECONDS = _env_int( "POLICYBENCH_THINKING_CLAUDE_REQUEST_TIMEOUT_SECONDS", 300 @@ -3020,6 +3029,10 @@ def _scenario_hash(scenarios: list[Scenario]) -> str: return hashlib.sha256(scenario_signature.encode("utf-8")).hexdigest() +def _scenario_countries(scenarios: list[Scenario]) -> set[str]: + return {(scenario.country or "us").lower() for scenario in scenarios} + + def _build_resume_metadata( *, task: str, @@ -3029,8 +3042,15 @@ def _build_resume_metadata( run_id: str | None, include_explanations: bool, env: dict | None = None, + policyengine_bundles: dict | None = None, ) -> dict: - countries = {(scenario.country or "us").lower() for scenario in scenarios} + """Metadata recorded beside a resumable output and compared on resume. + + ``policyengine_bundles``, when given, holds already-computed bundles keyed + by country (the supervisor's copy of its run's provenance file); otherwise + they are resolved for this process. + """ + countries = _scenario_countries(scenarios) return { "metadata_version": RESUME_METADATA_VERSION, "task": task, @@ -3051,7 +3071,13 @@ def _build_resume_metadata( single_output=task == "eval_no_tools_single_output", env=env, ), - "policyengine_bundles": policyengine_bundles_for_countries(countries), + "policyengine_bundles": ( + resolve_policyengine_bundles(countries, env=env) + if policyengine_bundles is None + else { + country: policyengine_bundles[country] for country in sorted(countries) + } + ), "response_contract": _response_contract_metadata(), "completion_budget_escalation": { "strategy": "double_on_length_with_missing_payload", diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 10dfb402..7a0724de 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -86,6 +86,44 @@ class ModelCard: "(effort medium), like GPT-5.6 Sol." ), ), + "gpt-6-sol": ModelCard( + litellm_id="gpt-6-sol", + answer_contract="tool", + thinking_budget=True, + expected_cost_per_scenario_usd=0.016, + notes=( + "Onboarded 2026-09-22, the day OpenAI announced it: forced tool " + "contract passed 3/3 (1,761 completion tokens) and 16/16 " + "whole-scenario (1,141 tokens) on the Responses API with " + "provider-default reasoning (effort medium), like GPT-6 Astra." + ), + ), + "gpt-6.1-sol": ModelCard( + litellm_id="gpt-6.1-sol", + answer_contract="tool", + thinking_budget=True, + expected_cost_per_scenario_usd=0.021, + notes=( + "Onboarded 2026-09-29, the day OpenAI announced it: forced tool " + "contract passed 3/3 (735 completion tokens) and 16/16 " + "whole-scenario (1,626 tokens) on the Responses API with " + "provider-default reasoning (effort medium, per the model page), " + "like GPT-6 Sol. Probed before its card existed, it got the " + "384-token non-thinking budget and stopped incomplete." + ), + ), + "gpt-6-luna": ModelCard( + litellm_id="gpt-6-luna", + answer_contract="tool", + thinking_budget=True, + expected_cost_per_scenario_usd=0.002, + notes=( + "Onboarded 2026-09-22, the day OpenAI announced it: forced tool " + "contract passed 3/3 (2,048 completion tokens) and 16/16 " + "whole-scenario (2,768 tokens) on the Responses API with " + "provider-default reasoning (effort medium), like GPT-6 Astra." + ), + ), "gpt-5.6-sol": ModelCard( litellm_id="gpt-5.6-sol", answer_contract="tool", @@ -181,6 +219,65 @@ class ModelCard: "1,984/1,984 parsed, no budget escalations)." ), ), + "claude-opus-5-5": ModelCard( + litellm_id="claude-opus-5-5", + answer_contract="json", + request_timeout_seconds=600, + thinking_budget=True, + provider_max_completion_tokens=128_000, + expected_cost_per_scenario_usd=0.057, + notes=( + "Onboarded 2026-09-22 (the Models API lists the id with " + "created_at 2026-09-21). The API rejects forced tool use on this " + "model with a 400 (tool_choice type tool/any 'not supported for " + "this model'), as on Fable 5.1, and a request with no thinking " + "parameter returns a thinking block (87 thinking tokens on a " + "one-question probe), so the row reasons at the provider " + "default and runs the JSON contract whole-scenario on the sync " + "path (the batch adapter supports the forced tool contract " + "only). Gauntlet: 3/3 and 16/16 parsed (1,664 and 2,489 " + "completion tokens, 15s and 21s); the gauntlet's cost estimate " + "is $0.057 per scenario at $4/$20 per 1M with $0.20 cache reads." + ), + ), + # Every field below comes from the 2026-09-28 onboarding record in + # results/local/adds202609/sonnet55/ (sonnet55_onboard.json, the + # thinking-default probe, and the Models API entry): + # - answer_contract "json": forced tool use returns a 400, so the + # forced-tool board request cannot be sent; the JSON contract passed. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). + # - thinking_budget True: a request with no thinking parameter returns a + # thinking block, so the row reasons at the API default (adaptive + # thinking, effort high) and thinking bills against the answer budget. + # - request_timeout_seconds 600: the value the other JSON-contract + # thinking rows (Fable 5.1, Opus 5.5) carry; the slowest probe took 17s, + # under the gauntlet's own 150s trigger for a longer timeout. + # - provider_max_completion_tokens 128_000: the Models API max_tokens. + # - expected_cost_per_scenario_usd 0.027: the gauntlet's full-scenario + # probe at $2/$10 per 1M. + "claude-sonnet-5-5": ModelCard( + litellm_id="claude-sonnet-5-5", + answer_contract="json", + request_timeout_seconds=600, + thinking_budget=True, + provider_max_completion_tokens=128_000, + expected_cost_per_scenario_usd=0.027, + notes=( + "Onboarded 2026-09-28, release day (the Models API lists the id " + "with created_at 2026-09-28). The API rejects forced tool use on " + "this model with a 400 (tool_choice type tool/any 'not supported " + "for this model'), as on Opus 5.5 and Fable 5.1, and a request " + "with no thinking parameter returns a thinking block (108 of 115 " + "output tokens on a one-question probe), so the row reasons at " + "the provider default (adaptive thinking, effort high) and runs " + "the JSON contract whole-scenario on the sync path (the batch " + "adapter supports the forced tool contract only). Gauntlet: 3/3 " + "and 16/16 parsed (1,523 and 2,389 completion tokens, 11s and " + "17s); the gauntlet's cost estimate is $0.027 per scenario at " + "$2/$10 per 1M with $0.20 cache reads." + ), + ), "xai/grok-4.5": ModelCard( litellm_id="xai/grok-4.5", answer_contract="tool", @@ -208,6 +305,22 @@ class ModelCard: "per 1M (litellm map). The 100-scenario run cost $8.70." ), ), + "xai/grok-4.7": ModelCard( + litellm_id="xai/grok-4.7", + answer_contract="tool", + request_timeout_seconds=1800, + thinking_budget=True, + expected_cost_per_scenario_usd=0.2, + notes=( + "Onboarded 2026-09-28: forced tool contract passed 3/3 (247s, " + "19,125 tokens) and 16/16 whole-scenario (472s, 32,960 tokens). " + "A first attempt with a 600s timeout passed 3/3 in 454s (39,192 " + "tokens) and then timed out on the whole-scenario probe, so " + "latency sits near 600s and the timeout is 1800s. Heavier than " + "Grok 4.6 (26,260 tokens on the 3-variable probe); at $2/$6 per " + "1M the gauntlet estimates $0.20 per scenario." + ), + ), "deepseek/deepseek-v4-pro": ModelCard( litellm_id="deepseek/deepseek-v4-pro", answer_contract="json", @@ -220,6 +333,20 @@ class ModelCard: answer_contract="json", thinking_budget=True, ), + "deepseek/deepseek-flash": ModelCard( + litellm_id="deepseek/deepseek-flash", + answer_contract="json", + thinking_budget=True, + expected_cost_per_scenario_usd=0.017, + notes=( + "Onboarded 2026-09-28 as DeepSeek V4.1 Flash (the deepseek-flash " + "name). The forced tool contract is rejected in thinking mode " + "('Thinking mode does not support this tool_choice'), as for the " + "V4 rows; the JSON contract passed 3/3 (80s, 19,313 tokens) and " + "16/16 whole-scenario (56s, 13,789 tokens). Thinking stays at " + "the provider default (on). Priced at the peak list rate." + ), + ), "openrouter/moonshotai/kimi-k2.6": ModelCard( litellm_id="openrouter/moonshotai/kimi-k2.6", answer_contract="json", diff --git a/policybench/output_scope.py b/policybench/output_scope.py new file mode 100644 index 00000000..084312b3 --- /dev/null +++ b/policybench/output_scope.py @@ -0,0 +1,218 @@ +"""Keep local taxes and credits out of the state outputs. + +PolicyBench's state outputs are state-only (``benchmark_specs.json``): + +- ``state_income_tax_before_refundable_credits``: "state individual income tax after + nonrefundable credits and before refundable credits, excluding local income and + payroll taxes"; +- ``state_refundable_credits``: "total refundable state individual income tax + credits". + +Local income taxes have their own output, ``local_income_tax``. + +policyengine-us builds each engine variable of those names by adding the variables +a parameter list names. On 2.15.17 three of those lists carry local entries +(``SCOPED_LISTS``): + +- ``gov.states.household.state_income_tax_before_refundable_credits``: + ``md_local_income_tax_before_refundable_credits`` (Maryland county income tax, + added by upstream #8888; the 2026-09-28 reference system removes it with + ``reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py``) and + ``nyc_income_tax_before_refundable_credits`` (New York City income tax, which + nothing removed); +- ``gov.states.household.state_refundable_credits``: ``nyc_refundable_credits``; +- ``gov.states.ca.tax.income.credits.refundable``: ``ca_sf_wftc``, San Francisco's + Working Families Tax Credit, which reaches ``state_refundable_credits`` through + ``ca_refundable_credits``. + +``reform`` removes every local entry, so the engine variables match the benchmark +outputs. Only the lists change; no formula or amount does. Maryland county tax still +reaches the federal SALT deduction through ``md_withheld_income_tax``'s county +estimate, and NYC tax net of NYC refundable credits (``nyc_income_tax``) is still in +``local_income_tax``. + +Every one of these is zero for every frozen household: ``in_nyc`` is true only in +the five NYC counties and ``in_san_francisco`` only in San Francisco County, and a +household with no county takes its state's alphabetically first county (Albany +County for New York, Alameda County for California). ``scope_violations`` is the +check: it returns each listed local entry that is nonzero in a simulation. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import Any + +STATE_AGGREGATE = "state_income_tax_before_refundable_credits" +STATE_AGGREGATE_PARAMETER = ( + "gov.states.household.state_income_tax_before_refundable_credits" +) +STATE_REFUNDABLE_PARAMETER = "gov.states.household.state_refundable_credits" +CA_REFUNDABLE_PARAMETER = "gov.states.ca.tax.income.credits.refundable" +# Each list, the local entries 2.15.17 puts in it, and state entries it must keep. +SCOPED_LISTS = { + STATE_AGGREGATE_PARAMETER: { + "local": ( + "md_local_income_tax_before_refundable_credits", + "nyc_income_tax_before_refundable_credits", + ), + "keeps": ( + "md_income_tax_before_refundable_credits", + "ny_income_tax_before_refundable_credits", + ), + }, + STATE_REFUNDABLE_PARAMETER: { + "local": ("nyc_refundable_credits",), + "keeps": ("ny_refundable_credits", "ca_refundable_credits"), + }, + CA_REFUNDABLE_PARAMETER: { + "local": ("ca_sf_wftc",), + "keeps": ("ca_eitc",), + }, +} +LOCAL_INCOME_TAX_COMPONENTS = SCOPED_LISTS[STATE_AGGREGATE_PARAMETER]["local"] +LOCAL_COMPONENTS = tuple( + name for spec in SCOPED_LISTS.values() for name in spec["local"] +) +# Spelled as engine variable names so a new local entry is caught by name: any +# list entry containing one of these fragments is local. +LOCAL_NAME_FRAGMENTS = ( + "_local_", + "nyc_", + "_sf_", + "san_francisco", + "philadelphia", + "kansas_city", + "st_louis", + "wilmington", + "denver", + "yonkers", + "multnomah", +) +SCOPE_INSTANT = "2026-01-01" +# Where the lists are rewritten: the benchmark year and the years after it. +SCOPE_PERIOD = "year:2026-01-01:10" +FIX_ID = "policybench_output_scope" +DESCRIPTION = ( + "Exclude Maryland county tax and New York City and San Francisco taxes and " + "credits from the state income tax and state refundable credit outputs " + "(output definition)" +) + + +def is_local_component(name: str) -> bool: + """Whether a list entry names a local (county or city) tax or credit. + + "state_and_local" (as in the SALT variables) is a state entry, not a local one. + """ + stripped = name.replace("state_and_local", "") + return name in LOCAL_COMPONENTS or any( + fragment in stripped for fragment in LOCAL_NAME_FRAGMENTS + ) + + +def scoped_components(components: Iterable[str]) -> list[str]: + """A list without its local entries, order kept.""" + return [name for name in components if not is_local_component(name)] + + +def local_components(components: Iterable[str]) -> list[str]: + """The local entries of a list, order kept.""" + return [name for name in components if is_local_component(name)] + + +def _parameter(parameters, path: str): + node = parameters + for part in path.split("."): + node = getattr(node, part) + return node + + +def listed_local_components( + parameters, path: str = STATE_AGGREGATE_PARAMETER, instant: str = SCOPE_INSTANT +) -> list[str]: + """The local entries one parameter list adds (default: the state income tax).""" + return local_components(_parameter(parameters, path)(instant)) + + +def all_listed_local_components(parameters, instant: str = SCOPE_INSTANT) -> list[str]: + """Every local entry in every scoped list, in ``SCOPED_LISTS`` order.""" + return [ + name + for path in SCOPED_LISTS + for name in listed_local_components(parameters, path, instant) + ] + + +def remove_local_components(parameters): + """``modify_parameters`` callback: drop every local entry from every list.""" + for path, spec in SCOPED_LISTS.items(): + param = _parameter(parameters, path) + current = list(param(SCOPE_INSTANT)) + kept = scoped_components(current) + if kept != current: + param.update(period=SCOPE_PERIOD, value=kept) + after = param(SCOPE_INSTANT) + assert not local_components(after), (path, after) + for name in spec["keeps"]: + assert name in after, (path, name) + return parameters + + +def installed_parameter_list(path: str = STATE_AGGREGATE_PARAMETER) -> list[str]: + """A list parameter of the installed policyengine-us, read from its YAML. + + The latest value is returned. The file is located without importing + policyengine_us and parsed without building a tax-benefit system (about 40 + seconds), so a test can check every engine upgrade for a new local entry. + """ + import importlib.util + from pathlib import Path + + import yaml + + spec = importlib.util.find_spec("policyengine_us") + if spec is None or not spec.submodule_search_locations: + raise ModuleNotFoundError("policyengine_us is not installed") + root = Path(next(iter(spec.submodule_search_locations))) + path_on_disk = root / "parameters" / (path.replace(".", "/") + ".yaml") + # BaseLoader keeps the 0000-01-01 key a string; the default loader rejects it. + data = yaml.load(path_on_disk.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + values = data["values"] + return list(values[sorted(values)[-1]]) + + +def scope_violations( + simulation: Any, + components: Sequence[str], + period: int | str, + tolerance: float = 0.005, +) -> dict[str, float]: + """Each listed local entry that is nonzero in ``simulation``, with its amount.""" + violations = {} + for name in components: + amount = float(simulation.calculate(name, period).sum()) + if abs(amount) > tolerance: + violations[name] = amount + return violations + + +def _build_reform(): + from policyengine_core.reforms import Reform + + class reform(Reform): + def apply(self): + self.modify_parameters(remove_local_components) + + reform.__module__ = __name__ + return reform + + +def __getattr__(name: str): + # ``reform`` is built on first access, as a fix module's ``reform`` is read + # (getattr(module, "reform")), so importing the helpers stays cheap: the CLI + # imports this module to build its parser. + if name == "reform": + globals()["reform"] = _build_reform() + return globals()["reform"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 2172eede..ab38c602 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -20,8 +20,8 @@ serving treatments, their evidence kinds, and the registry commit. * the frozen audit annotations dir (``manifest['audit_annotation_artifacts']``) -- the rows selected by the legacy threshold score, their adjudicated - failure sources, and the fact that zero rows are reference-suspect (no - PolicyEngine bugs found). + failure sources, and the developer adjudications that settle every + reference-suspect flag. The qmd imports ``r`` once in an ``#| echo: false`` setup cell and then every inline number is a ```{python} r.field``` placeholder, so a future @@ -32,36 +32,149 @@ import csv import json +import re from collections import Counter from functools import cached_property from pathlib import Path import pandas as pd +from policybench.reference_exclusions import ( + ENGINE_DEFECT, + LATER_LAW, + UNLISTED_INPUT, + exclusion_basis, + exclusion_keys, + load_reference_exclusions, +) from policybench.reference_exclusions import FILENAME as EXCLUSIONS_FILENAME -from policybench.reference_exclusions import exclusion_keys, load_reference_exclusions from policybench.snapshot_payload import read_run_payload +from policybench.spec import metric_type_for_output # ``paper_results`` lives in ``policybench/``; the repo root is one level up. ROOT = Path(__file__).resolve().parents[1] SNAPSHOT_DIR = ROOT / "paper" / "snapshot" / "20260501" +UPGRADE_VERIFICATION = ROOT / "reference_audit" / "2026-09-28" / "verification" +# The sweep that rechecked every reference on the newest policyengine-us +# release when PolicyBench checked PyPI before publishing, with the fix module +# the references were built with. +PUBLICATION_CHECK_SWEEP = UPGRADE_VERIFICATION / "latest_final_2170.csv" +# When the reference sweep began, each release's PyPI upload time, and when +# PolicyBench read PyPI for the publication check. +SWEEP_TIMING = UPGRADE_VERIFICATION / "sweep_timing.json" +# What each exclusion sweep re-run on the reference engine moves. +RERUN_SWEEPS = UPGRADE_VERIFICATION / "rerun_sweeps.json" + +NUMBER_WORDS = { + 0: "no", + 1: "one", + 2: "two", + 3: "three", + 4: "four", + 5: "five", + 6: "six", + 7: "seven", + 8: "eight", + 9: "nine", + 10: "ten", +} + + +def moves_beyond_tolerance(variable: str, before: float, after: float) -> bool: + """Whether a value change leaves the exact-match tolerance: more than $1 + for an amount output, any change for a 0/1 flag.""" + if metric_type_for_output(variable) == "amount": + return abs(after - before) > 1 + return after != before + + +def partition_rerun_sweep_moves( + summary: dict, excluded: frozenset[tuple[str, str]] | set[tuple[str, str]] +) -> dict[str, list[tuple[str, str, str]]]: + """Split what the re-run sweeps move (verification/rerun_sweeps.json). + + Each listed move is (sweep, scenario_id, variable), compared with the + sweep's own baseline (the same calculation without its fix or reading): + scored outputs beyond the exact-match tolerance, scored outputs moved + within it, and excluded outputs. + """ + groups: dict[str, list[tuple[str, str, str]]] = { + "scored_beyond_tolerance": [], + "scored_within_tolerance": [], + "excluded": [], + } + for sweep in summary["sweeps"]: + for move in sweep["moves"]: + if move["recomputed"] == move["baseline"]: + continue + key = (sweep["sweep"], move["scenario_id"], move["variable"]) + if (move["scenario_id"], move["variable"]) in excluded: + groups["excluded"].append(key) + elif moves_beyond_tolerance( + move["variable"], move["baseline"], move["recomputed"] + ): + groups["scored_beyond_tolerance"].append(key) + else: + groups["scored_within_tolerance"].append(key) + return groups + + +def partition_engine_upgrade_changes( + changes: list[dict], excluded: frozenset[tuple[str, str]] | set[tuple[str, str]] +) -> dict[str, list[dict]]: + """Split an engine upgrade's changed outputs into three disjoint groups. + + A change to an output the exclusion record now lists is a new exclusion. + A scored change moves beyond the exact-match tolerance ($1 for an amount, + any change for a 0/1 flag) or within it. Every change lands in exactly one + group, so the three counts add up to the revision's changed list. + """ + partition: dict[str, list[dict]] = { + "scored_changes": [], + "within_tolerance": [], + "new_exclusions": [], + } + for change in changes: + if (change["scenario_id"], change["variable"]) in excluded: + partition["new_exclusions"].append(change) + continue + moved = abs(change["regenerated"] - change["previous"]) + amount = metric_type_for_output(change["variable"]) == "amount" + beyond = moved > 1 if amount else moved > 0 + partition["scored_changes" if beyond else "within_tolerance"].append(change) + if sum(len(group) for group in partition.values()) != len(changes): + raise AssertionError( + "engine upgrade partition lost or duplicated a change: " + f"{ {name: len(group) for name, group in partition.items()} } " + f"of {len(changes)}" + ) + return partition + + # Human-readable model names for the frozen roster. Aliases that do not # appear here fall back to a humanized form of the PolicyBench id. MODEL_DISPLAY_NAMES = { "gpt-6-astra": "GPT-6 Astra", + "gpt-6-sol": "GPT-6 Sol", + "gpt-6.1-sol": "GPT-6.1 Sol", + "gpt-6-luna": "GPT-6 Luna", "gpt-5.6-sol": "GPT-5.6 Sol", "gpt-5.6-terra": "GPT-5.6 Terra", "gpt-5.6-luna": "GPT-5.6 Luna", "claude-fable-5.1": "Claude Fable 5.1", + "claude-opus-5.5": "Claude Opus 5.5", + "claude-sonnet-5.5": "Claude Sonnet 5.5", "claude-fable-5": "Claude Fable 5", "claude-sonnet-5": "Claude Sonnet 5", "ox-alpha": "GLM-5.3-Flash (preview)", "grok-4.5": "Grok 4.5", "grok-4.6": "Grok 4.6", + "grok-4.7": "Grok 4.7", "deepseek-v4-pro": "DeepSeek V4 Pro", "deepseek-v4-pro-0813": "DeepSeek V4 Pro 0813", "deepseek-v4-flash-0731": "DeepSeek V4 Flash 0731", + "deepseek-v4.1-flash": "DeepSeek V4.1 Flash", "claude-opus-5": "Claude Opus 5", "gemini-3.8-flash": "Gemini 3.8 Flash", "gemini-3.7-flash": "Gemini 3.7 Flash", @@ -198,6 +311,14 @@ def reference_meta(self) -> dict: meta = json.loads((run_dir / "reference_outputs.csv.meta.json").read_text()) return meta["policyengine_bundles"]["us"] + @cached_property + def sample_bundle(self) -> dict: + """Runtime metadata of the scenario draw: the certified dataset build + the benchmark households were sampled from.""" + run_dir = SNAPSHOT_DIR / "runs" / self.us_run_label + meta = json.loads((run_dir / "scenarios.csv.meta.json").read_text()) + return meta["policyengine_bundles"]["us"] + @cached_property def model_stats(self) -> list[dict]: """No-tools model rows, ranked by the exact-match headline metric. @@ -331,28 +452,38 @@ def model_response_date(self) -> str: @property def policyengine_version(self) -> str: - """policyengine.py version that generated the reference outputs.""" + """policyengine.py version the reference sidecar records for provenance; + the references themselves come from policyengine_us.Simulation.""" return self.manifest["reference_output_refresh"]["policyengine_version"] @property def policyengine_us_version(self) -> str: return self.manifest["reference_output_refresh"]["policyengine_us_version"] + @property + def reference_rebuilt_date(self) -> str: + """UTC date the frozen references were last regenerated.""" + refresh = self.manifest["reference_output_refresh"] + return (refresh.get("regenerated_at_utc") or refresh["generated_at_utc"])[:10] + + # The dataset accessors describe the households' source: the certified + # build the scenario draw sampled (and the population weights use). The + # manifest's reference_output_refresh records the reference runtime's + # default dataset instead, which computing a household's references never + # reads and which can be a later build. @property def dataset_id(self) -> str: """Populace dataset name, e.g. ``populace_us_2024``.""" - return self.manifest["reference_output_refresh"]["policyengine_us_dataset"] + return self.sample_bundle["default_dataset"] @property def dataset_build_id(self) -> str: """Certified populace build id, e.g. ``populace-us-2024-5da5a95-20260611``.""" - return self.manifest["reference_output_refresh"][ - "policyengine_us_data_build_id" - ] + return self.sample_bundle["certified_data_build_id"] @property def dataset_uri(self) -> str: - return self.manifest["reference_output_refresh"]["policyengine_us_dataset_uri"] + return self.sample_bundle["default_dataset_uri"] @property def dataset_label(self) -> str: @@ -408,11 +539,39 @@ def serving_evidence_caption(self) -> str: row["evidence"]["kind"] == "registry" for row in self.serving_config["models"].values() ) + # Newer fingerprints also pin the reasoning setup and timeout, so the + # registry supplies them only for the rows whose fingerprint omits them. + registry_keys = { + "reasoning setup": "reasoning_setup", + "timeouts": "request_timeout_seconds", + } + run_state_rows = [ + row + for row in self.serving_config["models"].values() + if row["evidence"]["kind"] == "run_state" + ] + fully_pinned = sum( + all( + registry_keys[label] not in row["registry_derived"] + for label in field_labels["registry_for_run_state"] + ) + for row in run_state_rows + ) + if not fully_pinned: + return ( + f"Supervised-run fingerprints pin {fingerprint_counts}. " + f"{registry_fields.capitalize()} for every row, and all fields for " + f"the other {registry_count} rows, are the harness registry as " + "frozen in the snapshot's serving-configuration file." + ) + other_fingerprinted = len(run_state_rows) - fully_pinned return ( - f"Supervised-run fingerprints pin {fingerprint_counts}. " - f"{registry_fields.capitalize()} for every row, and all fields for " - f"the other {registry_count} rows, are the harness registry as frozen " - "in the snapshot's serving-configuration file." + f"Supervised-run fingerprints pin {fingerprint_counts}; " + f"{registry_fields} for {_sentence_count(fully_pinned).lower()} rows. " + f"{registry_fields.capitalize()} for the other " + f"{_sentence_count(other_fingerprinted).lower()} fingerprinted rows, " + f"and all fields for the other {registry_count} rows, are the harness " + "registry as frozen in the snapshot's serving-configuration file." ) @cached_property @@ -504,6 +663,47 @@ def n_households(self) -> int: def n_households_fmt(self) -> str: return f"{self.n_households:,}" + def _benchmark_people(self) -> list[dict]: + """Every person in the frozen US scenarios, with their prompt inputs.""" + run_dir = SNAPSHOT_DIR / "runs" / self.us_run_label + scenarios = pd.read_csv(run_dir / "scenarios.csv") + people = [] + for text in scenarios["scenario_json"]: + scenario = json.loads(text) + people += scenario.get("adults", []) + scenario.get("children", []) + return people + + @property + def benchmark_person_count(self) -> int: + return len(self._benchmark_people()) + + @property + def disabled_person_count(self) -> int: + """People the prompt lists with the general ``is disabled`` fact.""" + return sum( + bool(person.get("inputs", {}).get("is_disabled")) + for person in self._benchmark_people() + ) + + @property + def program_disability_input_count(self) -> int: + """People carrying any program-specific disability input (the paper + says none do).""" + inputs = ( + "meets_ssi_disability_criteria", + "months_receiving_social_security_disability", + "is_permanently_and_totally_disabled", + "retired_on_total_disability", + "is_incapable_of_self_care", + "is_permanently_disabled_veteran", + "is_surviving_spouse_of_disabled_veteran", + "is_surviving_child_of_disabled_veteran", + ) + return sum( + any(person.get("inputs", {}).get(name) for name in inputs) + for person in self._benchmark_people() + ) + @property def n_output_groups(self) -> int: return self.manifest["scope"]["output_groups"]["us"] @@ -923,11 +1123,33 @@ def excluded_output_phrase(self) -> str: @property def excluded_outputs_by_input(self) -> dict[str, int]: + """Unlisted-input exclusions, counted by the input they turn on.""" counts: dict[str, int] = {} for entry in self.reference_exclusions: + if entry["reason_code"] != UNLISTED_INPUT: + continue counts[entry["unlisted_input"]] = counts.get(entry["unlisted_input"], 0) + 1 return counts + @property + def excluded_outputs_by_root_cause(self) -> dict[str, int]: + """Engine-defect exclusions, counted by root cause.""" + counts: dict[str, int] = {} + for entry in self.reference_exclusions: + if entry["reason_code"] != ENGINE_DEFECT: + continue + key = exclusion_basis(entry) + counts[key] = counts.get(key, 0) + 1 + return counts + + @property + def unlisted_input_exclusion_count(self) -> int: + return sum(self.excluded_outputs_by_input.values()) + + @property + def engine_defect_exclusion_count(self) -> int: + return sum(self.excluded_outputs_by_root_cause.values()) + @property def excluded_output_households_fmt(self) -> str: return f"{len({e['scenario_id'] for e in self.reference_exclusions}):,}" @@ -998,6 +1220,11 @@ def audit_opus_judged_case_count_fmt(self) -> str: entry = self.audit_judge_provenance["by_judge"]["claude-opus-5"] return f"{entry['cases']:,}" + @property + def audit_opus55_judged_case_count_fmt(self) -> str: + entry = self.audit_judge_provenance["by_judge"]["claude-opus-5-5"] + return f"{entry['cases']:,}" + @property def audit_sol_judged_case_count_fmt(self) -> str: entry = self.audit_judge_provenance["by_judge"]["gpt-5.6-sol"] @@ -1008,6 +1235,380 @@ def audit_developer_adjudications(self) -> dict: """Manifest summary of recorded developer adjudications.""" return self.manifest["audit_annotation_artifacts"]["developer_adjudications"] + @property + def audit_flagged_by_verdict(self) -> dict[str, int]: + """Judge-flagged cases by the developer's reference verdict.""" + return self.audit_developer_adjudications.get( + "judge_flagged_by_reference_verdict", {} + ) + + @property + def audit_flagged_case_count(self) -> int: + return sum(self.audit_flagged_by_verdict.values()) + + def audit_flagged_count(self, verdict: str) -> int: + return self.audit_flagged_by_verdict.get(verdict, 0) + + @property + def engine_defect_unflagged_count(self) -> int: + """Engine-defect exclusions no judge flagged: the fix sweeps found them.""" + return self.engine_defect_exclusion_count - self.audit_flagged_count( + "engine_defect" + ) + + @property + def snap_engine_defect_exclusion_count(self) -> int: + """SNAP outputs excluded for an engine defect (the SNAP rounding defects).""" + return sum( + 1 + for e in self.reference_exclusions + if e["reason_code"] == ENGINE_DEFECT and e["variable"] == "snap" + ) + + @cached_property + def fable51_auto_uplift_fmt(self) -> str: + """Claude Fable 5.1's tool_choice auto sensitivity minus its board row.""" + summary = json.loads( + ( + ROOT / "sensitivity" / "data" / "claude-fable-5-1-thinking.json" + ).read_text() + ) + return f"{summary['delta_exact']:.1f}" + + @property + def excluded_output_households_by_reason(self) -> dict[str, int]: + households: dict[str, set[str]] = {} + for entry in self.reference_exclusions: + households.setdefault(entry["reason_code"], set()).add(entry["scenario_id"]) + return {reason: len(ids) for reason, ids in households.items()} + + @cached_property + def reference_revisions(self) -> list[dict]: + """Revisions recorded in the frozen reference sidecar (oldest first).""" + run_dir = SNAPSHOT_DIR / "runs" / self.us_run_label + meta = json.loads((run_dir / "reference_outputs.csv.meta.json").read_text()) + return meta.get("revisions", []) + + @property + def regenerated_reference_keys(self) -> set[tuple[str, str]]: + """Scored outputs a convention or an upstream fix regenerated. + + These are the September 22 regenerations, made on the engine version + before the upgrade; the engine upgrade's own changes are counted by the + ``engine_upgrade_*`` properties. + """ + return { + (change["scenario_id"], change.get("variable", "snap")) + for revision in self.reference_revisions + if revision.get("kind", "convention") in {"convention", "upstream_fix"} + for change in revision["changed"] + } + + @cached_property + def engine_upgrade_revision(self) -> dict | None: + """The sidecar revision that moved the references to a newer engine.""" + upgrades = [ + revision + for revision in self.reference_revisions + if revision.get("kind") == "engine_upgrade" + ] + return upgrades[-1] if upgrades else None + + @property + def previous_policyengine_us_version(self) -> str: + """policyengine-us version behind the references before the upgrade, + the version the September 22 audit ran on.""" + revision = self.engine_upgrade_revision + if revision is None: + return self.policyengine_us_version + return revision["previous_engine_version"].removeprefix("policyengine-us ") + + @property + def engine_upgrade_date(self) -> str: + """UTC date PolicyBench rebuilt the references on the new engine: the + sidecar's ``regenerated_at_utc`` day, which is also the revision's own + ``date`` (tests/test_paper_results.py checks they agree).""" + if self.engine_upgrade_revision is None: + return "" + return self.reference_rebuilt_date + + def _engine_upgrade_changes(self) -> list[dict]: + revision = self.engine_upgrade_revision + return [] if revision is None else revision["changed"] + + @cached_property + def engine_upgrade_partition(self) -> dict[str, list[dict]]: + """The upgrade's changed outputs, split into scored changes beyond the + exact-match tolerance, scored changes within it, and new exclusions.""" + return partition_engine_upgrade_changes( + self._engine_upgrade_changes(), self._excluded_output_keys + ) + + @property + def engine_upgrade_scored_change_count(self) -> int: + """Scored references the upgrade moved beyond the exact-match tolerance.""" + return len(self.engine_upgrade_partition["scored_changes"]) + + @property + def engine_upgrade_within_tolerance_count(self) -> int: + """Scored references the upgrade moved within the $1 tolerance.""" + return len(self.engine_upgrade_partition["within_tolerance"]) + + @property + def engine_upgrade_new_exclusion_count(self) -> int: + """Outputs scored before the upgrade that it removed from scoring.""" + return len(self.engine_upgrade_partition["new_exclusions"]) + + @property + def excluded_outputs_by_engine_version(self) -> dict[str, int]: + """Excluded outputs by the policyengine-us version behind the value + each keeps: the version its exclusion was decided on.""" + return dict( + Counter( + entry["engine_version"].removeprefix("policyengine-us ") + for entry in self.reference_exclusions + ) + ) + + @property + def excluded_outputs_on_previous_engine_count(self) -> int: + return self.excluded_outputs_by_engine_version.get( + self.previous_policyengine_us_version, 0 + ) + + @property + def excluded_outputs_on_reference_engine_count(self) -> int: + return self.excluded_outputs_by_engine_version.get( + self.policyengine_us_version, 0 + ) + + @cached_property + def publication_check_policyengine_us_version(self) -> str: + """policyengine-us release of the sweep that rechecked every reference + before publication (reference_audit/2026-09-28/verification).""" + with PUBLICATION_CHECK_SWEEP.open(newline="") as source: + engines = {row["engine"] for row in csv.DictReader(source)} + if len(engines) != 1: + raise ValueError(f"{PUBLICATION_CHECK_SWEEP} mixes engines: {engines}") + return engines.pop() + + @cached_property + def sweep_timing(self) -> dict: + """reference_audit/2026-09-28/verification/sweep_timing.json.""" + return json.loads(SWEEP_TIMING.read_text()) + + @property + def reference_engine_uploaded_utc(self) -> str: + """PyPI upload time (UTC, HH:MM) of the reference engine's wheel.""" + uploaded = self.sweep_timing["pypi"]["wheel_uploaded_at_utc"] + return uploaded[self.policyengine_us_version][11:16] + + @property + def publication_check_pypi_read_date(self) -> str: + """UTC day PolicyBench read PyPI for the publication check.""" + pypi = self.sweep_timing["pypi"] + if pypi["newest_at_read"] != self.publication_check_policyengine_us_version: + raise ValueError( + f"{SWEEP_TIMING} names {pypi['newest_at_read']} as newest, but the " + f"check ran {self.publication_check_policyengine_us_version}" + ) + return pypi["read_at_utc"][:10] + + @property + def publication_check_pypi_read_utc(self) -> str: + """Time (UTC, HH:MM) PolicyBench read PyPI for the publication check.""" + return self.sweep_timing["pypi"]["read_at_utc"][11:16] + + @cached_property + def rerun_sweeps(self) -> dict: + """reference_audit/2026-09-28/verification/rerun_sweeps.json.""" + return json.loads(RERUN_SWEEPS.read_text()) + + @cached_property + def rerun_sweep_partition(self) -> dict[str, list[tuple[str, str, str]]]: + return partition_rerun_sweep_moves( + self.rerun_sweeps, self._excluded_output_keys + ) + + @property + def rerun_sweep_september_22_count(self) -> int: + """Sweeps of the September 22 audit re-run on the reference engine.""" + return sum( + 1 + for sweep in self.rerun_sweeps["sweeps"] + if sweep["september_22_root_cause"] is not None + ) + + @property + def rerun_sweep_september_22_count_word(self) -> str: + return NUMBER_WORDS[self.rerun_sweep_september_22_count] + + @property + def rerun_sweep_new_count(self) -> int: + """Sweeps first run on the reference engine.""" + return sum( + 1 + for sweep in self.rerun_sweeps["sweeps"] + if sweep["september_22_root_cause"] is None + ) + + @property + def rerun_sweep_scored_beyond_tolerance_count(self) -> int: + return len(self.rerun_sweep_partition["scored_beyond_tolerance"]) + + @property + def rerun_sweep_scored_within_tolerance_count(self) -> int: + return len(self.rerun_sweep_partition["scored_within_tolerance"]) + + @property + def rerun_sweep_scored_within_tolerance_count_word(self) -> str: + return NUMBER_WORDS[self.rerun_sweep_scored_within_tolerance_count] + + @cached_property + def rerun_sweep_new_excluded_outputs(self) -> list[tuple[str, str]]: + """Outputs that a sweep first run on the reference engine moves + beyond the exact-match tolerance, against its own baseline, and that + the exclusion record did not hold before the upgrade (no exclusion + decided on the previous engine).""" + previous = f"policyengine-us {self.previous_policyengine_us_version}" + held_before = { + (entry["scenario_id"], entry["variable"]) + for entry in self.reference_exclusions + if entry["engine_version"] == previous + } + return sorted( + { + (move["scenario_id"], move["variable"]) + for sweep in self.rerun_sweeps["sweeps"] + if sweep["september_22_root_cause"] is None + for move in sweep["moves"] + if (move["scenario_id"], move["variable"]) not in held_before + and moves_beyond_tolerance( + move["variable"], move["baseline"], move["recomputed"] + ) + } + ) + + @property + def rerun_sweep_new_excluded_count_word(self) -> str: + return NUMBER_WORDS[len(self.rerun_sweep_new_excluded_outputs)] + + @property + def engine_upgrade_rechecked_count(self) -> int: + """Excluded outputs whose value moved on the new engine and were + re-reviewed; they stay excluded.""" + revision = self.engine_upgrade_revision + return 0 if revision is None else len(revision["excluded_outputs_rechecked"]) + + def _regenerated_keys_of_kind(self, kind: str) -> set[tuple[str, str]]: + return { + (change["scenario_id"], change.get("variable", "snap")) + for revision in self.reference_revisions + if revision.get("kind", "convention") == kind + for change in revision["changed"] + } + + @property + def regenerated_reference_count(self) -> int: + return len(self.regenerated_reference_keys) + + @property + def regenerated_snap_reference_count(self) -> int: + return sum(1 for _, v in self.regenerated_reference_keys if v == "snap") + + @property + def regenerated_reference_household_count(self) -> int: + return len({scenario_id for scenario_id, _ in self.regenerated_reference_keys}) + + @property + def regenerated_non_snap_reference_count(self) -> int: + return sum( + 1 for _, variable in self.regenerated_reference_keys if variable != "snap" + ) + + @property + def regenerated_by_upstream_fix_count(self) -> int: + """References regenerated with a fix merged upstream after the freeze.""" + return len(self._regenerated_keys_of_kind("upstream_fix")) + + @property + def regenerated_by_convention_count(self) -> int: + return len(self._regenerated_keys_of_kind("convention")) + + @property + def upstream_fixed_root_causes(self) -> list[str]: + return sorted( + r["root_cause"] + for r in self.reference_revisions + if r.get("kind") == "upstream_fix" + ) + + @property + def upstream_fixed_root_cause_count(self) -> int: + return len(self.upstream_fixed_root_causes) + + @property + def upstream_fix_prs_fmt(self) -> str: + """The policyengine-us pull requests behind the upstream fixes, in order.""" + prs = sorted( + { + int(match) + for r in self.reference_revisions + if r.get("kind") == "upstream_fix" + for match in re.findall(r"policyengine-us#(\d+)", r["upstream"]) + } + ) + labels = [f"#{n}" for n in prs] + return ( + ", ".join(labels[:-1]) + f" and {labels[-1]}" + if len(labels) > 1 + else "".join(labels) + ) + + @property + def regenerated_references_by_source(self) -> dict[str, int]: + """Regenerated references per convention or upstream fix (may be none).""" + return { + revision.get("convention") or revision.get("root_cause"): len( + revision["changed"] + ) + for revision in self.reference_revisions + } + + @property + def publication_convention_count(self) -> int: + """Conventions in the reference sidecar, including any that moved no output.""" + return sum( + 1 + for r in self.reference_revisions + if r.get("kind", "convention") == "convention" + ) + + @property + def later_law_exclusion_count(self) -> int: + return sum( + 1 for e in self.reference_exclusions if e["reason_code"] == LATER_LAW + ) + + @property + def engine_defect_root_cause_count(self) -> int: + """Distinct root causes behind the engine-defect exclusions.""" + causes: set[str] = set() + for entry in self.reference_exclusions: + if entry["reason_code"] == ENGINE_DEFECT: + causes.update(exclusion_basis(entry).split("+")) + return len(causes) + + @property + def excluded_descriptive_row_count(self) -> int: + """Annotated rows on excluded outputs that carry a descriptive class.""" + return sum( + 1 + for row in self._excluded_output_annotation_rows + if row["failure_source"] + in {"prompt_ambiguity", "reference_engine_defect", "reference_later_law"} + ) + @property def audit_adjudicated_case_count_fmt(self) -> str: return f"{self.audit_developer_adjudications['cases']:,}" @@ -1139,6 +1740,13 @@ def top_failure_subtypes(self, n: int = 3) -> str: # platform.claude.com/docs/en/models/fable-5-1/overview ("Released # September 1, 2026") "claude-fable-5.1": "2026-09-01", + # Models API created_at 2026-09-21 (api.anthropic.com/v1/models, read + # 2026-09-22) + "claude-opus-5.5": "2026-09-21", + # platform.claude.com/docs/en/models/sonnet-5-5/overview ("Released + # September 28, 2026"); Models API created_at 2026-09-28 + # (api.anthropic.com/v1/models, read 2026-09-28) + "claude-sonnet-5.5": "2026-09-28", # anthropic.com/news/claude-fable-5-mythos-5 (2026-06-09) "claude-fable-5": "2026-06-09", # announced and available 2026-07-24 (fortune.com, bloomberg.com, @@ -1189,12 +1797,23 @@ def top_failure_subtypes(self, n: int = 3) -> str: # day (en.wikipedia.org/wiki/GPT-6_Astra citing Japan Today 2026-09-04); # the trusted-partner day is excluded under the public-availability rule "gpt-6-astra": "2026-09-04", + # openai.com/index/introducing-gpt-6-sol-and-luna (2026-09-22; API and + # ChatGPT availability the same day per techcrunch.com 2026-09-22) + "gpt-6-sol": "2026-09-22", + # OpenAI announced GPT-6.1 Sol at DevDay on 2026-09-29 (API id gpt-6.1-sol). + "gpt-6.1-sol": "2026-09-29", + "gpt-6-luna": "2026-09-22", # piunikaweb.com 2026-04-17 SuperGrok beta (paid public tier) "grok-4.3": "2026-04-17", # x.ai/news/grok-4-5; techcrunch.com 2026-07-08 "grok-4.5": "2026-07-08", # x.ai/news/grok-4-6 (2026-08-12) "grok-4.6": "2026-08-12", + # x.ai/news/grok-4-7 (2026-09-21; the post puts the model in Cursor, Grok + # Build and the Grok API that day). The API's language-models record + # carries created 2026-09-02; like grok-4.6's (created 2026-08-06), that + # predates the public launch and is not a release date. + "grok-4.7": "2026-09-21", # API public beta per secondary trackers (bighatgroup.com xai-weekly # 2026-06-03); no vendor-dated announcement exists "grok-build-0.1": "2026-05-29", @@ -1207,6 +1826,10 @@ def top_failure_subtypes(self, n: int = 3) -> str: # unsloth.ai/docs/models/deepseek-v4 (2026-07-31, 2026-08-13) "deepseek-v4-flash-0731": "2026-07-31", "deepseek-v4-pro-0813": "2026-08-13", + # api-docs.deepseek.com/news/news260910 ("DeepSeek-V4.1-Flash Release + # 2026/09/10", live on the API as deepseek-flash; MIT weights at + # huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, created 2026-09-10) + "deepseek-v4.1-flash": "2026-09-10", # verdent.ai kimi-k2.6 guide; huggingface.co/moonshotai/Kimi-K2.6 "kimi-k2.6": "2026-04-20", # simonwillison.net/2026/Jul/16/kimi-k3 (API launch; weights announced @@ -1240,9 +1863,11 @@ def top_failure_subtypes(self, n: int = 3) -> str: # Models whose weights are publicly downloadable. GLM-5.3's weights shipped # 2026-08-28 under the GLM-5.3 License (MIT-style with a security-review # condition for the largest Model-as-a-Service providers), two weeks after -# its API launch; the dated DeepSeek V4 checkpoints are MIT. Ox Alpha is not -# marked: the preview checkpoint itself was never published, and Z.ai's later -# identification of it as GLM-5.3-Flash is not a weights release of that row. +# its API launch; the dated DeepSeek V4 checkpoints and DeepSeek V4.1 Flash +# (huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, 2026-09-10) are MIT. +# Ox Alpha is not marked: the preview checkpoint itself was never +# published, and Z.ai's later identification of it as GLM-5.3-Flash is +# not a weights release of that row. # Qwen 3.7 Max and 3.8 Max # are API-only as of 2026-08-03 (3.8's weights are promised but unpublished); # Kimi K3's weights shipped on Hugging Face 2026-07-26/27 under a custom @@ -1253,6 +1878,7 @@ def top_failure_subtypes(self, n: int = 3) -> str: "deepseek-v4-flash", "deepseek-v4-flash-0731", "deepseek-v4-pro-0813", + "deepseek-v4.1-flash", "kimi-k2.6", "kimi-k3", "glm-5.2", diff --git a/policybench/policyengine_runtime.py b/policybench/policyengine_runtime.py index 49876fbb..a60e43b0 100644 --- a/policybench/policyengine_runtime.py +++ b/policybench/policyengine_runtime.py @@ -1,8 +1,12 @@ """PolicyEngine runtime provenance and model wiring.""" +import contextlib +import hashlib import json import os +import platform import re +import sys from functools import lru_cache from importlib import metadata from pathlib import Path @@ -13,6 +17,28 @@ "uk": "policyengine-uk", } +# A supervised run computes PolicyEngine provenance once and hands it to its +# per-scenario workers through this file, so a worker that only calls an LLM +# never imports policyengine. +POLICYENGINE_PROVENANCE_ENV = "POLICYBENCH_POLICYENGINE_PROVENANCE" +POLICYENGINE_PROVENANCE_FILENAME = "policyengine_provenance.json" +POLICYENGINE_PROVENANCE_FORMAT_VERSION = 1 +# Logged by a worker that could not reuse the file; the supervisor counts it. +POLICYENGINE_PROVENANCE_NOT_REUSED = "PolicyEngine provenance file not reused" +# Distributions whose metadata policyengine_release_bundle reads or whose code +# runs when it imports policyengine. +PROVENANCE_DISTRIBUTIONS = ( + "policyengine", + "policyengine-core", + "policyengine-us", + "policyengine-uk", +) +# policyengine.py's bundled release manifests, relative to its distribution. +# 4.16.1 ships one per country; 6.1.2 ships one bundle manifest that keeps each +# country's release under data_releases. +RELEASE_MANIFEST_PATH = "policyengine/data/release_manifests/{country}.json" +BUNDLE_MANIFEST_PATH = "policyengine/data/bundle/manifest.json" + DATA_PACKAGES = { "us": "policyengine-us-data", "uk": "policyengine-uk-data", @@ -237,11 +263,19 @@ def _load_raw_policyengine_manifest(country: str) -> dict[str, Any] | None: except metadata.PackageNotFoundError: return None manifest_path = Path( - distribution.locate_file(f"policyengine/data/release_manifests/{country}.json") + distribution.locate_file(RELEASE_MANIFEST_PATH.format(country=country)) ) - if not manifest_path.exists(): - return None - return json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest_path.exists(): + return json.loads(manifest_path.read_text(encoding="utf-8")) + # policyengine.py 6.x ships one bundle manifest; each country's data release + # keeps the per-country manifest's shape under data_releases. + bundle_path = Path(distribution.locate_file(BUNDLE_MANIFEST_PATH)) + if bundle_path.exists(): + bundle = json.loads(bundle_path.read_text(encoding="utf-8")) + release = (bundle.get("data_releases") or {}).get(country) + if isinstance(release, dict): + return release + return None def _default_dataset_uri_from_raw_manifest(raw_manifest: dict[str, Any]) -> str | None: @@ -389,6 +423,207 @@ def policyengine_bundles_for_countries(countries: set[str] | list[str]) -> dict: return bundles +def _sha256_or_none(path: Path) -> str | None: + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + + +# Hashed at import, so a fingerprint describes the code this process runs even +# if the file on disk changes afterwards. +_RUNTIME_SOURCE_SHA256 = _sha256_or_none(Path(__file__)) + + +def _distribution_inputs(name: str) -> dict[str, Any]: + try: + metadata_text = metadata.distribution(name).read_text("METADATA") + except metadata.PackageNotFoundError: + metadata_text = None + return { + "version": _package_version_or_none(name), + "direct_url": _package_direct_url_or_none(name), + "metadata_sha256": ( + hashlib.sha256(metadata_text.encode("utf-8")).hexdigest() + if metadata_text is not None + else None + ), + } + + +def _packaged_file_sha256(distribution: Any, relative_path: str) -> str | None: + if distribution is None: + return None + return _sha256_or_none(Path(distribution.locate_file(relative_path))) + + +def policyengine_provenance_inputs() -> dict[str, Any]: + """Fingerprint the environment ``policyengine_bundles_for_countries`` sees. + + Reads package metadata, files and environment flags only; importing + policyengine is the cost this exists to avoid. Recorded directly: each + PolicyEngine package's version, install URL and METADATA; the hash of + each release manifest file, in either layout policyengine.py ships (one + file per country, or one bundle manifest holding every country's + release), with None for each file the installed layout lacks; this + module; and the import flags. The environment holding everything + else (pydantic, requests, ...) is identified by ``sys.prefix``, though an + in-place upgrade of those packages is not detected. A provenance file is + reused only when its recorded fingerprint equals the reader's, so a + worker whose environment differs from the writer's recomputes instead. + """ + try: + policyengine_distribution = metadata.distribution("policyengine") + except metadata.PackageNotFoundError: + policyengine_distribution = None + release_manifests = { + country: _packaged_file_sha256( + policyengine_distribution, RELEASE_MANIFEST_PATH.format(country=country) + ) + for country in sorted(MODEL_PACKAGES) + } + return { + "python_version": platform.python_version(), + "python_prefix": sys.prefix, + "distributions": { + name: _distribution_inputs(name) for name in PROVENANCE_DISTRIBUTIONS + }, + "release_manifest_sha256": release_manifests, + "bundle_manifest_sha256": _packaged_file_sha256( + policyengine_distribution, BUNDLE_MANIFEST_PATH + ), + "policyengine_runtime_sha256": _RUNTIME_SOURCE_SHA256, + # Whether ``import policyengine`` succeeds decides which branch of + # policyengine_release_bundle records the US bundle. The skip flag + # stops policyengine/__init__.py importing the country models, and + # the token is sent with the Hugging Face request those imports make. + # Only the token's presence is recorded, never its value. + "environment": { + "POLICYENGINE_SKIP_COUNTRY_IMPORTS": os.environ.get( + "POLICYENGINE_SKIP_COUNTRY_IMPORTS" + ), + "HUGGING_FACE_TOKEN_set": bool(os.environ.get("HUGGING_FACE_TOKEN")), + }, + } + + +def _editable_provenance_distributions() -> list[str]: + """Editable installs change source without changing their fingerprint.""" + editable = [] + for name in PROVENANCE_DISTRIBUTIONS: + direct_url = _package_direct_url_or_none(name) or {} + if (direct_url.get("dir_info") or {}).get("editable"): + editable.append(name) + return editable + + +def _provenance_warning(message: str) -> None: + print(message, file=sys.stderr) + + +def write_policyengine_provenance( + path: str | Path, countries: set[str] | list[str] +) -> bool: + """Compute PolicyEngine bundles once and write them for workers to reuse. + + Returns True only when the written file reproduces the computation + exactly. Otherwise no file is left at ``path`` and False is returned, so + callers leave workers to compute provenance themselves. + """ + path = Path(path) + editable = _editable_provenance_distributions() + if editable: + path.unlink(missing_ok=True) + _provenance_warning( + f"PolicyEngine provenance file {path} not written: editable install " + f"of {', '.join(editable)}; workers will compute it themselves." + ) + return False + bundles = policyengine_bundles_for_countries(countries) + payload = { + "format_version": POLICYENGINE_PROVENANCE_FORMAT_VERSION, + "inputs": policyengine_provenance_inputs(), + "policyengine_bundles": bundles, + } + # Replace atomically: a worker left running by an earlier supervisor may + # read this file while a restarted supervisor rewrites it. + tmp_path = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path.write_text( + json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8" + ) + os.replace(tmp_path, path) + except OSError as error: + with contextlib.suppress(OSError): + tmp_path.unlink(missing_ok=True) + _provenance_warning( + f"PolicyEngine provenance file {path} not written " + f"({type(error).__name__}); workers will compute it themselves." + ) + return False + reread, reason = _read_policyengine_provenance(path, sorted(bundles)) + if reread != bundles: + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) + _provenance_warning( + f"PolicyEngine provenance file {path} removed: " + f"{reason or 'it does not round-trip exactly'}; workers will " + "compute it themselves." + ) + return False + return True + + +def _read_policyengine_provenance( + path: Path, countries: list[str] +) -> tuple[dict | None, str | None]: + """Return (bundles, None) from a reusable file, else (None, reason).""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + return None, f"could not read it ({type(error).__name__})" + if not isinstance(payload, dict): + return None, "it is not a JSON object" + if payload.get("format_version") != POLICYENGINE_PROVENANCE_FORMAT_VERSION: + return None, f"format_version {payload.get('format_version')!r} differs" + if payload.get("inputs") != policyengine_provenance_inputs(): + return None, "it was written for different PolicyEngine packages or code" + stored = payload.get("policyengine_bundles") + if not isinstance(stored, dict): + return None, "it has no policyengine_bundles object" + missing = [country for country in countries if country not in stored] + if missing: + return None, f"it has no bundle for {', '.join(missing)}" + return {country: stored[country] for country in countries}, None + + +def resolve_policyengine_bundles( + countries: set[str] | list[str], env: dict | None = None +) -> dict: + """Return ``policyengine_bundles_for_countries(countries)``, cheaply if possible. + + When ``POLICYBENCH_POLICYENGINE_PROVENANCE`` names a file that + ``write_policyengine_provenance`` wrote under the same fingerprint, its + bundles are returned without importing policyengine. In every other case + (unset, unreadable, fingerprint mismatch, or a missing country) the + bundles are computed in this process, as they were before the file + existed. + """ + source = os.environ if env is None else env + path = source.get(POLICYENGINE_PROVENANCE_ENV) + if path: + wanted = sorted({country.lower() for country in countries}) + bundles, reason = _read_policyengine_provenance(Path(path), wanted) + if bundles is not None: + return bundles + _provenance_warning( + f"{POLICYENGINE_PROVENANCE_NOT_REUSED} ({path}): {reason}; " + "computing it in this process." + ) + return policyengine_bundles_for_countries(countries) + + def runtime_metadata_for_country( country: str, *, diff --git a/policybench/prompt_contract_v2.py b/policybench/prompt_contract_v2.py new file mode 100644 index 00000000..30a58360 --- /dev/null +++ b/policybench/prompt_contract_v2.py @@ -0,0 +1,833 @@ +"""Opt-in US household prompt contract 2.1.0; no evaluator or v1 integration. + +This module renders supplied facts, not policy results. It uses no model registry, +simulation, network, or v1 prompt helpers. Unknown inputs remain visible and block +any claim that the result is a complete evaluation contract. See +``docs/prompt_contract_v2.md`` for the deliberately limited supported surface. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from policybench.scenarios import Scenario + +CONTRACT_VERSION = "2.1.0" + +# Proposed wording from the published spec, isolated from its v1 byte fingerprint. +_V2_OUTPUT_DEFINITIONS = { + "federal_income_tax_before_refundable_credits": "federal individual income tax " + "after nonrefundable credits and " + "before refundable credits. This " + "subtracts nonrefundable credits " + "actually used, including CDCC and " + "the nonrefundable portion of CTC " + "or other credits when applicable; " + "it does not subtract EITC or " + "refundable portions of credits " + "such as refundable CTC. Do not add " + "a dependent's separate income tax " + "return. Retain dependent-related " + "provisions on the specified " + "tax-unit return.", + "federal_refundable_credits": "total refundable federal income tax credits, " + "including EITC and refundable portions of credits " + "such as refundable CTC when applicable; exclude the " + "ACA Premium Tax Credit. Do not add a dependent's " + "separate income tax return. Retain dependent-related " + "provisions on the specified tax-unit return.", + "local_income_tax": "annual local income, wage, and earnings tax liability in the " + "separate local-income-tax output: NYC income tax, Philadelphia " + "wage tax, Kansas City earnings tax, and St. Louis earnings tax " + "where applicable. Do not add a dependent's separate income tax " + "return. Retain dependent-related provisions on the specified " + "tax-unit return. Explicitly applicable local wage or earnings " + "taxes include any listed person's taxable wages or earnings.", + "payroll_tax": "annual household employee-side payroll tax across all listed " + "people, including dependent wages: employee Social Security tax, " + "employee Medicare tax, Additional Medicare Tax, mandatory employee " + "state payroll taxes, and employee shares the employer chooses to " + "withhold under the stated state paid-leave/disability contribution " + "convention. Exclude shares paid by the employer, employer payroll " + "taxes, FUTA, employer unemployment-insurance taxes, and " + "self-employment tax", + "state_income_tax_before_refundable_credits": "state individual income tax after " + "nonrefundable credits and before " + "refundable credits, excluding local " + "income and payroll taxes. Do not add " + "a dependent's separate income tax " + "return. Retain dependent-related " + "provisions on the specified tax-unit " + "return.", + "state_refundable_credits": "total refundable state individual income tax credits. " + "Do not add a dependent's separate income tax return. " + "Retain dependent-related provisions on the specified " + "tax-unit return.", +} + +# Explicit exceptions to the generic unlisted-zero rule. These names are also +# used by the report-only sensitivity check; rendering an unknown is not coverage. +STATED_CONVENTION_INPUTS = frozenset( + { + "state_withheld_income_tax", + "takes_up_medicare_if_eligible", + "medicare_part_b_premium", + "medical_expense_health_insurance_premiums", + "state_paid_leave_employee_share", + "weekly_hours_worked_before_lsr", + "state_sales_tax", + "local_sales_tax", + } +) + +TASK_PREFACE = ( + "Estimate requested outputs under the declared PolicyEngine-US version and " + "the calculation conventions below. All listed people live together in the " + "stated household group. Unless dated explicitly, facts apply throughout " + "the tax-benefit year without changes in status or income volatility; wage " + "amounts are annual totals including overtime, and hourly rates are " + "straight-time rates. Unlisted numeric inputs, including unlisted integer " + "inputs, are zero and unlisted boolean inputs are false, except for the " + "explicitly listed unknown facts and the named calculation conventions. " + "Unknown facts and unknown provenance are never observed negatives. A " + "supplied value with unknown provenance remains a supplied value, not an " + "observation; an unknown value must not be replaced with zero or false. " + "A general disability indicator is a survey characteristic and does not " + "establish SSI disability, tax-specific disability, SSDI entitlement, or " + "Medicare eligibility. SSI disability uses the separately stated criteria " + "before the substantial-gainful-activity test; apply the earnings test " + "separately. SNAP receipt-based disability, tax-specific disability, and " + "self-care conditions are separate facts. Medicare eligibility is evaluated " + "on January 1; SSDI benefit months are measured as of that date. " + "Assume filing and full take-up of eligible requested benefits (Medicare " + "enrollment can be explicitly overridden), plus " + "eligible TANF/MOE noncash benefits used for SNAP categorical eligibility, " + "and apply those computed benefits in downstream calculations. This " + "exception permits computed receipt; it does not create substantive " + "eligibility or historical entitlement. Social Security, SSDI, and veterans " + "payments and histories are supplied inputs only. Infer no other income, " + "expenses, assets, receipt, rent, or coverage. The SSI output includes " + "federal SSI only; state supplements require a separate requested output. " + "If no weekly-hours input is supplied, this contract assumes 0 " + "hours/week, not an observed zero. Do not derive hours from annual wages or " + "an hourly rate. Supplied null hours remain unknown. These conventions are " + "declared assumptions, not verified engine behavior. " + "Every input read by a scored reference whose change by a plausible amount " + "moves that reference by more than $1 must be covered by a stated fact or " + "an explicit convention. A generic zero default does not cover a computed " + "engine estimate. Legacy sweep findings report gaps; they do not certify " + "readiness. Do not subtract employer-paid or employee after-tax premiums " + "from FICA wages. Employee pre-tax health premiums reduce income-tax and " + "FICA wages under the declared model." +) + +# Explicit labels and types, independent of the mutable engine input registry. +# The source names are always printed; no field is silently filtered or aliased. +_DISABILITY_LABELS = { + "is_disabled": "General disability indicator (survey characteristic)", + "is_blind": "Blindness indicator (not a program-specific disability finding)", + "meets_ssi_disability_criteria": ( + "SSI disability criteria before the substantial-gainful-activity test" + ), + "is_usda_disabled": "SNAP receipt-based disability status", + "is_permanently_and_totally_disabled": ( + "Permanent and total disability for IRC 152/22" + ), + "is_incapable_of_self_care": "Incapable of self-care for IRC 21", +} +_HOURS_LABELS = { + "hours_worked_last_week": "Usual weekly hours worked", + "weekly_hours_worked": "Weekly hours worked", + "weekly_hours_worked_before_lsr": "Usual weekly hours before labor-supply response", +} +_PERSON_MONEY_LABELS = { + "employment_income": "Gross wages and salaries", + "employer_sponsored_insurance_premiums": ( + "Employer-paid insurance premiums (not included in stated wages)" + ), + "pre_tax_health_insurance_premiums": "Employee pre-tax health insurance premiums", + "health_insurance_premiums_without_medicare_part_b": ( + "Employee after-tax health insurance premiums excluding Part B" + ), + "health_insurance_premiums": ( + "Employee after-tax health insurance premiums including Part B" + ), + "other_health_insurance_premiums": ( + "Employee after-tax other health insurance premiums" + ), + "medicare_part_b_premium": "Employee after-tax annual Part B premium, net of MSP", + "medical_expense_health_insurance_premiums": ( + "Employee after-tax medical-expense health insurance premium total" + ), + "financial_assistance": "Cash financial assistance from the named outside source", + "social_security_disability": "Social Security disability income", + "social_security_retirement": "Social Security retirement income", + "social_security_dependents": "Social Security dependent benefits", + "social_security_survivors": "Social Security survivor benefits", + "veterans_benefits": "Veterans benefits", + "ssi_reported": "Reported SSI income (supplied receipt, not computed SSI)", + "disability_benefits": "Disability benefits (unspecified program)", + "self_employment_income": "Self-employment income", + "bank_account_assets": "Bank account assets", + "stock_assets": "Stock assets", + "pre_subsidy_rent": "Pre-subsidy rent", + "real_estate_taxes": "Real estate taxes", + "home_mortgage_interest": "Home mortgage interest", + "tip_income": "Tip income included in gross wages and salaries above", +} +_PERSON_BOOL_LABELS = { + "is_tax_unit_head": "Tax unit head", + "is_tax_unit_spouse": "Tax unit spouse", + "is_unmarried_partner_of_household_head": "Unmarried partner of household head", + "has_esi": "Has employer-sponsored insurance", + "takes_up_medicare_if_eligible": "Part B enrollment if Medicare-eligible", + "is_surviving_spouse": "Surviving spouse indicator (see deciding facts below)", + "dependent_child_lives_in_home": "Dependent child lives in the home", + "state_paid_leave_employee_share_withheld": "Employer withholds employee share", +} +_PERSON_TEXT_LABELS = { + "financial_assistance_source": "Source of cash financial assistance" +} +_TAX_UNIT_MONEY_LABELS = { + "state_withheld_income_tax": ( + "Annual state income tax paid during the year for SALT" + ), + "state_sales_tax": "Annual state sales tax paid for SALT", + "local_sales_tax": "Annual local sales tax paid for SALT", +} +_OPTIONAL_PAYROLL_PROGRAMS = { + "MN": "MN Paid Leave", + "CO": "CO FAMLI", + "MA": "MA PFML", + "NY": "NY PFL/DBL", + "DE": "DE Paid Leave", + "ME": "ME PFML", + "VT": "VT child-care contribution", + "WA": "WA PFML", +} +_DURATION = "months_receiving_social_security_disability" +_PROVENANCE_FIELDS = frozenset(_DISABILITY_LABELS) | { + _DURATION, + "social_security_disability", +} +_PERSON_FIELDS = ( + _PROVENANCE_FIELDS + | _HOURS_LABELS.keys() + | _PERSON_MONEY_LABELS.keys() + | _PERSON_BOOL_LABELS.keys() + | _PERSON_TEXT_LABELS.keys() + | {"age", "hourly_wage", "spouse_death_year"} +) +# The existing Scenario defaults are evidence for these explicit true-only +# inputs; accepting false here would contradict this contract's fixed preamble. +_TAKEUP_ENTITIES = { + "takes_up_medicaid_if_eligible": "person", + "takes_up_ssi_if_eligible": "person", + "takes_up_aca_if_eligible": "tax_unit", + "takes_up_dc_ptc": "tax_unit", + "takes_up_eitc": "tax_unit", + "would_file_if_eligible_for_refundable_credit": "tax_unit", + "would_file_taxes_voluntarily": "tax_unit", + "takes_up_snap_if_eligible": "spm_unit", +} +_STATES = frozenset( + "AL AK AZ AR CA CO CT DE DC FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN " + "MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA " + "WV WI WY".split() +) +_FILING_STATUSES = { + "single": "Single", + "joint": "Joint", + "head_of_household": "Head of household", +} + + +class ContractInputError(ValueError): + """An input cannot be represented faithfully under this contract.""" + + +def _text(value: object, path: str) -> str: + if ( + type(value) is not str + or not value.strip() + or any(ord(char) < 32 or ord(char) == 127 for char in value) + or value.splitlines() != [value] + ): + raise ContractInputError(f"{path}: expected non-empty single-line text") + return value + + +def _mapping(value: object, path: str) -> Mapping: + if not isinstance(value, Mapping): + raise ContractInputError(f"{path}: expected a mapping") + for key in value: + if type(key) is not str or re.fullmatch(r"[a-z][a-z0-9_]*", key) is None: + raise ContractInputError(f"{path}: invalid input or person name {key!r}") + return value + + +def _number(value: object, path: str, *, integer: bool = False) -> int | float: + valid_types = (int,) if integer else (int, float) + if type(value) not in valid_types or ( + type(value) is float and not math.isfinite(value) + ): + expected = "integer" if integer else "finite number" + raise ContractInputError(f"{path}: expected {expected}, without coercion") + return value + + +def _numeric_text(value: int | float) -> str: + # repr preserves float precision, including sub-dollar and fractional hours. + # Add grouping only to the whole part; never round or convert ints to floats. + raw = str(value) + if "e" in raw.lower(): + return raw + whole, dot, fraction = raw.partition(".") + grouped = "-0" if whole == "-0" else f"{int(whole):,}" + return f"{grouped}{dot}{fraction}" + + +def _scalar_json(value: object, path: str) -> str: + if value is not None and type(value) not in (bool, int, float, str): + raise ContractInputError(f"{path}: unsupported non-scalar input") + if type(value) is float and not math.isfinite(value): + raise ContractInputError(f"{path}: unsupported non-finite input") + return json.dumps(value, ensure_ascii=True, allow_nan=False) + + +@dataclass(frozen=True) +class FactProvenance: + """Caller-supplied provenance, not independently verified by this renderer. + + Observed and imputed claims require a non-empty source. Unknown may also + name a source documenting missingness. A source is a label, not fetched. + """ + + kind: str + source: str | None = None + + def __post_init__(self) -> None: + if type(self.kind) is not str or self.kind not in ( + "observed", + "imputed", + "unknown", + ): + raise ContractInputError( + "provenance kind must be observed, imputed, or unknown" + ) + if self.source is not None: + _text(self.source, "provenance source") + if self.kind != "unknown" and self.source is None: + raise ContractInputError(f"{self.kind} provenance requires a source") + + +@dataclass(frozen=True) +class RenderedHouseholdContract: + """Review artifact, with unresolved fact and unsupported-input paths. + + Even empty diagnostic tuples do not certify board readiness. This slice + defines neither output/answer schemas nor engine-compatible unit mappings. + """ + + text: str + contract_id: str + unknown_facts: tuple[str, ...] + unsupported_inputs: tuple[str, ...] + + +def contract_identity() -> str: + """Hash source bytes plus canonical v2 definitions, excluding mutable v1 text.""" + source = Path(__file__).read_bytes() + definitions = json.dumps(v2_output_definitions(), sort_keys=True).encode("utf-8") + digest = hashlib.sha256(source + b"\n" + definitions).hexdigest() + return f"policybench-us-household-prompt/{CONTRACT_VERSION}:sha256:{digest}" + + +def v2_output_definitions() -> dict[str, str]: + """Return opt-in proposed wording; never read or alter the published v1 spec.""" + return dict(_V2_OUTPUT_DEFINITIONS) + + +def _label(name: str) -> str: + if name in _DISABILITY_LABELS: + return _DISABILITY_LABELS[name] + if name == _DURATION: + return "SSDI benefit months as of January 1" + if name in _HOURS_LABELS: + return _HOURS_LABELS[name] + if name in _PERSON_MONEY_LABELS: + return _PERSON_MONEY_LABELS[name] + if name in _PERSON_BOOL_LABELS: + return _PERSON_BOOL_LABELS[name] + if name in _PERSON_TEXT_LABELS: + return _PERSON_TEXT_LABELS[name] + if name in _TAX_UNIT_MONEY_LABELS: + return _TAX_UNIT_MONEY_LABELS[name] + if name == "spouse_death_year": + return "Spouse death year" + if name == "hourly_wage": + return "Straight-time hourly wage" + if name == "age": + return "Age" + if name in _TAKEUP_ENTITIES: + return "Filing/take-up convention" + return "Raw input" + + +def _value_text(name: str, value: object, path: str) -> str: + if name in _PERSON_TEXT_LABELS: + return _text(value, path) + if name == "spouse_death_year": + year = _number(value, path, integer=True) + if not 1 <= year <= 9999: + raise ContractInputError(f"{path}: expected a calendar year") + return str(year) + if name in _DISABILITY_LABELS or name in _PERSON_BOOL_LABELS: + if type(value) is not bool: + raise ContractInputError(f"{path}: expected boolean, without coercion") + return "yes" if value else "no" + if name in _TAKEUP_ENTITIES: + if value is not True: + raise ContractInputError( + f"{path}: contradicts fixed filing/take-up convention" + ) + return "yes" + if name == _DURATION or name == "age": + number = _number(value, path, integer=True) + if number < 0: + raise ContractInputError(f"{path}: must be nonnegative") + unit = "months" if name == _DURATION else "years" + return f"{_numeric_text(number)} {unit}" + if name in _HOURS_LABELS: + number = _number(value, path) + if not 0 <= number <= 168: + raise ContractInputError(f"{path}: hours must be between 0 and 168") + return f"{_numeric_text(number)} hours/week" + if ( + name in _PERSON_MONEY_LABELS + or name in _TAX_UNIT_MONEY_LABELS + or name == "hourly_wage" + ): + number = _number(value, path) + if ( + "premiums" in name + or name + in ("medicare_part_b_premium", "hourly_wage", "financial_assistance") + or name in _TAX_UNIT_MONEY_LABELS + ) and number < 0: + raise ContractInputError(f"{path}: must be nonnegative") + suffix = "/hour" if name == "hourly_wage" else "" + return f"${_numeric_text(number)}{suffix}" + return _scalar_json(value, path) + + +def _render_inputs( + inputs: Mapping, + entity: str, + prefix: str, + provenance: Mapping, + unknown: set[str], + unsupported: set[str], +) -> list[str]: + names = set(inputs) + if entity == "person": + names.update(_PROVENANCE_FIELDS) + lines = [] + for name in sorted(names): + path = f"{prefix}.{name}" + if name in _PERSON_FIELDS and entity != "person": + raise ContractInputError(f"{path}: {name} is a person input") + if name in _TAX_UNIT_MONEY_LABELS and entity != "tax_unit": + raise ContractInputError(f"{path}: {name} is a tax_unit input") + if name in _TAKEUP_ENTITIES and entity != _TAKEUP_ENTITIES[name]: + raise ContractInputError( + f"{path}: incorrect entity for filing/take-up input" + ) + supplied = name in inputs + value = inputs.get(name) + supported = ( + name in _PERSON_FIELDS + or name in _TAKEUP_ENTITIES + or name in _TAX_UNIT_MONEY_LABELS + ) + fact_provenance = provenance.get(name, FactProvenance("unknown")) + if not supported: + unsupported.add(path) + if value is None: + if name in _TAKEUP_ENTITIES: + raise ContractInputError( + f"{path}: unknown filing/take-up contradicts convention" + ) + if fact_provenance.kind != "unknown": + raise ContractInputError( + f"{path}: {fact_provenance.kind} provenance without a value" + ) + unknown.add(path) + rendered = "unknown" + notes = [ + "supplied null" if supplied else "not supplied", + "provenance: unknown", + ] + else: + rendered = _value_text(name, value, path) + notes = [] + if name in _PROVENANCE_FIELDS: + if fact_provenance.kind == "unknown": + unknown.add(path) + notes.extend( + [ + "supplied value", + "provenance: unknown", + "not an observed fact", + ] + ) + else: + notes.append(f"provenance: {fact_provenance.kind}") + if fact_provenance.source is not None: + notes.append(f"source: {fact_provenance.source}") + if not supported: + notes.extend(["unsupported input", "meaning and units not interpreted"]) + suffix = f" ({'; '.join(notes)})" if notes else "" + lines.append(f"- {_label(name)} [{name}]: {rendered}{suffix}") + return lines + + +def render_household_contract( + scenario: Scenario, + *, + policyengine_us_version: str, + provenance: Mapping[str, Mapping[str, FactProvenance]] | None = None, +) -> RenderedHouseholdContract: + """Render an existing US Scenario without modifying it or computing outcomes. + + Provenance keys are exact person names and supported disability/history + input names. Missing provenance stays unknown; no source is inferred from + ``source_dataset`` or ``metadata``. Unsupported scalar inputs are printed + verbatim as JSON with an explicit marker. Invalid supported inputs raise. + """ + if ( + type(policyengine_us_version) is not str + or re.fullmatch(r"\d+\.\d+\.\d+", policyengine_us_version) is None + ): + raise ContractInputError( + "policyengine_us_version: expected an exact x.y.z version" + ) + if scenario.country != "us": + raise ContractInputError("country: this contract supports US scenarios only") + if type(scenario.state) is not str or scenario.state not in _STATES: + raise ContractInputError("state: unsupported US state code") + year = _number(scenario.year, "year", integer=True) + if not 1 <= year <= 9999: + raise ContractInputError("year: expected a calendar year from 1 to 9999") + if ( + type(scenario.filing_status) is not str + or scenario.filing_status not in _FILING_STATUSES + ): + raise ContractInputError("filing_status: unsupported or unknown filing status") + _text(scenario.id, "scenario id") + _text(scenario.source_dataset, "source dataset") + if type(scenario.adults) is not list or not scenario.adults: + raise ContractInputError("adults: expected a non-empty list") + if type(scenario.children) is not list: + raise ContractInputError("children: expected a list") + people = scenario.adults + scenario.children + person_names = [] + for person in people: + name = _text(person.name, "person name") + _mapping({name: None}, "person name") + if name in person_names: + raise ContractInputError(f"duplicate person name: {name}") + person_names.append(name) + provenance = {} if provenance is None else _mapping(provenance, "provenance") + for person_name, facts in provenance.items(): + if person_name not in person_names: + raise ContractInputError(f"provenance: unknown person {person_name}") + for name, fact in _mapping(facts, f"provenance.{person_name}").items(): + if name not in _PROVENANCE_FIELDS or not isinstance(fact, FactProvenance): + raise ContractInputError( + f"provenance.{person_name}.{name}: unsupported fact or provenance" + ) + identity = contract_identity() + lines = [ + f"Household prompt contract: {identity}", + "Status: opt-in, unactivated household-fact slice; " + "not a complete evaluation contract.", + f"Declared calculation version: policyengine-us {policyengine_us_version} " + "(caller-supplied; not verified).", + "", + TASK_PREFACE, + "", + "Household:", + f"- scenario: {scenario.id}", + f"- state: {scenario.state}", + f"- tax year: {year}", + f"- Medicare/SSDI duration reference date: {year:04d}-01-01", + f"- supplied filing status: {_FILING_STATUSES[scenario.filing_status]}", + f"- source dataset label (not fact provenance): {scenario.source_dataset}", + "- marital-unit mapping: not defined by this slice; " + "do not infer couples from person names", + "- provenance annotations are caller-supplied, not independently verified", + ] + unknown: set[str] = set() + unsupported: set[str] = set() + for person in people: + prefix = f"person.{person.name}" + inputs = dict(_mapping(person.inputs, prefix)) + for duplicate in ("age", "employment_income"): + if duplicate in inputs: + raise ContractInputError( + f"{prefix}.{duplicate}: duplicates a dedicated Person field" + ) + # Dedicated fields are required, unlike nullable optional inputs. + _value_text("age", person.age, f"{prefix}.age") + _value_text( + "employment_income", person.employment_income, f"{prefix}.employment_income" + ) + inputs.update(age=person.age, employment_income=person.employment_income) + hours = [inputs[name] for name in _HOURS_LABELS if name in inputs] + if len(hours) > 1 and any(value != hours[0] for value in hours[1:]): + raise ContractInputError(f"{prefix}: conflicting weekly-hours inputs") + if ( + "spouse_death_year" in inputs + and inputs["spouse_death_year"] is not None + and _number( + inputs["spouse_death_year"], f"{prefix}.spouse_death_year", integer=True + ) + > year + ): + raise ContractInputError(f"{prefix}.spouse_death_year: future death year") + if ( + "state_paid_leave_employee_share_withheld" in inputs + and scenario.state not in _OPTIONAL_PAYROLL_PROGRAMS + ): + raise ContractInputError( + f"{prefix}: no optional employee-share program in {scenario.state}" + ) + lines.extend(["", f"Person {person.name}:"]) + lines.extend( + _render_inputs( + inputs, + "person", + prefix, + provenance.get(person.name, {}), + unknown, + unsupported, + ) + ) + if not hours: + lines.append( + "- Weekly hours: 0 hours/week (contract assumption; " + "no weekly-hours input supplied)" + ) + lines.append( + "- Usual weekly hours before labor-supply response " + "[weekly_hours_worked_before_lsr]: 0 hours/week (declared convention)." + ) + elif "weekly_hours_worked_before_lsr" not in inputs: + supplied_hours = inputs.get("hours_worked_last_week") + if supplied_hours is not None: + lines.append( + "- Usual weekly hours before labor-supply response " + "[weekly_hours_worked_before_lsr]: " + f"{_numeric_text(supplied_hours)} " + "hours/week (declared alias of supplied hours_worked_last_week)." + ) + else: + unknown.add(f"{prefix}.weekly_hours_worked_before_lsr") + lines.append( + "- Usual weekly hours before labor-supply response " + "[weekly_hours_worked_before_lsr]: unknown " + "(no non-null hours_worked_last_week or direct before-response " + "input; weekly_hours_worked alone does not establish it)." + ) + if "takes_up_medicare_if_eligible" not in inputs: + lines.append( + "- Part B enrollment [takes_up_medicare_if_eligible]: yes if eligible " + "(declared convention; includes Part B, not evidence of enrollment)" + ) + if "medicare_part_b_premium" not in inputs: + lines.append( + "- Employee after-tax annual Part B premium [medicare_part_b_premium]: " + "declared-model standard premium plus IRMAA, net of Medicare Savings " + "Program support; paid only while enrolled (declared convention). " + "Unlisted two-year-prior IRMAA MAGI is $0 " + "(declared convention; no prior-year income is supplied)." + ) + if year == 2026 and policyengine_us_version == "2.15.17": + lines.append( + "- 2026 Part B standard premium: $202.90/month, $2,434.80/year " + "before IRMAA and Medicare Savings Program support " + "(declared policyengine-us 2.15.17 convention)." + ) + if "medical_expense_health_insurance_premiums" not in inputs: + lines.append( + "- Employee after-tax medical premium total " + "[medical_expense_health_insurance_premiums]: " + "nonzero health_insurance_premiums overrides the component sum; " + "otherwise health_insurance_premiums_without_medicare_part_b " + "(zero if absent) plus medicare_part_b_premium while enrolled. " + "A supplied zero total in health_insurance_premiums still uses " + "the component sum (declared convention; do not double-count)." + ) + program = _OPTIONAL_PAYROLL_PROGRAMS.get(scenario.state) + if program: + choice = inputs.get("state_paid_leave_employee_share_withheld", True) + if choice is None: + description = "employer's withholding choice is unknown" + elif choice: + description = "employer withholds the full employee share" + else: + description = ( + "employer pays the employee share; no employee withholding" + ) + lines.append( + "- Employer contribution choice [state_paid_leave_employee_share]: " + f"{program}: {description} " + "(declared convention when not supplied; use the declared model's " + "employee-share parameters; NY DBL uses 52 weeks/year)." + ) + if inputs.get("is_surviving_spouse") is True: + # This slice represents one supplied tax unit. A future multi-unit + # adapter must scope the spouse check to this person's tax unit. + remarried = scenario.filing_status == "joint" or any( + member.inputs.get("is_tax_unit_spouse") is True for member in people + ) + if remarried: + lines.append( + "- The surviving spouse indicator concerns a prior deceased spouse;" + " " + "the current listed spouse is living (declared convention). " + "Apply remarriage/current joint-filing rules." + ) + if "spouse_death_year" not in inputs: + death_year = year - 1 if year > 1 else "unknown" + lines.append( + f"- Spouse death year [spouse_death_year]: {death_year} " + "(declared convention for the prior deceased spouse; " + "synthetic date, not an observed fact)." + ) + if year == 1: + unknown.add(f"{prefix}.spouse_death_year") + if "dependent_child_lives_in_home" not in inputs: + child_lives = "yes" if scenario.children else "no" + lines.append( + "- Dependent child lives in the home " + f"[dependent_child_lives_in_home]: {child_lives} " + "(declared convention: listed children are dependent children " + "living in the home). Apply the dated filing-status rules; " + "the surviving spouse label alone does not establish " + "qualifying surviving spouse status." + ) + if ( + "financial_assistance" in inputs + and "financial_assistance_source" not in inputs + ): + lines.append( + "- Source of financial assistance [financial_assistance_source]: " + "cash gifts from friends or relatives outside the household " + "(declared convention; count as SNAP unearned cash income)." + ) + for entity in ("tax_unit", "spm_unit", "household"): + inputs = _mapping(getattr(scenario, f"{entity}_inputs"), entity) + if inputs: + lines.extend(["", f"{entity} inputs:"]) + lines.extend( + _render_inputs(inputs, entity, entity, {}, unknown, unsupported) + ) + if entity == "tax_unit" and "state_withheld_income_tax" not in inputs: + lines.extend( + [ + "", + "Tax-unit SALT payment convention:", + "- Annual state income tax paid during the year " + "[state_withheld_income_tax]: the declared model's per-state AGI " + "withholding estimate, treated as paid; not the final state income " + "tax liability (declared convention for each supplied tax unit). " + "A supplied tax-unit value overrides this estimate. SALT chooses " + "the larger of state_withheld_income_tax + local_income_tax and " + "state_sales_tax + local_sales_tax, then adds real estate tax " + "and applies the deduction cap. Montana's separate " + "mt_withheld_income_tax reader still requires its own stated input " + "or convention before scoring.", + ] + ) + if entity == "tax_unit": + audited_sales_context = ( + year == 2026 and policyengine_us_version == "2.15.17" + ) + state_sales_known = ( + inputs["state_sales_tax"] is not None + if "state_sales_tax" in inputs + else audited_sales_context + ) + if "state_sales_tax" not in inputs: + if audited_sales_context: + lines.append( + "- Annual state sales tax paid [state_sales_tax]: use the " + "2025 IRS optional sales-tax table with the declared " + "reference's household income/size rules; hold published " + "2025 cells without uprating for 2026, as " + "latest_c_irs_sales_tax_2025 does (declared convention)." + ) + else: + unknown.add("tax_unit.state_sales_tax") + lines.append( + "- Annual state sales tax paid [state_sales_tax]: unknown " + "(no supplied amount or audited table convention for this " + "year and declared model version)." + ) + if "local_sales_tax" not in inputs: + if state_sales_known or scenario.state in { + "CT", + "DC", + "IN", + "KY", + "MA", + "MD", + "ME", + "MI", + "NJ", + "RI", + }: + lines.append( + "- Annual local sales tax paid [local_sales_tax]: 0 in CT, " + "DC, IN, KY, MA, MD, ME, MI, NJ and RI; otherwise 20% of " + "state_sales_tax (declared reference-model proxy, not " + "observed spending)." + ) + else: + unknown.add("tax_unit.local_sales_tax") + lines.append( + "- Annual local sales tax paid [local_sales_tax]: unknown " + "(state_sales_tax is unknown, so its 20% local proxy " + "cannot be evaluated)." + ) + lines.extend(["", "Proposed v2 output definitions (not an output request):"]) + for name, definition in sorted(v2_output_definitions().items()): + lines.append(f"- [{name}]: {definition}") + lines.extend( + [ + "", + "Contract limitations:", + "- Unsupported inputs remain visible and uninterpreted; " + "resolve them before evaluation.", + "- Resolve unknown values/provenance and unit mappings, then " + "validate engine assumptions before scoring.", + "- No output request, answer schema, reference certification, " + "or board activation is supplied here.", + ] + ) + return RenderedHouseholdContract( + text="\n".join(lines), + contract_id=identity, + unknown_facts=tuple(sorted(unknown)), + unsupported_inputs=tuple(sorted(unsupported)), + ) diff --git a/policybench/prompt_contract_v2_required_facts.py b/policybench/prompt_contract_v2_required_facts.py new file mode 100644 index 00000000..5b55926e --- /dev/null +++ b/policybench/prompt_contract_v2_required_facts.py @@ -0,0 +1,339 @@ +"""Offline v2 fact-coverage report using PR #196's real sweep comparisons. + +The evidence records actual 2.15.17 simulations, not newly computed results. +Replay feeds their baselines and perturbations through ``unlisted_input_sweep``'s +comparison/report logic. It is intentionally a report on the legacy fixtures, +not a proof that every possible engine input has been discovered. A release must +also rerun the full sweep on the proposed v2 reference builder and conventions. +""" + +from __future__ import annotations + +import re +from collections import defaultdict +from collections.abc import Mapping, Set +from typing import TYPE_CHECKING, Any + +from policybench import unlisted_input_sweep as sweep + +if TYPE_CHECKING: + from policybench.prompt_contract_v2 import RenderedHouseholdContract + from policybench.scenarios import Scenario + +PAYROLL_ESTIMATE = "state_paid_leave_employee_share" + + +def replay_recorded_sweep( + fixture: Mapping[str, Any], + scenarios: Mapping[str, Scenario], + references: Mapping[tuple[str, str], float], + exclusions: Mapping[tuple[str, str], dict], +) -> sweep.SweepReport: + """Run upstream comparison logic over every recorded household and output. + + Only moving perturbations were vendored; omitted readings are not fabricated + as no-ops. All 1,984 recorded baselines are present, including excluded outputs + that do not match their published references. The recorded payroll scope + perturbation uses upstream movement/exclusion functions because #196 has not + yet registered that employer choice as an estimate. + """ + import pandas as pd + + baselines = fixture["baselines"] + if set(baselines) != set(scenarios): + raise ValueError("recorded baselines do not cover every frozen scenario") + if { + (sid, variable) for sid, values in baselines.items() for variable in values + } != set(references): + raise ValueError("recorded baselines do not cover every frozen output") + readings: dict[str, dict[tuple, dict]] = defaultdict(dict) + payroll_rows = [] + for move in fixture["moves"]: + sid, variable = move["scenario_id"], move["variable"] + if sid not in scenarios: + raise ValueError(f"unknown scenario {sid}") + key = (sid, variable) + if key not in references: + raise ValueError(f"unknown output {key}") + if abs(move["reference"] - references[key]) > sweep.BASELINE_TOLERANCE: + raise ValueError(f"published reference does not match evidence: {key}") + baseline = move["baseline"] + if abs(baseline - baselines[sid][variable]) > sweep.BASELINE_TOLERANCE: + raise ValueError(f"recorded baseline does not match evidence: {key}") + if not sweep.output_moved( + baseline, move["value"], binary=sweep.is_binary_output(variable) + ): + raise ValueError(f"recorded perturbation does not move: {key}") + if move["estimate"] == PAYROLL_ESTIMATE: + record = exclusions.get(key) + names_input = sweep.exclusion_names_inputs(record, (PAYROLL_ESTIMATE,)) + payroll_rows.append( + { + **{column: "" for column in sweep.MOVE_COLUMNS}, + **move, + "delta": move["value"] - baseline, + "status": ( + "excluded_same_input" + if names_input + else "excluded_other_reason" + ) + if record + else "scored", + "scored": record is None, + "excluded_reason": record["reason_code"] if record else "", + "exclusion_names_input": names_input, + } + ) + continue + reading_key = (move["estimate"], move["reading"], move["kind"], move["variant"]) + reading = readings[sid].setdefault( + reading_key, + { + "estimate": move["estimate"], + "reading": move["reading"], + "kind": move["kind"], + "variant": move["variant"], + "detail": {"parts": move.get("parts", [])}, + "outputs": {}, + "locality_outputs": [], + "localities": [], + "override_text": move.get("override", ""), + "noop": False, + "converged": True, + "iterations": 0, + }, + ) + reading["outputs"][variable] = move["value"] + if move["status"] == "prompt_rules_out": + reading["locality_outputs"].append(variable) + results = [ + { + "scenario_id": sid, + "state": scenario.state, + "baseline": dict(baselines[sid]), + "local_taxes": {}, + "readings": list(readings[sid].values()), + "simulations": 0, + } + for sid, scenario in sorted(scenarios.items()) + ] + report = sweep.evaluate(results, pd.Series(dict(references)), dict(exclusions)) + if payroll_rows: + report.moves = pd.concat( + [report.moves, pd.DataFrame(payroll_rows, columns=sweep.MOVE_COLUMNS)], + ignore_index=True, + ) + report.summary = sweep.summarize( + results, report.baseline, report.moves, report.readings + ) + report.summary["evidence_mode"] = "recorded_simulation_replay" + report.summary["recorded_simulations"] = fixture["sources"]["unlisted_input_sweep"][ + "simulations" + ] + return report + + +def _person_fact_text(text: str, person_name: str) -> str: + """Select one person's bullet facts, stopping at the next entity header.""" + lines = text.splitlines() + try: + start = lines.index(f"Person {person_name}:") + 1 + except ValueError: + return "" + facts = [] + for line in lines[start:]: + if line and not line.startswith("- "): + break + facts.append(line) + return "\n".join(facts) + + +def _has_fact_marker(text: str, name: str) -> bool: + # A marker in another fact's value or a disclaimer is not a statement of + # this input. Match the actual bullet label, before its value begins. + return re.search(rf"(?m)^- [^\[\]\n]*\[{re.escape(name)}\]: ", text) is not None + + +def _input_is_stated( + name: str, + entity: str, + scenario: Scenario, + rendered: RenderedHouseholdContract, + conventions: Set[str], + override: str, +) -> bool: + blocked = set(rendered.unknown_facts) | set(rendered.unsupported_inputs) + # The payroll estimate is a scope choice rather than an engine variable. + source_names = ( + (name, "state_paid_leave_employee_share_withheld") + if name == PAYROLL_ESTIMATE + else (name,) + ) + if entity == "person": + all_people = scenario.adults + scenario.children + affected = { + person + for person, variable in re.findall( + r"([a-z][a-z0-9_]*)\.([a-z][a-z0-9_]*)=", override + ) + if variable in source_names + } + affected &= {person.name for person in all_people} + people = [ + person for person in all_people if not affected or person.name in affected + ] + # Supplied null/unknown/unsupported overrides defeat an absent-input + # convention. Merely placing the input name in the convention registry + # cannot turn a supplied unknown into a stated fact. + if any( + f"person.{person.name}.{source}" in blocked + or (source in person.inputs and person.inputs[source] is None) + for person in people + for source in source_names + ): + return False + return bool(people) and all( + ( + name in conventions + and _has_fact_marker( + _person_fact_text(rendered.text, person.name), name + ) + ) + or any( + source in person.inputs + and _has_fact_marker( + _person_fact_text(rendered.text, person.name), source + ) + for source in source_names + ) + for person in people + ) + values = getattr(scenario, f"{entity}_inputs") + if f"{entity}.{name}" in blocked or (name in values and values[name] is None): + return False + if name in conventions and _has_fact_marker(rendered.text, name): + return True + return ( + name in values + and values[name] is not None + and f"{entity}.{name}" not in blocked + and _has_fact_marker(rendered.text, name) + ) + + +def report_required_facts( + replay: sweep.SweepReport, + fixture: Mapping[str, Any], + scenarios: Mapping[str, Scenario], + rendered: Mapping[str, RenderedHouseholdContract], + stated_convention_inputs: Set[str], +) -> dict[str, Any]: + """List remaining individually moving inputs; keep compound readings distinct. + + Unsupported raw labels and unknown values/provenance never establish a fact. + A named convention counts only when supplied in ``stated_convention_inputs`` + and its exact ``[name]`` fact marker appears in this household's text. Person + inputs require that marker within every affected person's own section; another + person's convention never covers them. The generic unlisted-zero rule cannot + certify an engine estimate. Compound + literal readings can override newly stated conventions too, so their residual + inputs are reported separately, without assigning their full move to each. + """ + if set(rendered) != set(scenarios): + raise ValueError("rendered contracts do not cover every scenario") + estimates = {estimate["id"]: estimate for estimate in fixture["estimates"]} + individual: dict[str, dict[str, Any]] = {} + compound = [] + for move in replay.moves.to_dict("records"): + if move["status"] == "prompt_rules_out": + continue + sid = move["scenario_id"] + estimate_id = move["estimate"] + if estimate_id == sweep.COMBINED_ESTIMATE: + original = next( + entry + for entry in fixture["moves"] + if all( + entry[field] == move[field] + for field in ( + "scenario_id", + "variable", + "estimate", + "reading", + "variant", + ) + ) + ) + missing = sorted( + { + name + for part in original["parts"] + for name in estimates[part]["engine_inputs"] + if not _input_is_stated( + name, + estimates[part]["entity"], + scenarios[sid], + rendered[sid], + stated_convention_inputs, + move["override"], + ) + } + ) + compound.append( + {"output": [sid, move["variable"]], "unstated_inputs": missing} + ) + continue + estimate = estimates[estimate_id] + names = estimate["engine_inputs"] or [estimate_id] + missing = [ + name + for name in names + if not _input_is_stated( + name, + estimate["entity"], + scenarios[sid], + rendered[sid], + stated_convention_inputs, + move["override"], + ) + ] + if not missing: + continue + entry = individual.setdefault( + estimate_id, + { + "estimate": estimate_id, + "unstated_inputs": set(), + "outputs": set(), + "scored_outputs": set(), + }, + ) + entry["unstated_inputs"].update(missing) + entry["outputs"].add((sid, move["variable"])) + if move["status"] in ("scored", "acknowledged"): + entry["scored_outputs"].add((sid, move["variable"])) + remaining = [] + for estimate_id, entry in sorted(individual.items()): + outputs = sorted(entry["outputs"]) + scored = sorted(entry["scored_outputs"]) + remaining.append( + { + "estimate": estimate_id, + "unstated_inputs": sorted(entry["unstated_inputs"]), + "outputs": [list(output) for output in outputs], + "output_count": len(outputs), + "scored_output_count": len(scored), + } + ) + return { + "households": replay.summary["households"], + "outputs": replay.summary["outputs"], + "evidence_mode": replay.summary["evidence_mode"], + "remaining": remaining, + "compound_readings_requiring_rerun": compound, + "note": ( + "Report only: legacy exclusions remain. Compound readings change " + "multiple inputs and can override v2 conventions; rerun the complete " + "sweep under the proposed reference builder before activation." + ), + } diff --git a/policybench/reference_exclusions.py b/policybench/reference_exclusions.py index 80f55290..2906eb2b 100644 --- a/policybench/reference_exclusions.py +++ b/policybench/reference_exclusions.py @@ -1,16 +1,37 @@ -"""Reference outputs excluded from scoring because the reference is not determinate. +"""Reference outputs excluded from scoring because the reference is not sound. A benchmark output is scored only when its reference follows from the facts the -prompt states. When a reference instead depends on an engine input the -household facts never carried (so the prompt could not list it and the model -could not know it), the output is excluded from scoring for every model: no -model gains or loses from it. The record lives beside the reference CSV as -``reference_exclusions.json`` and travels with the run into the frozen -snapshot, where the manifest pins it. - -Each entry names the output, the unlisted input, the alternative reading a -careful reader could take of the stated facts, and the reference under both -readings as recomputed with the engine version that produced the references. +prompt states under the law for the benchmark year. Two cases fail that test, +and each output that fails is excluded from scoring for every model, so no +model gains or loses from it: + +``reference_depends_on_unlisted_input`` + The reference depends on an engine input the household facts never + carried, so the prompt could not list it and a careful reader could take + the stated facts the other way. The entry names the unlisted input and the + alternative reading, and records the reference under both readings. + +``reference_engine_defect`` + The engine that produced the reference misapplies the law on facts the + prompt does state. The entry names the root cause, the defect and the law, + and records the frozen reference beside the corrected value, computed with + the same engine version and a sandbox fix that implements the rule. It + also points to the upstream issue or fix. A later reference version that + uses a fixed engine brings the output back. + +``reference_law_published_after_freeze`` + The reference depends on a law or official parameter published after the + references were frozen, so the frozen value is the engine's projection + and no model could have known the governing figure when it answered. The + entry names what was published and when, and records the frozen + projection beside the value under the published figure. + +The record lives beside the reference CSV as ``reference_exclusions.json`` and +travels with the run into the frozen snapshot, where the manifest pins it. +Every entry carries ``alternative_reading`` and ``alternative_value``: for an +unlisted input they are the other reading and the reference under it; for an +engine defect they are the rule as the law states it and the corrected value; +for later-published law they are the published figure and the value under it. Exclusion is symmetric: rows whose answer happened to match the frozen reference leave the score along with rows that did not. """ @@ -23,12 +44,14 @@ import pandas as pd FILENAME = "reference_exclusions.json" -REASON_CODES = frozenset({"reference_depends_on_unlisted_input"}) -REQUIRED_FIELDS = ( +UNLISTED_INPUT = "reference_depends_on_unlisted_input" +ENGINE_DEFECT = "reference_engine_defect" +LATER_LAW = "reference_law_published_after_freeze" +REASON_CODES = frozenset({UNLISTED_INPUT, ENGINE_DEFECT, LATER_LAW}) +COMMON_FIELDS = ( "scenario_id", "variable", "reason_code", - "unlisted_input", "alternative_reading", "frozen_value", "alternative_value", @@ -36,6 +59,13 @@ "decided_on", "decided_by", ) +REQUIRED_FIELDS_BY_REASON = { + UNLISTED_INPUT: COMMON_FIELDS + ("unlisted_input",), + ENGINE_DEFECT: COMMON_FIELDS + ("root_cause", "defect", "law", "upstream"), + LATER_LAW: COMMON_FIELDS + ("root_cause", "published", "law"), +} +# Kept for readers that predate the engine-defect reason code. +REQUIRED_FIELDS = REQUIRED_FIELDS_BY_REASON[UNLISTED_INPUT] class ReferenceExclusionError(ValueError): @@ -60,17 +90,17 @@ def load_reference_exclusions(path: Path) -> list[dict]: raise ReferenceExclusionError(f"{path}: 'exclusions' must be a list") seen: set[tuple[str, str]] = set() for entry in entries: + if entry.get("reason_code") not in REASON_CODES: + raise ReferenceExclusionError( + f"{path}: unknown reason_code {entry.get('reason_code')!r}" + ) missing = [ field - for field in REQUIRED_FIELDS + for field in REQUIRED_FIELDS_BY_REASON[entry["reason_code"]] if field not in entry or entry[field] in (None, "") ] if missing: raise ReferenceExclusionError(f"{path}: entry missing {missing}: {entry}") - if entry["reason_code"] not in REASON_CODES: - raise ReferenceExclusionError( - f"{path}: unknown reason_code {entry['reason_code']!r}" - ) try: frozen = float(entry["frozen_value"]) alternative = float(entry["alternative_value"]) @@ -90,6 +120,13 @@ def load_reference_exclusions(path: Path) -> list[dict]: return entries +def exclusion_basis(entry: dict) -> str: + """What an exclusion turns on: the unlisted input or the engine root cause.""" + if entry["reason_code"] in (ENGINE_DEFECT, LATER_LAW): + return str(entry["root_cause"]) + return str(entry["unlisted_input"]) + + def exclusion_keys(exclusions: list[dict]) -> set[tuple[str, str]]: return {(str(e["scenario_id"]), str(e["variable"])) for e in exclusions} diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 03ffa029..aaeeec30 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -143,6 +143,15 @@ "wy_power_shelter_qualified", } +# Prompt-visible inputs PolicyEngine reads under another name. The prompt shows +# hours_worked_last_week as "usual weekly hours worked"; the SNAP work rules read +# weekly_hours_worked_before_lsr, which the prompt never shows. policyengine-us +# defaulted that input to 40 until #9261 (2026-08-12) made it 0, so a stated 40 +# hours stopped reaching the SNAP work tests. The stated value is copied to the +# engine name; a person with no stated hours keeps the engine default, which is +# also the prompt's rule for unlisted numbers (0). +PE_INPUT_ALIASES = {"hours_worked_last_week": "weekly_hours_worked_before_lsr"} + EXCLUDED_INPUT_PREFIXES = ( "takes_up_", "would_", @@ -476,6 +485,9 @@ def to_pe_household(self) -> dict: } for key, value in person.inputs.items(): person_data[key] = self._yearize(value) + for source, target in PE_INPUT_ALIASES.items(): + if source in person.inputs and target not in person.inputs: + person_data[target] = self._yearize(person.inputs[source]) for key, value in DEFAULT_TAKEUP_INPUTS["person"].items(): person_data.setdefault(key, self._yearize(value)) people[person.name] = person_data @@ -488,6 +500,9 @@ def to_pe_household(self) -> dict: } for key, value in person.inputs.items(): person_data[key] = self._yearize(value) + for source, target in PE_INPUT_ALIASES.items(): + if source in person.inputs and target not in person.inputs: + person_data[target] = self._yearize(person.inputs[source]) for key, value in DEFAULT_TAKEUP_INPUTS["person"].items(): person_data.setdefault(key, self._yearize(value)) people[person.name] = person_data diff --git a/policybench/supervisor.py b/policybench/supervisor.py index bc995ed8..095d175b 100644 --- a/policybench/supervisor.py +++ b/policybench/supervisor.py @@ -15,10 +15,29 @@ combined into ``/predictions.csv`` at the end; rerunning the same command skips completed scenarios and replays partially-complete ones from the response cache. + +For a single-country run (every supervised run in practice), PolicyEngine +provenance (the ``policyengine_bundles`` block of every scenario sidecar) is +computed once, in a fresh interpreter, into +``/policyengine_provenance.json`` and handed to workers through +``POLICYBENCH_POLICYENGINE_PROVENANCE``. What computing it costs depends on +the environment. It imports policyengine only when a country's installed +model package is the exact version policyengine.py pins for it. Under +policyengine.py 4.16.1, with ``POLICYENGINE_SKIP_COUNTRY_IMPORTS`` unset, +that import built the US and UK tax-benefit systems: about 1 GB of peak RSS +and 11-14 CPU-seconds per process when measured on 2026-09-28. Otherwise it +reads only package metadata and policyengine.py's bundled release manifest, +and imports no PolicyEngine module; that was the case on 2026-09-30 for +policyengine-us 2.15.17, which differs from the 2.2.1 that policyengine.py +6.1.2 pins. Either way, computing it once per run bounds the cost: a worker +that only calls an LLM does not pay it, and neither does the supervisor. +Without the file (a mixed-country run, or a failed write), workers and the +supervisor compute provenance themselves, as before. """ from __future__ import annotations +import contextlib import hashlib import json import math @@ -40,6 +59,11 @@ completion_budget_ceiling_for, explanation_chunk_size_for, ) +from policybench.policyengine_runtime import ( + POLICYENGINE_PROVENANCE_ENV, + POLICYENGINE_PROVENANCE_FILENAME, + POLICYENGINE_PROVENANCE_NOT_REUSED, +) from policybench.spend_ledger import ( SPEND_LEDGER_SUFFIX, count_budget_escalations, @@ -57,6 +81,16 @@ # cross this share of the budget. BUDGET_STOP_FRACTION = 0.9 TREATMENT_FINGERPRINT_VERSION = 3 +PROVENANCE_WRITER_TIMEOUT_SECONDS = 3600 +# Writes the run's PolicyEngine provenance file. It runs in a fresh +# interpreter, as each worker did, so a single-country run records what a +# worker computing the bundles itself would, and the supervisor needs no +# policyengine import of its own when the handoff succeeds. +PROVENANCE_WRITER = ( + "import sys\n" + "from policybench.policyengine_runtime import write_policyengine_provenance\n" + "sys.exit(0 if write_policyengine_provenance(sys.argv[1], sys.argv[2:]) else 3)\n" +) @dataclass @@ -69,6 +103,7 @@ class ScenarioResult: missing_predictions: int = 0 timed_out: bool = False seconds: float = 0.0 + policyengine_provenance_recomputed: bool = False @dataclass @@ -84,6 +119,7 @@ class RunState: started_at: float = 0.0 updated_at: float = 0.0 budget_escalation_count: int = 0 + policyengine_provenance_recomputed: int = 0 def projected_total_usd(self) -> float | None: if not self.completed: @@ -118,6 +154,8 @@ def __init__( self.max_rounds = max_rounds self.python = python or sys.executable self.env = {**os.environ, **(env or {})} + # Only a provenance file this supervisor wrote may reach its workers. + self.env.pop(POLICYENGINE_PROVENANCE_ENV, None) self.scenarios = self._load_scenarios() self.scenario_ids = [scenario.id for scenario in self.scenarios] self.initial_request_variables = self._load_initial_request_variables() @@ -135,6 +173,9 @@ def __init__( self._credits_checked_at = float("-inf") self._credits_spent = 0.0 self._credits_spent_offset = 0.0 + # Bundles read back from this run's provenance file; set together with + # self.env[POLICYENGINE_PROVENANCE_ENV] by _write_policyengine_provenance. + self._policyengine_bundles: dict | None = None # -- setup ------------------------------------------------------------- @@ -262,6 +303,9 @@ def _expected_scenario_metadata(self, index: int) -> dict: # Match the eval-no-tools CLI defaults used by _spawn, including # its sliced scenario list and the environment of the subprocess. + # PolicyEngine provenance comes from the copy read back when the run's + # provenance file was written, so a worker sidecar that departs from + # it (or a file changed afterwards) is caught here. return _build_resume_metadata( task="eval_no_tools_batch", scenarios=[self.scenarios[index]], @@ -270,6 +314,7 @@ def _expected_scenario_metadata(self, index: int) -> dict: run_id=None, include_explanations=True, env=self.env, + policyengine_bundles=self._policyengine_bundles, ) def _raise_stale_scenario_output( @@ -591,6 +636,10 @@ def write_heartbeat(self) -> None: "stopped_reason": self.state.stopped_reason, "projection_warning": self.projection_warning, "budget_escalation_count": self.state.budget_escalation_count, + "policyengine_provenance": self.env.get(POLICYENGINE_PROVENANCE_ENV), + "policyengine_provenance_recomputed": ( + self.state.policyengine_provenance_recomputed + ), "workload": self.workload, "treatment_fingerprint": self.treatment_fingerprint, "started_at": self.state.started_at, @@ -650,9 +699,11 @@ def _collect(self, index: int, started: float) -> ScenarioResult: missing = int(frame["prediction"].isna().sum()) if "prediction" in frame else 0 log = path.with_suffix(".log") timed_out = False + provenance_recomputed = False if log.exists(): text = log.read_text(errors="ignore") timed_out = "Timeout" in text or "timed out" in text + provenance_recomputed = POLICYENGINE_PROVENANCE_NOT_REUSED in text return ScenarioResult( scenario_id, index, @@ -662,12 +713,62 @@ def _collect(self, index: int, started: float) -> ScenarioResult: missing_predictions=missing, timed_out=timed_out, seconds=time.time() - started, + policyengine_provenance_recomputed=provenance_recomputed, ) # -- main loop ----------------------------------------------------------- + def _compute_policyengine_provenance( + self, path: Path, countries: list[str], env: dict + ) -> bool: + """Write the provenance file from a fresh worker interpreter.""" + try: + result = subprocess.run( + [self.python, "-c", PROVENANCE_WRITER, str(path), *countries], + env=env, + timeout=PROVENANCE_WRITER_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + return False + return result.returncode == 0 + + def _write_policyengine_provenance(self) -> None: + """Compute PolicyEngine provenance once and point workers at it. + + On any failure, workers compute provenance themselves and the + supervisor's expectation does too, as before the file existed. + """ + from policybench.eval_no_tools import _scenario_countries + + path = (self.run_dir / POLICYENGINE_PROVENANCE_FILENAME).resolve() + self.env.pop(POLICYENGINE_PROVENANCE_ENV, None) + self._policyengine_bundles = None + # A file left by an earlier supervisor must not outlive a failed write. + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) + countries = sorted(_scenario_countries(self.scenarios)) + # Each worker computed its one scenario's country in a fresh process. + # Whether ``import policyengine`` succeeds, and so which branch + # records the US bundle, can depend on what the same process looked + # up first, so the handoff covers single-country runs only. + if len(countries) != 1: + return + if not self._compute_policyengine_provenance(path, countries, dict(self.env)): + return + try: + bundles = json.loads(path.read_text(encoding="utf-8"))[ + "policyengine_bundles" + ] + except (OSError, ValueError, KeyError, TypeError): + return + if not isinstance(bundles, dict) or not set(countries) <= set(bundles): + return + self._policyengine_bundles = bundles + self.env[POLICYENGINE_PROVENANCE_ENV] = str(path) + def run(self, poll_seconds: float = 2.0) -> RunState: existing_state = self._validate_resume() + self._write_policyengine_provenance() if self._credits_baseline is not None and existing_state is not None: prior_spend = existing_state.get("spent_usd") if isinstance(prior_spend, (int, float)) and prior_spend > 0: @@ -714,6 +815,8 @@ def run(self, poll_seconds: float = 2.0) -> RunState: del in_flight[index] result = self._collect(index, started) self._record(result) + if result.policyengine_provenance_recomputed: + self.state.policyengine_provenance_recomputed += 1 if result.ok: self.state.completed.append(result.scenario_id) else: diff --git a/policybench/unlisted_input_sweep.py b/policybench/unlisted_input_sweep.py new file mode 100644 index 00000000..267e5b87 --- /dev/null +++ b/policybench/unlisted_input_sweep.py @@ -0,0 +1,2069 @@ +"""Sweep a frozen run's references against the engine's estimates of unlisted inputs. + +A US reference is a policyengine-us calculation on the facts the prompt lists. Where +a rule needs an input the prompt never lists, the engine supplies one: a default, a +formula estimate, or an assumed county. The prompt tells models to "treat any +unlisted numeric input as 0 and any other unlisted household fact, boolean, or +status input as false" (``policybench.prompts.TASK_PREFACE``). An output whose +reference moves when such an estimate is replaced by that literal reading, or by a +documented alternative reading, rests on an input the prompt never states. That is +the test of the ``reference_depends_on_unlisted_input`` exclusion rule +(``reference_exclusions.json``). + +The audits found each estimate on its own (r14's weekly hours on 2026-09-22, the +SALT withholding estimate and four others on 2026-10-05). ``ESTIMATES`` registers +them. ``run`` recomputes every output of a frozen run on the reference system that +built its references (a fix module, as ``reference_audit/2026-09-28/scripts/ +sweep.py`` defines one), first as published and then under each registered reading, +and reports every output that moves by more than the $1 exact-match tolerance or +flips a flag. Each move is marked scored or already excluded. + +Every simulation runs on one tax-benefit system per worker process: building the +2.15.17 system takes about 40 seconds and a household simulation about half a +second. + +The swept system adds ``policybench.output_scope.reform`` to the fix module, so its +state income tax and state refundable credit outputs exclude local taxes and +credits as the benchmark defines them. The run checks that the adapter changes no +published reference: each local entry it removes must be zero for every household. + +A county reading that switches on one of the engine's locality flags (in_nyc, +in_san_francisco, ...) makes an unlisted household fact true, which the prompt's +rule makes false. The county is simulated again with those flags held at their +reference values; an output whose move that undoes is reported as +``prompt_rules_out`` and does not fail ``--strict``. A move that survives depends +on the county itself and stays scored. + +Usage (``docs/runbook.md``, "Reference-build gates"): + + uv run policybench unlisted-input-sweep \\ + --fix reference_audit/2026-09-28/fixes/latest_final.py \\ + --fix-support reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json \\ + --out-dir results/local/unlisted_input_sweep \\ + --acknowledged reference_audit/unlisted_input_acknowledged.json --strict + +``--run-dir`` defaults to the frozen run, +``paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace``. +""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import math +import multiprocessing +import os +import re +import shutil +import sys +import tempfile +import time +from collections.abc import Callable, Iterable, Sequence +from concurrent.futures import ProcessPoolExecutor, as_completed +from dataclasses import dataclass, field +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from policybench import output_scope + +if TYPE_CHECKING: + import pandas as pd + +TOLERANCE = 1.0 +BASELINE_TOLERANCE = 1e-3 +NOOP_TOLERANCE = 1e-9 +FIXED_POINT_TOLERANCE = 0.005 +MAX_FIXED_POINT_ITERATIONS = 25 +LITERAL = "literal" +ALTERNATIVE = "alternative" +COMBINED_ESTIMATE = "all_literal_readings" +COMBINED_READING = "all_literal" + +# Inputs the references were built under another name. The frozen manifest stores +# partnership_se_income; policyengine-us 2.15.17 calls it +# partnership_self_employment_net_earnings. The 2026-09-28 reference builder +# (reference_audit/2026-09-28/scripts/build_references_latest.py) builds households +# with sweep.py's build_situation, which renames it. +REFERENCE_INPUT_RENAMES = { + "partnership_se_income": "partnership_self_employment_net_earnings" +} + +# Local income taxes whose only route to the federal SALT deduction is the +# withholding estimate. In policyengine-us 2.15.17 the income-tax line of SALT is +# state_withheld_income_tax plus local_income_tax. md_withheld_income_tax carries an +# AGI-based estimate of Maryland county tax (the county's rate times AGI less the +# largest single standard deduction); local_income_tax adds no Maryland entry, and +# the county liability variable feeds no SALT variable. Replacing the withholding +# estimate with what the household paid therefore has to add the county liability +# back. NYC tax already reaches SALT through local_income_tax (as nyc_income_tax, +# net of NYC refundable credits), so it is not added. +SALT_VIA_WITHHOLDING_ONLY = ("md_local_income_tax_before_refundable_credits",) + +DEFAULT_RUN_DIR = Path( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) + + +# --------------------------------------------------------------------------- +# Overrides and readings +# --------------------------------------------------------------------------- + +_GROUPS = { + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "household": "households", +} + + +def _canonical(value: Any) -> Any: + if isinstance(value, bool) or isinstance(value, str): + return value + if isinstance(value, int | float): + return float(value) + return value + + +def _same_value(a: Any, b: Any) -> bool: + """Equal and of the same kind: 0.0 and False are different settings.""" + a, b = _canonical(a), _canonical(b) + return type(a) is type(b) and a == b + + +@dataclass(frozen=True) +class Override: + """Input values a reading sets on top of the household as built for the reference. + + ``people`` maps a person's name in the situation to ``{variable: value}``; the + group fields map a variable to its value on the household's one unit. Values are + for the scenario year. + """ + + people: dict[str, dict[str, Any]] = field(default_factory=dict) + tax_unit: dict[str, Any] = field(default_factory=dict) + spm_unit: dict[str, Any] = field(default_factory=dict) + household: dict[str, Any] = field(default_factory=dict) + + def items(self) -> Iterable[tuple[str, str | None, str, Any]]: + """(entity, person or None, variable, value) for every value set.""" + for name in sorted(self.people): + for variable in sorted(self.people[name]): + yield "person", name, variable, self.people[name][variable] + for entity in _GROUPS: + values = getattr(self, entity) + for variable in sorted(values): + yield entity, None, variable, values[variable] + + def is_empty(self) -> bool: + return not any(True for _ in self.items()) + + def as_dict(self) -> dict[str, Any]: + out: dict[str, Any] = {} + if self.people: + out["people"] = { + name: {k: _canonical(v) for k, v in sorted(values.items())} + for name, values in sorted(self.people.items()) + } + for entity in _GROUPS: + values = getattr(self, entity) + if values: + out[entity] = {k: _canonical(v) for k, v in sorted(values.items())} + return out + + def key(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True) + + def describe(self) -> str: + parts = [] + for entity, person, variable, value in self.items(): + where = person if person is not None else entity + shown = f"{value:.2f}" if isinstance(value, float) else str(value) + parts.append(f"{where}.{variable}={shown}") + return "; ".join(parts) + + def merged(self, other: Override) -> Override: + """Both overrides at once; setting one variable to two values is an error.""" + people = {name: dict(values) for name, values in self.people.items()} + for name, values in other.people.items(): + target = people.setdefault(name, {}) + for variable, value in values.items(): + if variable in target and not _same_value(target[variable], value): + raise ValueError(f"conflicting values for {name}.{variable}") + target[variable] = value + groups = {} + for entity in _GROUPS: + mine = dict(getattr(self, entity)) + for variable, value in getattr(other, entity).items(): + if variable in mine and not _same_value(mine[variable], value): + raise ValueError(f"conflicting values for {entity}.{variable}") + mine[variable] = value + groups[entity] = mine + return Override(people=people, **groups) + + def apply(self, situation: dict, period: str) -> dict: + """A copy of ``situation`` with these values set for ``period``.""" + out = copy.deepcopy(situation) + for name, values in self.people.items(): + if name not in out["people"]: + raise KeyError(f"no person {name!r} in the situation") + for variable, value in values.items(): + out["people"][name][variable] = {period: value} + for entity, plural in _GROUPS.items(): + values = getattr(self, entity) + if not values: + continue + units = out[plural] + if len(units) != 1: + raise ValueError(f"expected one {entity}, found {len(units)}") + unit = next(iter(units.values())) + for variable, value in values.items(): + unit[variable] = {period: value} + return out + + +@dataclass +class ReadingPlan: + """One way to fill an unlisted input for one household.""" + + override: Override + variant: str = "" + simulation: Any = None + trace: list[dict[str, float]] = field(default_factory=list) + converged: bool = True + detail: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Reading: + """A reading of an unlisted input. + + ``kind`` is ``literal`` for the prompt's own rule (unlisted numbers 0, unlisted + facts false) and ``alternative`` for another reading the stated facts support, + used where the engine already applies the literal one. ``plan`` returns the + household's overrides (several for a reading with variants, none where the + reading cannot apply). + """ + + id: str + kind: str + description: str + plan: Callable[[HouseholdContext], list[ReadingPlan]] + + +@dataclass(frozen=True) +class UnlistedEstimate: + """An input the engine fills in that no PolicyBench prompt lists.""" + + id: str + engine_inputs: tuple[str, ...] + entity: str + engine_behavior: str + why_unlisted: str + found_in: str + readings: tuple[Reading, ...] + + def reading(self, reading_id: str) -> Reading: + for reading in self.readings: + if reading.id == reading_id: + return reading + raise KeyError(f"{self.id} has no reading {reading_id!r}") + + def describe(self) -> dict[str, Any]: + return { + "id": self.id, + "engine_inputs": list(self.engine_inputs), + "entity": self.entity, + "engine_behavior": self.engine_behavior, + "why_unlisted": self.why_unlisted, + "found_in": self.found_in, + "readings": [ + {"id": r.id, "kind": r.kind, "description": r.description} + for r in self.readings + ], + } + + +# --------------------------------------------------------------------------- +# Household context +# --------------------------------------------------------------------------- + + +def discover_locality_flags() -> tuple[str, ...]: + """Household flags that put a household in a locality with its own rules. + + policyengine-us defines them as ``in_*`` variables under ``variables/gov/local`` + (in_san_francisco, in_la, ...) and ``variables/household/demographic/geographic`` + (in_nyc, and in_ny_mctd_zone_2 for the Metropolitan Commuter Transportation + District outside the city). Most are formulas of county; a few (in_denver, + in_wilmington, in_yonkers) are inputs that default to false, which no county + reading switches on. They are read from the installed source tree, so no system + is built. The prompt states no locality, and its rule makes every unlisted + household fact false. + """ + spec = importlib.util.find_spec("policyengine_us") + if spec is None or not spec.submodule_search_locations: + return () + root = Path(next(iter(spec.submodule_search_locations))) / "variables" + names = set() + for pattern in ("gov/local/**/in_*.py", "household/demographic/geographic/in_*.py"): + names.update(path.stem for path in root.glob(pattern)) + return tuple(sorted(names)) + + +class Engine: + """The swept tax-benefit system and how to simulate a situation on it.""" + + def __init__( + self, + system: Any, + simulation_class: Any, + aggregate_local_components: Sequence[str] = (), + removed_local_components: Sequence[str] = (), + remaining_local_components: Sequence[str] | None = None, + county_states: frozenset[str] | None = None, + locality_flags: Sequence[str] | None = None, + ): + self.system = system + self.simulation_class = simulation_class + # Local taxes still in the swept state income tax aggregate. + self.aggregate_local_components = tuple(aggregate_local_components) + # Local entries the output-scope adapter removed from the state lists. + self.removed_local_components = tuple(removed_local_components) + # Local entries still in any state list (taxes and credits). + self.remaining_local_components = tuple( + aggregate_local_components + if remaining_local_components is None + else remaining_local_components + ) + self.county_states = county_states + if locality_flags is None: + locality_flags = [ + name for name in discover_locality_flags() if self.has_variable(name) + ] + self.locality_flags = tuple(locality_flags) + + @property + def remaining_local_credits(self) -> tuple[str, ...]: + """Local credits the swept system still counts in state refundable credits.""" + return tuple( + name + for name in self.remaining_local_components + if name not in output_scope.LOCAL_INCOME_TAX_COMPONENTS + and name not in self.aggregate_local_components + ) + + def simulate(self, situation: dict) -> Any: + return self.simulation_class( + tax_benefit_system=self.system, situation=situation + ) + + def has_variable(self, name: str) -> bool: + return name in self.system.variables + + def variable_entity(self, name: str) -> str: + entity = getattr(self.system.variables[name], "entity", None) + return getattr(entity, "key", "household") + + def parameter(self, path: str, instant: str) -> Any: + node = self.system.parameters + for part in path.split("."): + node = getattr(node, part) + return node(instant) + + def counties(self, state: str) -> list[str]: + """Every county the engine knows in ``state``, in the engine's order. + + policyengine-us gives a household with no county its state's first county in + this order (``first_county_in_state``: the alphabetically first value). + """ + enum = self.system.variables["county"].possible_values + members = [ + member + for member in enum + if member.name != "UNKNOWN" and str(member.value).endswith(f", {state}") + ] + return [member.name for member in sorted(members, key=lambda m: str(m.value))] + + +@dataclass +class HouseholdContext: + scenario: Any + situation: dict + variables: list[str] + engine: Engine + baseline: Any = None + simulations: int = 0 + _cache: dict[str, Any] = field(default_factory=dict) + + @property + def year(self) -> int: + return int(self.scenario.year) + + @property + def period(self) -> str: + return str(self.scenario.year) + + @property + def person_names(self) -> list[str]: + return list(self.situation["people"]) + + def total(self, simulation: Any, variable: str) -> float: + return float(simulation.calculate(variable, self.year).sum()) + + def by_person(self, simulation: Any, variable: str) -> dict[str, float]: + values = simulation.calculate(variable, self.year) + return { + name: float(values[index]) for index, name in enumerate(self.person_names) + } + + def _baseline_value(self, entity: str, person: str | None, variable: str) -> Any: + values = self.baseline.calculate(variable, self.year) + if hasattr(values, "decode_to_str"): + values = values.decode_to_str() + index = self.person_names.index(person) if entity == "person" else 0 + value = values[index] + return value if isinstance(value, str) else float(value) + + def is_noop(self, override: Override) -> bool: + """Whether every value ``override`` sets equals the reference's own value. + + Setting an input to the value the engine computes for it leaves every + output unchanged, so such a reading needs no simulation. + """ + for entity, person, variable, value in override.items(): + current = self._baseline_value(entity, person, variable) + if isinstance(current, str) or isinstance(value, str): + if str(current) != str(value): + return False + elif abs(float(current) - float(value)) > NOOP_TOLERANCE: + return False + return True + + def simulate(self, override: Override, *, cache: bool = True) -> Any: + """The household under ``override``. Kept for reuse unless ``cache`` is off + (the county reading's hundreds of variants keep only their outputs).""" + if override.is_empty() or self.is_noop(override): + return self.baseline + key = override.key() + if key in self._cache: + return self._cache[key] + self.simulations += 1 + simulation = self.engine.simulate(override.apply(self.situation, self.period)) + if cache: + self._cache[key] = simulation + return simulation + + def outputs(self, simulation: Any) -> dict[str, float]: + from policybench.ground_truth import ( + _extract_person_value, + _pe_variable_for_output, + ) + + values = {} + for variable in self.variables: + pe_variable = _pe_variable_for_output(variable, self.scenario.country) + values[variable] = float( + _extract_person_value( + simulation.calculate(pe_variable, self.year), + self.scenario, + variable, + ) + ) + return values + + +# --------------------------------------------------------------------------- +# Readings +# --------------------------------------------------------------------------- + + +def _zero_on_tax_unit(variable: str) -> Callable[[HouseholdContext], list[ReadingPlan]]: + def plan(ctx: HouseholdContext) -> list[ReadingPlan]: + return [ + ReadingPlan( + Override(tax_unit={variable: 0.0}), + detail={"engine_value": ctx.total(ctx.baseline, variable)}, + ) + ] + + return plan + + +def _zero_on_people(variable: str) -> Callable[[HouseholdContext], list[ReadingPlan]]: + def plan(ctx: HouseholdContext) -> list[ReadingPlan]: + values = ctx.by_person(ctx.baseline, variable) + people = {name: {variable: 0.0} for name, value in values.items() if value} + if not people: + return [] + return [ReadingPlan(Override(people=people), detail={"engine_value": values})] + + return plan + + +def salt_income_tax_paid( + ctx: HouseholdContext, simulation: Any, *, net: bool = False +) -> float: + """The state income tax a household pays during the year if it pays its liability. + + The engine's state income tax before refundable credits, less any local tax the + swept system's state aggregate still adds, plus the local tax whose only route + to SALT is the withholding estimate (Maryland county tax). ``net`` also + subtracts state refundable credits, less any local credit the swept system still + counts in them. Floored at zero. + + It inherits the engine aggregate's entries: Mississippi's is before + nonrefundable credits (ms_income_tax_before_credits_unit), and Washington's + adds its capital gains and millionaires taxes. + """ + amount = ctx.total(simulation, output_scope.STATE_AGGREGATE) + for name in ctx.engine.aggregate_local_components: + amount -= ctx.total(simulation, name) + for name in SALT_VIA_WITHHOLDING_ONLY: + if ctx.engine.has_variable(name): + amount += ctx.total(simulation, name) + if net: + amount -= ctx.total(simulation, "state_refundable_credits") + for name in ctx.engine.remaining_local_credits: + amount += ctx.total(simulation, name) + return max(0.0, amount) + + +def _withholding_fixed_point( + net: bool, +) -> Callable[[HouseholdContext], list[ReadingPlan]]: + def plan(ctx: HouseholdContext) -> list[ReadingPlan]: + estimate = ctx.total(ctx.baseline, "state_withheld_income_tax") + withheld = salt_income_tax_paid(ctx, ctx.baseline, net=net) + trace = [] + converged = False + override = Override(tax_unit={"state_withheld_income_tax": withheld}) + simulation = ctx.baseline + # A state's tax can read federal tax or the federal SALT deduction, so the + # amount paid is iterated until it reproduces itself. + for _ in range(MAX_FIXED_POINT_ITERATIONS): + override = Override(tax_unit={"state_withheld_income_tax": withheld}) + simulation = ctx.simulate(override) + implied = salt_income_tax_paid(ctx, simulation, net=net) + trace.append({"withheld": withheld, "implied": implied}) + if abs(implied - withheld) < FIXED_POINT_TOLERANCE: + converged = True + break + withheld = implied + return [ + ReadingPlan( + override, + simulation=simulation, + trace=trace, + converged=converged, + detail={ + "engine_value": estimate, + "paid": override.tax_unit["state_withheld_income_tax"], + }, + ) + ] + + return plan + + +def _stated_hours_alternative( + hours: float, +) -> Callable[[HouseholdContext], list[ReadingPlan]]: + target = "weekly_hours_worked_before_lsr" + + def plan(ctx: HouseholdContext) -> list[ReadingPlan]: + # Stated usual hours reach the target through PE_INPUT_ALIASES; a person + # without them keeps the engine default. + people = { + name: {target: hours} + for name, person in ctx.situation["people"].items() + if target not in person + } + if not people: + return [] + return [ReadingPlan(Override(people=people))] + + return plan + + +def _people_flag_alternative( + variable: str, + value: Any, + when: Callable[[HouseholdContext], dict[str, bool]], +) -> Callable[[HouseholdContext], list[ReadingPlan]]: + def plan(ctx: HouseholdContext) -> list[ReadingPlan]: + people = {name: {variable: value} for name, hit in when(ctx).items() if hit} + if not people: + return [] + return [ReadingPlan(Override(people=people))] + + return plan + + +def _receives_ssdi(ctx: HouseholdContext) -> dict[str, bool]: + values = ctx.by_person(ctx.baseline, "social_security_disability") + return {name: value > 0 for name, value in values.items()} + + +def _is_medicare_eligible(ctx: HouseholdContext) -> dict[str, bool]: + values = ctx.by_person(ctx.baseline, "is_medicare_eligible") + return {name: bool(value) for name, value in values.items()} + + +def _is_disabled(ctx: HouseholdContext) -> dict[str, bool]: + values = ctx.by_person(ctx.baseline, "is_disabled") + return {name: bool(value) for name, value in values.items()} + + +def _pre_tcja_mortgage(ctx: HouseholdContext) -> list[ReadingPlan]: + # deductible_mortgage_interest_tax_unit is the only variable that reads the + # origination years; it is interest times a deductible share, min(1, capped + # balance / balance), which raising a cap cannot lower. Where the share is + # already 1 (deductible equals interest), every output is unchanged, so no + # simulation is needed. Testing the share, not non-deductible interest, keeps + # this exact for negative interest too. + interest = ctx.total(ctx.baseline, "home_mortgage_interest_tax_unit") + deductible = ctx.total(ctx.baseline, "deductible_mortgage_interest_tax_unit") + capped = interest - deductible + if abs(capped) <= NOOP_TOLERANCE: + return [] + year = int( + ctx.engine.parameter( + "gov.irs.deductions.itemized.interest.mortgage.pre_tcja_origination_year", + f"{ctx.period}-01-01", + ) + ) + values = {} + for prefix in ("first", "second"): + if ctx.total(ctx.baseline, f"{prefix}_home_mortgage_balance") > 0: + values[f"{prefix}_home_mortgage_origination_year"] = year + if not values: + return [] + return [ + ReadingPlan( + Override(tax_unit=values), + detail={"capped_interest": capped, "origination_year": year}, + ) + ] + + +def _each_county(ctx: HouseholdContext) -> list[ReadingPlan]: + states = ctx.engine.county_states + if states is not None and ctx.scenario.state not in states: + return [] + return [ + ReadingPlan(Override(household={"county": county}), variant=county) + for county in ctx.engine.counties(ctx.scenario.state) + ] + + +ESTIMATES: tuple[UnlistedEstimate, ...] = ( + UnlistedEstimate( + id="state_withheld_income_tax", + engine_inputs=("state_withheld_income_tax",), + entity="tax_unit", + engine_behavior=( + "The income-tax line of the federal SALT deduction is " + "state_withheld_income_tax plus local_income_tax, and it counts when it " + "exceeds general sales tax. state_withheld_income_tax adds per-state " + "*_withheld_income_tax formulas on each person's federal AGI " + "(parameters/gov/states/household/state_withheld_income_tax.yaml); " + "Maryland's includes a county estimate. CO and MO formulas read it, and " + "HI, VA, SC, NM, ID, UT and AZ formulas read SALT." + ), + why_unlisted=( + "No prompt states state income tax withheld or paid during the year " + "(26 U.S.C. 164(a)(3), (b)(5))." + ), + found_in="reference_audit 2026-10-05 (PR #191)", + readings=( + Reading( + id="zero", + kind=LITERAL, + description=( + "No state income tax was withheld or paid (unlisted numeric " + "input = 0); SALT takes local income tax or general sales tax." + ), + plan=_zero_on_tax_unit("state_withheld_income_tax"), + ), + Reading( + id="liability", + kind=ALTERNATIVE, + description=( + "The household pays its own state income tax before refundable " + "credits during the year, with its Maryland county liability in " + "place of the county estimate, iterated to a fixed point." + ), + plan=_withholding_fixed_point(net=False), + ), + Reading( + id="net", + kind=ALTERNATIVE, + description=( + "As liability, net of state refundable credits and floored at 0." + ), + plan=_withholding_fixed_point(net=True), + ), + ), + ), + UnlistedEstimate( + id="local_sales_tax", + engine_inputs=("local_sales_tax",), + entity="tax_unit", + engine_behavior=( + "local_sales_tax is 0.2 times state_sales_tax (the optional state sales " + "tax table amount; for 2026 the reference system holds the 2025 IRS " + "table) outside CT, DC, IN, KY, MA, MD, ME, MI, NJ and RI, an " + "approximation of the locality's tax " + "(variables/gov/local/tax/sales/local_sales_tax.py). Federal SALT and " + "Hawaii's SALT deduction read it." + ), + why_unlisted="No prompt states a locality or a local sales tax rate.", + found_in="reference_audit 2026-10-05 (variants.py no_local_sales)", + readings=( + Reading( + id="zero", + kind=LITERAL, + description="No local general sales tax (unlisted numeric input = 0).", + plan=_zero_on_tax_unit("local_sales_tax"), + ), + ), + ), + UnlistedEstimate( + id="medicare_part_b_premium", + engine_inputs=("medicare_part_b_premium", "takes_up_medicare_if_eligible"), + entity="person", + engine_behavior=( + "takes_up_medicare_if_eligible defaults to true, so every " + "Medicare-eligible person is medicare_enrolled and is charged a modeled " + "medicare_part_b_premium, net of Medicare Savings Program coverage. " + "medical_expense_health_insurance_premiums adds it unless a direct " + "health_insurance_premiums input is set, which no frozen household does." + ), + why_unlisted=( + "No prompt states Medicare enrollment or a Part B premium, and the " + "prompt says not to infer unlisted expenses or health coverage." + ), + found_in="reference_audit 2026-10-05 (variants.py no_part_b)", + readings=( + Reading( + id="zero", + kind=LITERAL, + description="No Part B premium is paid (unlisted expense = 0).", + plan=_zero_on_people("medicare_part_b_premium"), + ), + Reading( + id="not_enrolled", + kind=ALTERNATIVE, + description=( + "A Medicare-eligible person is not enrolled, so nothing that " + "reads medicare_enrolled applies. The prompt lists no health " + "coverage but also says to assume program take-up, so this is " + "an alternative reading, not the literal one." + ), + plan=_people_flag_alternative( + "takes_up_medicare_if_eligible", False, _is_medicare_eligible + ), + ), + ), + ), + UnlistedEstimate( + id="county", + engine_inputs=("county",), + entity="household", + engine_behavior=( + "A household with no county takes its state's alphabetically first " + "county (first_county_in_state): Allegany County for Maryland, whose " + "county rate then sets the county withholding estimate and liability. " + "County also sets locality flags (in_nyc, in_san_francisco and others) " + "and county-level program rules." + ), + why_unlisted="Prompts state the state, never the county.", + found_in=( + "reference_audit 2026-09-28 (latest_md_local_output_scope) and 2026-10-05" + ), + readings=( + Reading( + id="each_county", + kind=ALTERNATIVE, + description=( + "Each other county of the household's state. A county that " + "switches on a locality flag (living in NYC or San Francisco) " + "makes an unlisted fact true, which the prompt's rule makes " + "false; its moves are marked prompt_rules_out." + ), + plan=_each_county, + ), + ), + ), + UnlistedEstimate( + id="weekly_hours_worked_before_lsr", + engine_inputs=("weekly_hours_worked_before_lsr",), + entity="person", + engine_behavior=( + "SNAP's work rules and the TANF rules of HI, MA, MT, DC and OK read " + "weekly_hours_worked_before_lsr (some through weekly_hours_worked). " + "Stated usual hours reach it through PE_INPUT_ALIASES; otherwise the " + "2.15.17 default is 0 (the literal reading; it was 40 before upstream " + "#9261)." + ), + why_unlisted="Most prompts list no hours worked.", + found_in="reference_audit 2026-09-22 root cause r14 (weekly hours)", + readings=( + Reading( + id="forty_hours", + kind=ALTERNATIVE, + description=( + "A person with no stated hours works 40 hours a week (the " + "1.755.4 default and the 22c board's reading)." + ), + plan=_stated_hours_alternative(40.0), + ), + ), + ), + UnlistedEstimate( + id="mortgage_origination_year", + engine_inputs=( + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", + ), + entity="tax_unit", + engine_behavior=( + "An origination year of 0 (the default) takes the post-TCJA " + "acquisition-debt cap ($750,000; $375,000 married filing separately) " + "for the mortgage interest deduction." + ), + why_unlisted=( + "Prompts list mortgage balances and interest, never when the mortgage " + "was taken out (26 U.S.C. 163(h)(3)(F))." + ), + found_in="reference_audit 2026-10-05 follow-up", + readings=( + Reading( + id="pre_tcja", + kind=ALTERNATIVE, + description=( + "The mortgage originated in or before the engine's pre-TCJA " + "origination year (2017), so the $1,000,000 cap applies." + ), + plan=_pre_tcja_mortgage, + ), + ), + ), + UnlistedEstimate( + id="months_receiving_social_security_disability", + engine_inputs=("months_receiving_social_security_disability",), + entity="person", + engine_behavior=( + "The input defaults to 0, so Medicare's under-65 route (24 months of " + "SSDI entitlement) is closed (the literal reading)." + ), + why_unlisted="Prompts list SSDI income, never how long it has been received.", + found_in=( + "reference_exclusions.json (head_medicare_eligible records, 2026-09-05)" + ), + readings=( + Reading( + id="twenty_four_months", + kind=ALTERNATIVE, + description=( + "A person with listed SSDI income has received it for 24 months." + ), + plan=_people_flag_alternative( + "months_receiving_social_security_disability", 24, _receives_ssdi + ), + ), + ), + ), + UnlistedEstimate( + id="meets_ssi_disability_criteria", + engine_inputs=("meets_ssi_disability_criteria",), + entity="person", + engine_behavior=( + "The input defaults to false (the literal reading), so a person listed " + "as disabled does not meet SSI's disability criterion." + ), + why_unlisted=( + "Prompts say a person is disabled, never whether the disability meets " + "the Social Security definition." + ), + found_in="reference_exclusions.json (snap and ssi records, 2026-09-05)", + readings=( + Reading( + id="disabled_meets_criteria", + kind=ALTERNATIVE, + description="A person listed as disabled meets SSI's criterion.", + plan=_people_flag_alternative( + "meets_ssi_disability_criteria", True, _is_disabled + ), + ), + ), + ), +) + +ESTIMATES_BY_ID = {estimate.id: estimate for estimate in ESTIMATES} + + +def select_estimates(ids: Sequence[str] | None) -> tuple[UnlistedEstimate, ...]: + if not ids: + return ESTIMATES + unknown = [i for i in ids if i not in ESTIMATES_BY_ID] + if unknown: + raise SystemExit( + f"Unknown estimate(s): {', '.join(unknown)}. " + f"Registered: {', '.join(ESTIMATES_BY_ID)}" + ) + return tuple(ESTIMATES_BY_ID[i] for i in ids) + + +def combined_literal_plan( + ctx: HouseholdContext, estimates: Sequence[UnlistedEstimate] +) -> list[ReadingPlan]: + """Every literal reading at once: the household exactly as the prompt lists it. + + Some estimates only matter together; the federal SALT deduction falls back to + the local sales tax estimate only once the withholding estimate is gone. + """ + override = Override() + parts = [] + for estimate in estimates: + for reading in estimate.readings: + if reading.kind != LITERAL: + continue + plans = reading.plan(ctx) + if len(plans) > 1 or any(p.variant or p.simulation for p in plans): + raise ValueError( + f"literal reading {estimate.id}/{reading.id} must be one override" + ) + for plan in plans: + if ctx.is_noop(plan.override): + continue + override = override.merged(plan.override) + parts.append(f"{estimate.id}/{reading.id}") + if len(parts) < 2: + return [] + return [ReadingPlan(override, detail={"parts": parts})] + + +# --------------------------------------------------------------------------- +# One household +# --------------------------------------------------------------------------- + + +def build_reference_situation(scenario, engine: Engine, patch=None) -> dict: + """The situation the reference builder simulates for ``scenario``.""" + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in REFERENCE_INPUT_RENAMES.items(): + if ( + old in person + and not engine.has_variable(old) + and engine.has_variable(new) + ): + person[new] = person.pop(old) + if patch is not None: + situation = patch(copy.deepcopy(situation), scenario) + return situation + + +@dataclass(frozen=True) +class HouseholdJob: + scenario_json: str + variables: tuple[str, ...] + estimate_ids: tuple[str, ...] + combined: bool = True + tolerance: float = TOLERANCE + + +def _reading_record( + estimate_id: str, + reading_id: str, + kind: str, + plan: ReadingPlan, + noop: bool, + outputs: dict[str, float] | None, + localities: Sequence[str] = (), + locality_outputs: Sequence[str] = (), +) -> dict[str, Any]: + return { + "estimate": estimate_id, + "reading": reading_id, + "kind": kind, + "variant": plan.variant, + "override": plan.override.as_dict(), + "override_text": plan.override.describe(), + "noop": noop, + "converged": plan.converged, + "iterations": len(plan.trace), + "trace": plan.trace, + "detail": plan.detail, + "localities": list(localities), + "locality_outputs": list(locality_outputs), + "outputs": outputs, + } + + +def sweep_household(job: HouseholdJob, engine: Engine, patch=None) -> dict[str, Any]: + """The reference and every applicable reading for one household.""" + from policybench.scenarios import scenario_from_dict + + started = time.perf_counter() + scenario = scenario_from_dict(json.loads(job.scenario_json)) + situation = build_reference_situation(scenario, engine, patch) + ctx = HouseholdContext( + scenario=scenario, + situation=situation, + variables=list(job.variables), + engine=engine, + ) + ctx.baseline = engine.simulate(copy.deepcopy(situation)) + baseline_outputs = ctx.outputs(ctx.baseline) + scope_components = dict.fromkeys( + engine.removed_local_components + engine.remaining_local_components + ) + local_taxes = { + name: ctx.total(ctx.baseline, name) + for name in scope_components + if engine.has_variable(name) + } + + def localities_on(simulation: Any) -> set[str]: + return { + flag for flag in engine.locality_flags if ctx.total(simulation, flag) > 0 + } + + baseline_localities = localities_on(ctx.baseline) + + def moved(variable: str, value: float) -> bool: + return output_moved( + baseline_outputs[variable], + value, + binary=is_binary_output(variable), + tolerance=job.tolerance, + ) + + def locality_only( + override: Override, switched_on: Sequence[str], values: dict[str, float] + ) -> list[str]: + """Outputs whose move is the switched-on locality's doing alone. + + The reading is simulated again with every flag it switched on held at its + reference value; an output that then stays put moved only through the + locality's own rules. One that still moves (NY SNAP's utility allowance + region reads the county directly) depends on the county itself. + """ + if any(engine.variable_entity(f) != "household" for f in switched_on): + return [] + held = override.merged(Override(household={f: False for f in switched_on})) + held_values = ctx.outputs(ctx.simulate(held, cache=False)) + return sorted( + variable + for variable, value in values.items() + if moved(variable, value) and not moved(variable, held_values[variable]) + ) + + estimates = [ESTIMATES_BY_ID[i] for i in job.estimate_ids] + readings = [] + output_cache: dict[str, tuple[dict[str, float], list[str], list[str]]] = {} + + def record_reading(estimate_id: str, reading_id: str, kind: str, plan: ReadingPlan): + noop = plan.override.is_empty() or ctx.is_noop(plan.override) + outputs = None + localities: list[str] = [] + locality_outputs: list[str] = [] + if not noop: + key = plan.override.key() + if key not in output_cache: + simulation = plan.simulation or ctx.simulate( + plan.override, cache=not plan.variant + ) + values = ctx.outputs(simulation) + switched_on = sorted(localities_on(simulation) - baseline_localities) + only = locality_only(plan.override, switched_on, values) + output_cache[key] = (values, switched_on, only if switched_on else []) + outputs, localities, locality_outputs = output_cache[key] + readings.append( + _reading_record( + estimate_id, + reading_id, + kind, + plan, + noop, + outputs, + localities, + locality_outputs, + ) + ) + + for estimate in estimates: + for reading in estimate.readings: + for plan in reading.plan(ctx): + record_reading(estimate.id, reading.id, reading.kind, plan) + if job.combined: + for plan in combined_literal_plan(ctx, estimates): + record_reading(COMBINED_ESTIMATE, COMBINED_READING, LITERAL, plan) + + return { + "scenario_id": scenario.id, + "state": scenario.state, + "baseline": baseline_outputs, + "local_taxes": local_taxes, + "baseline_localities": sorted(baseline_localities), + "readings": readings, + "simulations": 1 + ctx.simulations, + "aggregate_local_components": list(engine.aggregate_local_components), + "removed_local_components": list(engine.removed_local_components), + "remaining_local_components": list(engine.remaining_local_components), + "seconds": round(time.perf_counter() - started, 3), + } + + +# --------------------------------------------------------------------------- +# Worker processes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EngineConfig: + fix_path: str | None + output_scope_adapter: bool = True + county_states: tuple[str, ...] | None = None + + +def load_fix_module(path: str | Path): + """Import a fix module (``reform`` and/or ``patch(situation, scenario)``).""" + path = Path(path) + digest = hashlib.sha1(str(path).encode()).hexdigest()[:8] + name = f"policybench_fix_{path.stem}_{digest}" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def compose_reform(fix_reform, adapter: bool): + """The fix module's reform, then the output-scope adapter. + + Returns the composed Reform subclass and a dict that receives, when the system + is built, the local taxes the fix leaves in the state aggregate (the ones the + adapter removes). + """ + from policyengine_core.reforms import Reform + + seen: dict[str, list[str]] = {} + + class composed(Reform): + def apply(self): + if fix_reform is not None: + fix_reform.apply(self) + # The engine can apply a reform more than once; a later pass sees the + # lists the adapter already scoped, so keep the first observation. + seen.setdefault( + "listed", output_scope.all_listed_local_components(self.parameters) + ) + if adapter: + self.modify_parameters(output_scope.remove_local_components) + + return composed, seen + + +def build_engine(config: EngineConfig) -> tuple[Engine, Any]: + from policyengine_us import CountryTaxBenefitSystem, Simulation + + fix_reform = patch = None + if config.fix_path: + module = load_fix_module(config.fix_path) + fix_reform = getattr(module, "reform", None) + patch = getattr(module, "patch", None) + reform, seen = compose_reform(fix_reform, config.output_scope_adapter) + system = CountryTaxBenefitSystem(reform=reform) + listed = list(seen.get("listed", ())) + aggregate = output_scope.listed_local_components(system.parameters) + remaining = output_scope.all_listed_local_components(system.parameters) + removed = [name for name in listed if name not in remaining] + engine = Engine( + system=system, + simulation_class=Simulation, + aggregate_local_components=aggregate, + removed_local_components=removed, + remaining_local_components=remaining, + county_states=( + frozenset(config.county_states) + if config.county_states is not None + else None + ), + ) + return engine, patch + + +_WORKER: dict[str, Any] = {} + + +def _init_worker(config: EngineConfig) -> None: + engine, patch = build_engine(config) + _WORKER["engine"] = engine + _WORKER["patch"] = patch + + +def _run_job(job: HouseholdJob) -> dict[str, Any]: + return sweep_household(job, _WORKER["engine"], _WORKER["patch"]) + + +def _print_progress(done: int, total: int, result: dict[str, Any]) -> None: + print( + f"[{done}/{total}] {result['scenario_id']} {result['state']}: " + f"{result['simulations']} simulations in {result['seconds']:.0f} s", + file=sys.stderr, + flush=True, + ) + + +def run_jobs( + jobs: Sequence[HouseholdJob], + config: EngineConfig, + workers: int, + progress: Callable[[int, int, dict[str, Any]], None] | None = _print_progress, +) -> list[dict[str, Any]]: + """Run every job, building the system once per worker process. + + Results come back in completion order; ``evaluate`` sorts them. + """ + results = [] + + def collect(result: dict[str, Any]) -> None: + results.append(result) + if progress is not None: + progress(len(results), len(jobs), result) + + if workers <= 1: + _init_worker(config) + for job in jobs: + collect(_run_job(job)) + return results + os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") + context = multiprocessing.get_context("spawn") + with ProcessPoolExecutor( + max_workers=workers, + mp_context=context, + initializer=_init_worker, + initargs=(config,), + ) as pool: + futures = [pool.submit(_run_job, job) for job in jobs] + for future in as_completed(futures): + collect(future.result()) + return results + + +# --------------------------------------------------------------------------- +# Comparison and report +# --------------------------------------------------------------------------- + + +def is_binary_output(variable: str) -> bool: + from policybench.spec import metric_type_for_output + + return metric_type_for_output(variable) == "binary" + + +def output_moved( + baseline: float, value: float, *, binary: bool, tolerance: float = TOLERANCE +) -> bool: + """A flag moves when it flips; an amount when it moves past the tolerance.""" + if binary: + return round(baseline) != round(value) + return abs(value - baseline) > tolerance + + +def reproduces(reference: float, value: float, *, binary: bool) -> bool: + if binary: + return round(reference) == round(value) + return abs(value - reference) <= BASELINE_TOLERANCE + + +def exclusion_names_inputs(record: dict | None, inputs: Iterable[str]) -> bool: + """Whether an exclusion record's ``unlisted_input`` names one of ``inputs``. + + Matched as whole identifiers, so ``county`` does not match ``county_fips``. The + field is prose in some records, so a word such as ``county`` still matches + prose that uses it. + """ + if not record: + return False + tokens = set(re.findall(r"[A-Za-z0-9_]+", str(record.get("unlisted_input") or ""))) + return any(name in tokens for name in inputs) + + +def _acknowledgement_key(entry: dict) -> tuple[str, str, str, str]: + return ( + entry["scenario_id"], + entry["variable"], + entry["estimate"], + entry.get("reading", "*"), + ) + + +def load_acknowledgements(path: str | Path | None) -> dict[tuple, dict]: + """Scored moves already under review (a pending decision or ruling), by key. + + The file is ``{"acknowledged": [{"scenario_id", "variable", "estimate", + "reading" (optional; any reading if absent), "status", "note"}]}``. An + ``estimate`` of ``"*"`` covers every reading of the output. + """ + if not path: + return {} + entries = json.loads(Path(path).read_text(encoding="utf-8"))["acknowledged"] + out = {} + for entry in entries: + missing = {"scenario_id", "variable", "estimate", "status"} - set(entry) + if missing: + raise SystemExit( + f"acknowledgement {entry} is missing {', '.join(sorted(missing))}" + ) + out[_acknowledgement_key(entry)] = entry + return out + + +def _acknowledgement_for( + acknowledgements: dict[tuple, dict], + sid: str, + variable: str, + estimate: str, + reading: str, +) -> dict | None: + for key in ( + (sid, variable, estimate, reading), + (sid, variable, estimate, "*"), + (sid, variable, "*", "*"), + ): + if key in acknowledgements: + return acknowledgements[key] + return None + + +def _estimate_inputs(estimate_id: str, parts: Sequence[str] = ()) -> tuple[str, ...]: + if estimate_id == COMBINED_ESTIMATE: + names = [] + for part in parts: + names.extend(ESTIMATES_BY_ID[part.split("/")[0]].engine_inputs) + return tuple(names) + return ESTIMATES_BY_ID[estimate_id].engine_inputs + + +@dataclass +class SweepReport: + baseline: pd.DataFrame + moves: pd.DataFrame + readings: pd.DataFrame + values: pd.DataFrame + summary: dict[str, Any] + + +MOVE_COLUMNS = [ + "scenario_id", + "state", + "variable", + "estimate", + "reading", + "kind", + "variant", + "reference", + "baseline", + "value", + "delta", + "status", + "scored", + "excluded_reason", + "exclusion_names_input", + "acknowledged_status", + "localities", + "override", +] + + +def evaluate( + results: Sequence[dict[str, Any]], + reference: pd.Series, + exclusions: dict[tuple[str, str], dict], + acknowledgements: dict[tuple, dict] | None = None, + tolerance: float = TOLERANCE, +) -> SweepReport: + """Compare every reading with the reference as recomputed, and that with the CSV. + + ``reference`` is the published values indexed by (scenario_id, variable). A move + is measured against the recomputed reference (the swept system's own baseline), + so it shows what the reading alone does. + """ + import pandas as pd + + acknowledgements = acknowledgements or {} + baseline_rows = [] + move_rows = [] + reading_rows = [] + value_rows = [] + for result in sorted(results, key=lambda r: r["scenario_id"]): + sid = result["scenario_id"] + for variable, value in sorted(result["baseline"].items()): + key = (sid, variable) + record = exclusions.get(key) + published = float(reference[key]) + binary = is_binary_output(variable) + baseline_rows.append( + { + "scenario_id": sid, + "state": result["state"], + "variable": variable, + "reference": published, + "baseline": value, + "reproduces": reproduces(published, value, binary=binary), + "scored": record is None, + "excluded_reason": record["reason_code"] if record else "", + } + ) + for reading in result["readings"]: + moved_here = 0 + parts = reading["detail"].get("parts", ()) + inputs = _estimate_inputs(reading["estimate"], parts) + outputs = reading["outputs"] + if outputs is not None: + for variable, value in sorted(outputs.items()): + base = result["baseline"][variable] + value_rows.append( + { + "scenario_id": sid, + "variable": variable, + "estimate": reading["estimate"], + "reading": reading["reading"], + "variant": reading["variant"], + "baseline": base, + "value": value, + } + ) + binary = is_binary_output(variable) + if not output_moved( + base, value, binary=binary, tolerance=tolerance + ): + continue + moved_here += 1 + key = (sid, variable) + record = exclusions.get(key) + names_input = exclusion_names_inputs(record, inputs) + ack = _acknowledgement_for( + acknowledgements, + sid, + variable, + reading["estimate"], + reading["reading"], + ) + if record is not None: + status = ( + "excluded_same_input" + if names_input + else "excluded_other_reason" + ) + elif variable in reading.get("locality_outputs", ()): + status = "prompt_rules_out" + elif ack is not None: + status = "acknowledged" + else: + status = "scored" + move_rows.append( + { + "scenario_id": sid, + "state": result["state"], + "variable": variable, + "estimate": reading["estimate"], + "reading": reading["reading"], + "kind": reading["kind"], + "variant": reading["variant"], + "reference": float(reference[key]), + "baseline": base, + "value": value, + "delta": value - base, + "status": status, + "scored": record is None, + "excluded_reason": record["reason_code"] if record else "", + "exclusion_names_input": names_input, + "acknowledged_status": ack["status"] if ack else "", + "localities": ";".join(reading["localities"]), + "override": reading["override_text"], + } + ) + reading_rows.append( + { + "scenario_id": sid, + "state": result["state"], + "estimate": reading["estimate"], + "reading": reading["reading"], + "kind": reading["kind"], + "variant": reading["variant"], + "noop": reading["noop"], + "converged": reading["converged"], + "iterations": reading["iterations"], + "outputs_moved": moved_here, + "localities": ";".join(reading["localities"]), + "override": reading["override_text"], + "detail": json.dumps(reading["detail"], sort_keys=True), + } + ) + + baseline = pd.DataFrame(baseline_rows) + moves = pd.DataFrame(move_rows, columns=MOVE_COLUMNS) + readings = pd.DataFrame(reading_rows) + values = pd.DataFrame(value_rows) + summary = summarize(results, baseline, moves, readings) + return SweepReport(baseline, moves, readings, values, summary) + + +def summarize( + results: Sequence[dict[str, Any]], + baseline: pd.DataFrame, + moves: pd.DataFrame, + readings: pd.DataFrame, +) -> dict[str, Any]: + + mismatched = baseline[~baseline["reproduces"] & baseline["scored"]] + scope = [ + {"scenario_id": r["scenario_id"], "local_taxes": r["local_taxes"]} + for r in results + if any(abs(v) > 0.005 for v in r["local_taxes"].values()) + ] + unconverged = readings[~readings["converged"]] if not readings.empty else readings + per_reading = [] + if not readings.empty: + grouped = readings.groupby(["estimate", "reading", "kind"], sort=True) + for (estimate, reading, kind), frame in grouped: + active = frame[~frame["noop"]] + these = moves[ + (moves["estimate"] == estimate) & (moves["reading"] == reading) + ] + per_reading.append( + { + "estimate": estimate, + "reading": reading, + "kind": kind, + "households_planned": int(frame["scenario_id"].nunique()), + "households_changed": int(active["scenario_id"].nunique()), + "simulated_variants": int(len(active)), + "outputs_moved": int( + these[["scenario_id", "variable"]].drop_duplicates().shape[0] + ), + "scored_outputs_moved": int( + these[these["status"] == "scored"][["scenario_id", "variable"]] + .drop_duplicates() + .shape[0] + ), + } + ) + by_output = [] + if not moves.empty: + for (sid, variable), frame in moves.groupby( + ["scenario_id", "variable"], sort=True + ): + by_output.append( + { + "scenario_id": sid, + "variable": variable, + "status": _strongest_status(frame["status"]), + "excluded_reason": frame["excluded_reason"].iloc[0], + "estimates": sorted(set(frame["estimate"])), + "localities": sorted( + { + flag + for text in frame["localities"].fillna("") + for flag in str(text).split(";") + if flag + } + ), + "readings": sorted(set(frame["estimate"] + "/" + frame["reading"])), + "reference": float(frame["reference"].iloc[0]), + "baseline": float(frame["baseline"].iloc[0]), + "min_value": float(frame["value"].min()), + "max_value": float(frame["value"].max()), + } + ) + return { + "outputs": int(len(baseline)), + "scored_outputs": int(baseline["scored"].sum()) if len(baseline) else 0, + "households": int(len(results)), + "simulations": int(sum(r["simulations"] for r in results)), + "baseline_scored_mismatches": mismatched[ + ["scenario_id", "variable", "reference", "baseline"] + ].to_dict("records"), + "baseline_excluded_mismatches": int( + (~baseline["reproduces"] & ~baseline["scored"]).sum() + ) + if len(baseline) + else 0, + "scope_violations": scope, + "unconverged": unconverged[["scenario_id", "estimate", "reading"]].to_dict( + "records" + ) + if len(unconverged) + else [], + "per_reading": per_reading, + "moved_outputs": by_output, + "counts": { + status: sum(1 for o in by_output if o["status"] == status) + for status in _STATUS_ORDER + }, + } + + +# Strongest first: an output's status is the strongest of its moves. A move is +# scored unless the output is excluded, the reading switches on a locality fact the +# prompt's rule makes false (prompt_rules_out), or an --acknowledged entry covers it. +_STATUS_ORDER = ( + "scored", + "acknowledged", + "prompt_rules_out", + "excluded_other_reason", + "excluded_same_input", +) + + +def _strongest_status(statuses: Iterable[str]) -> str: + present = set(statuses) + for status in _STATUS_ORDER: + if status in present: + return status + return "" + + +def render_markdown(summary: dict[str, Any], moves: pd.DataFrame) -> str: + meta = summary["run"] + lines = [ + "# Unlisted-input sweep", + "", + f"- Engine: policyengine-us {meta['policyengine_us']} " + f"(policyengine-core {meta['policyengine_core']})", + f"- Reference system: `{meta['fix_id']}`" + + (" + `policybench_output_scope`" if meta["output_scope_adapter"] else ""), + f"- Run: {meta['scenarios']} ({summary['households']} households, " + f"{summary['outputs']} outputs, {summary['scored_outputs']} scored)", + f"- Simulations: {summary['simulations']} in {meta['seconds']:.0f} s " + f"on {meta['workers']} workers", + f"- Tolerance: ${meta['tolerance']:g} for amounts; flags move when they flip", + "", + "## Gates", + "", + f"- Baseline reproduces every scored reference: " + f"{'yes' if not summary['baseline_scored_mismatches'] else 'NO'} " + f"({len(summary['baseline_scored_mismatches'])} mismatches)", + f"- Local taxes and credits in the state outputs: " + f"{'none' if not summary['scope_violations'] else 'FOUND'}" + f" (removed by the adapter and zero in every household: " + f"{', '.join(meta['removed_local_components']) or 'none'}; " + f"still listed: {', '.join(meta['remaining_local_components']) or 'none'})", + f"- Every reference row recomputed: " + f"{'yes' if not summary.get('reference_rows_not_recomputed') else 'NO'}", + f"- Fixed points converged: {'yes' if not summary['unconverged'] else 'NO'}", + f"- Scored outputs that move: {summary['counts']['scored']}", + f"- Outputs that move only where a reading puts the household in a " + f"locality (prompt_rules_out): {summary['counts']['prompt_rules_out']}", + "", + "## Readings", + "", + "| Estimate | Reading | Kind | Households changed | Simulations | " + "Outputs moved | Scored |", + "|---|---|---|---:|---:|---:|---:|", + ] + for row in summary["per_reading"]: + lines.append( + f"| {row['estimate']} | {row['reading']} | {row['kind']} | " + f"{row['households_changed']} | {row['simulated_variants']} | " + f"{row['outputs_moved']} | {row['scored_outputs_moved']} |" + ) + lines += ["", "## Outputs that move", ""] + if not summary["moved_outputs"]: + lines.append("None.") + else: + lines += [ + "| Output | Status | Published | Recomputed | Range under readings " + "| Readings |", + "|---|---|---:|---:|---|---|", + ] + for row in summary["moved_outputs"]: + status = row["status"] + if row["excluded_reason"]: + status += f" ({row['excluded_reason']})" + elif row["status"] == "prompt_rules_out": + status += f" ({', '.join(row['localities'])})" + lines.append( + f"| {row['scenario_id']} {row['variable']} | {status} | " + f"{row['reference']:,.2f} | {row['baseline']:,.2f} | " + f"{row['min_value']:,.2f} to " + f"{row['max_value']:,.2f} | {', '.join(row['readings'])} |" + ) + lines.append("") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Command +# --------------------------------------------------------------------------- + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _package_version(name: str) -> str: + try: + return version(name) + except PackageNotFoundError: + return "not installed" + + +def assemble_fix_dir(fix: Path, support: Sequence[Path], destination: Path) -> Path: + """Copy the fix module's directory and support files into ``destination``. + + Fix modules load their parts and data files from their own directory + (``Path(__file__).with_name``); the 2026-09-28 sales tax convention reads + ``r19_irs_sales_tax_2025.json``, which lives in the 2026-09-22 directory. + """ + destination.mkdir(parents=True, exist_ok=True) + for path in sorted(fix.parent.iterdir()): + if path.is_file() and path.suffix in {".py", ".json"}: + shutil.copy2(path, destination / path.name) + for path in support: + shutil.copy2(path, destination / path.name) + return destination / fix.name + + +def _county_count(state: str) -> int: + try: + from policyengine_us.variables.household.demographic.geographic.county.county_enum import ( # noqa: E501 + County, + ) + except ImportError: # pragma: no cover - the engine moved the enum + return 0 + return sum(1 for c in County if str(c.value).endswith(f", {state}")) + + +def build_jobs( + scenarios: pd.DataFrame, + reference: pd.Series, + programs: Sequence[str], + estimates: Sequence[UnlistedEstimate], + county_states: frozenset[str] | None, + combined: bool = True, + tolerance: float = TOLERANCE, +) -> list[HouseholdJob]: + """One job per household, longest first so the pool's tail is short.""" + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + + estimate_ids = tuple(e.id for e in estimates) + jobs = [] + for _, row in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(row["scenario_json"])) + variables = tuple( + v + for v in expand_programs_for_scenario(list(programs), scenario) + if (scenario.id, v) in reference.index + ) + cost = 1 + if "county" in estimate_ids and ( + county_states is None or scenario.state in county_states + ): + cost += _county_count(scenario.state) + jobs.append( + ( + cost, + scenario.id, + HouseholdJob( + row["scenario_json"], variables, estimate_ids, combined, tolerance + ), + ) + ) + jobs.sort(key=lambda item: (-item[0], item[1])) + return [job for _, _, job in jobs] + + +def _resolve(run_dir: Path, explicit: str | None, name: str) -> Path: + return Path(explicit) if explicit else run_dir / name + + +def run(args) -> int: + """The ``unlisted-input-sweep`` command. Returns the process exit code.""" + import pandas as pd + + started = time.perf_counter() + run_dir = Path(args.run_dir) + scenarios_path = _resolve(run_dir, args.scenarios, "scenarios.csv") + reference_path = _resolve(run_dir, args.reference, "reference_outputs.csv") + meta_path = Path(f"{reference_path}.meta.json") + exclusions_path = _resolve(run_dir, args.exclusions, "reference_exclusions.json") + for path in (scenarios_path, reference_path, meta_path): + if not path.exists(): + raise SystemExit(f"Missing {path}") + + programs = json.loads(meta_path.read_text(encoding="utf-8"))["programs"] + reference = pd.read_csv(reference_path).set_index(["scenario_id", "variable"])[ + "value" + ] + exclusions = {} + if exclusions_path.exists(): + for record in json.loads(exclusions_path.read_text(encoding="utf-8"))[ + "exclusions" + ]: + exclusions[(record["scenario_id"], record["variable"])] = record + acknowledgements = load_acknowledgements(args.acknowledged) + scenarios = pd.read_csv(scenarios_path) + if args.scenario: + scenarios = scenarios[scenarios["scenario_id"].isin(args.scenario)] + if scenarios.empty: + raise SystemExit("No scenarios match --scenario") + if (scenarios["country"] != "us").any(): + raise SystemExit("The unlisted-input sweep covers US scenarios only") + scope_year = int(output_scope.SCOPE_INSTANT[:4]) + years = { + int(json.loads(raw).get("year", scope_year)) + for raw in scenarios["scenario_json"] + } + if years != {scope_year}: + raise SystemExit( + f"Scenario years {sorted(years)}: the output-scope adapter rewrites the " + f"{scope_year} lists only" + ) + + estimates = select_estimates(args.estimate) + county_states = ( + None + if args.county_states in (None, "all") + else frozenset(s.strip().upper() for s in args.county_states.split(",")) + ) + jobs = build_jobs( + scenarios, + reference, + programs, + estimates, + county_states, + combined=not args.no_combined, + tolerance=args.tolerance, + ) + # Every reference row of the swept households must be recomputed, or the + # baseline gate would pass over the rows a program expansion dropped. + swept = set(scenarios["scenario_id"]) + expected = {key for key in reference.index if key[0] in swept} + planned = { + (json.loads(job.scenario_json)["id"], variable) + for job in jobs + for variable in job.variables + } + not_recomputed = sorted(expected - planned) + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + fix = Path(args.fix) if args.fix else None + support = [Path(p) for p in args.fix_support or []] + workers = args.workers or max(1, min(16, (os.cpu_count() or 2) - 2)) + with tempfile.TemporaryDirectory(prefix="unlisted_input_sweep_fix_") as tmp: + fix_files = {} + fix_path = None + if fix is not None: + fix_path = assemble_fix_dir(fix, support, Path(tmp)) + fix_files = { + p.name: _sha256(p) for p in sorted(Path(tmp).iterdir()) if p.is_file() + } + config = EngineConfig( + fix_path=str(fix_path) if fix_path else None, + output_scope_adapter=not args.no_output_scope_adapter, + county_states=tuple(sorted(county_states)) if county_states else None, + ) + print( + f"policyengine-us {_package_version('policyengine-us')}; " + f"{len(jobs)} households; {len(estimates)} estimates; {workers} workers", + flush=True, + ) + results = run_jobs(jobs, config, workers) + scope = scope_lists(results) + + report = evaluate(results, reference, exclusions, acknowledgements, args.tolerance) + summary = report.summary + summary["reference_rows_not_recomputed"] = [list(key) for key in not_recomputed] + summary["run"] = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "policyengine_us": _package_version("policyengine-us"), + "policyengine_core": _package_version("policyengine-core"), + "fix": str(fix) if fix else None, + "fix_id": fix.stem if fix else "baseline", + "fix_files_sha256": fix_files, + "output_scope_adapter": not args.no_output_scope_adapter, + **scope, + "scenarios": str(scenarios_path), + "scenarios_sha256": _sha256(scenarios_path), + "reference": str(reference_path), + "reference_sha256": _sha256(reference_path), + "exclusions": str(exclusions_path) if exclusions_path.exists() else None, + "exclusions_sha256": _sha256(exclusions_path) + if exclusions_path.exists() + else None, + "acknowledged": args.acknowledged, + "tolerance": args.tolerance, + "county_states": sorted(county_states) if county_states else "all", + "combined_literal_reading": not args.no_combined, + "workers": workers, + "seconds": round(time.perf_counter() - started, 1), + } + summary["estimates"] = [e.describe() for e in estimates] + + report.moves.to_csv(out_dir / "moves.csv", index=False) + report.readings.to_csv(out_dir / "readings.csv", index=False) + report.baseline.to_csv(out_dir / "baseline.csv", index=False) + report.values.to_csv(out_dir / "values.csv.gz", index=False) + (out_dir / "summary.json").write_text( + json.dumps(summary, indent=1, sort_keys=True, default=_json_default) + "\n", + encoding="utf-8", + ) + (out_dir / "households.json").write_text( + json.dumps( + [ + {k: v for k, v in r.items() if k != "readings"} + | { + "readings": [ + {k: v for k, v in reading.items() if k != "outputs"} + for reading in r["readings"] + if not reading["noop"] + ] + } + for r in sorted(results, key=lambda r: r["scenario_id"]) + ], + indent=1, + sort_keys=True, + default=_json_default, + ) + + "\n", + encoding="utf-8", + ) + markdown = render_markdown(summary, report.moves) + (out_dir / "report.md").write_text(markdown, encoding="utf-8") + print(markdown) + return exit_code( + summary, + strict=args.strict, + allow_baseline_mismatch=args.allow_baseline_mismatch, + ) + + +SCOPE_KEYS = ( + "aggregate_local_components", + "removed_local_components", + "remaining_local_components", +) + + +def scope_lists(results: Sequence[dict[str, Any]]) -> dict[str, list[str]]: + """The local entries the swept system removed from, or keeps in, its state lists. + + Every worker builds the same system, so every household reports the same lists. + """ + lists = {tuple(tuple(r[key]) for key in SCOPE_KEYS) for r in results} + if len(lists) > 1: + raise RuntimeError(f"workers built different systems: {sorted(lists)}") + if not lists: + return {key: [] for key in SCOPE_KEYS} + return {key: list(values) for key, values in zip(SCOPE_KEYS, lists.pop())} + + +def exit_code( + summary: dict[str, Any], *, strict: bool, allow_baseline_mismatch: bool +) -> int: + """1 when the sweep itself fails a gate; 2 when --strict finds scored moves.""" + if summary["baseline_scored_mismatches"] and not allow_baseline_mismatch: + return 1 + if summary["scope_violations"] or summary["unconverged"]: + return 1 + if summary.get("reference_rows_not_recomputed"): + return 1 + if strict and summary["counts"]["scored"]: + return 2 + return 0 + + +def _json_default(value: Any) -> Any: + if isinstance(value, float) and math.isnan(value): + return None + if hasattr(value, "item"): + return value.item() + raise TypeError(f"not JSON serializable: {type(value).__name__}") + + +def add_arguments(parser) -> None: + parser.add_argument( + "--fix", + default=None, + help=( + "Fix module defining the reference system (reform and/or " + "patch(situation, scenario)), e.g. " + "reference_audit/2026-09-28/fixes/latest_final.py" + ), + ) + parser.add_argument( + "--fix-support", + action="append", + default=None, + help="Extra file the fix module reads from its directory (repeatable)", + ) + parser.add_argument( + "--run-dir", + default=str(DEFAULT_RUN_DIR), + help="Frozen run directory with scenarios.csv, reference_outputs.csv, " + "its .meta.json and reference_exclusions.json", + ) + parser.add_argument("--scenarios", default=None, help="Override scenarios.csv") + parser.add_argument( + "--reference", default=None, help="Override reference_outputs.csv" + ) + parser.add_argument( + "--exclusions", default=None, help="Override reference_exclusions.json" + ) + parser.add_argument("--out-dir", required=True) + parser.add_argument( + "--estimate", + action="append", + default=None, + help=f"Registered estimate to sweep (repeatable; default all): " + f"{', '.join(ESTIMATES_BY_ID)}", + ) + parser.add_argument( + "--scenario", action="append", default=None, help="Restrict to a scenario id" + ) + parser.add_argument( + "--county-states", + default="all", + help="States whose counties the county estimate sweeps: 'all' (default) " + "or a comma-separated list such as MD", + ) + parser.add_argument("--workers", type=int, default=None) + parser.add_argument("--tolerance", type=float, default=TOLERANCE) + parser.add_argument( + "--acknowledged", + default=None, + help="JSON of scored moves already under review, so --strict passes them", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit 2 when any scored output moves without an exclusion or " + "acknowledgement", + ) + parser.add_argument( + "--allow-baseline-mismatch", + action="store_true", + help="Do not fail when the recomputed reference differs from the CSV " + "(previewing a new engine)", + ) + parser.add_argument( + "--no-output-scope-adapter", + action="store_true", + help="Sweep the fix module's system without policybench.output_scope", + ) + parser.add_argument( + "--no-combined", + action="store_true", + help="Skip the reading that applies every literal reading at once", + ) diff --git a/pyproject.toml b/pyproject.toml index d85bca01..d40404bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,13 @@ description = "Benchmark: can AI models calculate tax/benefit outcomes without t readme = "README.md" license = "MIT" authors = [{name = "PolicyEngine", email = "hello@policyengine.org"}] -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "litellm[caching]>=1.30", "pandas>=2.0", "numpy>=1.24", - "policyengine[us]==4.16.1", + "policyengine==6.1.2", + "policyengine-us==2.15.17", "policyengine-uk==2.89.0", "anthropic>=0.115.1", "google-genai>=2.10.0", @@ -22,6 +23,7 @@ dependencies = [ [project.optional-dependencies] dev = [ + "hypothesis>=6.100", "pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.4", @@ -44,8 +46,15 @@ policybench = ["benchmark_specs.json", "population_weights.json"] [tool.ruff] line-length = 88 -target-version = "py310" -extend-exclude = ["app/public/paper/web/index.out.ipynb"] +target-version = "py311" +# reference_audit/ holds byte-exact records of the audit's sandbox modules: the +# reference sidecar pins their sha256, so they are never reformatted. +# docs/adds0928/review_evidence/ keeps a review's probe scripts as they ran. +extend-exclude = [ + "app/public/paper/web/index.out.ipynb", + "reference_audit", + "docs/adds0928/review_evidence", +] [tool.ruff.lint] select = ["E", "F", "I", "W"] diff --git a/reference_audit/2026-09-22/README.md b/reference_audit/2026-09-22/README.md new file mode 100644 index 00000000..c48c2814 --- /dev/null +++ b/reference_audit/2026-09-22/README.md @@ -0,0 +1,94 @@ +# Reference audit, September 22, 2026 + +This directory records how the September 22 references and exclusion record were built for the 42-model board: release `dashboard-data-20260922`, and its revisions of 2026-09-24 for one more engine defect (below). Release `dashboard-data-20260922b` excluded the one output that defect moves while its fix was open upstream; release `dashboard-data-20260922c` regenerates that output with the fix, which merged the same day. The current record has 26 regenerated references and 52 outputs excluded from scoring for every model, 41 of them added in this audit on 2026-09-22. The references come from policyengine-us 1.755.4 for tax year 2026, frozen on 2026-07-03. + +## The rules + +1. A scored reference follows from the stated facts and from law published before the 2026-07-03 freeze (Max, 2026-09-22). Where the engine projected a 2026 parameter with a price index, a publication convention regenerates the reference with the amount published before the freeze, or with the last amount published. A convention changes parameter values only. +2. An engine defect fixed in policyengine-us after the freeze is regenerated with its fix, on the same engine version (Max, 2026-09-23). Eight root causes qualify: `r04` (#8839), `r09` (#9301 and #9313), `r17` (#9363), `r26` and `r27` (#9318), `r28` and `r31` (#9162), and, from release `dashboard-data-20260922c`, `r33` (#9586). +3. An engine defect not fixed upstream excludes every output its fix moves by more than $1. +4. A reference that turns on an input or definition the prompt never states is excluded. + +## What is here + +- `root_causes.json` holds each root cause and publication convention: its class, the defect or unlisted input, the law, the alternative reading, the upstream status (`upstream_fixed: true` for rule 2) and, for a defect measured on top of other sources, `measured_against`. Classes: + - `engine_defect`: the engine misapplies the law on facts the prompt states. + - `unlisted_input`: the reference turns on an input or definition the prompt never states. + - `convention`: a publication convention (rule 1). + - `superseded`: withdrawn, with the reason. + - `regenerated`: the first SNAP rule, as recorded before the conventions generalized it; its note records how it was superseded. +- `fixes/` holds the sandbox modules. Each is a `policyengine_core` Reform or a situation patch on policyengine-us 1.755.4, evaluated for tax year 2026. The IRS sales tax, California, Idaho, Maryland, Minnesota, Michigan and Missouri conventions also set the 2025 amounts their held 2026 values carry forward. `r06` has no year guard, so it applies from 2025, when Wisconsin's retirement subtraction begins, and `r04_capital_gain_distributions.py`, a faithful backport of #8839, applies in every year, as upstream does. + - The `_v2` modules are the independently verified revisions. + - `r04_capital_gain_distributions.py` is the faithful backport of upstream #8839 and regenerates the r04 references. The verified `r04_capital_gain_distributions_v2.py` also extended Wisconsin's capital gain exclusion to the distributions, which #8839 does not; that part is `r32_wi_capital_gain_distributions.py`, a defect not fixed upstream. + - `r13_hold_fy2026_v2.py` is the superseded first SNAP module, which mixed the FY2026 hold with formula corrections. `r13_hold_fy2026_v3.py` is its hold alone, and the corrections are `r26`, `r27`, `r28` and `r31`, all fixed upstream. + - The `c13v3_*` modules combine the SNAP convention with a SNAP fix, to measure the fix on top of the convention. `c13v3_upstream_plus_r30.py` measures r30 against the published SNAP value instead (the convention with the SNAP fixes merged before #9586, `c13v3_plus_upstream_snap.py`); it moves the same two outputs. `c13v3_plus_upstream_snap.py` holds the SNAP fixes merged before #9586; with r33 added, `c13v3_upstream_plus_r33.py` is the SNAP configuration the references now apply. + - `cwi_plus_r04.py` combines the Wisconsin convention with #8839, the baseline r32 is measured against, and `cwi_plus_r04_r32.py` adds r32. + - `r29_*` are the net-income sensitivity readings. From release `dashboard-data-20260922c` the sensitivity combines them with r33 (`c13v3_r26_r28_r33`, `c13v3_r28_r29n_r33`, `c13v3_r28_r29c_r33`), as the references do; r33 moves only scenario_045 SNAP, to $0 under every procedure. + - `r18_hold_all_projections.py` is the projection screen: it holds every projected 2025–2026 parameter at its last encoded value. +- `sweep_moves.csv` lists every output each module changes when all 1,984 references are recomputed under it. + - `baseline` is the value the module is measured against, named by `measured_against`: the frozen reference (`frozen`); for the SNAP defects, the SNAP convention's reference (`c_snap_hold_fy2026`); for r32, which only matters once the distributions reach federal AGI, the Wisconsin convention plus #8839 (`c_wi_published_2026+r04_capital_gain_distributions`). That combination's own rows are listed with class `baseline`, so each r32 baseline can be checked. + - `moved_over_1` marks changes over the $1 exact-match tolerance, or a flipped flag. + - The unmodified harness reproduces all 1,984 pre-audit references frozen on 2026-07-03. +- `snap_net_income_sensitivity.csv` recomputes SNAP on top of the convention with the rounding fixes and r33, under three net-income procedures in every state: the engine's floor, the nearest dollar (as upstream #9318 does), and cents kept (7 CFR 273.10(e)(1)(ii)). No scored output differs by more than the dollar tolerance between the nearest-dollar and cents-kept readings. +- `scripts/` holds the tools that build the records: + - the harness, `sweep.py` and `run_one.sh`; + - the records builder, `build_records.py`, which writes `reference_exclusions.json` and `us_adjudications.json`; + - the regeneration script, `regen_references.py`, which writes `reference_outputs.csv` and its sidecar; + - the on-convention measurement, `make_on_convention.py`; + - this package's builder, `package_audit.py`. + + They run against the local results tree and a policyengine-us 1.755.4 environment; their paths are the ones used on 2026-09-22 and 2026-09-23. +- `verification/` holds the independent reports: + - `v1`–`v4`: GPT-6 Astra lanes re-derived each first-pass fix from primary sources and reran its sweep. + - `v5a`, `v5b`: sourced every projected parameter behind a moved output. + - `v6`: settled the three reference flags the September 22 judge raised on scored outputs. + - `v7`: the adversarial reviews of the first refreeze. They found that the first SNAP convention module also corrected the engine's SNAP arithmetic. + - `v8`: the Axiom rules engine's encoding of the SNAP allotment and minimum-benefit rounding (7 U.S.C. 2017(a)), given the engine's household size, eligibility and floored net income, against the rounding-only sandbox for the 19 households the first SNAP module changed, January to September; it does not test net income rounding. Also the rounding procedures in the state SNAP manuals for the states with affected households. + - `v9`: read-only Axiom encoding-preparation lanes. Each checked whether Axiom encodes a defect's governing provision and derived the affected households' values from the statute text. + +## How the records follow + +1. An output moved only by upstream-fixed defects (and conventions) is regenerated. The regenerated value applies every convention and upstream fix together. +2. Any other output an engine defect moves by more than $1 is excluded. The exclusion names the defects not fixed upstream and records: + - the frozen value; + - the corrected value: the output's unfixed defects' fixes applied together on top of every publication convention and upstream fix, as the published references are; + - the law and the upstream status. + + The SNAP defects are measured against the SNAP convention's reference, and r32 against the Wisconsin convention plus #8839: the value that would otherwise be published. +3. An unlisted-input root cause excludes every output its alternative reading moves, with the alternative value, computed the same way. An output that moves under both an unfixed defect and an unlisted input is recorded as the engine defect, and the note names the unlisted input. An exclusion carried over from 2026-09-05 keeps its classification; its alternative value is recomputed with every convention and upstream fix (the unlisted readings are the `u_*` situation patches), and its note names a defect not fixed upstream that also moves it. The later SNAP defects also move two such outputs, 023 and 100. +4. The reference sidecar (`reference_outputs.csv.meta.json`) lists one revision per convention and per upstream fix. Each revision lists the outputs that source changes, with its module's sha256 and, for a fix, the upstream pull request. An excluded output keeps its frozen value. +5. Every reference flag the September 22 judges raised carries a developer adjudication: `affirmed`, `regenerated`, `engine_defect` or `unlisted_input`. + +## Revision of 2026-09-24: SNAP child support treatment (`r33`, release `dashboard-data-20260922b`) + +`r33_snap_child_support_treatment` was recorded on 2026-09-24 as an engine defect not fixed upstream, so rule 3 applied and release `dashboard-data-20260922b` excluded the one output it moves for every model. Its fix merged the same day; the next section records release `dashboard-data-20260922c`, which regenerates the output under rule 2. This section describes the record as it stood for release `dashboard-data-20260922b`. + +- **Defect.** policyengine-us reads `gov.usda.snap.income.deductions.child_support` as the state option in 7 CFR 273.9(c)(17): when it is true, `snap_child_support_gross_income_deduction` and `snap_gross_test_income` take legally obligated child support paid to nonhousehold members out of gross income. When it is false, `snap_child_support_deduction` deducts the payments from net income, as 7 CFR 273.9(d)(5) requires of a state that does not take the option. The parameter's values carry the opposite meaning. For 2026, 1.755.4 excludes the payments from gross income in 37 jurisdictions that USDA's 17th SNAP State Options Report lists as deducting them, and deducts them in 9 of the 14 it lists as excluding them. Michigan is a deduction state in the 16th edition ("Treatment of Child Support Payments", p. 15) and the 17th (p. 21), as in the 14th. The 15th, which reports FY 2023 choices, lists it as an exclusion state; policyengine-us#9586 found no Michigan policy that adopted an exclusion for FY 2023 and keeps Michigan a deduction state in every year. Michigan's Bridges Eligibility Manual 556 (11-1-2025) enters child support at line 20, after gross income at line 10. The engine excludes it from gross income. +- **Upstream.** For release `dashboard-data-20260922b` the fix was open: PolicyEngine/policyengine-us#9586, head `3f156660320436e02258a94b40bc6e7ba1d7208e`. It changes the parameter's values and no formula. +- **Fix modules.** For release `dashboard-data-20260922b`, `fixes/r33_snap_child_support_treatment.py` embedded that head's `child_support.yaml` verbatim (it now embeds the merged file; see the next section): all 53 jurisdictions and every dated value. It replaces each jurisdiction's history from 2010-01-01. The reformed system's parameter matches the YAML value for value in every jurisdiction, and it changes the 2026 value in 46 jurisdictions. `fixes/c13v3_plus_r33.py` adds r33 to the SNAP convention, and `fixes/c13v3_upstream_plus_r33.py` adds it to the convention plus every upstream SNAP fix (the SNAP value actually published). +- **Sweep.** All 1,984 references were recomputed under each module. The four households that list child support paid are 014 (WV), 015 (IN), 045 (MI) and 074 (LA). On the frozen engine, r33 alone changes one output, `scenario_045` SNAP, from 287.68 to 0. On the SNAP convention it changes the same output, from 286.08 to 0. On the published SNAP configuration it again changes only that output, from 288.00 to 0. Every other output is unchanged to the cent under all three. `sweep_moves.csv` records the move against the SNAP convention (`measured_against` `c_snap_hold_fy2026`, as for r30), and `make_on_convention.py` writes both measurements. +- **The household.** The Michigan household is one person with $5,200 a year of child support paid and no elderly or disabled member. With the payments excluded, SNAP gross income is $2,589.63 a month, 198.6 percent of the poverty guideline in January. That is within the 200 percent gross limit of Michigan's broad-based categorical eligibility, so the household is eligible and receives the $24 monthly minimum. With the payments counted, gross income is $3,022.97, 231.8 percent of the guideline. The household is then not categorically eligible, fails the 130 percent gross income test, and receives $0. Net income is $1,102 a month either way. +- **Record.** + - `scenario_045` SNAP is excluded as `reference_engine_defect`, root cause r33, decided 2026-09-24. The exclusion names r28, fixed upstream, as also moving the output. + - An excluded output keeps its frozen value, so its reference returns from the regenerated 288.00 to the frozen 287.68. + - The frozen bundle's derivation narrative for that value called the annual $287.68 a monthly benefit and an average, credited the October minimum to changes in the standard and shelter deductions, and left out the child support the engine subtracts from gross income. `scripts/regen_references.py` rewrites it from the frozen engine's trace, as for 080 and 091: `FROZEN_NARRATIVES` holds the engine facts, `FROZEN_REQUIRED` the figures it must state ($433.33 and $287.68), and `HAND_CORRECTED` the published text, which replaces a writer draft that credited the October change to the poverty guideline. The narrative attributes the exclusion to PolicyEngine's child support parameter and sums the twelve monthly minimums, $23.84 for nine months and $24.3744 (8% of the projected $304.68 maximum) for three, to $287.68. `tests/test_reference_audit.py` checks it against the committed tables. + - The sidecar's `c_snap_hold_fy2026` and `r28_snap_min_allotment_rounding` revisions no longer list the output. No other reference and no other narrative changed. + - The new narrative re-rendered the case's judge prompt, which dropped its verdict. Claude Opus 5.5 re-judged the case on 2026-09-24 through `scripts/run_audit_claude.sh`, with four engine facts added to the case's grounding in the unified audit (`grounding.csv`): the child support parameter is true for Michigan and the engine reads true as excluding child support paid from gross income; the trace cites no Michigan statute, manual or state-option election, so a diagnosis describes the exclusion as what PolicyEngine or the reference does; and the gross income, net income and allotment figures. It returned `llm_error` and `taxable_income_or_deductions` and raised no reference flag, and each diagnosis compares the answer with the reference. A run the same day without those facts stated the engine's exclusion as Michigan's rule in most diagnoses and was not used. The developer adjudication keeps the verdict beside the `engine_defect` decision. +- **Counts.** + - Exclusions go from 52 to 53. Engine-defect exclusions go from 28 to 29, across 12 root causes in 21 households, 9 of them never flagged. + - Regenerated references go from 26 to 25: 12 SNAP, 22 by conventions and 14 by upstream fixes, in 24 households. + - Adjudications go from 63 to 64. Every model is scored on 1,931 outputs. + - When #9586 merges, rule 2 applies: the reference is regenerated with the fix, and the output returns to scoring at $0. + +## Revision of 2026-09-24: the r33 fix merged (release `dashboard-data-20260922c`) + +PolicyEngine/policyengine-us#9586 was squash-merged on 2026-09-24 as `d9e801df417352b8246a4c292a19ec082a518790`, so rule 2 applies to r33: its output is regenerated with the fix, not excluded. + +- **Root cause.** `root_causes.json` marks r33 `upstream_fixed`, with `upstream` naming the merged pull request and a dated note on both releases. r33 is the eighth root cause fixed upstream. +- **Fix modules.** `fixes/r33_snap_child_support_treatment.py` now embeds the merged `child_support.yaml`. Between the head the previous revision used and the merge, the pull request changed North Carolina, Missouri and Louisiana to exclusion states from 2010, Vermont's fiscal year 2024 to a deduction, and several states' values before 2017; Michigan is a deduction state throughout, as before. The 2026 value still changes in 46 jurisdictions. +- **Sweep.** All 1,984 references were recomputed with the merged values: r33 still moves only `scenario_045` SNAP, to $0, on the frozen engine, on the SNAP convention and on the published SNAP configuration. The Louisiana change reaches `scenario_074`, a Louisiana household that pays child support, but its SNAP is $0 with or without it; no household in the benchmark is in Vermont. +- **Record.** + - `scenario_045` SNAP is regenerated at $0 with r28 and r33 (`regenerated_by_fix`), and its exclusion and the adjudication added for it on 2026-09-24 are removed. `scripts/regen_references.py` holds the regenerated value's derivation, read from an engine run with the fix. + - The reference sidecar dates every revision 2026-09-22, the audit wave's date; the r33 revision's `upstream` field gives the fix's merge date, 2026-09-24. + - The SNAP net-income sensitivity adds r33 to its three procedures (`c13v3_r26_r28_r33`, `c13v3_r28_r29n_r33`, `c13v3_r28_r29c_r33`); r33 moves only `scenario_045` SNAP, to $0 under every procedure. +- **Counts.** Exclusions go from 53 to 52, engine-defect exclusions from 29 to 28 (11 root causes, 20 households, 8 never flagged), and adjudications from 64 to 63. Regenerated references go from 25 to 26: 13 SNAP, 23 by conventions and 15 by upstream fixes, in 24 households. Every model is scored on 1,932 outputs, as on release `dashboard-data-20260922`; the one difference from that release is `scenario_045` SNAP's reference, $0 instead of $288. diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r26.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r26.py new file mode 100644 index 00000000..03b32a5d --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r26.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r26: r13_hold_fy2026_v3 plus r26_snap_contribution_rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding']] +FIX_ID = 'c13v3_plus_r26' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r26_snap_contribution_rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r27.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r27.py new file mode 100644 index 00000000..666227d5 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r27.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r27: r13_hold_fy2026_v3 plus r27_snap_net_income_rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r27_snap_net_income_rounding']] +FIX_ID = 'c13v3_plus_r27' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r27_snap_net_income_rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r28.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r28.py new file mode 100644 index 00000000..8be817c8 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r28.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r28: r13_hold_fy2026_v3 plus r28_snap_min_allotment_rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r28_snap_min_allotment_rounding']] +FIX_ID = 'c13v3_plus_r28' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r28_snap_min_allotment_rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r30.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r30.py new file mode 100644 index 00000000..9e1c9925 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r30.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r30: r13_hold_fy2026_v3 plus r30_snap_heat_and_eat_sua, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r30_snap_heat_and_eat_sua']] +FIX_ID = 'c13v3_plus_r30' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r30_snap_heat_and_eat_sua' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r31.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r31.py new file mode 100644 index 00000000..f965f053 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r31.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r31: r13_hold_fy2026_v3 plus r31_snap_income_limit_rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r31_snap_income_limit_rounding']] +FIX_ID = 'c13v3_plus_r31' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r31_snap_income_limit_rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_r33.py b/reference_audit/2026-09-22/fixes/c13v3_plus_r33.py new file mode 100644 index 00000000..a27fad43 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_r33.py @@ -0,0 +1,25 @@ +"""c13v3_plus_r33: r13_hold_fy2026_v3 plus r33_snap_child_support_treatment, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r33_snap_child_support_treatment']] +FIX_ID = 'c13v3_plus_r33' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r33_snap_child_support_treatment' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_plus_upstream_snap.py b/reference_audit/2026-09-22/fixes/c13v3_plus_upstream_snap.py new file mode 100644 index 00000000..4fb784c8 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_plus_upstream_snap.py @@ -0,0 +1,25 @@ +"""c13v3_plus_upstream_snap: r13_hold_fy2026_v3 plus every SNAP fix merged upstream (r26, r27, r28, r31), to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding', 'r27_snap_net_income_rounding', 'r28_snap_min_allotment_rounding', 'r31_snap_income_limit_rounding']] +FIX_ID = 'c13v3_plus_upstream_snap' +DESCRIPTION = 'r13_hold_fy2026_v3 plus every SNAP fix merged upstream (r26, r27, r28, r31)' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r26_r28.py b/reference_audit/2026-09-22/fixes/c13v3_r26_r28.py new file mode 100644 index 00000000..7bb322f5 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r26_r28.py @@ -0,0 +1,25 @@ +"""c13v3_r26_r28: r13_hold_fy2026_v3 plus r26 and r28 (engine floor on net income), to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding', 'r28_snap_min_allotment_rounding']] +FIX_ID = 'c13v3_r26_r28' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r26 and r28 (engine floor on net income)' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r26_r28_r33.py b/reference_audit/2026-09-22/fixes/c13v3_r26_r28_r33.py new file mode 100644 index 00000000..67c98345 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r26_r28_r33.py @@ -0,0 +1,25 @@ +"""c13v3_r26_r28_r33: r13_hold_fy2026_v3 plus r26 and r28 (engine floor on net income), to measure the defect on top of the SNAP convention, with r33 (policyengine-us#9586, merged 2026-09-24), which the references apply from release dashboard-data-20260922c.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding', 'r28_snap_min_allotment_rounding', 'r33_snap_child_support_treatment']] +FIX_ID = 'c13v3_r26_r28_r33' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r26 and r28 (engine floor on net income), with r33' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r28_r29c.py b/reference_audit/2026-09-22/fixes/c13v3_r28_r29c.py new file mode 100644 index 00000000..8643a9cf --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r28_r29c.py @@ -0,0 +1,25 @@ +"""c13v3_r28_r29c: r13_hold_fy2026_v3 plus r28 and cents-kept net income with r26 rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r28_snap_min_allotment_rounding', 'r29_snap_net_cents']] +FIX_ID = 'c13v3_r28_r29c' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r28 and cents-kept net income with r26 rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r28_r29c_r33.py b/reference_audit/2026-09-22/fixes/c13v3_r28_r29c_r33.py new file mode 100644 index 00000000..c43e889a --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r28_r29c_r33.py @@ -0,0 +1,25 @@ +"""c13v3_r28_r29c_r33: r13_hold_fy2026_v3 plus r28 and cents-kept net income with r26 rounding, to measure the defect on top of the SNAP convention, with r33 (policyengine-us#9586, merged 2026-09-24), which the references apply from release dashboard-data-20260922c.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r28_snap_min_allotment_rounding', 'r29_snap_net_cents', 'r33_snap_child_support_treatment']] +FIX_ID = 'c13v3_r28_r29c_r33' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r28 and cents-kept net income with r26 rounding, with r33' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r28_r29n.py b/reference_audit/2026-09-22/fixes/c13v3_r28_r29n.py new file mode 100644 index 00000000..d3a42d36 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r28_r29n.py @@ -0,0 +1,25 @@ +"""c13v3_r28_r29n: r13_hold_fy2026_v3 plus r28 and nearest-dollar net income with r26 rounding, to measure the defect on top of the SNAP convention.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r28_snap_min_allotment_rounding', 'r29_snap_net_nearest']] +FIX_ID = 'c13v3_r28_r29n' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r28 and nearest-dollar net income with r26 rounding' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_r28_r29n_r33.py b/reference_audit/2026-09-22/fixes/c13v3_r28_r29n_r33.py new file mode 100644 index 00000000..a9ec2b8b --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_r28_r29n_r33.py @@ -0,0 +1,25 @@ +"""c13v3_r28_r29n_r33: r13_hold_fy2026_v3 plus r28 and nearest-dollar net income with r26 rounding, to measure the defect on top of the SNAP convention, with r33 (policyengine-us#9586, merged 2026-09-24), which the references apply from release dashboard-data-20260922c.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r28_snap_min_allotment_rounding', 'r29_snap_net_nearest', 'r33_snap_child_support_treatment']] +FIX_ID = 'c13v3_r28_r29n_r33' +DESCRIPTION = 'r13_hold_fy2026_v3 plus r28 and nearest-dollar net income with r26 rounding, with r33' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r30.py b/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r30.py new file mode 100644 index 00000000..2083b588 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r30.py @@ -0,0 +1,25 @@ +"""c13v3_upstream_plus_r30: the SNAP convention, the upstream SNAP fixes and r30, for the SNAP sensitivity of r30.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding', 'r27_snap_net_income_rounding', 'r28_snap_min_allotment_rounding', 'r31_snap_income_limit_rounding', 'r30_snap_heat_and_eat_sua']] +FIX_ID = 'c13v3_upstream_plus_r30' +DESCRIPTION = 'c13v3_plus_upstream_snap plus r30_snap_heat_and_eat_sua' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r33.py b/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r33.py new file mode 100644 index 00000000..3380ddf3 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/c13v3_upstream_plus_r33.py @@ -0,0 +1,25 @@ +"""c13v3_upstream_plus_r33: the SNAP convention with every upstream SNAP fix, r33 included since policyengine-us#9586 merged (2026-09-24), to measure r33 against the SNAP value published before it (c13v3_plus_upstream_snap).""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r13_hold_fy2026_v3', 'r26_snap_contribution_rounding', 'r27_snap_net_income_rounding', 'r28_snap_min_allotment_rounding', 'r31_snap_income_limit_rounding', 'r33_snap_child_support_treatment']] +FIX_ID = 'c13v3_upstream_plus_r33' +DESCRIPTION = 'c13v3_plus_upstream_snap plus r33_snap_child_support_treatment' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/cwi_plus_r04.py b/reference_audit/2026-09-22/fixes/cwi_plus_r04.py new file mode 100644 index 00000000..5709ee01 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/cwi_plus_r04.py @@ -0,0 +1,25 @@ +"""cwi_plus_r04: the Wisconsin convention plus the #8839 backport, the baseline r32 is measured against.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r19_wi_convention', 'r04_capital_gain_distributions']] +FIX_ID = 'cwi_plus_r04' +DESCRIPTION = 'r19_wi_convention plus r04_capital_gain_distributions' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/cwi_plus_r04_r32.py b/reference_audit/2026-09-22/fixes/cwi_plus_r04_r32.py new file mode 100644 index 00000000..13752d7c --- /dev/null +++ b/reference_audit/2026-09-22/fixes/cwi_plus_r04_r32.py @@ -0,0 +1,25 @@ +"""cwi_plus_r04_r32: cwi_plus_r04 plus r32_wi_capital_gain_distributions, to measure r32 on top of it.""" +import importlib.util +from pathlib import Path + +from policyengine_core.reforms import Reform + +_HERE = Path(__file__).resolve().parent + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_PARTS = [_load(n) for n in ['r19_wi_convention', 'r04_capital_gain_distributions', 'r32_wi_capital_gain_distributions']] +FIX_ID = 'cwi_plus_r04_r32' +DESCRIPTION = 'r19_wi_convention plus r04_capital_gain_distributions plus r32_wi_capital_gain_distributions' + + +class reform(Reform): + def apply(self): + for part in _PARTS: + part.reform.apply(self) diff --git a/reference_audit/2026-09-22/fixes/ma_part_a_loss_offset.py b/reference_audit/2026-09-22/fixes/ma_part_a_loss_offset.py new file mode 100644 index 00000000..c55adae9 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/ma_part_a_loss_offset.py @@ -0,0 +1,122 @@ +"""2026-only sandbox: carry the existing MA Part A loss offset into tax. + +M.G.L. c. 62, section 2(c)(2), (c)(4): capital losses reduce Part A +interest/dividends, subject to the combined $2,000 limit already calculated +by ma_part_a_agi. Section 2(f) taxes adjusted, not gross, Part A income. +https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2 + +The primary reform retains the baseline's Part B interest classification to +isolate the confirmed dividend defect. The companion module selects the +alternative reading that the unspecified interest is ordinary Part A interest. +Neither reform restores the bank-interest exemption repealed from 2024. + +This is a narrow tax-base connection fix, not a rewrite of MA capital-loss +netting, carryovers, collectibles, or Part C. All inherited AGI rules remain. +Only scenario_081 is in MA in the 100-household frozen bundle. +""" + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import add, max_, min_ +from policyengine_us.variables.gov.states.ma.tax.income.gross_income.ma_part_a_gross_income import ( + ma_part_a_gross_income as OriginalGross, +) +from policyengine_us.variables.gov.states.ma.tax.income.taxable_income.ma_part_a_taxable_dividend_income import ( + ma_part_a_taxable_dividend_income as OriginalDividends, +) +from policyengine_us.variables.gov.states.ma.tax.income.taxable_income.ma_part_a_div_excess_exemption import ( + ma_part_a_div_excess_exemption as OriginalDividendExcess, +) +from policyengine_us.variables.gov.states.ma.tax.income.taxable_income.ma_part_a_taxable_capital_gains_income import ( + ma_part_a_taxable_capital_gains_income as OriginalGains, +) +from policyengine_us.variables.gov.states.ma.tax.income.taxable_income.ma_part_a_cg_excess_exemption import ( + ma_part_a_cg_excess_exemption as OriginalGainsExcess, +) + +FIX_ID = "ma_part_a_loss_offset" +DESCRIPTION = "2026 MA: use loss-adjusted Part A dividends in the tax and exemption allocation." + + +def build(*, ordinary_interest=False): + def gross_interest_and_dividends(tax_unit, period): + variables = ["dividend_income"] + if ordinary_interest: + variables.append("taxable_interest_income") + return add(tax_unit, period, variables) + + def adjusted_interest_and_dividends(tax_unit, period): + # ST losses reduce interest/dividends; LT losses first absorb ST gains. + # Thus the remaining 5% component cannot exceed either gross I&D or + # total Part A AGI. Reuse the engine's existing $2,000 loss calculation. + return min_( + gross_interest_and_dividends(tax_unit, period), + tax_unit("ma_part_a_agi", period), + ) + + class ma_part_a_gross_income(OriginalGross): + def formula(tax_unit, period, parameters): + base = OriginalGross.formula(tax_unit, period, parameters) + if period.start.year != 2026 or not ordinary_interest: + return base + return base + add(tax_unit, period, ["taxable_interest_income"]) + + class ma_part_a_taxable_dividend_income(OriginalDividends): + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return OriginalDividends.formula(tax_unit, period, parameters) + return max_( + 0, + adjusted_interest_and_dividends(tax_unit, period) + - tax_unit("ma_part_b_excess_exemption", period), + ) + + class ma_part_a_div_excess_exemption(OriginalDividendExcess): + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return OriginalDividendExcess.formula(tax_unit, period, parameters) + return max_( + 0, + tax_unit("ma_part_b_excess_exemption", period) + - adjusted_interest_and_dividends(tax_unit, period), + ) + + class ma_part_a_taxable_capital_gains_income(OriginalGains): + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return OriginalGains.formula(tax_unit, period, parameters) + return max_( + 0, + tax_unit("ma_part_a_agi", period) + - adjusted_interest_and_dividends(tax_unit, period) + - tax_unit("ma_part_a_div_excess_exemption", period), + ) + + class ma_part_a_cg_excess_exemption(OriginalGainsExcess): + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return OriginalGainsExcess.formula(tax_unit, period, parameters) + return max_( + 0, + tax_unit("ma_part_a_div_excess_exemption", period) + - ( + tax_unit("ma_part_a_agi", period) + - adjusted_interest_and_dividends(tax_unit, period) + ), + ) + + class PartALossOffset(Reform): + def apply(self): + for variable in ( + ma_part_a_taxable_dividend_income, + ma_part_a_div_excess_exemption, + ma_part_a_taxable_capital_gains_income, + ma_part_a_cg_excess_exemption, + ): + self.update_variable(variable) + if ordinary_interest: + self.update_variable(ma_part_a_gross_income) + + return PartALossOffset + + +reform = build() diff --git a/reference_audit/2026-09-22/fixes/ma_part_a_ordinary_interest.py b/reference_audit/2026-09-22/fixes/ma_part_a_ordinary_interest.py new file mode 100644 index 00000000..edb8ab2c --- /dev/null +++ b/reference_audit/2026-09-22/fixes/ma_part_a_ordinary_interest.py @@ -0,0 +1,19 @@ +"""2026 alternative: the prompt's unspecified $56 interest is ordinary Part A. + +Includes the confirmed dividend-loss fix. Under M.G.L. c. 62 section 2(b)(1), +ordinary taxable interest belongs in Part A unless a statutory exception applies. +This is an alternative source reading, not proof that the bank was outside MA. +The benchmark's sole MA case has no U.S.-obligation interest or other exception. +""" + +import importlib.util +from pathlib import Path + +_path = Path(__file__).with_name("ma_part_a_loss_offset.py") +_spec = importlib.util.spec_from_file_location("ma_loss_offset", _path) +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) + +FIX_ID = "ma_part_a_ordinary_interest" +DESCRIPTION = "2026 MA loss-offset fix plus the ordinary Part A reading of unspecified interest." +reform = _module.build(ordinary_interest=True) diff --git a/reference_audit/2026-09-22/fixes/r01_ira_compensation.py b/reference_audit/2026-09-22/fixes/r01_ira_compensation.py new file mode 100644 index 00000000..ef428aa6 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r01_ira_compensation.py @@ -0,0 +1,222 @@ +"""r01_ira_compensation: IRA compensation limit, spousal IRA rule, head/spouse-only +traditional IRA deduction, and the 2026 IRA dollar limits (policyengine-us 1.755.4). + +Defect (engine_defect): 1.755.4 caps IRA contributions only at the dollar limit +(ira_contribution_limit = limit.ira + catch-up). It never applies the compensation +limit of IRC 219(b)(1)(B), and the tax unit's above-the-line deduction +(`adds = gov.irs.ald.deductions`) sums traditional_ira_contributions over every +member of the tax unit, dependents included. + +Rules encoded (tax year 2026), each read this session on law.cornell.edu/uscode/text/26/219 +and IRS Publication 590-A (2025 edition, irs.gov/publications/p590a): + +1. 219(b)(1): the IRA limit is the lesser of (A) the deductible amount and + (B) the compensation includible in the individual's gross income. + 408A(c)(2) makes the same 219(b)(1) limit (less traditional contributions) + the Roth IRA limit, so the cap applies to the combined traditional + Roth + amount. The engine's existing proportional scaling (ira_contribution_scale) + then allocates the combined limit between traditional and Roth. +2. 219(f)(1) compensation = wages includible in gross income (Pub 590-A: the + amount properly shown in W-2 box 1; engine: irs_employment_income, i.e. + employment_income less pre-tax 401(k)/403(b)/health/HSA payroll deductions) + plus earned income under 401(c)(2) (Pub 590-A: net earnings from self- + employment reduced by the deductible part of SE tax and the deduction for + SE retirement plan contributions). Pub 590-A: a net SE loss is not + subtracted from wages, so SE earnings are floored at 0. Pensions, annuities, + IRA distributions, Social Security, interest, dividends, capital gains and + rents are not compensation. The engine does not include alimony_income in + IRS gross income, so the 219(f)(1) pre-2019 alimony rule adds nothing here. +3. 219(c) (Kay Bailey Hutchison spousal IRA): on a joint return, an individual + whose compensation is less than the spouse's uses own compensation plus the + spouse's compensation reduced by the spouse's IRA deduction, designated + nondeductible contributions and Roth IRA contributions + (219(c)(1)(B)(ii)(I)-(III)), i.e. the spouse's total IRA contributions. +4. 219(a): the deduction is for the individual's own contributions. On a return + the deduction belongs to the filer and, on a joint return, the spouse; a + dependent's contributions are deducted (if at all) on the dependent's own + return, never on the parents' return. traditional_ira_contributions is + zeroed for anyone who is not the tax-unit head or spouse. +5. 219(b)(5)(A)/(B) dollar amounts for 2026 from IRS Notice 2025-67 ("2026 + Amounts Relating to Retirement Plans and IRAs"; irs.gov/pub/irs-drop/n-25-67.pdf, + text read this session): deductible amount increased from $7,000 to $7,500; + the 219(b)(5)(B)(ii) age-50 catch-up increased from $1,000 to $1,100 + (indexed since 2024 by 219(b)(5)(C)(iii), added by SECURE 2.0). IRS news + release IR-2025-111 (Nov. 13, 2025) states the same two figures. 1.755.4 + projects $7,000 and $1,000 for 2026. + +Out of scope and left untouched: the 219(g) active-participant phase-out (r02), +the 408A(c)(3) Roth income phase-out, and the 401(c)(2)(A)(i) material-services +test (assumed met for Schedule C/F and partnership SE earnings). +""" + +from policyengine_core.periods import period as make_period +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r01_ira_compensation" +DESCRIPTION = ( + "Cap combined traditional+Roth IRA contributions at 219(f)(1) compensation " + "(219(b)(1)(B), 408A(c)(2)) with the 219(c) spousal rule; deduct traditional " + "IRA contributions only for the tax-unit head and spouse (219(a)); set the " + "2026 IRA dollar limit to $7,500 and catch-up to $1,100 (Notice 2025-67)." +) + +SE_EARNINGS_SOURCES = [ + # Same 1402(a) sources the engine uses in taxable_self_employment_income. + "self_employment_income", + "sstb_self_employment_income", + "farm_operations_income", + "partnership_self_employment_net_earnings", +] + + +class ira_compensation(Variable): + value_type = float + entity = Person + label = "Compensation for IRA contribution limits" + unit = USD + documentation = ( + "IRC 219(f)(1) compensation: wages includible in gross income plus " + "401(c)(2) earned income from self-employment (net earnings less the " + "deductible part of SE tax and SE retirement plan deductions), with a " + "net SE loss not offsetting wages (IRS Pub. 590-A)." + ) + definition_period = YEAR + reference = ( + "https://www.law.cornell.edu/uscode/text/26/219#f_1", + "https://www.law.cornell.edu/uscode/text/26/401#c_2", + "https://www.irs.gov/publications/p590a", + ) + + def formula(person, period, parameters): + wages = person("irs_employment_income", period) + se_earnings = ( + add(person, period, SE_EARNINGS_SOURCES) + - person("self_employment_tax_ald_person", period) + - person("self_employed_pension_contribution_ald_person", period) + ) + return wages + max_(0, se_earnings) + + +def _dollar_limit(person, period, parameters): + p = parameters(period).gov.irs.gross_income.retirement_contributions + catch_up_eligible = person("age", period) >= p.catch_up.age_threshold + return p.limit.ira + where(catch_up_eligible, p.catch_up.limit.ira, 0) + + +def _make_ira_contribution_limit(spousal_rule: bool = True): + class ira_contribution_limit(Variable): + value_type = float + entity = Person + label = "IRA contribution limit" + unit = USD + documentation = ( + "Combined traditional and Roth IRA limit: the lesser of the " + "219(b)(5) deductible amount (with the age-50 catch-up) and " + "219(f)(1) compensation, with the 219(c) spousal rule on joint " + "returns (219(b)(1), 219(c), 408A(c)(2))." + ) + definition_period = YEAR + reference = ( + "https://www.law.cornell.edu/uscode/text/26/219#b_1", + "https://www.law.cornell.edu/uscode/text/26/219#c", + "https://www.law.cornell.edu/uscode/text/26/408A#c_2", + ) + + def formula(person, period, parameters): + dollar = _dollar_limit(person, period, parameters) + comp = person("ira_compensation", period) + if not spousal_rule: + return min_(dollar, comp) + head_or_spouse = person("is_tax_unit_head_or_spouse", period) + joint = person.tax_unit("tax_unit_is_joint", period) + # The spouse's compensation (head and spouse only). + unit_comp = person.tax_unit.sum(comp * head_or_spouse) + spouse_comp = where(head_or_spouse, unit_comp - comp, 0) + # 219(c)(2): applies on a joint return to the spouse with less + # compensation. The other spouse is limited by own compensation, + # so the other spouse's total IRA contributions are + # min(desired traditional + Roth, dollar, own compensation). + desired = add( + person, + period, + [ + "traditional_ira_contributions_desired", + "roth_ira_contributions_desired", + ], + ) + own_contribution = min_(desired, min_(dollar, comp)) + unit_contribution = person.tax_unit.sum(own_contribution * head_or_spouse) + spouse_contribution = where( + head_or_spouse, unit_contribution - own_contribution, 0 + ) + spousal = joint & head_or_spouse & (comp < spouse_comp) + # 219(c)(1)(B): own compensation plus the spouse's compensation + # reduced by the spouse's deduction, nondeductible and Roth + # contributions. + spousal_comp = comp + max_(spouse_comp - spouse_contribution, 0) + return min_(dollar, where(spousal, spousal_comp, comp)) + + return ira_contribution_limit + + +def _make_traditional_ira_contributions(dependent_mask: bool = True): + class traditional_ira_contributions(Variable): + value_type = float + entity = Person + label = "Traditional IRA contributions" + unit = USD + documentation = ( + "Traditional IRA contributions deductible on this tax unit's " + "return: desired contributions scaled to the combined IRA limit " + "(dollar and compensation), for the tax-unit head and spouse only " + "(219(a))." + ) + definition_period = YEAR + reference = ( + "https://www.law.cornell.edu/uscode/text/26/219#a", + "https://www.law.cornell.edu/uscode/text/26/219#b", + ) + + def formula(person, period, parameters): + desired = person("traditional_ira_contributions_desired", period) + scale = person("ira_contribution_scale", period) + if not dependent_mask: + return desired * scale + head_or_spouse = person("is_tax_unit_head_or_spouse", period) + return desired * scale * head_or_spouse + + return traditional_ira_contributions + + +def _set_2026_dollar_limits(parameters): + # IRS Notice 2025-67: 219(b)(5)(A) $7,500; 219(b)(5)(B)(ii) $1,100 for 2026. + rc = parameters.gov.irs.gross_income.retirement_contributions + year_2026 = make_period("year:2026-01-01:1") + rc.limit.ira.update(period=year_2026, value=7_500) + rc.catch_up.limit.ira.update(period=year_2026, value=1_100) + return parameters + + +def build_reform( + dollar_limits: bool = True, + compensation_cap: bool = True, + spousal_rule: bool = True, + dependent_mask: bool = True, +): + """Build the fix, or a component of it for attribution runs.""" + + class r01_ira_compensation_reform(Reform): + def apply(self): + if dollar_limits: + self.modify_parameters(_set_2026_dollar_limits) + if compensation_cap: + self.update_variable(ira_compensation) + self.update_variable(_make_ira_contribution_limit(spousal_rule)) + if dependent_mask: + self.update_variable(_make_traditional_ira_contributions(True)) + + return r01_ira_compensation_reform + + +reform = build_reform() diff --git a/reference_audit/2026-09-22/fixes/r01_ira_compensation_v2.py b/reference_audit/2026-09-22/fixes/r01_ira_compensation_v2.py new file mode 100644 index 00000000..7c4a152a --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r01_ira_compensation_v2.py @@ -0,0 +1,117 @@ +"""r01 verification correction: restrict the deduction, preserve contributions. + +The original r01 zeros traditional_ira_contributions for a dependent even when +that person has compensation and can lawfully contribute under IRC 219(a)/(b). +The restriction belongs to the parent's return, not to that contribution. + +Reuse the reviewed original compensation and spousal calculation, but retain +PolicyEngine's unmasked underlying contribution variable. A separate +traditional_ira_deduction represents the head's and joint spouse's deduction on +this tax unit. Replace the IRA item in the federal ALD list so all its consumers +(AGI, person AGI, section 86 MAGI, student-loan MAGI and Medicaid AGI) use that +return-level restriction consistently. Preserve MA disallowance and MS federal +conformity by replacing the deduction item in those lists too. + +The new compensation cap can fall below $1. Replace the existing scale +denominator floor of $1 with a zero-only guard, avoiding over-reduction of +positive requested contributions below $1. + +All activation and parameter substitutions are confined to calendar year 2026. +The original 219(g) phase-out and Roth income-phase-out omissions stay outside +this root cause; r02 can overwrite traditional_ira_deduction when composed. + +Authorities checked in verification: +- https://www.law.cornell.edu/uscode/text/26/219 (a), (b), (c), (f) +- https://www.law.cornell.edu/uscode/text/26/408A (c)(2) +- https://www.irs.gov/publications/p590a (compensation and spousal contribution) +- https://www.irs.gov/pub/irs-drop/n-25-67.pdf (p. 4: $7,500 and $1,100) +- https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2 + (d)(1)(F): Massachusetts excludes the IRC 219 deduction. +- https://www.dor.ms.gov/sites/default/files/tax-forms/individual/80100251%202.pdf + (line 50): Mississippi IRA adjustment follows federal deductibility. +""" +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +from policyengine_core.periods import period as make_period +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +_spec = spec_from_file_location("r01_verified_original", Path(__file__).with_name("r01_ira_compensation.py")) +_original = module_from_spec(_spec) +_spec.loader.exec_module(_original) + +FIX_ID = "r01_ira_compensation_v2" +DESCRIPTION = "2026 r01 compensation/spousal/dollar limits, with head/joint-spouse-only IRA deduction separated from actual contributions." +OLD = "traditional_ira_contributions" +NEW = "traditional_ira_deduction" +YEAR_2026 = make_period("year:2026-01-01:1") +_compensation_limit = _original._make_ira_contribution_limit(True) + + +class ira_contribution_limit(Variable): + value_type = float + entity = Person + label = "IRA contribution limit (r01 verification)" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219" + + def formula(person, period, parameters): + if period.start.year != 2026: + return _original._dollar_limit(person, period, parameters) + return _compensation_limit.formula(person, period, parameters) + + +class ira_contribution_scale(Variable): + value_type = float + entity = Person + label = "IRA contribution scale with a zero-only denominator guard" + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#b_1" + + def formula(person, period, parameters): + desired = add(person, period, [OLD + "_desired", "roth_ira_contributions_desired"]) + denominator = max_(desired, 1) if period.start.year != 2026 else where(desired > 0, desired, 1) + return min_(person("ira_contribution_limit", period) / denominator, 1) + + +class traditional_ira_deduction(Variable): + value_type = float + entity = Person + label = "Traditional IRA deduction on this tax unit (before section 219(g))" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#a" + + def formula(person, period, parameters): + contributions = person(OLD, period) + if period.start.year != 2026: + return contributions + claimant = person("is_tax_unit_head", period) | ( + person("is_tax_unit_spouse", period) + & person.tax_unit("tax_unit_is_joint", period) + ) + return contributions * claimant + + +def modify_parameters(parameters): + _original._set_2026_dollar_limits(parameters) + # Explicit period updates restore the original list in 2027. + for node in ( + parameters.gov.irs.ald.deductions, + parameters.gov.states.ma.tax.income.ald.disallowed, + parameters.gov.states.ms.tax.income.adjustments.adjustments, + ): + values = [NEW if value == OLD else value for value in node("2026-01-01")] + node.update(period=YEAR_2026, value=values) + return parameters + + +class reform(Reform): + def apply(self): + self.update_variable(_original.ira_compensation) + self.update_variable(ira_contribution_limit) + self.update_variable(ira_contribution_scale) + self.update_variable(traditional_ira_deduction) + self.modify_parameters(modify_parameters) diff --git a/reference_audit/2026-09-22/fixes/r02_ira_219g.py b/reference_audit/2026-09-22/fixes/r02_ira_219g.py new file mode 100644 index 00000000..481b0def --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r02_ira_219g.py @@ -0,0 +1,451 @@ +"""r02: apply the IRC 219(g) active-participant phase-out to the traditional IRA deduction. + +Defect in policyengine-us 1.755.4 +--------------------------------- +`gov.irs.ald.deductions` (2026-01-01 list) names `traditional_ira_contributions` +directly as an above-the-line deduction. That variable is +`traditional_ira_contributions_desired * ira_contribution_scale`, which applies +only the combined 219(b)(1)(A) / 408A(c)(2) dollar cap. Nothing in the package +tests active participation or 219(g); the whole contribution is deducted at any +income. (Issue #8388 / PR #8416 left 219(g) "out of scope"; no follow-up exists.) + +Law encoded (tax year 2026) +--------------------------- +* IRC 219(g)(1): if the individual or the individual's spouse is an active + participant, the dollar limitations in 219(b)(1)(A) and 219(c)(1)(A) are + reduced (not below zero) by the amount in 219(g)(2). +* 219(g)(2)(A): reduction = limitation x (MAGI - applicable dollar amount) / + $10,000 ($20,000 on a joint return). +* 219(g)(2)(B): no limitation is reduced below $200 unless it is reduced to zero. +* 219(g)(2)(C): an amount determined under (g)(2) that is not a multiple of $10 + is rounded to the next lowest $10 (Pub. 590-A Worksheet 1-2 line 4 states the + same thing as rounding the remaining limit UP to the next $10). +* 219(g)(3)(A): MAGI is AGI determined after sections 86 and 469 and without + regard to sections 85(c), 135, 137, 221 and 911 or the 219 deduction itself. + Engine terms: gross income with full unemployment compensation (85(c) + disregarded; `unemployment_compensation` instead of the AGI-dependent + `taxable_unemployment_compensation`), plus taxable Social Security figured + WITHOUT the IRA deduction (Pub. 590-A Appendix B, Worksheet 1 lines 1-17), minus + every 2026 ALD except the IRA deduction, `student_loan_interest_ald` (221), + `us_bonds_for_higher_ed` (135) and `qualified_adoption_assistance_expense` + (137). The engine has no 911 exclusion. 931/933 items stay subtracted because + 219(g)(3)(A) does not add them back (Pub. 590-A Worksheet 1-1 lines 1-6). +* 219(g)(3)(B), 219(g)(7), 219(g)(8), and IRS Notice 2025-67 (IR-2025-111, + Nov. 13, 2025), 2026 amounts: + - active participant, joint return or qualifying widow(er): $129,000-$149,000 + - active participant, single or head of household: $81,000-$91,000 + - active participant, married filing separately: $0-$10,000 (not indexed) + - not an active participant, spouse is (219(g)(7)), joint: $242,000-$252,000 + Width is $20,000 for an active participant on a joint (or QSS, per Notice + 2025-67 and Pub. 590-A Worksheet 1-2's 35%/40% multipliers) return, and + $10,000 otherwise (219(g)(2)(A)(ii), 219(g)(7)(B)). +* 219(g)(4): spouses who file separately and live apart all year are not treated + as married; the engine flag `cohabitating_spouses` (default False) marks the + lived-together case. +* 219(g)(5): an active participant includes a participant in a 401(a) plan, a + 403(b) annuity contract, a SEP (408(k)) and a SIMPLE (408(p)). Pub. 590-A: a + person is covered by a defined contribution plan for a year in which "amounts + are contributed or allocated to your account"; the Form W-2 box 13 + instructions say the same for employee contributions. Treated as active + participation here: any positive `traditional_401k_contributions`, + `roth_401k_contributions`, `traditional_403b_contributions` or + `roth_403b_contributions` (elective deferrals, pre-tax or designated Roth, are + contributions added to the participant's 401(a)/403(b) account), provided the + person has wages or self-employment earnings to defer from (under a 401(k) + arrangement the employee elects to have the employer contribute instead of + paying cash, 401(k)(2)(A), and 415(c)(1)(B) caps annual additions at 100% of + compensation; the engine does not cap deferrals at compensation), or any positive `self_employed_pension_contributions` (the + engine's SEP/SIMPLE/Keogh input, already capped at SE income). The deferral + variables are the engine's post-402(g)-limit values, positive exactly when the + desired input is positive. No other engine input identifies plan coverage, so + employer-only contributions and defined-benefit coverage cannot be seen. + REQUIRE_COMPENSATION_FOR_DEFERRALS = False gives the "engine inputs only" + reading; in the 100 PolicyBench households it changes only scenario_003. + +Changes +------- +1. New Person variable `traditional_ira_deduction` = min(traditional IRA + contributions, the 219(g)-reduced deductible amount). The deductible amount + before reduction is the 219(b)(5) dollar limit plus catch-up, computed from + the same parameters and formula as the engine's `ira_contribution_limit`, so + the separate stale-2026-limit issue (engine $7,000/$1,000 vs. law + $7,500/$1,100, fixed in r01) is NOT changed by this fix. +2. `gov.irs.ald.deductions` (2026 onward): `traditional_ira_contributions` -> + `traditional_ira_deduction`. Every reader of the list follows: AGI, + adjusted_gross_income_person, Section 86 MAGI (which correctly includes the + 219 deduction), the 221 MAGI, taxable_uc_agi, Medicaid AGI. +3. State lists that name the old variable as a deduction are kept consistent: + - MA `gov.states.ma.tax.income.ald.disallowed` also disallows the new + variable, so MA behavior is unchanged (M.G.L. c.62 s.2(d)(1)(F)). + - MS `gov.states.ms.tax.income.adjustments.adjustments` (line 50) uses the new + variable: the 2025 MS resident instructions (Form 80-100, line 50) allow IRA + payments only "to the extent contributions are deductible for federal + income tax purposes". + `traditional_ira_contributions` itself is unchanged, so readers that need + contributions (saver's credit under 25B/219(e), MN/UT household-income + add-backs, PR, CRFB contrib, Riverside GR) are untouched. +""" + +from __future__ import annotations + +from policyengine_core.parameters import ParameterNode +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r02_ira_219g" +DESCRIPTION = ( + "IRC 219(g) active-participant phase-out of the traditional IRA deduction " + "(2026 ranges from Notice 2025-67; MAGI per 219(g)(3)(A) with Section 86 " + "benefits figured before the IRA deduction per Pub. 590-A App. B; reduction " + "rounded down to $10, $200 floor). Active participant = any 401(k)/403(b) " + "elective deferral (traditional or Roth) by a person with wages or SE " + "earnings, or any SEP/SIMPLE/Keogh contribution." +) + +START = "2026-01-01" +OLD = "traditional_ira_contributions" +NEW = "traditional_ira_deduction" + +# ALDs 219(g)(3)(A)(ii) disregards: the 219 deduction itself, 221, 135, 137. +MAGI_DISREGARDED_ALDS = ( + OLD, + NEW, + "student_loan_interest_ald", + "us_bonds_for_higher_ed", + "qualified_adoption_assistance_expense", +) + +ELECTIVE_DEFERRALS = [ + "traditional_401k_contributions", + "roth_401k_contributions", + "traditional_403b_contributions", + "roth_403b_contributions", +] +# Earned-income sources an elective deferral can come out of: wages, or +# 401(c)(2) self-employment earnings for a self-employed 401(k). +SE_EARNINGS_SOURCES = [ + "self_employment_income", + "sstb_self_employment_income", + "farm_operations_income", + "partnership_self_employment_net_earnings", +] +# Set False to reproduce the "engine inputs only" reading (deferral inputs +# count even when the person has no compensation to defer). +REQUIRE_COMPENSATION_FOR_DEFERRALS = True + +_FS = ("SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE") + + +def _by_fs(values_2025: dict, values_2026: dict) -> dict: + return { + fs: {"values": {"2025-01-01": values_2025[fs], "2026-01-01": values_2026[fs]}} + for fs in _FS + } + + +# IRS Notice 2025-67 section on 219(g) (2025 values from Notice 2024-80 as +# restated in Notice 2025-67 "increased from" clauses and Pub. 590-A (2025) +# Worksheet 1-2). +PARAMETER_DATA = { + "description": "IRC 219(g) active-participant phase-out (sandbox fix r02).", + # Applicable dollar amount when the contributor is an active participant. + "covered_start": _by_fs( + dict(SINGLE=79_000, HEAD_OF_HOUSEHOLD=79_000, JOINT=126_000, + SURVIVING_SPOUSE=126_000, SEPARATE=0), + dict(SINGLE=81_000, HEAD_OF_HOUSEHOLD=81_000, JOINT=129_000, + SURVIVING_SPOUSE=129_000, SEPARATE=0), + ), + # 219(g)(2)(A)(ii): $20,000 on a joint return (QSS per Notice 2025-67 and + # Pub. 590-A Worksheet 1-2), $10,000 otherwise. + "covered_width": _by_fs( + dict(SINGLE=10_000, HEAD_OF_HOUSEHOLD=10_000, JOINT=20_000, + SURVIVING_SPOUSE=20_000, SEPARATE=10_000), + dict(SINGLE=10_000, HEAD_OF_HOUSEHOLD=10_000, JOINT=20_000, + SURVIVING_SPOUSE=20_000, SEPARATE=10_000), + ), + # 219(g)(7)(A): contributor not active, spouse active (joint return). + "spouse_covered_start": {"values": {"2025-01-01": 236_000, "2026-01-01": 242_000}}, + # 219(g)(7)(B). + "spouse_covered_width": {"values": {"2025-01-01": 10_000, "2026-01-01": 10_000}}, + # 219(g)(3)(B)(iii): MFS (lived with spouse), not indexed. + "separate_start": {"values": {"2025-01-01": 0, "2026-01-01": 0}}, + "separate_width": {"values": {"2025-01-01": 10_000, "2026-01-01": 10_000}}, + # 219(g)(2)(C) and (B). + "rounding": {"values": {"2025-01-01": 10}}, + "floor": {"values": {"2025-01-01": 200}}, +} + + +class ira_active_participant(Variable): + value_type = bool + entity = Person + label = "Active participant in an employer plan for IRC 219(g)" + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#g_5" + + def formula(person, period, parameters): + deferrals = add(person, period, ELECTIVE_DEFERRALS) > 0 + if REQUIRE_COMPENSATION_FOR_DEFERRALS: + # A 401(k) deferral is pay the employee elects to have contributed + # instead of received in cash (401(k)(2)(A)), and 415(c)(1)(B) caps + # annual additions at 100% of compensation. With no wages or + # self-employment earnings nothing can be deferred, no amount + # reaches the account, and the person is not an active participant + # through that input (the engine does not cap deferrals at + # compensation; irs_employment_income floors at 0). + wages = max_(0, person("employment_income", period)) + se = max_(0, add(person, period, SE_EARNINGS_SOURCES)) + deferrals = deferrals & ((wages + se) > 0) + # self_employed_pension_contributions is already capped at + # self-employment income by the engine (415(c) limit variable). + sep = person("self_employed_pension_contributions", period) > 0 + return deferrals | sep + + +class ira_219g_taxable_social_security(Variable): + """Section 86 taxable benefits figured without the IRA deduction. + + Pub. 590-A Appendix B, Worksheet 1 lines 1-17. Same inputs and thresholds as + the engine's taxable_ss_magi / tax_unit_taxable_social_security, except that + the IRA deduction is not subtracted (it is what is being computed). + """ + + value_type = float + entity = TaxUnit + label = "Taxable Social Security for 219(g) MAGI (before the IRA deduction)" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#g_3_A" + + def formula(tax_unit, period, parameters): + irs = parameters(period).gov.irs + p = irs.social_security.taxability + # taxable_ss_magi, with the IRA deduction removed from the ALDs. + sources = [ + s + for s in irs.gross_income.sources + if s not in ["taxable_social_security", "taxable_unemployment_compensation"] + ] + if "taxable_unemployment_compensation" in irs.gross_income.sources: + sources.append("unemployment_compensation") + sources.append("tax_exempt_interest_income") + person = tax_unit.members + not_dependent = ~person("is_tax_unit_dependent", period) + gross = 0 + for source in sources: + gross += not_dependent * max_(0, add(person, period, [source])) + gross = tax_unit.sum(gross) + revoked = p.income.revoked_deductions + deductions = [ + d + for d in irs.ald.deductions + if d not in revoked and d not in (OLD, NEW) + ] + ss_magi = max_(0, gross - add(tax_unit, period, deductions)) + # tax_unit_taxable_social_security, with that MAGI. + gross_ss = tax_unit("tax_unit_social_security_for_taxability", period) + combined = ss_magi + p.combined_income_ss_fraction * gross_ss + filing_status = tax_unit("filing_status", period) + separate = filing_status == filing_status.possible_values.SEPARATE + cohabitating = tax_unit("cohabitating_spouses", period) + base = where( + separate & cohabitating, + p.threshold.base.separate_cohabitating, + p.threshold.base.main[filing_status], + ) + adjusted_base = where( + separate & cohabitating, + p.threshold.adjusted_base.separate_cohabitating, + p.threshold.adjusted_base.main[filing_status], + ) + excess = max_(0, combined - base) + over_adjusted = max_(0, combined - adjusted_base) + tier1 = min_(p.rate.base.benefit_cap * gross_ss, p.rate.base.excess * excess) + bracket = min_(tier1, p.rate.additional.bracket * (adjusted_base - base)) + tier2 = min_( + p.rate.additional.excess * over_adjusted + bracket, + p.rate.additional.benefit_cap * gross_ss, + ) + return select( + [combined < base, combined < adjusted_base], [0, tier1], default=tier2 + ) + + +class ira_219g_magi(Variable): + value_type = float + entity = TaxUnit + label = "Modified AGI for the IRC 219(g) IRA deduction phase-out" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#g_3_A" + + def formula(tax_unit, period, parameters): + irs = parameters(period).gov.irs + sources = [ + s + for s in irs.gross_income.sources + if s not in ["taxable_social_security", "taxable_unemployment_compensation"] + ] + if "taxable_unemployment_compensation" in irs.gross_income.sources: + # 219(g)(3)(A)(ii): without regard to 85(c). + sources.append("unemployment_compensation") + person = tax_unit.members + not_dependent = ~person("is_tax_unit_dependent", period) + gross = 0 + for source in sources: + gross += not_dependent * max_(0, add(person, period, [source])) + gross = tax_unit.sum(gross) + # 219(g)(3)(A)(i): after section 86 (benefits figured before the IRA + # deduction, Pub. 590-A App. B Worksheet 1 line 17). + gross += tax_unit("ira_219g_taxable_social_security", period) + deductions = [d for d in irs.ald.deductions if d not in MAGI_DISREGARDED_ALDS] + magi = gross - add(tax_unit, period, deductions) + if parameters(period).gov.contrib.ubi_center.basic_income.taxable: + magi += add(tax_unit, period, ["basic_income"]) + return magi + + +class ira_219g_deductible_limit(Variable): + value_type = float + entity = Person + label = "Traditional IRA deductible amount after the IRC 219(g) reduction" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219#g" + + def formula(person, period, parameters): + p = parameters(period).gov.irs.ald.ira_219g + # The dollar limitation 219(g)(1) reduces: the 219(b)(1)(A) deductible + # amount (219(b)(5)(A) plus the 219(b)(5)(B) catch-up), computed from + # the same parameters and formula as the engine's ira_contribution_limit + # (identical to it in 1.755.4). Read from parameters rather than from + # ira_contribution_limit so that, if another fix adds the 219(b)(1)(B) + # compensation cap to ira_contribution_limit, 219(g) still reduces only + # the dollar amount, as the statute says; compensation stays a separate + # cap already embedded in traditional_ira_contributions. + rc = parameters(period).gov.irs.gross_income.retirement_contributions + catch_up_eligible = person("age", period) >= rc.catch_up.age_threshold + limit = rc.limit.ira + where(catch_up_eligible, rc.catch_up.limit.ira, 0) + tax_unit = person.tax_unit + filing_status = tax_unit("filing_status", period) + fs = filing_status.possible_values + joint = filing_status == fs.JOINT + separate = filing_status == fs.SEPARATE + # 219(g)(4): MFS spouses who lived apart all year are not treated as + # married (single range, spouse's participation disregarded). + separate_married = separate & tax_unit("cohabitating_spouses", period) + separate_apart = separate & ~separate_married + self_active = person("ira_active_participant", period) + is_head = person("is_tax_unit_head", period) + is_spouse = person("is_tax_unit_spouse", period) + head_active = tax_unit.any(is_head & self_active) + spouse_active = tax_unit.any(is_spouse & self_active) + other_active = where(is_head, spouse_active, where(is_spouse, head_active, False)) + # 219(g)(7) applies to a married contributor who is not active but + # whose spouse is. (For MFS the engine keeps the spouse in another tax + # unit, so this branch can only fire on a joint return here.) + spouse_route = ~self_active & other_active & (joint | separate_married) + covered = self_active | spouse_route + # 219(g)(3)(B): applicable dollar amount; (2)(A)(ii) width. + self_start = where( + separate_married, + p.separate_start, + where(separate_apart, p.covered_start.SINGLE, p.covered_start[filing_status]), + ) + self_width = where( + separate_married, + p.separate_width, + where(separate_apart, p.covered_width.SINGLE, p.covered_width[filing_status]), + ) + start = where( + self_active, + self_start, + where(separate_married, p.separate_start, p.spouse_covered_start), + ) + width = where( + self_active, + self_width, + where(separate_married, p.separate_width, p.spouse_covered_width), + ) + magi = tax_unit("ira_219g_magi", period) + excess = max_(0, magi - start) + full_phase_out = excess >= width + raw_reduction = limit * min_(1, excess / width) + # 219(g)(2)(C): round the reduction down to the next lowest $10. + reduction = np.floor(np.round(raw_reduction, 6) / p.rounding) * p.rounding + reduced = max_(0, limit - reduction) + reduced = where(full_phase_out, 0, reduced) + # 219(g)(2)(B): $200 floor unless the limitation is reduced to zero. + reduced = where((reduced > 0) & (reduced < p.floor), p.floor, reduced) + return where(covered, reduced, limit) + + +class traditional_ira_deduction(Variable): + value_type = float + entity = Person + label = "Traditional IRA deduction (IRC 219, with the 219(g) phase-out)" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/219" + + def formula(person, period, parameters): + contributions = person(OLD, period) + return min_(contributions, person("ira_219g_deductible_limit", period)) + + +def _swap_from(node, start: str, transform) -> None: + """Apply transform to the list in force at `start` and every later dated value.""" + entries = sorted(node.values_list, key=lambda e: e.instant_str) + starts = [e.instant_str for e in entries] + if start not in starts: + # Split the entry in force at `start` so that a value begins there. + later = [s for s in starts if s > start] + stop = instant(later[0]).offset(-1, "day") if later else None + node.update(start=instant(start), stop=stop, value=list(node(start))) + entries = sorted(node.values_list, key=lambda e: e.instant_str) + starts = [e.instant_str for e in entries] + for i, entry in enumerate(entries): + if entry.instant_str < start: + continue + stop = instant(starts[i + 1]).offset(-1, "day") if i + 1 < len(entries) else None + node.update(start=instant(entry.instant_str), stop=stop, value=transform(list(entry.value))) + + +def modify_parameters(parameters): + if "ira_219g" not in parameters.gov.irs.ald.children: + parameters.gov.irs.ald.add_child( + "ira_219g", ParameterNode("gov.irs.ald.ira_219g", data=PARAMETER_DATA) + ) + _swap_from( + parameters.gov.irs.ald.deductions, + START, + lambda v: [NEW if x == OLD else x for x in v], + ) + _swap_from( + parameters.gov.states.ma.tax.income.ald.disallowed, + START, + lambda v: v + [NEW] if NEW not in v else v, + ) + _swap_from( + parameters.gov.states.ms.tax.income.adjustments.adjustments, + START, + lambda v: [NEW if x == OLD else x for x in v], + ) + return parameters + + +REFORM_VARIABLES = [ + ira_active_participant, + ira_219g_taxable_social_security, + ira_219g_magi, + ira_219g_deductible_limit, + traditional_ira_deduction, +] + + +class reform(Reform): + def apply(self): + for variable in REFORM_VARIABLES: + self.update_variable(variable) + self.modify_parameters(modify_parameters) diff --git a/reference_audit/2026-09-22/fixes/r02_ira_219g_v2.py b/reference_audit/2026-09-22/fixes/r02_ira_219g_v2.py new file mode 100644 index 00000000..173220cd --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r02_ira_219g_v2.py @@ -0,0 +1,77 @@ +"""Verified r02, with deduction-list substitutions confined to tax year 2026. + +The original phase-out arithmetic and MAGI calculations are reused unchanged. +Its `_swap_from` also replaces every future ALD list and carries unindexed 2026 +thresholds forward. This sandbox verifies 2026 only, so restore baseline lists +outside that year. No household input or contribution amount is changed. + +`mask_dependents` is enabled only by the r01+r02 composition, retaining r01's +return-level restriction while keeping the original contribution variable. +Standalone r02 still changes only active-participant deductibility. + +Sources: IRC 219(g); IRS Notice 2025-67 p.4; IRS Pub.590-A Worksheets 1-1, +1-2 and Appendix B; MA General Laws ch.62 sec.2(d)(1)(F); MS 2025 Form 80-100 +line 50. See the verification report for links and the scope limitations of +inferring coverage from the benchmark's elective-deferral inputs. +""" +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from policyengine_core.parameters import ParameterNode +from policyengine_core.periods import period as make_period +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +_spec = spec_from_file_location('r02_verified_original', Path(__file__).with_name('r02_ira_219g.py')) +_original = module_from_spec(_spec) +_spec.loader.exec_module(_original) +FIX_ID = 'r02_ira_219g_v2' +DESCRIPTION = 'IRC 219(g) phase-out, confined to 2026; same 2026 arithmetic as original r02.' +YEAR_2026 = make_period('year:2026-01-01:1') +OLD, NEW = _original.OLD, _original.NEW + + +def modify_parameters(parameters): + if 'ira_219g' not in parameters.gov.irs.ald.children: + parameters.gov.irs.ald.add_child('ira_219g', ParameterNode('gov.irs.ald.ira_219g', data=_original.PARAMETER_DATA)) + for node in ( + parameters.gov.irs.ald.deductions, + parameters.gov.states.ma.tax.income.ald.disallowed, + parameters.gov.states.ms.tax.income.adjustments.adjustments, + ): + values = [NEW if value == OLD else value for value in node('2026-01-01')] + node.update(period=YEAR_2026, value=values) + return parameters + + +def build_reform(mask_dependents=False): + class traditional_ira_deduction(Variable): + value_type = float + entity = Person + label = 'Traditional IRA deduction (verified 2026 phase-out)' + unit = USD + definition_period = YEAR + reference = 'https://www.law.cornell.edu/uscode/text/26/219' + + def formula(person, period, parameters): + contributions = person(OLD, period) + if period.start.year != 2026: + return contributions + result = min_(contributions, person('ira_219g_deductible_limit', period)) + if mask_dependents: + claimant = person('is_tax_unit_head', period) | ( + person('is_tax_unit_spouse', period) + & person.tax_unit('tax_unit_is_joint', period) + ) + result = result * claimant + return result + + class r02_verified_reform(Reform): + def apply(self): + for variable in _original.REFORM_VARIABLES[:-1]: + self.update_variable(variable) + self.update_variable(traditional_ira_deduction) + self.modify_parameters(modify_parameters) + return r02_verified_reform + + +reform = build_reform() diff --git a/reference_audit/2026-09-22/fixes/r03_estate_income__qbi_false_v2.py b/reference_audit/2026-09-22/fixes/r03_estate_income__qbi_false_v2.py new file mode 100644 index 00000000..6804c9af --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r03_estate_income__qbi_false_v2.py @@ -0,0 +1,32 @@ +"""2026 estate-income inclusion plus the unlisted-QBI-status-false reading. + +IRC 199A(c)(3) and Treasury Regulation 1.199A-6(d) require qualifying +trade-or-business items. A generic estate-income label does not establish +that status. Preserve an explicitly supplied qualification status and all +other years. This is an input interpretation, not a universal legal default. +""" + +import importlib.util +from pathlib import Path + +_spec = importlib.util.spec_from_file_location( + "r03_estate_income_base_v2", Path(__file__).with_name("r03_estate_income_v2.py") +) +_base = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_base) + +FIX_ID = "r03_estate_income__qbi_false_v2" +DESCRIPTION = "2026 estate income inclusion; set only unlisted estate QBI status false." +reform = _base.reform + + +def patch(situation, scenario): + if scenario.year != 2026: + return situation + for person in situation["people"].values(): + if "2026" not in person.get("estate_income", {}): + continue + # Each scenario uses annual string keys. Preserve explicit values. + qualification = person.setdefault("estate_income_would_be_qualified", {}) + qualification.setdefault("2026", False) + return situation diff --git a/reference_audit/2026-09-22/fixes/r03_estate_income_v2.py b/reference_audit/2026-09-22/fixes/r03_estate_income_v2.py new file mode 100644 index 00000000..9386bb20 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r03_estate_income_v2.py @@ -0,0 +1,27 @@ +"""Include Schedule E estate/trust income in gross income for 2026 only. + +IRC 61(a)(14), 102(b), 652(a), and 662(a) include the beneficiary's +taxable estate/trust income. This sandbox changes only the 2026 source list. +It does not alter QBI qualification or loss treatment. +""" + +from policyengine_core.reforms import Reform + +FIX_ID = "r03_estate_income_v2" +DESCRIPTION = "Append estate_income to federal gross-income sources in 2026 only." + + +def _add_estate_income(parameters): + node = parameters.gov.irs.gross_income.sources + current = list(node("2026-01-01")) + if "estate_income" not in current: + node.update(period="2026", value=current + ["estate_income"]) + return parameters + + +class EstateIncomeInGrossIncome2026(Reform): + def apply(self): + self.modify_parameters(_add_estate_income) + + +reform = EstateIncomeInGrossIncome2026 diff --git a/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions.py b/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions.py new file mode 100644 index 00000000..da2ab896 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions.py @@ -0,0 +1,138 @@ +"""r04: count capital gain distributions reported without Schedule D (Form 1040 line 7a). + +Backport of upstream policyengine-us PR #8839 (merge commit 491642087f, merged +2026-07-05, fixes issue #8828) to policyengine-us 1.755.4. The three files the PR +changes are byte-identical in 1.755.4 and in the PR's parent commit +(491642087f^), so the backport is the PR's own diff, applied as a reform. + +Defect in 1.755.4 +----------------- +`non_sch_d_capital_gains` ("Capital gains not reported on Schedule D", PUF E01100) +is read only by has_qdiv_or_ltcg, dwks09 and dwks10. Those Schedule D worksheet +lines feed regular_tax_before_credits, which only alternative_minimum_tax reads. +The input is absent from gov.irs.gross_income.sources, so it never reaches +irs_gross_income, AGI, or Section 86 provisional income (taxable_ss_magi). It is +absent from net_capital_gain, so it never reaches the preferential-rate base +(capital_gains_excluded_from_taxable_income / capital_gains_tax). It is absent +from gov.irs.investment.income.sources, so it never reaches net investment income. + +Law +--- +* IRC 61(a)(3): gross income includes gains derived from dealings in property. +* IRC 852(b)(3)(B): a capital gain dividend "shall be treated by the shareholders + as a gain from the sale or exchange of a capital asset held for more than 1 + year", i.e. long-term capital gain, which is net capital gain for the IRC 1(h) + rates. +* 2025 Form 1040 instructions (i1040gi), line 7, Exception 1: when the only + capital gains are 1099-DIV box 2a distributions, enter them on line 7a and + check "Schedule D not required" on line 7b, so they are part of total income + and AGI. The Social Security Benefits Worksheet line 3 includes line 7a + (IRC 86(b)(2) provisional income). The Qualified Dividends and Capital Gain Tax + Worksheet line 3 ("No" branch) enters line 7a in the preferential-rate base. +* Form 8960 instructions, line 5a: net gain from disposition of property combines + Form 1040 line 7a and Schedule 1 line 4, so the distributions are net + investment income under IRC 1411(c)(1)(A)(iii). + +Changes (exactly PR #8839's three code changes, nothing else) +-------------------------------------------------------------- +1. gov.irs.gross_income.sources: insert non_sch_d_capital_gains after + capital_gains (upstream position). Every reader of this list moves with it: + irs_gross_income, taxable_ss_magi, taxable_uc_agi, dependent_gross_income, + medicaid_irs_gross_income, student_loan_interest_ald_magi, + capped_qualified_tuition_expenses_ald. +2. gov.irs.investment.income.sources: append non_sch_d_capital_gains (upstream + position). Readers: net_investment_income (NIIT, MN NIIT, and the EITC / + NJ EITC investment-income tests). +3. net_capital_gain: add non_sch_d_capital_gains to the returned total, as the + upstream formula does (no Schedule D netting on the no-Schedule-D path). + +The PR's changelog entry and YAML tests are not code and are not backported. +dwks09 / dwks10 / has_qdiv_or_ltcg already read the input in 1.755.4 and upstream +did not change them. +""" + +from __future__ import annotations + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r04_capital_gain_distributions" +DESCRIPTION = ( + "Backport policyengine-us PR #8839 (issue #8828): non_sch_d_capital_gains " + "(Form 1040 line 7a capital gain distributions, IRC 852(b)(3)(B) long-term " + "gain) enters gov.irs.gross_income.sources (AGI, Section 86 provisional " + "income), gov.irs.investment.income.sources (NIIT, EITC investment-income " + "test), and net_capital_gain (IRC 1(h) preferential-rate base)." +) + +INPUT = "non_sch_d_capital_gains" + + +class net_capital_gain(Variable): + """Upstream main formula after PR #8839 (net_capital_gain.py lines 20-38).""" + + value_type = float + entity = TaxUnit + label = "Net capital gain" + unit = USD + documentation = ( + "The excess of net long-term capital gain over net short-term capital" + 'loss, plus qualified dividends (the definition of "net capital gain"' + "which applies to 26 U.S.C. § 1(h) from § 1(h)(11))." + ) + definition_period = YEAR + reference = dict( + title="26 U.S. Code § 1222(11)", + href="https://www.law.cornell.edu/uscode/text/26/1222#11", + ) + + def formula(tax_unit, period, parameters): + lt_capital_gain = max_(0, add(tax_unit, period, ["long_term_capital_gains"])) + st_capital_loss = max_(0, -add(tax_unit, period, ["short_term_capital_gains"])) + investment_income_election = add( + tax_unit, + period, + ["investment_income_elected_form_4952"], + ) + net_cap_gain = max_( + 0, + lt_capital_gain - st_capital_loss - investment_income_election, + ) + qual_div_income = add(tax_unit, period, ["qualified_dividend_income"]) + # Capital gain distributions reported without Schedule D (Form 1040 + # line 7 with the box checked) are long-term gains under IRC + # 852(b)(3)(B) and enter the preferential-rate base directly, with no + # Schedule D netting available on that filing path. + non_sch_d_capital_gains = add(tax_unit, period, [INPUT]) + return net_cap_gain + qual_div_income + non_sch_d_capital_gains + + +def _add_source(node, after: str | None) -> None: + """Add INPUT to every dated value of a list parameter, keeping each date range.""" + entries = sorted(node.values_list, key=lambda e: e.instant_str) + starts = [e.instant_str for e in entries] + for i, entry in enumerate(entries): + values = list(entry.value) + if INPUT in values: + continue + if after is not None and after in values: + values.insert(values.index(after) + 1, INPUT) + else: + values.append(INPUT) + stop = ( + instant(starts[i + 1]).offset(-1, "day") if i + 1 < len(entries) else None + ) + node.update(start=instant(entry.instant_str), stop=stop, value=values) + + +def modify_parameters(parameters): + _add_source(parameters.gov.irs.gross_income.sources, after="capital_gains") + _add_source(parameters.gov.irs.investment.income.sources, after=None) + return parameters + + +class reform(Reform): + def apply(self): + self.update_variable(net_capital_gain) + self.modify_parameters(modify_parameters) diff --git a/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions_v2.py b/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions_v2.py new file mode 100644 index 00000000..78b837e6 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r04_capital_gain_distributions_v2.py @@ -0,0 +1,85 @@ +"""2026 distribution-income correction, including the WI exclusion and §1(h)(2). + +Federal inclusion follows policyengine-us PR #8839, with distributions included +before the investment-income election reduction. IRC 1(h)(2) applies even if +Schedule D is not required (2025 Schedule D Tax Worksheet, lines 7-9). +Wisconsin Schedule SB instructions, line 5, require the 30% exclusion for mutual +fund/REIT capital gain distributions; adding federal AGI without it overstates WI +income. Existing engine amounts/rates and unrelated behavior are preserved. + +Sources: +https://www.law.cornell.edu/uscode/text/26/852#b_3_B +https://www.law.cornell.edu/uscode/text/26/1#h_2 +https://www.irs.gov/instructions/i1040sd +https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=2 +""" +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r04_capital_gain_distributions_v2" +DESCRIPTION = "2026 non-Schedule-D distributions: federal inclusion, election reduction, Wisconsin capital-gain exclusion." +SOURCE = "non_sch_d_capital_gains" + + +class net_capital_gain(Variable): + value_type = float + entity = TaxUnit + label = "Net capital gain" + unit = USD + definition_period = YEAR + reference = "https://www.law.cornell.edu/uscode/text/26/1#h_2" + + def formula(tax_unit, period, parameters): + lt = max_(0, add(tax_unit, period, ["long_term_capital_gains"])) + st_loss = max_(0, -add(tax_unit, period, ["short_term_capital_gains"])) + election = add(tax_unit, period, ["investment_income_elected_form_4952"]) + distributions = add(tax_unit, period, [SOURCE]) if period.start.year == 2026 else 0 + qualified_dividends = add(tax_unit, period, ["qualified_dividend_income"]) + return max_(0, lt - st_loss + distributions - election) + qualified_dividends + + +class wi_capital_gain_loss_subtraction(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin capital gain/loss subtraction from federal AGI" + unit = USD + definition_period = YEAR + defined_for = StateCode.WI + reference = "https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=2" + + def formula(tax_unit, period, parameters): + st = add(tax_unit, period, ["short_term_capital_gains"]) + sources = ["long_term_capital_gains"] + if period.start.year == 2026: + sources.append(SOURCE) + lt = add(tax_unit, period, sources) + total = max_(0, st + lt) + fraction = parameters(period).gov.states.wi.tax.income.subtractions.capital_gain.fraction + reduction = min_(total, max_(0, lt)) * fraction + # Preserve original arithmetic when SOURCE is absent. + wi_gain = total - reduction + return max_(0, total - wi_gain) + + +def modify_parameters(parameters): + for node, after in [ + (parameters.gov.irs.gross_income.sources, "capital_gains"), + (parameters.gov.irs.investment.income.sources, None), + ]: + values = list(node(instant("2026-01-01"))) + if SOURCE in values: + continue + if after in values: + values.insert(values.index(after) + 1, SOURCE) + else: + values.append(SOURCE) + node.update(start=instant("2026-01-01"), stop=instant("2026-12-31"), value=values) + return parameters + + +class reform(Reform): + def apply(self): + self.update_variable(net_capital_gain) + self.update_variable(wi_capital_gain_loss_subtraction) + self.modify_parameters(modify_parameters) diff --git a/reference_audit/2026-09-22/fixes/r05_nj_worker_ui_v2.py b/reference_audit/2026-09-22/fixes/r05_nj_worker_ui_v2.py new file mode 100644 index 00000000..6ee202bb --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r05_nj_worker_ui_v2.py @@ -0,0 +1,80 @@ +"""2026-only NJ worker UI/WF/SWF repair, independently verified. + +NJDOL 2026 worker UI 0.003825, WF/SWF 0.000425, wage base $44,800: +https://www.nj.gov/labor/ea/employer-services/rate-info/ +UI coverage and withholding: N.J.S.A. 43:21-7(d)(1)(D)-(F). +https://www.nj.gov/labor/myunemployment/assets/pdfs/UI_statute.pdf + +Retains the original fix's covered nongovernmental-employer assumption, since +the input model has no governmental-reimbursable employer indicator. Such +workers instead have UI rate 0.000825. Does not alter employer contributions. +Unlike v1, this sandbox repair does not apply a 2026 base to later years or +invent historical zero contributions. Only benchmark tax year 2026 changes. +""" + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r05_nj_worker_ui_v2" +DESCRIPTION = "Add covered NJ workers' UI and WF/SWF contributions for 2026 only." + + +class nj_employee_unemployment_taxable_wages(Variable): + value_type = float + entity = Person + label = "New Jersey worker UI and WF/SWF taxable wages (2026 repair)" + definition_period = YEAR + unit = USD + defined_for = StateCode.NJ + + def formula(person, period, parameters): + if period.start.year != 2026: + return 0 + return min_(max_(0, person("employment_income", period)), 44_800) + + +class nj_employee_unemployment_insurance_contribution(Variable): + value_type = float + entity = Person + label = "New Jersey worker unemployment insurance contribution (2026 repair)" + definition_period = YEAR + unit = USD + defined_for = StateCode.NJ + + def formula(person, period, parameters): + return 0.003825 * person("nj_employee_unemployment_taxable_wages", period) + + +class nj_employee_workforce_fund_contribution(Variable): + value_type = float + entity = Person + label = "New Jersey worker WF/SWF contribution (2026 repair)" + definition_period = YEAR + unit = USD + defined_for = StateCode.NJ + + def formula(person, period, parameters): + return 0.000425 * person("nj_employee_unemployment_taxable_wages", period) + + +class nj_employee_state_payroll_tax(Variable): + value_type = float + entity = Person + label = "New Jersey employee state payroll tax" + definition_period = YEAR + unit = USD + defined_for = StateCode.NJ + adds = [ + "nj_employee_temporary_disability_insurance_contribution", + "nj_employee_family_leave_insurance_contribution", + "nj_employee_unemployment_insurance_contribution", + "nj_employee_workforce_fund_contribution", + ] + + +class reform(Reform): + def apply(self): + self.update_variable(nj_employee_unemployment_taxable_wages) + self.update_variable(nj_employee_unemployment_insurance_contribution) + self.update_variable(nj_employee_workforce_fund_contribution) + self.update_variable(nj_employee_state_payroll_tax) diff --git a/reference_audit/2026-09-22/fixes/r067_adult_dependent_nonchild.py b/reference_audit/2026-09-22/fixes/r067_adult_dependent_nonchild.py new file mode 100644 index 00000000..4671f2a3 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r067_adult_dependent_nonchild.py @@ -0,0 +1,38 @@ +"""2026 sensitivity: adult tax dependents are not children of their claimant. + +This is the reverse bound of the omitted adult-dependent relationship, not a +universal adult-dependent rule. 42 CFR 435.603(f)(2)(i) sends a dependent other +than the claimant's spouse or natural/adopted/step child to non-filer rules. +Only the Medicaid relationship predicate changes; all other encoding remains. + +Primary source published before the reference freeze: +https://www.govinfo.gov/content/pkg/CFR-2025-title42-vol4/pdf/CFR-2025-title42-vol4-sec435-603.pdf +""" + +from policyengine_core.reforms import Reform +from policyengine_us.variables.gov.hhs.medicaid.income.medicaid_claimed_by_parent_in_tax_unit import ( + medicaid_claimed_by_parent_in_tax_unit as original_relationship, +) + +FIX_ID = "r067_adult_dependent_nonchild" +DESCRIPTION = ( + "For 2026 only, interpret adult tax dependents as people other than their " + "claimant's natural/adopted/step children for Medicaid household rules." +) + + +class medicaid_claimed_by_parent_in_tax_unit(original_relationship): + def formula_2026_01_01(person, period, parameters): + encoded = original_relationship.formula(person, period, parameters) + adult_dependent = person("is_tax_unit_dependent", period) & ( + person("age", period) >= 18 + ) + return encoded & ~adult_dependent + + def formula_2027_01_01(person, period, parameters): + return original_relationship.formula(person, period, parameters) + + +class reform(Reform): + def apply(self): + self.update_variable(medicaid_claimed_by_parent_in_tax_unit) diff --git a/reference_audit/2026-09-22/fixes/r067_adult_dependent_relationship.py b/reference_audit/2026-09-22/fixes/r067_adult_dependent_relationship.py new file mode 100644 index 00000000..4216fede --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r067_adult_dependent_relationship.py @@ -0,0 +1,43 @@ +"""2026 sensitivity: read adult tax dependents as children of their claimant. + +This is an alternative interpretation of an unlisted relationship, not a rule +that every adult dependent is a child. 42 CFR 435.603(b), (d), and (f)(2) +include a claimed natural/adopted/step child in the claimant's MAGI household +without an age limit. Section (f)(2)(i) instead sends other dependents to the +non-filer rules. The frozen prompts do not distinguish these relationships. + +Only the Medicaid relationship predicate changes. Ages, SSI disability +criteria, income definitions, and all unrelated formulas remain as encoded. +The original formula applies before and after calendar 2026. + +Primary source (2025 edition, printed before the reference freeze): +https://www.govinfo.gov/content/pkg/CFR-2025-title42-vol4/pdf/CFR-2025-title42-vol4-sec435-603.pdf +""" + +from policyengine_core.reforms import Reform +from policyengine_us.variables.gov.hhs.medicaid.income.medicaid_claimed_by_parent_in_tax_unit import ( + medicaid_claimed_by_parent_in_tax_unit as original_relationship, +) + +FIX_ID = "r067_adult_dependent_relationship" +DESCRIPTION = ( + "For 2026 only, interpret adult tax dependents as natural/adopted/step " + "children of their claimant for Medicaid MAGI household construction." +) + + +class medicaid_claimed_by_parent_in_tax_unit(original_relationship): + def formula_2026_01_01(person, period, parameters): + encoded = original_relationship.formula(person, period, parameters) + adult_dependent = person("is_tax_unit_dependent", period) & ( + person("age", period) >= 18 + ) + return encoded | adult_dependent + + def formula_2027_01_01(person, period, parameters): + return original_relationship.formula(person, period, parameters) + + +class reform(Reform): + def apply(self): + self.update_variable(medicaid_claimed_by_parent_in_tax_unit) diff --git a/reference_audit/2026-09-22/fixes/r06_wi_act15_before_refundable_v2.py b/reference_audit/2026-09-22/fixes/r06_wi_act15_before_refundable_v2.py new file mode 100644 index 00000000..9e55a710 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r06_wi_act15_before_refundable_v2.py @@ -0,0 +1,300 @@ +"""r06 v2: Wisconsin elected-path reporting with optimized joint pooling. + +V2 correction +------------- +For a joint return with both spouses age 67+, line 16 has a pooled limit. +The original fix imposed proportional allocation between spouses, which can +unnecessarily reduce the separate line-17 subtractions. V2 allocates the pooled +subtraction first against income exceeding the existing line-17 entitlement. +The maximum preserved line-17 total is the lesser of that entitlement and +qualifying retirement income remaining after line 16. Single and non-pooled +paths retain the original arithmetic; no age, income base, or dollar parameter +is changed. + +Root cause +---------- +policyengine-us 1.755.4 models the 2025 Wis. Act 15 retirement income +subtraction (Wis. Stat. 71.05(6)(b)54m; Schedule SB line 16) only inside +``wi_income_tax``, which returns min(standard net tax, exclusion-path tax). +The benchmarked output ``state_income_tax_before_refundable_credits`` sums +``wi_income_tax_before_refundable_credits``, which is always the +no-subtraction path (max(0, wi_income_tax_before_credits - +wi_non_refundable_credits)). When the filer elects line 16, the benchmark +therefore reports a tax that appears on no return the filer would file, and +state_income_tax != before_refundable - refundable. + +Rule encoded (tax year 2026) +---------------------------- +1. Election (Wis. Stat. 71.05(6)(b)54m.a-d, created by 2025 Wis. Act 15, + "for taxable years beginning after December 31, 2024", no sunset). A + filer with a head or spouse aged 67+ may subtract up to $24,000 of + qualified-plan and IRA distributions per individual ($48,000 for a joint + return when both spouses are 67+); claiming it forfeits every s. 71.07 + credit (54m.d). The 2025 Schedule SB instructions, line 16, apply the + forfeiture to Schedule CR and Form 1 lines 13-20 and 30-35, i.e. every + nonrefundable and refundable credit the engine models, including the + homestead credit (Form 1 line 33). The filer elects line 16 only when it + strictly lowers net tax (tax after all credits); ties stay on the standard + path. This is the engine's own comparison in ``wi_income_tax``. +2. Exclusion-path tax. Line 16 is a subtraction from federal AGI in computing + Wisconsin adjusted gross income (Wis. Stat. 71.01(13) includes every + s. 71.05(6) modification; Form 1 line 7 "Wisconsin income" = line 5 minus + Schedule SB line 50). The sliding-scale standard deduction is keyed to + Wisconsin AGI (Wis. Stat. 71.05(22)(dp)1; Form 1 line 8 is looked up on + line 7), so it is recomputed on the post-subtraction income. This is the + same correction upstream made after 1.755.4 in commit 35bbe1ba0c + (PolicyEngine/policyengine-us#8817/#8818). The $5,000 retirement income + subtraction (Wis. Stat. 71.05(6)(b)54, Schedule SB line 17) applies + "except as provided under subd. 54m": the 2025 Schedule SB line 17 + worksheet line 4 removes amounts already subtracted on line 16, so the + engine's line-17 amount for each person is capped at that person's + remaining qualified-plan and IRA income after line 16. Exemptions and rates + are unchanged. No credits of any kind are applied on this path. +3. Outputs on the elected path: + wi_income_tax_before_refundable_credits = exclusion-path tax (no + nonrefundable credits) and wi_refundable_credits = 0 when line 16 is + elected; otherwise both keep the engine's standard-path values. The + engine's wi_income_tax formula is untouched; with these two inputs it + returns exactly the elected-path net tax, so state_income_tax again equals + state_income_tax_before_refundable_credits - state_refundable_credits. + +Left alone: wi_agi, wi_standard_deduction, wi_taxable_income, +wi_income_tax_before_credits and wi_non_refundable_credits stay the engine's +standard-path intermediates. The engine's line-17 base (pension only, not IRA +distributions) and its per-person line-16 eligibility are not changed. + +Set EXCLUSION_TAX_MODE = "engine" to keep the 1.755.4 exclusion-path tax +(taxable income minus line 16, standard deduction not recomputed, no line-17 +offset) and change only the election plumbing; used to show the two variants +agree on this bundle. +""" + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r06_wi_act15_before_refundable_v2" +DESCRIPTION = ( + "Preserve the largest permitted line 17 subtraction under joint line 16 pooling. " + "Wisconsin: report wi_income_tax_before_refundable_credits and " + "wi_refundable_credits on the path the filer elects under Wis. Stat. " + "71.05(6)(b)54m (Sch. SB line 16), with the exclusion-path tax computed " + "on post-subtraction Wisconsin AGI (standard deduction per 71.05(22)(dp), " + "line 17 net of line 16) and all credits forfeited (54m.d)." +) + +EXCLUSION_TAX_MODE = "corrected" # or "engine" + + +def _wi_rate_tax(tax_unit, period, parameters, taxable_income): + fstatus = tax_unit("filing_status", period) + statuses = fstatus.possible_values + p = parameters(period).gov.states.wi.tax.income + return select( + [ + fstatus == statuses.SINGLE, + fstatus == statuses.JOINT, + fstatus == statuses.SURVIVING_SPOUSE, + fstatus == statuses.SEPARATE, + fstatus == statuses.HEAD_OF_HOUSEHOLD, + ], + [ + p.rates.single.calc(taxable_income), + p.rates.joint.calc(taxable_income), + p.rates.joint.calc(taxable_income), + p.rates.separate.calc(taxable_income), + p.rates.head_of_household.calc(taxable_income), + ], + ) + + +def _wi_standard_deduction_at(tax_unit, period, parameters, wagi): + # Same formula as 1.755.4 variables/.../wi_standard_deduction.py, evaluated + # at a caller-supplied Wisconsin AGI. + fstatus = tax_unit("filing_status", period) + statuses = fstatus.possible_values + deduction = parameters(period).gov.states.wi.tax.income.deductions + max_amount = deduction.standard.max[fstatus] + phase_out_amount = select( + [ + fstatus == statuses.SINGLE, + fstatus == statuses.JOINT, + fstatus == statuses.SURVIVING_SPOUSE, + fstatus == statuses.SEPARATE, + fstatus == statuses.HEAD_OF_HOUSEHOLD, + ], + [ + deduction.standard.phase_out.single.calc(wagi), + deduction.standard.phase_out.joint.calc(wagi), + deduction.standard.phase_out.joint.calc(wagi), + deduction.standard.phase_out.separate.calc(wagi), + deduction.standard.phase_out.head_of_household.calc(wagi), + ], + ) + return max_(0, max_amount - phase_out_amount) + + +class wi_retirement_income_exclusion_line17_offset(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin line 17 subtraction lost when line 16 is claimed" + unit = USD + definition_period = YEAR + reference = ( + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54", + "https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=8", + ) + defined_for = StateCode.WI + + def formula(tax_unit, period, parameters): + # Line 17 worksheet line 4 includes line 16, so each person's line-17 + # amount is capped at their qualified-plan + IRA income left after + # their share of line 16. + p = parameters(period).gov.states.wi.tax.income.subtractions + px = p.retirement_income.exclusion + person = tax_unit.members + age = person("age", period) + pension = person("taxable_pension_income", period) + ira = person("taxable_ira_distributions", period) + + # Line 16 total and eligibility match the existing engine. For joint + # pooling, maximize remaining line 17 rather than imposing proportions. + head_or_spouse_16 = person("is_tax_unit_head_or_spouse", period) + eligible_16 = (age >= px.min_age) * head_or_spouse_16 + ret_16 = (pension + ira) * eligible_16 + filing_status = tax_unit("filing_status", period) + pooled = (filing_status == filing_status.possible_values.JOINT) & ( + tax_unit.sum(eligible_16) >= 2 + ) + total_16 = tax_unit.sum(ret_16) + pooled_amount = min_(px.max_amount.joint, total_16) + # Non-pooled returns retain the original per-person allocation. + line16_person = min_(px.max_amount.single, ret_16) + + # Engine's line 17 per person (wi_retirement_income_subtraction.py). + psri = p.retirement_income + head_or_spouse_17 = ~person("is_tax_unit_dependent", period) + line17_person = min_( + psri.max_amount, pension * (age >= psri.min_age) * head_or_spouse_17 + ) * tax_unit.project( + tax_unit("wi_retirement_income_subtraction_agi_eligible", period) + ) + remaining = max_(0, (pension + ira) * head_or_spouse_17 - line16_person) + line17_on_path = min_(line17_person, remaining) + nonpooled_offset = tax_unit.sum(line17_person - line17_on_path) + + # Schedule SB line 16 permits a pooled limit regardless of each + # spouse's income. Line 17 is limited to each spouse's remaining + # retirement income; no rule requires proportional line 16 allocation. + # Allocate line 16 to income exceeding the existing line 17 entitlement + # first. Its maximum retained total is min(existing entitlement, + # retirement income left after line16). Preserve the engine's original + # pension-only line17 scope and the non-pooled path. + pooled_line17 = tax_unit.sum(line17_person * eligible_16) + pooled_remaining = max_(0, total_16 - pooled_amount) + pooled_offset = max_(0, pooled_line17 - pooled_remaining) + return where(pooled, pooled_offset, nonpooled_offset) + + +class wi_retirement_income_exclusion_tax(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin retirement income exclusion path tax" + unit = USD + definition_period = YEAR + reference = ( + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54m/a", + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/01/13", + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/22", + "https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=7", + ) + defined_for = StateCode.WI + + def formula(tax_unit, period, parameters): + line16 = tax_unit("wi_retirement_income_exclusion_amount", period) + if EXCLUSION_TAX_MODE == "engine": + taxinc = max_(0, tax_unit("wi_taxable_income", period) - line16) + return _wi_rate_tax(tax_unit, period, parameters, taxinc) + offset = tax_unit("wi_retirement_income_exclusion_line17_offset", period) + wagi = tax_unit("wi_agi", period) - line16 + offset + sd = _wi_standard_deduction_at(tax_unit, period, parameters, wagi) + exemption = tax_unit("wi_exemption", period) + taxinc = max_(0, wagi - sd - exemption) + return _wi_rate_tax(tax_unit, period, parameters, taxinc) + + +class wi_retirement_income_exclusion_elected(Variable): + value_type = bool + entity = TaxUnit + label = "Wisconsin filer elects the Schedule SB line 16 subtraction" + definition_period = YEAR + reference = ( + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54m/d", + "https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=7", + ) + defined_for = StateCode.WI + + def formula(tax_unit, period, parameters): + p = parameters(period).gov.states.wi.tax.income + if not p.subtractions.retirement_income.exclusion.in_effect: + return tax_unit.filled_array(False) + standard_before_refundable = max_( + 0, + tax_unit("wi_income_tax_before_credits", period) + - tax_unit("wi_non_refundable_credits", period), + ) + standard_refundable = add(tax_unit, period, p.credits.refundable) + standard_net = standard_before_refundable - standard_refundable + exclusion_net = tax_unit("wi_retirement_income_exclusion_tax", period) + line16 = tax_unit("wi_retirement_income_exclusion_amount", period) + return (line16 > 0) & (exclusion_net < standard_net) + + +class wi_income_tax_before_refundable_credits(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin income tax before refundable credits" + unit = USD + definition_period = YEAR + reference = ( + "https://www.revenue.wi.gov/TaxForms2025/2025-Form1f.pdf", + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54m/a", + ) + defined_for = StateCode.WI + + def formula(tax_unit, period, parameters): + standard = max_( + 0, + tax_unit("wi_income_tax_before_credits", period) + - tax_unit("wi_non_refundable_credits", period), + ) + elected = tax_unit("wi_retirement_income_exclusion_elected", period) + exclusion_tax = tax_unit("wi_retirement_income_exclusion_tax", period) + return where(elected, exclusion_tax, standard) + + +class wi_refundable_credits(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin refundable credits" + unit = USD + definition_period = YEAR + reference = ( + "https://www.revenue.wi.gov/TaxForms2025/2025-Form1f.pdf", + "https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54m/d", + ) + defined_for = StateCode.WI + + def formula(tax_unit, period, parameters): + p = parameters(period).gov.states.wi.tax.income.credits + standard = add(tax_unit, period, p.refundable) + elected = tax_unit("wi_retirement_income_exclusion_elected", period) + return where(elected, 0, standard) + + +class reform(Reform): + def apply(self): + self.update_variable(wi_retirement_income_exclusion_line17_offset) + self.update_variable(wi_retirement_income_exclusion_elected) + self.update_variable(wi_retirement_income_exclusion_tax) + self.update_variable(wi_income_tax_before_refundable_credits) + self.update_variable(wi_refundable_credits) diff --git a/reference_audit/2026-09-22/fixes/r07_idaho_health_premiums_v2.py b/reference_audit/2026-09-22/fixes/r07_idaho_health_premiums_v2.py new file mode 100644 index 00000000..52742280 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r07_idaho_health_premiums_v2.py @@ -0,0 +1,99 @@ +"""Idaho 2026 premium subtraction with a consistent deduction election. + +Idaho Code 63-3022P excludes premiums already deducted or accounted for: +https://legislature.idaho.gov/statutesrules/idstat/Title63/T63CH30/SECT63-3022P/ +Form 39R line 18 worksheet, printed pages 34-35, and Form 40 deduction election, +printed page 8, EIN00046 03-02-2026 (2025 instructions): +https://tax.idaho.gov/wp-content/uploads/forms/EIN00046/EIN00046_03-02-2026.pdf + +The standard route keeps all otherwise available premium subtraction. The +itemized route loses the portion used as an itemized medical deduction. Compare +both combined reductions, rather than comparing the deductions in isolation. +Ties retain the standard deduction, as in v1's strict comparison. This is only +the 2026 benchmark repair; unrelated federal medical-expense modeling remains. +""" + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r07_idaho_health_premiums_v2" +DESCRIPTION = "Idaho health premium subtraction with consistent 2026 deduction election." +SUBTRACTION = "id_health_insurance_premiums_subtraction" + + +def available_and_itemized_portion(tax_unit, period): + premiums = add(tax_unit, period, ["medical_expense_health_insurance_premiums"]) + elsewhere = tax_unit("self_employed_health_insurance_ald", period) + available = max_(0, premiums - elsewhere) + portion = min_(available, tax_unit("medical_expense_deduction", period)) + return available, portion + + +class id_itemizes_after_health_premiums(Variable): + value_type = bool + entity = TaxUnit + label = "Idaho itemization benefits taxpayer after premium subtraction (2026 repair)" + definition_period = YEAR + defined_for = StateCode.ID + + def formula(tax_unit, period, parameters): + available, portion = available_and_itemized_portion(tax_unit, period) + mandatory = tax_unit("separate_filer_itemizes", period) & (available > 0) + return mandatory | ( + tax_unit("id_itemized_deductions", period) + > tax_unit("standard_deduction", period) + portion + ) + + +class id_health_insurance_premiums_subtraction(Variable): + value_type = float + entity = TaxUnit + label = "Idaho health insurance premiums subtraction" + unit = USD + definition_period = YEAR + defined_for = StateCode.ID + + def formula(tax_unit, period, parameters): + available, portion = available_and_itemized_portion(tax_unit, period) + return available - where( + tax_unit("id_itemizes_after_health_premiums", period), portion, 0 + ) + + +class id_deductions(Variable): + value_type = float + entity = TaxUnit + label = "Idaho deductions" + unit = USD + definition_period = YEAR + defined_for = StateCode.ID + + def formula(tax_unit, period, parameters): + itemized = tax_unit("id_itemized_deductions", period) + standard = tax_unit("standard_deduction", period) + if period.start.year != 2026: + return max_(itemized, standard) + return where( + tax_unit("id_itemizes_after_health_premiums", period), itemized, standard + ) + + +def _modify(parameters): + node = parameters.gov.states.id.tax.income.subtractions.subtractions + current = list(node(instant("2026-01-01"))) + if SUBTRACTION not in current: + node.update( + start=instant("2026-01-01"), + stop=instant("2026-12-31"), + value=current + [SUBTRACTION], + ) + return parameters + + +class reform(Reform): + def apply(self): + self.update_variable(id_itemizes_after_health_premiums) + self.update_variable(id_health_insurance_premiums_subtraction) + self.update_variable(id_deductions) + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-22/fixes/r08_eitc_earned_income_deferrals_v2.py b/reference_audit/2026-09-22/fixes/r08_eitc_earned_income_deferrals_v2.py new file mode 100644 index 00000000..68d8ff13 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r08_eitc_earned_income_deferrals_v2.py @@ -0,0 +1,195 @@ +"""r08: EITC / ACTC earned income must exclude wages that are not includible in gross income. + +Law +--- +IRC 32(c)(2)(A)(i) defines EITC earned income as "wages, salaries, tips, and +other employee compensation, but only if such amounts are includible in gross +income for the taxable year" (clause added by EGTRRA, Pub. L. 107-16, sec. +303(b), effective for tax years beginning after 2001; unchanged in the current +Cornell LII text). A traditional 401(k)/403(b) elective deferral is excluded +from gross income by IRC 402(e)(3) (up to the 402(g) limit, which the engine +already applies through elective_deferral_contribution_scale), and cafeteria-plan +health premiums / HSA payroll contributions are excluded by IRC 125 / 106. +The IRS EIC worksheet (Form 1040 instructions, Step 5 line 1) and the Schedule +8812 Earned Income Worksheet (line 1a) both start from Form 1040 line 1z, i.e. +W-2 box 1, which omits those amounts. + +IRC 24(d)(1)(B)(i) computes the refundable CTC (ACTC) phase-in on "earned +income (within the meaning of section 32)", so the ACTC follows the same base. +In policyengine-us 1.755.4 that happens automatically: +ctc_phase_in_relevant_earnings reads eitc_earned_income. + +State credits built on the same definition +----------------------------------------- +State EITCs that are a percentage of the federal EITC (MT, VA, NY, WI, CO, ...) +move downstream with no further change. State credits that recompute a +federal-style EITC read filer_adjusted_earnings instead, which in 1.755.4 is +also built from gross employment_income: + - California CalEITC and Young Child Tax Credit. R&TC 17052(c)(4)(A) keeps + the IRC 32(c)(2)(A)(i) "includible in gross income" clause and adds "and + only if such amounts are subject to withholding pursuant to Division 6 ... + of the Unemployment Insurance Code"; FTB 3514 (2025) line 13 takes W-2 box + 16 California wages. EDD DE 231TP lists employee 401(k) contributions as + "Not subject" to PIT withholding and "Not reportable" as PIT wages when not + includible in California gross income (CUIC 13009(i)(1), 13009(q), + 13009.5). R&TC 17052.1 (YCTC) uses the same earned income. + - the state_eitc_helpers federal-style EITC (CO ITIN / under-25 branches, IL, + IN, DC with qualifying child), IN decoupled, MN WFC, OK frozen-2020 federal + EITC, WA WFTC: each mirrors the IRC 32 computation. +For 2026 filer_adjusted_earnings uses federal box-1 wages outside CA; +CA uses state-taxable wages, retaining payroll HSA contributions. Its only other +consumer, the AMT kiddie-tax exemption cap (IRC 59(j), earned income per IRC +911(d)(2)), is not an EITC/ACTC base; amt_exemption is re-pointed at the +original gross sum so the AMT is left exactly as it was. + +Everything else is unchanged: the self-employment terms, the SE-tax ALD, the +per-person zero floor, and the person-level earned_income variable (used by the +CDCC, IRA limits, benefit programs, ...). +""" + +# Independent verifier corrections: constrain this sandbox to 2026; retain +# California-taxable payroll HSA contributions in state earned income. +# Sources: FTB 3514 line 13 (W-2 box 16), EDD DE 231EB p. 3 HSA row: +# https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html +# https://edd.ca.gov/siteassets/files/pdf_pub_ctr/de231eb.pdf +from policyengine_us.variables.gov.irs.credits.earned_income.eitc_earned_income import eitc_earned_income as baseline_eitc +from policyengine_us.variables.gov.irs.income.filer_adjusted_earnings import filer_adjusted_earnings as baseline_filer +from policyengine_us.variables.gov.irs.tax.federal_income.alternative_minimum_tax.exemption.amt_exemption import amt_exemption as baseline_amt +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r08_eitc_earned_income_deferrals_v2" +DESCRIPTION = ( + "2026 EITC/ACTC and federal-style state credit earnings exclude qualifying " + "pre-tax payroll contributions under IRC 32(c)(2)(A)(i), 24(d)(1)(B)(i), " + "and 402(e)(3). California earnings retain payroll HSA contributions, " + "which are state-taxable wages under R&TC 17052(c)(4)(A) and EDD DE 231EB." +) + + +class eitc_earned_income(Variable): + value_type = float + entity = TaxUnit + label = "Earned income for the EITC" + unit = USD + definition_period = YEAR + reference = ( + "https://www.law.cornell.edu/uscode/text/26/32#c_2", + "https://www.law.cornell.edu/uscode/text/26/402#e_3", + "https://www.irs.gov/instructions/i1040gi", + ) + + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return baseline_eitc.formula(tax_unit, period, parameters) + earned_income_sources = [ + # IRC 32(c)(2)(A)(i): wages only if includible in gross income, + # i.e. W-2 box 1 (employment_income less pre_tax_contributions). + "irs_employment_income", + "self_employment_income", + "sstb_self_employment_income", + "farm_operations_income", + "partnership_self_employment_net_earnings", + ] + gross_earned_income = sum( + tax_unit_non_dep_sum(source, tax_unit, period) + for source in earned_income_sources + ) + self_employment_tax_ald = tax_unit_non_dep_sum( + "self_employment_tax_ald_person", tax_unit, period + ) + return max_(0, gross_earned_income - self_employment_tax_ald) + + +class filer_adjusted_earnings(Variable): + value_type = float + entity = TaxUnit + definition_period = YEAR + label = "Filer earned income adjusted for self-employment tax" + unit = USD + + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return baseline_filer.formula(tax_unit, period, parameters) + # Same as 1.755.4 (sum over non-dependents of adjusted_earnings) except + # that wages excluded from gross income (pre_tax_contributions, as + # already subtracted in irs_employment_income) are removed first. + person = tax_unit.members + p = parameters(period).gov.irs.ald.misc + se_adjustment = ( + (1 - p.self_emp_tax_adj) + * p.employer_share + * person("self_employment_tax", period) + ) + excluded_wages = person("employment_income", period) - person( + "irs_employment_income", period + ) + # Federal box 1 excludes payroll HSA; California box 16 does not. + # Compute the CA exclusion directly so the existing wage zero floor + # also behaves correctly when claimed payroll deductions exceed wages. + ca_exclusions = add(person, period, [ + "traditional_401k_contributions", + "traditional_403b_contributions", + "pre_tax_health_insurance_premiums", + ]) + ca_wages = max_(0, person("employment_income", period) - ca_exclusions) + excluded_wages = where( + person.household("state_code", period) == StateCode.CA, + person("employment_income", period) - ca_wages, + excluded_wages, + ) + adjusted = max_( + 0, person("earned_income", period) - excluded_wages - se_adjustment + ) + is_dependent = person("is_tax_unit_dependent", period) + return tax_unit.sum(adjusted * ~is_dependent) + + +class amt_exemption(Variable): + value_type = float + entity = TaxUnit + definition_period = YEAR + label = "Alternative Minimum Tax exemption" + unit = USD + documentation = ( + "AMT exemption amount after phase-out and kiddie tax adjustments. " + "Form 6251, Line 5." + ) + reference = [ + "https://www.law.cornell.edu/uscode/text/26/55#d", + "https://www.irs.gov/instructions/i6251", + ] + + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return baseline_amt.formula(tax_unit, period, parameters) + # Verbatim 1.755.4 formula, except the kiddie-tax cap reads the + # original gross adjusted-earnings sum so this fix leaves the AMT + # unchanged. + p = parameters(period).gov.irs.income.amt + phase_out = p.exemption.phase_out + filing_status = tax_unit("filing_status", period) + amt_income = tax_unit("amt_income", period) + base_exemption_amount = p.exemption.amount[filing_status] + income_excess = max_(0, amt_income - phase_out.start[filing_status]) + exemption_phase_out = phase_out.rate * income_excess + reduced_exemption_amount = max_( + 0, + base_exemption_amount - exemption_phase_out, + ) + kiddie_tax_applies = tax_unit("amt_kiddie_tax_applies", period) + adj_earnings = tax_unit_non_dep_sum("adjusted_earnings", tax_unit, period) + child_amount = p.exemption.child.amount + exemption_cap = where( + kiddie_tax_applies, + adj_earnings + child_amount, + np.inf, + ) + return min_(reduced_exemption_amount, exemption_cap) + + +class reform(Reform): + def apply(self): + self.update_variable(eitc_earned_income) + self.update_variable(filer_adjusted_earnings) + self.update_variable(amt_exemption) diff --git a/reference_audit/2026-09-22/fixes/r09_ny_rptc_rent_cap_v2.py b/reference_audit/2026-09-22/fixes/r09_ny_rptc_rent_cap_v2.py new file mode 100644 index 00000000..88f664ec --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r09_ny_rptc_rent_cap_v2.py @@ -0,0 +1,58 @@ +"""NY IT-214 verification fix, limited to the assigned 2026 benchmark. + +The original fix's undated formula applies the 2025+ Part RR tables even in +2024. Preserve baseline formulas outside 2026, and retain the verified original +2026 calculation. This is a sandbox correction, not a complete historical model. + +Sources: Tax Law 606(e)(1)(A)(ii), (1)(F)(ii), (3)(B), and (7), +https://www.nysenate.gov/legislation/laws/TAX/606 ; 2025 IT-214 and IT-214-I, +https://www.tax.ny.gov/pdf/current_forms/it/it214_fill_in.pdf and +https://www.tax.ny.gov/pdf/current_forms/it/it214i.pdf . +""" + +import numpy as np +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 +from policyengine_us.variables.gov.states.ny.tax.income.credits.ny_real_property_tax_credit import ( + ny_real_property_tax_credit as BaselineNYRPTC, +) + +FIX_ID = "r09_ny_rptc_rent_cap_v2" +DESCRIPTION = "NY IT-214 renter cap and Part RR rules; sandbox correction limited to 2026." + + +def closed_band(income, tops, values): + index = np.searchsorted(np.asarray(tops, dtype=float), income, side="left") + return np.append(np.asarray(values, dtype=float), 0.0)[index] + + +class ny_real_property_tax_credit(BaselineNYRPTC): + defined_for = StateCode.NY + + def formula(tax_unit, period, parameters): + if period.start.year != 2026: + return BaselineNYRPTC.formula(tax_unit, period, parameters) + + p = parameters(period).gov.states.ny.tax.income.credits.real_property_tax + elderly = tax_unit.any(tax_unit.members("age", period) >= p.elderly_age) + rent = add(tax_unit, period, ["rent"]) + taxes = add(tax_unit, period, ["real_estate_taxes"]) + assessed_value = add(tax_unit, period, ["assessed_property_value"]) + income = np.floor(max_(tax_unit("adjusted_gross_income", period), 0) + 0.5) + scale = p.excess_real_property_tax + rate = closed_band(income, list(scale.thresholds[1:]) + [p.max_agi], list(scale.amounts)) + excess = max_(0, taxes + rent * p.rent_tax_equivalent - income * rate) + eligible = ((assessed_value <= p.max_property_value) + & (rent <= p.max_rent) & (income <= p.max_agi)) + amount = where( + elderly, + closed_band(income, [3000, 5000, 7000, 9000, 11000, 14000, 18000], + [375, 330, 300, 260, 230, 200, 150]), + closed_band(income, [5000, 9000, 14000, 18000], [75, 70, 60, 50]), + ) + return where(eligible & (excess > 0), amount, 0) + + +class reform(Reform): + def apply(self): + self.update_variable(ny_real_property_tax_credit) diff --git a/reference_audit/2026-09-22/fixes/r10_wi_homestead_income.py b/reference_audit/2026-09-22/fixes/r10_wi_homestead_income.py new file mode 100644 index 00000000..60a8ff93 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r10_wi_homestead_income.py @@ -0,0 +1,103 @@ +"""r10: Wisconsin homestead credit household income omits nontaxable income that +Wis. Stat. 71.52(6) and the 2025 Schedule H count. + +Engine (policyengine-us 1.755.4, unchanged on upstream/main 16832c046d): +wi_homestead_income = sum(gov.states.wi.tax.income.credits.homestead.income.sources) +- $500 x dependents, with sources = [adjusted_gross_income, tax_exempt_interest_income, +tax_exempt_social_security, tax_exempt_pension_income, +tax_exempt_unemployment_compensation, ssi, tanf]. Income that never reaches federal +AGI and is not on that list is dropped from household income. + +Law. Wis. Stat. 71.52(5): household income is all income of household members, less +$500 per dependent. 71.52(6): "Income" is Wisconsin AGI plus, to the extent not +already in Wisconsin AGI, maintenance payments, support money, cash public +assistance, "the gross amount of any pension or annuity (including railroad +retirement benefits, all payments received under the federal social security act and +veterans disability pensions)", nontaxable interest, worker's compensation, +unemployment insurance, "the gross amount of 'loss of time' insurance", +"compensation and other cash benefits received from the United States for past or +present service in the armed forces", scholarship and fellowship gifts or income, and +other listed items. Exclusions are limited to gifts from natural persons, Title XX +reimbursements, relief in kind, and deferred/nonrecognized home-sale gains. +2025 Wisconsin Schedule H instructions (I-016i, revenue.wi.gov/TaxForms2025/ +2025-ScheduleH-inst.pdf), which apply unchanged to 2026 claims: + 9d "GROSS amount of ALL pensions and annuities ... Include veterans' pensions, + disability payments ... and nontaxable IRA, SEP, SIMPLE, and qualified plan + distributions. Both taxable and nontaxable amounts must be included" + (rollovers and 1035 exchanges excepted) + 9h nontaxable scholarship and fellowship income, educational grants + 9i court-ordered child support, family maintenance; post-2018 alimony that is no + longer taxable "must be included in household income on line 9i" + 11b workers' compensation, income continuation, and loss of time insurance + +Fix. Append to the source list every engine variable that (a) is an income type the +statute and Schedule H count, (b) does not reach the engine's federal-AGI base or the +existing list (checked empirically: $10,000 of each moves adjusted_gross_income by 0), +and (c) is nonzero for at least one bundle household: + + survivor_benefits 71.52(6) pension/annuity, worker's comp, armed-forces + benefits; Sch H 9d / 11b. (CPS SRVS_VAL: survivor + pensions, railroad retirement, worker comp, annuities) + workers_compensation 71.52(6) "worker's compensation"; Sch H 11b + veterans_benefits 71.52(6) "veterans disability pensions", "compensation + and other cash benefits ... for ... service in the + armed forces"; Sch H 9d + child_support_received 71.52(6) "support money"; Sch H 9i + alimony_income 71.52(6) "maintenance payments"; Sch H 9i note + disability_benefits 71.52(6) "gross amount of 'loss of time' insurance"; + Sch H 9d "disability payments", 11b income continuation + educational_assistance 71.52(6) "scholarship and fellowship gifts or income"; + Sch H 9h + tax_exempt_retirement_distributions + 71.52(6) gross pension/annuity; Sch H 9d nontaxable IRA, + SEP, SIMPLE, qualified plan distributions (the bundle + has tax_exempt_ira_distributions) + +Not added (see the structured result): financial_assistance (engine label "cash +financial assistance from outside the household"; 71.52(6) excludes gifts from natural +persons, so the prompt facts do not settle it), estate_income (taxable income that the +engine also drops from federal AGI; a federal-AGI defect, not a homestead one), and the +71.52(6) add-backs of excluded deferrals, IRA/Keogh deductions, depreciation and +disqualified losses (Sch H 9e, 9f, 11e-11j), which are deductions reversed rather than +income types. A work/ variant shows none of these moves any bundle output. + +Only wi_homestead_income changes; its consumers are wi_homestead_eligible and +wi_homestead_credit. No parameter values change. +""" + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +FIX_ID = "r10_wi_homestead_income" +DESCRIPTION = ( + "WI homestead credit household income (Wis. Stat. 71.52(5)-(6), 2025 Schedule H " + "lines 9d, 9h, 9i, 11b) adds survivor benefits, worker's compensation, veterans " + "benefits, child support, alimony, disability benefits, scholarships/educational " + "assistance and nontaxable retirement-account distributions that never reach AGI." +) + +ADDED_SOURCES = [ + "survivor_benefits", + "workers_compensation", + "veterans_benefits", + "child_support_received", + "alimony_income", + "disability_benefits", + "educational_assistance", + "tax_exempt_retirement_distributions", +] + + +class reform(Reform): + def apply(self): + def modify(parameters): + node = parameters.gov.states.wi.tax.income.credits.homestead.income.sources + current = list(node("2026-01-01")) + node.update( + start=instant("2021-01-01"), + stop=instant("2100-12-31"), + value=current + [s for s in ADDED_SOURCES if s not in current], + ) + return parameters + + self.modify_parameters(modify) diff --git a/reference_audit/2026-09-22/fixes/r11_ca_itemized_conformity_v2.py b/reference_audit/2026-09-22/fixes/r11_ca_itemized_conformity_v2.py new file mode 100644 index 00000000..d2c50185 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r11_ca_itemized_conformity_v2.py @@ -0,0 +1,222 @@ +"""r11 v2: Bound every adjustment to 2026, including regular-tax formulas. + +Independent verification found the original undated replacement variables +affected 2025 and 2027 while its AMT parameter edit affected only 2026. +This version preserves the original 2026 operations and baseline formulas +outside 2026. SB 711 (Stats. 2025 ch. 231), sections 1 and 11, had already +established the cited conformity date and miscellaneous-deduction exception +before the July 3, 2026 reference freeze. +https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260SB711 + +California itemized deductions stop inheriting two federal-only 2026 rules. + +Defect (policyengine-us 1.755.4) +-------------------------------- +variables/gov/states/ca/tax/income/deductions/itemized/ +ca_itemized_deductions_pre_limitation.py builds the California (Schedule CA (540) +Part II) total as + + itemized_deductions_less_salt + ca_investment_interest_expense_deduction + + real_estate_taxes - investment_interest_expense + +and itemized_deductions_less_salt sums the federal list in +parameters/gov/irs/deductions/itemized_deductions.yaml, which includes the +federal charitable_deduction and misc_deduction. For 2026 that carries two +federal-only rules into California: + + * charitable_deduction applies the OBBBA 0.5%-of-AGI floor + (IRC 170(b)(1)(I); gov.irs.deductions.itemized.charity.floor.applies = true + from 2026-01-01, amount 0.005) and the federal 60% cash ceiling + (gov.irs.deductions.itemized.charity.ceiling.all = 0.6). + * misc_deduction returns 0 because gov.irs.deductions.itemized.misc.applies is + false from 2018 (IRC 67(g) suspension, made permanent by OBBBA). + +California law for tax year 2026 +-------------------------------- + * Conformity date: R&TC 17024.5(a)(1)(Q): the IRC "as enacted on" January 1, + 2025 for taxable years beginning on or after January 1, 2025 (text as amended + by Stats. 2026 ch. 236, SB 1435, effective 2026-09-14). OBBBA (P.L. 119-21, + enacted 2025-07-04) is after that date, and FTB's 2025 Schedule CA (540) + instructions say "In general, California R&TC does not conform to the OBBBA." + R&TC 17201(a) brings in IRC Part VI (incl. section 170) "except as otherwise + provided". + * Charitable contributions: no 0.5% floor (it exists only in the post-OBBBA + IRC). Overall ceiling 50% of federal AGI: FTB 2025 Schedule CA (540) + instructions, lines 11 and 12, "California limits the amount of your + deduction to 50% of your federal AGI." For 2026 this is also what the IRC as + of 2025-01-01 gives: the 60% cash limit of 170(b)(1)(G) applied only to years + "before January 1, 2026", and R&TC 17024.5(h)(6) makes a provision that + becomes inoperative after the specified date inoperative for California too, + leaving the 170(b)(1)(A) 50% limit. "AGI" in AGI-based limitations is federal + AGI: R&TC 17024.5(h)(2)(A). + * Miscellaneous itemized deductions: R&TC 17076(a) applies IRC 67 (the 2% floor); + R&TC 17076(c) (as amended by Stats. 2025 ch. 231, SB 711): "Section 67(g) of + the Internal Revenue Code ... shall not apply." FTB 2025 Schedule CA (540) + instructions, lines 19-22: the federal suspension "California law does not + conform"; form lines 22-25: total job expenses/misc, line 23 = federal + Form 1040 line 11b (federal AGI), line 24 = 2% of line 23, line 25 = line 22 + minus line 24. + * CA AMT: 2025 Schedule P (540) instructions, Part I line 5 "Miscellaneous + itemized deductions: Enter on this line the amount from Schedule CA (540), + Part II, line 25." The engine already lists misc_deduction as this add-back + (parameters/gov/states/ca/tax/income/amt/amti/sources.yaml, "line 5"); once + California's own line-25 amount is allowed for regular tax, the line-5 + add-back must use it too. The installed total-AMTI formula separately adds + all pre-limitation deductions, instead of reversing only the high-income + limitation. That inherited issue is outside this narrow fix; all five + California bundle households have zero AMT before and after this fix. + +What this fix changes (tax year 2026 only; nothing federal is touched) +---------------------------------------------------------------------- + 1. New ca_charitable_deduction: the engine's own federal charitable_deduction + computation (same inputs, same non-cash sub-ceilings 0.5 / 0.3 of AGI from + gov.irs.deductions.itemized.charity.ceiling) with the floor branch removed + and the overall ceiling set to 50% of federal (positive) AGI. + 2. New ca_misc_deduction: total_misc_deductions (engine sources + unreimbursed_business_employee_expenses + tax_preparation_fees) above + gov.irs.deductions.itemized.misc.floor (0.02, IRC 67(a)) x federal positive + AGI -- i.e. the engine's own misc_deduction formula without the 67(g) switch. + 3. ca_itemized_deductions_pre_limitation: same adds/subtracts as 1.755.4, plus + ca_charitable_deduction + ca_misc_deduction and minus the federal + charitable_deduction + misc_deduction that itemized_deductions_less_salt + carried in. + 4. gov.states.ca.tax.income.amt.amti.sources for 2026: misc_deduction -> + ca_misc_deduction (Schedule P line 5 = Schedule CA line 25). +The R&TC 17077 high-income limitation (ca_itemized_deductions), the itemize / +standard choice (ca_deductions), and everything else stay as the engine has them. +""" + +from __future__ import annotations + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r11_ca_itemized_conformity_v2" +DESCRIPTION = ( + "California itemized deductions (Schedule CA (540) Part II) use a California " + "charitable deduction (IRC 170 as of 2025-01-01: no OBBBA 0.5%-of-AGI floor, " + "50%-of-federal-AGI ceiling) and allow miscellaneous itemized deductions above " + "2% of federal AGI (R&TC 17076(c): IRC 67(g) does not apply), instead of the " + "federal 2026 charitable_deduction and misc_deduction; the CA AMT line-5 " + "add-back uses the same California misc amount." +) + +# FTB 2025 Schedule CA (540) instructions, lines 11-12; IRC 170(b)(1)(A) as of the +# California specified date (R&TC 17024.5(a)(1)(Q), (h)(6)). +CA_CHARITY_CEILING_FRACTION_OF_FEDERAL_AGI = 0.5 + + +class ca_charitable_deduction(Variable): + value_type = float + entity = TaxUnit + label = "California charitable contribution deduction" + unit = USD + definition_period = YEAR + defined_for = StateCode.CA + reference = ( + "https://www.ftb.ca.gov/forms/2025/2025-540-ca-instructions.html", + "https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17024.5", + "https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17201", + ) + + end = "2026-12-31" + + def formula_2026(tax_unit, period, parameters): + # Same computation as the engine's federal charitable_deduction + # (1.755.4) without the OBBBA floor branch, with California's overall + # 50%-of-federal-AGI ceiling in place of the federal 60%. + cash_donations = add(tax_unit, period, ["charitable_cash_donations"]) + non_cash_donations = add(tax_unit, period, ["charitable_non_cash_donations"]) + non_cash_to_non_50_pct = add( + tax_unit, period, ["charitable_non_cash_donations_non_50_pct_orgs"] + ) + non_cash_to_50_pct = non_cash_donations - non_cash_to_non_50_pct + federal_agi = tax_unit("positive_agi", period) + p = parameters(period).gov.irs.deductions.itemized.charity + capped_non_cash = min_( + non_cash_to_50_pct, p.ceiling.non_cash * federal_agi + ) + min_( + non_cash_to_non_50_pct, + p.ceiling.non_cash_to_non_50_pct_org * federal_agi, + ) + return min_( + capped_non_cash + cash_donations, + CA_CHARITY_CEILING_FRACTION_OF_FEDERAL_AGI * federal_agi, + ) + + +class ca_misc_deduction(Variable): + value_type = float + entity = TaxUnit + label = "California miscellaneous itemized deductions above 2% of federal AGI" + unit = USD + definition_period = YEAR + defined_for = StateCode.CA + reference = ( + "https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17076", + "https://www.ftb.ca.gov/forms/2025/2025-540-ca.pdf", + ) + + end = "2026-12-31" + + def formula_2026(tax_unit, period, parameters): + # Schedule CA (540) Part II lines 22-25: IRC 67 applies (R&TC 17076(a)), + # 67(g) does not (R&TC 17076(c)); line 23 is federal AGI. + p = parameters(period).gov.irs.deductions.itemized.misc + expenses = tax_unit("total_misc_deductions", period) + federal_agi = tax_unit("positive_agi", period) + return max_(0, expenses - p.floor * federal_agi) + + +class ca_itemized_deductions_pre_limitation(Variable): + value_type = float + entity = TaxUnit + label = "California pre-limitation itemized deductions" + unit = USD + definition_period = YEAR + reference = ( + "https://www.ftb.ca.gov/forms/2025/2025-540-ca-instructions.html" + ) + defined_for = StateCode.CA + + def formula(tax_unit, period, parameters): + # Reproduce the baseline adds/subtracts outside the audited year. + additions = [ + "itemized_deductions_less_salt", + "ca_investment_interest_expense_deduction", + "real_estate_taxes", + ] + subtractions = ["investment_interest_expense"] + if period.start.year == 2026: + additions += ["ca_charitable_deduction", "ca_misc_deduction"] + subtractions += ["charitable_deduction", "misc_deduction"] + # Keep v1's exact operation order to avoid changing 2026 rounding. + result = add(tax_unit, period, additions) + for variable in subtractions: + result = result - add(tax_unit, period, [variable]) + return result + + +def _modify(parameters): + node = parameters.gov.states.ca.tax.income.amt.amti.sources + current = list(node(instant("2026-01-01"))) + # Reforms can be applied more than once while the system is built; keep the + # edit idempotent. + if "ca_misc_deduction" in current: + return parameters + updated = ["ca_misc_deduction" if s == "misc_deduction" else s for s in current] + node.update( + start=instant("2026-01-01"), + stop=instant("2026-12-31"), + value=updated, + ) + return parameters + + +class reform(Reform): + def apply(self): + self.update_variable(ca_charitable_deduction) + self.update_variable(ca_misc_deduction) + self.update_variable(ca_itemized_deductions_pre_limitation) + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v2.py b/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v2.py new file mode 100644 index 00000000..539fcf6b --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v2.py @@ -0,0 +1,101 @@ +"""Apply the benchmark's FY2026 SNAP financial schedule for calendar 2026. + +This is the user-directed reference-freeze convention, not a statement that +FY2026 law remains effective after September 2026. Compared with r13_hold_fy2026, +hold the SNAP poverty guideline as well, and retain the published FY2026 minimum +in October-December (including Hawaii's $41). Do not change other programs' FPG. +The required full-year arithmetic check also exposed California's final-net +rounding choice: round net income to the nearest dollar before applying 30%. + +Sources: USDA FY2026 COLA memorandum, attachment pp. 3-7 (signed 2025-08-14); +7 USC 2017(a); 7 CFR 273.10(e)(2)(ii)(A),(C); CDSS's statewide summary of +MPP 63-503.311 (p. 1, dated 2015-08-04): +https://www.cdss.ca.gov/shd/res/pdf/ParaRegs-Food-Stamps-Income.pdf +Santa Clara County's CalFresh benefit computation instructions corroborate both +elderly/disabled and other household procedures: +https://stgenssa.sccgov.org/debs/program_handbooks/calfresh/assets/CalFresh/Budgeting_Concepts/Computation.htm +Preserve other states' existing net-income treatment and state minimum overrides. +Parameter and variable changes are limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.income.snap_fpg import snap_fpg as _base_fpg +from policyengine_us.variables.gov.usda.snap.income.snap_net_income import snap_net_income as _base_net +from policyengine_us.variables.gov.usda.snap.snap_min_allotment import snap_min_allotment as _base_min +from policyengine_us.variables.gov.usda.snap.snap_expected_contribution import snap_expected_contribution as _base_contribution + +FIX_ID = 'r13_hold_fy2026_v2' +DESCRIPTION = 'Hold FY2026 SNAP schedules, including SNAP FPG and published minimum, for calendar 2026; round contribution up and California final net income to the nearest dollar.' +FY2026_MINIMUM = { + 'CONTIGUOUS_US': 24, 'GU': 35, 'VI': 31, + 'AK_URBAN': 31, 'AK_RURAL_1': 39, 'AK_RURAL_2': 48, 'HI': 41, +} + + +def _hold(parameters): + index = parameters.gov.usda.snap.uprating + index.update(start=instant('2026-10-01'), stop=instant('2026-12-31'), + value=index('2025-10-01')) + return parameters + + +class snap_fpg(_base_fpg): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_fpg.formula(spm_unit, period, parameters) + # Preserve this month's unit size while selecting the FY2026 schedule. + size = spm_unit('snap_unit_size', period) + state = spm_unit.household('state_group_str', period.this_year) + p = parameters('2025-10-01').gov.hhs.fpg + return (p.first_person[state] + p.additional_person[state] * (size - 1)) / MONTHS_IN_YEAR + + +class snap_min_allotment(_base_min): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_min.formula(spm_unit, period, parameters) + p = parameters(period).gov.usda.snap.min_allotment + region = spm_unit.household('snap_region_str', period) + federal = np.array([FY2026_MINIMUM[str(r)] for r in np.asarray(region)], dtype=float) + minimum = (spm_unit('snap_unit_size', period) <= p.maximum_household_size) * federal + state = spm_unit.household('state_code_str', period) + dc = parameters(period).gov.states.dc.dhs.snap.min_allotment + if dc.in_effect: + minimum = where(state == 'DC', dc.amount, minimum) + md = parameters(period).gov.states.md.usda.snap.min_allotment + if md.in_effect: + minimum = where((state == 'MD') & spm_unit('md_snap_elderly_present', period), md.amount, minimum) + nj = parameters(period).gov.states.nj.snap + if nj.in_effect: + minimum = where(state == 'NJ', nj.amount, minimum) + return minimum + + +class snap_expected_contribution(_base_contribution): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_contribution.formula(spm_unit, period, parameters) + rate = parameters(period).gov.usda.snap.expected_contribution + net = np.floor(spm_unit('snap_net_income', period)) + return np.ceil(np.round(net * rate, 2)) + + +class snap_net_income(_base_net): + def formula(spm_unit, period): + net = _base_net.formula(spm_unit, period) + if period.start.year != 2026: + return net + state = spm_unit.household('state_code_str', period) + return where(state == 'CA', np.floor(net + 0.5), net) + + +class reform(Reform): + def apply(self): + self.modify_parameters(_hold) + self.update_variable(snap_fpg) + self.update_variable(snap_min_allotment) + self.update_variable(snap_expected_contribution) + self.update_variable(snap_net_income) diff --git a/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v3.py b/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v3.py new file mode 100644 index 00000000..d2b85e91 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v3.py @@ -0,0 +1,40 @@ +"""Hold the FY2026 SNAP schedule for calendar 2026 (publication convention only). + +The user-directed reference-freeze convention, not a statement that FY2026 law +remains effective after September 2026: October-December 2026 use the FY2026 +SNAP figures, the last USDA published before the 2026-07-03 freeze, in place of +policyengine-us 1.755.4's projected FY2027 figures. It holds the SNAP uprating +index, and with it the maximum allotments, deductions and minimum the engine +derives from it. The poverty guideline is not held: the engine already uses the +2026 HHS guideline (published in January 2026, before the freeze) from October. + +This is r13_hold_fy2026_v2 without its formula corrections and without v2's +poverty-guideline and published-minimum overrides. The corrections are engine +defects fixed in policyengine-us after the freeze, each in its own module and +applied with this convention when the references are regenerated: +r26_snap_contribution_rounding (7 CFR 273.10(e)(2)(ii)(A)), +r27_snap_net_income_rounding (7 CFR 273.10(e)(1)(ii)), both #9318; +r28_snap_min_allotment_rounding (7 CFR 273.10(e)(2)(ii)(C)) and +r31_snap_income_limit_rounding (7 CFR 273.9(a)(3)), both #9162. + +Sources: USDA FY2026 SNAP COLA memorandum, attachment pp. 3-7 (signed +2025-08-14); 7 USC 2017(a). Parameter changes are limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = 'r13_hold_fy2026_v3' +DESCRIPTION = 'Hold the FY2026 SNAP uprating index for calendar 2026.' +def _hold(parameters): + index = parameters.gov.usda.snap.uprating + index.update(start=instant('2026-10-01'), stop=instant('2026-12-31'), + value=index('2025-10-01')) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_hold) diff --git a/reference_audit/2026-09-22/fixes/r14_unlisted_weekly_hours_v2_v2.py b/reference_audit/2026-09-22/fixes/r14_unlisted_weekly_hours_v2_v2.py new file mode 100644 index 00000000..d107582b --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r14_unlisted_weekly_hours_v2_v2.py @@ -0,0 +1,124 @@ +"""r14 independent-verification revision: hours-input sensitivity, not an exact award. + +Executable patch and reform are unchanged from r14_unlisted_weekly_hours_v2. +For each person with no prompt-visible hours field, use the prompt's numeric +zero default. Retain 1.755.4's other policy and assume no unlisted general-work +noncompliance (7 CFR 273.7(a)(1), (f)(1)). This removes the artificial work- +registration sanctions that v1 introduced for scenarios 012, 032 and 038. + +The combined test changes the SNAP outputs for 056 and 112 to zero in the +1.755.4 sandbox. Those zeros are sensitivity results, not uniquely established +2026 legal entitlements. The unchanged ABAWD implementation does not track +prior countable months or geographic waivers. 7 U.S.C. 2015(o)(2) limits benefits +after three countable months in a 36-month period; zero hours alone does not +establish immediate ineligibility. NJ's official guidance also reports geographic +waivers during 2026. The scenarios do not state prior ABAWD months or locality. +The two exclusions rest on an unlisted hours dependency under otherwise frozen +engine assumptions; they must not be presented as validated zero-dollar awards. + +Sources checked in the independent verification: +- https://www.ecfr.gov/current/title-7/section-273.7 +- https://www.law.cornell.edu/uscode/text/7/2015#o +- https://www.nj.gov/humanservices/dfd/news/federal-changes/ +""" + +from __future__ import annotations + +from policybench.scenarios import is_excluded_prompt_input_name +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 (Variable, Person, MONTH, ...) + +FIX_ID = "r14_unlisted_weekly_hours_v2_v2" +DESCRIPTION = ( + "Set weekly_hours_worked_before_lsr to 0 for every person whose prompt lists no " + "weekly hours (prompt: 'Treat any unlisted numeric input as 0'; 1.755.4 " + "defaults it to 40), and treat non-exempt SNAP work registrants as compliant " + "unless flagged noncompliant (7 CFR 273.7(a)(1), (f)(1); upstream 48a10d4d43), " + "to isolate hours-input sensitivity under otherwise frozen engine assumptions; " + "zero outputs do not establish exact legal entitlements." +) + +HOURS_FIELDS = ( + "hours_worked_last_week", + "weekly_hours_worked", + "weekly_hours_worked_before_lsr", + "hours_worked", +) +TARGET = "weekly_hours_worked_before_lsr" + + +def listed_hours_fields(person) -> list[str]: + """Hours fields that the prompt actually shows for this person.""" + return [ + field + for field in HOURS_FIELDS + if field in person.inputs and not is_excluded_prompt_input_name(field) + ] + + +def patch(situation: dict, scenario) -> dict: + year = str(scenario.year) + for person in list(scenario.adults) + list(scenario.children): + if listed_hours_fields(person): + continue + situation["people"][person.name][TARGET] = {year: 0.0} + return situation + + +class is_snap_work_registration_noncompliant(Variable): + value_type = bool + entity = Person + label = "SNAP work registration noncompliant" + definition_period = MONTH + default_value = False + documentation = ( + "Whether a non-exempt SNAP work registrant has refused or failed without " + "good cause to comply with the work requirements (7 CFR 273.7(f)(1)). " + "Backport of the upstream hook from commit 48a10d4d43." + ) + reference = "https://www.ecfr.gov/current/title-7/section-273.7#p-273.7(f)(1)" + + +class meets_snap_general_work_requirements(Variable): + value_type = bool + entity = Person + label = "Person is eligible for SNAP benefits via general work requirements" + definition_period = MONTH + reference = ( + "https://www.law.cornell.edu/cfr/text/7/273.7#a_1", + "https://www.law.cornell.edu/cfr/text/7/273.7#f_1", + ) + + def formula(person, period, parameters): + # Identical to 1.755.4 except the compliance line (see module docstring). + p = parameters(period).gov.usda.snap.work_requirements.general + age = person("monthly_age", period) + weekly_hours_worked = person("weekly_hours_worked_before_lsr", period.this_year) + worked_exempted_age = p.age_threshold.exempted.calc(age) + is_disabled = person("is_disabled", period) + is_dependent = person("is_tax_unit_dependent", period) + is_child = age < p.age_threshold.caring_dependent_child + has_child = person.spm_unit.any(is_dependent & is_child) + has_incapacitated_person = person.spm_unit.any( + person("is_incapable_of_self_care", period) + ) + is_working = weekly_hours_worked >= p.weekly_hours_threshold + exempted = ( + worked_exempted_age + | is_disabled + | has_child + | has_incapacitated_person + | is_working + ) + compliant = person("is_snap_work_program_participant", period) | ~person( + "is_snap_work_registration_noncompliant", period + ) + return exempted | compliant + + +class reform(Reform): + def apply(self): + # update_variable adds a missing variable and replaces an existing one, so + # apply() stays idempotent if it runs more than once on a system. + self.update_variable(is_snap_work_registration_noncompliant) + self.update_variable(meets_snap_general_work_requirements) diff --git a/reference_audit/2026-09-22/fixes/r15_snap_mortgage_interest_v2.py b/reference_audit/2026-09-22/fixes/r15_snap_mortgage_interest_v2.py new file mode 100644 index 00000000..6d657a75 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r15_snap_mortgage_interest_v2.py @@ -0,0 +1,112 @@ +"""r15 v2: alternate SNAP shelter reading, with tax-unit-local input fallback. + +Classification: unlisted_input, not a proved engine-law defect. The benchmark +prompts list mortgage interest but do not explicitly state that the encumbered +home is the SNAP household's occupied shelter. This module implements the +alternative reading that first-home interest (or person interest where the +tax unit has no populated interest structure) belongs to the occupied home. +It does not establish that reading as the unique interpretation of the prompt. + +7 CFR 273.9(d)(6)(ii)(A) allows continuing mortgage charges, including interest, +for occupied shelter. Paragraph (D) also permits certain temporarily vacant +homes; first/second-home labels alone do not resolve those occupancy facts. +https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/subpart-D/section-273.9 +7 USC 2014(e)(6)(A)-(B) gives the 50% excess-shelter test and the cap exception +for households with an elderly or disabled member. +https://uscode.house.gov/view.xhtml?req=(title:7%20section:2014%20edition:prelim) + +As in v1, the listed interest is a lower bound of total payments; unlisted +principal is not invented. The SNAP-only increment is max(interest minus +mortgage_payments, 0), preventing double counting of the reported payment. +The first-versus-person fallback now occurs per tax unit before SPM-unit +aggregation. The remainder of the original SNAP shelter formula is unchanged. +No statutory dollar parameter is introduced. All 100 bundle scenarios have +one tax unit, so this correction should leave the benchmark sweep unchanged. +""" + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r15_snap_mortgage_interest_v2" +DESCRIPTION = ( + "Alternative occupied-home reading of listed mortgage interest for SNAP; " + "choose first-home versus person inputs per tax unit before aggregation." +) + + +class snap_mortgage_interest_shelter_cost(Variable): + value_type = float + entity = SPMUnit + label = "SNAP shelter cost from listed mortgage interest beyond mortgage_payments" + unit = USD + definition_period = YEAR + reference = ( + "https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/subpart-D/section-273.9#p-273.9(d)(6)(ii)(A)", + ) + + def formula(spm_unit, period, parameters): + person = spm_unit.members + head = person("is_tax_unit_head", period) + first = person.tax_unit("first_home_mortgage_interest", period) + second = person.tax_unit("second_home_mortgage_interest", period) + # Choose the input representation separately for each tax unit. + # A populated structure for one tax unit must not suppress another + # tax unit's person-only mortgage interest within the same SPM unit. + interest_by_person = where( + first + second > 0, + head * first, + person("home_mortgage_interest", period), + ) + occupied_home_interest = spm_unit.sum(interest_by_person) + mortgage_payments = spm_unit("mortgage_payments", period) + return max_(occupied_home_interest - mortgage_payments, 0) + + +class snap_excess_shelter_expense_deduction(Variable): + value_type = float + entity = SPMUnit + definition_period = MONTH + documentation = ( + "Excess shelter expense deduction for calculating SNAP benefit amount" + ) + label = "SNAP shelter deduction" + reference = ("United States Code, Title 7, Section 2014(e)(6)",) + unit = USD + + def formula(spm_unit, period, parameters): + # Identical to policyengine-us 1.755.4 except for the mortgage-interest + # term added to housing_cost. + p = parameters(period).gov.usda.snap.income.deductions.excess_shelter_expense + net_income_pre_shelter = spm_unit("snap_net_income_pre_shelter", period) + subtracted_income = p.income_share_disregard * net_income_pre_shelter + mortgage_interest = ( + spm_unit("snap_mortgage_interest_shelter_cost", period.this_year) + / MONTHS_IN_YEAR + ) + housing_cost = ( + add(spm_unit, period, ["snap_utility_allowance", "housing_cost"]) + + mortgage_interest + ) + uncapped_ded = max_(housing_cost - subtracted_income, 0) + state_group = spm_unit.household("snap_region_str", period) + ded_cap = p.cap[state_group] + capped_ded = min_(uncapped_ded, ded_cap) + has_elderly_disabled = spm_unit("has_usda_elderly_disabled", period) + non_homeless_shelter_deduction = where( + has_elderly_disabled, uncapped_ded, capped_ded + ) + state = spm_unit.household("state_code_str", period) + homeless_deduction = p.homeless.deduction * p.homeless.available[state] + return where( + spm_unit.household("is_homeless", period) + & (housing_cost > 0) + & (homeless_deduction > non_homeless_shelter_deduction), + homeless_deduction, + non_homeless_shelter_deduction, + ) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_mortgage_interest_shelter_cost) # adds it; safe on re-apply + self.update_variable(snap_excess_shelter_expense_deduction) diff --git a/reference_audit/2026-09-22/fixes/r16_survivor_benefits_federal_v2.py b/reference_audit/2026-09-22/fixes/r16_survivor_benefits_federal_v2.py new file mode 100644 index 00000000..39902711 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r16_survivor_benefits_federal_v2.py @@ -0,0 +1,100 @@ +"""r16: count survivor_benefits (non-Social Security survivor benefits) in federal gross income. + +Defect +------ +policyengine-us 1.755.4 defines survivor_benefits +(variables/household/income/person/general/survivor_benefits.py) as a bare input, +"Survivor benefits other than Social Security survivor benefits", with no formula. +No IRS income parameter lists it: gov.irs.gross_income.sources +(parameters/gov/irs/gross_income/sources.yaml) holds taxable_pension_income, +taxable_retirement_distributions, etc., but not survivor_benefits. So the amount +never reaches irs_gross_income, adjusted_gross_income, the IRC 86 provisional +income in taxable_ss_magi, the OBBBA senior-deduction MAGI, Medicaid's +medicaid_irs_gross_income, or any state tax that starts from federal AGI. The +engine does count it as unearned income for SNAP, SSI, TANF, WIC, HUD and school +meals, and as a household benefit (gov.household.household_benefits). + +Rule encoded (the "taxable survivor pension/annuity" reading) +------------------------------------------------------------- +IRC 61(a)(8), (10) include annuities and pensions in gross income; IRC 72 taxes +annuity payments less the exclusion ratio; IRC 402(a) taxes qualified-plan +distributions to the distributee, including a beneficiary. IRS Pub. 575 (for +2025 returns, fetched 2026-09-22 from irs.gov/pub/irs-pdf/p575.pdf), "Survivors +and Beneficiaries": "Benefits paid to you as a survivor under a joint and +survivor annuity must be included in your gross income"; a survivor of an +employee excludes only the part of each payment that recovers the employee's +investment in the contract. IRS Pub. 525 (for 2025 returns), "Survivor +Benefits": "In most cases, payments made by or for an employer because of an +employee's death must be included in income." + +This is an alternative factual reading, not a default legal presumption. The +prompt gives neither payer nor cost basis. Assume a survivor annuity from a +noncontributory employer plan (or one whose cost was fully recovered), so the +full amount is taxable. A qualified public-safety survivor annuity can instead +be wholly exempt under IRC 101(h), even if VA benefits are excluded by the +source dataset definition. Those payer facts were not shown in the prompt. The fix appends survivor_benefits to +gov.irs.gross_income.sources. irs_gross_income, taxable_ss_magi, +dependent_gross_income, medicaid_irs_gross_income, taxable_uc_agi and the ALD +MAGI helpers all iterate over that list, so every federal-AGI consumer picks it +up the same way it picks up taxable_pension_income. Nothing else changes: +survivor_benefits stays where it already is in the benefit-program income lists, +and the WI homestead income source list is left alone (r10 handles that). + +The opposite reading is also lawful for some payers, which is why the taxability +is an unlisted fact rather than an engine error on stated facts: + * VA Dependency and Indemnity Compensation and other VA survivor benefits are + exempt: 38 U.S.C. 5301(a); Pub. 525 "Veterans' benefits". + * Workers' compensation survivor benefits are exempt: IRC 104(a)(1); Treas. + Reg. 1.104-1(b); Pub. 525 ("The exemption also applies to your survivors"). + * Survivor annuities for public safety officers killed in the line of duty are + exempt: IRC 101(h). + * Life-insurance death proceeds are exempt: IRC 101(a) (installments taxable + only on the interest element, IRC 101(d)). + +Not done here: states that build gross income from their own source lists +(AL, AR, IA, MS, OK, PR, MT elderly credit) would still miss survivor_benefits; +no benchmark household with survivor_benefits lives in those states. State +pension/retirement subtractions keyed to taxable_pension_income (e.g. the WI +67+ retirement income exclusion) do not see survivor_benefits under this fix; +see verify/work/r16_trace.py for the input-move sensitivity that routes the +amount through taxable_private_pension_income instead. In the current bundle +that sensitivity leaves the two moved benchmark outputs unchanged. This +module is only validated for the 2026 benchmark households, not as a general +implementation of every survivor-benefit category. The scenario_108 homestead +credit movement overlaps r10 and does not require federal taxability ambiguity. +""" + +from __future__ import annotations + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +FIX_ID = "r16_survivor_benefits_federal_v2" +DESCRIPTION = ( + "Treat survivor_benefits (non-Social Security survivor benefits) as a fully " + "taxable pension/annuity for federal purposes by appending it to " + "gov.irs.gross_income.sources (IRC 61(a)(8), (10), 72, 402(a); IRS Pub. 575 " + "Survivors and Beneficiaries). Flows to AGI, IRC 86 provisional income and " + "every state tax that starts from federal AGI." +) + +SOURCE = "survivor_benefits" + + +class _SurvivorBenefitsInGrossIncome(Reform): + def apply(self): + def modify(parameters): + node = parameters.gov.irs.gross_income.sources + current = list(node("2026-01-01")) + if SOURCE not in current: + node.update( + start=instant("2026-01-01"), + stop=instant("2026-12-31"), + value=current + [SOURCE], + ) + return parameters + + self.modify_parameters(modify) + + +reform = _SurvivorBenefitsInGrossIncome diff --git a/reference_audit/2026-09-22/fixes/r17_caleitc_agi_comparison.py b/reference_audit/2026-09-22/fixes/r17_caleitc_agi_comparison.py new file mode 100644 index 00000000..bca49a53 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r17_caleitc_agi_comparison.py @@ -0,0 +1,78 @@ +"""CalEITC: the second credit lookup at adjusted gross income (FTB 3514 worksheet lines 3-5). + +California's earned income tax credit follows the federal structure: when adjusted gross +income exceeds the income at which the credit starts to phase out, the filer looks the +credit up a second time at AGI and takes the smaller of the two amounts (Cal. R&TC +17052(c); FTB 3514 instructions, CalEITC worksheet lines 3-5). policyengine-us 1.755.4's +ca_eitc phases out on filer_adjusted_earnings only; ca_eitc_eligible tests AGI solely +against the credit's maximum income. A filer whose AGI exceeds earnings in the phase-out +range (for example, one with a taxable retirement distribution) gets too large a credit. + +This module keeps the 1.755.4 schedule and adds the AGI lookup, for 2026 only. +Env R17_AGI = "federal" (default: adjusted_gross_income) or "ca" (ca_agi) picks the AGI. +""" + +import os + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "r17_caleitc_agi_comparison" +AGI_VARIABLE = "ca_agi" if os.environ.get("R17_AGI") == "ca" else "adjusted_gross_income" + + +def _credit(income, child_count, p): + phase_in_rate = p.phase_in.rate.calc(child_count) * p.adjustment.factor + phase_in_max_income = p.earned_income_amount.calc(child_count) + phased_in_amount = min_(income, phase_in_max_income) * phase_in_rate + phase_out_min_income = p.phase_out.start.calc(child_count) + phase_out_rate = p.phase_out.rate.calc(child_count) * p.adjustment.factor + second_phase_out_start_eitc = p.phase_out.final.start.calc(child_count) + maximum_eitc = phase_in_max_income * phase_in_rate + earnings_range_of_first_phase_out = ( + maximum_eitc - second_phase_out_start_eitc + ) / phase_out_rate + second_phase_out_start = phase_out_min_income + earnings_range_of_first_phase_out + second_phase_out_end = p.phase_out.final.end + phase_out_income = min_( + max_(0, income - phase_out_min_income), earnings_range_of_first_phase_out + ) + amount_after_first_phase_out = phased_in_amount - phase_out_income * phase_out_rate + share = min_( + (income - second_phase_out_start) / (second_phase_out_end - second_phase_out_start), + 1, + ) + return where( + income > second_phase_out_start, + amount_after_first_phase_out * (1 - share), + amount_after_first_phase_out, + ) + + +class ca_eitc(Variable): + value_type = float + entity = TaxUnit + label = "CalEITC" + unit = USD + definition_period = YEAR + defined_for = "ca_eitc_eligible" + + def formula(tax_unit, period, parameters): + p = parameters(period).gov.states.ca.tax.income.credits.earned_income + earned_income = tax_unit("filer_adjusted_earnings", period) + child_count = tax_unit("eitc_child_count", period) + by_earnings = _credit(earned_income, child_count, p) + if period.start.year != 2026: + return by_earnings + agi = tax_unit(AGI_VARIABLE, period) + by_agi = _credit(max_(agi, 0), child_count, p) + return where( + agi > p.phase_out.start.calc(child_count), + min_(by_earnings, by_agi), + by_earnings, + ) + + +class reform(Reform): + def apply(self): + self.update_variable(ca_eitc) diff --git a/reference_audit/2026-09-22/fixes/r18_hold_all_projections.py b/reference_audit/2026-09-22/fixes/r18_hold_all_projections.py new file mode 100644 index 00000000..f9161a20 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r18_hold_all_projections.py @@ -0,0 +1,90 @@ +"""Every projected parameter value held at the parameter's last explicit value. + +Audit of the reference convention "a scored reference follows from the stated facts +and from law published before the 2026-07-03 reference freeze". policyengine-us +1.755.4 fills parameter values past each parameter's last explicit (YAML) value by +uprating: it multiplies the last explicit value by the growth of an index (CPI, +chained CPI, SNAP's thrifty-food-plan index, California CPI, ...), most of whose +2026 values are forecasts. A reference that moves when those projections are removed +depends on a projection rather than on a published amount, and needs one of: + - the amount was published before the freeze and the engine did not encode it: + an engine defect (the projection may or may not match the published amount); + - the amount was published after the freeze: the held convention applies (the + last amount published before the freeze), as for SNAP (r13_hold_fy2026) and + California (r12_hold_ca_2025). + +Projected entries are found by loading the parameter tree a second time with +policyengine-core's uprate_parameters disabled and diffing each parameter's +values_list instants. The reform drops the projected entries, so each parameter +carries its last explicit value forward. + +Env: + HOLD_PREFIXES comma-separated parameter-name prefixes to restrict the hold to + (default: every parameter); used to attribute a move to a family. + HOLD_LIST path to write the list of projected (parameter, instant) entries. +""" + +from __future__ import annotations + +import json +import os + +import policyengine_us.system as pe_system +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform +from policyengine_us import CountryTaxBenefitSystem + +FIX_ID = "r18_hold_all_projections" +LAST_DATE = "2026-12-31" + + +def _instants(system) -> dict[str, set[str]]: + return { + p.name: {v.instant_str for v in p.values_list} + for p in system.parameters.get_descendants() + if isinstance(p, Parameter) + } + + +def _projected() -> dict[str, set[str]]: + uprated = _instants(pe_system.system) + original = pe_system.uprate_parameters + pe_system.uprate_parameters = lambda parameters: parameters + try: + raw = _instants(CountryTaxBenefitSystem()) + finally: + pe_system.uprate_parameters = original + prefixes = [p for p in os.environ.get("HOLD_PREFIXES", "").split(",") if p] + projected = {} + for name, instants in uprated.items(): + if prefixes and not any(name.startswith(prefix) for prefix in prefixes): + continue + extra = { + i for i in instants - raw.get(name, set()) if i <= LAST_DATE and i >= "2025-01-01" + } + if extra: + projected[name] = extra + return projected + + +PROJECTED = _projected() +if os.environ.get("HOLD_LIST"): + with open(os.environ["HOLD_LIST"], "w") as f: + json.dump({k: sorted(v) for k, v in sorted(PROJECTED.items())}, f, indent=1) + + +def _hold(parameters): + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in PROJECTED: + drop = PROJECTED[parameter.name] + parameter.values_list = [ + v for v in parameter.values_list if v.instant_str not in drop + ] + if parameter.parent is not None: + parameter.parent.clear_parent_cache() + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_hold) diff --git a/reference_audit/2026-09-22/fixes/r19_ca_convention.py b/reference_audit/2026-09-22/fixes/r19_ca_convention.py new file mode 100644 index 00000000..3800c691 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_ca_convention.py @@ -0,0 +1,83 @@ +"""California's last published indexed amounts at the 2026-07-03 freeze. + +2025 FTB sources (all amounts held for tax year 2026): +https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2025/10.html +https://www.ftb.ca.gov/forms/2025/2025-540-tax-rate-schedules.pdf +https://www.ftb.ca.gov/forms/2025/2025-540-booklet.html +https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html +https://www.ftb.ca.gov/forms/2026/2026-540-es-instructions.html + +The CalEITC final-phase credit breakpoints are computed from RTC 17052(o)'s +2019 $200/$505 bases and FTB's published annual indexing factors, rounding +annually under RTC 17041(h): 1.4%, 4.4%, 8.3%, 3.1%, 3.3%, 3.0% (2020-25). +This gives $252/$636, not the engine's projected 2025 $257/$649. +They are statutory derived amounts, not amounts printed by name in Form 3514. +Renter's joint/HOH/surviving cap is $107,988; engine 2025 has $107,987. + +Only parameters change: existing withholding and credit formulas remain. +HOLD_PREFIXES optionally restricts the parameter overrides for attribution. +""" + +import os +from policyengine_core.parameters import Parameter +from policyengine_core.periods import period +from policyengine_core.reforms import Reform + +FIX_ID = "r19_ca_convention" +DESCRIPTION = "California published 2025 amounts carried into 2026, correcting 2025 CalEITC forecast breakpoints." +P = "gov.states.ca.tax.income." +VALUES = {} + +BRACKETS = { + "single": [11079, 26264, 41452, 57542, 72724, 371479, 445771, 742953], + "separate": [11079, 26264, 41452, 57542, 72724, 371479, 445771, 742953], + "joint": [22158, 52528, 82904, 115084, 145448, 742958, 891542, 1485906], + "surviving_spouse": [22158, 52528, 82904, 115084, 145448, 742958, 891542, 1485906], + "head_of_household": [22173, 52530, 67716, 83805, 98990, 505208, 606251, 1010417], +} +for status, values in BRACKETS.items(): + for i, value in enumerate(values, 1): + VALUES[P + f"rates.{status}[{i}].threshold"] = value + +VALUES.update({ + P + "exemptions.amount": 153, + P + "exemptions.dependent_amount": 475, + P + "credits.earned_income.eligibility.max_investment_income": 4814, + P + "credits.earned_income.phase_out.final.start[0].amount": 252, + P + "credits.earned_income.phase_out.final.start[1].amount": 636, + P + "credits.foster_youth.amount[1].amount": 1189, + P + "credits.foster_youth.phase_out.start": 27425, + P + "credits.young_child.amount": 1189, + P + "credits.young_child.loss_threshold": 35640, + P + "credits.young_child.phase_out.start": 27425, +}) +for i, value in enumerate([4661, 6998, 9823]): + for branch in ["earned_income_amount", "phase_out.start"]: + VALUES[P + f"credits.earned_income.{branch}[{i}].amount"] = value +for status in ["SINGLE", "SEPARATE", "JOINT", "SURVIVING_SPOUSE", "HEAD_OF_HOUSEHOLD"]: + VALUES[P + f"credits.renter.income_cap.{status}"] = ( + 53994 if status in ["SINGLE", "SEPARATE"] else 107988 + ) + + +def modify(parameters): + prefixes = [s.strip() for s in os.environ.get("HOLD_PREFIXES", "").split(",") if s.strip()] + seen = set() + for param in parameters.get_descendants(): + if isinstance(param, Parameter) and param.name in VALUES: + if prefixes and not any(param.name.startswith(s) for s in prefixes): + continue + param.update(period=period("year:2026-01-01:1"), value=VALUES[param.name]) + # Repair the projected 2025 breakpoints too, so the held value is + # sourced law rather than another engine forecast. + if ".earned_income.phase_out.final.start[" in param.name: + param.update(period=period("year:2025-01-01:1"), value=VALUES[param.name]) + seen.add(param.name) + expected = {name for name in VALUES if not prefixes or any(name.startswith(s) for s in prefixes)} + assert seen == expected, sorted(expected - seen) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(modify) diff --git a/reference_audit/2026-09-22/fixes/r19_id_convention.py b/reference_audit/2026-09-22/fixes/r19_id_convention.py new file mode 100644 index 00000000..d03bc9a4 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_id_convention.py @@ -0,0 +1,80 @@ +"""Idaho parameters under PolicyBench's 2026-07-03 publication freeze. + +Tax thresholds: retain the last Idaho-published amounts found, $4,811/$9,622, +in the 2025 Form 40 instructions, revision 2026-03-02, page 9. A 2026 +Commission-published threshold/factor was not verified; this limitation is +documented in sweep/verify/r19_id_report.md. + +Retirement caps: Idaho Code 63-3022A defines the maximum via Social Security's +full-retirement-age benefit. SSA's 2026 fact sheet, linked from its 2025-10-24 +release, publishes $4,152/month: $49,824/year single and $74,736 with spouse. +These are law-derived amounts, not a located Idaho 2026 publication. The +optional ID_RETIREMENT_CAP_MODE=held sensitivity instead uses the published +2025 caps; no benchmark household is eligible for this deduction. + +Sources: +https://tax.idaho.gov/wp-content/uploads/forms/EIN00046/EIN00046_03-02-2026.pdf +https://legislature.idaho.gov/statutesrules/idstat/title63/t63ch30/sect63-3022a/ +https://www.ssa.gov/cola/factsheets/2026.html +https://www.ssa.gov/news/en/press/releases/2025-10-24.html +""" + +import os + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + + +FIX_ID = "r19_id_convention" +DESCRIPTION = "Hold Idaho tax thresholds; use pre-freeze SSA-derived retirement caps." + +THRESHOLDS = { + "single": 4_811, + "separate": 4_811, + "joint": 9_622, + "head_of_household": 9_622, + "surviving_spouse": 9_622, +} +CAPS_2025 = { + "SINGLE": 48_216, + "SEPARATE": 0, + "JOINT": 72_324, + "HEAD_OF_HOUSEHOLD": 48_216, + "SURVIVING_SPOUSE": 48_216, +} +CAPS_2026 = { + "SINGLE": 49_824, + "SEPARATE": 0, + "JOINT": 74_736, + "HEAD_OF_HOUSEHOLD": 49_824, + "SURVIVING_SPOUSE": 49_824, +} + + +def _modify(parameters): + caps = CAPS_2025 if os.environ.get("ID_RETIREMENT_CAP_MODE") == "held" else CAPS_2026 + values = { + f"gov.states.id.tax.income.main.{status}[1].threshold": value + for status, value in THRESHOLDS.items() + } + values.update( + { + f"gov.states.id.tax.income.deductions.retirement_benefits.cap.{status}": value + for status, value in caps.items() + } + ) + found = set() + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in values: + parameter.update(period="2026", value=values[parameter.name]) + found.add(parameter.name) + if parameter.name.endswith("retirement_benefits.cap.SEPARATE"): + # The only projected Idaho 2025 entry. Its zero is correct. + parameter.update(period="2025", value=0) + assert found == set(values), f"Missing Idaho parameters: {set(values) - found}" + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json b/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json new file mode 100644 index 00000000..f327044e --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json @@ -0,0 +1,6530 @@ +{ + "AK": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "AL": [ + [ + 306, + 405, + 453, + 492, + 525, + 555, + 581, + 605, + 628, + 657, + 695, + 728, + 759, + 787, + 816, + 846, + 874, + 902, + 1052 + ], + [ + 371, + 491, + 547, + 594, + 635, + 670, + 702, + 731, + 758, + 793, + 838, + 877, + 914, + 949, + 983, + 1020, + 1053, + 1086, + 1265 + ], + [ + 416, + 550, + 613, + 666, + 711, + 751, + 787, + 820, + 849, + 889, + 938, + 983, + 1024, + 1061, + 1100, + 1142, + 1179, + 1215, + 1416 + ], + [ + 453, + 597, + 666, + 723, + 773, + 816, + 854, + 890, + 922, + 964, + 1018, + 1066, + 1111, + 1152, + 1193, + 1238, + 1279, + 1317, + 1535 + ], + [ + 483, + 638, + 712, + 772, + 825, + 870, + 911, + 949, + 983, + 1029, + 1086, + 1137, + 1184, + 1228, + 1273, + 1319, + 1363, + 1405, + 1635 + ], + [ + 527, + 696, + 776, + 842, + 899, + 949, + 993, + 1034, + 1071, + 1120, + 1182, + 1239, + 1290, + 1338, + 1385, + 1437, + 1484, + 1528, + 1779 + ] + ], + "AR": [ + [ + 420, + 584, + 665, + 733, + 793, + 846, + 894, + 937, + 978, + 1032, + 1100, + 1163, + 1221, + 1274, + 1329, + 1387, + 1442, + 1494, + 1790 + ], + [ + 486, + 676, + 770, + 849, + 917, + 978, + 1033, + 1084, + 1131, + 1193, + 1273, + 1345, + 1412, + 1473, + 1537, + 1605, + 1668, + 1726, + 2070 + ], + [ + 530, + 737, + 840, + 926, + 1000, + 1067, + 1127, + 1183, + 1234, + 1302, + 1388, + 1467, + 1540, + 1607, + 1677, + 1750, + 1819, + 1884, + 2258 + ], + [ + 565, + 786, + 895, + 986, + 1065, + 1136, + 1200, + 1259, + 1314, + 1386, + 1478, + 1562, + 1639, + 1711, + 1785, + 1864, + 1937, + 2006, + 2403 + ], + [ + 593, + 826, + 939, + 1036, + 1119, + 1193, + 1261, + 1323, + 1380, + 1455, + 1553, + 1641, + 1721, + 1798, + 1875, + 1957, + 2034, + 2106, + 2524 + ], + [ + 634, + 882, + 1003, + 1106, + 1195, + 1274, + 1346, + 1412, + 1474, + 1554, + 1657, + 1752, + 1837, + 1918, + 2001, + 2089, + 2170, + 2248, + 2693 + ] + ], + "AZ": [ + [ + 335, + 463, + 525, + 578, + 624, + 664, + 701, + 734, + 766, + 806, + 859, + 907, + 951, + 991, + 1034, + 1078, + 1119, + 1159, + 1383 + ], + [ + 386, + 532, + 605, + 665, + 718, + 765, + 806, + 845, + 882, + 928, + 989, + 1044, + 1095, + 1142, + 1190, + 1241, + 1289, + 1334, + 1592 + ], + [ + 419, + 579, + 658, + 724, + 781, + 832, + 877, + 920, + 959, + 1010, + 1077, + 1136, + 1191, + 1243, + 1296, + 1351, + 1404, + 1452, + 1734 + ], + [ + 446, + 615, + 700, + 769, + 830, + 885, + 933, + 978, + 1020, + 1074, + 1145, + 1209, + 1266, + 1321, + 1377, + 1436, + 1492, + 1544, + 1843 + ], + [ + 467, + 646, + 733, + 806, + 870, + 927, + 979, + 1026, + 1069, + 1127, + 1200, + 1267, + 1328, + 1385, + 1444, + 1507, + 1565, + 1620, + 1934 + ], + [ + 498, + 688, + 781, + 859, + 927, + 988, + 1043, + 1093, + 1139, + 1200, + 1279, + 1351, + 1416, + 1477, + 1539, + 1606, + 1668, + 1725, + 2061 + ] + ], + "CA": [ + [ + 412, + 564, + 637, + 698, + 752, + 798, + 841, + 879, + 915, + 963, + 1023, + 1079, + 1128, + 1175, + 1223, + 1274, + 1320, + 1365, + 1619 + ], + [ + 476, + 650, + 734, + 805, + 866, + 921, + 970, + 1015, + 1056, + 1110, + 1180, + 1243, + 1301, + 1355, + 1410, + 1469, + 1522, + 1574, + 1867 + ], + [ + 519, + 709, + 801, + 877, + 944, + 1003, + 1057, + 1106, + 1151, + 1211, + 1286, + 1355, + 1418, + 1477, + 1537, + 1600, + 1659, + 1715, + 2034 + ], + [ + 552, + 754, + 852, + 934, + 1005, + 1067, + 1124, + 1176, + 1225, + 1288, + 1368, + 1442, + 1508, + 1571, + 1635, + 1702, + 1765, + 1825, + 2164 + ], + [ + 580, + 792, + 895, + 981, + 1055, + 1121, + 1181, + 1235, + 1286, + 1352, + 1437, + 1514, + 1584, + 1649, + 1716, + 1787, + 1853, + 1915, + 2272 + ], + [ + 619, + 845, + 955, + 1046, + 1126, + 1196, + 1259, + 1318, + 1372, + 1442, + 1532, + 1615, + 1690, + 1760, + 1831, + 1907, + 1977, + 2043, + 2423 + ] + ], + "CO": [ + [ + 170, + 233, + 263, + 289, + 312, + 331, + 349, + 366, + 381, + 400, + 427, + 449, + 470, + 491, + 511, + 532, + 551, + 571, + 678 + ], + [ + 197, + 271, + 307, + 336, + 363, + 386, + 406, + 426, + 443, + 466, + 496, + 523, + 547, + 571, + 594, + 619, + 642, + 664, + 790 + ], + [ + 216, + 297, + 336, + 369, + 397, + 422, + 445, + 466, + 485, + 511, + 543, + 573, + 600, + 626, + 651, + 678, + 704, + 728, + 865 + ], + [ + 232, + 317, + 358, + 394, + 425, + 451, + 475, + 498, + 519, + 545, + 581, + 612, + 641, + 668, + 696, + 725, + 752, + 778, + 924 + ], + [ + 244, + 334, + 378, + 414, + 447, + 475, + 501, + 524, + 546, + 575, + 611, + 645, + 675, + 704, + 732, + 763, + 792, + 819, + 974 + ], + [ + 261, + 357, + 405, + 445, + 478, + 509, + 536, + 562, + 585, + 615, + 655, + 691, + 723, + 754, + 785, + 818, + 848, + 877, + 1043 + ] + ], + "CT": [ + [ + 378, + 519, + 589, + 647, + 698, + 742, + 782, + 820, + 854, + 899, + 957, + 1009, + 1057, + 1103, + 1149, + 1197, + 1243, + 1286, + 1531 + ], + [ + 416, + 572, + 649, + 712, + 768, + 817, + 861, + 902, + 940, + 990, + 1053, + 1111, + 1164, + 1214, + 1264, + 1317, + 1368, + 1415, + 1685 + ], + [ + 441, + 605, + 687, + 754, + 812, + 865, + 912, + 955, + 995, + 1048, + 1115, + 1176, + 1232, + 1285, + 1338, + 1394, + 1447, + 1497, + 1783 + ], + [ + 459, + 631, + 715, + 785, + 847, + 901, + 950, + 994, + 1036, + 1091, + 1161, + 1225, + 1283, + 1338, + 1393, + 1452, + 1507, + 1559, + 1856 + ], + [ + 474, + 651, + 738, + 810, + 874, + 930, + 980, + 1027, + 1069, + 1126, + 1198, + 1264, + 1324, + 1380, + 1438, + 1499, + 1556, + 1609, + 1916 + ], + [ + 495, + 679, + 770, + 846, + 911, + 970, + 1023, + 1070, + 1115, + 1174, + 1249, + 1318, + 1381, + 1439, + 1499, + 1563, + 1622, + 1678, + 1998 + ] + ], + "DC": [ + [ + 363, + 493, + 557, + 609, + 655, + 695, + 731, + 765, + 796, + 837, + 889, + 935, + 978, + 1019, + 1059, + 1103, + 1143, + 1181, + 1398 + ], + [ + 398, + 541, + 611, + 669, + 719, + 764, + 804, + 841, + 874, + 919, + 976, + 1028, + 1074, + 1119, + 1164, + 1212, + 1255, + 1297, + 1536 + ], + [ + 421, + 573, + 647, + 708, + 761, + 807, + 850, + 889, + 925, + 972, + 1032, + 1087, + 1136, + 1183, + 1231, + 1281, + 1327, + 1371, + 1623 + ], + [ + 439, + 596, + 673, + 736, + 792, + 841, + 885, + 925, + 963, + 1012, + 1074, + 1131, + 1183, + 1231, + 1281, + 1332, + 1381, + 1427, + 1689 + ], + [ + 453, + 615, + 695, + 760, + 818, + 867, + 913, + 955, + 993, + 1044, + 1108, + 1167, + 1221, + 1269, + 1321, + 1375, + 1425, + 1473, + 1743 + ], + [ + 472, + 641, + 724, + 792, + 852, + 904, + 952, + 994, + 1035, + 1088, + 1155, + 1216, + 1271, + 1323, + 1376, + 1432, + 1485, + 1534, + 1815 + ] + ], + "DE": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "FL": [ + [ + 379, + 524, + 596, + 656, + 709, + 755, + 797, + 836, + 871, + 918, + 979, + 1034, + 1084, + 1131, + 1179, + 1230, + 1278, + 1323, + 1582 + ], + [ + 433, + 599, + 681, + 751, + 810, + 863, + 911, + 956, + 996, + 1050, + 1119, + 1182, + 1240, + 1294, + 1349, + 1408, + 1461, + 1513, + 1810 + ], + [ + 468, + 649, + 738, + 812, + 877, + 935, + 987, + 1035, + 1080, + 1137, + 1213, + 1281, + 1344, + 1401, + 1461, + 1525, + 1584, + 1640, + 1961 + ], + [ + 497, + 688, + 782, + 861, + 929, + 990, + 1046, + 1097, + 1144, + 1205, + 1285, + 1357, + 1423, + 1485, + 1549, + 1616, + 1679, + 1738, + 2078 + ], + [ + 519, + 719, + 818, + 901, + 972, + 1036, + 1094, + 1148, + 1196, + 1261, + 1344, + 1420, + 1489, + 1554, + 1620, + 1691, + 1756, + 1818, + 2174 + ], + [ + 550, + 764, + 868, + 956, + 1032, + 1100, + 1162, + 1218, + 1270, + 1339, + 1427, + 1507, + 1581, + 1649, + 1720, + 1795, + 1865, + 1931, + 2309 + ] + ], + "GA": [ + [ + 235, + 324, + 368, + 404, + 436, + 464, + 490, + 513, + 535, + 564, + 600, + 633, + 663, + 692, + 721, + 752, + 781, + 808, + 964 + ], + [ + 270, + 373, + 423, + 466, + 503, + 535, + 565, + 591, + 616, + 649, + 692, + 730, + 765, + 798, + 832, + 867, + 901, + 931, + 1112 + ], + [ + 295, + 406, + 461, + 507, + 547, + 583, + 614, + 644, + 671, + 708, + 754, + 795, + 834, + 869, + 906, + 944, + 981, + 1015, + 1212 + ], + [ + 313, + 432, + 491, + 539, + 582, + 619, + 654, + 686, + 714, + 753, + 801, + 846, + 887, + 924, + 964, + 1004, + 1043, + 1080, + 1289 + ], + [ + 328, + 453, + 515, + 566, + 610, + 650, + 687, + 719, + 749, + 790, + 841, + 888, + 930, + 970, + 1012, + 1054, + 1095, + 1133, + 1353 + ], + [ + 350, + 483, + 548, + 603, + 651, + 694, + 731, + 767, + 799, + 842, + 897, + 947, + 992, + 1035, + 1079, + 1124, + 1168, + 1209, + 1442 + ] + ], + "HI": [ + [ + 397, + 537, + 604, + 660, + 709, + 752, + 790, + 826, + 858, + 901, + 956, + 1005, + 1050, + 1093, + 1135, + 1181, + 1224, + 1263, + 1491 + ], + [ + 475, + 641, + 721, + 788, + 845, + 897, + 942, + 984, + 1023, + 1073, + 1138, + 1197, + 1251, + 1301, + 1352, + 1406, + 1456, + 1503, + 1772 + ], + [ + 528, + 712, + 801, + 875, + 939, + 995, + 1046, + 1093, + 1135, + 1192, + 1263, + 1329, + 1388, + 1443, + 1500, + 1560, + 1615, + 1668, + 1965 + ], + [ + 570, + 769, + 864, + 943, + 1013, + 1073, + 1128, + 1178, + 1225, + 1285, + 1362, + 1433, + 1496, + 1556, + 1617, + 1681, + 1741, + 1797, + 2117 + ], + [ + 605, + 816, + 917, + 1001, + 1074, + 1138, + 1196, + 1249, + 1298, + 1363, + 1444, + 1519, + 1586, + 1649, + 1713, + 1781, + 1845, + 1904, + 2244 + ], + [ + 654, + 882, + 991, + 1083, + 1161, + 1231, + 1293, + 1351, + 1404, + 1473, + 1561, + 1641, + 1714, + 1782, + 1851, + 1925, + 1993, + 2058, + 2423 + ] + ], + "IA": [ + [ + 401, + 551, + 626, + 687, + 740, + 787, + 830, + 869, + 906, + 954, + 1015, + 1070, + 1121, + 1169, + 1218, + 1269, + 1317, + 1363, + 1623 + ], + [ + 464, + 638, + 723, + 794, + 856, + 910, + 960, + 1005, + 1047, + 1102, + 1173, + 1237, + 1296, + 1351, + 1407, + 1466, + 1521, + 1574, + 1874 + ], + [ + 507, + 696, + 789, + 866, + 933, + 993, + 1047, + 1096, + 1142, + 1201, + 1279, + 1349, + 1413, + 1473, + 1534, + 1599, + 1658, + 1715, + 2041 + ], + [ + 539, + 741, + 840, + 922, + 993, + 1056, + 1114, + 1167, + 1215, + 1279, + 1361, + 1435, + 1503, + 1567, + 1632, + 1700, + 1765, + 1825, + 2172 + ], + [ + 567, + 778, + 882, + 968, + 1043, + 1109, + 1170, + 1225, + 1276, + 1343, + 1429, + 1507, + 1578, + 1644, + 1713, + 1785, + 1852, + 1916, + 2280 + ], + [ + 605, + 831, + 940, + 1033, + 1113, + 1183, + 1247, + 1306, + 1361, + 1432, + 1523, + 1608, + 1683, + 1754, + 1827, + 1903, + 1975, + 2043, + 2431 + ] + ], + "ID": [ + [ + 504, + 671, + 752, + 818, + 875, + 926, + 971, + 1013, + 1051, + 1101, + 1165, + 1223, + 1276, + 1324, + 1374, + 1426, + 1475, + 1521, + 1782 + ], + [ + 611, + 813, + 911, + 991, + 1060, + 1121, + 1176, + 1226, + 1273, + 1332, + 1410, + 1480, + 1543, + 1601, + 1661, + 1724, + 1783, + 1838, + 2152 + ], + [ + 688, + 914, + 1023, + 1113, + 1190, + 1258, + 1320, + 1376, + 1428, + 1495, + 1581, + 1658, + 1730, + 1796, + 1863, + 1933, + 1999, + 2061, + 2411 + ], + [ + 747, + 994, + 1112, + 1210, + 1294, + 1368, + 1434, + 1495, + 1551, + 1624, + 1717, + 1802, + 1879, + 1950, + 2022, + 2099, + 2170, + 2237, + 2617 + ], + [ + 798, + 1061, + 1187, + 1292, + 1381, + 1460, + 1530, + 1595, + 1655, + 1734, + 1832, + 1922, + 2005, + 2080, + 2157, + 2239, + 2315, + 2387, + 2791 + ], + [ + 871, + 1158, + 1295, + 1409, + 1506, + 1591, + 1669, + 1740, + 1805, + 1889, + 1998, + 2095, + 2184, + 2267, + 2351, + 2439, + 2523, + 2600, + 3040 + ] + ], + "IL": [ + [ + 356, + 485, + 548, + 600, + 646, + 686, + 722, + 755, + 785, + 826, + 876, + 924, + 966, + 1005, + 1047, + 1090, + 1129, + 1168, + 1383 + ], + [ + 421, + 573, + 646, + 707, + 760, + 806, + 849, + 888, + 923, + 971, + 1031, + 1086, + 1135, + 1181, + 1229, + 1279, + 1325, + 1370, + 1621 + ], + [ + 466, + 633, + 714, + 781, + 840, + 892, + 938, + 981, + 1020, + 1071, + 1137, + 1198, + 1252, + 1303, + 1356, + 1411, + 1462, + 1510, + 1787 + ], + [ + 503, + 681, + 769, + 841, + 903, + 959, + 1008, + 1054, + 1097, + 1152, + 1223, + 1288, + 1346, + 1400, + 1456, + 1515, + 1570, + 1623, + 1918 + ], + [ + 532, + 722, + 813, + 891, + 957, + 1015, + 1067, + 1116, + 1161, + 1220, + 1294, + 1363, + 1425, + 1482, + 1542, + 1604, + 1661, + 1716, + 2030 + ], + [ + 576, + 780, + 878, + 961, + 1033, + 1096, + 1153, + 1204, + 1253, + 1316, + 1396, + 1470, + 1537, + 1599, + 1662, + 1730, + 1792, + 1851, + 2188 + ] + ], + "IN": [ + [ + 408, + 564, + 640, + 704, + 759, + 807, + 852, + 893, + 930, + 980, + 1044, + 1102, + 1155, + 1204, + 1255, + 1308, + 1359, + 1406, + 1677 + ], + [ + 475, + 655, + 743, + 817, + 880, + 937, + 989, + 1037, + 1081, + 1137, + 1212, + 1279, + 1340, + 1397, + 1455, + 1518, + 1576, + 1631, + 1945 + ], + [ + 520, + 716, + 812, + 894, + 964, + 1026, + 1082, + 1133, + 1181, + 1244, + 1324, + 1398, + 1465, + 1527, + 1591, + 1659, + 1723, + 1782, + 2126 + ], + [ + 555, + 764, + 867, + 953, + 1028, + 1094, + 1154, + 1210, + 1260, + 1327, + 1413, + 1491, + 1563, + 1629, + 1697, + 1770, + 1837, + 1901, + 2267 + ], + [ + 584, + 804, + 912, + 1002, + 1082, + 1151, + 1214, + 1271, + 1325, + 1395, + 1486, + 1568, + 1643, + 1713, + 1785, + 1862, + 1933, + 2000, + 2385 + ], + [ + 625, + 860, + 976, + 1072, + 1157, + 1231, + 1298, + 1360, + 1418, + 1493, + 1589, + 1677, + 1757, + 1832, + 1909, + 1991, + 2067, + 2138, + 2549 + ] + ], + "KS": [ + [ + 417, + 572, + 647, + 710, + 765, + 813, + 857, + 898, + 934, + 984, + 1046, + 1103, + 1155, + 1203, + 1253, + 1306, + 1355, + 1400, + 1666 + ], + [ + 493, + 675, + 764, + 838, + 903, + 960, + 1010, + 1058, + 1102, + 1160, + 1233, + 1300, + 1361, + 1418, + 1477, + 1538, + 1595, + 1649, + 1961 + ], + [ + 544, + 745, + 844, + 925, + 996, + 1059, + 1116, + 1169, + 1217, + 1281, + 1361, + 1435, + 1502, + 1565, + 1629, + 1697, + 1761, + 1821, + 2163 + ], + [ + 586, + 801, + 907, + 994, + 1070, + 1138, + 1199, + 1255, + 1307, + 1375, + 1461, + 1542, + 1614, + 1681, + 1750, + 1823, + 1891, + 1955, + 2322 + ], + [ + 619, + 848, + 960, + 1052, + 1132, + 1204, + 1268, + 1327, + 1382, + 1454, + 1546, + 1630, + 1706, + 1777, + 1850, + 1928, + 1999, + 2067, + 2455 + ], + [ + 668, + 914, + 1034, + 1133, + 1221, + 1297, + 1367, + 1430, + 1489, + 1567, + 1666, + 1756, + 1837, + 1914, + 1993, + 2076, + 2153, + 2226, + 2644 + ] + ], + "KY": [ + [ + 388, + 537, + 610, + 672, + 726, + 773, + 817, + 856, + 893, + 940, + 1002, + 1059, + 1110, + 1159, + 1208, + 1260, + 1309, + 1355, + 1620 + ], + [ + 443, + 613, + 698, + 768, + 829, + 884, + 932, + 978, + 1020, + 1074, + 1146, + 1211, + 1268, + 1324, + 1380, + 1440, + 1496, + 1549, + 1852 + ], + [ + 479, + 664, + 755, + 831, + 898, + 957, + 1009, + 1058, + 1104, + 1164, + 1240, + 1310, + 1374, + 1434, + 1495, + 1560, + 1621, + 1678, + 2007 + ], + [ + 507, + 703, + 799, + 879, + 951, + 1013, + 1069, + 1121, + 1169, + 1232, + 1313, + 1387, + 1455, + 1518, + 1583, + 1652, + 1716, + 1777, + 2126 + ], + [ + 530, + 734, + 836, + 920, + 993, + 1059, + 1118, + 1172, + 1223, + 1289, + 1374, + 1451, + 1522, + 1588, + 1655, + 1727, + 1795, + 1858, + 2223 + ], + [ + 563, + 779, + 887, + 976, + 1054, + 1123, + 1186, + 1244, + 1297, + 1367, + 1457, + 1540, + 1615, + 1685, + 1757, + 1833, + 1904, + 1972, + 2359 + ] + ], + "LA": [ + [ + 313, + 434, + 493, + 543, + 587, + 625, + 661, + 693, + 722, + 762, + 812, + 858, + 900, + 939, + 980, + 1022, + 1062, + 1100, + 1317 + ], + [ + 356, + 494, + 563, + 620, + 670, + 714, + 754, + 791, + 825, + 871, + 928, + 980, + 1028, + 1074, + 1119, + 1168, + 1214, + 1257, + 1505 + ], + [ + 386, + 535, + 609, + 671, + 726, + 774, + 817, + 857, + 893, + 943, + 1005, + 1062, + 1114, + 1163, + 1213, + 1265, + 1315, + 1361, + 1631 + ], + [ + 409, + 567, + 645, + 711, + 768, + 818, + 865, + 907, + 946, + 997, + 1063, + 1124, + 1180, + 1231, + 1284, + 1341, + 1393, + 1442, + 1728 + ], + [ + 427, + 592, + 674, + 743, + 803, + 856, + 905, + 948, + 989, + 1044, + 1113, + 1176, + 1234, + 1287, + 1343, + 1402, + 1457, + 1509, + 1806 + ], + [ + 453, + 629, + 715, + 788, + 852, + 908, + 960, + 1006, + 1050, + 1107, + 1181, + 1248, + 1309, + 1367, + 1425, + 1488, + 1546, + 1601, + 1918 + ] + ], + "MA": [ + [ + 318, + 432, + 486, + 532, + 572, + 607, + 639, + 668, + 695, + 730, + 775, + 816, + 853, + 888, + 923, + 961, + 995, + 1028, + 1216 + ], + [ + 362, + 491, + 552, + 605, + 650, + 690, + 726, + 759, + 789, + 829, + 879, + 926, + 969, + 1007, + 1048, + 1091, + 1130, + 1168, + 1380 + ], + [ + 390, + 529, + 597, + 653, + 702, + 744, + 784, + 820, + 852, + 896, + 951, + 1000, + 1046, + 1089, + 1131, + 1178, + 1221, + 1260, + 1491 + ], + [ + 412, + 560, + 631, + 691, + 742, + 787, + 829, + 866, + 901, + 947, + 1004, + 1057, + 1106, + 1151, + 1196, + 1245, + 1290, + 1332, + 1575 + ], + [ + 431, + 585, + 659, + 721, + 775, + 823, + 865, + 905, + 941, + 988, + 1049, + 1105, + 1155, + 1201, + 1249, + 1300, + 1348, + 1391, + 1645 + ], + [ + 457, + 619, + 699, + 765, + 822, + 871, + 917, + 959, + 997, + 1047, + 1112, + 1170, + 1224, + 1273, + 1324, + 1377, + 1428, + 1475, + 1743 + ] + ], + "MD": [ + [ + 301, + 413, + 468, + 515, + 555, + 590, + 623, + 652, + 679, + 715, + 761, + 803, + 841, + 877, + 914, + 953, + 988, + 1023, + 1219 + ], + [ + 350, + 481, + 545, + 599, + 646, + 687, + 724, + 759, + 790, + 832, + 885, + 933, + 977, + 1019, + 1061, + 1106, + 1148, + 1187, + 1414 + ], + [ + 386, + 529, + 599, + 658, + 709, + 754, + 794, + 832, + 866, + 912, + 970, + 1024, + 1071, + 1117, + 1164, + 1213, + 1258, + 1301, + 1549 + ], + [ + 413, + 567, + 642, + 704, + 759, + 806, + 850, + 891, + 927, + 976, + 1038, + 1095, + 1147, + 1194, + 1244, + 1297, + 1346, + 1391, + 1655 + ], + [ + 436, + 598, + 677, + 743, + 800, + 851, + 897, + 939, + 978, + 1030, + 1095, + 1155, + 1210, + 1260, + 1312, + 1367, + 1419, + 1467, + 1745 + ], + [ + 469, + 643, + 727, + 798, + 860, + 914, + 964, + 1008, + 1051, + 1106, + 1176, + 1240, + 1298, + 1353, + 1409, + 1467, + 1522, + 1575, + 1873 + ] + ], + "ME": [ + [ + 305, + 413, + 465, + 509, + 546, + 580, + 609, + 637, + 663, + 696, + 738, + 777, + 812, + 845, + 878, + 914, + 947, + 978, + 1155 + ], + [ + 346, + 468, + 528, + 577, + 619, + 658, + 692, + 723, + 752, + 789, + 837, + 882, + 921, + 958, + 995, + 1036, + 1073, + 1108, + 1308 + ], + [ + 374, + 506, + 570, + 623, + 669, + 709, + 746, + 780, + 810, + 851, + 903, + 950, + 993, + 1033, + 1073, + 1117, + 1157, + 1194, + 1411 + ], + [ + 395, + 534, + 601, + 658, + 706, + 749, + 788, + 824, + 856, + 899, + 954, + 1003, + 1048, + 1091, + 1133, + 1179, + 1222, + 1261, + 1489 + ], + [ + 412, + 558, + 628, + 687, + 737, + 782, + 823, + 859, + 893, + 937, + 995, + 1047, + 1094, + 1137, + 1183, + 1230, + 1275, + 1316, + 1554 + ], + [ + 437, + 590, + 665, + 727, + 780, + 828, + 870, + 909, + 945, + 992, + 1053, + 1108, + 1158, + 1203, + 1251, + 1302, + 1349, + 1392, + 1643 + ] + ], + "MI": [ + [ + 342, + 470, + 533, + 586, + 632, + 672, + 709, + 742, + 774, + 816, + 867, + 915, + 959, + 999, + 1042, + 1086, + 1127, + 1166, + 1389 + ], + [ + 392, + 540, + 612, + 673, + 726, + 773, + 814, + 853, + 890, + 936, + 996, + 1052, + 1102, + 1149, + 1196, + 1247, + 1295, + 1340, + 1596 + ], + [ + 427, + 587, + 666, + 731, + 789, + 840, + 886, + 927, + 967, + 1018, + 1084, + 1144, + 1197, + 1248, + 1301, + 1356, + 1408, + 1456, + 1736 + ], + [ + 453, + 624, + 707, + 777, + 838, + 892, + 940, + 985, + 1027, + 1081, + 1151, + 1214, + 1271, + 1326, + 1381, + 1440, + 1495, + 1547, + 1843 + ], + [ + 474, + 653, + 741, + 814, + 878, + 934, + 986, + 1033, + 1075, + 1133, + 1206, + 1273, + 1333, + 1390, + 1448, + 1510, + 1567, + 1622, + 1933 + ], + [ + 505, + 696, + 789, + 867, + 935, + 995, + 1049, + 1099, + 1146, + 1206, + 1284, + 1355, + 1420, + 1480, + 1542, + 1608, + 1669, + 1726, + 2058 + ] + ], + "MN": [ + [ + 366, + 513, + 586, + 648, + 702, + 749, + 793, + 833, + 870, + 919, + 982, + 1040, + 1093, + 1142, + 1192, + 1246, + 1297, + 1345, + 1619 + ], + [ + 414, + 582, + 665, + 735, + 796, + 851, + 900, + 945, + 988, + 1044, + 1115, + 1181, + 1241, + 1297, + 1355, + 1416, + 1473, + 1527, + 1840 + ], + [ + 447, + 628, + 717, + 792, + 858, + 917, + 971, + 1020, + 1065, + 1125, + 1203, + 1274, + 1339, + 1399, + 1461, + 1527, + 1589, + 1648, + 1986 + ], + [ + 471, + 662, + 757, + 837, + 907, + 969, + 1025, + 1077, + 1125, + 1189, + 1270, + 1346, + 1414, + 1478, + 1544, + 1614, + 1679, + 1741, + 2098 + ], + [ + 492, + 691, + 790, + 873, + 945, + 1010, + 1069, + 1124, + 1174, + 1241, + 1325, + 1405, + 1476, + 1543, + 1611, + 1684, + 1752, + 1817, + 2191 + ], + [ + 520, + 730, + 836, + 923, + 1000, + 1069, + 1131, + 1189, + 1242, + 1313, + 1403, + 1486, + 1561, + 1632, + 1705, + 1782, + 1854, + 1923, + 2319 + ] + ], + "MO": [ + [ + 263, + 363, + 411, + 452, + 487, + 519, + 547, + 574, + 598, + 630, + 670, + 708, + 741, + 773, + 805, + 840, + 872, + 903, + 1077 + ], + [ + 307, + 423, + 480, + 528, + 570, + 606, + 640, + 670, + 699, + 736, + 784, + 828, + 867, + 904, + 942, + 982, + 1020, + 1055, + 1259 + ], + [ + 337, + 465, + 528, + 580, + 626, + 666, + 703, + 736, + 768, + 808, + 861, + 909, + 953, + 993, + 1035, + 1080, + 1121, + 1160, + 1384 + ], + [ + 361, + 498, + 565, + 621, + 670, + 713, + 753, + 788, + 822, + 865, + 922, + 973, + 1020, + 1063, + 1108, + 1156, + 1199, + 1242, + 1482 + ], + [ + 381, + 525, + 596, + 655, + 707, + 753, + 793, + 832, + 866, + 913, + 972, + 1027, + 1075, + 1121, + 1169, + 1219, + 1265, + 1310, + 1563 + ], + [ + 408, + 563, + 639, + 703, + 759, + 807, + 852, + 893, + 930, + 980, + 1044, + 1102, + 1155, + 1203, + 1254, + 1308, + 1359, + 1406, + 1678 + ] + ], + "MS": [ + [ + 611, + 819, + 918, + 1000, + 1071, + 1134, + 1191, + 1243, + 1290, + 1353, + 1432, + 1504, + 1570, + 1630, + 1693, + 1759, + 1820, + 1877, + 2205 + ], + [ + 741, + 991, + 1111, + 1210, + 1295, + 1371, + 1438, + 1500, + 1558, + 1632, + 1727, + 1815, + 1893, + 1966, + 2040, + 2119, + 2193, + 2262, + 2653 + ], + [ + 833, + 1111, + 1245, + 1356, + 1451, + 1536, + 1611, + 1680, + 1744, + 1827, + 1934, + 2030, + 2117, + 2199, + 2282, + 2370, + 2452, + 2529, + 2964 + ], + [ + 904, + 1206, + 1351, + 1471, + 1574, + 1666, + 1748, + 1822, + 1892, + 1982, + 2097, + 2202, + 2296, + 2385, + 2474, + 2569, + 2658, + 2741, + 3212 + ], + [ + 965, + 1286, + 1441, + 1568, + 1679, + 1775, + 1863, + 1943, + 2016, + 2112, + 2234, + 2346, + 2447, + 2540, + 2635, + 2737, + 2830, + 2919, + 3420 + ], + [ + 1051, + 1400, + 1568, + 1707, + 1827, + 1932, + 2027, + 2113, + 2194, + 2297, + 2430, + 2551, + 2661, + 2762, + 2866, + 2976, + 3078, + 3174, + 3718 + ] + ], + "MT": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "NC": [ + [ + 345, + 465, + 522, + 570, + 611, + 648, + 680, + 711, + 738, + 774, + 821, + 862, + 901, + 936, + 973, + 1010, + 1046, + 1080, + 1271 + ], + [ + 413, + 555, + 624, + 680, + 729, + 772, + 811, + 847, + 879, + 923, + 977, + 1028, + 1072, + 1115, + 1158, + 1203, + 1245, + 1286, + 1512 + ], + [ + 460, + 617, + 694, + 757, + 810, + 859, + 902, + 941, + 978, + 1026, + 1087, + 1142, + 1192, + 1238, + 1287, + 1336, + 1383, + 1428, + 1679 + ], + [ + 497, + 667, + 748, + 817, + 875, + 927, + 974, + 1017, + 1055, + 1107, + 1172, + 1232, + 1286, + 1336, + 1388, + 1442, + 1493, + 1541, + 1811 + ], + [ + 528, + 709, + 795, + 867, + 929, + 984, + 1034, + 1080, + 1121, + 1175, + 1245, + 1308, + 1365, + 1419, + 1474, + 1530, + 1584, + 1635, + 1921 + ], + [ + 573, + 768, + 862, + 939, + 1007, + 1066, + 1120, + 1169, + 1214, + 1273, + 1348, + 1417, + 1479, + 1536, + 1594, + 1657, + 1715, + 1769, + 2079 + ] + ], + "ND": [ + [ + 284, + 391, + 444, + 488, + 526, + 561, + 591, + 619, + 646, + 679, + 724, + 764, + 800, + 835, + 869, + 907, + 941, + 974, + 1162 + ], + [ + 330, + 455, + 516, + 567, + 611, + 650, + 686, + 718, + 748, + 788, + 839, + 886, + 928, + 968, + 1008, + 1051, + 1091, + 1129, + 1346 + ], + [ + 362, + 498, + 565, + 619, + 668, + 712, + 751, + 786, + 820, + 862, + 918, + 969, + 1016, + 1058, + 1103, + 1150, + 1193, + 1235, + 1472 + ], + [ + 386, + 531, + 602, + 662, + 714, + 760, + 801, + 839, + 874, + 920, + 980, + 1034, + 1084, + 1129, + 1176, + 1227, + 1274, + 1317, + 1570 + ], + [ + 406, + 560, + 634, + 697, + 752, + 799, + 843, + 884, + 920, + 969, + 1031, + 1089, + 1140, + 1188, + 1238, + 1291, + 1340, + 1386, + 1651 + ], + [ + 435, + 599, + 679, + 745, + 804, + 856, + 903, + 945, + 985, + 1037, + 1104, + 1165, + 1220, + 1271, + 1325, + 1381, + 1434, + 1484, + 1767 + ] + ], + "NE": [ + [ + 339, + 474, + 541, + 598, + 647, + 691, + 730, + 767, + 801, + 846, + 903, + 956, + 1003, + 1048, + 1095, + 1144, + 1189, + 1233, + 1483 + ], + [ + 389, + 544, + 622, + 687, + 742, + 793, + 839, + 880, + 920, + 971, + 1037, + 1098, + 1153, + 1204, + 1257, + 1313, + 1366, + 1416, + 1703 + ], + [ + 422, + 591, + 675, + 745, + 806, + 861, + 911, + 957, + 999, + 1055, + 1126, + 1192, + 1252, + 1308, + 1366, + 1427, + 1484, + 1539, + 1850 + ], + [ + 448, + 628, + 716, + 791, + 856, + 914, + 967, + 1016, + 1060, + 1119, + 1195, + 1265, + 1328, + 1388, + 1449, + 1514, + 1575, + 1633, + 1963 + ], + [ + 469, + 657, + 751, + 829, + 897, + 958, + 1013, + 1063, + 1111, + 1173, + 1252, + 1325, + 1392, + 1454, + 1518, + 1586, + 1650, + 1710, + 2058 + ], + [ + 499, + 699, + 798, + 880, + 954, + 1019, + 1078, + 1131, + 1181, + 1247, + 1332, + 1410, + 1481, + 1547, + 1615, + 1688, + 1755, + 1819, + 2189 + ] + ], + "NH": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "NJ": [ + [ + 402, + 556, + 631, + 694, + 748, + 797, + 842, + 883, + 920, + 969, + 1032, + 1090, + 1143, + 1191, + 1242, + 1296, + 1346, + 1392, + 1664 + ], + [ + 451, + 624, + 708, + 779, + 841, + 896, + 944, + 990, + 1033, + 1088, + 1159, + 1224, + 1283, + 1339, + 1394, + 1455, + 1511, + 1564, + 1868 + ], + [ + 483, + 667, + 759, + 835, + 901, + 960, + 1013, + 1061, + 1107, + 1166, + 1242, + 1312, + 1375, + 1434, + 1495, + 1559, + 1620, + 1677, + 2002 + ], + [ + 508, + 702, + 797, + 877, + 947, + 1008, + 1064, + 1115, + 1163, + 1226, + 1305, + 1378, + 1445, + 1507, + 1571, + 1639, + 1702, + 1762, + 2104 + ], + [ + 527, + 729, + 829, + 912, + 984, + 1048, + 1106, + 1160, + 1210, + 1274, + 1357, + 1433, + 1503, + 1567, + 1634, + 1704, + 1770, + 1832, + 2189 + ], + [ + 556, + 768, + 872, + 960, + 1036, + 1104, + 1165, + 1221, + 1273, + 1342, + 1429, + 1509, + 1582, + 1650, + 1720, + 1795, + 1864, + 1930, + 2304 + ] + ], + "NM": [ + [ + 366, + 509, + 580, + 639, + 691, + 737, + 779, + 817, + 852, + 899, + 959, + 1015, + 1064, + 1111, + 1159, + 1210, + 1257, + 1302, + 1561 + ], + [ + 413, + 575, + 655, + 721, + 780, + 832, + 878, + 922, + 962, + 1015, + 1083, + 1145, + 1200, + 1253, + 1307, + 1365, + 1419, + 1470, + 1761 + ], + [ + 444, + 617, + 704, + 775, + 838, + 894, + 944, + 990, + 1034, + 1090, + 1163, + 1229, + 1290, + 1347, + 1405, + 1466, + 1524, + 1578, + 1892 + ], + [ + 467, + 650, + 740, + 817, + 883, + 940, + 994, + 1043, + 1088, + 1148, + 1224, + 1294, + 1358, + 1418, + 1479, + 1544, + 1605, + 1661, + 1992 + ], + [ + 486, + 676, + 771, + 849, + 918, + 979, + 1035, + 1086, + 1132, + 1194, + 1274, + 1347, + 1414, + 1476, + 1539, + 1607, + 1670, + 1730, + 2073 + ], + [ + 513, + 714, + 812, + 896, + 968, + 1033, + 1091, + 1145, + 1194, + 1259, + 1344, + 1420, + 1490, + 1555, + 1623, + 1694, + 1760, + 1823, + 2186 + ] + ], + "NV": [ + [ + 392, + 536, + 607, + 665, + 716, + 762, + 802, + 839, + 873, + 919, + 977, + 1030, + 1078, + 1122, + 1169, + 1217, + 1262, + 1305, + 1550 + ], + [ + 453, + 619, + 701, + 768, + 827, + 878, + 925, + 968, + 1008, + 1060, + 1127, + 1188, + 1243, + 1295, + 1348, + 1404, + 1456, + 1505, + 1786 + ], + [ + 494, + 675, + 764, + 837, + 901, + 958, + 1008, + 1055, + 1099, + 1156, + 1229, + 1295, + 1355, + 1412, + 1469, + 1530, + 1586, + 1640, + 1947 + ], + [ + 525, + 718, + 812, + 891, + 959, + 1019, + 1073, + 1123, + 1169, + 1230, + 1307, + 1378, + 1442, + 1502, + 1563, + 1628, + 1689, + 1746, + 2072 + ], + [ + 551, + 755, + 853, + 935, + 1006, + 1069, + 1127, + 1179, + 1228, + 1291, + 1372, + 1446, + 1514, + 1577, + 1641, + 1709, + 1773, + 1833, + 2175 + ], + [ + 589, + 804, + 910, + 998, + 1074, + 1142, + 1202, + 1258, + 1310, + 1378, + 1464, + 1544, + 1615, + 1682, + 1751, + 1824, + 1891, + 1955, + 2321 + ] + ], + "NY": [ + [ + 246, + 339, + 386, + 425, + 458, + 487, + 515, + 539, + 563, + 592, + 631, + 666, + 699, + 729, + 760, + 792, + 823, + 852, + 1018 + ], + [ + 274, + 379, + 431, + 473, + 511, + 544, + 575, + 602, + 628, + 661, + 705, + 744, + 780, + 813, + 848, + 885, + 919, + 952, + 1136 + ], + [ + 292, + 404, + 460, + 506, + 545, + 581, + 613, + 643, + 670, + 707, + 753, + 795, + 834, + 869, + 906, + 945, + 982, + 1016, + 1214 + ], + [ + 307, + 423, + 481, + 530, + 572, + 609, + 643, + 674, + 703, + 740, + 789, + 834, + 873, + 911, + 950, + 991, + 1029, + 1065, + 1273 + ], + [ + 318, + 440, + 500, + 549, + 593, + 633, + 667, + 700, + 729, + 769, + 819, + 865, + 907, + 945, + 986, + 1029, + 1068, + 1106, + 1321 + ], + [ + 334, + 462, + 525, + 577, + 624, + 664, + 701, + 734, + 766, + 807, + 860, + 908, + 952, + 993, + 1035, + 1080, + 1121, + 1161, + 1387 + ] + ], + "OH": [ + [ + 372, + 513, + 583, + 641, + 692, + 736, + 777, + 814, + 849, + 894, + 953, + 1005, + 1054, + 1099, + 1146, + 1194, + 1240, + 1284, + 1532 + ], + [ + 420, + 580, + 659, + 724, + 782, + 832, + 878, + 920, + 960, + 1010, + 1077, + 1136, + 1191, + 1242, + 1295, + 1351, + 1403, + 1451, + 1733 + ], + [ + 452, + 625, + 709, + 780, + 841, + 896, + 945, + 990, + 1033, + 1088, + 1159, + 1223, + 1282, + 1336, + 1393, + 1453, + 1509, + 1562, + 1864 + ], + [ + 476, + 658, + 747, + 822, + 887, + 944, + 996, + 1044, + 1089, + 1147, + 1221, + 1290, + 1352, + 1410, + 1469, + 1531, + 1590, + 1646, + 1965 + ], + [ + 497, + 686, + 779, + 856, + 924, + 984, + 1038, + 1088, + 1134, + 1195, + 1273, + 1344, + 1409, + 1469, + 1530, + 1596, + 1657, + 1715, + 2047 + ], + [ + 525, + 724, + 823, + 905, + 976, + 1039, + 1097, + 1150, + 1198, + 1262, + 1345, + 1420, + 1488, + 1552, + 1617, + 1686, + 1751, + 1812, + 2163 + ] + ], + "OK": [ + [ + 318, + 432, + 487, + 534, + 574, + 609, + 642, + 671, + 699, + 734, + 780, + 822, + 859, + 894, + 930, + 968, + 1003, + 1037, + 1229 + ], + [ + 381, + 517, + 584, + 639, + 687, + 729, + 768, + 802, + 835, + 877, + 931, + 981, + 1026, + 1067, + 1110, + 1156, + 1198, + 1238, + 1465 + ], + [ + 425, + 577, + 651, + 712, + 766, + 812, + 855, + 894, + 929, + 977, + 1037, + 1092, + 1142, + 1188, + 1236, + 1287, + 1333, + 1377, + 1630 + ], + [ + 459, + 624, + 704, + 770, + 828, + 878, + 924, + 966, + 1005, + 1056, + 1121, + 1180, + 1234, + 1285, + 1335, + 1390, + 1441, + 1489, + 1761 + ], + [ + 488, + 663, + 748, + 819, + 880, + 934, + 983, + 1028, + 1068, + 1123, + 1192, + 1255, + 1312, + 1365, + 1420, + 1478, + 1531, + 1582, + 1871 + ], + [ + 531, + 720, + 812, + 889, + 955, + 1014, + 1066, + 1114, + 1159, + 1218, + 1293, + 1361, + 1423, + 1481, + 1540, + 1602, + 1660, + 1715, + 2028 + ] + ], + "OR": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "PA": [ + [ + 351, + 474, + 534, + 583, + 626, + 664, + 698, + 729, + 758, + 795, + 843, + 887, + 926, + 963, + 1000, + 1041, + 1078, + 1112, + 1311 + ], + [ + 397, + 535, + 602, + 658, + 706, + 747, + 786, + 821, + 853, + 896, + 950, + 998, + 1043, + 1085, + 1127, + 1172, + 1214, + 1252, + 1477 + ], + [ + 427, + 575, + 647, + 707, + 758, + 803, + 844, + 882, + 916, + 962, + 1020, + 1072, + 1120, + 1165, + 1210, + 1258, + 1303, + 1345, + 1584 + ], + [ + 449, + 605, + 681, + 743, + 798, + 846, + 889, + 928, + 965, + 1013, + 1073, + 1128, + 1179, + 1226, + 1274, + 1324, + 1371, + 1416, + 1668 + ], + [ + 468, + 631, + 709, + 775, + 831, + 880, + 925, + 967, + 1004, + 1054, + 1117, + 1175, + 1227, + 1276, + 1325, + 1378, + 1427, + 1474, + 1736 + ], + [ + 494, + 665, + 748, + 818, + 876, + 929, + 976, + 1020, + 1059, + 1112, + 1178, + 1239, + 1294, + 1346, + 1397, + 1453, + 1505, + 1554, + 1830 + ] + ], + "RI": [ + [ + 370, + 507, + 575, + 631, + 678, + 722, + 761, + 796, + 830, + 872, + 928, + 979, + 1025, + 1067, + 1112, + 1159, + 1201, + 1243, + 1478 + ], + [ + 423, + 581, + 657, + 721, + 777, + 826, + 870, + 911, + 949, + 998, + 1061, + 1119, + 1172, + 1221, + 1271, + 1324, + 1374, + 1421, + 1689 + ], + [ + 460, + 630, + 713, + 782, + 842, + 896, + 943, + 988, + 1029, + 1083, + 1151, + 1214, + 1270, + 1323, + 1378, + 1436, + 1489, + 1540, + 1830 + ], + [ + 487, + 668, + 756, + 830, + 893, + 950, + 1000, + 1047, + 1091, + 1148, + 1220, + 1286, + 1347, + 1403, + 1460, + 1521, + 1578, + 1632, + 1939 + ], + [ + 511, + 699, + 791, + 868, + 935, + 994, + 1047, + 1096, + 1142, + 1200, + 1277, + 1347, + 1410, + 1469, + 1528, + 1592, + 1651, + 1708, + 2029 + ], + [ + 543, + 743, + 841, + 922, + 993, + 1056, + 1113, + 1165, + 1213, + 1276, + 1357, + 1430, + 1497, + 1560, + 1624, + 1692, + 1755, + 1814, + 2155 + ] + ], + "SC": [ + [ + 351, + 484, + 549, + 604, + 652, + 694, + 732, + 767, + 799, + 842, + 897, + 947, + 991, + 1034, + 1078, + 1123, + 1166, + 1206, + 1439 + ], + [ + 402, + 555, + 629, + 692, + 745, + 794, + 838, + 877, + 914, + 963, + 1026, + 1083, + 1134, + 1182, + 1232, + 1285, + 1333, + 1380, + 1645 + ], + [ + 437, + 601, + 682, + 749, + 808, + 860, + 908, + 951, + 991, + 1044, + 1111, + 1173, + 1229, + 1281, + 1334, + 1392, + 1445, + 1495, + 1782 + ], + [ + 462, + 637, + 723, + 794, + 857, + 912, + 962, + 1007, + 1050, + 1106, + 1177, + 1242, + 1302, + 1357, + 1415, + 1475, + 1530, + 1584, + 1888 + ], + [ + 484, + 667, + 757, + 832, + 897, + 955, + 1006, + 1054, + 1099, + 1157, + 1232, + 1300, + 1362, + 1420, + 1480, + 1543, + 1602, + 1657, + 1975 + ], + [ + 514, + 708, + 803, + 883, + 952, + 1014, + 1068, + 1119, + 1167, + 1229, + 1308, + 1380, + 1446, + 1508, + 1571, + 1638, + 1700, + 1759, + 2097 + ] + ], + "SD": [ + [ + 381, + 519, + 586, + 642, + 691, + 734, + 773, + 808, + 841, + 885, + 939, + 990, + 1036, + 1079, + 1122, + 1169, + 1212, + 1252, + 1485 + ], + [ + 460, + 626, + 707, + 775, + 833, + 885, + 931, + 974, + 1014, + 1065, + 1132, + 1192, + 1247, + 1299, + 1352, + 1407, + 1458, + 1507, + 1786 + ], + [ + 515, + 701, + 791, + 866, + 931, + 989, + 1041, + 1090, + 1133, + 1191, + 1265, + 1333, + 1394, + 1451, + 1510, + 1572, + 1630, + 1684, + 1996 + ], + [ + 558, + 760, + 857, + 939, + 1009, + 1072, + 1128, + 1180, + 1228, + 1291, + 1371, + 1444, + 1510, + 1572, + 1636, + 1703, + 1765, + 1824, + 2160 + ], + [ + 595, + 809, + 913, + 1000, + 1075, + 1142, + 1201, + 1257, + 1308, + 1375, + 1459, + 1538, + 1609, + 1674, + 1742, + 1813, + 1879, + 1942, + 2299 + ], + [ + 647, + 879, + 993, + 1087, + 1169, + 1241, + 1306, + 1366, + 1421, + 1494, + 1586, + 1671, + 1747, + 1819, + 1892, + 1969, + 2041, + 2109, + 2497 + ] + ], + "TN": [ + [ + 537, + 728, + 822, + 898, + 965, + 1024, + 1077, + 1125, + 1171, + 1230, + 1305, + 1373, + 1436, + 1494, + 1553, + 1616, + 1675, + 1730, + 2044 + ], + [ + 640, + 866, + 976, + 1067, + 1147, + 1217, + 1280, + 1338, + 1391, + 1460, + 1550, + 1631, + 1705, + 1774, + 1844, + 1918, + 1988, + 2053, + 2425 + ], + [ + 710, + 962, + 1084, + 1184, + 1273, + 1350, + 1420, + 1484, + 1543, + 1620, + 1719, + 1809, + 1891, + 1967, + 2045, + 2128, + 2204, + 2277, + 2689 + ], + [ + 766, + 1036, + 1168, + 1277, + 1371, + 1454, + 1529, + 1599, + 1662, + 1746, + 1852, + 1949, + 2037, + 2119, + 2203, + 2292, + 2374, + 2453, + 2896 + ], + [ + 811, + 1099, + 1238, + 1354, + 1454, + 1543, + 1622, + 1695, + 1763, + 1851, + 1964, + 2067, + 2160, + 2246, + 2336, + 2429, + 2518, + 2600, + 3070 + ], + [ + 877, + 1188, + 1339, + 1463, + 1572, + 1667, + 1753, + 1832, + 1905, + 2001, + 2123, + 2233, + 2334, + 2427, + 2524, + 2625, + 2720, + 2809, + 3316 + ] + ], + "TX": [ + [ + 388, + 535, + 607, + 668, + 721, + 768, + 809, + 849, + 885, + 931, + 992, + 1047, + 1098, + 1145, + 1193, + 1244, + 1292, + 1336, + 1595 + ], + [ + 452, + 624, + 708, + 778, + 839, + 893, + 942, + 987, + 1029, + 1084, + 1155, + 1219, + 1277, + 1331, + 1387, + 1447, + 1502, + 1555, + 1854 + ], + [ + 496, + 682, + 775, + 852, + 919, + 978, + 1032, + 1082, + 1127, + 1187, + 1264, + 1334, + 1398, + 1458, + 1519, + 1584, + 1645, + 1702, + 2030 + ], + [ + 529, + 729, + 828, + 910, + 981, + 1045, + 1102, + 1155, + 1203, + 1267, + 1350, + 1425, + 1493, + 1557, + 1622, + 1691, + 1756, + 1817, + 2167 + ], + [ + 557, + 768, + 871, + 958, + 1033, + 1100, + 1160, + 1216, + 1266, + 1334, + 1421, + 1500, + 1571, + 1638, + 1707, + 1780, + 1848, + 1912, + 2281 + ], + [ + 596, + 823, + 933, + 1026, + 1106, + 1177, + 1242, + 1301, + 1356, + 1428, + 1520, + 1605, + 1682, + 1754, + 1827, + 1905, + 1978, + 2046, + 2440 + ] + ], + "UT": [ + [ + 375, + 508, + 573, + 627, + 673, + 715, + 753, + 786, + 819, + 859, + 912, + 961, + 1004, + 1045, + 1087, + 1130, + 1172, + 1211, + 1432 + ], + [ + 445, + 603, + 680, + 744, + 799, + 848, + 893, + 933, + 970, + 1020, + 1082, + 1138, + 1190, + 1239, + 1288, + 1341, + 1388, + 1435, + 1696 + ], + [ + 493, + 668, + 754, + 825, + 886, + 939, + 989, + 1034, + 1074, + 1129, + 1198, + 1261, + 1318, + 1372, + 1427, + 1485, + 1538, + 1588, + 1878 + ], + [ + 531, + 720, + 811, + 888, + 954, + 1012, + 1064, + 1113, + 1157, + 1216, + 1290, + 1358, + 1420, + 1477, + 1536, + 1597, + 1655, + 1710, + 2021 + ], + [ + 563, + 763, + 860, + 940, + 1010, + 1072, + 1128, + 1179, + 1226, + 1288, + 1367, + 1438, + 1504, + 1564, + 1627, + 1692, + 1754, + 1812, + 2141 + ], + [ + 608, + 824, + 929, + 1016, + 1091, + 1158, + 1218, + 1273, + 1323, + 1390, + 1476, + 1553, + 1623, + 1689, + 1756, + 1827, + 1893, + 1955, + 2310 + ] + ], + "VA": [ + [ + 241, + 332, + 377, + 414, + 447, + 475, + 502, + 525, + 547, + 577, + 613, + 648, + 678, + 708, + 737, + 769, + 798, + 826, + 985 + ], + [ + 276, + 381, + 433, + 475, + 513, + 545, + 575, + 602, + 628, + 661, + 704, + 743, + 779, + 811, + 846, + 883, + 916, + 948, + 1129 + ], + [ + 301, + 414, + 470, + 516, + 557, + 592, + 625, + 655, + 682, + 719, + 765, + 807, + 846, + 883, + 919, + 959, + 995, + 1030, + 1227 + ], + [ + 319, + 440, + 499, + 548, + 591, + 630, + 663, + 696, + 724, + 763, + 812, + 857, + 899, + 936, + 976, + 1018, + 1056, + 1093, + 1303 + ], + [ + 334, + 461, + 523, + 575, + 619, + 659, + 696, + 729, + 760, + 800, + 851, + 899, + 941, + 982, + 1023, + 1066, + 1107, + 1146, + 1366 + ], + [ + 356, + 491, + 557, + 611, + 660, + 702, + 740, + 776, + 808, + 851, + 907, + 957, + 1002, + 1045, + 1089, + 1135, + 1179, + 1220, + 1453 + ] + ], + "VT": [ + [ + 270, + 373, + 422, + 465, + 502, + 533, + 563, + 590, + 614, + 647, + 690, + 727, + 763, + 795, + 829, + 863, + 897, + 928, + 1106 + ], + [ + 292, + 403, + 457, + 503, + 542, + 577, + 608, + 638, + 664, + 700, + 745, + 786, + 825, + 859, + 896, + 933, + 969, + 1003, + 1196 + ], + [ + 306, + 421, + 478, + 526, + 568, + 604, + 638, + 667, + 696, + 733, + 780, + 824, + 863, + 900, + 937, + 978, + 1015, + 1050, + 1252 + ], + [ + 317, + 436, + 495, + 544, + 587, + 625, + 659, + 690, + 719, + 758, + 806, + 851, + 892, + 930, + 969, + 1010, + 1049, + 1086, + 1294 + ], + [ + 325, + 448, + 508, + 558, + 602, + 641, + 675, + 708, + 738, + 777, + 828, + 873, + 915, + 954, + 994, + 1037, + 1077, + 1113, + 1327 + ], + [ + 336, + 463, + 525, + 577, + 623, + 662, + 699, + 732, + 763, + 803, + 856, + 903, + 947, + 987, + 1028, + 1072, + 1113, + 1152, + 1373 + ] + ], + "WA": [ + [ + 430, + 594, + 675, + 743, + 802, + 855, + 903, + 947, + 987, + 1040, + 1108, + 1171, + 1228, + 1281, + 1335, + 1393, + 1447, + 1498, + 1790 + ], + [ + 492, + 680, + 773, + 851, + 919, + 979, + 1033, + 1084, + 1129, + 1190, + 1268, + 1341, + 1406, + 1465, + 1528, + 1594, + 1656, + 1714, + 2049 + ], + [ + 532, + 737, + 838, + 922, + 996, + 1061, + 1120, + 1174, + 1225, + 1291, + 1375, + 1453, + 1523, + 1589, + 1657, + 1728, + 1796, + 1860, + 2222 + ], + [ + 565, + 782, + 889, + 978, + 1056, + 1125, + 1187, + 1245, + 1298, + 1368, + 1458, + 1541, + 1615, + 1685, + 1757, + 1833, + 1903, + 1970, + 2356 + ], + [ + 591, + 818, + 930, + 1024, + 1105, + 1177, + 1243, + 1303, + 1359, + 1432, + 1526, + 1612, + 1690, + 1763, + 1838, + 1918, + 1993, + 2063, + 2466 + ], + [ + 628, + 869, + 988, + 1088, + 1174, + 1251, + 1320, + 1384, + 1443, + 1521, + 1621, + 1712, + 1796, + 1874, + 1953, + 2038, + 2116, + 2192, + 2619 + ] + ], + "WI": [ + [ + 328, + 454, + 516, + 568, + 613, + 653, + 690, + 723, + 755, + 795, + 847, + 895, + 938, + 979, + 1021, + 1064, + 1106, + 1145, + 1368 + ], + [ + 378, + 523, + 594, + 654, + 706, + 752, + 794, + 832, + 867, + 914, + 974, + 1029, + 1079, + 1125, + 1173, + 1224, + 1271, + 1316, + 1573 + ], + [ + 411, + 569, + 647, + 711, + 768, + 818, + 863, + 905, + 943, + 994, + 1059, + 1119, + 1173, + 1224, + 1276, + 1331, + 1382, + 1431, + 1710 + ], + [ + 437, + 604, + 687, + 756, + 816, + 869, + 917, + 962, + 1002, + 1056, + 1125, + 1189, + 1246, + 1300, + 1355, + 1414, + 1469, + 1520, + 1816 + ], + [ + 458, + 634, + 720, + 792, + 855, + 911, + 962, + 1008, + 1051, + 1108, + 1180, + 1246, + 1307, + 1363, + 1421, + 1482, + 1540, + 1593, + 1903 + ], + [ + 488, + 675, + 767, + 844, + 911, + 970, + 1024, + 1073, + 1119, + 1179, + 1256, + 1326, + 1391, + 1450, + 1512, + 1577, + 1638, + 1696, + 2026 + ] + ], + "WV": [ + [ + 375, + 532, + 610, + 677, + 735, + 787, + 835, + 878, + 919, + 972, + 1041, + 1105, + 1163, + 1217, + 1273, + 1332, + 1388, + 1441, + 1747 + ], + [ + 435, + 617, + 709, + 786, + 854, + 915, + 970, + 1021, + 1068, + 1130, + 1211, + 1285, + 1352, + 1416, + 1481, + 1550, + 1615, + 1677, + 2033 + ], + [ + 475, + 675, + 776, + 860, + 935, + 1001, + 1061, + 1117, + 1169, + 1238, + 1325, + 1407, + 1481, + 1550, + 1621, + 1697, + 1768, + 1836, + 2227 + ], + [ + 507, + 720, + 828, + 918, + 997, + 1068, + 1132, + 1192, + 1248, + 1320, + 1415, + 1501, + 1580, + 1654, + 1731, + 1812, + 1888, + 1960, + 2377 + ], + [ + 533, + 758, + 870, + 966, + 1050, + 1124, + 1192, + 1254, + 1313, + 1389, + 1489, + 1580, + 1662, + 1741, + 1822, + 1907, + 1987, + 2063, + 2503 + ], + [ + 570, + 810, + 931, + 1033, + 1123, + 1202, + 1276, + 1343, + 1405, + 1487, + 1592, + 1690, + 1779, + 1863, + 1949, + 2040, + 2127, + 2208, + 2679 + ] + ], + "WY": [ + [ + 247, + 343, + 391, + 431, + 465, + 496, + 524, + 549, + 573, + 604, + 645, + 681, + 715, + 745, + 778, + 812, + 844, + 874, + 1047 + ], + [ + 281, + 391, + 445, + 490, + 529, + 565, + 596, + 626, + 653, + 689, + 734, + 776, + 814, + 850, + 887, + 925, + 962, + 996, + 1193 + ], + [ + 304, + 421, + 480, + 529, + 572, + 610, + 645, + 676, + 706, + 744, + 793, + 839, + 880, + 918, + 959, + 1000, + 1040, + 1077, + 1290 + ], + [ + 321, + 446, + 508, + 560, + 605, + 645, + 681, + 715, + 746, + 787, + 839, + 887, + 930, + 972, + 1014, + 1058, + 1100, + 1138, + 1365 + ], + [ + 335, + 466, + 530, + 585, + 632, + 674, + 712, + 746, + 779, + 822, + 876, + 927, + 973, + 1015, + 1059, + 1106, + 1149, + 1190, + 1426 + ], + [ + 354, + 494, + 563, + 619, + 669, + 714, + 755, + 791, + 826, + 871, + 929, + 982, + 1031, + 1075, + 1122, + 1172, + 1218, + 1261, + 1511 + ] + ] +} diff --git a/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_convention.py b/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_convention.py new file mode 100644 index 00000000..3983ba4e --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_convention.py @@ -0,0 +1,49 @@ +"""Use the latest published IRS optional sales-tax table at the July 3 freeze. + +The IRS directory dates the 2025 Schedule A instructions December 18, 2025 +(the PDF is dated December 8); exact first-publication day is unverified. No 2026 edition was found as of September 22, 2026. +Use all 5,814 state/family-size/income cells from the published 2025 edition +for both 2025 and the held 2026 convention. The companion JSON is extracted +from pages 13-17; absent-state tables are zero under worksheet line 1. + +Sources: +https://www.irs.gov/pub/irs-pdf/i1040sca.pdf +https://www.irs.gov/downloads/irs-pdf?order=uri&page=24&sort=asc + +Only parameter values change. The existing local_sales_tax = .2 * state_sales_tax +proxy remains in place; this audit does not establish any household's local rate. +""" + +import json +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "r19_irs_sales_tax_convention" +DESCRIPTION = "Published 2025 optional state sales-tax tables held for the 2026 freeze" +TABLES = json.loads(Path(__file__).with_name("r19_irs_sales_tax_2025.json").read_text()) + + +def _convention(parameters): + table = parameters.gov.irs.deductions.itemized.salt_and_real_estate.state_sales_tax_table.tax + assert len(TABLES) == 51 + for state, sizes in TABLES.items(): + # CountryTaxBenefitSystem applies the reform both before and after + # homogenization. Four zero-table states are absent from raw YAML; + # homogenization fills them, and the second application sets them. + if state not in table.children: + assert state in {"DE", "MT", "NH", "OR"} + continue + assert len(sizes) == 6 + for size, brackets in enumerate(sizes, 1): + assert len(brackets) == 19 + for bracket, amount in enumerate(brackets, 1): + parameter = table.children[state].children[str(size)].children[str(bracket)] + parameter.update(period="2025", value=amount) + parameter.update(period="2026", value=amount) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-22/fixes/r19_md_convention.py b/reference_audit/2026-09-22/fixes/r19_md_convention.py new file mode 100644 index 00000000..655e7259 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_md_convention.py @@ -0,0 +1,60 @@ +"""Maryland parameter audit at the 2026-07-03 reference freeze. + +The return-deduction/CDCC holds are provisional: no final 2026 publication +was located. See sweep/verify/r19_md_report.md. The withholding correction +is supported by the 2026 Employer Withholding Guide, revised December 2025, +and the dated 2026-04-09 US DOI payroll announcement. + +Sources: +https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/instructions/withholding/2026/withholding-guide.pdf +https://ibc.doi.gov/HRD/Payroll/Announcements/04-09-26 +https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/instructions/2025/resident-booklet.pdf +https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/forms/2025/502cr.pdf +https://mgaleg.maryland.gov/2025RS/Chapters_noln/CH_604_hb0352e.pdf + +Set MD_VERIFIED_ONLY=1 to omit the provisional return/CDCC holds and retain +only the published withholding correction (the source of scenario 078). +No withholding rates, exemptions, or formula behavior are changed. +""" + +import os + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "r19_md_convention" +DESCRIPTION = "Maryland published withholding deduction and provisional return/CDCC holds" +STATUSES = ("SINGLE", "SEPARATE", "JOINT", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE") +DOUBLE = {"JOINT", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE"} +ROOT = "gov.states.md.tax.income." + + +def _modify(parameters): + values = {} + for status in STATUSES: + # This legacy table's only live 2025/26 reader is the withholding + # proxy, which always reads SINGLE. Set the complete table to the + # published withholding amount (the guide has no status split). + values[ROOT + "deductions.standard.max." + status] = {2025: 3350, 2026: 3400} + if os.environ.get("MD_VERIFIED_ONLY") != "1": + # Return deductions: last published amounts. Min is obsolete + # after 2024, so it is deliberately not assigned fictitious limits. + values[ROOT + "deductions.standard.flat_deduction.amount." + status] = { + 2026: 6700 if status in DOUBLE else 3350 + } + values[ROOT + "credits.cdcc.eligibility.agi_cap." + status] = { + 2026: 174300 if status == "JOINT" else 112100 + } + values[ROOT + "credits.cdcc.eligibility.refundable_agi_cap." + status] = { + 2026: 91400 if status == "JOINT" else 60900 + } + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in values: + for year, value in values[parameter.name].items(): + parameter.update(period=str(year), value=value) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-22/fixes/r19_mi_convention.py b/reference_audit/2026-09-22/fixes/r19_mi_convention.py new file mode 100644 index 00000000..9c3a1f46 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_mi_convention.py @@ -0,0 +1,70 @@ +"""Michigan amounts in Treasury's pre-freeze 2026 withholding guide. + +Form 446, revision February 2026: personal exemption $5,900; private +retirement maximum $67,610 single/$135,220 joint (pages 1-2): +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2026/446_Withholding-Guide_2026.pdf +2025 Form 446, revision January 2025, gives $5,800 and $65,897/$131,794: +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2025/446_Withholding-Guide_2025.pdf + +2025 other tables: +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040-Book.pdf +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040CR-7-Book.pdf +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040CR-7.pdf + +The January 2026 home-heating form and its instruction tables supply the last +published amounts. The FY2027 proposed LIHEAP plan says tax-year 2026 forms +will be available in January. For homestead, disability, and senior investment +deductions, a 2026 publication/date was not located: their holds are explicitly +provisional. Set MI_VERIFIED_ONLY=1 to omit the held tables and their 2025 +repairs, retaining only the six verified 2026 personal/retirement leaves. +""" + +import os + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "r19_mi_convention" +DESCRIPTION = "Michigan published 2026 amounts and documented/provisional holds." +ROOT = "gov.states.mi.tax.income." +STATUSES = ("SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE") +PUBLISHED_2026 = {ROOT + "exemptions.personal": 5_900} +HELD_2025 = { + ROOT + "exemptions.disabled.amount.base": 3_400, + ROOT + "credits.homestead_property_tax.cap": 1_900, + ROOT + "credits.homestead_property_tax.household_resources_limit": 71_500, + ROOT + "credits.homestead_property_tax.property_value_limit": 165_400, + ROOT + "credits.homestead_property_tax.reduction.start": 62_500, + ROOT + "credits.home_heating.additional_exemption.amount": 212, + ROOT + "credits.home_heating.alternate.heating_costs.cap": 3_765, +} +for status in STATUSES: + PUBLISHED_2026[ROOT + "deductions.retirement_benefits.tier_one.amount." + status] = ( + 135_220 if status == "JOINT" else 67_610 + ) + HELD_2025[ROOT + "deductions.interest_dividends_capital_gains.amount." + status] = ( + 29_376 if status == "JOINT" else 14_688 + ) +for i, amount in enumerate((604, 815, 1027, 1239, 1451, 1662)): + HELD_2025[ROOT + f"credits.home_heating.standard.base[{i}].amount"] = amount +for i, amount in enumerate((18_592, 25_018, 31_449, 34_227)): + HELD_2025[ROOT + f"credits.home_heating.alternate.household_resources.cap[{i}].amount"] = amount + + +def _convention(parameters): + for parameter in parameters.get_descendants(): + if not isinstance(parameter, Parameter): + continue + if parameter.name in PUBLISHED_2026: + parameter.update(period="2026", value=PUBLISHED_2026[parameter.name]) + if parameter.name in HELD_2025 and os.environ.get("MI_VERIFIED_ONLY") != "1": + # Also correct projected 2025 entries rather than carrying a + # forecast forward as though it were a published 2025 amount. + for year in (2025, 2026): + parameter.update(period=str(year), value=HELD_2025[parameter.name]) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-22/fixes/r19_mn_convention.py b/reference_audit/2026-09-22/fixes/r19_mn_convention.py new file mode 100644 index 00000000..6de0bcac --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_mn_convention.py @@ -0,0 +1,129 @@ +"""Minnesota published-parameter convention, reference freeze 2026-07-03. + +Primary sources: +2026, dated 2025-12-01 (linked by agency release 2025-12-16): +https://www.revenue.state.mn.us/sites/default/files/2025-12/inflation-adjusted-amounts-2026.pdf +2025, dated 2024-12-11: +https://www.revenue.state.mn.us/sites/default/files/2024-12/inflation-adjusted-amounts-2025.pdf + +Every projected Minnesota parameter covered by those publications is explicitly +set for 2026. The 51 projected 2025 renter entries are explicitly set for 2025. +Marriage-credit cap is treated separately; see report and source note below. +Only parameter values change, leaving all formula behavior unchanged. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "r19_mn_convention" +DESCRIPTION = "Minnesota amounts published before the 2026-07-03 reference freeze" +PREFIX = "gov.states.mn.tax.income." + + +def _status(single, joint, head=None, separate=None): + return dict( + SINGLE=single, + JOINT=joint, + HEAD_OF_HOUSEHOLD=single if head is None else head, + SEPARATE=single if separate is None else separate, + SURVIVING_SPOUSE=joint, + ) + + +VALUES_2026 = {} + + +def _statuses(path, values): + VALUES_2026.update({PREFIX + path + "." + key: value for key, value in values.items()}) + + +_statuses("amt.fractional_income_threshold", _status(73100, 97470, separate=48740)) +_statuses("deductions.standard.base", _status(15300, 30600, 23000)) +_statuses("deductions.standard.extra", _status(2000, 1600, separate=1600)) +for deduction in ("standard", "itemized"): + _statuses(f"deductions.{deduction}.reduction.agi_threshold.low", _status(244400, 244400, separate=122200)) + _statuses(f"deductions.{deduction}.reduction.agi_threshold.high", _status(337800, 337800, separate=168900)) +_statuses("exemptions.agi_threshold", _status(244500, 366700, 305600, 183350)) +_statuses("subtractions.pension_income.cap", _status(13850, 27690)) +for subtraction in ("pension_income", "social_security"): + _statuses(f"subtractions.{subtraction}.reduction.start", _status(86410, 110780, separate=55390)) +# The source explicitly labels both alternate-subtraction tables Not Indexed. +_statuses("subtractions.social_security.alternative_amount", _status(4560, 5840, separate=2920)) +_statuses("subtractions.social_security.income_amount", _status(69250, 88630, separate=44315)) +for name, value in { + "exemptions.amount": 5300, + "credits.cdcc.phaseout_threshold": 65610, + "credits.cwfc.ctc.amount": 1800, + "credits.cwfc.phase_out.threshold.joint": 38770, + "credits.cwfc.phase_out.threshold.other": 32680, + "credits.cwfc.wfc.phase_in[1].threshold": 9690, + "credits.cwfc.wfc.additional.amount[1].amount": 1020, + "credits.cwfc.wfc.additional.amount[2].amount": 2330, + "credits.cwfc.wfc.additional.amount[3].amount": 2770, +}.items(): + VALUES_2026[PREFIX + name] = value + +# Earliest located 2026 agency form: near-final draft 2026-08-03, cap $1,894. +# Hold the $1,851 cap printed in the 2025 final draft dated 2025-10-15. +# An earlier 2026 publication was not located; see report's explicit limitation. +# https://www.revenue.state.mn.us/sites/default/files/2026-08/m1ma-26-grid.pdf +# https://www.revenue.state.mn.us/sites/default/files/2025-10/m1ma-25-grid-0.pdf +VALUES_2026[PREFIX + "credits.marriage.maximum_amount"] = 1851 + +RATE_THRESHOLDS_2026 = { + "single": (33310, 109430, 203150), + "joint": (48700, 193480, 337930), + "separate": (24350, 96740, 168965), + "head_of_household": (41010, 164800, 270060), + "surviving_spouse": (48700, 193480, 337930), +} +for status, thresholds in RATE_THRESHOLDS_2026.items(): + for index, value in enumerate(thresholds, 1): + VALUES_2026[f"{PREFIX}rates.{status}[{index}].threshold"] = value + +# Full renter schedule, compressed only where adjacent values are equal in the +# agency table; indices match the existing engine's three single-amount scales. +RENTER_VALUES = { + 2025: { + "claimant_share": (6670, 15530, 22160, 31030, 37690, 46540, 53180, 62060, 68720, 77570), + "percent_of_income": (8860, 11070, 15530, 19960, 24360, 28820, 31030, 33240, 37690, 39890, 77570), + "max_thresholds": (8860, 11070, 15530, 19960, 22160, 24360, 28820, 62060, 64260, 66480, 68720, 70920, 73140, 75350, 77570), + "max_amounts": (2720, 2640, 2580, 2500, 2440, 2380, 2300, 2240, 2040, 1830, 1550, 1360, 1220, 680, 270), + }, + 2026: { + "claimant_share": (6820, 15880, 22670, 31740, 38540, 47590, 54390, 63470, 70280, 79330), + "percent_of_income": (9060, 11320, 15880, 20410, 24920, 29470, 31740, 34000, 38540, 40800, 79330), + "max_thresholds": (9060, 11320, 15880, 20410, 22670, 24920, 29470, 63470, 65720, 68000, 70280, 72530, 74810, 77070, 79330), + "max_amounts": (2780, 2700, 2640, 2560, 2490, 2430, 2360, 2290, 2080, 1870, 1590, 1390, 1250, 690, 270), + }, +} + + +def _renters(year): + result = {} + values = RENTER_VALUES[year] + for scale in ("claimant_share", "percent_of_income"): + for index, value in enumerate(values[scale], 1): + result[f"{PREFIX}credits.renters.{scale}[{index}].threshold"] = value + for index, value in enumerate(values["max_thresholds"], 1): + result[f"{PREFIX}credits.renters.max_credit[{index}].threshold"] = value + for index, value in enumerate(values["max_amounts"]): + result[f"{PREFIX}credits.renters.max_credit[{index}].amount"] = value + return result + + +VALUES_2025 = _renters(2025) +VALUES_2026.update(_renters(2026)) + + +def _convention(parameters): + indexed = {p.name: p for p in parameters.get_descendants() if isinstance(p, Parameter)} + for year, values in ((2025, VALUES_2025), (2026, VALUES_2026)): + for name, value in values.items(): + indexed[name].update(period=str(year), value=value) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-22/fixes/r19_mo_convention.py b/reference_audit/2026-09-22/fixes/r19_mo_convention.py new file mode 100644 index 00000000..a06ab846 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_mo_convention.py @@ -0,0 +1,49 @@ +"""Missouri's published 2026 income-tax brackets, frozen-law audit. + +Source: https://dor.mo.gov/forms/Withholding%20Formula_2026.pdf (annual table). +The DOR forms index dates the 2026 formula 2025-11-21; the same brackets +were republished by USDA NFC on 2026-05-14, before the 2026-07-03 freeze: +https://help.nfc.usda.gov/bulletins/2026/1773783048.htm +2025: https://dor.mo.gov/forms/2025%20Tax%20Chart_2025.pdf (2025-12-23). + +All brackets are shared by every filing status. Zero and infinity are +unchanged but explicitly set, including their projected 2025 entries. +The unrelated projected public-pension cap is documented in the report; +its 2026 publication date remains unverified, so this module does not +claim a 2026 convention value for it. The sourced 2025 cap is repaired to +$47,633 only for 2025. Neither MO household has public pensions. +""" + +from policyengine_core.reforms import Reform +from policyengine_us.system import system as _baseline_system + +FIX_ID = "r19_mo_convention" +DESCRIPTION = "Use Missouri income-tax thresholds published before 2026-07-03." + +# CountryTaxBenefitSystem applies reforms before and after uprating. Preserve +# the frozen 2026 cap explicitly so the 2025-only repair cannot change its forecast. +BASELINE_2026_PENSION_CAP = float( + _baseline_system.parameters("2026-01-01").gov.states.mo.tax.income.deductions + .social_security_and_public_pension.mo_max_social_security_benefit +) + +THRESHOLDS_2025 = (0, 1313, 2626, 3939, 5252, 6565, 7878, 9191, float("inf")) +THRESHOLDS_2026 = (0, 1348, 2696, 4044, 5392, 6740, 8088, 9436, float("inf")) + + +def _convention(parameters): + scale = parameters.gov.states.mo.tax.income.rates + for year, thresholds in ((2025, THRESHOLDS_2025), (2026, THRESHOLDS_2026)): + for bracket, value in zip(scale.brackets, thresholds): + bracket.threshold.update(period=str(year), value=value) + pension = parameters.gov.states.mo.tax.income.deductions.social_security_and_public_pension + pension.mo_max_social_security_benefit.update(period="2025", value=47633) + pension.mo_max_social_security_benefit.update( + period="2026", value=BASELINE_2026_PENSION_CAP + ) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-22/fixes/r19_wi_convention.py b/reference_audit/2026-09-22/fixes/r19_wi_convention.py new file mode 100644 index 00000000..bd666ba0 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r19_wi_convention.py @@ -0,0 +1,65 @@ +"""Wisconsin 2026 indexed parameters published before the reference freeze. + +Primary source: Wisconsin DOR 2026 Form 1-ES instructions, pp. 2-3, +https://www.revenue.wi.gov/TaxForms2026/2026-Form1-ES-inst.pdf +Revision R. 1-26; laws as of January 16, 2026. The official 2026 forms +index records February 6, 2026, 08:08 for the final instructions. + +Replace all 22 projected Wisconsin entries, retaining existing formula and +filing-status mappings. SURVIVING_SPOUSE retains the engine's JOINT alias; +Wisconsin's separate surviving-spouse filing-status issue is outside this +parameter-only audit. No Wisconsin 2025 entry is projected in r18's inventory. +""" + +import os + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + + +FIX_ID = "r19_wi_convention" +DESCRIPTION = "Wisconsin 2026 standard deductions and brackets from published 1-ES" + +ROOT = "gov.states.wi.tax.income." +VALUES = { + ROOT + "deductions.standard.max.SINGLE": 13_960, + ROOT + "deductions.standard.max.JOINT": 25_840, + ROOT + "deductions.standard.max.SEPARATE": 12_280, + ROOT + "deductions.standard.max.HEAD_OF_HOUSEHOLD": 18_030, + ROOT + "deductions.standard.max.SURVIVING_SPOUSE": 25_840, + ROOT + "deductions.standard.phase_out.single[1].threshold": 20_120, + ROOT + "deductions.standard.phase_out.joint[1].threshold": 29_040, + ROOT + "deductions.standard.phase_out.separate[1].threshold": 13_780, + ROOT + "deductions.standard.phase_out.head_of_household[1].threshold": 20_120, + ROOT + "deductions.standard.phase_out.head_of_household[2].threshold": 58_827, +} +for _status, _thresholds in { + "single": (15_110, 51_950, 332_720), + "head_of_household": (15_110, 51_950, 332_720), + "joint": (20_150, 69_260, 443_630), + "separate": (10_080, 34_630, 221_820), +}.items(): + for _index, _amount in enumerate(_thresholds, start=1): + VALUES[ROOT + f"rates.{_status}[{_index}].threshold"] = _amount + + +def _convention(parameters): + # Optional prefix restriction provides causal attribution without copying + # the reform or modifying the triage harness. + prefixes = tuple(filter(None, os.environ.get("WI_PREFIXES", "").split(","))) + seen = set() + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in VALUES: + if prefixes and not parameter.name.startswith(prefixes): + continue + parameter.update(period="2026", value=VALUES[parameter.name]) + seen.add(parameter.name) + expected = {name for name in VALUES if not prefixes or name.startswith(prefixes)} + if seen != expected: + raise ValueError(f"Missing Wisconsin parameters: {sorted(expected - seen)}") + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-22/fixes/r24_disability_benefits_taxable.py b/reference_audit/2026-09-22/fixes/r24_disability_benefits_taxable.py new file mode 100644 index 00000000..7ef2a9c5 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r24_disability_benefits_taxable.py @@ -0,0 +1,44 @@ +"""r24: count disability_benefits (employment disability benefits) in federal gross income. + +policyengine-us 1.755.4 defines disability_benefits +(variables/household/income/person/general/disability_benefits.py) as "Disability +benefits from employment (not Social Security), except for worker's compensation", an +input with no formula, and gov.irs.gross_income.sources does not list it, so the +amount never reaches federal gross income, adjusted gross income, IRC 86 provisional +income, Medicaid MAGI, or any state tax that starts from federal AGI. + +Whether such benefits are taxable depends on who paid for the coverage: benefits from +employer-paid coverage are included in gross income (IRC 105(a)), while benefits from +coverage the employee paid for with after-tax money are excluded (IRC 104(a)(3)). The +prompt lists the benefits without saying who paid for the coverage, so taxability is +an unlisted fact. This module encodes the employer-paid reading by appending +disability_benefits to gov.irs.gross_income.sources for 2026, the same way r16 treats +survivor_benefits; every federal-AGI consumer picks it up. +""" + +from __future__ import annotations + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +FIX_ID = "r24_disability_benefits_taxable" +SOURCE = "disability_benefits" + + +class _DisabilityBenefitsInGrossIncome(Reform): + def apply(self): + def modify(parameters): + node = parameters.gov.irs.gross_income.sources + current = list(node("2026-01-01")) + if SOURCE not in current: + node.update( + start=instant("2026-01-01"), + stop=instant("2026-12-31"), + value=current + [SOURCE], + ) + return parameters + + self.modify_parameters(modify) + + +reform = _DisabilityBenefitsInGrossIncome diff --git a/reference_audit/2026-09-22/fixes/r25_niit_excluded.py b/reference_audit/2026-09-22/fixes/r25_niit_excluded.py new file mode 100644 index 00000000..ecc34687 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r25_niit_excluded.py @@ -0,0 +1,32 @@ +"""r25: read "federal income tax before refundable credits" as excluding the NIIT. + +The prompt defines the output as "federal individual income tax after nonrefundable +credits and before refundable credits". policyengine-us 1.755.4's +income_tax_before_refundable_credits adds net_investment_income_tax (IRC 1411, chapter +2A) to income_tax_before_credits. On Form 1040 the NIIT enters on Schedule 2, Part II +("Other taxes"), after line 22 ("tax after nonrefundable credits"), beside +self-employment tax, which the benchmark scores as a separate output. So a careful +reader can take the definition either way. This module encodes the reading that +excludes the NIIT by setting its 2026 rate to zero. +""" + +from __future__ import annotations + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +FIX_ID = "r25_niit_excluded" + + +class _NiitExcluded(Reform): + def apply(self): + def modify(parameters): + parameters.gov.irs.investment.net_investment_income_tax.rate.update( + start=instant("2026-01-01"), stop=instant("2026-12-31"), value=0 + ) + return parameters + + self.modify_parameters(modify) + + +reform = _NiitExcluded diff --git a/reference_audit/2026-09-22/fixes/r26_snap_contribution_rounding.py b/reference_audit/2026-09-22/fixes/r26_snap_contribution_rounding.py new file mode 100644 index 00000000..eb5a61d2 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r26_snap_contribution_rounding.py @@ -0,0 +1,33 @@ +"""Make the SNAP allotment a whole-dollar amount (engine defect). + +7 CFR 273.10(e)(2)(ii)(A): if 30 percent of net income ends in cents, the State +agency either rounds that 30 percent up to the next whole dollar or leaves it +unrounded and rounds the allotment down to the next lower dollar. With a +whole-dollar maximum allotment both methods give the same allotment. +policyengine-us 1.755.4 does neither: snap_expected_contribution is +floor(snap_net_income) * 0.3, and the allotment keeps the cents. This fix takes +method (1). Split out of r13_hold_fy2026_v2, where the verifier added it to +reproduce USDA's allotment arithmetic. Limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.snap_expected_contribution import snap_expected_contribution as _base_contribution + +FIX_ID = 'r26_snap_contribution_rounding' +DESCRIPTION = 'Round the SNAP expected contribution (30% of net income) up to the next whole dollar, per 7 CFR 273.10(e)(2)(ii)(A).' + + +class snap_expected_contribution(_base_contribution): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_contribution.formula(spm_unit, period, parameters) + rate = parameters(period).gov.usda.snap.expected_contribution + net = np.floor(spm_unit('snap_net_income', period)) + return np.ceil(np.round(net * rate, 2)) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_expected_contribution) diff --git a/reference_audit/2026-09-22/fixes/r27_snap_net_income_rounding.py b/reference_audit/2026-09-22/fixes/r27_snap_net_income_rounding.py new file mode 100644 index 00000000..d085eb3d --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r27_snap_net_income_rounding.py @@ -0,0 +1,33 @@ +"""Round SNAP net income to the nearest dollar in every state (engine defect, fixed upstream). + +7 CFR 273.10(e)(1)(ii): a state agency rounds each income calculation to the +nearest dollar (A), or follows its TANF procedure, which may keep cents (B). +policyengine-us 1.755.4 floors net income in every state, which matches neither. +PolicyEngine/policyengine-us#9318 (merged 2026-08-25, after the reference freeze) +rounds SNAP net income to the nearest dollar in every state; this fix applies the +same rule. California's CalFresh rounds final net income to the nearest dollar +(CDSS summary of MPP 63-503.311); New Jersey rounds income and deductions to whole +dollars; New York and North Carolina keep cents. Recomputing with cents kept in +every state moves no scored reference. Limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.income.snap_net_income import snap_net_income as _base_net + +FIX_ID = 'r27_snap_net_income_rounding' +DESCRIPTION = 'Round SNAP net income to the nearest dollar in every state, as policyengine-us#9318 does.' + + +class snap_net_income(_base_net): + def formula(spm_unit, period): + net = _base_net.formula(spm_unit, period) + if period.start.year != 2026: + return net + return np.floor(net + 0.5) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_net_income) diff --git a/reference_audit/2026-09-22/fixes/r28_snap_min_allotment_rounding.py b/reference_audit/2026-09-22/fixes/r28_snap_min_allotment_rounding.py new file mode 100644 index 00000000..6b3bdadd --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r28_snap_min_allotment_rounding.py @@ -0,0 +1,48 @@ +"""Round the SNAP minimum allotment to the nearest whole dollar (engine defect, fixed upstream). + +7 CFR 273.10(e)(2)(ii)(C): "The minimum benefit is 8 percent of the maximum +allotment for a household of one, rounded to the nearest whole dollar." +policyengine-us 1.755.4 returns 8 percent of the maximum without rounding: $23.84 +a month in FY2026, where USDA published $24. PolicyEngine/policyengine-us#9162 +(merged 2026-07-28, after the reference freeze) applies the same rounding. This +fix follows the CFR's wording; for Hawaii it gives $40 against USDA's published +$41 (no benchmark household is in Hawaii). The state minimum overrides (DC, MD, +NJ) are unchanged. Limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.snap_min_allotment import snap_min_allotment as _base_min + +FIX_ID = 'r28_snap_min_allotment_rounding' +DESCRIPTION = 'Round the SNAP minimum allotment (8% of the one-person maximum) to the nearest whole dollar, per 7 CFR 273.10(e)(2)(ii)(C).' + + +class snap_min_allotment(_base_min): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_min.formula(spm_unit, period, parameters) + snap = parameters(period).gov.usda.snap + region = spm_unit.household('snap_region_str', period) + relevant_max = snap.max_allotment.main[region][ + str(snap.min_allotment.relevant_max_allotment_household_size) + ] + eligible = spm_unit('snap_unit_size', period) <= snap.min_allotment.maximum_household_size + minimum = eligible * np.floor(snap.min_allotment.rate * relevant_max + 0.5) + state = spm_unit.household('state_code_str', period) + dc = parameters(period).gov.states.dc.dhs.snap.min_allotment + if dc.in_effect: + minimum = where(state == 'DC', dc.amount, minimum) + md = parameters(period).gov.states.md.usda.snap.min_allotment + if md.in_effect: + minimum = where((state == 'MD') & spm_unit('md_snap_elderly_present', period), md.amount, minimum) + nj = parameters(period).gov.states.nj.snap + if nj.in_effect: + minimum = where(state == 'NJ', nj.amount, minimum) + return minimum + + +class reform(Reform): + def apply(self): + self.update_variable(snap_min_allotment) diff --git a/reference_audit/2026-09-22/fixes/r29_snap_net_cents.py b/reference_audit/2026-09-22/fixes/r29_snap_net_cents.py new file mode 100644 index 00000000..ef263f06 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r29_snap_net_cents.py @@ -0,0 +1,23 @@ +"""Sensitivity: net income keeps its cents in every state (a TANF procedure that includes cents, 7 CFR 273.10(e)(1)(ii)(B)), with the 30 percent contribution rounded up (r26).""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.snap_expected_contribution import snap_expected_contribution as _base_contribution + +FIX_ID = 'r29_snap_net_cents' +DESCRIPTION = 'Net income keeps cents in every state, contribution rounded up.' + + +class snap_expected_contribution(_base_contribution): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_contribution.formula(spm_unit, period, parameters) + rate = parameters(period).gov.usda.snap.expected_contribution + net = spm_unit('snap_net_income', period) + return np.ceil(np.round(net * rate, 2)) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_expected_contribution) diff --git a/reference_audit/2026-09-22/fixes/r29_snap_net_nearest.py b/reference_audit/2026-09-22/fixes/r29_snap_net_nearest.py new file mode 100644 index 00000000..5881d4d6 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r29_snap_net_nearest.py @@ -0,0 +1,23 @@ +"""Sensitivity: net income rounded to the nearest dollar in every state (7 CFR 273.10(e)(1)(ii)(A), applied to final net income), with the 30 percent contribution rounded up (r26).""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.snap_expected_contribution import snap_expected_contribution as _base_contribution + +FIX_ID = 'r29_snap_net_nearest' +DESCRIPTION = 'Net income to the nearest dollar in every state, contribution rounded up.' + + +class snap_expected_contribution(_base_contribution): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_contribution.formula(spm_unit, period, parameters) + rate = parameters(period).gov.usda.snap.expected_contribution + net = spm_unit('snap_net_income', period) + return np.ceil(np.round(np.floor(net + 0.5) * rate, 2)) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_expected_contribution) diff --git a/reference_audit/2026-09-22/fixes/r30_snap_heat_and_eat_sua.py b/reference_audit/2026-09-22/fixes/r30_snap_heat_and_eat_sua.py new file mode 100644 index 00000000..f409c211 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r30_snap_heat_and_eat_sua.py @@ -0,0 +1,36 @@ +"""Limit the heat-and-eat standard utility allowance to elderly or disabled households (engine defect). + +P.L. 119-21 sec. 10103(a) (approved 2025-07-04) amended 7 U.S.C. +2014(e)(6)(C)(iv)(I): a LIHEAP payment makes a household eligible for the +standard utility allowance only if the household has an elderly or disabled +member (the text inserts "with an elderly or disabled member" after +"households"). policyengine-us 1.755.4 still grants the allowance to every household in a state +whose always_standard flag models heat-and-eat (Pennsylvania's has been true since +2015-10-01). This fix keeps the flag only for households with a USDA elderly or +disabled member; households with heating or cooling expenses keep the allowance +through the engine's own expense path. Limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.income.deductions.shelter.snap_state_using_standard_utility_allowance import ( + snap_state_using_standard_utility_allowance as _base, +) + +FIX_ID = 'r30_snap_heat_and_eat_sua' +DESCRIPTION = 'Grant the heat-and-eat standard utility allowance only to households with an elderly or disabled member, per P.L. 119-21 sec. 10103.' + + +class snap_state_using_standard_utility_allowance(_base): + def formula(spm_unit, period, parameters): + flag = _base.formula(spm_unit, period, parameters) + if period.start.year != 2026: + return flag + elderly_disabled = spm_unit('has_usda_elderly_disabled', period.this_year) + return np.asarray(flag).astype(bool) & np.asarray(elderly_disabled).astype(bool) + + +class reform(Reform): + def apply(self): + self.update_variable(snap_state_using_standard_utility_allowance) diff --git a/reference_audit/2026-09-22/fixes/r31_snap_income_limit_rounding.py b/reference_audit/2026-09-22/fixes/r31_snap_income_limit_rounding.py new file mode 100644 index 00000000..8caf4e02 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r31_snap_income_limit_rounding.py @@ -0,0 +1,42 @@ +"""Round the SNAP gross and net income standards up to whole dollars (engine defect). + +7 CFR 273.9(a)(3): the monthly income standards are the poverty guideline times +130 percent (gross) or 100 percent (net), divided by 12 and rounded up to the next +whole dollar. policyengine-us 1.755.4 compares income to the unrounded standard +through an income-to-poverty ratio. Fixed upstream in PolicyEngine/policyengine-us +#9162 (merged 2026-07-28), after the reference freeze. Limited to calendar 2026. +""" +import numpy as np + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * +from policyengine_us.variables.gov.usda.snap.eligibility.meets_snap_gross_income_test import meets_snap_gross_income_test as _base_gross +from policyengine_us.variables.gov.usda.snap.eligibility.meets_snap_net_income_test import meets_snap_net_income_test as _base_net + +FIX_ID = 'r31_snap_income_limit_rounding' +DESCRIPTION = 'Round the SNAP gross and net income standards up to the next whole dollar, per 7 CFR 273.9(a)(3).' + + +class meets_snap_gross_income_test(_base_gross): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_gross.formula(spm_unit, period, parameters) + p = parameters(period).gov.usda.snap.income.limit + limit = np.ceil(np.round(p.gross * spm_unit('snap_fpg', period), 4)) + income = spm_unit('snap_gross_income', period) + return spm_unit('has_usda_elderly_disabled', period) | (income <= limit) + + +class meets_snap_net_income_test(_base_net): + def formula(spm_unit, period, parameters): + if period.start.year != 2026: + return _base_net.formula(spm_unit, period, parameters) + p = parameters(period).gov.usda.snap.income.limit + limit = np.ceil(np.round(p.net * spm_unit('snap_fpg', period), 4)) + return spm_unit('snap_net_income', period) <= limit + + +class reform(Reform): + def apply(self): + self.update_variable(meets_snap_gross_income_test) + self.update_variable(meets_snap_net_income_test) diff --git a/reference_audit/2026-09-22/fixes/r32_wi_capital_gain_distributions.py b/reference_audit/2026-09-22/fixes/r32_wi_capital_gain_distributions.py new file mode 100644 index 00000000..6332b868 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r32_wi_capital_gain_distributions.py @@ -0,0 +1,58 @@ +"""r32: Wisconsin's 30% capital-gain exclusion omits capital gain distributions reported without Schedule D. + +Split out of r04_capital_gain_distributions_v2 on 2026-09-23. policyengine-us PR +#8839 (r04) puts non_sch_d_capital_gains (Form 1040 line 7a distributions) into +federal AGI, which Wisconsin starts from, but leaves Wisconsin's capital gain +subtraction reading long_term_capital_gains only. Upstream main still does, so this +defect is not fixed upstream and is measured on top of the Wisconsin publication +convention plus the #8839 backport (cwi_plus_r04), the value that would otherwise +be published. + +Law: Wis. Stat. 71.05(6)(b)9 subtracts 30% of the net capital gain on assets held +more than one year. A capital gain distribution is long-term capital gain (IRC +852(b)(3)(B)), and the 2025 Schedule SB instructions, line 5, apply the 30% +exclusion to capital gain distributions reported directly on the federal return. + +Change: in tax year 2026, the Schedule WD long-term gain the formula reads adds +non_sch_d_capital_gains. The engine's formula is otherwise unchanged; without +distributions it computes exactly the 1.755.4 value. Periods: 2026 only. + +Sources: +https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/9 +https://www.law.cornell.edu/uscode/text/26/852#b_3_B +https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=2 +""" + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r32_wi_capital_gain_distributions" +DESCRIPTION = "2026 Wisconsin capital gain subtraction includes capital gain distributions reported without Schedule D." +SOURCE = "non_sch_d_capital_gains" + + +class wi_capital_gain_loss_subtraction(Variable): + value_type = float + entity = TaxUnit + label = "Wisconsin capital gain/loss subtraction from federal AGI" + unit = USD + definition_period = YEAR + defined_for = StateCode.WI + reference = "https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=2" + + def formula(tax_unit, period, parameters): + stcg_net = add(tax_unit, period, ["short_term_capital_gains"]) + sources = ["long_term_capital_gains"] + if period.start.year == 2026: + sources.append(SOURCE) + ltcg_net = add(tax_unit, period, sources) + totcg = max_(0, stcg_net + ltcg_net) + fraction = parameters(period).gov.states.wi.tax.income.subtractions.capital_gain.fraction + cg_reduction = min_(totcg, max_(0, ltcg_net)) * fraction + wi_cg = totcg - cg_reduction + return max_(0, totcg - wi_cg) + + +class reform(Reform): + def apply(self): + self.update_variable(wi_capital_gain_loss_subtraction) diff --git a/reference_audit/2026-09-22/fixes/r33_snap_child_support_treatment.py b/reference_audit/2026-09-22/fixes/r33_snap_child_support_treatment.py new file mode 100644 index 00000000..90b15b69 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/r33_snap_child_support_treatment.py @@ -0,0 +1,117 @@ +"""Treat child support paid as USDA's State Options Reports record it (engine defect, fixed upstream). + +7 CFR 273.9(c)(17) lets a state exclude legally obligated child support paid to +nonhousehold members from gross income; a state that does not take the option +deducts the payments when computing net income under 7 CFR 273.9(d)(5). +policyengine-us 1.755.4 reads gov.usda.snap.income.deductions.child_support as +"exclude from gross income" (snap_child_support_gross_income_deduction and +snap_gross_test_income subtract the payments before the gross income test when +it is true), but its values carry the opposite meaning: they were entered in +2022 as "can be deducted from net income" and relabeled in 2023 without being +flipped. From 2021-10-01 the engine therefore excludes child support from gross +income in the deduction states, Michigan among them, and deducts it in the +exclusion states. USDA's SNAP State Options Report lists Michigan as a +deduction state (16th edition, "Treatment of Child Support Payments", p. 15; +17th edition, p. 21), and Michigan's Bridges Eligibility Manual 556 subtracts +child support at line 20, after gross income (line 10). + +The fix is merged upstream: PolicyEngine/policyengine-us#9586, squash-merged on +2026-09-24 as d9e801df417352b8246a4c292a19ec082a518790. VALUES below are that +commit's policyengine_us/parameters/gov/usda/snap/income/deductions/child_support.yaml, +every jurisdiction and date (true = excluded from gross income), copied +verbatim. The merge changes no formula's logic: its three variable files change +only documentation, references and comments. The module replaces each +jurisdiction's history from 2010-01-01 with them, as the merged file does +(every jurisdiction's first date is 2010-01-01). + +2026-09-24: this module first embedded the open pull request's head 3f15666 +(release dashboard-data-20260922b, which excluded the one output r33 moves). +The merged values differ from that head for 2026 only in Louisiana and North +Carolina, which now exclude child support from gross income; recomputed under +the merged values, the output r33 moves is still only scenario_045 SNAP. +""" + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +FIX_ID = "r33_snap_child_support_treatment" +DESCRIPTION = ( + "Set the SNAP child support exclusion flag to USDA's State Options Report " + "values, as PolicyEngine/policyengine-us#9586 (merged as d9e801d) does." +) +UPSTREAM_PR = "PolicyEngine/policyengine-us#9586" +UPSTREAM_MERGE_COMMIT = "d9e801df417352b8246a4c292a19ec082a518790" +PARAMETER = "gov.usda.snap.income.deductions.child_support" + +# policyengine-us at d9e801d (#9586 merged): jurisdiction -> ((effective date, excluded), ...) +VALUES = { + "AK": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "AL": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "AR": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "AZ": (("2010-01-01", True), ("2013-09-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "CA": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "CO": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "CT": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "DC": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "DE": (("2010-01-01", False), ("2010-06-10", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "FL": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "GA": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "GU": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "HI": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "IA": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "ID": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "IL": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "IN": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "KS": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "KY": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "LA": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "MA": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "MD": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "ME": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "MI": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "MN": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "MO": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "MS": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "MT": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NC": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "ND": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NE": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NH": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NJ": (("2010-01-01", False), ("2013-11-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "NM": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NV": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "NY": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "OH": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "OK": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "OR": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-01-19", True), ("2023-10-01", True), ("2024-10-01", True),), + "PA": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "RI": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "SC": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "SD": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "TN": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "TX": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "UT": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "VA": (("2010-01-01", False), ("2017-10-01", False), ("2018-10-31", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "VI": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "VT": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "WA": (("2010-01-01", True), ("2017-10-01", True), ("2022-10-01", True), ("2023-10-01", True), ("2024-10-01", True),), + "WI": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "WV": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), + "WY": (("2010-01-01", False), ("2017-10-01", False), ("2022-10-01", False), ("2023-10-01", False), ("2024-10-01", False),), +} + + +def _apply(parameters): + node = parameters.gov.usda.snap.income.deductions.child_support + for state, history in VALUES.items(): + child = node.children[state] + # Each update without a stop replaces the values from its start on, + # so applying the dates in order reproduces the upstream history. + for start, value in history: + child.update(start=instant(start), value=value) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_apply) diff --git a/reference_audit/2026-09-22/fixes/u_ssdi_months.py b/reference_audit/2026-09-22/fixes/u_ssdi_months.py new file mode 100644 index 00000000..723fb914 --- /dev/null +++ b/reference_audit/2026-09-22/fixes/u_ssdi_months.py @@ -0,0 +1,25 @@ +"""Alternative reading behind the 2026-09-05 Medicare exclusions. + +The prompt lists Social Security disability income but never how many months it +has been received (engine input months_receiving_social_security_disability, +integer-typed, never carried by the data). The alternative reading gives every +person with listed SSDI income 24 months of receipt, the Medicare threshold. +Situation patch only; applies to tax year 2026. +""" + +import copy + +FIX_ID = "u_ssdi_months" +DESCRIPTION = "Read listed SSDI income as 24 or more months of receipt." +YEAR = "2026" + + +def patch(situation, scenario): + situation = copy.deepcopy(situation) + for person in situation["people"].values(): + if person.get("social_security_disability", {}).get(YEAR, 0) > 0: + person["months_receiving_social_security_disability"] = {YEAR: 24} + return situation + + +reform = None diff --git a/reference_audit/2026-09-22/fixes/u_ssi_disability_criteria.py b/reference_audit/2026-09-22/fixes/u_ssi_disability_criteria.py new file mode 100644 index 00000000..0e275d4f --- /dev/null +++ b/reference_audit/2026-09-22/fixes/u_ssi_disability_criteria.py @@ -0,0 +1,24 @@ +"""Alternative reading behind the 2026-09-05 SSI-disability exclusions. + +The prompt lists a general "is disabled" fact and never SSI's disability +criterion (engine input meets_ssi_disability_criteria, which the certified +microdata never sets). The alternative reading sets that input for every person +the prompt lists as disabled. Situation patch only; applies to tax year 2026. +""" + +import copy + +FIX_ID = "u_ssi_disability_criteria" +DESCRIPTION = "Read 'is disabled' as meeting SSI's disability criterion." +YEAR = "2026" + + +def patch(situation, scenario): + situation = copy.deepcopy(situation) + for person in situation["people"].values(): + if person.get("is_disabled", {}).get(YEAR): + person["meets_ssi_disability_criteria"] = {YEAR: True} + return situation + + +reform = None diff --git a/reference_audit/2026-09-22/root_causes.json b/reference_audit/2026-09-22/root_causes.json new file mode 100644 index 00000000..df2539be --- /dev/null +++ b/reference_audit/2026-09-22/root_causes.json @@ -0,0 +1,346 @@ +{ + "_about": "One entry per root cause from the 2026-09-22 reference triage. build_records.py turns these, the verified sweep outputs in sweep/out/, and the case verdicts in verdicts.json into reference_exclusions.json entries and us_adjudications.json entries. The text fields appear on policybench.org beside each excluded output. 'upstream' is filled in once the issue is filed.", + "r01_ira_compensation": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine deducts traditional IRA contributions above the compensation limit and counts a dependent's contributions on the filers' return", + "law": "26 U.S.C. 219(a), 219(b)(1)(B), 219(c), 219(f)(1); IRS Notice 2025-67", + "alternative_reading": "A filer deducts traditional IRA contributions only up to their own compensation (wages plus net self-employment earnings), or the couple's combined compensation on a joint return, and a dependent's contributions never reach the parents' return.", + "upstream": "" + }, + "r02_ira_219g": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine deducts traditional IRA contributions without the active-participant phase-out", + "law": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)", + "alternative_reading": "A filer who defers wages into a 401(k) or 403(b) is an active participant, and above the 2026 phase-out range ($81,000 to $91,000 single or head of household, $129,000 to $149,000 joint) deducts nothing.", + "upstream": "" + }, + "r03_estate_income": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine leaves estate income out of gross income while counting it as qualified business income", + "law": "26 U.S.C. 61(a)(14), 662(a), 199A(c)(3)(A)(ii)", + "alternative_reading": "Income from an interest in an estate is gross income to the beneficiary; the corrected value adds it to AGI and treats it as non-qualified business income, as the prompt's rule that unlisted flags are false implies.", + "upstream": "related: PolicyEngine/policyengine-us#9304" + }, + "r04_capital_gain_distributions": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine leaves capital gain distributions reported without Schedule D out of gross income", + "law": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7", + "alternative_reading": "Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.", + "upstream": "fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze", + "upstream_fixed": true, + "fix_note": "Regenerated with r04_capital_gain_distributions.py, the faithful backport of #8839 (federal gross income, net investment income and the preferential-rate base). The verified r04_capital_gain_distributions_v2.py also extended Wisconsin's capital gain exclusion to the distributions; #8839 does not, so that part is r32." + }, + "r05_nj_worker_ui": { + "class": "engine_defect", + "subtype": "payroll_tax_base", + "defect": "PolicyEngine leaves the New Jersey worker unemployment and workforce contributions out of employee payroll tax", + "law": "N.J.S.A. 43:21-7(d)(1); NJDOL 2026 contribution rates", + "alternative_reading": "New Jersey workers contribute 0.3825% of wages to unemployment insurance and 0.0425% to the workforce funds, up to the 2026 wage base of $44,800, alongside the TDI and FLI contributions.", + "upstream": "" + }, + "r06_wi_act15_before_refundable": { + "class": "engine_defect", + "subtype": "state_local_rule", + "defect": "PolicyEngine's Wisconsin tax before refundable credits ignores the retirement income exclusion the filer elects, which its final Wisconsin tax applies", + "law": "Wis. Stat. 71.05(6)(b)54m (2025 Wisconsin Act 15); 2025 Schedule SB line 16", + "alternative_reading": "A filer aged 67 or older who elects the retirement income exclusion computes Wisconsin tax on the reduced income and forfeits the credits, so the tax before refundable credits follows the elected path.", + "upstream": "" + }, + "r07_idaho_health_premiums": { + "class": "engine_defect", + "subtype": "state_local_rule", + "defect": "PolicyEngine omits Idaho's subtraction for health insurance premiums the taxpayer pays", + "law": "Idaho Code 63-3022P; Idaho Form 39R", + "alternative_reading": "Idaho subtracts health insurance premiums the taxpayer pays that are not otherwise deducted or excluded.", + "upstream": "" + }, + "r08_eitc_earned_income_deferrals": { + "class": "engine_defect", + "subtype": "credit_phaseout", + "defect": "PolicyEngine counts elective 401(k) deferrals as earned income for the EITC and refundable child tax credit", + "law": "26 U.S.C. 32(c)(2)(A)(i), 24(d)(1)(B)(i), 402(e)(3); Cal. R&TC 17052(c)(4)(A)", + "alternative_reading": "EITC earned income counts wages only if they are includible in gross income, which excludes elective deferrals (W-2 box 1 wages); the refundable child tax credit and state credits built on the EITC follow.", + "upstream": "", + "not_confirmed": { + "scenario_023:state_refundable_credits": "The verifier found the CalEITC increase an artifact of the missing AGI lookup (r17), not a confirmed correction." + } + }, + "r09_ny_rptc_rent_cap": { + "class": "engine_defect", + "subtype": "state_local_rule", + "defect": "PolicyEngine applies New York's $450-a-month renter cap to a quarter of rent instead of the rent itself", + "law": "N.Y. Tax Law 606(e)(7)(D); Form IT-214", + "alternative_reading": "A renter whose adjusted rent averages more than $450 a month gets no real property tax credit.", + "upstream": "fixed in PolicyEngine/policyengine-us#9301 (issue #9298) and PolicyEngine/policyengine-us#9313 (the 2025 flat credit tables the sandbox module also applies), after the reference freeze", + "upstream_fixed": true + }, + "r10_wi_homestead_income": { + "class": "unlisted_input", + "subtype": "state_local_rule", + "unlisted_input": "source of survivor_benefits (pension or annuity, or another survivor payment)", + "alternative_reading": "The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out.", + "upstream": "", + "verification": "Independent verifier (2026-09-22) rejected the engine-defect framing: the listed input does not establish a countable subtype." + }, + "r11_ca_itemized_conformity": { + "class": "engine_defect", + "subtype": "state_local_rule", + "defect": "PolicyEngine applies the federal charitable deduction floor and the suspension of miscellaneous deductions to California itemized deductions", + "law": "Cal. R&TC 17024.5 (conformity date), 17076", + "alternative_reading": "California itemizers deduct charitable gifts without the federal 0.5%-of-AGI floor and deduct miscellaneous expenses above 2% of AGI.", + "upstream": "" + }, + "r12_ca_2026_indexing": { + "class": "superseded", + "note": "Withdrawn 2026-09-22. The independent verifier rejected the module (wrong CalEITC anchors, unverified cutoff), and California's 2026 factor (June 2026 CCPI, published by DIR on 2026-08-12) came out after the reference freeze. Under the benchmark's rule the 2026 California amounts hold the 2025 amounts: convention c_ca_hold_2025." + }, + "r14_unlisted_weekly_hours_v2": { + "class": "unlisted_input", + "subtype": "categorical_eligibility", + "unlisted_input": "weekly_hours_worked_before_lsr", + "alternative_reading": "The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.", + "upstream": "PolicyEngine/policyengine-us#9254 (default changed to 0 after the reference freeze)" + }, + "r15_snap_mortgage_interest": { + "class": "unlisted_input", + "subtype": "taxable_income_or_deductions", + "unlisted_input": "whether the listed home mortgage interest is on the home the SNAP household occupies", + "alternative_reading": "The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.", + "upstream": "", + "verification": "Independent verifier (2026-09-22) reclassified this from an engine defect to an unlisted input." + }, + "r16_survivor_benefits_federal": { + "class": "unlisted_input", + "subtype": "taxable_income_or_deductions", + "unlisted_input": "taxability of survivor_benefits", + "alternative_reading": "The prompt lists survivor benefits other than Social Security without saying whether they are taxable; the reference treats them as nontaxable, while a survivor pension or annuity is taxable under 26 U.S.C. 72.", + "upstream": "" + }, + "r13_snap_fy2027_parameters": { + "class": "regenerated", + "note": "Not excluded. Max (2026-09-22): grading any model on figures published after the freeze is unfair, and excluding them would leave no positive-benefit SNAP case scored. The SNAP references are regenerated with October to December 2026 held at the FY2026 figures, the last published before the 2026-07-03 freeze, with the statutory rounding of the minimum allotment and allotment (fix r13_hold_fy2026). No model's explanation used FY2027 figures. Verified fix: r13_hold_fy2026_v2 (also holds the SNAP poverty guidelines and the published FY2026 minimum allotment; bounded to calendar 2026). Superseded 2026-09-22: the SNAP convention is now r13_hold_fy2026_v3 (the FY2026 uprating hold alone), and the engine's SNAP rounding defects are root causes r26, r27, r28 and r31, which are fixed upstream and regenerated with the fixes." + }, + "c_snap_hold_fy2026": { + "class": "convention", + "fix": "r13_hold_fy2026_v3", + "outputs": "snap", + "rule": "SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.", + "basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14)", + "grounding": "The reference is the ANNUAL 2026 SNAP total: the sum of twelve monthly allotments, January through December 2026. Every month uses the FY2026 SNAP figures (maximum allotment, standard deduction, excess shelter cap, utility allowances); October-December keep them because they were the last USDA published before the benchmark's 2026-07-03 reference freeze. The allotment is a whole-dollar amount: net income is rounded to the nearest dollar, 30% of it is rounded up to the next dollar, and the minimum benefit is 8% of the one-person maximum rounded to the nearest dollar ($24 a month). State the monthly allotment and the annual total separately and never call the annual total a monthly or quarterly amount." + }, + "c_ca_hold_2025": { + "class": "convention", + "fix": "r19_ca_convention", + "outputs": "California income tax and credits", + "rule": "California's 2026 indexed income tax amounts hold the 2025 amounts FTB published (October 2025 Tax News; 2025 Form 540 and 3514 booklets), the last published before the 2026-07-03 reference freeze; the 2026 factor rests on June 2026 California CPI (BLS 2026-07-14; DIR 2026-08-12). The CalEITC final phase-out breakpoints take their statutory 2025 values ($252/$636), which policyengine-us 1.755.4 had projected ($257/$649).", + "basis": "Cal. R&TC 17041(h), 17052(o); FTB Tax News, October 2025; FTB 2026 Form 540-ES instructions", + "published": "FTB Tax News October 2025; 2026 Form 540-ES instructions use the 2025 amounts", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5a.md) sourced every value.", + "grounding": "California's 2026 indexed income tax amounts (bracket thresholds, exemption credits, renter's credit income limits, credit parameters) equal the 2025 amounts California published, the last published before the benchmark's 2026-07-03 reference freeze; the 2026 indexing factor rests on June 2026 California CPI, published after it. Where the reference is a federal amount, California income tax enters it through the state and local tax deduction." + }, + "r17_caleitc_agi_comparison": { + "class": "engine_defect", + "subtype": "credit_phaseout", + "defect": "PolicyEngine phases out the CalEITC on earned income only, without the second lookup at adjusted gross income", + "law": "Cal. R&TC 17052(a), incorporating IRC 32(a)(2)(B); FTB 3514 instructions, CalEITC worksheet", + "alternative_reading": "When adjusted gross income exceeds the income at which the CalEITC starts to phase out, the credit is looked up again at AGI and the smaller amount applies.", + "upstream": "fixed in PolicyEngine/policyengine-us#9363 (merged 2026-09-01), after the reference freeze", + "upstream_fixed": true + }, + "c_irs_sales_tax_2025": { + "class": "convention", + "fix": "r19_irs_sales_tax_convention", + "outputs": "federal income tax of filers who deduct general sales tax", + "rule": "The optional state sales tax tables for 2026 are the 2025 tables the IRS published in the 2025 Instructions for Schedule A (posted 2025-12-18), the last published before the 2026-07-03 reference freeze; no 2026 edition had been published by 2026-09-22. policyengine-us 1.755.4 projected 2026 tables from its 2023 edition with chained CPI, which is not the IRS method.", + "basis": "IRS, 2025 Instructions for Schedule A (Form 1040), Optional State Sales Tax Tables", + "published": "2025-12-18", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5a.md) extracted all 5,814 cells from the IRS PDF.", + "grounding": "The general sales tax deduction uses the IRS optional state sales tax table from the 2025 Instructions for Schedule A (the last edition published before the benchmark's 2026-07-03 reference freeze), plus PolicyEngine's local sales tax component of 20% of the state table amount; the filer takes the larger of the standard deduction and itemized deductions." + }, + "c_wi_published_2026": { + "class": "convention", + "fix": "r19_wi_convention", + "outputs": "Wisconsin income tax", + "rule": "Wisconsin's 2026 standard deduction and tax brackets are the amounts DOR published before the 2026-07-03 reference freeze (2026 Form 1-ES instructions, rev. 1-26; 2026 WT-4A worksheet, rev. 11-25), not policyengine-us 1.755.4's CPI projections.", + "basis": "Wisconsin DOR, 2026 Form 1-ES instructions (R. 1-26), pp. 2-3; 2026 WT-4A worksheet (R. 11-25), p. 2", + "published": "2026 Form 1-ES instructions (R. 1-26); 2026 WT-4A (R. 11-25)", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5a.md).", + "grounding": "Wisconsin's 2026 standard deduction and tax brackets are the amounts the Wisconsin DOR published for 2026 before the benchmark's 2026-07-03 reference freeze." + }, + "c_id_hold_2025": { + "class": "convention", + "fix": "r19_id_convention", + "outputs": "Idaho income tax", + "rule": "Idaho's 2026 zero-rate taxable income thresholds hold the 2025 amounts ($4,811 single; $9,622 joint and head of household) the Tax Commission published, the last located before the 2026-07-03 reference freeze; no 2026 publication was found by 2026-09-22. The retirement-benefit caps take their statutory 2026 values; they do not affect any benchmark household.", + "basis": "Idaho Code 63-3024(3), 63-3022A", + "published": "2025 Form 40 instructions (EIN00046, rev. 2026-03-02)", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5a.md); the absence of a 2026 publication is provisional.", + "grounding": "Idaho's 2026 taxable income threshold for the zero rate equals the 2025 amount the Idaho State Tax Commission published, the last located before the benchmark's 2026-07-03 reference freeze." + }, + "c_mn_published_2026": { + "class": "convention", + "fix": "r19_mn_convention", + "outputs": "Minnesota income tax and credits", + "rule": "Minnesota's 2026 inflation-adjusted amounts are those the Department of Revenue published on 2025-12-01 (announced 2025-12-16), before the 2026-07-03 reference freeze, not policyengine-us 1.755.4's CPI projections; the 2025 renter's credit schedule takes its published values. The marriage credit cap, first located in an August 2026 draft, holds its 2025 amount.", + "basis": "Minnesota Department of Revenue, 2026 inflation-adjusted amounts (2025-12-01), pp. 1-3, 5", + "published": "2025-12-01", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5b.md)", + "grounding": "Minnesota's 2026 standard deduction, brackets, exemptions and credit amounts are the inflation-adjusted amounts the Minnesota Department of Revenue published in December 2025, before the benchmark's 2026-07-03 reference freeze." + }, + "c_md_2026": { + "class": "convention", + "fix": "r19_md_convention", + "outputs": "Maryland income tax; federal tax through the Maryland withholding proxy", + "rule": "Maryland's 2026 withholding allowance is the $3,400 the Comptroller published in the 2026 Employer Withholding Guide (revised December 2025); the 2026 flat standard deduction holds the published 2025 amounts ($3,350; $6,700), because no 2026 return amount was located before the 2026-07-03 reference freeze (provisional); the child and dependent care credit caps hold their 2025 amounts.", + "basis": "Md. Tax-Gen. 10-217 (Chapter 604 of 2025); Comptroller Tax Alert rev. 2025-12-22; 2026 Employer Withholding Guide", + "published": "2025-12 (withholding guide)", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5b.md); the 2026 standard-deduction hold is provisional.", + "grounding": "Maryland's 2026 standard deduction equals the flat 2025 amount ($3,350 single, $6,700 joint or head of household), the last located before the benchmark's 2026-07-03 reference freeze; Maryland withholding uses the published 2026 allowance of $3,400, and Maryland income tax enters federal amounts through the state and local tax deduction." + }, + "c_mi_published_2026": { + "class": "convention", + "fix": "r19_mi_convention", + "outputs": "Michigan income tax and credits", + "rule": "Michigan's 2026 personal exemption ($5,900) and retirement limits are those Treasury published in the 2026 Form 446 (revised February 2026), before the 2026-07-03 reference freeze; amounts with no located 2026 publication (disability exemption, senior investment limits, homestead and home heating tables) hold their published 2025 values, which also replace the engine's projected 2025 entries.", + "basis": "Michigan Treasury Form 446 (2026, rev. February 2026); 2025 MI-1040 and MI-1040CR-7 instructions", + "published": "2026-02", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5b.md)", + "grounding": "Michigan's 2026 personal exemption is $5,900 per exemption, the amount Michigan Treasury published in February 2026, before the benchmark's 2026-07-03 reference freeze." + }, + "c_mo_published_2026": { + "class": "convention", + "fix": "r19_mo_convention", + "outputs": "Missouri income tax", + "rule": "Missouri's 2026 income tax brackets are those the Department of Revenue published in its 2026 withholding formula (dated 2025-11-21), before the 2026-07-03 reference freeze, not policyengine-us 1.755.4's CPI projections.", + "basis": "Missouri DOR 2026 Withholding Formula (2025-11-21), annual table p. 2", + "published": "2025-11-21", + "verification": "Independent verifier (GPT-6 Astra, 2026-09-22, sweep/verify/report-v5b.md)", + "grounding": "Missouri's 2026 income tax brackets are those the Missouri Department of Revenue published in November 2025, before the benchmark's 2026-07-03 reference freeze." + }, + "r20_adult_dependent_child": { + "class": "unlisted_input", + "subtype": "household_unit_or_filing_status", + "unlisted_input": "the relationship of an adult tax dependent to the filers who claim them", + "alternative_reading": "The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)).", + "upstream": "", + "verification": "Investigated by GPT-6 Astra (2026-09-22, sweep/verify/report-v6-067.md); the scenario encodes no parent-child link." + }, + "r21_adult_dependent_nonchild": { + "class": "unlisted_input", + "subtype": "household_unit_or_filing_status", + "unlisted_input": "the relationship of an adult tax dependent to the filers who claim them", + "alternative_reading": "The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household.", + "upstream": "", + "verification": "Investigated by GPT-6 Astra (2026-09-22, sweep/verify/report-v6-067.md)." + }, + "r22_ma_part_a_loss_offset": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine taxes Massachusetts Part A dividends gross, without the short-term capital loss offset its own Part A AGI computes", + "law": "M.G.L. c. 62 sec. 2(b), 2(c)(2)(a), 2(c)(4), 2(f); DOR TIR 02-21", + "alternative_reading": "Massachusetts applies excess short-term capital losses against Part A interest and dividends, up to $2,000 of combined losses, before taxing Part A income.", + "upstream": "", + "verification": "Investigated by GPT-6 Astra (2026-09-22, sweep/verify/report-v6-081.md); release 2.8.0 and upstream main unchanged." + }, + "r23_ma_interest_source": { + "class": "unlisted_input", + "subtype": "taxable_income_or_deductions", + "unlisted_input": "whether listed taxable interest comes from a Massachusetts bank deposit (Part B) or is ordinary interest (Part A)", + "alternative_reading": "The prompt does not say the interest is from a Massachusetts bank; as ordinary interest it falls in Part A (M.G.L. c. 62 sec. 2(b)(1)), where the short-term capital loss also offsets it.", + "upstream": "", + "verification": "Investigated by GPT-6 Astra (2026-09-22, sweep/verify/report-v6-081.md)." + }, + "r24_disability_benefits_taxability": { + "class": "unlisted_input", + "subtype": "taxable_income_or_deductions", + "unlisted_input": "who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable", + "alternative_reading": "The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them.", + "upstream": "", + "verification": "Raised as prompt_ambiguity rows by the Opus 5.5 judge on scenario_107 federal (2026-09-22); swept with r24_disability_benefits_taxable over all 1,984 outputs." + }, + "r25_niit_in_federal_output": { + "class": "unlisted_input", + "subtype": "other", + "unlisted_input": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say", + "alternative_reading": "The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.", + "upstream": "", + "verification": "Raised as prompt_ambiguity rows by the Opus 5.5 judge on scenario_020 federal (2026-09-22); swept with r25_niit_excluded over all 1,984 outputs." + }, + "r26_snap_contribution_rounding": { + "class": "engine_defect", + "subtype": "thresholds_rates", + "defect": "PolicyEngine subtracts 30% of net income from the maximum allotment without making the allotment a whole-dollar amount", + "law": "7 CFR 273.10(e)(2)(ii)(A)", + "alternative_reading": "If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.", + "upstream": "fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze", + "measured_against": "c_snap_hold_fy2026", + "upstream_fixed": true + }, + "r28_snap_min_allotment_rounding": { + "class": "engine_defect", + "subtype": "thresholds_rates", + "defect": "PolicyEngine sets the SNAP minimum allotment at 8% of the one-person maximum without rounding ($23.84 a month for FY2026)", + "law": "7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)", + "alternative_reading": "The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).", + "upstream": "fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze", + "measured_against": "c_snap_hold_fy2026", + "upstream_fixed": true + }, + "r27_snap_net_income_rounding": { + "class": "engine_defect", + "subtype": "thresholds_rates", + "defect": "PolicyEngine floors SNAP net income in every state, which matches neither procedure 7 CFR 273.10(e)(1)(ii) allows", + "law": "7 CFR 273.10(e)(1)(ii); California MPP 63-503.311 (CDSS summary)", + "alternative_reading": "SNAP net income is rounded to the nearest dollar, as policyengine-us#9318 does in every state and California's CalFresh rules require; recomputing with cents kept instead moves no scored reference.", + "upstream": "fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze", + "upstream_fixed": true, + "measured_against": "c_snap_hold_fy2026", + "supersedes": "r27_ca_snap_net_income_rounding (California only)" + }, + "r30_snap_heat_and_eat_sua": { + "class": "engine_defect", + "subtype": "categorical_eligibility", + "defect": "PolicyEngine grants the heat-and-eat standard utility allowance to households without an elderly or disabled member, which P.L. 119-21 sec. 10103 ended", + "law": "7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04)", + "alternative_reading": "A LIHEAP payment makes a household eligible for the standard utility allowance only if the household has an elderly or disabled member; other households take the allowance only if they incur heating or cooling costs.", + "upstream": "", + "measured_against": "c_snap_hold_fy2026", + "exclusion_note": "The prompt says the head is disabled but states none of the routes 7 U.S.C. 2012(j) lists for a disabled member, such as receipt of SSI or Social Security disability benefits, so the engine and this record treat the household as having no elderly or disabled member (verification/v9_axiom_us-2014-sua.md). Read the other way, the LIHEAP payment would still confer the standard utility allowance, and the value the publication conventions and upstream SNAP fixes give, 3,576.00 (the reference it would carry if scored), would stand; the output is excluded either way." + }, + "r31_snap_income_limit_rounding": { + "class": "engine_defect", + "subtype": "thresholds_rates", + "defect": "PolicyEngine compares SNAP income to the unrounded gross and net income standards", + "law": "7 CFR 273.9(a)(3)", + "alternative_reading": "The monthly gross and net income standards are rounded up to the next whole dollar.", + "upstream": "fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze", + "upstream_fixed": true, + "measured_against": "c_snap_hold_fy2026" + }, + "r32_wi_capital_gain_distributions": { + "class": "engine_defect", + "subtype": "taxable_income_or_deductions", + "defect": "PolicyEngine's Wisconsin capital gain subtraction leaves out capital gain distributions reported without Schedule D, so once they reach federal AGI Wisconsin taxes all of them instead of 70%", + "law": "Wis. Stat. 71.05(6)(b)9; 26 U.S.C. 852(b)(3)(B); 2025 Wisconsin Schedule SB instructions, line 5", + "alternative_reading": "Wisconsin subtracts 30% of net long-term capital gain, and a capital gain distribution reported without Schedule D is long-term capital gain.", + "upstream": "", + "measured_against": "c_wi_published_2026+r04_capital_gain_distributions", + "split_from": "r04_capital_gain_distributions_v2 (2026-09-23): #8839 fixed the federal inclusion only" + }, + "r33_snap_child_support_treatment": { + "class": "engine_defect", + "subtype": "state_local_rule", + "defect": "PolicyEngine reads its SNAP child support flag as the option to exclude child support paid from gross income, but the values carry the opposite meaning: for 2026 it excludes the payments from gross income in 37 jurisdictions that USDA's State Options Report lists as deducting them from net income, Michigan among them, and deducts them in 9 of the 14 it lists as excluding them", + "law": "7 CFR 273.9(c)(17); 7 CFR 273.9(d)(5); USDA SNAP State Options Report, 16th edition (June 2024), Treatment of Child Support Payments, p. 15, and 17th edition, p. 21 (Michigan: child support income expense deduction); Michigan Bridges Eligibility Manual 556 (11-1-2025), line 20", + "alternative_reading": "A state that does not take the option in 7 CFR 273.9(c)(17) counts legally obligated child support paid to nonhousehold members in gross income and deducts it when computing net income (7 CFR 273.9(d)(5)). USDA's 16th and 17th State Options Reports list Michigan among those states (the 15th lists it among the exclusion states), and Michigan's budget (BEM 556) subtracts child support at line 20, after gross income at line 10.", + "upstream": "fixed in PolicyEngine/policyengine-us#9586 (merged 2026-09-24)", + "measured_against": "c_snap_hold_fy2026", + "decided_on": "2026-09-24", + "upstream_fixed": true, + "note": "2026-09-24: first recorded while the fix was open, so release dashboard-data-20260922b excluded scenario_045 SNAP for every model (rule 3). PolicyEngine merged policyengine-us#9586 the same day (squash commit d9e801df417352b8246a4c292a19ec082a518790), so from release dashboard-data-20260922c rule 2 applies: the output is regenerated with the merged parameter values (fix module r33_snap_child_support_treatment.py) and scored for every model at $0. Counted in gross income, the $433.33 a month of child support the worker pays leaves SNAP gross income at $3,022.97, 231.8 percent of the poverty guideline in January, above the 200 percent limit of Michigan's broad-based categorical eligibility and SNAP's 130 percent gross income test, and the household has no elderly or disabled member." + } +} diff --git a/reference_audit/2026-09-22/scripts/build_records.py b/reference_audit/2026-09-22/scripts/build_records.py new file mode 100644 index 00000000..2e092324 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/build_records.py @@ -0,0 +1,673 @@ +"""Build the reference-exclusion and adjudication records from the 2026-09-22 triage. + +Inputs: + root_causes.json per root cause: class, defect, law, correct rule, upstream + sweep/out/.csv verified sweeps (moved = |delta| > $1 or a flipped flag) + verdicts.json the per-case triage verdicts for the 25 flagged cases + the final (post-judge) row annotations and case notes + verdict.meta.json sidecars (judge model per case) + the existing record (kept verbatim; new entries appended) + the existing record (kept; flagged entries get a verdict) + +Writes the two records to --out-dir and prints a summary. Run with the +policyengine-us 1.755.4 triage venv so multi-cause outputs can be recomputed +under their combined fixes: + + cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/r33-20260922c \\ + .venv-pe1755/bin/python build_records.py --annotations --out-dir records/ + +(2026-09-22/23 ran from policybench-wt/opus55; the 2026-09-24 revision, which adds +r33, from policybench-wt/r33-20260922b while its fix was open, and from +policybench-wt/r33-20260922c once policyengine-us#9586 merged: r33 is then +upstream_fixed, so its output is regenerated, not excluded.) +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE / "sweep")) +from sweep import BUNDLE, YEAR, build_situation # noqa: E402 + +from policybench.ground_truth import ( # noqa: E402 + _extract_person_value, + _pe_variable_for_output, +) +from policybench.scenarios import scenario_from_dict # noqa: E402 + +DECIDED_ON = "2026-09-22" +# Dates of this audit's records. A root cause added after the September 22 wave +# carries its own "decided_on" in root_causes.json (r33: 2026-09-24), which its +# exclusions and adjudications take; every run rebuilds all of them. (r33 is +# fixed upstream since 2026-09-24, so it has none from release 20260922c on.) +REVISED_ON = ("2026-09-24",) +WAVE_DATES = (DECIDED_ON, *REVISED_ON) +ENGINE = "policyengine-us 1.755.4" +AUDIT_CASES = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/audit/cases" +) +# The fix module whose sweep defines each root cause's moved set and values: the +# independently verified v2 modules (sweep/verify reports, 2026-09-22). r10 keeps +# its original module, which encodes the alternative reading of the unlisted input. +# r04 is the faithful backport of upstream #8839; the Wisconsin exclusion the v2 +# module also applied is r32, which is not fixed upstream (split 2026-09-23). +SWEEP_FOR = { + "r01_ira_compensation": "r01_ira_compensation_v2", + "r02_ira_219g": "r02_ira_219g_v2", + "r03_estate_income": "r03_estate_income__qbi_false_v2", + "r04_capital_gain_distributions": "r04_capital_gain_distributions", + "r05_nj_worker_ui": "r05_nj_worker_ui_v2", + "r06_wi_act15_before_refundable": "r06_wi_act15_before_refundable_v2", + "r07_idaho_health_premiums": "r07_idaho_health_premiums_v2", + "r08_eitc_earned_income_deferrals": "r08_eitc_earned_income_deferrals_v2", + "r09_ny_rptc_rent_cap": "r09_ny_rptc_rent_cap_v2", + "r10_wi_homestead_income": "r10_wi_homestead_income", + "r11_ca_itemized_conformity": "r11_ca_itemized_conformity_v2", + "r14_unlisted_weekly_hours_v2": "r14_unlisted_weekly_hours_v2_v2", + "r15_snap_mortgage_interest": "r15_snap_mortgage_interest_v2", + "r16_survivor_benefits_federal": "r16_survivor_benefits_federal_v2", + "r17_caleitc_agi_comparison": "r17_caleitc_agi_comparison", + "r20_adult_dependent_child": "r067_adult_dependent_relationship", + "r21_adult_dependent_nonchild": "r067_adult_dependent_nonchild", + "r22_ma_part_a_loss_offset": "ma_part_a_loss_offset", + "r23_ma_interest_source": "ma_part_a_ordinary_interest", + "r24_disability_benefits_taxability": "r24_disability_benefits_taxable", + "r25_niit_in_federal_output": "r25_niit_excluded", + "r26_snap_contribution_rounding": "r26_snap_contribution_rounding", + "r27_snap_net_income_rounding": "r27_snap_net_income_rounding", + "r28_snap_min_allotment_rounding": "r28_snap_min_allotment_rounding", + "r30_snap_heat_and_eat_sua": "r30_snap_heat_and_eat_sua", + "r31_snap_income_limit_rounding": "r31_snap_income_limit_rounding", + "r32_wi_capital_gain_distributions": "r32_wi_capital_gain_distributions", + "r33_snap_child_support_treatment": "r33_snap_child_support_treatment", +} +# The SNAP defects are measured on top of the SNAP publication convention +# (c_snap_hold_fy2026), since the convention's value is what would otherwise be +# published: _on_c13v3.csv compares the convention alone with the +# convention plus the defect's fix, and moved means more than $1. r32 needs the +# distributions in federal AGI, so it is measured on top of the Wisconsin +# convention plus #8839 (r32_on_cwi_r04.csv). make_on_convention.py writes both. +MEASURED_BY = { + "r26_snap_contribution_rounding": "r26_on_c13v3", + "r27_snap_net_income_rounding": "r27_on_c13v3", + "r28_snap_min_allotment_rounding": "r28_on_c13v3", + "r30_snap_heat_and_eat_sua": "r30_on_c13v3", + "r31_snap_income_limit_rounding": "r31_on_c13v3", + "r32_wi_capital_gain_distributions": "r32_on_cwi_r04", + "r33_snap_child_support_treatment": "r33_on_c13v3", +} +REASON = { + "engine_defect": "reference_engine_defect", + "unlisted_input": "reference_depends_on_unlisted_input", + "later_law": "reference_law_published_after_freeze", +} +ADJUDICATED_SOURCE = { + "engine_defect": "reference_engine_defect", + "unlisted_input": "prompt_ambiguity", + "later_law": "reference_later_law", +} +# Precedence when one output moves under several root causes: an engine +# defect makes the frozen reference wrong on the stated facts, which settles +# the exclusion; an unlisted input comes next; later-published law last. +PRECEDENCE = ("engine_defect", "unlisted_input", "later_law") + +# The four flagged references the triage affirmed. Reasoning is the case's +# verdict in one or two sentences; the basis is the law it rests on. +AFFIRMED = { + ("scenario_028", "child3_chip_eligible"): { + "subtype": "health_coverage", + "basis": "42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7", + "reasoning": ( + "Child 3 has employer-sponsored group coverage, which bars CHIP; the " + "household's income passes Pennsylvania's CHIP limit, so the judge's " + "income-limit hypothesis fails." + ), + }, + ("scenario_056", "state_refundable_credits"): { + "subtype": "credit_phaseout", + "basis": "N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32", + "reasoning": ( + "New Jersey bases the earned income credit for filers 18 and older who " + "fail only the federal age test on the federal maximum credit for " + "filers without a qualifying child, and pays it as a flat yearly " + "amount: 40% of the 2026 $664 maximum is $265.60. The judge's " + "phase-in reading fails." + ), + }, + ("scenario_086", "federal_income_tax_before_refundable_credits"): { + "subtype": "taxable_income_or_deductions", + "basis": "26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12", + "reasoning": ( + "The 2026 educator expense cap is $350, so the head's $337.50 is fully " + "deductible; the judge's $300 cap is not the law." + ), + }, + ("scenario_076", "state_income_tax_before_refundable_credits"): { + "subtype": "state_local_rule", + "basis": "Idaho Code 63-3029L (taxable years 2018 through 2025); Idaho Administrative Bulletin, July 1, 2026, Docket 35-0101-2601", + "reasoning": ( + "Idaho's $205 child tax credit applies only to taxable years beginning " + "in 2018 through 2025, so it does not reduce 2026 tax; the judge's " + "missing-credit hypothesis fails. The reference applies the Idaho " + "zero-rate threshold held at its published 2025 amount." + ), + }, + ("scenario_118", "state_refundable_credits"): { + "subtype": "state_local_rule", + "basis": "N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025", + "reasoning": ( + "From 2025 New York keys the real property tax credit to federal AGI and " + "a flat table; with federal AGI of $0 and property tax above 3.5% of " + "it, a filer 65 or older receives $375. The judge applied the repealed " + "household-gross-income rule." + ), + }, +} + + +def load_fix(fix_id: str): + path = HERE / "sweep" / "fixes" / f"{fix_id}.py" + spec = importlib.util.spec_from_file_location(f"fix_{fix_id}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return getattr(module, "reform", None), getattr(module, "patch", None) + + +def flatten(reform) -> list: + if reform is None: + return [] + if isinstance(reform, (tuple, list)): + return [r for item in reform for r in flatten(item)] + return [reform] + + +_SYSTEMS: dict[tuple[str, ...], tuple] = {} + + +def _system_for(fixes: list[str]): + """The tax-benefit system and situation patches for a set of fixes. + + Only the latest system is kept (one is several gigabytes), so callers + compute outputs grouped by their fix set. + """ + from policyengine_us import CountryTaxBenefitSystem + + key = tuple(fixes) + if key not in _SYSTEMS: + _SYSTEMS.clear() + reforms, patches = [], [] + for fix in fixes: + reform, patch = load_fix(fix) + reforms.extend(flatten(reform)) + if patch is not None: + patches.append(patch) + system = CountryTaxBenefitSystem(reform=tuple(reforms)) if reforms else None + _SYSTEMS[key] = (system, patches) + return _SYSTEMS[key] + + +def combined_value(scenario_id: str, variable: str, fixes: list[str]) -> float: + """Recompute one output with every listed fix module applied together.""" + from policyengine_us import Simulation + + system, patches = _system_for(fixes) + scenarios = pd.read_csv(BUNDLE / "scenarios.csv") + row = scenarios.loc[scenarios["scenario_id"] == scenario_id] + scenario = scenario_from_dict(json.loads(row["scenario_json"].iloc[0])) + situation = build_situation(scenario) + for patch in patches: + situation = patch(copy.deepcopy(situation), scenario) + sim = ( + Simulation(tax_benefit_system=system, situation=situation) + if system is not None + else Simulation(situation=situation) + ) + pe_variable = _pe_variable_for_output(variable, "us") + return float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, variable) + ) + + +PENDING_JUDGE: list[str] = [] + + +def compute_many(requests: list[tuple[tuple[str, str], list[str]]]) -> dict: + """Values for (output, fixes) requests, one tax-benefit system at a time.""" + values = {} + for key, fixes in sorted(requests, key=lambda item: (item[1], item[0])): + values[(key, tuple(fixes))] = combined_value(key[0], key[1], fixes) + return values + + +# The alternative reading of each unlisted input behind the 2026-09-05 +# exclusions, as a situation patch (sweep/fixes/u_*.py). +UNLISTED_PATCH = { + "meets_ssi_disability_criteria": "u_ssi_disability_criteria", + "months_receiving_social_security_disability": "u_ssdi_months", +} +FLAG_SOURCE_EARLIER_RUN = ( + "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); " + "the case's current verdict.json does not flag it" +) + + +def judge_flag_for(scenario_id: str, variable: str) -> bool: + """Whether the case's current verdict.json flags the reference as suspect.""" + path = AUDIT_CASES / f"us__{scenario_id}__{variable}" / "verdict.json" + return bool(path.exists() and json.loads(path.read_text()).get("reference_suspect")) + + +def judge_verdict_for(scenario_id: str, variable: str) -> tuple[str, str]: + """The judge's own class for the case, from its verdict.json. + + Not us_case_notes.csv: apply_adjudications has already replaced the judge's + class there with the adjudicated one by the time this script reads it. + """ + path = AUDIT_CASES / f"us__{scenario_id}__{variable}" / "verdict.json" + if not path.exists(): + PENDING_JUDGE.append(f"{scenario_id}:{variable}") + return "pending", "pending" + verdict = json.loads(path.read_text()) + return verdict["case_failure_source"], verdict["case_failure_subtype"] + + +def judge_model_for(scenario_id: str, variable: str) -> str: + meta = AUDIT_CASES / f"us__{scenario_id}__{variable}" / "verdict.meta.json" + if not meta.exists(): + # The case awaits a re-judge; the final build runs after judging and refuses this. + PENDING_JUDGE.append(f"{scenario_id}:{variable}") + return "pending" + reported = json.loads(meta.read_text()).get("judge_model_reported") or [] + models = [m for m in reported if "haiku" not in m] or reported + return models[-1] if models else "unknown" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--annotations", required=True, type=Path) + parser.add_argument("--exclusions", type=Path, default=BUNDLE / "reference_exclusions.json") + parser.add_argument("--adjudications", type=Path) + parser.add_argument("--out-dir", required=True, type=Path) + parser.add_argument("--allow-pending-judge", action="store_true") + args = parser.parse_args() + adjudications_path = args.adjudications or args.annotations / "us_adjudications.json" + + causes = json.loads((HERE / "root_causes.json").read_text()) + exclusions_doc = json.loads(args.exclusions.read_text()) + existing = {(e["scenario_id"], e["variable"]) for e in exclusions_doc["exclusions"]} + cases = pd.read_csv(args.annotations / "us_case_notes.csv") + case_index = cases.set_index(["scenario_id", "variable"]) + + flagged = set( + map( + tuple, + cases.loc[ + cases["reference_suspect"].astype(str).str.lower() == "true", + ["scenario_id", "variable"], + ].values, + ) + ) + # The wave's first Opus 5.5 pass flagged 25 references; re-judging a case after + # GPT-6 rows joined it can drop a flag already adjudicated. A flag the wave + # raised stays recorded as raised. + first_pass = HERE / "flagged_sept22_first_pass.json" + if first_pass.exists(): + flagged |= {tuple(k.split(":")) for k in json.loads(first_pass.read_text())} + # Every flag any judge run of the wave raised (the verdicts-board42*.json + # files, collected into flagged_sept22_wave.json), for the same reason. + wave = HERE / "flagged_sept22_wave.json" + if wave.exists(): + flagged |= {tuple(k.split(":")) for k in json.loads(wave.read_text())} + + # 1. Every moved output and the root causes that move it. + moved: dict[tuple[str, str], dict] = {} + for cause, fix in SWEEP_FOR.items(): + if causes[cause]["class"] not in PRECEDENCE: + continue + not_confirmed = causes[cause].get("not_confirmed", {}) + frame = pd.read_csv(HERE / "sweep" / "out" / f"{MEASURED_BY.get(cause, fix)}.csv") + for _, r in frame[frame["moved"]].iterrows(): + key = (r["scenario_id"], r["variable"]) + if f"{key[0]}:{key[1]}" in not_confirmed: + continue + item = moved.setdefault(key, {"frozen": float(r["frozen"]), "causes": {}}) + item["causes"][cause] = float(r["recomputed"]) + + # The publication conventions and the upstream fixes: the references are + # regenerated under all of them together (regen_references.py), so an + # excluded output's corrected value applies all of them with its own fixes. + # A source that changes nothing for a household leaves its value alone, and + # one that only matters in combination (a SNAP rounding fix once r30 cuts the + # allotment below the maximum) still applies. + published_fixes = [ + cause["fix"] + for cause in causes.values() + if isinstance(cause, dict) and cause.get("class") == "convention" + ] + [ + SWEEP_FOR[name] + for name, cause in causes.items() + if isinstance(cause, dict) and cause.get("upstream_fixed") + ] + + new_exclusions, new_adjudications, summary = [], [], [] + regenerated_by_fix = [] + decisions = {} + for key, item in sorted(moved.items()): + if key in existing: + continue + # Rule (Max, 2026-09-23): a defect fixed in policyengine-us after the + # freeze is regenerated with the fix (regen_references.py), not excluded. + # An output any unfixed defect or unlisted input moves stays excluded. + if all(causes[c].get("upstream_fixed") for c in item["causes"]): + regenerated_by_fix.append((key, sorted(item["causes"]))) + continue + # The unfixed causes decide the exclusion; a defect fixed upstream only + # enters the corrected value, since the regenerated reference applies it. + fixed = sorted(c for c in item["causes"] if causes[c].get("upstream_fixed")) + classes = { + c: causes[c]["class"] for c in item["causes"] if c not in fixed + } + klass = next(k for k in PRECEDENCE if k in classes.values()) + primary = sorted(c for c, k in classes.items() if k == klass) + # Corrected value: the primary class's fixes applied together, and for an + # engine defect also any later-law fix on the same output, on top of every + # publication convention and upstream fix, as the published references are. + applied = primary + sorted( + c for c, k in classes.items() if klass == "engine_defect" and k == "later_law" + ) + decisions[key] = (klass, primary, [SWEEP_FOR[c] for c in applied]) + # One system per fix set, computed in fix-set order. + corrected_values = {} + for key in sorted(decisions, key=lambda k: (decisions[k][2], k)): + fixes = decisions[key][2] + corrected_values[key] = combined_value(key[0], key[1], fixes + published_fixes) + + for key, (klass, primary, fixes) in sorted(decisions.items()): + item = moved[key] + corrected = corrected_values[key] + texts = [causes[c] for c in primary] + others = sorted(set(item["causes"]) - set(primary)) + decided = max(t.get("decided_on", DECIDED_ON) for t in texts) + entry = { + "scenario_id": key[0], + "variable": key[1], + "reason_code": REASON[klass], + "root_cause": "+".join(primary), + "alternative_reading": " ".join(t["alternative_reading"] for t in texts), + "frozen_value": item["frozen"], + "alternative_value": round(corrected, 6), + "engine_version": ENGINE, + "decided_on": decided, + "decided_by": "developer", + } + if klass == "engine_defect": + entry["defect"] = "; ".join(t["defect"] for t in texts) + entry["law"] = "; ".join(t["law"] for t in texts) + entry["upstream"] = "; ".join(t["upstream"] for t in texts if t["upstream"]) or "to be filed" + elif klass == "unlisted_input": + entry["unlisted_input"] = " and ".join(t["unlisted_input"] for t in texts) + else: + entry["published"] = "; ".join(t["published"] for t in texts) + entry["law"] = "; ".join(t["law"] for t in texts) + if klass != "unlisted_input": + entry.pop("unlisted_input", None) + else: + entry.pop("root_cause", None) + notes = [ + "Corrected value applies " + " and ".join(fixes) + + (" together" if len(fixes) > 1 else "") + + " with every publication convention and upstream fix." + ] + notes += [causes[c]["exclusion_note"] for c in primary if causes[c].get("exclusion_note")] + if others: + notes.append( + "The output also moves under " + + ", ".join( + f"{c} ({causes[c]['class'].replace('_', ' ')}" + + (", fixed upstream" if causes[c].get("upstream_fixed") else "") + + ")" + for c in others + ) + + "." + ) + entry["note"] = " ".join(notes) + new_exclusions.append(entry) + + verdict = {"engine_defect": "engine_defect", "unlisted_input": "unlisted_input", "later_law": "later_law"}[klass] + new_adjudications.append( + { + "country": "us", + "scenario_id": key[0], + "variable": key[1], + "judge_model": judge_model_for(*key), + "judge_failure_source": judge_verdict_for(*key)[0], + "judge_failure_subtype": judge_verdict_for(*key)[1], + "adjudicated_failure_source": ADJUDICATED_SOURCE[klass], + "adjudicated_failure_subtype": causes[primary[0]]["subtype"], + "adjudicated_on": decided, + "adjudicator": "developer", + "excluded_from_scoring": True, + "judge_reference_suspect": key in flagged, + "reference_verdict": verdict, + "reference_basis": entry.get("law") or entry.get("unlisted_input"), + "reasoning": ( + f"The frozen reference is {item['frozen']:,.2f}; " + f"{'the corrected value' if klass == 'engine_defect' else 'the alternative'} is " + f"{corrected:,.2f}. " + + (entry.get("defect", "") + "." if klass == "engine_defect" else texts[0]["alternative_reading"]) + ).strip(), + } + ) + summary.append((key, klass, "+".join(primary), item["frozen"], corrected)) + + # 2. Affirmed references. + for key, spec in AFFIRMED.items(): + new_adjudications.append( + { + "country": "us", + "scenario_id": key[0], + "variable": key[1], + "judge_model": judge_model_for(*key), + "judge_failure_source": judge_verdict_for(*key)[0], + "judge_failure_subtype": judge_verdict_for(*key)[1], + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": spec["subtype"], + "adjudicated_on": DECIDED_ON, + "adjudicator": "developer", + "judge_reference_suspect": key in flagged, + "reference_verdict": "affirmed", + "reference_basis": spec["basis"], + "reasoning": spec["reasoning"], + } + ) + + # 2b. Flagged references the regeneration replaced: they stay scored. + revisions = json.loads( + (HERE.parent / "reference_v12" / "reference_outputs.csv.meta.json").read_text() + )["revisions"] + regenerated = {} + for revision in revisions: + for change in revision["changed"]: + regenerated.setdefault((change["scenario_id"], change.get("variable", "snap")), []).append((revision, change)) + done = {(e["scenario_id"], e["variable"]) for e in new_adjudications} + for key in sorted(flagged): + if key in done or key not in regenerated: + continue + sources = regenerated[key] + change = sources[0][1] + parts, bases = [], [] + for revision, _ in sources: + if revision.get("kind") == "upstream_fix": + cause = causes[revision["root_cause"]] + parts.append( + f"it corrects an engine defect fixed upstream ({cause['upstream']}): {cause['alternative_reading']}" + ) + bases.append(cause["law"]) + else: + convention = causes[revision["convention"]] + parts.append(f"it applies the publication rule: {convention['rule']}") + bases.append(convention["basis"]) + new_adjudications.append( + { + "country": "us", + "scenario_id": key[0], + "variable": key[1], + "judge_model": judge_model_for(*key), + "judge_failure_source": judge_verdict_for(*key)[0], + "judge_failure_subtype": judge_verdict_for(*key)[1], + "adjudicated_failure_source": "llm_error", + # A convention changes parameter values, so the models' misses + # against the regenerated reference are threshold and rate + # errors; where an upstream fix also regenerates the reference, + # the judge's own subtype stands. + "adjudicated_failure_subtype": ( + judge_verdict_for(*key)[1] + if any(r.get("kind") == "upstream_fix" for r, _ in sources) + else "thresholds_rates" + ), + "adjudicated_on": DECIDED_ON, + "adjudicator": "developer", + "judge_reference_suspect": True, + "reference_verdict": "regenerated", + "reference_basis": "; ".join(bases), + "reasoning": ( + f"The frozen {change['frozen']:,.2f} is regenerated as " + f"{change['regenerated']:,.2f}: " + "; ".join(parts) + ), + } + ) + + # 3. Existing adjudications: flagged ones gain their reference verdict, and + # an entry whose case was re-judged after it was recorded takes the current + # judge's model, date and verdict (the adjudication keeps the verdict it + # resolves beside the decision). + adjudications_doc = json.loads(adjudications_path.read_text()) + # This wave's entries are rebuilt from scratch on every run; keep only the + # earlier waves' records, so an output this run no longer excludes loses + # the adjudication a previous run gave it. + adjudications_doc["adjudications"] = [ + e for e in adjudications_doc["adjudications"] if e.get("adjudicated_on") not in WAVE_DATES + ] + refreshed = [] + for entry in adjudications_doc["adjudications"]: + key = (entry["scenario_id"], entry["variable"]) + meta_path = AUDIT_CASES / f"us__{key[0]}__{key[1]}" / "verdict.meta.json" + if meta_path.exists() and key in case_index.index: + meta = json.loads(meta_path.read_text()) + judged_on = str(meta.get("judged_at_utc", ""))[:10] + if judged_on and judged_on > str(entry.get("judged_on_utc", "")): + entry["judge_model"] = judge_model_for(*key) + entry["judged_on_utc"] = judged_on + refreshed.append(f"{key[0]}:{key[1]}") + # The judge's class is always the case's current verdict.json, which the + # freezer checks (verify_adjudications_keep_judge_verdicts). + if (AUDIT_CASES / f"us__{key[0]}__{key[1]}" / "verdict.json").exists(): + source, subtype = judge_verdict_for(*key) + entry["judge_failure_source"] = source + entry["judge_failure_subtype"] = subtype + if key in flagged and not entry.get("reference_verdict"): + entry["judge_reference_suspect"] = True + entry["reference_verdict"] = "unlisted_input" + entry["reference_basis"] = "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905" + new_keys = {(e["scenario_id"], e["variable"]) for e in new_adjudications} + kept = [e for e in adjudications_doc["adjudications"] if (e["scenario_id"], e["variable"]) not in new_keys] + adjudications_doc["adjudications"] = kept + new_adjudications + # The 2026-09-05 exclusions keep their classification; their alternative + # value is recomputed like every other corrected value, on top of every + # publication convention and upstream fix, and a defect not fixed upstream + # that also moves the output is named in the note. + carried = [e for e in exclusions_doc["exclusions"] if e["decided_on"] not in WAVE_DATES] + requests = [] + for e in carried: + key = (e["scenario_id"], e["variable"]) + requests.append((key, [UNLISTED_PATCH[e["unlisted_input"]]] + published_fixes)) + for c in sorted(moved.get(key, {}).get("causes", {})): + if causes[c]["class"] == "engine_defect" and not causes[c].get("upstream_fixed"): + requests += [(key, published_fixes), (key, [SWEEP_FOR[c]] + published_fixes)] + carried_values = compute_many(requests) + carried_sentences = {} + for e in carried: + key = (e["scenario_id"], e["variable"]) + fixes = [UNLISTED_PATCH[e["unlisted_input"]]] + published_fixes + value = carried_values[(key, tuple(fixes))] + notes = [e["note"]] if e.get("note") else [] + if abs(value - float(e["alternative_value"])) > 1e-6: + notes.append( + f"The 2026-09-05 record gave {float(e['alternative_value']):,.2f} under that " + f"reading on the frozen engine; the {DECIDED_ON} audit recomputes it with " + "every publication convention and upstream fix." + ) + carried_sentences.setdefault(key, []).append( + f"The {DECIDED_ON} audit recomputes the alternative as {value:,.2f} with every " + f"publication convention and upstream fix (the frozen-engine value above was " + f"{float(e['alternative_value']):,.2f})." + ) + e["alternative_value"] = round(value, 6) + for c in sorted(moved.get(key, {}).get("causes", {})): + if causes[c]["class"] == "engine_defect" and not causes[c].get("upstream_fixed"): + base = carried_values[(key, tuple(published_fixes))] + fixed_value = carried_values[(key, tuple([SWEEP_FOR[c]] + published_fixes))] + sentence = ( + f"{c} (engine defect, not fixed upstream) also moves this output on " + f"the stated facts, from {base:,.2f} to {fixed_value:,.2f}; the record " + "keeps its 2026-09-05 classification." + ) + notes.append(sentence) + carried_sentences.setdefault(key, []).append(sentence) + if notes: + e["note"] = " ".join(notes) + exclusions_doc["exclusions"] = exclusions_doc["exclusions"] + new_exclusions + # judge_reference_suspect records every flag the wave raised; where the + # case's current verdict does not carry it, the entry says which run did. + # The carried 2026-09-05 adjudications state the recomputed alternative too, + # so their reasoning agrees with the exclusion record (replaced, not + # appended again, on each run). + marker = f" The {DECIDED_ON} audit recomputes the alternative" + for entry in adjudications_doc["adjudications"]: + key = (entry["scenario_id"], entry["variable"]) + if entry.get("adjudicated_on") not in WAVE_DATES: + reasoning = entry["reasoning"].split(marker)[0] + reasoning = reasoning.split(" r30_snap_heat_and_eat_sua (engine defect")[0] + if key in carried_sentences: + reasoning = reasoning + " " + " ".join(carried_sentences[key]) + entry["reasoning"] = reasoning + for entry in adjudications_doc["adjudications"]: + key = (entry["scenario_id"], entry["variable"]) + entry.pop("judge_reference_suspect_source", None) + if entry.get("judge_reference_suspect") and not judge_flag_for(*key): + entry["judge_reference_suspect_source"] = FLAG_SOURCE_EARLIER_RUN + + args.out_dir.mkdir(parents=True, exist_ok=True) + (args.out_dir / "reference_exclusions.json").write_text(json.dumps(exclusions_doc, indent=2) + "\n") + (args.out_dir / "us_adjudications.json").write_text(json.dumps(adjudications_doc, indent=2) + "\n") + unresolved = flagged - {(e["scenario_id"], e["variable"]) for e in adjudications_doc["adjudications"] if e.get("reference_verdict")} + for key, klass, cause, frozen, corrected in summary: + print(f"{key[0]} {key[1]:46s} {klass:15s} {cause:60s} {frozen:>11.2f} -> {corrected:>11.2f}") + print( + f"exclusions: {len(exclusions_doc['exclusions'])} ({len(new_exclusions)} new); " + f"adjudications: {len(adjudications_doc['adjudications'])}; " + f"flagged cases without a verdict: {sorted(unresolved)}" + ) + if regenerated_by_fix: + print(f"regenerated with an upstream fix, not excluded ({len(regenerated_by_fix)}): " + + ", ".join(f"{k[0]}:{k[1]} [{'+'.join(c)}]" for k, c in regenerated_by_fix)) + refreshed = [k for k in refreshed if tuple(k.split(":")) not in new_keys] + if refreshed: + print(f"refreshed judge fields on {len(refreshed)} existing adjudications: {refreshed}") + if PENDING_JUDGE: + print(f"WARNING: {len(PENDING_JUDGE)} records name a case awaiting its re-judge: {PENDING_JUDGE}") + if not args.allow_pending_judge: + raise SystemExit("re-judge those cases first (or pass --allow-pending-judge for a dry run)") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-22/scripts/make_on_convention.py b/reference_audit/2026-09-22/scripts/make_on_convention.py new file mode 100644 index 00000000..45146718 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/make_on_convention.py @@ -0,0 +1,91 @@ +"""Measure defects on top of the value that would otherwise be published. + +The SNAP convention (c_snap_hold_fy2026, fix r13_hold_fy2026_v3) regenerates SNAP +references with the FY2026 schedule held for October-December. The engine's SNAP +defects (r26, r27, r28, r30, r31, r33) are measured against that convention's value: +an output is moved when the convention plus the defect's fix differs from the +convention alone by more than $1 (a binary output: when the flag flips). The +Wisconsin defect r32 only matters once #8839 puts the distributions in federal +AGI, so it is measured against the Wisconsin convention plus the #8839 backport +(cwi_plus_r04). Reads the sweeps in sweep/out/ and writes: + + sweep/out/_on_.csv frozen, baseline, recomputed, delta, moved + sweep/out/snap_net_income_sensitivity.csv + the convention with r26, r28 and r33 applied under three net-income + procedures (the engine's floor, nearest dollar, cents kept) in every state, + against the convention alone (r33 since policyengine-us#9586 merged on + 2026-09-24; it moves only scenario_045 SNAP, to $0 under every procedure) + sweep/out/r30_on_published_snap.csv, sweep/out/r33_on_published_snap.csv + r30 and r33 against the convention with the SNAP rounding fixes merged + upstream (c13v3_plus_upstream_snap), the SNAP value published before + r33's own fix merged (policyengine-us#9586, 2026-09-24): a check that each + moves the same outputs either way + +Run after the sweeps (./run_one.sh for r13_hold_fy2026_v3, c13v3_plus_r26, +c13v3_plus_r27, c13v3_plus_r28, c13v3_plus_r30, c13v3_plus_r31, c13v3_plus_r33, +c13v3_r26_r28_r33, c13v3_r28_r29n_r33, c13v3_r28_r29c_r33, c13v3_plus_upstream_snap, +c13v3_upstream_plus_r30, c13v3_upstream_plus_r33, cwi_plus_r04, cwi_plus_r04_r32): + + python3 make_on_convention.py +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +OUT = Path(__file__).resolve().parent / "sweep" / "out" +KEY = ["scenario_id", "variable"] +# output name: (baseline sweep, baseline plus the defect's fix) +MEASUREMENTS = { + "r26_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r26"), + "r27_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r27"), + "r28_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r28"), + "r30_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r30"), + "r31_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r31"), + "r33_on_c13v3": ("r13_hold_fy2026_v3", "c13v3_plus_r33"), + "r32_on_cwi_r04": ("cwi_plus_r04", "cwi_plus_r04_r32"), + "r30_on_published_snap": ("c13v3_plus_upstream_snap", "c13v3_upstream_plus_r30"), + "r33_on_published_snap": ("c13v3_plus_upstream_snap", "c13v3_upstream_plus_r33"), +} +PROCEDURES = { + "floor": "c13v3_r26_r28_r33", + "nearest": "c13v3_r28_r29n_r33", + "cents": "c13v3_r28_r29c_r33", +} + + +def main() -> None: + for name, (base, combined) in MEASUREMENTS.items(): + baseline = pd.read_csv(OUT / f"{base}.csv").set_index(KEY) + frame = pd.read_csv(OUT / f"{combined}.csv").set_index(KEY) + out = pd.DataFrame(index=frame.index) + out["state"] = frame["state"] + out["frozen"] = frame["frozen"] + out["baseline"] = baseline["recomputed"] + out["recomputed"] = frame["recomputed"] + out["delta"] = out["recomputed"] - out["baseline"] + binary = out.index.get_level_values("variable").str.endswith("_eligible") + out["moved"] = out["delta"].abs() > 1 + out.loc[binary, "moved"] = out.loc[binary, "delta"] != 0 + out.reset_index().to_csv(OUT / f"{name}.csv", index=False) + print(name, int(out["moved"].sum()), f"moved on top of {base}") + convention = pd.read_csv(OUT / "r13_hold_fy2026_v3.csv").set_index(KEY) + + table = pd.DataFrame(index=convention.index) + table["state"] = convention["state"] + table["convention"] = convention["recomputed"] + for name, fix in PROCEDURES.items(): + table[name] = pd.read_csv(OUT / f"{fix}.csv").set_index(KEY)["recomputed"] + changed = ( + table[list(PROCEDURES)].sub(table["convention"], axis=0).abs().max(axis=1) + > 1e-6 + ) + table = table[changed] + table.reset_index().to_csv(OUT / "snap_net_income_sensitivity.csv", index=False) + print(len(table), "outputs differ from the convention under some procedure") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-22/scripts/package_audit.py b/reference_audit/2026-09-22/scripts/package_audit.py new file mode 100644 index 00000000..51236100 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/package_audit.py @@ -0,0 +1,185 @@ +"""Assemble the committed reference-audit record from the triage tree. + +Writes (default: the worktree's reference_audit/2026-09-22): the root +causes, every fix module the records and references rest on, the per-root-cause +sweep of all 1,984 references (sweep_moves.csv), the SNAP net-income sensitivity, +the build scripts and the verification reports. Run after build_records.py, +regen_references.py and make_on_convention.py: + + python3 package_audit.py [--dest ] +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +from pathlib import Path + +import pandas as pd + +from make_on_convention import MEASUREMENTS + +HERE = Path(__file__).resolve().parent +FIXES = HERE / "sweep" / "fixes" +OUT = HERE / "sweep" / "out" +WK = Path("/Users/maxghenis/PolicyEngine/_wk") +# 2026-09-22/23: policybench-wt/opus55; 2026-09-24 (r33): policybench-wt/r33-20260922b, +# then policybench-wt/r33-20260922c once #9586 merged. +DEST = Path("/Users/maxghenis/PolicyEngine/policybench-wt/r33-20260922c/reference_audit/2026-09-22") + +# Modules the recorded ones load at run time, and the measurement modules. +EXTRA_FIXES = [ + "r18_hold_all_projections.py", + # Superseded: the first SNAP convention module the v7 reviews examined. + "r13_hold_fy2026_v2.py", + "r19_irs_sales_tax_2025.json", + "r01_ira_compensation.py", + "r02_ira_219g.py", + "r03_estate_income_v2.py", + "c13v3_plus_r26.py", + "c13v3_plus_r27.py", + "c13v3_plus_r28.py", + "c13v3_plus_r30.py", + "c13v3_plus_r31.py", + "c13v3_plus_upstream_snap.py", + "c13v3_upstream_plus_r30.py", + "c13v3_plus_r33.py", + "c13v3_upstream_plus_r33.py", + "cwi_plus_r04.py", + "cwi_plus_r04_r32.py", + # Superseded for r04 on 2026-09-23: #8839 plus the Wisconsin part now in r32. + "r04_capital_gain_distributions_v2.py", + "c13v3_r26_r28.py", + "c13v3_r28_r29n.py", + "c13v3_r28_r29c.py", + # The net-income sensitivity with r33, from release dashboard-data-20260922c. + "c13v3_r26_r28_r33.py", + "c13v3_r28_r29n_r33.py", + "c13v3_r28_r29c_r33.py", + "r29_snap_net_nearest.py", + "r29_snap_net_cents.py", + # The unlisted readings behind the 2026-09-05 exclusions (situation patches). + "u_ssi_disability_criteria.py", + "u_ssdi_months.py", +] +SCRIPTS = [ + HERE / "sweep" / "sweep.py", + HERE / "sweep" / "run_one.sh", + HERE / "build_records.py", + HERE / "regen_references.py", + HERE / "make_on_convention.py", + HERE / "package_audit.py", +] +VERIFICATION = { + "v1_r01.md": WK / "pb-triage-verify/sweep/verify/work/r01_report.md", + "v1_r02.md": WK / "pb-triage-verify/sweep/verify/work/r02_legal.md", + "v1_r03.md": WK / "pb-triage-verify/sweep/verify/work/r03_report.md", + "v1_r16.md": WK / "pb-triage-verify/sweep/verify/work/r16_report.md", + "v2_r04_r05_r06_r07_r10.md": WK / "pb-triage-verify-v2/sweep/verify/REPORT.md", + "v3_r08_r09_r11_r12.md": WK / "pb-triage-verify-v3/sweep/verify/REPORT.md", + "v4_r13_r14_r15.md": WK / "pb-triage-verify-v4/sweep/verify/SNAP_VERIFICATION.md", + "v5a_projection_irs_wi_id_ca.md": WK / "pb-triage-verify-v5a/sweep/verify/r19_projected_parameter_audit.md", + "v5b_projection_mn_md_mi_mo_il.md": WK / "pb-triage-verify-v5b/sweep/verify/r19_report.md", + "v6_flag_067.md": WK / "pb-flag-v6-067/report.md", + "v6_flag_076.md": WK / "pb-flag-v6-076/report.md", + "v6_flag_081.md": WK / "pb-flag-v6-081/report.md", + "v7_review_data.md": WK / "pb-review-b6a3238/report-data.md", + "v7_review_claims.md": WK / "pb-review-b6a3238/report-claims.md", + "v8_axiom_snap_rounding.md": WK / "axiom-pb-parity/snap/report.md", + "v8_snap_state_rounding.md": WK / "snap-state-rounding/report.md", + "v9_axiom_us-219.md": WK / "axenc-pb/reports/axiom-us-219.md", + "v9_axiom_us-32c2.md": WK / "axenc-pb/reports/axiom-us-32c2.md", + "v9_axiom_us-662.md": WK / "axenc-pb/reports/axiom-us-662.md", + "v9_axiom_ca-17076.md": WK / "axenc-pb/reports/axiom-ca-17076.md", + "v9_axiom_ca-17052.md": WK / "axenc-pb/reports/axiom-ca-17052.md", + "v9_axiom_id-63-3022p.md": WK / "axenc-pb/reports/axiom-id-63-3022p.md", + "v9_axiom_ma-62-2.md": WK / "axenc-pb/reports/axiom-ma-62-2.md", + "v9_axiom_ca-mpp-snap.md": WK / "axenc-pb/reports/axiom-ca-mpp-snap.md", + "v9_axiom_us-852-capgain.md": WK / "axenc-pb/reports/axiom-us-852-capgain.md", + "v9_axiom_nj-43-21-7.md": WK / "axenc-pb/reports/axiom-nj-43-21-7.md", + "v9_axiom_wi-71-05-54m.md": WK / "axenc-pb/reports/axiom-wi-71-05-54m.md", + "v9_axiom_ny-606e.md": WK / "axenc-pb/reports/axiom-ny-606e.md", + "v9_axiom_us-2014-sua.md": WK / "axenc-pb/reports/axiom-us-2014-sua.md", +} + + +def recorded_fixes(causes: dict) -> tuple[dict[str, str], dict[str, str]]: + source = (HERE / "build_records.py").read_text() + sweep_for = dict(re.findall(r'"(r\d\d_[a-z0-9_]+)": "([a-z0-9_]+)"', source.split("MEASURED_BY")[0])) + measured_by = dict(re.findall(r'"(r\d\d_[a-z0-9_]+)": "([a-z0-9_]+)"', source.split("MEASURED_BY = {")[1].split("}")[0])) + return sweep_for, measured_by + + +def sweep_moves(causes: dict, sweep_for: dict, measured_by: dict) -> pd.DataFrame: + rows = [] + entries = [(k, causes[k]["class"], v) for k, v in sweep_for.items() if k in causes] + entries += [(k, "convention", v["fix"]) for k, v in causes.items() if isinstance(v, dict) and v.get("class") == "convention"] + entries += [("r18_hold_all_projections", "screen", "r18_hold_all_projections")] + # A defect measured on top of a combination of sources that is not itself a + # convention (r32: the WI convention plus #8839) gets that combination's own + # rows, class "baseline", so its baseline values can be checked. + convention_fixes = {v["fix"] for v in causes.values() if isinstance(v, dict) and v.get("class") == "convention"} + for cause, measured in measured_by.items(): + base = MEASUREMENTS[measured][0] + if base not in convention_fixes: + entries.append((causes[cause]["measured_against"], "baseline", base)) + for cause, klass, fix in entries: + measured = measured_by.get(cause) + frame = pd.read_csv(OUT / f"{measured or fix}.csv") + baseline = frame["baseline"] if measured else frame["frozen"] + delta = frame["recomputed"] - baseline + for i in frame.index[delta.abs() > 1e-6]: + r = frame.loc[i] + rows.append({ + "root_cause": cause, + "class": klass, + "fix_module": f"{fix}.py", + "measured_against": causes[cause]["measured_against"] if measured else "frozen", + "scenario_id": r["scenario_id"], + "state": r["state"], + "variable": r["variable"], + "frozen": round(float(r["frozen"]), 6), + "baseline": round(float(baseline[i]), 6), + "recomputed": round(float(r["recomputed"]), 6), + "delta": round(float(delta[i]), 6), + "moved_over_1": bool(r["moved"]), + }) + return pd.DataFrame(rows).sort_values(["root_cause", "scenario_id", "variable"]) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dest", type=Path, default=DEST) + args = parser.parse_args() + dest = args.dest + causes = json.loads((HERE / "root_causes.json").read_text()) + sweep_for, measured_by = recorded_fixes(causes) + + if (dest / "fixes").exists(): + shutil.rmtree(dest / "fixes") + for sub in ("fixes", "scripts", "verification"): + (dest / sub).mkdir(parents=True, exist_ok=True) + modules = set(f"{v}.py" for v in sweep_for.values()) + modules |= {f"{v['fix']}.py" for v in causes.values() if isinstance(v, dict) and v.get("class") == "convention"} + modules |= set(EXTRA_FIXES) + for name in sorted(modules): + shutil.copy2(FIXES / name, dest / "fixes" / name) + shutil.copy2(HERE / "root_causes.json", dest / "root_causes.json") + for script in SCRIPTS: + shutil.copy2(script, dest / "scripts" / script.name) + missing = [] + for name, source in VERIFICATION.items(): + if source.exists(): + shutil.copy2(source, dest / "verification" / name) + else: + missing.append(name) + moves = sweep_moves(causes, sweep_for, measured_by) + moves.to_csv(dest / "sweep_moves.csv", index=False) + shutil.copy2(OUT / "snap_net_income_sensitivity.csv", dest / "snap_net_income_sensitivity.csv") + print(f"{len(modules)} fix files, {len(moves)} sweep rows, verification missing: {missing}") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-22/scripts/regen_references.py b/reference_audit/2026-09-22/scripts/regen_references.py new file mode 100644 index 00000000..16714272 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/regen_references.py @@ -0,0 +1,597 @@ +"""Regenerate the scored references under the benchmark's publication rules. + +Rule (Max, 2026-09-22): a scored reference follows from the stated facts and from +law published before the 2026-07-03 reference freeze. Where policyengine-us 1.755.4 +used a projection of an amount published after the freeze, the reference takes the +last amount published before it. The conventions are the root_causes.json entries +of class "convention"; each names the fix module that implements it: + c_snap_hold_fy2026 SNAP October-December 2026 at the FY2026 figures + (sweep/fixes/r13_hold_fy2026_v3.py) + c_ca_hold_2025 California's 2026 indexed amounts at the published 2025 + amounts (sweep/fixes/r19_ca_convention.py) + ... and the other parameter conventions root_causes.json lists + +Rule (Max, 2026-09-23): an engine defect fixed in policyengine-us after the freeze +(root_causes.json "upstream_fixed": true) is regenerated with its verified sandbox +fix, not excluded. Defects not fixed upstream stay excluded (build_records.py). + +Every output is recomputed with all conventions and upstream fixes applied +together. A scored output whose value changes takes the regenerated value; an +excluded output keeps the frozen value its exclusion record names. The sidecar +gains one revision per convention and per upstream fix, listing the outputs that +source moves on its own (a SNAP fix is measured against the SNAP convention). + +Step 1 (policyengine-us 1.755.4 triage venv) writes the reference CSV, the sidecar, +and the engine traces of every changed output: + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \\ + .venv-pe1755/bin/python regen_references.py references \\ + --exclusions records/reference_exclusions.json --out-dir ../reference_v12 + +Step 1b (the same venv) traces the frozen engine for the excluded outputs whose +narrative is rewritten from the frozen trace (FROZEN_NARRATIVES): + ... regen_references.py frozen-traces --out-dir ../reference_v12 + +Step 2 (policybench venv, which has litellm) rewrites the derivation narratives of +changed outputs whose explanation does not already carry the regenerated value, +restores the frozen bundle's narrative for an excluded output whose explanation +carries another value (an output regenerated by an earlier run and excluded +since), and rewrites the FROZEN_NARRATIVES from the frozen trace (a +HAND_CORRECTED narrative replaces the writer's draft, which the log keeps). A +regenerated output in REGENERATED_NARRATIVES is grounded in the engine facts +stated there. --frozen limits the FROZEN_NARRATIVES rewritten (with no names, +none): + ANTHROPIC_API_KEY=... PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \\ + /Users/maxghenis/PolicyEngine/policybench/.venv/bin/python \\ + regen_references.py narratives --out-dir ../reference_v12 \\ + --exclusions records/reference_exclusions.json \\ + --explanations /us_case_reference_explanations.csv + +(2026-09-22/23 ran from policybench-wt/opus55; the 2026-09-24 revisions from +policybench-wt/r33-20260922b, which excluded r33's output, and +policybench-wt/r33-20260922c, which regenerates it now that #9586 has merged.) +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import importlib.util +import json +import sys +import types +from pathlib import Path + +import pandas as pd + +HERE = Path(__file__).resolve().parent +FIXES = HERE / "sweep" / "fixes" +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) +# The frozen v1.1 bundle (reference_outputs.csv sha256 b9136a15...), read-only. Not +# adds202609/publish/: the fold overwrites that dir with the regenerated references. +YEAR = 2026 +DATE = "2026-09-22" +ENGINE = "policyengine-us 1.755.4" +RULE = ( + "A scored reference follows from the stated facts and from law published before " + "the 2026-07-03 reference freeze." +) +PRISTINE_EXPLANATIONS = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/annotations/" + "us_case_reference_explanations.csv" +) +# Excluded outputs whose frozen-bundle narrative misstated the engine's path, +# rewritten from the frozen engine's trace. The rendered trace drops zero-valued +# nodes, so it never shows why net income is zero; the grounding states the +# engine facts, each read from a 1.755.4 run of the frozen engine. +FROZEN_NARRATIVES = { + ("scenario_091", "state_income_tax_before_refundable_credits"): ( + "The household lists $1,170 of capital gain distributions reported without " + "Schedule D, and the engine leaves them out of gross income, so no capital " + "gains reach federal or Wisconsin adjusted gross income. " + "Federal gross income is $35,055.84: $26,073.84 of wages ($27,000 less " + "$926.16 of pre-tax contributions), $4,806 of taxable interest and $4,176 " + "of dividends. The engine leaves the $1,170 of capital gain distributions " + "reported without Schedule D out of gross income, so they reach neither " + "federal nor Wisconsin income. A $43.28 traditional IRA deduction gives " + "federal adjusted gross income of $35,012.56, which is also Wisconsin " + "adjusted gross income: with no capital gains in it, Wisconsin's capital " + "gain subtraction is $0. The Wisconsin standard deduction is $12,067.66 " + "and the personal exemption $700, leaving Wisconsin taxable income of " + "$22,244.90 and tax of $843.66 before refundable credits; no " + "nonrefundable credit applies." + ), + ("scenario_080", "snap"): ( + "Monthly SNAP gross income is $301.33: $300 of financial assistance and " + "$1.33 of dividends; capital gains are not counted. The SNAP standard " + "deduction is $209 a month for January to September and $213.68 for " + "October to December. The engine gives every SNAP household in " + "Pennsylvania the standard utility allowance, $857 a month, through a " + "state flag that models the heat-and-eat practice; it does not check the " + "household's utility bills or its energy assistance payment. The allowance " + "is the household's only shelter cost and is not deducted in full: it " + "enters the excess shelter deduction, which is capped because the engine " + "counts no elderly or disabled member for SNAP, at $744 a month for " + "January to September and $760.68 for October to December. That capped " + "deduction exceeds the income left after the standard deduction ($92.33 " + "and $87.65 a month), so net income is $0 in every month. The allowance " + "is what brings net income to $0: without it the household has no shelter " + "cost and no shelter deduction. " + "With net income of $0 the allotment is the one-person maximum: $298 a " + "month for January to September and $304.68 for October to December, the " + "engine's projected fiscal-year 2027 figure, for $3,596.04 a year. SNAP " + "countable assets are $19,827.65, the bank account balance alone. The " + "household fails the SNAP asset test but is categorically eligible " + "through Pennsylvania's TANF non-cash program, whose income and asset " + "tests it meets; that asset test adds vehicle value only in Texas, so the " + "$3,570 vehicle value counts nowhere." + ), +} + + +# Figures a frozen narrative must state; a narrative missing one is regenerated +# with an explicit instruction, and the step fails if it still omits it. +FROZEN_REQUIRED = { + ("scenario_091", "state_income_tax_before_refundable_credits"): ["1,170"], +} + +# Regenerated outputs whose narrative is grounded in engine facts stated here, +# in place of the grounding of the revisions that list them (one narrative per +# output, however many sources move it). Every figure is read from a 1.755.4 run +# of the published configuration: every convention and every upstream fix +# together, as step 1 regenerates the references. +REGENERATED_NARRATIVES = { + # Regenerated (r33, release dashboard-data-20260922c). The fix merged in + # policyengine-us#9586 on 2026-09-24; the release before it + # (dashboard-data-20260922b) excluded this output and published the frozen + # engine's narrative for $287.68. Figures from employment_income, + # child_support_expense, snap_gross_income, + # snap_child_support_gross_income_deduction, snap_child_support_deduction, + # snap_fpg, snap_gross_income_fpg_ratio, meets_snap_gross_income_test, + # has_usda_elderly_disabled, is_tanf_non_cash_eligible, + # meets_snap_categorical_eligibility, the SNAP deductions, snap_net_income, + # is_snap_eligible and snap in 2026-01, 2026-09, 2026-10 and 2026-12. + ("scenario_045", "snap"): ( + "The reference applies the fix for an engine defect fixed upstream " + "(PolicyEngine/policyengine-us#9586, merged 2026-09-24): PolicyEngine's " + "SNAP child support parameter had excluded child support paid from gross " + "income in states that deduct it from net income, Michigan among them. " + "With the fix the parameter is false for Michigan, so the engine counts " + "the $433.33 a month the worker pays in legally obligated child support " + "($5,200 a year) in SNAP gross income, which is the $3,022.97 of monthly " + "wages, and deducts the $433.33 when computing net income. Gross income " + "is 231.8% of the federal poverty guideline for January to September " + "($1,304.17 a month) and 227.3% for October to December ($1,330). The " + "household has no elderly or disabled member, so it must pass SNAP's 130% " + "gross income test, which it fails. It is not categorically eligible: " + "Michigan's TANF non-cash program, the engine's route to broad-based " + "categorical eligibility, has a gross income limit of 200% of the " + "guideline. The household is therefore ineligible in every month, and " + "the annual benefit is $0. Its net income, $1,102 a month after the $209 " + "standard deduction, a $604.59 earned income deduction (20% of wages), " + "the $433.33 child support deduction and a $673.98 excess shelter " + "deduction, would pass the net income test, but a household without an " + "elderly or disabled member must also pass the gross income test or be " + "categorically eligible." + ), +} +# Figures a regenerated narrative must state (as FROZEN_REQUIRED). +REGENERATED_REQUIRED = { + ("scenario_045", "snap"): ["433.33", "3,022.97", "231.8", "200%", "$0"], +} + +# Narratives corrected by hand against the engine's trace, used in place of the +# writer's draft (which the step still generates and prints, so the run log +# keeps it). scenario_045 SNAP: 2026-09-24 (dashboard-data-20260922b, +# narratives_run7.txt), the frozen engine's $287.68, which the draft credited +# to the poverty guideline and the standard deduction; replaced the same day +# (dashboard-data-20260922c, narratives_run8.txt) by the regenerated $0 below. +HAND_CORRECTED = { + ("scenario_045", "snap"): ( + "PolicyEngine computes an annual 2026 SNAP amount of $0 for this " + "one-person Michigan household. The worker earns $3,022.97 a month in " + "wages and pays $433.33 a month ($5,200 a year) in legally obligated " + "child support. With the fix merged in policyengine-us #9586, " + "PolicyEngine's child support parameter for Michigan counts child " + "support paid in SNAP gross income and deducts it when computing net " + "income, so gross income is the full $3,022.97: 231.8% of the federal " + "poverty guideline from January to September ($1,304.17) and 227.3% " + "from October ($1,330). The household has no elderly or disabled " + "member, so it must pass SNAP's 130% gross income test, which it fails, " + "and it is not categorically eligible through Michigan's TANF non-cash " + "program, whose gross income limit is 200% of the guideline. It is " + "therefore ineligible in every month, and the benefit for the year is " + "$0. Its net income, $1,102 a month after the $209 standard deduction, " + "a $604.59 earned income deduction, the $433.33 child support deduction " + "and a $673.98 excess shelter deduction, would pass the net income " + "test, but without an elderly or disabled member the household must " + "also pass the gross income test or qualify categorically." + ), +} + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def conventions() -> dict[str, dict]: + causes = json.loads((HERE / "root_causes.json").read_text()) + return {k: v for k, v in causes.items() if isinstance(v, dict) and v.get("class") == "convention"} + + +def upstream_fixes() -> dict[str, dict]: + """Engine defects fixed in policyengine-us after the freeze: regenerated, not excluded. + + Each carries its verified fix module (build_records.SWEEP_FOR) and the sweep + that defines what it moves on its own: against the frozen reference, or, for + the SNAP defects, against the SNAP convention (build_records.MEASURED_BY). + """ + import re + + causes = json.loads((HERE / "root_causes.json").read_text()) + source = (HERE / "build_records.py").read_text() + sweep_for = dict(re.findall(r'"(r\d\d_[a-z0-9_]+)": "([a-z0-9_]+)"', source.split("MEASURED_BY")[0])) + measured_by = dict(re.findall(r'"(r\d\d_[a-z0-9_]+)": "([a-z0-9_]+)"', source.split("MEASURED_BY = {")[1].split("}")[0])) + return { + k: {**v, "fix": sweep_for[k], "measured": measured_by.get(k)} + for k, v in causes.items() + if isinstance(v, dict) and v.get("class") == "engine_defect" and v.get("upstream_fixed") + } + + +def load_reform(fix: str): + spec = importlib.util.spec_from_file_location(f"conv_{fix}", FIXES / f"{fix}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.reform + + +def compute_all(system) -> pd.DataFrame: + """Every reference output under one tax-benefit system, as the sweep computes it.""" + sys.path.insert(0, str(HERE / "sweep")) + from sweep import build_situation # noqa: E402 + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + from policyengine_us import Simulation + + meta = json.loads((BUNDLE / "reference_outputs.csv.meta.json").read_text()) + frozen = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index(["scenario_id", "variable"])["value"] + rows = [] + for _, srow in pd.read_csv(BUNDLE / "scenarios.csv").iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + sim = Simulation(tax_benefit_system=system, situation=build_situation(scenario)) + for variable in expand_programs_for_scenario(meta["programs"], scenario): + if (scenario.id, variable) not in frozen.index: + continue + value = _extract_person_value( + sim.calculate(_pe_variable_for_output(variable, "us"), YEAR), scenario, variable + ) + rows.append({"scenario_id": scenario.id, "variable": variable, "value": float(value)}) + return pd.DataFrame(rows).set_index(["scenario_id", "variable"])["value"] + + +def references(args) -> None: + from policyengine_us import CountryTaxBenefitSystem + + convs = conventions() + fixes = upstream_fixes() + reference = pd.read_csv(BUNDLE / "reference_outputs.csv") + excluded = { + (e["scenario_id"], e["variable"]) + for e in json.loads(Path(args.exclusions).read_text())["exclusions"] + } + sources = {name: c["fix"] for name, c in convs.items()} | {name: f["fix"] for name, f in fixes.items()} + reforms = {name: load_reform(fix) for name, fix in sources.items()} + combined = compute_all(CountryTaxBenefitSystem(reform=tuple(reforms.values()))) + # What each source moves on its own (from its verified sweep). + alone, alone_value = {}, {} + for name, fix in sources.items(): + measured = fixes.get(name, {}).get("measured") + sweep = pd.read_csv(HERE / "sweep" / "out" / f"{measured or fix}.csv") + baseline = sweep["baseline"] if measured else sweep["frozen"] + moved = sweep.loc[(sweep["recomputed"] - baseline).abs() > 1e-6] + alone[name] = set(map(tuple, moved[["scenario_id", "variable"]].values)) + alone_value[name] = moved.set_index(["scenario_id", "variable"])["recomputed"].to_dict() + + changed = {name: [] for name in sources} + for idx, row in reference.iterrows(): + key = (row["scenario_id"], row["variable"]) + new = float(combined[key]) + if key in excluded or abs(new - float(row["value"])) <= 1e-6: + continue + owners = [name for name in sources if key in alone[name]] + if not owners: + raise SystemExit(f"{key} changes under the combined sources but under none alone") + # A source that moves an output alone must give the combined value (no + # interaction), or the attribution is wrong. + if len(owners) == 1 and abs(alone_value[owners[0]][key] - new) > 1e-3: + raise SystemExit(f"{key}: combined {new} differs from {owners[0]} alone {alone_value[owners[0]][key]}") + for name in owners: + changed[name].append( + {"scenario_id": key[0], "variable": key[1], "frozen": float(row["value"]), "regenerated": new} + ) + reference.loc[idx, "value"] = new + + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + reference.to_csv(out / "reference_outputs.csv", index=False) + meta = json.loads((BUNDLE / "reference_outputs.csv.meta.json").read_text()) + meta["reference_csv_sha256"] = sha256(out / "reference_outputs.csv") + meta["revisions"] = [ + { + "date": DATE, + "kind": "convention", + "convention": name, + "outputs": c["outputs"], + "rule": f"{RULE} {c['rule']}", + "basis": c["basis"], + "engine_version": ENGINE, + "fix_module": f"{c['fix']}.py", + "fix_module_sha256": sha256(FIXES / f"{c['fix']}.py"), + "applied_together_with": sorted(set(sources) - {name}), + "excluded_outputs_untouched": True, + "changed": changed[name], + } + for name, c in convs.items() + ] + [ + { + "date": DATE, + "kind": "upstream_fix", + "root_cause": name, + "outputs": ", ".join(sorted({c["variable"] for c in changed[name]})) or "none", + "rule": ( + "An engine defect fixed in policyengine-us after the reference freeze is " + f"regenerated with the fix. {f['alternative_reading']}" + ), + "defect": f["defect"], + "basis": f["law"], + "upstream": f["upstream"], + "engine_version": ENGINE, + "fix_module": f"{f['fix']}.py", + "fix_module_sha256": sha256(FIXES / f"{f['fix']}.py"), + "measured_against": "c_snap_hold_fy2026" if f.get("measured") else "frozen", + "applied_together_with": sorted(set(sources) - {name}), + "excluded_outputs_untouched": True, + "changed": changed[name], + } + for name, f in fixes.items() + ] + (out / "reference_outputs.csv.meta.json").write_text(json.dumps(meta, indent=2) + "\n") + + # Engine traces of the changed outputs, for the narratives in step 2. + sys.modules.setdefault("litellm", types.ModuleType("litellm")) + sys.path.insert(0, str(HERE / "sweep")) + from sweep import build_situation # noqa: E402 + + from policybench.case_reference_explanations import _find_target_tree, _render_trace + from policybench.ground_truth import _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + from policyengine_us import Simulation + + system = CountryTaxBenefitSystem(reform=tuple(reforms.values())) + scenarios = pd.read_csv(BUNDLE / "scenarios.csv").set_index("scenario_id") + traces = {} + for name, items in changed.items(): + for item in items: + scenario = scenario_from_dict(json.loads(scenarios.loc[item["scenario_id"], "scenario_json"])) + sim = Simulation(tax_benefit_system=system, situation=build_situation(scenario)) + sim.trace = True + pe_variable = _pe_variable_for_output(item["variable"], "us") + sim.calculate(pe_variable, YEAR) + tree = _find_target_tree(sim.tracer.trees, pe_variable) + traces[f"{item['scenario_id']}|{item['variable']}"] = { + "pe_variable": pe_variable, + "trace": "\n".join(_render_trace(tree)) if tree else "", + } + (out / "reference_traces.json").write_text(json.dumps(traces, indent=1)) + print(f"sha256 {meta['reference_csv_sha256']}") + for name, items in changed.items(): + print(f"{name}: {len(items)} regenerated") + for item in items: + print(f" {item['scenario_id']} {item['variable']:46s} {item['frozen']:>11.2f} -> {item['regenerated']:>11.2f}") + + +def frozen_traces(args) -> None: + """Trace the frozen engine for FROZEN_NARRATIVES into reference_traces.json.""" + sys.modules.setdefault("litellm", types.ModuleType("litellm")) + sys.path.insert(0, str(HERE / "sweep")) + from sweep import build_situation # noqa: E402 + + from policybench.case_reference_explanations import _find_target_tree, _render_trace + from policybench.ground_truth import _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + from policyengine_us import Simulation + + out = Path(args.out_dir) + traces = json.loads((out / "reference_traces.json").read_text()) + scenarios = pd.read_csv(BUNDLE / "scenarios.csv").set_index("scenario_id") + for scenario_id, variable in FROZEN_NARRATIVES: + scenario = scenario_from_dict(json.loads(scenarios.loc[scenario_id, "scenario_json"])) + sim = Simulation(situation=build_situation(scenario)) + sim.trace = True + pe_variable = _pe_variable_for_output(variable, "us") + sim.calculate(pe_variable, YEAR) + tree = _find_target_tree(sim.tracer.trees, pe_variable) + traces[f"frozen|{scenario_id}|{variable}"] = { + "pe_variable": pe_variable, + "trace": "\n".join(_render_trace(tree)) if tree else "", + } + (out / "reference_traces.json").write_text(json.dumps(traces, indent=1)) + print(f"traced {len(FROZEN_NARRATIVES)} frozen outputs") + + +def narratives(args) -> None: + import litellm + + from policybench.case_reference_explanations import ( + MAX_TOKENS, + REFERENCE_MODEL, + TEMPERATURE, + _prompt, + _scenario_summary, + ) + + out = Path(args.out_dir) + meta = json.loads((out / "reference_outputs.csv.meta.json").read_text()) + traces = json.loads((out / "reference_traces.json").read_text()) + scenarios = pd.read_csv(BUNDLE / "scenarios.csv").set_index("scenario_id", drop=False) + explanations = pd.read_csv(args.explanations) + todo = [] + queued = set() + for revision in meta["revisions"]: + for item in revision["changed"]: + key = (item["scenario_id"], item["variable"]) + mask = (explanations["scenario_id"] == item["scenario_id"]) & ( + explanations["variable"] == item["variable"] + ) + current = explanations.loc[mask, "reference_value"] + if len(current) and abs(float(current.iloc[0]) - item["regenerated"]) <= 1e-6: + continue + # An output several sources move is listed in each one's revision; + # it gets one narrative. REGENERATED_NARRATIVES grounds it in the + # engine facts of every source together. + if key in queued: + continue + queued.add(key) + source = ( + {"regenerated_grounding": REGENERATED_NARRATIVES[key]} + if key in REGENERATED_NARRATIVES + else revision + ) + todo.append((source, item)) + frozen = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index(["scenario_id", "variable"])["value"] + # --frozen names the FROZEN_NARRATIVES to rewrite; omitted, all of them; + # given with no names, none (2026-09-24, release dashboard-data-20260922c, + # which rewrote only a regenerated narrative). + only = set(FROZEN_NARRATIVES) if args.frozen is None else set(args.frozen) + for (scenario_id, variable), grounding in FROZEN_NARRATIVES.items(): + if f"{scenario_id}:{variable}" not in only and (scenario_id, variable) not in only: + continue + todo.append( + ( + {"frozen_grounding": grounding}, + { + "scenario_id": scenario_id, + "variable": variable, + "regenerated": float(frozen[(scenario_id, variable)]), + "trace_key": f"frozen|{scenario_id}|{variable}", + }, + ) + ) + + def grounding_for(revision) -> str: + if "frozen_grounding" in revision: + return revision["frozen_grounding"] + if "regenerated_grounding" in revision: + return revision["regenerated_grounding"] + if revision["kind"] == "convention": + return conventions()[revision["convention"]]["grounding"] + cause = json.loads((HERE / "root_causes.json").read_text())[revision["root_cause"]] + return ( + f"The reference applies the fix for an engine defect fixed upstream " + f"({cause['upstream']}): {cause['alternative_reading']}" + ) + + async def one(revision, item, extra=""): + row = scenarios.loc[item["scenario_id"]] + traced = traces[item.get("trace_key", f"{item['scenario_id']}|{item['variable']}")] + trace = traced["trace"] + prompt = _prompt( + "us", + _scenario_summary(row), + item["variable"], + traced["pe_variable"], + item["regenerated"], + YEAR, + trace, + grounding=grounding_for(revision) + extra, + ) + response = await litellm.acompletion( + model=REFERENCE_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + ) + text = response.choices[0].message.content.strip() + lines = text.split("\n") + if lines and lines[0].lstrip().startswith("#"): + text = "\n".join(lines[1:]).strip() + return item, text, len(trace.splitlines()) + + async def run_all(): + return await asyncio.gather(*(one(c, item) for c, item in todo)) + + results = asyncio.run(run_all()) + for index, (item, text, n_lines) in enumerate(results): + key = (item["scenario_id"], item["variable"]) + required = FROZEN_REQUIRED.get(key, []) or REGENERATED_REQUIRED.get(key, []) + for attempt in range(3): + missing = [figure for figure in required if figure not in text] + if not missing: + break + extra = ( + " The narrative must state these figures and the fact behind each: " + + ", ".join(missing) + + "." + ) + item, text, n_lines = asyncio.run(one(todo[index][0], item, extra)) + missing = [figure for figure in required if figure not in text] + if missing: + raise SystemExit(f"{key}: narrative still omits {missing}") + if key in HAND_CORRECTED: + print(f"--- draft replaced by HAND_CORRECTED: {key[0]} {key[1]}\n{text}\n") + text = HAND_CORRECTED[key] + assert not [figure for figure in required if figure not in text], key + results[index] = (item, text, n_lines) + for item, text, n_lines in results: + mask = (explanations["scenario_id"] == item["scenario_id"]) & ( + explanations["variable"] == item["variable"] + ) + explanations.loc[mask, "reference_value"] = item["regenerated"] + explanations.loc[mask, "trace_lines"] = n_lines + explanations.loc[mask, "explanation"] = text + explanations.loc[mask, "error"] = pd.NA + print(f"--- {item['scenario_id']} {item['variable']} ({item['regenerated']:.2f})\n{text}\n") + # An excluded output keeps its frozen reference, so its narrative is the + # frozen bundle's unless FROZEN_NARRATIVES rewrote it above. + excluded = { + (e["scenario_id"], e["variable"]) + for e in json.loads(Path(args.exclusions).read_text())["exclusions"] + } + pristine = pd.read_csv(PRISTINE_EXPLANATIONS).set_index(["scenario_id", "variable"]) + restored = [] + for idx, row in explanations.iterrows(): + key = (row["scenario_id"], row["variable"]) + if key not in excluded or key in FROZEN_NARRATIVES: + continue + if abs(float(row["reference_value"]) - float(frozen[key])) <= 1e-6: + continue + for column in explanations.columns: + if column in pristine.columns: + explanations.loc[idx, column] = pristine.loc[key, column] + restored.append(f"{key[0]}:{key[1]}") + explanations.to_csv(out / "us_case_reference_explanations.csv", index=False) + print(f"rewrote {len(todo)} narratives; restored the frozen narrative of {restored}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("step", choices=["references", "frozen-traces", "narratives"]) + parser.add_argument("--exclusions") + parser.add_argument("--explanations") + # Rewrite only these FROZEN_NARRATIVES (scenario_id:variable); omitted, all; + # given with no names, none. + parser.add_argument("--frozen", nargs="*") + parser.add_argument("--out-dir", required=True) + args = parser.parse_args() + {"references": references, "frozen-traces": frozen_traces, "narratives": narratives}[args.step](args) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-22/scripts/run_one.sh b/reference_audit/2026-09-22/scripts/run_one.sh new file mode 100755 index 00000000..fb6cfbb1 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/run_one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Run one fix sweep: run_one.sh +# PB_WORKTREE names the branch checkout whose policybench code the sweep imports +# (2026-09-22/23: policybench-wt/opus55; 2026-09-24: policybench-wt/r33-20260922b, +# then policybench-wt/r33-20260922c once #9586 merged). +WT="${PB_WORKTREE:-/Users/maxghenis/PolicyEngine/policybench-wt/r33-20260922c}" +cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep || exit 1 +PYTHONPATH="$WT" OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + ../.venv-pe1755/bin/python sweep.py --fix "fixes/$1.py" --out "out/$1.csv" > "out/$1.log" 2>&1 +echo "$1 rc=$? $(tail -1 "out/$1.log")" diff --git a/reference_audit/2026-09-22/scripts/sweep.py b/reference_audit/2026-09-22/scripts/sweep.py new file mode 100644 index 00000000..a226f1c4 --- /dev/null +++ b/reference_audit/2026-09-22/scripts/sweep.py @@ -0,0 +1,141 @@ +"""Recompute every PolicyBench reference under one fix and list the outputs that move. + +A fix is a Python module that defines either or both of: + reform a policyengine_core Reform subclass (or None) + patch(situation, scenario) returns an edited copy of the situation dict +and optionally FIX_ID and DESCRIPTION strings. + +Run with the policyengine-us 1.755.4 triage venv: + cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ + ../.venv-pe1755/bin/python sweep.py --fix fixes/r01_ira_compensation.py --out out/r01.csv + (no --fix: the baseline, which must reproduce all 1,984 frozen references) + +The output CSV has one row per reference output: scenario_id, variable, frozen, +recomputed, delta, moved. An amount output moves when |delta| > 1 (the exact-match +tolerance); a binary output moves when the flag flips. The script prints the +moved rows and a summary line. +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +from policybench.ground_truth import _extract_person_value, _pe_variable_for_output +from policybench.scenarios import scenario_from_dict +from policybench.spec import expand_programs_for_scenario + +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) +# The frozen v1.1 bundle (reference_outputs.csv sha256 b9136a15...), read-only. Not +# adds202609/publish/: the fold overwrites that dir with the regenerated references. +YEAR = 2026 +BINARY_SUFFIXES = ("_eligible",) +# The v1.1 reference run renamed this input for policyengine-us 1.755.4 +# (results/local/v1_1_runbook.md step 3; v1_1/us/scenarios_patched.csv). +RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} + + +def build_situation(scenario) -> dict: + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in RENAME.items(): + if old in person: + person[new] = person.pop(old) + return situation + + +def load_fix(path: str | None): + if not path: + return None, None, "baseline" + spec = importlib.util.spec_from_file_location("fix_module", path) + module = importlib.util.module_from_spec(spec) + sys.modules["fix_module"] = module + spec.loader.exec_module(module) + return ( + getattr(module, "reform", None), + getattr(module, "patch", None), + getattr(module, "FIX_ID", Path(path).stem), + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--fix") + parser.add_argument("--out", required=True) + parser.add_argument("--scenarios", nargs="*", help="restrict to these scenario ids") + args = parser.parse_args() + + from policyengine_us import CountryTaxBenefitSystem, Simulation + + reform, patch, fix_id = load_fix(args.fix) + # Build the reformed system once and share it across households: passing + # reform= to each Simulation re-applies it to a fresh copy every time. + system = CountryTaxBenefitSystem(reform=reform) if reform is not None else None + meta = json.loads((BUNDLE / "reference_outputs.csv.meta.json").read_text()) + programs = meta["programs"] + frozen = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index( + ["scenario_id", "variable"] + )["value"] + scenarios = pd.read_csv(BUNDLE / "scenarios.csv") + if args.scenarios: + scenarios = scenarios[scenarios["scenario_id"].isin(args.scenarios)] + + rows = [] + for _, srow in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + situation = build_situation(scenario) + if patch is not None: + situation = patch(copy.deepcopy(situation), scenario) + sim = ( + Simulation(tax_benefit_system=system, situation=situation) + if system is not None + else Simulation(situation=situation) + ) + for variable in expand_programs_for_scenario(programs, scenario): + key = (scenario.id, variable) + if key not in frozen.index: + continue + pe_variable = _pe_variable_for_output(variable, "us") + value = float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, variable) + ) + ref = float(frozen[key]) + delta = value - ref + binary = variable.endswith(BINARY_SUFFIXES) + moved = (round(value) != round(ref)) if binary else abs(delta) > 1.0 + rows.append( + { + "fix": fix_id, + "scenario_id": scenario.id, + "state": srow["state"], + "variable": variable, + "frozen": ref, + "recomputed": value, + "delta": delta, + "moved": moved, + } + ) + out = pd.DataFrame(rows) + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + out.to_csv(args.out, index=False) + moved = out[out["moved"]] + with pd.option_context("display.width", 200, "display.max_rows", 500): + print(moved[["scenario_id", "state", "variable", "frozen", "recomputed", "delta"]].to_string(index=False)) + print( + f"SUMMARY fix={fix_id} outputs={len(out)} moved={len(moved)} " + f"small_nonzero_deltas={int(((out['delta'].abs() > 1e-6) & ~out['moved']).sum())}" + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-22/snap_net_income_sensitivity.csv b/reference_audit/2026-09-22/snap_net_income_sensitivity.csv new file mode 100644 index 00000000..f9522602 --- /dev/null +++ b/reference_audit/2026-09-22/snap_net_income_sensitivity.csv @@ -0,0 +1,17 @@ +scenario_id,variable,state,convention,floor,nearest,cents +scenario_008,snap,NJ,15121.1982421875,15120.0,15108.0,15108.0 +scenario_012,snap,MS,4894.79931640625,4884.0,4884.0,4884.0 +scenario_023,snap,CA,422.3997802734375,420.0,408.0,420.0 +scenario_027,snap,CT,286.0799865722656,288.0,288.0,288.0 +scenario_030,snap,TX,286.0799865722656,288.0,288.0,288.0 +scenario_038,snap,LA,7215.59912109375,7212.0,7212.0,7212.0 +scenario_045,snap,MI,286.0799865722656,0.0,0.0,0.0 +scenario_054,snap,NC,6068.400390625,6060.0,6060.0,6060.0 +scenario_057,snap,LA,2628.0,2628.0,2628.0,2616.0 +scenario_073,snap,MI,286.0799865722656,288.0,288.0,288.0 +scenario_079,snap,AZ,2386.800048828125,2376.0,2376.0,2376.0 +scenario_100,snap,MT,8566.7998046875,8556.0,8556.0,8556.0 +scenario_108,snap,WI,286.0799865722656,288.0,288.0,288.0 +scenario_109,snap,FL,7935.59912109375,7932.0,7932.0,7932.0 +scenario_112,snap,TX,286.0799865722656,288.0,288.0,288.0 +scenario_118,snap,NY,2877.600341796875,2868.0,2868.0,2868.0 diff --git a/reference_audit/2026-09-22/sweep_moves.csv b/reference_audit/2026-09-22/sweep_moves.csv new file mode 100644 index 00000000..505ff485 --- /dev/null +++ b/reference_audit/2026-09-22/sweep_moves.csv @@ -0,0 +1,166 @@ +root_cause,class,fix_module,measured_against,scenario_id,state,variable,frozen,baseline,recomputed,delta,moved_over_1 +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_005,CA,state_income_tax_before_refundable_credits,41051.511719,41051.511719,41219.867188,168.355469,True +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_022,CA,federal_income_tax_before_refundable_credits,11131.327148,11131.327148,11113.570312,-17.756836,True +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_022,CA,state_income_tax_before_refundable_credits,2439.650146,2439.650146,2505.870117,66.219971,True +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_023,CA,state_income_tax_before_refundable_credits,6.798477,6.798477,12.776718,5.978241,True +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_023,CA,state_refundable_credits,148.310867,148.310867,141.889847,-6.421021,True +c_ca_hold_2025,convention,r19_ca_convention.py,frozen,scenario_099,CA,state_income_tax_before_refundable_credits,4493.743164,4493.743164,4640.777344,147.03418,True +c_id_hold_2025,convention,r19_id_convention.py,frozen,scenario_007,ID,state_income_tax_before_refundable_credits,755.77832,755.77832,761.557007,5.778687,True +c_id_hold_2025,convention,r19_id_convention.py,frozen,scenario_053,ID,state_income_tax_before_refundable_credits,2435.27832,2435.27832,2441.057129,5.778809,True +c_id_hold_2025,convention,r19_id_convention.py,frozen,scenario_076,ID,state_income_tax_before_refundable_credits,6806.786621,6806.786621,6818.344238,11.557617,True +c_irs_sales_tax_2025,convention,r19_irs_sales_tax_convention.py,frozen,scenario_000,TX,federal_income_tax_before_refundable_credits,2883.493652,2883.493652,2906.453613,22.959961,True +c_irs_sales_tax_2025,convention,r19_irs_sales_tax_convention.py,frozen,scenario_020,TX,federal_income_tax_before_refundable_credits,68056.710938,68056.710938,68334.734375,278.023438,True +c_md_2026,convention,r19_md_convention.py,frozen,scenario_068,MD,state_income_tax_before_refundable_credits,1252.967773,1252.967773,1255.342773,2.375,True +c_md_2026,convention,r19_md_convention.py,frozen,scenario_078,MD,federal_income_tax_before_refundable_credits,24772.693359,24772.693359,24780.613281,7.919922,True +c_mi_published_2026,convention,r19_mi_convention.py,frozen,scenario_045,MI,state_income_tax_before_refundable_credits,1288.837524,1288.837524,1290.962524,2.125,True +c_mn_published_2026,convention,r19_mn_convention.py,frozen,scenario_122,MN,state_income_tax_before_refundable_credits,1028.216553,1028.216553,1025.541504,-2.675049,True +c_mo_published_2026,convention,r19_mo_convention.py,frozen,scenario_093,MO,state_income_tax_before_refundable_credits,3389.650879,3389.650879,3388.245605,-1.405273,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_008,NJ,snap,15246.905273,15246.905273,15121.198242,-125.707031,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_012,MS,snap,4952.089355,4952.089355,4894.799316,-57.290039,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_023,CA,snap,461.339722,461.339722,422.39978,-38.939941,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_027,CT,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_030,TX,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_038,LA,snap,7286.944336,7286.944336,7215.599121,-71.345215,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_043,CO,snap,3596.039795,3596.039795,3576.0,-20.039795,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_045,MI,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_054,NC,snap,6125.688965,6125.688965,6068.400391,-57.288574,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_057,LA,snap,2669.217041,2669.217041,2628.0,-41.217041,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_066,VA,snap,3596.039795,3596.039795,3576.0,-20.039795,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_073,MI,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_079,AZ,snap,2428.017334,2428.017334,2386.800049,-41.217285,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_080,PA,snap,3596.039795,3596.039795,3576.0,-20.039795,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_100,MT,snap,8625.889648,8625.889648,8566.799805,-59.089844,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_108,WI,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_109,FL,snap,8020.554199,8020.554199,7935.599121,-84.955078,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_112,TX,snap,287.683167,287.683167,286.079987,-1.60318,True +c_snap_hold_fy2026,convention,r13_hold_fy2026_v3.py,frozen,scenario_118,NY,snap,2903.94043,2903.94043,2877.600342,-26.340088,True +c_wi_published_2026,convention,r19_wi_convention.py,frozen,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,284.740906,279.234802,-5.506104,True +c_wi_published_2026,convention,r19_wi_convention.py,frozen,scenario_064,WI,state_income_tax_before_refundable_credits,4605.99707,4605.99707,4598.476074,-7.520996,True +c_wi_published_2026,convention,r19_wi_convention.py,frozen,scenario_091,WI,state_income_tax_before_refundable_credits,843.661499,843.661499,838.155457,-5.506042,True +c_wi_published_2026+r04_capital_gain_distributions,baseline,cwi_plus_r04.py,frozen,scenario_042,WI,federal_income_tax_before_refundable_credits,1979.158325,1979.158325,2361.964844,382.806519,True +c_wi_published_2026+r04_capital_gain_distributions,baseline,cwi_plus_r04.py,frozen,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,284.740906,464.1828,179.441895,True +c_wi_published_2026+r04_capital_gain_distributions,baseline,cwi_plus_r04.py,frozen,scenario_051,LA,state_income_tax_before_refundable_credits,814.950012,814.950012,820.349976,5.399963,True +c_wi_published_2026+r04_capital_gain_distributions,baseline,cwi_plus_r04.py,frozen,scenario_064,WI,state_income_tax_before_refundable_credits,4605.99707,4605.99707,4598.476074,-7.520996,True +c_wi_published_2026+r04_capital_gain_distributions,baseline,cwi_plus_r04.py,frozen,scenario_091,WI,state_income_tax_before_refundable_credits,843.661499,843.661499,895.81311,52.151611,True +r01_ira_compensation,engine_defect,r01_ira_compensation_v2.py,frozen,scenario_003,TX,federal_income_tax_before_refundable_credits,22154.699219,22154.699219,22400.654297,245.955078,True +r01_ira_compensation,engine_defect,r01_ira_compensation_v2.py,frozen,scenario_064,WI,federal_income_tax_before_refundable_credits,4439.289551,4439.289551,4441.455078,2.165527,True +r01_ira_compensation,engine_defect,r01_ira_compensation_v2.py,frozen,scenario_064,WI,state_income_tax_before_refundable_credits,4605.99707,4605.99707,4607.14209,1.14502,True +r01_ira_compensation,engine_defect,r01_ira_compensation_v2.py,frozen,scenario_085,PA,federal_income_tax_before_refundable_credits,2322.76001,2322.76001,2322.848633,0.088623,False +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_005,CA,federal_income_tax_before_refundable_credits,106505.898438,106505.898438,107198.335938,692.4375,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_005,CA,state_income_tax_before_refundable_credits,41051.511719,41051.511719,41276.824219,225.3125,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_049,NH,federal_income_tax_before_refundable_credits,30543.908203,30543.908203,30702.589844,158.681641,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_052,TX,federal_income_tax_before_refundable_credits,104211.40625,104211.40625,104225.257812,13.851562,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_082,NY,federal_income_tax_before_refundable_credits,9563.052734,9563.052734,9564.916992,1.864258,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_082,NY,state_income_tax_before_refundable_credits,5598.552246,5598.552246,5599.143555,0.591309,False +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_099,CA,federal_income_tax_before_refundable_credits,10679.75,10679.75,10711.484375,31.734375,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_099,CA,state_income_tax_before_refundable_credits,4493.743164,4493.743164,4505.339844,11.59668,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_110,OH,federal_income_tax_before_refundable_credits,23897.443359,23897.443359,23941.082031,43.638672,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_110,OH,state_income_tax_before_refundable_credits,4057.472168,4057.472168,4062.927002,5.454834,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_120,CT,federal_income_tax_before_refundable_credits,40021.816406,40021.816406,40028.320312,6.503906,True +r02_ira_219g,engine_defect,r02_ira_219g_v2.py,frozen,scenario_120,CT,state_income_tax_before_refundable_credits,11917.183594,11917.183594,11919.057617,1.874023,True +r03_estate_income,engine_defect,r03_estate_income__qbi_false_v2.py,frozen,scenario_039,VA,federal_income_tax_before_refundable_credits,1345.510498,1345.510498,8596.02832,7250.517822,True +r03_estate_income,engine_defect,r03_estate_income__qbi_false_v2.py,frozen,scenario_039,VA,state_income_tax_before_refundable_credits,514.498413,514.498413,1975.798218,1461.299805,True +r03_estate_income,engine_defect,r03_estate_income__qbi_false_v2.py,frozen,scenario_110,OH,federal_income_tax_before_refundable_credits,23897.443359,23897.443359,25700.166016,1802.722656,True +r03_estate_income,engine_defect,r03_estate_income__qbi_false_v2.py,frozen,scenario_110,OH,state_income_tax_before_refundable_credits,4057.472168,4057.472168,4231.353027,173.880859,True +r04_capital_gain_distributions,engine_defect,r04_capital_gain_distributions.py,frozen,scenario_042,WI,federal_income_tax_before_refundable_credits,1979.158325,1979.158325,2361.964844,382.806519,True +r04_capital_gain_distributions,engine_defect,r04_capital_gain_distributions.py,frozen,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,284.740906,469.688782,184.947876,True +r04_capital_gain_distributions,engine_defect,r04_capital_gain_distributions.py,frozen,scenario_051,LA,state_income_tax_before_refundable_credits,814.950012,814.950012,820.349976,5.399963,True +r04_capital_gain_distributions,engine_defect,r04_capital_gain_distributions.py,frozen,scenario_091,WI,state_income_tax_before_refundable_credits,843.661499,843.661499,901.319092,57.657593,True +r05_nj_worker_ui,engine_defect,r05_nj_worker_ui_v2.py,frozen,scenario_008,NJ,payroll_tax,2162.76001,2162.76001,2276.659912,113.899902,True +r06_wi_act15_before_refundable,engine_defect,r06_wi_act15_before_refundable_v2.py,frozen,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,284.740906,0.0,-284.740906,True +r07_idaho_health_premiums,engine_defect,r07_idaho_health_premiums_v2.py,frozen,scenario_007,ID,state_income_tax_before_refundable_credits,755.77832,755.77832,645.53833,-110.23999,True +r07_idaho_health_premiums,engine_defect,r07_idaho_health_premiums_v2.py,frozen,scenario_053,ID,state_income_tax_before_refundable_credits,2435.27832,2435.27832,2170.27832,-265.0,True +r08_eitc_earned_income_deferrals,engine_defect,r08_eitc_earned_income_deferrals_v2.py,frozen,scenario_023,CA,state_refundable_credits,148.310867,148.310867,174.968216,26.657349,True +r08_eitc_earned_income_deferrals,engine_defect,r08_eitc_earned_income_deferrals_v2.py,frozen,scenario_100,MT,federal_refundable_credits,2878.09668,2878.09668,822.288269,-2055.808411,True +r08_eitc_earned_income_deferrals,engine_defect,r08_eitc_earned_income_deferrals_v2.py,frozen,scenario_100,MT,state_refundable_credits,473.177704,473.177704,164.457657,-308.720047,True +r08_eitc_earned_income_deferrals,engine_defect,r08_eitc_earned_income_deferrals_v2.py,frozen,scenario_119,VA,federal_refundable_credits,2854.008789,2854.008789,3366.07373,512.064941,True +r08_eitc_earned_income_deferrals,engine_defect,r08_eitc_earned_income_deferrals_v2.py,frozen,scenario_119,VA,state_income_tax_before_refundable_credits,1859.09436,1859.09436,1756.681396,-102.412964,True +r09_ny_rptc_rent_cap,engine_defect,r09_ny_rptc_rent_cap_v2.py,frozen,scenario_104,NY,state_refundable_credits,375.0,375.0,0.0,-375.0,True +r10_wi_homestead_income,unlisted_input,r10_wi_homestead_income.py,frozen,scenario_108,WI,state_refundable_credits,271.391602,271.391602,0.0,-271.391602,True +r11_ca_itemized_conformity,engine_defect,r11_ca_itemized_conformity_v2.py,frozen,scenario_005,CA,state_income_tax_before_refundable_credits,41051.511719,41051.511719,40522.707031,-528.804688,True +r11_ca_itemized_conformity,engine_defect,r11_ca_itemized_conformity_v2.py,frozen,scenario_022,CA,state_income_tax_before_refundable_credits,2439.650146,2439.650146,1902.585449,-537.064697,True +r11_ca_itemized_conformity,engine_defect,r11_ca_itemized_conformity_v2.py,frozen,scenario_099,CA,state_income_tax_before_refundable_credits,4493.743164,4493.743164,4429.90918,-63.833984,True +r14_unlisted_weekly_hours_v2,unlisted_input,r14_unlisted_weekly_hours_v2_v2.py,frozen,scenario_056,NJ,snap,1140.0,1140.0,0.0,-1140.0,True +r14_unlisted_weekly_hours_v2,unlisted_input,r14_unlisted_weekly_hours_v2_v2.py,frozen,scenario_112,TX,snap,287.683167,287.683167,0.0,-287.683167,True +r15_snap_mortgage_interest,unlisted_input,r15_snap_mortgage_interest_v2.py,frozen,scenario_056,NJ,snap,1140.0,1140.0,1956.239868,816.239868,True +r15_snap_mortgage_interest,unlisted_input,r15_snap_mortgage_interest_v2.py,frozen,scenario_118,NY,snap,2903.94043,2903.94043,3596.039795,692.099365,True +r16_survivor_benefits_federal,unlisted_input,r16_survivor_benefits_federal_v2.py,frozen,scenario_062,FL,federal_income_tax_before_refundable_credits,0.0,0.0,4328.386719,4328.386719,True +r16_survivor_benefits_federal,unlisted_input,r16_survivor_benefits_federal_v2.py,frozen,scenario_108,WI,state_refundable_credits,271.391602,271.391602,0.0,-271.391602,True +r17_caleitc_agi_comparison,engine_defect,r17_caleitc_agi_comparison.py,frozen,scenario_023,CA,state_refundable_credits,148.310867,148.310867,99.460068,-48.8508,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_000,TX,federal_income_tax_before_refundable_credits,2883.493652,2883.493652,2894.623535,11.129883,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_005,CA,state_income_tax_before_refundable_credits,41051.511719,41051.511719,41219.867188,168.355469,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_007,ID,state_income_tax_before_refundable_credits,755.77832,755.77832,761.557007,5.778687,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_008,NJ,snap,15246.905273,15246.905273,15121.198242,-125.707031,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_012,MS,snap,4952.089355,4952.089355,4894.799316,-57.290039,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_020,TX,federal_income_tax_before_refundable_credits,68056.710938,68056.710938,68113.203125,56.492188,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_022,CA,federal_income_tax_before_refundable_credits,11131.327148,11131.327148,11113.570312,-17.756836,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_022,CA,state_income_tax_before_refundable_credits,2439.650146,2439.650146,2505.870117,66.219971,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_023,CA,snap,461.339722,461.339722,422.39978,-38.939941,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_023,CA,state_income_tax_before_refundable_credits,6.798477,6.798477,12.776718,5.978241,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_023,CA,state_refundable_credits,148.310867,148.310867,140.9216,-7.389267,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_027,CT,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_030,TX,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_032,MN,state_refundable_credits,2187.600098,2187.600098,2129.199951,-58.400146,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_038,LA,snap,7286.944336,7286.944336,7215.599121,-71.345215,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,284.740906,303.714417,18.973511,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_043,CO,snap,3596.039795,3596.039795,3576.0,-20.039795,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_045,MI,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_045,MI,state_income_tax_before_refundable_credits,1288.837524,1288.837524,1295.212524,6.375,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_053,ID,state_income_tax_before_refundable_credits,2435.27832,2435.27832,2441.057129,5.778809,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_054,NC,snap,6125.688965,6125.688965,6068.400391,-57.288574,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_057,LA,snap,2669.217041,2669.217041,2628.0,-41.217041,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_064,WI,state_income_tax_before_refundable_credits,4605.99707,4605.99707,4660.806641,54.80957,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_066,VA,snap,3596.039795,3596.039795,3576.0,-20.039795,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_068,MD,state_income_tax_before_refundable_credits,1252.967773,1252.967773,1255.342773,2.375,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_070,IL,state_income_tax_before_refundable_credits,1650.561279,1650.561279,1654.273804,3.712524,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_073,MI,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_076,ID,state_income_tax_before_refundable_credits,6806.786621,6806.786621,6818.344238,11.557617,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_078,MD,federal_income_tax_before_refundable_credits,24772.693359,24772.693359,24771.373047,-1.320312,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_079,AZ,snap,2428.017334,2428.017334,2386.800049,-41.217285,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_080,PA,snap,3596.039795,3596.039795,3576.0,-20.039795,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_091,WI,state_income_tax_before_refundable_credits,843.661499,843.661499,862.635071,18.973572,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_093,MO,state_income_tax_before_refundable_credits,3389.650879,3389.650879,3397.625488,7.974609,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_099,CA,state_income_tax_before_refundable_credits,4493.743164,4493.743164,4640.777344,147.03418,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_100,MT,snap,8625.889648,8625.889648,8566.799805,-59.089844,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_108,WI,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_109,FL,snap,8020.554199,8020.554199,7935.599121,-84.955078,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_112,TX,snap,287.683167,287.683167,286.079987,-1.60318,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_118,NY,snap,2903.94043,2903.94043,2877.600342,-26.340088,True +r18_hold_all_projections,screen,r18_hold_all_projections.py,frozen,scenario_122,MN,state_income_tax_before_refundable_credits,1028.216553,1028.216553,1044.266479,16.049927,True +r20_adult_dependent_child,unlisted_input,r067_adult_dependent_relationship.py,frozen,scenario_064,WI,dependent1_medicaid_eligible,1.0,1.0,0.0,-1.0,True +r20_adult_dependent_child,unlisted_input,r067_adult_dependent_relationship.py,frozen,scenario_067,IN,dependent1_medicaid_eligible,1.0,1.0,0.0,-1.0,True +r21_adult_dependent_nonchild,unlisted_input,r067_adult_dependent_nonchild.py,frozen,scenario_064,WI,dependent2_medicaid_eligible,0.0,0.0,1.0,1.0,True +r22_ma_part_a_loss_offset,engine_defect,ma_part_a_loss_offset.py,frozen,scenario_081,MA,state_income_tax_before_refundable_credits,8238.40625,8238.40625,8232.900391,-5.505859,True +r23_ma_interest_source,unlisted_input,ma_part_a_ordinary_interest.py,frozen,scenario_081,MA,state_income_tax_before_refundable_credits,8238.40625,8238.40625,8230.100586,-8.305664,True +r24_disability_benefits_taxability,unlisted_input,r24_disability_benefits_taxable.py,frozen,scenario_002,WA,federal_income_tax_before_refundable_credits,0.0,0.0,1890.300049,1890.300049,True +r24_disability_benefits_taxability,unlisted_input,r24_disability_benefits_taxable.py,frozen,scenario_107,OH,federal_income_tax_before_refundable_credits,0.0,0.0,211.232422,211.232422,True +r24_disability_benefits_taxability,unlisted_input,r24_disability_benefits_taxable.py,frozen,scenario_121,SC,federal_income_tax_before_refundable_credits,0.0,0.0,848.184937,848.184937,True +r24_disability_benefits_taxability,unlisted_input,r24_disability_benefits_taxable.py,frozen,scenario_121,SC,state_income_tax_before_refundable_credits,0.0,0.0,212.233505,212.233505,True +r25_niit_in_federal_output,unlisted_input,r25_niit_excluded.py,frozen,scenario_005,CA,federal_income_tax_before_refundable_credits,106505.898438,106505.898438,100783.921875,-5721.976562,True +r25_niit_in_federal_output,unlisted_input,r25_niit_excluded.py,frozen,scenario_020,TX,federal_income_tax_before_refundable_credits,68056.710938,68056.710938,67834.070312,-222.640625,True +r25_niit_in_federal_output,unlisted_input,r25_niit_excluded.py,frozen,scenario_052,TX,federal_income_tax_before_refundable_credits,104211.40625,104211.40625,103992.984375,-218.421875,True +r25_niit_in_federal_output,unlisted_input,r25_niit_excluded.py,frozen,scenario_120,CT,federal_income_tax_before_refundable_credits,40021.816406,40021.816406,39858.417969,-163.398438,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_008,NJ,snap,15246.905273,15121.198242,15120.0,-1.198242,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_012,MS,snap,4952.089355,4894.799316,4884.0,-10.799316,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_023,CA,snap,461.339722,422.39978,420.0,-2.39978,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_038,LA,snap,7286.944336,7215.599121,7212.0,-3.599121,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_054,NC,snap,6125.688965,6068.400391,6060.0,-8.400391,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_079,AZ,snap,2428.017334,2386.800049,2376.0,-10.800049,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_100,MT,snap,8625.889648,8566.799805,8556.0,-10.799805,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_109,FL,snap,8020.554199,7935.599121,7932.0,-3.599121,True +r26_snap_contribution_rounding,engine_defect,r26_snap_contribution_rounding.py,c_snap_hold_fy2026,scenario_118,NY,snap,2903.94043,2877.600342,2868.0,-9.600342,True +r27_snap_net_income_rounding,engine_defect,r27_snap_net_income_rounding.py,c_snap_hold_fy2026,scenario_008,NJ,snap,15246.905273,15121.198242,15117.598633,-3.599609,True +r27_snap_net_income_rounding,engine_defect,r27_snap_net_income_rounding.py,c_snap_hold_fy2026,scenario_012,MS,snap,4952.089355,4894.799316,4891.200195,-3.599121,True +r27_snap_net_income_rounding,engine_defect,r27_snap_net_income_rounding.py,c_snap_hold_fy2026,scenario_023,CA,snap,461.339722,422.39978,418.799927,-3.599854,True +r27_snap_net_income_rounding,engine_defect,r27_snap_net_income_rounding.py,c_snap_hold_fy2026,scenario_038,LA,snap,7286.944336,7215.599121,7212.0,-3.599121,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_027,CT,snap,287.683167,286.079987,288.0,1.920013,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_030,TX,snap,287.683167,286.079987,288.0,1.920013,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_045,MI,snap,287.683167,286.079987,288.0,1.920013,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_073,MI,snap,287.683167,286.079987,288.0,1.920013,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_108,WI,snap,287.683167,286.079987,288.0,1.920013,True +r28_snap_min_allotment_rounding,engine_defect,r28_snap_min_allotment_rounding.py,c_snap_hold_fy2026,scenario_112,TX,snap,287.683167,286.079987,288.0,1.920013,True +r30_snap_heat_and_eat_sua,engine_defect,r30_snap_heat_and_eat_sua.py,c_snap_hold_fy2026,scenario_080,PA,snap,3596.039795,3576.0,3244.799561,-331.200439,True +r30_snap_heat_and_eat_sua,engine_defect,r30_snap_heat_and_eat_sua.py,c_snap_hold_fy2026,scenario_100,MT,snap,8625.889648,8566.799805,6936.0,-1630.799805,True +r32_wi_capital_gain_distributions,engine_defect,r32_wi_capital_gain_distributions.py,c_wi_published_2026+r04_capital_gain_distributions,scenario_042,WI,state_income_tax_before_refundable_credits,284.740906,464.1828,408.698425,-55.484375,True +r32_wi_capital_gain_distributions,engine_defect,r32_wi_capital_gain_distributions.py,c_wi_published_2026+r04_capital_gain_distributions,scenario_091,WI,state_income_tax_before_refundable_credits,843.661499,895.81311,878.515747,-17.297363,True +r33_snap_child_support_treatment,engine_defect,r33_snap_child_support_treatment.py,c_snap_hold_fy2026,scenario_045,MI,snap,287.683167,286.079987,0.0,-286.079987,True diff --git a/reference_audit/2026-09-22/verification/v1_r01.md b/reference_audit/2026-09-22/verification/v1_r01.md new file mode 100644 index 00000000..f72e7486 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v1_r01.md @@ -0,0 +1,35 @@ +## r01_ira_compensation + +**Verdict: accept_with_changes.** All three proposed exclusions are confirmed. The original sweep reproduces exactly, but the module over-applies its dependent restriction to actual contributions. Its compensation cap also exposes an existing scaling error for positive requested amounts below $1. The corrected module separates deductions from contributions, fixes that denominator, and confines activation to 2026. These corrections do not change this bundle's exclusions. + +**What I read and checked.** I read the complete original `sweep/fixes/r01_ira_compensation.py`, the complete sweep harness, both 003/064 federal investigations in the original `triage/verdicts.json`, and the relevant installed 1.755.4 formulas/parameters. Installed paths below are relative to `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +- The minimum of compensation and the dollar ceiling is required by §219(b)(1); §219(a) assigns the deduction to the individual. Section 219(c) allows only a lower-compensation joint spouse to use remaining spousal compensation after that spouse's IRA contributions. The original `comp < spouse_comp`, joint test, and subtraction of the higher earner's capped combined contributions implement these rules for the stated inputs. Section 219(f)(1) excludes pensions/annuities and includes qualifying earned income; (b)(5)(B) sets the age-50 catch-up test. [26 USC 219](https://www.law.cornell.edu/uscode/text/26/219). +- The combined traditional/Roth ceiling follows [26 USC 408A(c)(2)](https://www.law.cornell.edu/uscode/text/26/408A#c_2). Proportional allocation between desired traditional and Roth amounts is the inherited engine convention, not a statutory allocation requirement. All compensation reductions in this bundle are to zero, so that convention does not affect any moved result. +- The four self-employment sources match `variables/gov/irs/tax/self_employment/taxable_self_employment_income.py`. Deductible SE tax and retirement-plan contributions reduce earned income under [26 USC 401(c)(2)(A)(v)–(vi)](https://www.law.cornell.edu/uscode/text/26/401#c_2). I checked the two person-level deduction formulas and `irs_employment_income.py`, which deducts pretax contributions from wages and floors wages at zero. +- [IRS Publication 590-A, “What Is Compensation?”](https://www.irs.gov/publications/p590a) supports using W-2 box 1 after the box 11 adjustment, reducing net SE earnings by the retirement-plan and deductible SE-tax deductions, and not offsetting wages with an SE loss. Its compensation discussion also includes taxable pre-2019 alimony, combat pay, and certain graduate/postdoctoral payments. Those exceptions are not implemented by this module. The only bundle alimony-income recipient is 056, which has no IRA request; no bundle person supplied combat/stipend/fellowship income. I do not treat the module as a complete general IRA implementation. +- The only newly hard-coded dollar parameters, **$7,500** and **$1,100**, match [Notice 2025-67, page 4](https://www.irs.gov/pub/irs-drop/n-25-67.pdf). I also read `parameters/gov/irs/gross_income/retirement_contributions/catch_up/age_threshold.yaml`, which supplies **50**. The original parameter update correctly applies those amounts only during 2026. No contributor in this bundle approaches even the old ceiling: the largest requested combined amount is $2,852, so the dollar update causes none of the exclusions. + +**Scope and correction.** I scanned all **100 scenarios / 177 people**, including all **29 people with IRA requests in 24 households**, using `r01_trace.py`. Every person matched a separate scalar implementation of the compensation, dollar, spousal, and return-claimant rules within $0.002. The five people whose contributions change are the heads in 003, 004, 073, and 085, plus 064's dependent1. All five have zero compensation. The wage-free head in 089 properly retains $252.448 through the spouse's $87,127.15625 compensation; there is no blanket exclusion of nonworking spouses. All scenarios are 2026; filing statuses are 58 single, 35 joint, and 7 head of household. + +The original masks `traditional_ira_contributions` itself. An earning dependent may lawfully contribute, even though the parents cannot deduct that contribution. That variable also feeds contribution-based consumers such as the Saver's Credit and Minnesota/territorial calculations. V2 retains actual contributions and introduces `traditional_ira_deduction`, restricted to the head or joint spouse. Replacing the IRA item in `gov.irs.ald.deductions` ensures consistent treatment in aggregate AGI, person AGI, Social Security MAGI, student-loan MAGI, unemployment MAGI, and Medicaid AGI; I read those consumers. An ALD-total-only adjustment would leave the other consumers wrong. + +V2 also preserves the state lists' deduction semantics: Massachusetts disallows the new deduction as it disallowed the old IRA item, consistent with [MGL c.62 §2(d)(1)(F)](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2). Mississippi substitutes the federal deductible amount, as [official 2025 Form 80-100 instructions, line 50/page 12](https://www.dor.ms.gov/sites/default/files/tax-forms/individual/80100251%202.pdf) prescribe. This is preservation of the existing state treatment, not a state-rate reform; I did not obtain 2026 Mississippi instructions. All three list substitutions restore the baseline list outside 2026. + +The other correction is a zero-only denominator guard in `ira_contribution_scale`: with $0.50 compensation and $0.80 desired, the old `max(desired, 1)` denominator allows $0.40, whereas the lawful cap is $0.50. No requested combined amount in this bundle is below $1. The existing 219(g) phase-out and Roth-income phase-out omissions remain outside this module's root cause. + +**Every moved row.** Amounts below retain six decimals where helpful. “Federal” means `federal_income_tax_before_refundable_credits`; “state” means `state_income_tax_before_refundable_credits`. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 003 (TX) | Federal | 22,154.699219 | 22,400.654297 | yes | Both spouses have zero compensation; remove the head's $1,117.984 traditional IRA deduction. | +| 064 (WI) | Federal | 4,439.289551 | 4,441.455078 | yes | Remove dependent1's $18.032 deduction; the head's $108.192 remains. | +| 064 (WI), unflagged | State | 4,605.997070 | 4,607.142090 | yes | The same federal-AGI increase raises WI AGI and phases down the WI standard deduction. | + +For 003, the independent trace gives ALD $2,699.983887 → $1,582, AGI $182,172.265625 → $183,290.25, and taxable income $149,972.265625 → $151,090.25. For 064, ALD is $19,926.224609 → $19,908.191406 and AGI $114,403.367188 → $114,421.406250. The WI standard deduction falls from $8,759.121094 to $8,755.552734, and WI taxable income rises from $102,144.25 to $102,165.851563. I read `wi_agi.py`, `wi_standard_deduction.py`, `wi_taxable_income.py`, and `wi_income_tax_before_refundable_credits.py`; that is the complete observed downstream path. [Wisconsin's official Form 1](https://www.revenue.wi.gov/TaxForms2025/2025-Form1f.pdf) also begins with federal AGI. I am confirming the cause of the delta, not independently certifying all unchanged WI parameters. + +**Missed rows.** None above the sweep's $1/binary threshold. The defect also reaches 004 and 073, but their federal/state income-tax outputs remain zero after the correction; the full sweep shows no other changed scored output for them. For 085, removing its $0.396704 deduction changes AGI from $45,573 to $45,573.738281; federal tax rises **$0.088623**, below tolerance. These are affected households without missed exclusions. + +**Reproduction and artifacts.** The independently rerun original CSV has **1,984 parsed rows exactly equal** to `sweep/out/r01_ira_compensation.csv`: three moved rows and one substantive nonzero delta below tolerance. The two flagged federal recomputations exactly match the investigator's stored corrected values. Complete all-person evidence is in `sweep/verify/work/r01_person_scope.csv`; household inputs and before/after intermediates are in `sweep/verify/work/r01_trace.json` and `r01_trace.log`. + +Corrected module: `sweep/fixes/r01_ira_compensation_v2.py`. Corrected sweep: `sweep/verify/work/r01_ira_compensation_v2.csv`. Final v2 and adversarial verification results are recorded below after completion. diff --git a/reference_audit/2026-09-22/verification/v1_r02.md b/reference_audit/2026-09-22/verification/v1_r02.md new file mode 100644 index 00000000..07431b09 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v1_r02.md @@ -0,0 +1,32 @@ +## r02_ira_219g + +**Verdict: accept_with_changes.** The 2026 phase-out rules support the moved rows. The v2 confines the ALD-list replacements to 2026; the original replaces every subsequent list while carrying the same 2026 thresholds forward. Standalone r02 deliberately retains the engine's stale IRA contribution dollar limits; the combined module supplies the separately verified 2026 amounts. + +### Legal, parameter, and line-by-line checks + +I read the entire original module and its combined loader. The following covers every rule-bearing block, rather than treating the module's explanatory comments as evidence. + +| Code block | Independent check | +|---|---| +| `PARAMETER_DATA`, `_by_fs` | Notice 2025-67 p.4 confirms 2026 active-worker ranges $81,000–$91,000 (single/HOH), $129,000–$149,000 (joint/QSS), $0–$10,000 (MFS living together), and $242,000–$252,000 for a noncovered joint filer with a covered spouse. Its prior-year comparisons also support the encoded 2025 starts $79,000/$126,000/$236,000. The $10,000/$20,000 widths agree. | +| `ira_active_participant` | Positive traditional or Roth 401(k)/403(b) deferrals are evidence of plan participation; SEP/SIMPLE/Keogh contributions also count. This is a coverage inference from available inputs, not an exhaustive legal definition. | +| `ira_219g_taxable_social_security` | Read against Appendix B Worksheet 1 and §86: taxable benefits are figured before the IRA deduction, with one-half of benefits in combined income, 50%/85% tiers and caps, and filing-status thresholds. The helper reads these existing engine parameters rather than hard-coding new amounts. | +| `ira_219g_magi`, `MAGI_DISREGARDED_ALDS` | §219(g)(3)(A) requires AGI after §§86/469 and disregards §§85(c),135,137,221,911 and the IRA deduction. The helper replaces taxable unemployment with total unemployment, computes Social Security before the IRA deduction, and removes the relevant ALDs. It keeps §931/933 deductions in final IRA MAGI while revoking them for the §86 calculation. No §911 input is implemented in this engine. Positive income/loss handling otherwise follows the existing engine. | +| `ira_219g_deductible_limit` | The phase-out applies to the dollar limit, independently of compensation. Filing separately while apart all year uses the single range and disregards spouse coverage. The joint spouse-only route uses its own $10,000 width. Reduction is rounded down to $10; the remaining positive limit is at least $200 until the upper endpoint, where it is zero. Contribution amounts below that minimum remain the binding cap. | +| `traditional_ira_deduction` | Deduction is the lesser of actual traditional contributions and the reduced dollar limit. Underlying contributions remain available to contribution-based provisions. | +| `_swap_from`, `modify_parameters`, `reform.apply` | Federal ALDs use the new deduction. Massachusetts excludes it; Mississippi uses federal deductibility. Original `_swap_from` operates from 2026 onward, not solely in 2026; v2 uses an explicit 2026 period and restores surrounding lists. | + +Sources actually opened: [IRC §219](https://www.law.cornell.edu/uscode/text/26/219), [Notice 2025-67](https://www.irs.gov/pub/irs-drop/n-25-67.pdf), [IRS Publication 590-A, 2025 edition, Worksheets 1-1/1-2 and Appendix B](https://www.irs.gov/publications/p590a), [IRC §86](https://www.law.cornell.edu/uscode/text/26/86), [Treas. Reg. §1.219-2(b)–(e)](https://www.law.cornell.edu/cfr/text/26/1.219-2), and [IRS employer-plan coverage guidance](https://www.irs.gov/retirement-plans/are-you-covered-by-an-employers-retirement-plan). The regulation's older blanket-deduction language does not replace the current statute's phase-out. + +For state list substitutions I checked [Massachusetts DOR's federal/state differences guidance](https://www.mass.gov/info-details/differences-between-ma-and-federal-tax-law-for-personal-income) and [Mississippi's official 2025 Form 80-100 instructions, p.12, line 50](https://www.dor.ms.gov/sites/default/files/tax-forms/individual/80100251%202.pdf). Those are the documents read; I do not claim a published 2026 Mississippi instruction booklet. Downstream AGI linkage is supported by [California R&TC §17201(b)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17201.), [Schedule CA line 20 instructions](https://www.ftb.ca.gov/forms/2025/2025-540-ca-instructions.html), [Ohio §5747.01(A), effective March 5, 2026](https://codes.ohio.gov/ohio-revised-code/section-5747.01), and [Connecticut's 2026 §12-701 supplement](https://www.cga.ct.gov/2026/sup/chap_229.htm). + +### Scope findings + +The bundle contains 100 households: 58 single, 35 joint and seven heads of household; none file separately or as qualifying surviving spouses. Twenty-four households list desired traditional IRA contributions. Coverage cannot be inferred just from pension receipts, educator expenses, or the absence of elective deferrals. Defined-benefit eligibility, employer-only allocations and coverage of a spouse in another tax unit are not represented by this fix's coverage proxy. These are limitations outside the demonstrated bundle facts, not additional confirmed missed exclusions. + +The compensation gate has a narrow justification for scenario 003: both adults have zero wages and self-employment earnings, so the listed *desired* 401(k) amounts do not establish a feasible current elective deferral. It must not be generalized to a claim that a retiree without current wages can never be an active participant: employer allocations and plan-year timing can confer coverage. R01 independently eliminates 003's IRA deduction for lack of compensation, and the combined result does not depend on using 003's infeasible deferrals as coverage evidence. + +The original module changes 2027-and-later federal/MA/MS lists using 2026 thresholds without later-year verification. `sweep/fixes/r02_ira_219g_v2.py` limits those substitutions to 2026. It reuses the reviewed 2026 arithmetic, preserving standalone r02's scope. The combined v2 additionally retains r01's head/joint-spouse deduction restriction without suppressing a dependent's underlying contribution. + +R02 does not repair unrelated capital-gain/loss, estate-income, state-itemization, or Roth-eligibility defects. In particular, the scenario 005 California and scenario 110 federal verdicts contain multiple corrections; their headline corrected values are not expected to equal a phase-out-only sweep. Scenario 049's investigated phase-out-only value is $30,702.59, matching this sweep within $0.01. The verdicts' explicitly isolated r02 sensitivities are $41,276.82 for 005 California and $23,941.08 for 110 federal; these also match within $0.01. + diff --git a/reference_audit/2026-09-22/verification/v1_r03.md b/reference_audit/2026-09-22/verification/v1_r03.md new file mode 100644 index 00000000..ad4bcbd6 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v1_r03.md @@ -0,0 +1,36 @@ +## r03_estate_income + +**Verdict: accept_with_changes.** The 2026 gross-income correction is supported. The original reform also rewrites 2010–2109, beyond this verification's 2026 scope; the replacement confines the change to 2026. This date guard does not change the bundle's results. Ohio's state result has the separate business-income qualification described below. + +**Legal and parameter checks.** I read every line of `sweep/fixes/r03_estate_income.py`. The executable change copies the source list at 2026-01-01, appends `estate_income` unless already present, updates a 100-year period beginning in 2010, and installs that parameter reform. The module contains no dollar limit, tax rate, deduction amount, or income threshold. Its 2010 date matches the sole effective date in the installed `parameters/gov/irs/gross_income/sources.yaml`; the 100-year span is an implementation choice, not a tax-law parameter. + +- [IRC 61(a)(14)](https://www.govinfo.gov/link/uscode/26/61) includes income from estates and trusts. The current text uses paragraph 14, and its amendment note explains the 2017 renumbering. +- [IRC 102(a)–(b)](https://www.law.cornell.edu/uscode/text/26/102) distinguishes inherited property from taxable income from that property, including beneficiary income under subchapter J. The module is appropriate for the engine's estate-income field; it is not a rule to tax every inheritance or distribution of corpus. +- [IRC 662(a)–(b)](https://www.law.cornell.edu/uscode/text/26/662) includes required/current and other qualifying distributions up to distributable net income and preserves their character. [IRC 652(a)–(b)](https://www.law.cornell.edu/uscode/text/26/652) provides the corresponding rule for simple trusts. These provisions do not make tax-exempt income taxable or convert capital gains to ordinary income. The input is the beneficiary's taxable estate/trust income, not total estate receipts before applying those rules. +- [IRS Schedule E instructions, Part III](https://www.irs.gov/instructions/i1040se), and [Schedule K-1 (Form 1041) instructions](https://www.irs.gov/instructions/i1041sk1) describe beneficiary reporting and preserve separate interest/dividend/capital-gain categories. Both pages identify themselves as 2025 instructions; I use them for reporting mechanics, not 2026 numeric parameters. +- I read `/Users/maxghenis/PolicyEngine/policyengine-us-data/policyengine_us_data/datasets/puf/puf.py:324`, which maps `estate_income` to `E26390 - E26400`. In the installed engine, `estate_income.py` is an input, `loss_ald.py` labels estate/trust amounts as Schedule E items and already deducts negative amounts, and `irs_gross_income.py` sums only nonnegative sources for nondependents. Appending this source therefore fills the positive-income omission without newly deducting estate losses. +- The Social Security downstream effect follows [IRC 86(a)–(b)](https://www.law.cornell.edu/uscode/text/26/86). I read `taxable_ss_magi.py`, which uses the same gross-income list, adds positive amounts, and then subtracts the permitted adjustments. Medicaid gross-income and student-loan MAGI consumers likewise add positive amounts; none of these read paths duplicates an estate loss. + +**Scope.** I parsed all 100 scenario JSON records from the supplied `scenarios.csv`; all use 2026. Exactly two contain `estate_income`: scenario_039's 61-year-old Virginia head has $25,950 and scenario_110's 51-year-old Ohio head has $6,322.94140625. Neither has children, a spouse, another estate-income recipient, an estate loss, or a listed estate-QBI-status input. There are no additional bundle households to which this omission should apply. The reform does not add a state restriction, which is appropriate for this federal income source. I verified the negative-income handling in source code, rather than claiming a negative-estate-income household test in this bundle. + +**State qualification.** Virginia starts from federal AGI ([Va. Code 58.1-321](https://law.lis.virginia.gov/vacode/title58.1/chapter3/section58.1-321/)) and subtracts federally taxed Social Security ([58.1-322.02(3)](https://law.lis.virginia.gov/vacode/title58.1/chapter3/section58.1-322.02/)). Its subtraction list supplies no general estate-income exclusion. Ohio likewise starts from federal AGI, but [Ohio Rev. Code 5747.01(A)(28), (B)](https://codes.ohio.gov/ohio-revised-code/section-5747.01) allows up to $250,000 of individual business income to be deducted. Ohio's official [IT 1040 instructions, “Trust Income” examples](https://dam.assets.ohio.gov/image/upload/tax.ohio.gov/forms/ohio_individual/individual/2021/pit-it1040-booklet.pdf) explain that operating income retains business character through a trust, while an investment trust's income can be nonbusiness. The installed Ohio model does not implement that deduction: the comment in `variables/gov/states/oh/tax/income/credits/joint_filing_credit/oh_joint_filing_credit_potential.py` says so, and I read the AGI/deduction chain. Therefore the Ohio movement is attributable to this gross-income correction, but $4,231.35 is not certified as a fully corrected legal tax under a business-income interpretation. The nonbusiness interpretation tested by the companion variant supports inclusion without that offset. This limitation is separate from the reform's causal isolation; I did not add an unrelated Ohio business-deduction reform. + +**Independent results and trace.** Pending completion of trace and replacement sweeps. + +**Missed rows:** none found in the bundle. **Replacement:** `sweep/fixes/r03_estate_income_v2.py`. + +## r03_estate_income__qbi_false + +**Verdict: accept_with_changes.** The non-QBI interpretation is defensible and implements the prompt's treatment of an unlisted boolean. The original patch also overwrites an explicitly supplied qualification value, replaces its entire year mapping, and injects a 2026 value even when passed a different-year scenario. The replacement preserves listed values and other years, and imports the 2026-only gross-income reform. + +**Legal and parameter checks.** I read every line of `sweep/fixes/r03_estate_income__qbi_false.py`: it imports its sibling's reform unchanged; the only extra computational operation sets `estate_income_would_be_qualified = {"2026": False}` for each person having an estate-income key. There are no hard-coded monetary parameters. `False` is an input interpretation, not a general legal rule for estate income. + +[IRC 199A(c)(1), (3)(A)–(B)](https://www.law.cornell.edu/uscode/text/26/199A) requires qualifying trade-or-business items that enter taxable income and excludes ordinary investment dividends, capital gains, and nonbusiness interest. [Treas. Reg. 1.199A-6(d)](https://www.law.cornell.edu/cfr/text/26/1.199A-6) passes qualifying business items from an estate/trust to beneficiaries; it does not make all estate distributions QBI. [IRS Schedule K-1 instructions, box 14 code I](https://www.irs.gov/instructions/i1041sk1) similarly require apportioned qualifying items. Those rules support both business and nonbusiness possibilities for an otherwise unspecified estate-income source; there is no evidence in either household's listed facts of the required business character. I read `/Users/maxghenis/PolicyEngine/policybench-wt/opus55/policybench/prompts.py:19`, which instructs that unlisted nonnumeric facts/booleans/statuses are false, and its estate-income label mapping. The engine's `estate_income_would_be_qualified.py` defaults to `True`; `qualified_business_income.py` multiplies each source by its qualification flag. + +The base module's inclusion and state-law checks above also apply here. I additionally checked the 2026 single $16,100 standard deduction and $12,400/$50,400/$105,700 ordinary bracket thresholds against [Rev. Proc. 2025-32, sections 4.01 and 4.14](https://www.irs.gov/pub/irs-drop/rp-25-32.pdf) for the numerical traces; this variant does not override those parameters. Neither the new minimum QBI deduction nor the QBI phase-out changes the conclusions here: the corrected positive deductions exceed the minimum, and the alternative has no positive QBI. + +**Scope.** The same two estate-income heads are the entire bundle population reached. Their qualification booleans are both unlisted, so the explicit-input-overwrite defect does not change any of the 100 household results. The patch changes only estate QBI status, retaining other QBI-source defaults so the experiment stays focused on this input. Negative business income in scenario_039 remains present; the existing engine floors total QBI at zero after estate income is removed. No household with only unrelated business income is patched. + +**Independent results and trace.** Pending completion of trace and replacement sweeps. + +**Missed rows:** none found in the bundle. **Replacement:** `sweep/fixes/r03_estate_income__qbi_false_v2.py`. diff --git a/reference_audit/2026-09-22/verification/v1_r16.md b/reference_audit/2026-09-22/verification/v1_r16.md new file mode 100644 index 00000000..d0067683 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v1_r16.md @@ -0,0 +1,43 @@ +## r16_survivor_benefits_federal + +**Verdict: accept_with_changes.** Accept the two moved reference outputs under the module's stated fully taxable survivor-annuity reading. The executable reform is unchanged in v2; its documentation corrects a statutory subsection and makes the factual assumption explicit. Scenario 062 federal tax has two defensible readings and supports an `unlisted_input` exclusion. Scenario 108 refundable credits overlaps the separate Wisconsin homestead-income defect; its correction does not depend on establishing federal-tax ambiguity. + +### Legal and parameter checks + +I read all 93 lines of the original module, the harness, the two households in the bundle, `triage/verdicts.json`'s `scenario_108__state_refundable_credits` investigation, and the engine source files identified below. The original module has one executable change: append the person-level `survivor_benefits` source to the federal gross-income source list, only for January 1–December 31, 2026. It preserves existing sources and avoids a duplicate append. There are no hard-coded dollar amounts, rates, ages, or filing thresholds in this reform. + +The legal checks were: + +- The current [IRC §61(a)](https://uscode.house.gov/view.xhtml?edition=prelim&req=granuleid%3AUSC-prelim-title26-chapter1-subchapterB) includes annuities in **(8)** and pensions in **(10)**. The original's `(9)-(10)` citation for annuities/pensions is inaccurate; v2 corrects it. This is a citation correction, not a change in the inclusion rule. +- [IRC §72(a)–(b)](https://uscode.house.gov/view.xhtml?edition=prelim&f=treesort&num=1267&req=tax) includes annuity receipts except the exclusion attributable to contract investment. [IRC §402(a)](https://uscode.house.gov/view.xhtml?req=%28title%3A26+section%3A402+edition%3Aprelim%29) applies §72 to distributions from qualifying employees' trusts to beneficiaries. These retrieved current Code texts identify laws in effect in September 2026. [IRS Publication 575, “Survivors and Beneficiaries”](https://www.irs.gov/publications/p575) explains that survivor annuities follow the decedent's taxable/nontaxable treatment; recovered cost or no employee cost supports full taxation. The publication read is the 2025 edition, not a claimed 2026 edition. +- Full taxation is a **possible factual reading**, not a rule that missing basis proves zero basis. The prompt does not identify the payer, plan type, employee cost, or recovery history. V2 explicitly supplies the fully taxable employer-annuity reading for this sensitivity exercise. +- The opposite tax treatment is supported by [38 USC §5301(a)(1)](https://uscode.house.gov/view.xhtml?edition=prelim&f=treesort&jumpTo=true&num=0&req=%28title%3A38+section%3A5301+edition%3Aprelim%29+OR+%28granuleid%3AUSC-prelim-title38-section5301%29) and the [VA's description of tax-free DIC](https://www.va.gov/family-and-caregiver-benefits/survivor-compensation/dependency-indemnity-compensation/). [IRS Publication 525, “Veterans' benefits” and “Workers' Compensation”](https://www.irs.gov/publications/p525) also documents VA exclusions and qualifying workers' compensation paid to survivors. I checked the survivor-specific workers' compensation provision in [Treas. Reg. §1.104-1(b), official 2025 CFR](https://www.govinfo.gov/content/pkg/CFR-2025-title26-vol2/pdf/CFR-2025-title26-vol2.pdf). +- The current [IRC §101(a), (d), and (h)](https://uscode.house.gov/view.xhtml?req=%28title%3A26+section%3A101+edition%3Aprelim%29) provides death-benefit and qualifying public-safety survivor-annuity exclusions, with conditions. These are possible categories, not a claim that this household actually qualifies for any one. The §101(h) survivor-annuity reading also establishes a nontaxable possibility without depending on whether the source dataset's definition excludes VA payments. I did not infer actual payer history from the CPS metadata. +- For the Wisconsin downstream output, [Schedule H instructions, line 9d](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleH-inst.pdf) include gross pensions/annuities, including nontaxable amounts; eligibility step 1d gives the $24,680 income ceiling. The [Wisconsin DOR retired-persons FAQ, dated February 4, 2026](https://www.revenue.wi.gov/Pages/FAQS/pcs-retired.aspx) corroborates the ceiling and the $24,000 retirement subtraction beginning in 2025 for eligible people age 67 or older, with credits forfeited if claimed. [Schedule SB line 16](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf) limits that subtraction to qualified-plan/IRA income. The module changes no Wisconsin parameters. Direct retrieval of the Wisconsin legislative statute pages failed; I do not claim to have independently read their current text. + +### Scope and facts + +The all-household `scenarios.csv` scan found exactly two nonzero `survivor_benefits` inputs: scenario 062 (FL, $28,800) and scenario 108 (WI, $10,392). Scenarios 048 and 085 have `social_security_survivors` but no `survivor_benefits`; the reform does not change that distinct Social Security input. Both affected people are age 85, disabled, single tax-unit heads with surviving-spouse flags. The reform covers every person possessing the input in 2026, rather than changing a state's law or treating all Social Security survivors as taxable annuitants. + +For **062**, the prompt formatter renders separate lines for Social Security survivor benefits ($21,312), survivor benefits ($28,800), and taxable private pension income ($11,280). Other relevant inputs are farm rent $4,792.94140625 and taxable interest $257. The separate pension line does not identify the survivor line's payer or basis: it can be the person's own pension alongside a deceased spouse's survivor annuity, or alongside nontaxable survivor compensation. A qualified public-safety survivor annuity can be a recurring $28,800 annual payment, so the stated full-year facts do not force a lump-sum/life-insurance interpretation. Thus both fully taxable and wholly exempt interpretations are defensible from the prompt. These are alternatives, not findings about the household's true benefit source. + +For **108**, $7,656 wages, $12,934 Social Security survivors, and $10,392 additional survivors sum to $30,982 of relevant homestead income. The generic survivor amount is absent from both the baseline federal AGI and homestead additional-source lists. R16 reaches the homestead calculation once through AGI. The gross-pension reading independently makes the reference credit wrong even when the pension is federally exempt. Count this output once in the exclusion union; it is also the separately identified homestead defect. + +The source-list method does not classify the amount as `taxable_pension_income`. Consequently, it is **not a general implementation of state retirement subtractions**. I checked Wisconsin's pension-dependent subtraction formulas and performed the pension-routing sensitivity below for the only affected Wisconsin household. No claim is made about a new household in another state, all benefit categories, or years other than 2026. + +### Reproduction and every moved output + +The independent original sweep produced 1,984 outputs, two moved rows, and no smaller nonzero deltas. Every CSV row, not just the moved set, exactly matched the supplied `sweep/out/r16_survivor_benefits_federal.csv`; see `sweep/verify/work/r16_comparison.json`. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_062 (FL) | federal_income_tax_before_refundable_credits | 0.000000 | 4328.386719 | Yes, conditional on taxable-annuity reading | $28,800 enters federal income and increases the taxable part of Social Security; the omitted payer/basis facts admit a wholly exempt alternative. | +| scenario_108 (WI) | state_refundable_credits | 271.391602 | 0.000000 | Yes; overlaps homestead defect | $10,392 enters homestead income once through AGI, bringing it to $30,982 and above the $24,680 ceiling. | + +TRACE_RESULTS_PENDING + +**Missed rows:** None in the 100-household bundle for this alternative federal-tax treatment: only the two households carry the input, and all 1,984 original-reform outputs were recomputed. This finding does not certify the reform for other fact patterns. + +**V2 path:** `sweep/fixes/r16_survivor_benefits_federal_v2.py`. It preserves executable calculation logic and corrects/clarifies the documentation described above. V2_SWEEP_PENDING + +**Evidence files:** `sweep/verify/work/r16_survivor_benefits_federal.csv`, `r16_survivor_benefits_federal_v2.csv`, `r16_comparison.json`, `r16_trace.py`, `r16_trace.json`, and `r16_prompts.txt`. Engine paths read are under `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`: `parameters/gov/irs/gross_income/sources.yaml`; `variables/household/income/person/general/survivor_benefits.py`; `variables/household/income/person/retirement/taxable_pension_income.py`; `variables/gov/irs/income/taxable_income/adjusted_gross_income/irs_gross_income/social_security/taxable_ss_magi.py`; `variables/gov/irs/credits/elderly_and_disabled/section_22_income.py`; Wisconsin `subtractions/wi_retirement_income*.py`; and `credits/homestead/wi_homestead_income.py`, `wi_homestead_credit.py`, plus its income-source parameter. The prompt description is produced from `/Users/maxghenis/PolicyEngine/policybench-wt/opus55/policybench/prompts.py`. diff --git a/reference_audit/2026-09-22/verification/v2_r04_r05_r06_r07_r10.md b/reference_audit/2026-09-22/verification/v2_r04_r05_r06_r07_r10.md new file mode 100644 index 00000000..6eeb4a1e --- /dev/null +++ b/reference_audit/2026-09-22/verification/v2_r04_r05_r06_r07_r10.md @@ -0,0 +1,365 @@ +# Independent verification of PolicyBench reference fixes + +Verification date: September 22, 2026. Sandbox: policyengine-us 1.755.4; +benchmark tax year: 2026; assigned branch: `triage-verify`. + +I read the sweep brief first, inspected each complete module and its relevant +engine consumers, checked the cited legal sources, scanned all 100 scenario +records, and independently ran all five originals and five corrected modules. +Every original rerun is byte-for-byte identical to the builder's CSV, including +all 1,984 output rows and the moved flags. Numerical agreement does not establish +that a legal mapping is valid; the module sections distinguish those findings. +Every original and revised CSV also contains the same two unmoved scenario-112 +residuals of about `-5.68e-14` dollars, in federal refundable credits and payroll +tax, from floating-point decimal representation. No other nonzero unmoved deltas +were observed. These do not change the harness's moved set. + +All writes are in the assigned workspace. In this report, `triage/` means +`/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/`; +`sweep/` means the local assigned workspace's `sweep/`, except where an original +external path is explicitly identified. Original modules, outputs and source +inputs are unchanged, verified against the recorded SHA-256 hashes. + +The unchanged original sweep harness is preserved in `work/sweep_original.py`. +The wrapper and serialized runner cache reads from the immutable engine +installation and deep-copy parsed parameter data for each consumer. They do not +alter formulas or parameter values. Initial runs encountered prolonged loading +under filesystem pressure; only completed reruns are counted here. Execution, +exact comparisons, and diagnostic assertions are retained under `work/`; see +`work/final_checks.json` and `README.md` for reproduction details. + +The tables report calculations in the specified sandbox with unrelated engine +behavior held constant. They do not certify every other aspect of these tax +returns. In particular, the Wisconsin section identifies an inherited 2026 +standard-deduction parameter mismatch. Where 2025 official instructions supply +form mechanics, each section identifies the standing statutory rule used for +2026; no unread final 2026 instruction booklet is implied. + +The requested commit could not be created because this session makes `.git` +read-only: `git add` failed when creating `.git/index.lock` with `Operation not +permitted`. All deliverables remain uncommitted on the assigned branch. Nothing +was pushed, filed, or posted, and no history was rewritten. +The older initialization-diagnostic runner (PID `40328`) could not be stopped +with the available session interrupts or sandbox signal permissions; its status +and the completed/resumed-run provenance are documented in `README.md`. + +## r04_capital_gain_distributions + +**Verdict: accept_with_changes.** The federal distribution inclusion follows the +three production changes in PR #8839. Two qualifications prevent accepting the +original module as a complete correction: its Wisconsin outputs omit the matching +30% distribution subtraction, and its preferential-base formula mishandles a +Form 4952 election on those distributions. The latter does not appear in the four +bundle households with this input. + +**Legal and parameter checks.** Read every line of the original module, its three +upstream production changes, and the installed consumers needed to explain the +moved outputs. [IRC 61(a)(3)](https://www.law.cornell.edu/uscode/text/26/61) and +[852(b)(3)(B)](https://www.law.cornell.edu/uscode/text/26/852) establish inclusion +and long-term character. The [Form 1040 instructions](https://www.irs.gov/instructions/i1040gi) +confirm direct line-7a reporting, inclusion in the Social Security worksheet, and +the preferential worksheet's no-Schedule-D path. +[IRC 86](https://www.law.cornell.edu/uscode/text/26/86) supplies the provisional-income +rule; [IRC 1411](https://www.law.cornell.edu/uscode/text/26/1411) and +[Form 8960 line 5a](https://www.irs.gov/instructions/i8960) support investment-income +inclusion. The form instructions read are for 2025; the standing statutory rules +are the basis for their application here to 2026. + +The original hard-codes no dollar threshold or tax rate. The independent federal +arithmetic uses the 2026 amounts in [Rev. Proc. 2025-32](https://www.irs.gov/pub/irs-drop/rp-25-32.pdf): +single standard deduction $16,100, aged addition $2,050, 10% bracket through +$12,400, 12% next bracket, and $49,450 zero-rate capital-gain threshold. The +$6,000 senior deduction comes from [IRC 151(d)(5)(C)](https://www.law.cornell.edu/uscode/text/26/151). +These parameters are not changed by either version of r04. + +**Two code findings.** Wisconsin's [Schedule SB instructions, page 2, line 5](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf#page=2) +expressly allow subtracting 30% of mutual-fund/REIT capital-gain distributions. +[DOR Publication 103, January 2026, section 3.A](https://www.revenue.wi.gov/DOR%20Publications/pb103.pdf) +confirms the treatment. The installed Wisconsin subtraction reads only the ordinary +short/long-term gain inputs, so original r04 puts 100% of the distributions into +Wisconsin income. This explains why its state amounts require revision even though +the rows genuinely depend on the named capital-gain omission. + +Separately, [IRC 1(h)(2)](https://www.law.cornell.edu/uscode/text/26/1#h_2) reduces +preferential gain by the investment-income election. The [Schedule D Tax Worksheet](https://www.irs.gov/instructions/i1040sd) +applies that reduction even when Schedule D itself is unnecessary. The original +subtracts the election before adding these distributions. V2 adds them before the +reduction. It preserves the inherited qualified-dividend treatment; it does not +claim to repair all other Form 4952 behavior. + +**PR comparison.** The requested `gh pr diff 8839 --repo PolicyEngine/policyengine-us` +failed to connect to GitHub. I instead read the local merge diff for `491642087f`, +which identifies PR #8839, without changing the upstream checkout. The diff and +commit metadata are saved under `work/r04_capital_gain_distributions/`. +Independently compared all three installed production files with that merge's +parent: each is byte-for-byte identical. The checks and SHA-256 values are saved +in `work/r04_capital_gain_distributions/pr_parent_comparison.json`. + +**Scope and missed rows.** Scanning all 100 scenarios finds this input only in +013 (AZ, $72), 042 (WI, $3,753), 051 (LA, $180), and 091 (WI, $1,170). No such +household carries a short/long-term gain or Form 4952 election alongside it. The +v2 changes source-list values and Wisconsin treatment only in 2026. The original +source-list modifier changes every historical dated entry; that is broader than +needed for this benchmark. Neither version assigns a distribution to a person +whose input is zero. + +The IRS line-7a reporting exception requires no capital losses; distributions +must instead be netted on Schedule D when such losses exist. Although the generic +input schema can represent both, that combination is outside the stated +non-Schedule-D reporting meaning. The Form 4952 election counterexample is valid: +the [Schedule D Tax Worksheet instructions](https://www.irs.gov/instructions/i1040sd) +expressly require that worksheet for a positive line-4g election even when +Schedule D itself is unnecessary. + +**Independent sweeps.** Ran both modules across all 100 households and 1,984 +outputs using the preserved original harness. The original rerun is byte-for-byte +identical to the stored CSV: four moved outputs. The only other nonzero deltas +are the two shared scenario-112 floating-point residuals described in the report +introduction. V2 +keeps the same four-row moved set. It changes only the two Wisconsin recomputed +amounts relative to the original fix; all other output values agree exactly. + +**Every moved row.** “Confirmed” here confirms the named root cause warrants a +movement; the two Wisconsin amounts from the original module need the revision +shown in the following table. Other engine behavior is held constant; these are +not certifications of every unrelated parameter in the whole return. + +| Scenario | Variable | Frozen | Original recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 042 | federal_income_tax_before_refundable_credits | $1,979.16 | $2,361.96 | Yes | $3,753 enters gross/provisional income and the preferential base; additional taxable Social Security raises ordinary tax. | +| 042 | state_income_tax_before_refundable_credits | $284.74 | $469.69 | Yes | Distribution reaches Wisconsin AGI, but original amount omits its 30% subtraction. | +| 051 | state_income_tax_before_refundable_credits | $814.95 | $820.35 | Yes | Unflagged: Louisiana includes the $180 distribution; unchanged 3% rate adds $5.40. | +| 091 | state_income_tax_before_refundable_credits | $843.66 | $901.32 | Yes | Unflagged: $1,170 affects Wisconsin AGI and standard-deduction phaseout; original amount omits its 30% subtraction. | + +Louisiana's rate is supported by [RS 47:32](https://www.legis.la.gov/legis/Law.aspx?d=101946) +and [DOR income-tax guidance](https://revenue.louisiana.gov/individuals/general-resources/individual-income-tax/). + +| Wisconsin output | Original fix | V2 | V2 minus frozen | +|---|---:|---:|---:| +| 042 state before refundable credits | $469.68878173828125 | $414.204345703125 | +$129.46343994140625 | +| 091 state before refundable credits | $901.319091796875 | $884.0218505859375 | +$40.3603515625 | + +These are r04-only calculations. R06 makes 042's before-refundable output zero +when the Act 15 election is also modeled; its section records both combinations. + +Independent arithmetic in `work/r04_capital_gain_distributions/hand_check.json` +reproduces all four original movements within one cent. For 042, taxable Social +Security increases by `0.85 × $3,753 = $3,190.05`; the 12% ordinary rate adds +$382.806 of federal tax. Wisconsin's unchanged 12% deduction phase-out and 4.4% +marginal bracket give `distribution × 1.12 × 0.044`; v2 first multiplies the +distribution by 70%. Louisiana gives `$180 × 3% = $5.40`. The Wisconsin rates +and phase-out are also checked against the official 2026 instructions in the r06 +section; this arithmetic holds the sandbox's existing standard-deduction maximum +constant. + +**Calculation traces and missed rows.** The completed `audit.json` records the +full inputs and baseline/original/v2 calculations for all four households with +distributions. In 042, the original fix raises taxable Social Security from +$13,655.95 to $16,846.00, AGI from $43,637.65 to $50,580.71, and the preferential +base from $928 to $4,681. Capital-gains tax remains zero; ordinary tax supplies +the federal movement. Wisconsin's original-fix AGI is $33,734.71; v2 subtracts +$1,125.90 and gives $32,608.81, also changing the sliding standard deduction. +In 091, v2 subtracts $351 from original-fix Wisconsin AGI of $36,182.56, yielding +$35,831.56. Those intermediates reproduce the two revised amounts above. + +No missed moved row was found. In 013, AGI rises from $6,736 to $6,808, but taxable +Social Security, federal taxable income and Arizona before-refundable tax remain +zero. In 051, federal taxable income and preferential gain both rise by $180; +ordinary tax remains $2,620 and capital-gains tax remains zero. In 091, those two +amounts both rise by $1,170; federal before-refundable tax remains $1,349.9987793. +Thus their unchanged federal outputs have an explicit calculation explanation. + +**Adversarial checks.** All five upstream PR integration cases passed for each +of the original and v2 modules, ten case/variant executions. With $5,000 of +distributions and a $5,000 Form 4952 election, the original incorrectly retains +$5,000 of preferential gain and computes $5,020 tax; v2 removes that preferential +gain and computes $5,620. A separate Wisconsin $1,000 distribution produces the +required $300 subtraction under v2. The zero-distribution control and a 2025 +outside-year control match the untouched baseline under v2. Assertions and full +results are in `work/r04_capital_gain_distributions/audit.py` and `audit.json`. + +The corrected module is `sweep/fixes/r04_capital_gain_distributions_v2.py`. +Detailed source notes are in `work/r04_capital_gain_distributions/source_checks.md`. +The r06 section reports the Act 15 interaction on 042. + +## r05_nj_worker_ui + +**Verdict: accept_with_changes.** The 2026 benchmark correction is supported. The original module also changes later years using a frozen 2026 wage base; v2 limits the repair to 2026. + +**Legal and parameter checks.** I read the complete module, including its parameter insertion, three person-level formulas, state restriction, and replacement payroll aggregate. [NJDOL's live rate table](https://www.nj.gov/labor/ea/employer-services/rate-info/) confirms 2026 worker UI **0.003825**, WF/SWF **0.000425**, and their **$44,800** base. Its 2025 table confirms the module's **$43,300** base and the same two rates. The unchanged 2026 TDI/FLI rates are **0.0019/0.0023**, with a separate **$171,100** base. The table also publishes a **$46,400** UI/WF/SWF base for 2027: carrying $44,800 forward is incorrect. The module's zero entries before 2025 are unverified placeholders, inconsistent with the page's positive 2024 worker rates. + +I read [N.J.S.A. 43:21-7(d)(1)(D)–(F) and 43:21-7e](https://www.nj.gov/labor/myunemployment/assets/pdfs/UI_statute.pdf): the 0.3825% worker rate dates from July 1, 2004; employers withhold it; a separate 0.0825% rate covers State/reimbursable governmental employers; excess contributions across employers are refundable. [NJ Taxation's credit guidance](https://www.nj.gov/treasury/taxation/njit16.shtml) confirms separate worker caps for joint filers and the published 2025 UI/WF/SWF maximum of $184.02. This supports capping each person's annual wages rather than household wages. The floor at zero prevents negative contributions. + +**Scope.** The complete bundle scan found exactly four NJ households: 008, 036, 056, and 095. Only 008 has wages: the spouse earns $26,800; the head's $4,000 is self-employment. All people in the other three households have zero wages. Their pension, interest, assistance, and self-employment inputs do not create worker UI/WF/SWF liabilities. There is therefore no other NJ wage household that should move. `defined_for = StateCode.NJ` gates all four changed/new variables. The original employer-side UI parameter and TDI/FLI formulas remain unchanged. + +The 0.3825% assumption is limited to covered private/contributory-government employment. Employer type and work state are not supplied in these scenarios. The correction inherits the engine's covered-employment/state convention; it does not establish the higher rate for reimbursable governmental employment. That alternate listed rate would still produce a positive omitted contribution on 008. + +**Moved rows.** Amounts below are rounded for display; the verification CSV preserves full precision. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 008 | payroll_tax | $2,162.76 | $2,276.66 | Yes | Spouse wages $26,800 add $102.51 UI + $11.39 WF/SWF; no contribution applies to the head's self-employment. | + +**Missed rows:** none among the four NJ households under the stated employment convention. + +The computed trace keeps the spouse's Social Security tax at $1,661.60, Medicare tax at $388.60, TDI at $50.92 and FLI at $61.64. Only the NJ employee component changes, from $112.56 to $226.46. The head and all six children have zero new worker contributions. The other three NJ households retain zero payroll tax, including 056's $4,340 and 095's $4,198.50 of computed self-employment income. + +**v2:** `sweep/fixes/r05_nj_worker_ui_v2.py`. It reproduces the 2026 arithmetic and restores baseline behavior in every other year. Its only monetary/rate constants are the independently checked $44,800, 0.003825, and 0.000425. No existing fix or output was edited. + +**Validation:** Both full sweeps completed over all **1,984 outputs**. `sweep/verify/work/r05_nj_worker_ui.csv` is byte-identical to the builder's original CSV. `sweep/verify/work/r05_nj_worker_ui_v2.csv` has identical values and moved flags after omitting the fix identifier: the same single moved row, with no added or removed exclusions. Each version passed **135 assertions** at $0.01 tolerance: all four NJ household traces plus 40 wage/state/year cases covering negative and zero wages, wages below/at/above the cap, NJ versus NY, and 2024–2027. Those cases confirm the original carries $44,800 into 2027 and v2 adds nothing outside 2026. Evidence: `sweep/verify/work/r05_nj_worker_ui/trace.json`, `sweep/verify/work/r05_nj_worker_ui_v2/trace.json`, shared script `sweep/verify/work/r05_nj_worker_ui/trace_nj_id.py`, and `source_checks.md` in that directory. + +## r06_wi_act15_before_refundable + +**Verdict: accept_with_changes** — scenario 042's correction is supported. The original module unnecessarily reduces the allowed line-17 subtraction for some joint filers because it imposes proportional allocation of pooled line-16 income. The v2 corrects that allocation while preserving the existing income bases and other paths. Both full sweeps and the runtime audit passed. + +**Legal and parameter checks.** I read the entire original module, the 1.755.4 Wisconsin formulas it copies or calls, and the retirement/standard-deduction parameter YAML files. The source root is `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +- The saved statute `triage/scenario_042__state_income_tax_before_refundable_credits/fresh_stat_54m.html`, read directly, supplies §71.05(6)(b)54m: taxable years after 2024; minimum age 67; qualified-plan and IRA distributions; $24,000 individual cap; pooled $48,000 for a joint return with both spouses age 67; exclusion of already-exempt retirement payments; and credit forfeiture. The module obtains those ages/caps from engine parameters rather than introducing new dollar parameters. Fresh Wisconsin DOR [retirement guidance](https://www.revenue.wi.gov/Pages/FAQS/pcs-retired.aspx) confirms both the $24,000/$48,000 subtraction and the separate age-65/$5,000 subtraction. +- The 2025 [Schedule SB instructions](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf), pages 7–8, restrict credits on the line-16 path and cap each spouse's line-17 subtraction at retirement income remaining after line 16. The module correctly suppresses credits but adds an unsupported proportional allocation. Its pension-only line-17 base is inherited from the engine; the v2 preserves that base. +- The [April 2026 Wisconsin Tax Bulletin 233](https://www.revenue.wi.gov/WisconsinTaxBulletin/233-04-30-WTB.pdf), page 4, describes Act 174's effective April 4, 2026 clarification: age 67 by year-end, no credits or eligible credit carryovers when claiming this subtraction, and prorating for part-year residents. The module models the full-year resident cases in this bundle. Its election helper uses the existing parameter effective in 2025, consistent with the subtraction's applicability to 2026. +- The [2026 Form 1-ES instructions](https://www.revenue.wi.gov/TaxForms2026/2026-Form1-ES-inst.pdf), pages 2–3, calculate standard deduction from Wisconsin income and then deduct exemptions before applying rates. Recomputing standard deduction after the retirement subtraction is therefore appropriate. They specify a $13,960 single maximum, 12% phase-out above $20,120, $700 personal exemption plus $250 at age 65, and rates 3.5%, 4.4%, 5.3%, and 7.65%. The sandbox instead uses a projected $13,870 maximum and $19,993.059944 phase-out threshold. Its first single tax bracket ends at $15,012.691559 rather than the published $15,110. These inherited projections are preserved; neither module is a general update to 2026 Wisconsin parameters. They cannot change scenario 042's zero-tax conclusion, since its post-subtraction income is below either standard deduction maximum. The runtime parameter dump is in `evidence.json`. +- The saved `fresh_stat_71079.html` gives §71.07(9)(b): 12% of the first $2,500 of qualifying property tax for this single filer, hence the $300 credit on the standard path. That credit must be forgone on the retirement-subtraction path. +- The implementation's `EXCLUSION_TAX_MODE` defaults to `"corrected"`. Besides routing the reported outputs onto the chosen path, it recomputes the sliding deduction and reverses overlapping line-17 amounts. The `"engine"` option retains the old exclusion-tax arithmetic. Choosing the lower tax is an engine optimization convention, not a statutory requirement to make a particular election; retaining the standard path on a tie is also a convention. + +**Scope and causal trace.** I read all four Wisconsin rows from the complete 100-row `scenarios.csv`: 042, 064, 091, and 108. Scenario 042's head is 79 and receives $19,200 taxable IRA distributions plus $4,886 private pension: $24,086 of modeled qualifying retirement income, capped at $24,000. The other Wisconsin heads/spouses are 48/47 (064), 22 (091), and 85 (108). Scenario 108 has no modeled taxable pension/IRA distributions and already has zero before-refundable tax; its survivor-benefit/homestead issue is r10. No additional bundle row was identified for this root cause. All rewritten variables have Wisconsin scope. + +I independently summed 042's non-Social-Security income inputs: $5,920 + $928 + $175 + $19,200 + $4,886 − $1,127.294067 = $29,981.705933. After the $24,000 subtraction, Wisconsin income is $5,981.705933. The standard deduction plus exemption exceeds that amount, so taxable income and before-refundable tax are zero. The runtime trace confirms the $24,000 subtraction, zero line-17 offset, and zero refundable credits. The unmodified engine reports $284.740906 because its before-refundable output stays on the other path. The independent arithmetic is saved in `sweep/verify/work/r06_wi_act15_before_refundable/hand_check.json`; runtime traces are in `evidence.json`. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_042 | state_income_tax_before_refundable_credits | 284.74090576171875 | 0.00 | Yes | Age-79 filer can subtract $24,000 of pension/IRA income; income after subtraction is below standard deduction plus exemption, and foregoing the $300 school property credit still reduces tax to zero. | + +**Interaction with r04.** Direct arithmetic from the same inputs gives post-retirement-subtraction Wisconsin income of $9,734.705933 with original r04. Including the Wisconsin 30% capital-gain-distribution subtraction in r04_v2 gives $8,608.805933. Both remain below deduction plus exemption. Runtime runs of `r04 + r06` and `r04_v2 + r06` each returned exactly $0 for 042's state before-refundable tax. The separate deltas must not be added together: the r04 standard-path increase disappears once the retirement-subtraction path is used. + +The zero result does not require treating the $4,886 private pension as a qualified-plan payment: using only the explicitly taxable $19,200 IRA distribution leaves $10,781.705933 of Wisconsin income under r06 alone, $14,534.705933 with original r04, or $13,408.805933 with r04_v2. All three remain below the sandbox deduction plus exemption of $14,820. + +**Adversarial allocation finding.** The saved §71.05(6)(b)54m.c text permits a pooled $48,000 subtraction when both joint filers are 67; it does not require proportional allocation between them. Wisconsin DOR's [practitioner Q&A, questions 7 and 15](https://www.revenue.wi.gov/Pages/TaxPro/2026/TaxPractitonerFall2025QandA.aspx), confirms that one spouse can supply the full $48,000 and that line-17 eligibility uses federal AGI. I infer from the pooled limit and the per-spouse line-17 worksheet that the alternative allocation below is permitted. Proportional allocation is possible, but imposing it can prevent the module from finding the lower-tax permitted return. + +Two age-70 spouses receive qualified pensions of $55,000 and $5,000. A deductible $40,000 active-farm loss gives $20,000 federal AGI, satisfying the joint line-17 income test. They also receive $100,000 of federally exempt municipal interest taxable by Wisconsin; [Schedule AD instructions, line 1](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleAD-Inst.pdf) support that addition. The interest raises Wisconsin income without changing the federal-AGI eligibility test. + +| Calculation | Original proportional allocation | Permitted allocation | +|---|---:|---:| +| Line 16, head / spouse | $44,000 / $4,000 | $48,000 / $0 | +| Remaining pension, head / spouse | $11,000 / $1,000 | $7,000 / $5,000 | +| Retained line 17, head / spouse | $5,000 / $1,000 | $5,000 / $5,000 | +| Line-17 offset added back | $4,000 | $0 | +| Wisconsin income on subtraction path | $66,000 | $62,000 | + +`allocation_hand_check.py` independently enumerated every feasible whole-dollar allocation of the $48,000 subtraction and found the maximum retained line-17 total of $10,000, achieved by allocating $48,000 to the head. The exact inputs and arithmetic are saved in `allocation_hand_check.json`. + +The engine audit reproduced the original $4,000 offset and tax of $1,833.5634765625. The v2 produced a zero offset and tax of $1,622.754150390625, a $210.809326171875 reduction using the sandbox's inherited parameters. Independently overriding only the original module's line-17 offset to the hand-computed zero produced exactly the same tax as the v2. Federal AGI remained $20,000 and the original line-17 base remained $10,000 on all three runs. + +The v2 computes the minimum required offset for pooled returns: the existing eligible spouses' line-17 entitlement minus the retirement income remaining after the pooled line-16 subtraction, floored at zero. Equivalently, it preserves the smaller of those two totals. This is attainable by using line 16 first against retirement income exceeding existing line-17 entitlement. The non-pooled formula, eligibility, pension-only line-17 base, and parameter values are unchanged. None of this bundle's Wisconsin households has two age-eligible spouses, so this correction is outside the recorded moved set. + +**Year and state scope.** The election helper checks `in_effect`, false before 2025. The constructor backdates earliest parameter values to 2015. The 2024 synthetic case retained its $1,117.7103271484375 state tax under baseline, original r06, and v2. The Idaho synthetic case was also unchanged. Tests covered ages 66 and 67, preservation of refundable credits when the standard path wins, overlapping line-17 income, and a positive-tax subtraction path. + +**Verification.** The independently rerun original CSV is byte-identical to the builder's `out/r06_wi_act15_before_refundable.csv`: all 1,984 rows and the sole moved row match. The v2 sweep has the same 1,984 rows, values, deltas, and moved flags; only the `fix` column differs. `sweep_comparison.json` records no differences for either run. The runtime audit passed **1,589 checks with zero failures**, including required-variable calculations, Wisconsin/state accounting identities on corrected paths, original/v2 equality on all bundle and non-pooled test cases, and the joint allocation counterexample. Both exclusion-tax modes agree on the bundle's state outputs. Results are in `sweep/verify/work/r06_wi_act15_before_refundable/evidence.json` and `assertions.json`; sweep CSVs are `sweep/verify/work/r06_wi_act15_before_refundable.csv` and `sweep/verify/work/r06_wi_act15_before_refundable_v2.csv`. + +**Missed rows:** none identified from the complete input scan, Wisconsin traces, or full sweeps. The allocation counterexample is synthetic and outside this bundle. + +**V2 module:** `sweep/fixes/r06_wi_act15_before_refundable_v2.py`, correcting joint pooled allocation. It preserves the bundle's sole moved row and every recomputed bundle value. + +## r07_idaho_health_premiums + +**Verdict: accept_with_changes.** Both benchmark exclusions are supported. The original module's deduction comparison can select itemizing when the standard deduction plus retained premium subtraction is more beneficial; v2 coordinates those two calculations. + +**Legal and parameter checks.** I read the complete module and the official-page capture of [Idaho Code 63-3022P](https://legislature.idaho.gov/statutesrules/idstat/Title63/T63CH30/SECT63-3022P/) at `triage/scenario_053__state_income_tax_before_refundable_credits/id_63-3022P.html` under the original results directory. It allows taxpayer-paid medical insurance for the taxpayer/spouse/dependents unless already deducted or accounted for. The captured amendment history ends in 2003; the live legislature endpoint was inaccessible in this run. + +I read the live [Idaho instructions dated March 2, 2026](https://tax.idaho.gov/wp-content/uploads/forms/EIN00046/EIN00046_03-02-2026.pdf), tax year 2025: Form 39R line 18 and worksheet, printed pages 34–35, and Form 40 deductions, page 8. Checked: taxpayer-paid premiums; exclusion of pre-tax, business-deducted and Idaho MSA payments; Medicare rules; itemized medical priority; worksheet lines 7–10; full premium treatment when taking the Idaho standard deduction; and choosing the deduction that benefits the taxpayer. The [official instructions index](https://tax.idaho.gov/taxes/income-tax/individual-income/forms/individual-income-tax-instructions/) listed 2025 as latest. I did not read a published 2026 Form 39R; application to 2026 rests on the standing statute and current administrative mechanics. No new tax rate or dollar limit is hard-coded in this fix. + +**Engine mapping and scope.** I read `medical_expense_health_insurance_premiums`, both premium aggregate inputs, `pre_tax_health_insurance_premiums`, the employer-premium definition, self-employed premium/ALD formulas, the medical deduction, and the Idaho deduction/SALT path in 1.755.4. Direct premiums take precedence over the non-Medicare-plus-Part-B aggregate. The duplicate `other_health_insurance_premiums` input is not added again. Employer premiums are expressly employer-paid. Pre-tax premiums are separate and are not added. Subtracting the ALD prevents this new subtraction from repeating that deduction. The existing federal medical expense aggregate does not itself net the ALD; that inherited federal limitation is outside this repair. + +Only Idaho tax units are eligible, and the subtraction-list change is bounded to 2026. The bundle has exactly three Idaho households: 007 and 053 have taxpayer premium inputs of $2,080 and $5,000; 076 has only employer-paid premiums and no taxpayer premium input. No other Idaho household was overlooked. All three use the standard deduction on their supplied facts. + +**Moved rows.** Amounts below are rounded for display; the verification CSV preserves full precision. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 007 | state_income_tax_before_refundable_credits | $755.78 | $645.54 | Yes | Unflagged: $2,080 taxpayer premiums become a subtraction; $2,080 × 5.3% = $110.24 less tax. | +| 053 | state_income_tax_before_refundable_credits | $2,435.28 | $2,170.28 | Yes | $5,000 taxpayer premiums become a subtraction; $5,000 × 5.3% = $265 less tax. | + +The 5.3% is the unchanged engine rate, also stated in the read official-page capture `triage/scenario_053__state_income_tax_before_refundable_credits/id_63-3024.html`, subsection (2). These deltas do not rely on adding the $6,309.28/$3,389.28 employer-paid premiums or treating the duplicated personal-premium labels as separate payments. + +The computed traces establish each change directly. In 007, federal AGI stays $49,064.125 and the pre-existing taxable-Social-Security subtraction stays $13,784.125; adding premiums lowers Idaho AGI from $35,280 to $33,200 and taxable income from $19,180 to $17,100. In 053, federal AGI stays $66,968.68, Idaho subtractions rise from zero to $5,000, and taxable income falls from $50,868.68 to $45,868.68. Both retain the $16,100 standard deduction and have zero self-employed health insurance ALD. In 076, the premium aggregate and new subtraction are zero; its $6,806.79 state tax is unchanged. + +**Adversarial finding and v2.** The synthetic single filer has $60,000 wages, $10,000 personal premiums and $15,000 charitable donations. The computed itemized deduction is $20,200 and the medical deduction is $5,500. V1 selects itemizing, leaving a $4,500 premium subtraction: combined reductions are $24,700, taxable income is $35,300, and tax is $1,610.1383056640625. V2 selects the $16,100 standard deduction plus the full $10,000 premium subtraction: combined reductions are $26,100, taxable income is $33,900, and tax is $1,535.9383544921875. The **$74.20** improvement follows from the $1,400 additional income reduction at 5.3%; it matches an independent run that directly supplies the standard deduction and full premium subtraction. The premium-free control is unchanged at $2,065.938232421875 under baseline, original and v2. + +V2 compares combined deductions and premium subtraction, retains mandatory itemization for `separate_filer_itemizes` when premiums are available, and uses the resulting choice consistently. It changes `id_deductions` only in 2026. The corrected module is `sweep/fixes/r07_idaho_health_premiums_v2.py`. + +**Missed bundle rows:** none. The election boundary is outside the actual three-household Idaho sample. + +**Validation:** Both full sweeps completed over all **1,984 outputs**. `sweep/verify/work/r07_idaho_health_premiums.csv` is byte-identical to the builder's original CSV. `sweep/verify/work/r07_idaho_health_premiums_v2.csv` has identical values and moved flags after omitting the fix identifier: the same two moved rows, with no added or removed exclusions. All **5 original and 7 v2 assertions** passed at $0.01 tolerance, covering all three premium subtractions, the unchanged premium-free control and the independently supplied standard-deduction alternative. Full values are in `sweep/verify/work/r07_idaho_health_premiums/trace.json` and `sweep/verify/work/r07_idaho_health_premiums_v2/trace.json`; the shared script is `sweep/verify/work/r05_nj_worker_ui/trace_nj_id.py`. Source checks are in `sweep/verify/work/r05_nj_worker_ui/source_checks.md`. + +## r10_wi_homestead_income + +**Verdict: reject** as an unconditional basis for excluding scenario 108's reference. The arithmetic under the pension/annuity interpretation is correct, but the module maps a broader input to that legal category without establishing its subtype. It also adds dependents' income and applies its 2026 source list from 2021 through 2100. A conservative partial v2 is supplied; it does not establish that the original $271.39 reference is correct. + +**What I read.** I read the complete original module; `SWEEP_BRIEF.md`; the scenario 108 entry in `triage/verdicts.json`; all four Wisconsin scenario JSON records; the complete 100-household input bundle; the sandbox definitions of all eight added source variables, retirement-distribution components, all four `wi_homestead_*` variables, and `policyengine_core/commons/formulas.py`'s aggregation. I read the seven homestead parameter YAML files used by the moved result. Engine paths below are relative to `triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +**Legal and parameter checks.** I read the Legislature's saved primary text at `triage/scenario_108__state_refundable_credits/s71_52.html` and `s71_54.html`; the page states that the statutes are updated through 2025 Act 247 and effective September 4, 2026. Section 71.52's history ends with 2023 Act 12. Live Legislature URLs failed during this verification, so I am not claiming a fresh fetch of those pages. The relevant permanent sources are [§71.52](https://docs.legis.wisconsin.gov/statutes/statutes/71/viii/52) and [§71.54](https://docs.legis.wisconsin.gov/statutes/statutes/71/viii/54). I independently fetched [2025 Schedule H instructions](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleH-inst.pdf), [DOR's household definitions](https://www.revenue.wi.gov/Pages/FAQS/ise-homedef.aspx), and [January 2026 Publication 127](https://www.revenue.wi.gov/DOR%20Publications/pb127.pdf). I did not obtain a 2026 Schedule H instruction booklet; the date-valid statutory text is the basis for applying the income definition to 2026. + +| Added engine source | Definition and legal check | Finding | +|---|---|---| +| `survivor_benefits` | Engine says only non-Social-Security survivor benefits. Section 71.52(6) and Schedule H 9d count pensions/annuities; 11b counts workers' compensation. | The label does not identify a countable subtype. Census source codes also include estate/trust payments and an unspecified category; see the material ambiguity below. Omitted from v2 pending subtype evidence. | +| `workers_compensation` | Named directly in §71.52(6); Schedule H 11b. | Mapping supported. | +| `veterans_benefits` | Engine documents benefits from past military service. Section 71.52(6) counts veterans' disability pensions and cash armed-forces benefits; Schedule H 9d/9h. | Cash benefit mapping supported; this is not authority to count separately described in-kind services or exempt insurance proceeds. | +| `child_support_received` | Engine says value of child-support benefits received, without a court-order restriction. Schedule H 9i includes court-ordered support and excludes voluntary support. | Overbroad. Omitted from v2 because no legal-status discriminator is provided. No WI bundle case has this input. | +| `alimony_income` | Section 71.52(6) maintenance payments; Schedule H 9i expressly includes post-2018 nontaxable alimony. | Mapping supported; federal-AGI reach checked in the probe script. | +| `disability_benefits` | Engine documents employment disability benefits, excluding Social Security and workers' compensation. Schedule H 9d/11b covers disability pensions, income continuation and loss-of-time benefits. | Employment-income replacement mapping supported. A separately identified lump-sum personal injury/insurance payment would require separate classification. | +| `educational_assistance` | Engine documents scholarships, grants and other educational assistance. Section 71.52(6), Schedule H 9h cover scholarships, fellowships and educational grants. | Scholarships/grants count, but the source is broader: the CPS ED_VAL mapping and Census definitions encompass financial assistance from friends/relatives. Such personal gifts need not be scholarships. Omitted from v2 without subtype; no WI record has the input. | +| `tax_exempt_retirement_distributions` | Adds IRA, 401(k), SEP and 403(b) nontaxable distributions. The IRA input is documented as qualifying Roth withdrawals. Schedule H 9d includes nontaxable qualified-plan/IRA withdrawals and excludes rollovers/§1035 exchanges. | Bundle's $4,700 Roth IRA withdrawal in 042 is countable. A rollover must not be passed as an ordinary withdrawal. | + +I also checked that `tax_exempt_pension_income` sums only public/private pension inputs, while the newly added retirement-distribution aggregate sums IRA/401(k)/SEP/403(b) inputs; the definitions do not duplicate those branches. The IRS gross-income source list and AGI formula contain none of the five retained v2 additions; the only IRS references found for veterans, workers’ compensation and nontaxable retirement distributions concern the sales-tax deduction table, not AGI. The retained workers’ compensation, veterans, alimony and disability variables are separate inputs. The historical CPS mapping explicitly excludes worker-compensation source code 1 from disability benefits, preventing duplication of `WC_VAL` in that branch. The original module changes no monetary parameter, but its hard-coded `2021-01-01`–`2100-12-31` effective interval is wider than this verification. For the observed computation, I checked the existing $500 dependent deduction against §71.52(5); $24,680 ceiling, $8,060 phase-out starting income, 8.785% phase-out rate, 80% credit rate and $1,460 property-tax cap against §71.54. The existing 20% rent factor agrees with §71.52(8) only when heat is included; otherwise the law uses 25%. Scenario 108 does not list heat inclusion. Counting all $10,392 of survivor benefits still makes the credit zero under either rent factor, because income exceeds the ceiling and the $2,013.6977 phase-out exceeds the property-tax cap. + +**Material source ambiguity in 108.** The scenario has $7,656 wages, $12,934 Social Security survivor benefits and a separate $10,392 generic `survivor_benefits` amount. Those are distinct listed inputs; no source identifies the latter as a pension, annuity, worker's-compensation benefit, inherited principal, or trust income. The [Census SUR_SC1 source-code definition](https://api.census.gov/data/2024/cps/asec/mar/variables/SUR_SC1.json) includes regular estate/trust payments, payments from annuities/paid-up life insurance, and an other/unknown category. The scenario does not carry this source code. The Census definition expressly excludes lump-sum inheritances and insurance payments, so I do not rely on those as a plausible interpretation of this CPS-derived amount. Periodic trust distributions can, however, include principal; the concrete primary-source example below is the basis for the narrower remaining ambiguity. I have not determined this household’s actual subtype. + +I independently read the local historical CPS construction at `policyengine-us-data` commit `42ed5d4`, `policyengine_us_data/datasets/cps/cps.py`: it assigns `person.SRVS_VAL` directly to `survivor_benefits` and retains no `SUR_SC1` source classification. The current [Census subject-definitions page](https://www.census.gov/programs-surveys/cps/technical-documentation/subject-definitions.html) directs readers to its [income definitions, Appendix A, printed A-3](https://www2.census.gov/programs-surveys/cps/methodology/Appendix%20A%20P60-200%20income%20measurement%20definition.pdf). These describe periodic estate/trust payments without restricting the definition to taxable K-1 income, while excluding lump-sum inheritances/insurance. + +A concrete primary-source counterexample to “regular means taxable in full” appears in [Treasury Decision 9102, §1.643(a)-3(e), examples 11–12](https://www.irs.gov/irb/2004-05_IRB): a trust makes an annual $20,000 unitrust distribution; under example 12 it has $5,000 ordinary income, and the trustee consistently allocates the remainder to principal without passing capital gains through in distributable net income. This is a regular trust distribution with a principal component; it is not evidence that 108 has that trust. The inference is limited: neither the CPS definition nor the retained engine input identifies the distribution’s income/principal split. Missing facts are the benefit source, whether it is a pension/annuity contract, and, for a trust/estate payment, the K-1 income and principal components. A wholly taxable trust payment or pension/annuity still supports the original $0 recomputation. + +Publication 127, section 2.B(11), printed page 14, says not to include inheritances and life-insurance proceeds in household income. DOR's [Estates, Trusts and Fiduciaries FAQ, question 16](https://www.revenue.wi.gov/Pages/FAQS/ise-estate.aspx), updated August 27, 2026, distinguishes inherited assets from income passed through on a K-1. Thus the triage assertion that every estate/trust survivor payment necessarily enters Wisconsin AGI is not established. This finding does **not** treat pension/annuity payments as exempt: their gross amount is countable under the statute. I also read [Tax 14.03](https://www.law.cornell.edu/regulations/wisconsin/Wis-Admin-Code-SS-Tax-14-03), which distinguishes annuities from nontaxable lump-sum insurance proceeds; the primary official-code URL was unavailable, so the independently fetched DOR guidance above supplies the primary-source support for the finding. + +**Scope.** The existing variable is restricted to Wisconsin, and its downstream consumers are homestead eligibility and credit. The original parameter extension sums the added person variables across *all tax-unit members*. Section 71.52(4) and DOR's definition restrict the statutory household to claimant and spouse. Consequently a dependent's newly added survivor, disability or scholarship income can incorrectly reduce the claimant's credit. The v2 masks additions to tax-unit head/spouse and leaves pre-existing baseline sources unchanged. I checked the exact head/spouse indicator formulas and the explicit bundle role inputs: only head is selected in 042/091/108, while 064 selects head/spouse and excludes its three dependents. The mask is appropriate for these bundle households; partial-year marriage or spouses maintaining separate homes would require additional facts outside this bundle. It applies only in 2026. + +I scanned all 100 household JSON records; the complete source-by-person inventory is `sweep/verify/work/r10_wi_homestead_income/input_scan.json`. Only two Wisconsin records have any added input: 042 has $4,700 tax-exempt IRA distributions, and 108 has $10,392 survivor benefits. Other nonzero added inputs occur in FL, IN, AZ, TX, WA, WV, MI, NY, PA, ID, SC, NJ, LA, OH, NC and MD and therefore do not belong in a Wisconsin exclusion set. In particular the dependent disability benefit occurs in IN scenario 067, not a Wisconsin benchmark row. + +I reviewed the completed independent baseline/original/v2 traces for all four WI cases: 042, 064, 091 and 108. The first three have household income above the ceiling before any added sources. Their listed retirement-contribution/disqualified-loss omissions cannot create another positive credit: adding those back increases income. The original module also leaves the pre-existing federal-AGI-versus-Wisconsin-AGI base and deduction-reversal defects alone; v2 does not claim to fully implement every Schedule H line. + +**Sweep and moved-row trace.** The independent original sweep completed for all 100 households and 1,984 outputs. I compared file bytes: `sweep/verify/work/r10_wi_homestead_income.csv` is identical to the builder’s `sweep/out/r10_wi_homestead_income.csv`, including its sole moved row. The v2 sweep also completed for all 1,984 outputs and has zero moved rows; scenario 108 returns to its frozen value. Both CSVs contain the same two unmoved TX112 floating-point residuals of approximately −5.68e−14. The completed calculations are in `sweep/verify/work/r10_wi_homestead_income/trace.json`. Independent ordinary arithmetic gives baseline credit `0.8 × (1,440 − 0.08785 × (20,590 − 8,060)) = 271.3916`. Under the countable-survivor interpretation, household income becomes `7,656 + 12,934 + 10,392 = 30,982`; both the ceiling and phase-out then force zero. + +| Scenario | Variable | Frozen | Recomputed by original | Confirmed yes/no | Reason | +|---|---|---:|---:|---|---| +| 108 | `state_refundable_credits` | 271.3916015625 | 0.00 | **No** | Correct conditional arithmetic, but generic survivor benefits do not establish which $10,392 is legally countable. | + +**Missed rows:** no additional moved benchmark row is established from the source inventory. Scenario 042's omitted $4,700 does change intermediate homestead income while its credit remains zero. For 108, subtype evidence is needed rather than assuming either the frozen or corrected value is legally unique. + +**V2:** `sweep/fixes/r10_wi_homestead_income_v2.py`. It retains five supported source mappings, omits generic survivor benefits, undifferentiated child support and undifferentiated educational assistance, and restricts added income to claimant/spouse and tax year 2026. It is a conservative partial correction. It does not resolve or certify scenario 108; that row must not be treated as a verified engine-defect exclusion on this evidence alone. + +**Additional completed checks.** All 16 isolated source/person probes and the 2025/2027 controls passed. Adding $10,000 of each source to the head or child leaves federal AGI unchanged. The original raises homestead income by $10,000 for either person; v2 adds the five retained sources only for the head and adds none of the three ambiguous sources. Original workers’ compensation add-backs also affect 2025 and 2027; v2 reproduces baseline income in both years. + +| WI scenario | Baseline homestead income | Original income | V2 income | Homestead credit: baseline / original / v2 | +|---|---:|---:|---:|---:| +| 042 | 59,561.703125 | 64,261.703125 | 64,261.703125 | 0 / 0 / 0 | +| 064 | 112,903.3671875 | 112,903.3671875 | 112,903.3671875 | 0 / 0 / 0 | +| 091 | 35,012.5625 | 35,012.5625 | 35,012.5625 | 0 / 0 / 0 | +| 108 | 20,590 | 30,982 | 20,590 | 271.3916015625 / 0 / 271.3916015625 | + +The original-to-v2 benchmark difference is therefore exactly scenario 108’s refundable-credit output, increasing by $271.3916015625. This is a conservative withdrawal of an unverified source classification, not certification of the frozen amount. + +## Summary + +Original and v2 counts are separate sweep results. + +The original sweeps contain eight distinct moved scenario/output pairs; the v2 +sweeps contain seven. The overlap is 042 Wisconsin before-refundable tax, reached +by both r04 and r06. Their deltas must not be added: the combined Act 15 election +produces zero. The conservative r10 v2's zero moved rows do not certify the frozen +108 value; they leave that source-classification question unresolved. + +| Module | Verdict | Original moved outputs | V2 moved outputs | Material finding | +|---|---|---:|---:|---| +| r04_capital_gain_distributions | accept_with_changes | 4 | 4 | Federal omission confirmed; correct Wisconsin's corresponding 30% subtraction and the investment-income election ordering. | +| r05_nj_worker_ui | accept_with_changes | 1 | 1 | 008 payroll correction confirmed; confine verified rates/base to 2026. | +| r06_wi_act15_before_refundable | accept_with_changes | 1 | 1 | 042 election path confirmed; preserve the permitted line-17 deduction when allocating the pooled joint line-16 subtraction. | +| r07_idaho_health_premiums | accept_with_changes | 2 | 2 | Both premium-subtraction rows confirmed; coordinate the itemization election with the subtraction. | +| r10_wi_homestead_income | reject | 1 | 0 | 108's survivor-benefit subtype is unresolved; partial v2 also narrows source and household scope. | diff --git a/reference_audit/2026-09-22/verification/v3_r08_r09_r11_r12.md b/reference_audit/2026-09-22/verification/v3_r08_r09_r11_r12.md new file mode 100644 index 00000000..982ce271 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v3_r08_r09_r11_r12.md @@ -0,0 +1,248 @@ +# Independent verification: EITC, New York, and California + +Verified September 22, 2026 in the assigned `pb-triage-verify-v3` workspace, on branch `triage-verify`. Amounts are annual US dollars for tax year 2026 unless a different year is stated. Original source modules and builder outputs were read in the supplied triage tree; all new files are in this workspace. SHA-256 checks confirm all four original modules and all four builder CSVs are unchanged. + +**Decision:** accept r08, r09, and r11 with the documented changes; reject r12 as submitted. The two disputed movements are both scenario 023's California refundable-credit output, under different fixes. Its CalEITC calculation has an independent missing AGI comparison, and r12 also uses incorrect credit anchors. Neither module verifies scenario 023’s California refundable-credit movement as a scoring exclusion. The r12 v2 is explicitly a partial parameter correction, not a certified replacement CalEITC calculation. + +All four original and all four v2 sweeps cover all 100 households and 1,984 outputs each. Every original rerun exactly matches its builder CSV, including frozen values, recomputed values, deltas, and moved flags. V2 preserves the original 2026 outputs for r08, r09, and r11; r12 v2 changes scenario 023's refundable credit and reduces its moved set from five outputs to four. See [machine-readable comparisons](work/comparison.json). + +The source `sweep.py`, requested PE 1.755.4 interpreter, scenario adapter, input renaming, output mapping, and $1/binary movement rule were retained. Initial literal runs encountered disk/memory pressure and unusually slow engine initialization. The completed r08/r09 runs use the public `Reform(initialized_baseline)` constructor within the unchanged harness to reuse initialization; these formula-only reforms do not change parameters or uprating. Both r11 and both r12 sweeps use the full `CountryTaxBenefitSystem(reform=...)` constructor. This distinction is recorded in [the serial driver](work/warm_queue.py) and its [execution status](work/warm_queue_status.json). The first r12 follow-up trace failed on an invalid variable name; the repaired trace and its parameter-equivalence checks are documented in [the trace method](work/r12_warm_trace_method.json) and [output comparisons](work/r12_trace_comparison.json). A later trace hit the engine’s parameter-cache deepcopy error; the final trace runner clears derived caches before constructing reforms, without changing parameter values or formulas. Traces and independent arithmetic are separate evidence from sweep equality. + +The tables confirm attribution to each named correction within the benchmark's existing input and continuous-calculation conventions. Other inherited defects and unresolved legal amounts are identified explicitly. No combined multi-fix tax return was certified. The later-published CPI input is distinguished from defects knowable at the July 3 freeze. + +The requested commits could not be created: Git reported `Unable to create .../.git/index.lock: Operation not permitted`; this environment makes the assigned workspace's `.git` read-only. No history was rewritten, nothing was pushed or posted, and existing workspace files were preserved. + +## r08_eitc_earned_income_deferrals + +**Verdict: accept_with_changes.** The federal wage correction and its MT/VA effects are supported. California payroll HSA contributions must remain in California wages; v2 fixes that over-application and restricts the sandbox to 2026. The unflagged CA row is **not confirmed as a legally corrected output**: its increase depends on an inherited missing AGI comparison. The full original/v2 sweeps and all 100 household traces are complete. + +I read the whole original module, the sweep brief/harness, the three relevant investigator verdicts (100 federal, 100 MT, 119 VA), and the installed formulas for federal/state earned income, payroll deductions, the ACTC phase-in, AMT exemption, CalEITC eligibility/amount, and MT/VA EITCs. I scanned all 100 household inputs and calculated baseline/original/v2 traces using `work/r08_trace.py`. + +**Legal and parameter checks.** + +- IRC 32(c)(2)(A) counts taxable employee compensation and self-employment earnings net of the section 164(f) deduction. Section 32(a)(2) phases out on the larger of AGI and earned income. Section 402(e)(3) excludes qualified traditional salary deferrals; Roth deferrals remain taxable. I checked the statutory text and the IRS's taxable-pay explanation and 2026 W-2 treatment. Sources: [IRC 32](https://www.law.cornell.edu/uscode/text/26/32), [IRC 402](https://www.law.cornell.edu/uscode/text/26/402), [IRS Publication 596, earned income](https://www.irs.gov/publications/p596), [2026 W-2/W-3 instructions, boxes 1/3/12](https://www.irs.gov/instructions/iw2w3). +- IRC 24(d)(1)(B)(i) uses section 32 earned income for ACTC. The 15% rate and $2,500 threshold follow 24(d) and 24(h)(6). Schedule 8812's earned-income worksheet starts at Form 1040 line 1z and deducts the deductible self-employment tax. The existing `ctc_phase_in_relevant_earnings` reads `eitc_earned_income`, so no separate ACTC override is needed. Sources: [IRC 24](https://www.law.cornell.edu/uscode/text/26/24), [IRS Schedule 8812 instructions](https://www.irs.gov/instructions/i1040s8). +- The module changes **no numerical tax parameter**. Its zeros are nonnegative floors and its one is an arithmetic identity in the preserved SE adjustment. The 2026 $24,500 elective-deferral limit is inherited, not hard-coded here; all three moved households' traditional-plus-Roth deferrals are below it. IRS 2026 W-2 instructions support the limit. The relevant inherited two-child EITC maximum $7,316 and unmarried phase-out start $23,890 match [Revenue Procedure 2025-32, section 4.06](https://www.irs.gov/pub/irs-drop/rp-25-32.pdf); 40% phase-in and 21.06% phase-out match IRC 32(b). +- The engine's payroll list contains only traditional 401(k), traditional 403(b), explicitly pre-tax health premiums, and payroll HSA contributions. Employer-paid ESI premiums and Roth contributions are **not** subtracted by this fix. Federal exclusions for qualifying health/HSA payroll benefits are consistent with [2026 IRS Publication 15-B](https://www.irs.gov/pub/irs-pdf/p15b.pdf). The fix retains the engine's contribution caps, self-employment terms, dependent mask, and nonnegative floors. +- Montana's 2026 match is 20% of federal EITC, verified in [MT DOR's HB 337 explanation](https://revenuefiles.mt.gov/news/recent-news/HB-337). Virginia's nonrefundable match is 20%, and its refundable alternative is also 20% in 2025–2026, under [Va. Code 58.1-339.8(B)(2)–(3)](https://law.lis.virginia.gov/vacode/title58.1/chapter3/section58.1-339.8/). Their formulas/rates are untouched. +- **California uses state wages, not unconditionally federal box 1.** R&TC 17052(c)(4)(A) retains taxable compensation and adds the CA-withholding requirement. FTB 3514 line 13 uses W-2 box 16; YCTC uses the same earned income. EDD excludes qualifying traditional deferrals but taxes payroll HSA contributions. Sources: [R&TC 17052](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?sectionNum=17052.&lawCode=RTC), [FTB 3514, line 13 and YCTC line 23](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html), [EDD DE 231EB, HSA rows on page 3](https://edd.ca.gov/siteassets/files/pdf_pub_ctr/de231eb.pdf), [EDD wage definitions](https://edd.ca.gov/en/payroll_taxes/What_Are_Wages). Thus the 401(k) subtraction in 023 is correct, but the original generic shared-wage change over-applies to California HSA contributions. +- Other direct consumers of `filer_adjusted_earnings` are state EITC computations and the AMT exemption. I read the caller search and checked federal incorporation in [Washington RCW 82.08.0206(2)(b)](https://lawfilesext.leg.wa.gov/law/RCW/RCW%20%2082%20%20TITLE/RCW%20%2082%20.%2008%20%20CHAPTER/RCW%20%2082%20.%2008%20.0206.htm), [DC Code 47-1806.04(f)(4)](https://code.dccouncil.gov/us/dc/council/code/sections/47-1806.04), [Minnesota 290.0671](https://www.revisor.mn.gov/statutes/cite/290.0671), and the taxable-wage starting lines in [Illinois expanded EITC instructions](https://tax.illinois.gov/forms/incometax/currentyear/individual/il-1040-schedule-il-e-eic-instr.html) and [Oklahoma 511-EIC](https://oklahoma.gov/content/dam/ok/en/tax/documents/forms/individuals/current/511-EIC.pdf). The report does not independently certify unchanged state eligibility rules or credit percentages. Repointing `amt_exemption` to the original sum of `adjusted_earnings` is algebraically identical to its baseline input, preserving AMT rather than adopting an EITC definition for that separate tax. + +**Scope and the unflagged CA result.** + +The original replacements have no year restriction. The relevant taxable-pay rule is longstanding, but an undated override is broader than this 2026 audit; v2 explicitly delegates to the original formulas outside 2026. V2 excludes only traditional deferrals and explicitly pre-tax health premiums when computing California state wages, retaining payroll HSA amounts. Federal earned income still excludes qualifying HSA payroll contributions. + +FTB 3514's worksheet lines 3–5 require a second credit lookup at federal AGI and selection of the smaller amount when AGI exceeds the first threshold. Baseline `ca_eitc` instead phases out only on `filer_adjusted_earnings`; `ca_eitc_eligible` merely tests whether AGI exceeds the maximum. In scenario 023, the $8,000 taxable 403(b) distribution makes AGI exceed both original gross wages and corrected wages. That AGI is unchanged by r08. Consequently, with the required AGI comparison in place, reducing wages in this phase-out region would **not increase** CalEITC. The observed +$26.66 is a mechanical deferral effect in an already defective credit formula, not confirmation of the resulting $174.97 as lawful. I have not expanded this wage-base patch to fix that separate AGI defect or the separate indexing defect reviewed in r12. + +**Every moved row.** Both independent sweeps reproduce these values. All 1,984 frozen/recomputed/delta cells and moved flags match the builder exactly; v2 is identical on this bundle. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_023 | state_refundable_credits | 148.31086731 | 174.96821594 | No | Correct wage exclusion, but missing AGI comparison makes the claimed increase legally unsupported. | +| scenario_100 | federal_refundable_credits | 2,878.09667969 | 822.28826904 | Yes | Remove $3,859 traditional deferral from two-child EITC/ACTC earnings. | +| scenario_100 | state_refundable_credits | 473.17770386 | 164.45765686 | Yes | MT receives 20% of corrected federal EITC. | +| scenario_119 | federal_refundable_credits | 2,854.00878906 | 3,366.07373047 | Yes | Correct earned income lowers the phase-out base to the larger unchanged AGI. | +| scenario_119 | state_income_tax_before_refundable_credits | 1,859.09436035 | 1,756.68139648 | Yes | Larger federal EITC increases VA's nonrefundable 20% credit. | + +For 100, earned income falls from $5,914.720703 to $2,055.720703. The EITC phase-in yields $822.288281; ACTC earnings are below $2,500, so its $512.208105 phase-in disappears. For 119, earned income falls from $55,000 to $51,912.800049. AGI is approximately $52,568.544052 (wages less traditional 401(k) and IRA plus $800 interest); that larger AGI becomes the phase-out base. Reducing the base increases EITC by approximately `0.2106 * (55000 - 52568.544052) = $512.06`. The engine trace confirms EITC $764.233887 → $1,276.298828 while ACTC stays $2,089.774902. This explains the direction and isolates the cause of the unflagged federal movement. Independent arithmetic is saved in `work/r08_hand_checks.py` and `work/r08_hand_checks.json`. These decimal calculations preserve the benchmark's continuous formula; engine float32 values can differ by fractions of a cent. + +**Scope, missed rows, and v2 difference.** No additional moved row was missed in the 100-household scan. Earned income changes in 20 households. Four other households with pre-tax contribution amounts (003, 004, 073, 085) have no wages to remove. Outside 023/100/119, the affected households are 005, 015, 018, 026, 037, 043, 049, 052, 064, 070, 082, 089, 091, 093, 099, 110, and 120. Their EITC/ACTC outputs remain unchanged: credits are zero through eligibility/phaseout, or the ACTC is already constrained elsewhere. In particular, 043 is a childless 66-year-old outside the modeled federal/Colorado age eligibility, and 026's ACTC remains $2,013.489258 after its wages fall. AGI and AMT exemption are identical in baseline/original/v2 for **all 100 households**. Every bundle trace is identical between original and v2. Evidence: `work/r08_validation.json` and `work/r08_traces.json`. + +No bundle household supplies payroll HSA or pre-tax health-premium inputs. I therefore added a $1,000 payroll HSA contribution to a copy of 023. Federal earned income is $13,664.166016 under both original and v2. California earned income is incorrectly $13,664.166016 under the original but $14,664.166016 under v2, correctly retaining the state-taxable HSA amount. This verifies the scope correction; the credit amounts in that diagnostic still inherit the separate AGI defect. Evidence: `work/r08_hsa_counterfactual.json`. + +The original and v2 sweeps each produce 1,984 outputs, five moved rows, and no small nonzero deltas. They use the untouched source harness with the public `Reform(initialized_baseline)` constructor to reuse the already loaded engine; r08 changes no parameters or uprating. `work/warm_queue.py` records this construction method. `work/r08_validate.py` checks population invariants and the HSA counterfactual. The v2 changes **no bundle output**; its changes affect households with payroll HSA contributions and restore baseline formulas outside 2026. + +**V2:** `sweep/fixes/r08_eitc_earned_income_deferrals_v2.py`. It corrects the California HSA scope and year scope only; it does not certify the CA 023 credit amount. Evidence script: `sweep/verify/work/r08_trace.py`. + +Installed source root: `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. Files read include `variables/gov/irs/credits/earned_income/{eitc,eitc_earned_income}.py`, `variables/gov/irs/credits/ctc/refundable/ctc_phase_in_relevant_earnings.py`, `variables/gov/irs/income/{adjusted_earnings,filer_adjusted_earnings}.py`, `variables/gov/irs/income/taxable_income/adjusted_gross_income/irs_gross_income/{irs_employment_income,pre_tax_contributions,earned_income/earned_income}.py`, `parameters/gov/irs/gross_income/pre_tax_contributions.yaml`, `variables/household/expense/retirement/elective_deferral_contribution_scale.py`, `variables/gov/irs/tax/federal_income/alternative_minimum_tax/exemption/amt_exemption.py`, `variables/gov/states/ca/tax/income/credits/earned_income/{ca_eitc,ca_eitc_eligible}.py`, and MT/VA `credits/eitc` formulas. + +## r09_ny_rptc_rent_cap + +**Verdict: accept_with_changes (year scope).** The sole 2026 exclusion is confirmed. The original applies the post-2024 tables in earlier years; v2 confines the sandbox correction to 2026 and produces exactly the same benchmark outputs. + +I read the complete original module, `SWEEP_BRIEF.md`, `sweep.py`, the 1.755.4 `ny_real_property_tax_credit` and `rent` formulas, the `assessed_property_value` definition, all YAML parameters under `gov/states/ny/tax/income/credits/real_property_tax`, the two relevant entries in `triage/verdicts.json`, and all five NY households in the published `scenarios.csv`. These source files are under `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/`. + +Legal and parameter checks: + +- The $450 average monthly **adjusted rent** limit is separate from the 25% rent-to-property-tax conversion. For full-year rent this is $5,400 annually. The $18,000 FAGI and $85,000 full property value limits include equality. Age 65 includes a claimed dependent. The amount is a flat band amount after a strictly positive excess-tax test. These checks match [Tax Law 606(e)(1), (3)(B), and (7)](https://www.nysenate.gov/legislation/laws/TAX/606). +- I checked every Table A top/amount pair: 3,000/375; 5,000/330; 7,000/300; 9,000/260; 11,000/230; 14,000/200; 18,000/150. Table B pairs are 5,000/75; 9,000/70; 14,000/60; 18,000/50. Table 1 uses the Table A tops and rates 0.035, 0.040, 0.045, 0.050, 0.055, 0.060, 0.065. Each upper boundary belongs to the lower band, and rates are selected rather than accumulated. These values and the strict gate agree with [2025 IT-214, Tables 1/A/B and lines 18–20](https://www.tax.ny.gov/pdf/current_forms/it/it214_fill_in.pdf). +- Flooring negative FAGI to zero and rounding nonnegative FAGI half-up to whole dollars follow line 8 and the general completion instructions in [2025 IT-214-I](https://www.tax.ny.gov/pdf/current_forms/it/it214i.pdf). The instructions also distinguish adjusted rent from rent including utilities/board, require market rather than assessed property value, and require qualifying tax payments net of STAR credit. +- These are 2025-and-later rules, including 2026, rather than a 2026 inflation update. I checked [DTF's Part RR explanation](https://www.tax.ny.gov/legal/2025/pit-corp-changes.htm), current statutory effective-year language, and the locally saved enacted bill excerpts `triage/scenario_118__state_refundable_credits/part_rr.txt` and `part_rr_rest.txt` (Part RR sections 1–9). I read the old 50% rate and maximum-chart YAML values; the all-components reform does not use them for 2026. They belong to its diagnostic component-off path. + +Scope findings: + +The replacement is NY-only and only updates `ny_real_property_tax_credit`. Its six named components are broader than the rent comparison alone, but all target the same IT-214 calculation and the module explicitly describes them. The original uses an undated formula, applying post-2024 flat tables in earlier years. A 2024 probe with age 72, FAGI $2,500, and $1,000 property tax returned **$341 baseline, $375 original, $341 v2**. The v2 delegates to the original 1.755.4 formula outside 2026. This preserves baseline behavior outside the verification task; it does not claim to correct the engine's entire historical implementation. Probes also confirmed baseline preservation in 2025 and 2027. + +The baseline's use of assessed value rather than full market value, absence of separate utility/board adjustments, and absence of several explicit residency/dependency/property-exemption tests remain modeling limitations. I did not expand this fix to unrelated eligibility inputs. The bundle uses full-year situations. In the five NY households I read, no adjusted-rent utility inclusion, real-property market value, or STAR credit is provided. The owner at 118 has property taxes but no assessed value input; the benchmark's omitted-input convention supplies zero. + +I also read PE's tax-unit entity and head/spouse/dependent formulas and `Scenario.to_pe_household()`. TaxUnit has a generic member role; the default dependent flag is `~head & ~spouse`, so non-filer members are dependents by construction unless explicitly overridden. The any-member age test therefore relies on that grouping convention. An explicitly unclaimed third adult could violate the convention; this is not a general legal dependency validator (the old `~is_dependent` age test could also count such an adult). None of the five NY scenarios contains an elderly non-filer whose dependency is overridden: they contain a head alone, head/spouse, or head and one age-1 child. No bundle over-application was identified from this grouping issue. + +I scanned all 100 scenario records: all use 2026, and these are the only NY cases. The following are independently calculated PE values; rent and taxes are summed across members. + +| Scenario | FAGI | Annual rent | Property tax | Baseline RPTC | Original / v2 RPTC | Scope finding | +|---|---:|---:|---:|---:|---:|---| +| 004 | 343.72 | 0 | 0 | 0 | 0 / 0 | No qualifying payment. | +| 071 | 5,078.00 | 0 | 0 | 0 | 0 / 0 | No qualifying payment, including through its housing-assistance input. | +| 082 | 117,585.15 | 0 | 9,216.53 | 0 | 0 / 0 | FAGI exceeds $18,000; other state refundable credits remain $650.50. | +| 104 | 0 | 6,483.53 | 0 | 375 | 0 / 0 | Rent exceeds the cap. | +| 118 | 0 | 0 | 1,633.50 | 375 | 375 / 375 | Age 74; positive property tax exceeds 3.5% of zero FAGI. | + +Every moved row: + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_104 | state_refundable_credits | 375.00 | 0.00 | Yes | $6,483.529296875 annual rent / 12 = $540.29410807 monthly, above $450; the $1,620.88232422 tax equivalent does not determine the rent cap. | + +The only nonzero listed income for 104 is $14,096 Social Security retirement and $20,520 veterans benefits. My trace gives taxable Social Security and FAGI both zero; `rent` equals `pre_subsidy_rent`. No utility inclusion or housing subsidy is listed. The rent-cap-only reform gives **$0**, while the Part-RR-only reform leaves **$375**. Thus this row moves solely because of the rent-cap comparison, and exactly matches the investigator's corrected zero. Scenario **118 stays at $375** under baseline, rent-cap-only, Part-RR-only, the original full reform, and v2. + +**Missed rows: none identified.** Among the five NY cases only 104 has rent; the other payment/income combinations above explain why none of the additional Part RR corrections changes their outputs. No non-NY output moved. + +**Reproduction and tests.** Both completed sweeps contain **1,984 outputs** and exactly one moved row, with zero smaller nonzero deltas. Every original recomputed value and moved flag exactly matches `sweep/out/r09_ny_rptc_rent_cap.csv`; every v2 recomputed value and flag exactly matches the original rerun. I compared the CSVs independently with the standard-library CSV reader, and the trace script also asserts those comparisons. All **35 synthetic probes** passed: Table A/B boundaries, $3,000.49/$3,000.50 rounding, negative FAGI, the $18,000 limit, rent above $5,400, equality versus positive excess, a claimed age-65 dependent, NY-only applicability, and other-year preservation. + +The initial direct commands were interrupted during costly system initialization under memory pressure. The completed runs used the unchanged original `sweep.py` via `runpy` in a warmed PE 1.755.4 process, supplying the public `Reform(initialized_baseline)` constructor for these formula-only reforms. Household construction and all 1,984 reference calculations remained the original harness's. This is system reuse, not a claim that the initial command completed unchanged. `r09_commands.txt` records the attempts and method; the successful output appears in `r11_all.log`. + +Corrected module: `sweep/fixes/r09_ny_rptc_rent_cap_v2.py`. Sweep outputs: `sweep/verify/work/r09_ny_rptc_rent_cap.csv` and `sweep/verify/work/r09_ny_rptc_rent_cap_v2.csv`. **V2 changes no 2026 exclusion or amount.** + +Reproduction script: `sweep/verify/work/r09_verify.py`; complete household inputs, component calculations, comparison results, and synthetic probes: `sweep/verify/work/r09_evidence.json`. + +| Module | Verdict | Confirmed moved rows | Missed rows | Required change | +|---|---|---:|---|---| +| r09_ny_rptc_rent_cap | accept_with_changes | 1 | None identified | Limit the reform to the assigned 2026 year; benchmark outputs unchanged. | + +## r11_ca_itemized_conformity + +**Verdict: accept_with_changes** — the California charity/miscellaneous-deduction correction is supported; the original module's undated variable replacements are not restricted to 2026, despite its description. V2 bounds them to the audited year. Both full sweeps and all moved-row traces are complete. V2 preserves every 2026 output and restores baseline results in the 2025/2027 boundary checks. + +I read the complete original module, `SWEEP_BRIEF.md`, `sweep.py`, the `scenario_005__state_income_tax_before_refundable_credits` verdict, the relevant installed engine formulas/parameters, and all five California records in the 100-household `scenarios.csv`. + +**Legal and parameter checks.** + +- **Conformity and the freeze date:** R&TC 17201(a) incorporates IRC Part VI subject to California exceptions. R&TC 17024.5(a)(1)(Q) uses the IRC as enacted January 1, 2025 for 2025 and later taxable years; (h)(2)(A) uses federal AGI for AGI limitations; (h)(6) preserves scheduled expirations. These provisions support excluding the later OBBBA floor. The module cites the September 14, 2026 SB 1435 amendment, but the relevant conformity date and provisions were already enacted by **SB 711 on October 1, 2025**, before the July 3, 2026 reference freeze. This is an engine defect, not a newly published parameter. Sources: [R&TC 17201](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17201.), [R&TC 17024.5](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17024.5), [SB 711, sections 1 and 11](https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202520260SB711). +- **Charity:** I checked the explicit `0.5` overall cap and the inherited noncash caps `0.5` and `0.3`. IRC 170(b)(1)(A)/(B), as of the conformity date, establishes the ordinary 50%/30% contribution categories; the pre-OBBBA 60% cash rule expires before 2026. OBBBA section 70425 adds the federal `0.005` floor for 2026 and makes the 60% limit permanent. California's January 2025 conformity date does not incorporate those amendments. FTB's 2025 Schedule CA instructions, revised April 2026, lines 11–12 expressly use 50% of federal AGI. Sources: [2024 IRC 170](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partVI-sec170.htm), [Public Law 119-21, section 70425](https://www.govinfo.gov/content/pkg/PLAW-119publ21/html/PLAW-119publ21.htm), [FTB Schedule CA instructions](https://www.ftb.ca.gov/forms/2025/2025-540-ca-instructions.html). +- **Miscellaneous deductions:** R&TC 17076(a)/(c) applies the 2% floor and rejects IRC 67(g). Schedule CA Part II lines 19–25 includes employee expenses and tax-preparation fees, calculates 2% of federal AGI with a zero minimum, then subtracts that amount with a zero minimum. Therefore the module's `max(0, expenses - .02 * positive_agi)` matches these inputs. The installed sources list contains exactly employee expenses and tax-preparation fees; its floor parameter is `0.02`. Sources: [R&TC 17076](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17076.), [FTB Schedule CA, page 6](https://www.ftb.ca.gov/forms/2025/2025-540-ca.pdf). +- **AMT and state-only substitution:** Schedule P line 5 adds back Schedule CA line 25, so replacing federal `misc_deduction` with `ca_misc_deduction` in the AMTI source list is necessary. The installed `ca_amti_adjustments` formula already selects this list only when California itemized deductions exceed the standard deduction; otherwise it adds back the standard deduction. Sources: [Schedule P instructions, line 5](https://www.ftb.ca.gov/forms/2025/2025-540-p-instructions.html), [Schedule P, page 1](https://www.ftb.ca.gov/forms/2025/2025-540-p.pdf). The formula preserves the existing California investment-interest/property-tax adjustments and higher-income limitation, and replaces only the federal charity/miscellaneous components. + +The reused charitable formula is not a complete model of every gift category: IRC 170(b)(1)(C)/(D) also has capital-gain-property limits of 30%/20%, which the installed federal formula explicitly says it does not fully model. This is inherited behavior. All five California records have zero noncash gifts designated for non-50%-limit organizations; the three donors' amounts are well below even 20% of their AGI. No charity cap determines any moved row. + +**Scope and missed rows.** + +The input scan found exactly `005`, `022`, `023`, `031`, and `099` in California, all in 2026. Only `005`, `022`, and `099` report gifts or miscellaneous expenses. `023` and `031` have zero charity, employee expenses, and tax-preparation fees. The state guards are present on all three replacement variables. No additional California household with these named inputs was missed. Every bundle record is in 2026, each CSV state agrees with its scenario JSON, and no household/tax-unit/SPM state override is supplied. The scan is saved in `work/r11_input_scan.json`; baseline/fix traces for all five California cases are in `work/r11_ca_itemized_conformity_traces.json`. California AMT stays zero in every case. + +The original formula and adds/subtracts replacements apply without a year condition, while its AMTI parameter edit alone is limited to January 1–December 31, 2026. A copy of scenario 005 recalculated for 2025 changes tax from $41,160.1171875 to $40,873.8125; for 2027 it changes $40,767.16796875 to $40,236.78515625. These are synthetic year checks, not additional bundle rows. V2 adds a 2026 start/end to the new helper variables and restores the original engine calculation in the replacement total for other years. It also fixes the preexisting concatenated metadata URLs and records the pre-freeze SB 711 authority. It does not change the 2026 arithmetic order. + +**Independent arithmetic.** `work/r11_hand_checks.py` rebuilds each moved household's AGI from its inputs while retaining the unrelated baseline IRA/401(k) choices. It computes restored deductions of $5,686.06229 (005), $6,713.30651 (022), and $797.92522 (099). Multiplying by the unchanged marginal rates (9.3%, 8%, 8%) gives tax reductions of $528.803793, $537.064521, and $63.834018. Each agrees with the original CSV within $0.001. The completed original sweep and traces independently reproduce the same effects. + +**Every moved row.** All amounts below were independently recomputed. All 1,984 original-rerun CSV records equal the builder CSV exactly: three moved rows, no small nonzero deltas. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_005 | state_income_tax_before_refundable_credits | 41,051.51171875 | 40,522.70703125 | Yes | Restore $2,573.53 gifts plus $3,112.53 miscellaneous deduction; 9.3% marginal rate. | +| scenario_022 | state_income_tax_before_refundable_credits | 2,439.650146484375 | 1,902.58544921875 | Yes | Restore $545.75 charitable floor plus $6,167.56 miscellaneous deduction; 8% marginal rate. | +| scenario_099 | state_income_tax_before_refundable_credits | 4,493.7431640625 | 4,429.9091796875 | Yes | Restore $797.92 charitable floor at 8%; employee expenses remain below 2% of AGI. | + +The direct traces show: + +- **005:** federal/California AGI $536,764.50; state itemized deductions $15,338.790039 → $21,024.851562, and taxable income $521,425.71875 → $515,739.65625. The $1,941.21 higher-income limitation is unchanged. The rerun result is within $0.003 of the investigator's $40,522.71. +- **022, unflagged:** federal AGI $109,149.1640625, California AGI $91,524.4140625. The formula correctly uses **federal** AGI for both floors. Charity $14,847.666016 → $15,393.412109; miscellaneous deduction $0 → $6,167.560547. State itemized deductions $23,676.666016 → $30,389.972656; taxable income $67,847.75 → $61,134.441406. +- **099, unflagged:** federal AGI $159,585.046875; charity $11,823.075195 → $12,621.00. Employee expenses $1,262.647095 remain below the $3,191.70 floor. Itemized deductions $20,476.574219 → $21,274.50; taxable income $139,108.46875 → $138,310.546875. + +An additional inherited AMTI defect limits broader claims about this sandbox. `ca_pre_exemption_amti.py` adds **all** pre-limitation deductions to taxable income; Schedule P line 18 instead reverses the high-income deduction limitation. As a result, this fix increases modeled AMTI in 005/022 when it adds back miscellaneous expenses. The builder's explanation that omitting the new add-back would lower modeled AMTI is inaccurate for this engine. This does not affect any reported row: AMT is zero before and after in all five California cases. V2 leaves this separate AMTI formula unchanged and corrects its explanatory comment. [FTB Schedule P, line 18 instructions](https://www.ftb.ca.gov/forms/2025/2025-540-p-instructions.html). + +**Reproduction and evidence.** + +The original sweep completed through `work/r11_run_all.py`, invoking the unchanged absolute source `sweep.py`. Its output is `work/r11_ca_itemized_conformity.csv`. Independent traces use the public `Reform` API to clone the already initialized baseline, avoiding repeated initialization; every traced state-tax value agrees with the full sweep. The v2 sweep also invokes the unchanged harness with full `CountryTaxBenefitSystem` construction. Its 1,984-output comparison passed. A subsequent detailed trace raised an `IndexError` while indexing a copied cached parameter node. The successful trace-only retry, `work/r11_r12_trace_retry.py`, used a fresh interpreter and cleared derived parameter caches before cloning reforms. `work/r11_compare.py` passed: all 1,984 original/v2/builder output rows agree numerically; every original/v2 CA tax trace agrees with its sweep; federal AGI, federal tax, and CA refunds stay unchanged; CA AMT is zero throughout; and the v2 2025/2027 synthetic results equal baseline exactly. These assertions are saved in `work/r11_validation.json`. The first executions stalled reading installed Python bytecode and were interrupted; the retry uses `PYTHONPYCACHEPREFIX` pointing to an unused workspace path plus `PYTHONDONTWRITEBYTECODE=1`. No source environment file was modified. + +Installed engine files read are beneath `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`: `variables/gov/irs/income/taxable_income/deductions/itemizing/{charitable_deduction,misc_deduction,total_misc_deductions}.py`; `variables/gov/states/ca/tax/income/deductions/{ca_deductions.py,itemized/ca_itemized_deductions.py,itemized/ca_itemized_deductions_pre_limitation.py}`; `variables/gov/states/ca/tax/income/alternative_minimum_tax/{ca_amt.py,amti/ca_amti.py,amti/ca_amti_adjustments.py,amti/ca_pre_exemption_amti.py,exemption/ca_amt_exemption.py}`; `parameters/gov/irs/deductions/itemized/{charity/{floor,ceiling},misc}/`; and `parameters/gov/states/ca/tax/income/amt/`. + +**V2 module:** `sweep/fixes/r11_ca_itemized_conformity_v2.py`. **V2 differences in moved outputs:** none; all 1,984 output records match the original sweep numerically, and all trace/year-boundary assertions passed. + +## r12_ca_2026_indexing + +**Verdict: reject as submitted.** The annual-liability indexing is supported, but the refundable-credit branch combines incorrect CalEITC starting amounts with an unverified earnings cutoff. The original also changes parameters outside 2026. A narrower v2 preserves the supported parameter changes and leaves the uncertain cutoff/reduction parameters at baseline. Its credit output is not a certification of the complete legal CalEITC calculation. + +I read `SWEEP_BRIEF.md`, the complete original module, `sweep.py`, the scenario CSV, the scenario 005 entry in `verdicts.json`, the engine's CalEITC and withholding formulas and relevant parameter files. The original module, harness and triage evidence are under `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/`. The scenario bundle is `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/publish/us_full_run_20260612_policyengine_4_16_1_populace/us/scenarios.csv`. Originals were not edited. The independent trace script, inputs, and computed parameter values are in `sweep/verify/work/r12_*` in the assigned workspace. + +### Legal rules and parameter checks + +Joint/surviving-spouse rate schedules were also checked against [R&TC 17045](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17045.): tax is twice the tax on half of income. + +The factor is supported directly by [DIR's percentage-change table](https://www.dir.ca.gov/oprl/CPI/PresentCCPIchange.PDF), which reports **3.4% for June 2026 California all-urban consumers**, dated August 12, 2026. [DIR's index table](https://www.dir.ca.gov/OPRL/CPI/PresentCCPI.PDF) confirms June 2025 **353.044** and June 2026 **364.969**. Thus the module's factor 1.034 is supported independently of its historical-rounding claim. + +I checked the statutory June-to-June adjustment and nearest-dollar rule in [R&TC 17041(h)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17041.), standard deductions and twice-single rule in [17073.5(d)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17073.5.), exemptions and twice-single joint credit in [17054(i)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17054.), exemption phaseout in [17054.1(f)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17054.1.), itemized-limitation threshold in [17077(c)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17077.), AMT in [17062(b)(7)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17062.), and renter AGI caps/twice-single rule in [17053.5(j)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17053.5.). Local text copies read are the corresponding `sweep/work/r12_ca_2026_indexing/src/rtc*.txt` files in the original triage tree. + +All 2026 amounts below are **my statutory calculations from verified FTB 2025 inputs and DIR's 3.4% factor**, not amounts I read in a published FTB 2026 tax table. I could not retrieve a search-discovered 2026 indexing memo hosted at `https://www.caltax.org/regulatory-issues/ftb/2026-PIT-Indexing-Memo.pdf`; I make no claim about its contents or whether FTB has published the 2026 amounts elsewhere. + +| Parameter/status | Verified 2025 input | Computed 2026 | +|---|---|---| +| Standard deduction, single/separate; joint/surviving/HOH | 5,706; 11,412 | 5,900; 11,800 | +| Personal/senior/blind exemption per applicable individual; dependent | 153; 475 | 158; 491 | +| Single/separate bracket breakpoints | 11,079; 26,264; 41,452; 57,542; 72,724; 371,479; 445,771; 742,953 | 11,456; 27,157; 42,861; 59,498; 75,197; 384,109; 460,927; 768,213 | +| Joint/surviving breakpoints | Twice single | 22,912; 54,314; 85,722; 118,996; 150,394; 768,218; 921,854; 1,536,426 | +| HOH bracket breakpoints | 22,173; 52,530; 67,716; 83,805; 98,990; 505,208; 606,251; 1,010,417 | 22,927; 54,316; 70,018; 86,654; 102,356; 522,385; 626,864; 1,044,771 | +| Itemized/exemption phaseout AGI, single/separate; joint/surviving; HOH | 252,203; 504,411; 378,310 | 260,778; 521,561; 391,173 | +| AMT exemption, single/HOH; separate; joint/surviving | 92,749; 61,830; 123,667 | 95,902; 63,932; 127,872 | +| AMT phaseout start, same ordering | 347,808; 231,868; 463,745 | 359,633; 239,752; 479,512 | +| AMT end, start plus four times exemption | Derived from 25% phaseout | 743,241; 495,480; 991,000 | +| Renter AGI cap, single/separate; joint/surviving/HOH | 53,994; 107,988 | 55,830; 111,660 | +| YCTC/FYTC maximum; phaseout start; YCTC loss limit | 1,189; 27,425; 35,640 | 1,229; 28,357; 36,852 | +| CalEITC earned-income/first phaseout amounts, 0/1/2+ children | 4,661; 6,998; 9,823 | 4,819; 7,236; 10,157 | +| CalEITC investment-income ceiling | 4,814 | 4,978 | + +Sources checked for those base amounts: [FTB October 2025 indexing](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2025/10.html) for standard deductions, exemption credits, renter caps and schedules; [2025 Form 540 booklet](https://www.ftb.ca.gov/forms/2025/2025-540-booklet.html) for AGI thresholds and filing statuses; [2025 Schedule P instructions, line 22 worksheet](https://www.ftb.ca.gov/forms/2025/2025-540-p-instructions.html) for all AMT values and 25% phaseout; [2025 FTB 3514 booklet](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html) for the credit parameters. The corresponding saved source files under original `sweep/work/r12_ca_2026_indexing/` were also read. + +The credit indexing authority is [R&TC 17052(e),(o)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052.), [17052.1(a),(b)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052.1.), and [17052.2(a)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052.2.). Three findings prevent accepting the submitted credit branch: + +1. **Wrong CalEITC anchors.** The hard-coded 2025 final-phaseout credits **257/649** come from projected engine parameters. Starting with the statute's 2019 **200/505** and applying actual annual factors with annual nearest-dollar rounding yields **252/636 for 2025**, then **261/658 for 2026**. The submitted module instead produces **266/671**. Rates used were 2020 1.4%, 2021 4.4% ([FTB October 2021](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/october-2021/indexing.html)); 2022 8.3%, 2023 3.1% ([FTB October 2023](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2023/10.html)); 2024 3.3%, 2025 3.0% (October 2025 source above). This is a demonstrated numerical error, not just missing documentation. +2. **Unsupported final cutoff/reduction.** The 2025 cutoff **32,901** is in FTB 3514, but directly multiplying it by 1.034 to get **34,020** does not follow an explicit statutory indexing instruction for that cutoff. Section 17052(o)(4) indexes specified credit/income amounts and retains the applicable phaseout percentages. The cutoff is an output of that computation/table. I did not establish the precise 2026 FTB cutoff, so I do not certify 34,020. Consequently the proposed YCTC graduated reduction **21.70** is also unverified. V2 leaves these parameters at baseline. +3. **FYTC omission and pre-existing model approximation.** The submitted module changes YCTC's graduated reduction but omits FYTC's corresponding reduction even though the two statutory rules have the same endpoint condition. Separately, `ca_eitc.py` uses only earned income and continuous interpolation: the [3514 worksheet, lines 3–5](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html) also tests federal AGI and takes the smaller credit. Scenario 023 has taxable retirement income and is exposed to that separate omission. V2 does not silently fix it. + +**Freeze date:** the correct observed factor was not determinable on **2026-07-03**. DIR's [publication notice](https://www.dir.ca.gov/oprl/CPI/11-07-2019%28cpi%29.pdf) states an approximately seven-week publication lag after the reference month; underlying June CPI data were embargoed until [July 14, 2026](https://www.bls.gov/news.release/archives/cpi_07142026.htm). The [2026 FTB estimated-tax instructions](https://www.ftb.ca.gov/forms/2026/2026-540-es-instructions.html) expressly use 2025 deductions/rates for estimates. Thus replacement of projected CPI amounts is `projection_superseded`. The six filing-status breakdown parameters remain at their explicit 2025 values despite file-level uprating metadata; this is distinguishable from the final 3.4% factor becoming available. Knowing the indexing law on July 3 did not reveal June's published factor, so the full later-observed dollar change cannot be attributed to an error knowable at freeze. + +### Scope and withholding + +The only California households in the 100-row scenario file are 005, 022, 023, 031 and 099; I read all their inputs. Across all 100 records, CSV and JSON states agree, and none supplies a `state_name`, `state_code`, or `ca_*` input override. Updates target the California income-tax subtree and declare a calendar-2026 period, but the constructor applies reforms before and after uprating. My runtime comparison found **two changed parameter leaves in 2025 and 61 in 2027** for the original. In 2025, CalEITC final-phaseout anchors change from baseline 257/649 to the raw 2019 values 200/505. In 2027, the first single bracket changes from 11,629.176791 to 11,079 and the personal exemption from 160.597892 to 153. The constructor's first reform pass inserts a raw-value restoration at 2027; `uprate_parameters.py` starts at the latest defined date, so it skips generation of the anchors' intervening historical values and replaces future projections with stale amounts. This is actual year over-application. + +V2 seeds each edited parameter with fully initialized baseline history before replacing 2026. The same runtime comparison found **zero changed leaves in 2025 and 2027** for v2, and 91 changed leaves in 2026 (original: 93). Evidence: `sweep/verify/work/r12_year_scope.json`, produced by `r12_final_verify.py`. I checked those three years, not every possible year. The withholding override changes only the 2026 parameter source and keeps the original formula otherwise. + +Preserving withholding is appropriate to isolate annual-liability indexing. It must not be described as verified EDD withholding: [EDD's 2026 Method B](https://edd.ca.gov/siteassets/files/pdf_pub_ctr/26methb.pdf) applies wages, filing status and withholding allowances, including a 5,706 single standard deduction and annual breakpoints based on 2025 amounts; the engine proxy applies single liability rates to each person's AGI, and its baseline brackets already include a projected increase. The module preserves that pre-existing approximation. Actual withholding is not supplied in these scenarios. + +The scan of all `gov.states.ca.cpi` references in the installed California income-tax parameter subtree found no additional CPI-indexed parameter family omitted by `modify`. The FYTC graduated reduction omission has no bundle exposure: every California adult is at least 28, outside the statutory 18–25 FYTC range. Scenario 031 has no earnings: pension 13,608 plus IRA distribution 800 less alimony 1,165 gives AGI 13,243. My arithmetic gives gross state tax 75.37 before and 73.43 after indexing, both absorbed by personal/senior credits exceeding 312; no credit eligibility change is identified. No missed household was identified from the input scan and arithmetic that should cross the harness's $1 threshold from the confirmed parameter changes. + +### Reproduction and every moved output + +**Both full sweeps completed.** The original rerun reproduces all **1,984 rows exactly**, including its **five moved outputs**, in `sweep/verify/work/r12_ca_2026_indexing.csv`. The original harness and full `CountryTaxBenefitSystem` constructor were used. V2 produces **1,984 outputs, four moved, one smaller nonzero delta**, in `r12_ca_2026_indexing_v2.csv`. `r12_compare.py` verified complete scenario/variable coverage and equality of every original rerun field; its saved result is `r12_comparison.json`. Earlier duplicate runs were interrupted during engine loading to reduce system memory pressure. Bytecode caching options changed caching only; all outputs were written only to the assigned workspace. + +**Detailed traces completed for all five California households.** The initial scratch trace's incorrect federal variable name was corrected to `income_tax_refundable_credits`. The successful trace used the public `Reform(initialized baseline)` constructor after clearing derived parameter caches, then verified all **484 California parameter leaves, including 454 numeric leaves**, against baseline plus the independently saved original full-constructor differences for 2026. Every value matched; **no parameter restoration was needed**. All **40** baseline/submitted state-tax, state-refund, federal-tax and federal-refund comparisons match the completed original sweep exactly. Evidence: `r12_warm_trace_method.json`, `r12_trace.json` and `r12_trace_comparison.json`. + +The traces also confirm unchanged per-person California withholding in all five households. AMT, YCTC and FYTC are zero before and after throughout; CalEITC is nonzero only for 023. Scenario 031's actual itemized deduction is 1,441.574951, below both standard deductions, confirming the zero-tax explanation above. Thus no moved row is attributable to an AMT, renter-cap, YCTC or FYTC eligibility change. + +The v2 runner saved its completed CSV and year-scope JSON, but a clean process exit was not observed after the final scope messages; its next statement cancels a diagnostic traceback timer. Ctrl-C did not end that cleanup wait, and a narrowly targeted termination attempt failed because the sandbox could not enumerate processes. The saved sweep and comparison results are complete. + +I completed separate executable arithmetic in `sweep/verify/work/r12_hand_checks.py` and saved `r12_hand_checks.json`. It reconstructs income and relevant deductions from the household inputs, applies the independently checked 2026 parameters and rate schedules, and reproduces all four moved state-liability values within one cent (maximum discrepancy 0.00154). It also reproduces 031's zero and independently calculates the submitted 023 credit change. This arithmetic retains unrelated sandbox IRA, charity-floor and miscellaneous-deduction treatments. The old projected factor is recovered from the baseline first bracket saved in my own full-constructor scope comparison, divided by its explicit 2025 source value. The table's yes means that the indexing root cause is supported by this arithmetic, not that the full tax reference is otherwise legally correct. + +For unflagged 022, the scenario is single, age 77, without children; its two exemption units are personal plus senior, despite the separate person input `is_surviving_spouse=true`. For unflagged 099, two adults and two children produce two personal and two dependent credits. Both continue to itemize, so their standard-deduction increases do not account for the moved liability. + +| Scenario | Variable | Frozen | Submitted recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 005 | state_income_tax_before_refundable_credits | 41,051.511719 | 40,800.011719 | yes | Itemized deductions rise 1,029; exemption phaseout increments fall 13→7, producing credits 232; indexed gross tax is 41,032.01. | +| 022 | state_income_tax_before_refundable_credits | 2,439.650146 | 2,406.939941 | yes | Taxable income stays 67,847.75; gross tax falls 29.645 and personal/senior credits rise 3.065. | +| 023 | state_income_tax_before_refundable_credits | 6.798477 | 0.126724 | yes | Standard deduction 5,706→5,900; gross tax 223.266→218.127; personal credit 156.467→158; renter credit remains 60. | +| 023 | state_refundable_credits | 148.310867 | 154.870132 | no | Wrong final-phaseout anchors and an unverified earnings cutoff drive this CalEITC amount. | +| 099 | state_income_tax_before_refundable_credits | 4,493.743164 | 4,420.917480 | yes | Taxable income stays 139,108.47; joint-bracket gross tax falls 59.290; two personal/two dependent credits rise 13.535. | + +### V2 and remaining limits + +Path: `sweep/fixes/r12_ca_2026_indexing_v2.py`; output: `sweep/verify/work/r12_ca_2026_indexing_v2.csv`. V2 corrects the two proven CalEITC anchor errors, removes the unsupported cutoff/graduated-reduction updates, and preserves the fully initialized baseline parameter histories outside 2026. It is a narrower, source-supported parameter experiment, **not a complete replacement reference calculation**. The exact 2026 FTB credit table remains unresolved, as does the engine's separate CalEITC AGI-test defect. + +The completed v2 sweep leaves all four annual-liability changes exactly intact and changes only 023's refundable amount among all 1,984 outputs: **154.870132→147.914139**. Compared with frozen 148.310867, v2's delta is **−0.396729**, below the harness's $1 threshold, so its moved set has four rows. `r12_compare.py` asserts identical scenario/variable coverage and exactly this one difference; results are in `r12_comparison.json`. I also attempted a statutory cutoff reconstruction using indexed section 17052(o)(2) income amounts and rates calculated to end at 30,000 in 2022; it did not reproduce FTB's 2025 cutoff/table exactly. `r12_cutoff_exploration.json` records that failed validation; I did not substitute its speculative 2026 endpoints. + +The investigator's 005 corrected value **40,522.71** in `verdicts.json` concerns r11's charity/miscellaneous deduction fix, not r12's independent indexing experiment. Matching that number is therefore not a valid r12 acceptance test. + +## Summary + +| Module | Verdict | Original moved rows | Confirmed / not confirmed | V2 moved rows | Main finding | +|---|---|---:|---|---:|---| +| r08_eitc_earned_income_deferrals | accept_with_changes | 5 | 4 yes; CA 023 refund no | 5 | Federal/MT/VA attribution verified; retain CA-taxable payroll HSA contributions; bound year scope. | +| r09_ny_rptc_rent_cap | accept_with_changes | 1 | 1 yes | 1 | 104 exceeds the actual-rent cap; 118 remains $375; bound year scope. | +| r11_ca_itemized_conformity | accept_with_changes | 3 | 3 yes | 3 | All three deduction changes verified, including unflagged 022/099; bound year scope. | +| r12_ca_2026_indexing | reject | 5 | 4 annual-liability movements yes; CA 023 refund no | 4 | Wrong CalEITC anchors, unverified endpoint, and year leakage; final CPI unavailable at freeze. | diff --git a/reference_audit/2026-09-22/verification/v4_r13_r14_r15.md b/reference_audit/2026-09-22/verification/v4_r13_r14_r15.md new file mode 100644 index 00000000..7b0a582f --- /dev/null +++ b/reference_audit/2026-09-22/verification/v4_r13_r14_r15.md @@ -0,0 +1,194 @@ +# Independent SNAP reference-fix verification + +Verified September 22, 2026, in policyengine-us 1.755.4. All three original modules reproduce their supplied sweep results across 100 households and 1,984 outputs each. All three verdicts are **accept_with_changes**, with the limits stated below. The revised r13 amount for California 023 is **$408**, correcting the original fix's $420. The r14 and r15 revisions leave their respective moved sets and amounts unchanged. + +I read `SWEEP_BRIEF.md` first and used its original harness, the specified `.venv-pe1755` interpreter and `policybench-wt/opus55` imports. Source inputs were read from `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep`; the household bundle was `results/local/adds202609/publish/us_full_run_20260612_policyengine_4_16_1_populace/us/scenarios.csv`. All newly written files are under the assigned workspace `/Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v4/sweep/`. Corrected modules are in its `fixes/` directory; scratch and sweep outputs are in `verify/work/`. + +The full sweeps compare each module independently with the frozen reference. They do not combine fixes. “Confirmed” in a moved-row table means the stated root cause was traced; qualifications about legal entitlement or source coverage are explicit in each section. There are 17 module/row incidences covering 15 distinct SNAP outputs. + +The [verification checks](work/verification_checks.json) confirm all original values and moved flags, all revised comparisons, and unchanged SHA-256 hashes for 13 source files. The no-fix baseline has no moved output; two CSV decimal round-trip differences are only 5.7e-14 dollars. [The rerun script](work/run_verification.sh) reproduces the three original and three revised full sweeps. Evidence includes all-month r13 traces and four independent hand calculations, r14 five-mode controls and monthly traces, and r15 mortgage inventory, traces and real-engine aggregation probes. The module sections identify the legal sources actually read and remaining limits; in particular, retained MT $799 and PA $857 utility amounts could not be corroborated against retrieved primary sources. + +## r13_hold_fy2026 + +**Verdict: accept_with_changes.** The index hold has two gaps: SNAP poverty guidelines update through a separate HHS parameter, and the published FY2026 minimum lookup expires in September, giving Hawaii $40 rather than $41 in October. The original also extends the index hold through September 2027. The full-year trace exposed an additional California rounding error: 023 should be $408 under the held schedule, not $420. The revision addresses these issues within calendar 2026. Its full 1,984-output sweep retains the same 13 moved rows; only 023 differs from the original fix, by −$12. + +The hold is the benchmark's requested convention. It is not a claim that FY2026 figures legally remain effective after September 30, 2026. I read the complete hold module and its imported `r13_snap_fy2027_parameters.py`, including component selection and every formula reachable with `round_min,round_allot`. The FY2027 hard-coded tables are inactive; I did not treat their numerical contents as evidence for this convention. + +### Legal and parameter checks + +- [7 USC 2017(a)](https://usc-cdn.house.gov/view.xhtml?edition=prelim&num=0&req=granuleid%3AUSC-prelim-title7-section2017) specifies the 30% income reduction, whole-dollar allotment rounding, and the one-/two-person minimum at 8% of the one-person Thrifty Food Plan, rounded to the nearest dollar. [7 CFR 273.10(e)(2)(ii)(A),(C)](https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/subpart-D/section-273.10) permits rounding the contribution up or rounding the resulting allotment down. `ceil(round(floor(net) * .3, 2))` correctly adds the contribution-rounding step to the retained engine income calculation. The cent rounding protects exact dollars from floating-point noise; it does not change these whole-dollar-net products. +- The earlier `floor(net)` is not a universal legal rule. Section 273.10(e)(1)(ii) gives state rounding alternatives. [CDSS's statewide summary of MPP 63-503.311](https://www.cdss.ca.gov/shd/res/pdf/ParaRegs-Food-Stamps-Income.pdf), p. 1, dated August 4, 2015, requires retaining cents through calculation and rounding final net income half up. [Santa Clara County's CalFresh benefit computation instructions](https://stgenssa.sccgov.org/debs/program_handbooks/calfresh/assets/CalFresh/Budgeting_Concepts/Computation.htm) corroborate both ordinary and elderly/disabled household procedures. This matters for 023: $876.5096 becomes $877, contribution $264, benefit $34/month or **$408/year**, versus the original hold's $420. V2 adds this California-only final-net rounding rule. Both final-net methods give the same contribution in the four other hand calculations below; no national nearest-dollar assumption was imposed. +- I read the existing USDA FY2026 COLA memo at `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/scenario_112__snap/cola_fy26.pdf` and its `cola_fy26_pdf.txt` extraction, pp. 3–7. Its August 14, 2025 digital signature supports the module's date. All seven active minimum constants match p. 5: contiguous/DC **24**, Guam **35**, Virgin Islands **31**, Alaska urban/rural1/rural2 **31/39/48**, Hawaii **41**. The published minimum is appropriate for Hawaii; simply rounding 8% of the already-rounded maximum ($506) does not reproduce that table. +- The memo's contiguous maximum allotments, sizes 1–8, are **298, 546, 785, 994, 1,183, 1,421, 1,571, 1,789**, additional person **218**. Standard deductions are **209/209/209/223/261/299** for sizes 1–6+, shelter cap **744**, homeless deduction **198.99**. All 100 max/additional/standard/cap cells checked against raw engine YAML match the memo; the full source/value record is [r13_parameters_source_values.json](work/r13_parameters_source_values.json). +- `system.py` applies reforms before uprating and again after setup. `uprate_parameters.py` calculates projections from an explicit anchor and its index, so changing the SNAP index can intercept its dependent projections. The index's **321.2** is an engine forecast anchor, not a statutory benefit amount. Copying the previous index is a mechanical way to make the growth factor one. +- The separate [parameter audit](work/r13_parameter_findings.md) identifies the standard/limited/individual utility paths and lists the state sources actually read. Standard allowances have explicit FY2026 values and no SNAP uprating metadata; limited and individual allowances use the held index. The fix changes no utility entitlement rule. +- The instantiated [parameter CSV](work/r13_parameters_all.csv) contains **1,305 SNAP leaves and 25 related leaves**. All 1,305 SNAP leaves are numerically identical under the hold in January, September, October and December 2026. This includes all **787 utility leaves**; **238** of those project in baseline October. Overall, **339 monetary leaves plus the index** change from baseline in October/December 2026. No alternative uprater or explicit FY2027 amount bypasses the held index within `gov.usda.snap`. The FPG and Hawaii-minimum problems described below lie outside that claim. +- State override code is copied from baseline. NJ's **$95** minimum is supported by [NJ DHS](https://www.nj.gov/humanservices/njsnap/apply/eligibility/); DC's **$30** by [DC Code 4-261.04](https://code.dccouncil.gov/us/dc/council/code/sections/4-261.04). Maryland's retained **$40** and age-62 test are outdated: [2024 SB35](https://mgaleg.maryland.gov/mgawebsite/Legislation/Details/sb0035?ys=2024RS) establishes $50 and age 60. Neither Maryland scenario (068, age 31; 078, age 19) invokes this override, and there is no DC scenario. This independent baseline issue was not folded into the r13 revision. + +### Scope and revision + +All 100 scenarios are in 34 contiguous states. The independent sweep is byte-identical to the supplied result: **1,984 outputs, 13 moved SNAP rows, six sub-$1 differences, no other output changes**. Eleven moved rows were not judge-flagged; only 057 and 066 are present in `verdicts.json`, and 057's investigation addresses a separate SSI-disability reading. The investigator's FY2027-based values (066: $3,600; 112: $291) are not expected to equal this new FY2026-hold convention (066: $3,576; 112: $288). + +`snap_fpg.py` uses 2025 HHS guidelines before October and 2026 guidelines thereafter. The latter are **15,960 + 5,680 per additional person**, compared with FY2026 SNAP's **15,650 + 5,500**. They bypass the SNAP index. Although the 2026 HHS publication predates the reference freeze, using it in October fails the requested all-twelve-month FY2026 SNAP schedule. SNAP's regular and BBCE income tests both use `snap_fpg`; the revision changes this variable without changing global HHS or unrelated TANF guidelines. + +Hawaii's minimum error cannot affect this bundle because it has no Hawaii household. The original index update extends to September 2027, and the rounding replacements apply to other years. The revision bounds its index and all four variable replacements to calendar 2026 and removes the global `R13_COMPONENTS` side effect/import dependency. Year guards were checked with the adapter probes; the full engine sweeps calculate 2026, so they do not establish runtime equality outside that year. + +Income limits are still not rounded up to published whole-dollar standards; that is a separate retained baseline issue. An initial suspicion that asset limits were CPI-projected was disproved by the instantiated parameter audit: they remain the memo's **$3,000/$4,500**. The raw YAML parent has uprating metadata but does not propagate it to the two leaves. `meets_snap_asset_test` is annual and uses January parameters. This illustrates why the report distinguishes source inspection from instantiated values. + +**Revision:** [sweep/fixes/r13_hold_fy2026_v2.py](../fixes/r13_hold_fy2026_v2.py). The first revision (FPG/Hawaii/year scope only) reproduced all 1,984 original-fix values exactly; its preserved CSV is [r13_hold_fy2026_v2_before_ca.csv](work/r13_hold_fy2026_v2_before_ca.csv). The [final sweep](work/r13_hold_fy2026_v2.csv) also fixes California's final-net rounding and differs only at 023 SNAP: $420 → $408. It has 13 moved rows and the same six sub-$1 changes. [Forty-five actual-formula adapter probes](work/r13_revision_probes.json) pass all seven regional minima in January/September/October/December, year guards, FPG date/current unit size, contribution rounding, and California/state rounding boundaries. These adapter probes are not represented as full synthetic engine runs. + +### Independent hand calculations + +These use the scenario's stated annual income/rent, the memo's figures and the 20% earned-income deduction / 50% shelter threshold in 7 CFR 273.10(e)(1). All four have zero listed utility expenses and no applicable other deduction. The complete monthly trace confirms eligibility and the income/deduction amounts for every month. The [Decimal calculation script](work/r13_hand_calculations.py) produces [these exact arithmetic results](work/r13_hand_calculations.json), and [trace assertions](work/r13_trace_checks.json) independently check agreement. + +| Scenario | Gross/month | Earned deduction | Standard | Net before shelter | Shelter deduction | Final net | Rounded 30% | Maximum | 12-month amount | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 012 MS | 1,833.3333 | 366.6667 | 209 | 1,257.6667 | 0 | 1,257.6667 | 378 | 785 | 4,884 | +| 054 NC | 1,425 | 285 | 209 | 931 | 0 | 931 | 280 | 785 | 6,060 | +| 066 VA | 43.3333 | 8.6667 | 209 | 0 | 430 | 0 | 0 | 298 | 3,576 | +| 109 FL | 2,500 | 500 | 261 | 1,739 | 0 | 1,739 | 522 | 1,183 | 7,932 | + +For 012, monthly rent $450 is below half of $1,257.6667; for 109, $800 is below half of $1,739. For 066, the standard deduction already exhausts income; its $430 rent cannot reduce net below zero. Virginia's $14,000 bank balance does not defeat BBCE: [Virginia Code 63.2-801](https://law.lis.virginia.gov/vacodeupdates/title63.2/section63.2-801/) sets 200% FPG and no asset limit. The head explicitly lists 40 weekly hours. The four results use the same monthly figures twelve times under the benchmark convention. + +### Moved rows and missed rows + +The [2,400-row trace](work/r13_hold_fy2026_trace.csv) covers baseline and original hold, every household, all twelve months. All 100 held SNAP outputs are constant across the year and sum to the swept annual result. For every moved row, unit size, earned/unearned/gross income, earned-income, dependent-care, child-support and excess-medical deductions, housing cost, utility allowance and work status are unchanged versus baseline in the same month; every moved household remains eligible. The maximum/standard/cap hold and contribution rounding explain the complete change. “Confirmed” below covers that causal chain and the displayed held-schedule allotment arithmetic; 023 is marked no because its original recomputed amount fails the California rounding check. It does not certify unrelated SSI, disability, utility-entitlement or income-source assumptions retained from 1.755.4. Montana's retained $799 and Pennsylvania's $857 utility amounts remain unverified against a retrieved primary source, as recorded in the parameter audit. + +| Scenario | Variable | Frozen | Recomputed, original fix | Confirmed | One-line reason | +|---|---|---:|---:|---|---| +| 008 NJ, unflagged | snap | $15,246.91 | $15,120.00 | Yes | Eight members; held maximum $1,789 minus $529 contribution = $1,260 × 12; $299 standard deduction, no shelter deduction. | +| 012 MS, unflagged | snap | $4,952.09 | $4,884.00 | Yes | $785 maximum − $378 contribution = $407 × 12; hand calculation above. | +| 023 CA, unflagged | snap | $461.34 | $420.00 | No | Root-cause movement is traced, but $876.51 net must round to $877; corrected held amount is $34 × 12 = $408. | +| 038 LA, unflagged | snap | $7,286.94 | $7,212.00 | Yes | $994 maximum − $393 contribution = $601 × 12; $1,309.833 net after $223 standard and earned deduction. | +| 043 CO, unflagged | snap | $3,596.04 | $3,576.00 | Yes | Age 66; $340 medical deduction already exhausts income before shelter; $298 maximum × 12. | +| 054 NC, unflagged | snap | $6,125.69 | $6,060.00 | Yes | $785 − $280 = $505 × 12; hand calculation above. | +| 057 LA | snap | $2,669.22 | $2,628.00 | Yes | Retained monthly earned $866.667 plus unearned $605.750; net $1,090.083, so ($546 − $327) × 12. | +| 066 VA | snap | $3,596.04 | $3,576.00 | Yes | $520 annual wages are exhausted by deductions; $298 × 12; hand calculation above. | +| 079 AZ, unflagged | snap | $2,428.02 | $2,376.00 | Yes | Retained unearned $1,511 minus $209 standard and $145 medical = $1,157 net; ($546 − $348) × 12. | +| 080 PA, unflagged | snap | $3,596.04 | $3,576.00 | Yes | $301.333 gross minus $209 standard and $744 shelter leaves zero net; $298 × 12. | +| 100 MT, unflagged | snap | $8,625.89 | $8,556.00 | Yes | Retained $799 utility amount gives $453.636 shelter deduction and $237.092 net; ($785 − $72) × 12; utility-source limitation above. | +| 109 FL, unflagged | snap | $8,020.55 | $7,932.00 | Yes | $1,183 − $522 = $661 × 12; hand calculation above. | +| 118 NY, unflagged | snap | $2,903.94 | $2,868.00 | Yes | $805 pre-shelter minus $610.625 shelter = $194.375 net; ($298 − $59) × 12; r15 mortgage interpretation not applied. | + +For NJ 008, simply rounding the final fractional engine net would be the wrong state procedure. [N.J.A.C. 10:87-6.16(b),(c)](https://www.law.cornell.edu/regulations/new-jersey/N-J-A-C-10-87-6-16) rounds individual income and deduction amounts: self-employment $333, wages $2,233, interest $1+$8, earned deduction $513 and standard $299 give net $1,763 and the same $529 contribution. This is why v2's new final-net rule is restricted to California. + +**Missed rows:** no additional moved output was found by the full scenario scan or either revision sweep. The FPG leak changes 068 MD's October net-income-test flag ($1,319.235 net; FPG $1,304.167 → $1,330), but gross eligibility fails and SNAP remains $0. Six positive minimum cases—027, 030, 045, 073, 108, 112—become $288 from $287.6831665, only **+$0.3168335**, below the $1 movement criterion. They are affected but correctly absent from the moved set. NJ 056 remains $1,140 because the state $95 minimum binds. The other 80 households remain at zero. Hawaii has no bundle household. Among all five California households, only 023 changes an output in the final revision; 031's calculated contribution rises by $1 but SNAP stays zero. + +## r14_unlisted_weekly_hours_v2 + +**Verdict: accept_with_changes.** Accept the two-output hours-dependency exclusion. Revise the module's claim that its recomputed zeros are established amounts under law. The correction is documentation only; it preserves the narrow input-dependency experiment. + +**Legal and parameter checks.** I read the original module in full, its v1 predecessor, the sweep harness, the two relevant `verdicts.json` entries, the prompt and scenario input filtering, and 1.755.4's general/ABAWD/person/household work-requirement formulas and SNAP unit-size formula. Engine paths below are relative to `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +- `HOURS_FIELDS`, `listed_hours_fields`, and `patch`: the only hours field actually present in the bundle is `hours_worked_last_week`. `/Users/maxghenis/PolicyEngine/policybench-wt/opus55/policybench/prompts.py` renders it as usual weekly hours and specifies zero for unlisted numeric inputs and false for other unlisted facts. `scenarios.py` excludes `weekly_hours_worked` and `weekly_hours_worked_before_lsr`. `variables/household/income/person/weekly_hours_worked.py` gives the latter a default of 40. Thus the hard-coded replacement 0 comes from the benchmark instructions, not a legal presumption about unemployed people. The code changes the annual hours input only for persons without a prompt-visible hours field. +- The added monthly, person-level noncompliance input defaults false. The replacement general-work formula retains every existing exemption expression and changes only compliance to `participant | ~noncompliant`. Registration, assigned E&T/workfare participation, and sanctions are distinct under [7 CFR 273.7(a)(1), (f)(1)](https://www.ecfr.gov/current/title-7/section-273.7). Lack of program participation alone does not establish a sanction. In this bundle neither participation nor noncompliance is listed, so assuming no sanction is consistent with the prompt's false default. +- I checked the retained parameter files `parameters/gov/usda/snap/work_requirements/general/weekly_hours_threshold.yaml` (30), `general/age_threshold/caring_dependent_child.yaml` (6), and `general/age_threshold/exempted.yaml` (under 16 and 60 or older) against [7 CFR 273.7(b)(1)](https://www.ecfr.gov/current/title-7/section-273.7). These numbers agree. The retained exemption code is incomplete as a general implementation: the regulation also includes an earnings substitute, specified 16–17-year-olds, students, UI recipients, TANF compliance and treatment participation; household presence is not necessarily individual caregiving. The patch intentionally does not repair those separate defects, and its no-sanction default makes those omissions immaterial to the general-work result here. +- The unchanged ABAWD threshold file `abawd/weekly_hours_threshold.yaml` is 20, consistent with [7 U.S.C. 2015(o)(2)](https://www.law.cornell.edu/uscode/text/7/2015#o). The work-requirement statute preserves the initial three countable months in 36 months, exemptions, and geographic waivers. The module introduces no monetary parameter. Its 40-hour reference is an engine default, not a statutory requirement. I also read the upstream formula at commit `48a10d4d43`; the compliance expression matches that backport. + +I checked the installed `.venv-pelatest` package metadata (2.8.0) and its same general-work formula. It contains that compliance expression plus an `is_usda_disabled` exemption; the latter is absent from the reviewed fix, as its documentation says. + +**Correction required.** The original docstring says the remaining rows change in law, but the unchanged 1.755.4 ABAWD formula tests current hours/exemptions without a prior-months clock or waiver geography. Zero hours alone does not establish immediate loss of every month's benefits. [Texas Works Handbook A-1960](https://fhb.hhs.texas.gov/handbooks/texas-works-handbook/a-1960-regaining-eligibility) expressly preserves three initial months. [New Jersey's official 2026 guidance](https://www.nj.gov/humanservices/dfd/news/federal-changes/) says all counties except Morris were waived through January 31 and Camden City/Cape May County remain waived from February 1. Neither scenario states prior ABAWD months; 056 does not state locality. Its legal amount cannot be established as $0 from those facts. These are still valid sensitivity findings and unlisted-input exclusions; they are not validated replacement entitlements. I did not extend this fix into the separate waiver or benefit-history defects. + +The relevant independent-investigator comparisons are the **hours-zero alternatives in the verdict narratives**. The top-level `investigate.corrected_value` fields are $1,956.24 for 056's mortgage experiment and $291 for 112's separate minimum-allotment/FY2027 calculation; r14 does not reproduce those different experiments. Both narratives expressly record $0 under the zero-hours reading in 1.755.4. + +The new module is [`sweep/fixes/r14_unlisted_weekly_hours_v2_v2.py`](../fixes/r14_unlisted_weekly_hours_v2_v2.py). It revises the docstring and description and uses a distinct `FIX_ID`; executable functions/classes are unchanged. The double suffix follows the requested `_v2.py` naming for an input whose identifier already ends `_v2`. + +**Scope.** A raw scan of all 100 scenarios finds 177 persons: 104 have no hours field (67 in the `adults` arrays and 37 in `children` arrays), and 73 have a listed `hours_worked_last_week` (72 and one respectively). These are scenario role groups, not legal age classifications; the child arrays include adult dependents. No person lists either internal weekly-hours field or a SNAP work-participation/noncompliance input. All scenario years are 2026. I also inspected all 11 listed hours values below 30; only scenario_023 has positive frozen SNAP, and its head is explicitly disabled. The function itself is year-generic and the reform is system-wide; conclusions here are limited to this bundle and do not establish safe use in other years or situations with explicit sanctions. A participant with an explicit, different work-registration violation would need more specific sanction handling than the copied `participant | ~noncompliant` expression. + +**Independent computations.** I reran the original module over all 100 households and 1,984 outputs. The resulting CSV is byte-identical to the original `out/r14_unlisted_weekly_hours_v2.csv` (174,158 bytes): two moved rows and no small nonzero deltas. A separate full sweep of the documentation revision matches every field except its deliberately different fix identifier. + +I independently computed five versions for all 100 households: baseline, hours only, compliance only, both changes, and both changes with every listed hours value mapped to the internal hours input. The compliance-only control reproduces all 1,984 frozen outputs exactly. Mapping the 73 listed values adds no difference across all 1,984 outputs. The independently computed hours-only SNAP amounts reproduce all 100 original v1 SNAP rows. For the five affected cases I recorded every person's relevant flags and monthly household calculations for all twelve months (300 scenario/mode/month records). + +**Every moved row.** “Confirmed” below concerns the causal hours dependency; it does not certify the recomputed value as the unique legal benefit. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| scenario_056 (NJ) | snap | $1,140.00 | $0.00 | Yes | Unlisted hours become 0 for the single age-20 head; general compliance remains true, while the engine's ABAWD gate fails in all twelve months. | +| scenario_112 (TX) | snap | $287.6831665 | $0.00 | Yes | Unlisted hours become 0 for the single age-31 head; general compliance remains true, while the engine's ABAWD gate fails in all twelve months. | + +Both households have no children. In every month, baseline hours are 40; with v2 they are 0, `meets_snap_general_work_requirements` remains true, `meets_snap_abawd_work_requirements` becomes false, final person work eligibility fails, SNAP unit size changes from one to zero, and household SNAP eligibility becomes false. For 056 the baseline is $95 each month; for 112 it is $23.84000015 in January–September and $24.37439537 in October–December. The isolated compliance change leaves both annual amounts exactly unchanged. The individual traces show no disability, pregnancy, student or other modeled non-age work-registration exemption for either head. In 112 the engine also removes employment income from countable income after excluding the head; that downstream proration does not restore eligibility. + +**V1-only rows.** The independent decomposition confirms v2's exclusion of all three from the moved set: + +| Scenario | Baseline | Hours only (v1) | Compliance only | Combined (v2) | Causal finding | +|---|---:|---:|---:|---:|---| +| 012 MS | $4,952.0893555 | $2,068.0173340 | $4,952.0893555 | $4,952.0893555 | v1 excludes the 18-year-old pregnant spouse solely through general work registration, reducing SNAP unit size from three to two. | +| 032 MN | $0.00 | $73.1231842 | $0.00 | $0.00 | v1 excludes the 18-year-old spouse through general work registration; a two-person unit becomes categorically eligible in October and gets three minimum allotments. | +| 038 LA | $7,286.9443359 | $2,725.9172363 | $7,286.9443359 | $7,286.9443359 | v1 excludes both adults through general work registration, reducing unit size from four to two and changing income proration. | + +The monthly records establish that restoring general compliance restores unit size and the frozen output in all three. In 032 and 038 the standalone ABAWD flags do fall when hours become zero, but that flag is bypassed in the final person-work formula because the household contains a dependent under 14. Thus those standalone flag changes do not cause the v1 output differences. For 032, v1's October benefit is $24.37439537 per month; January–September remain zero, giving $73.1231842 annually. These are engine-mechanism checks, not endorsements of every unchanged eligibility or rounding rule. + +**Missed rows.** None in the independently swept bundle for this hours-reading experiment. The complete output comparison and the explicit-listed-hours challenge identify exactly 056 and 112. Other positive SNAP households with absent hours remain unaffected because the changed persons have modeled age/disability/caregiving exemptions or the household's child-related work treatment applies; several positive households have explicit hours that are deliberately preserved. No additional output appears when the preserved listed hours are mapped correctly. This finding does not claim to identify all unrelated SNAP reference defects. + +**Evidence files.** `sweep/verify/work/r14/verify_r14.py` is the independent five-mode experiment; `static_scan.json`, `static_checks.json`, and `person_scan.json` record the person scans and comparison of executable ASTs. The exact original-harness rerun is `sweep/verify/work/r14_unlisted_weekly_hours_v2.csv`; the documentation revision is `sweep/verify/work/r14_unlisted_weekly_hours_v2_v2.csv`. `annual_decomposition.csv`, `control_outputs.csv`, and `monthly_trace.json` hold the computed controls and monthly traces. `summarize_r14.py` checks every original/revised row and all controls; its passing results are in `verification_summary.json` and `verification_summary.txt` under the same r14 work directory. The revision changes no amount or moved-set membership. + +## r15_snap_mortgage_interest + +**Verdict: accept_with_changes. Classification: unlisted_input.** Treat the two moved outputs as sensitive to an alternative reading of the listed housing facts. The prompt and input definitions do not establish that the mortgaged home is the occupied SNAP shelter, so this is not a demonstrated engine-law defect or a unique corrected entitlement. The v2 module also corrects an aggregation error outside this bundle: one tax unit's structured mortgage inputs must not suppress another tax unit's person-level inputs in the same SPM unit. + +### Legal, input, and line-by-line checks + +I read the entire original module, the sweep harness, the 056 triage verdict, and the relevant installed 1.755.4 definitions, and scanned all 100 scenario records. Installed source root: `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +- Original lines 1–49 and 54–60 describe the legal claim and input conversion. [7 CFR 273.9(d)(6)(ii)(A)](https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/subpart-D/section-273.9#p-273.9(d)(6)(ii)(A)) permits mortgage interest as an occupied-shelter charge. Paragraph (D) separately covers specified temporary absences. The law does **not** make every tax-deductible mortgage an occupied-shelter cost. I read the current eCFR text displayed as current through September 18, 2026, and the original agent's saved `sweep/work/r15_snap_mortgage_interest/ecfr_273_9.xml`; both contain the same mortgage/occupancy rule. Attempts to open January 1, 2026 eCFR failed; I do not claim to have inspected that historical rendering. +- Lines 63–85 add an annual SPM-unit variable and calculate an incremental expense. Subtracting already reported `mortgage_payments`, floored at zero, implements a payment lower bound without counting the same interest twice. No extra principal is imputed. But lines 76–83 select the input representation after summing every tax unit; this can discard another unit's person-only interest. V2 makes that selection before SPM aggregation. +- Input semantics were checked in `variables/household/expense/person/home_mortgage_interest.py` and `variables/household/expense/tax_unit/mortgage_interest_structure.py`: the first field covers mortgage interest reported or unreported on Form 1098; the latter describes first/second home acquisition mortgages used for the federal deduction. Neither defines SNAP occupancy. Omitting `second_home_mortgage_interest` is consistent with the chosen occupied-first-home reading, but does not prove that a second home can never qualify under the regulation's temporary-absence rule. +- Lines 88–129 retain the installed `variables/gov/usda/snap/income/deductions/shelter/snap_excess_shelter_expense_deduction.py` calculation, adding only the new annual interest increment divided by `MONTHS_IN_YEAR`. [7 USC 2014(e)(6)(A)–(B)](https://uscode.house.gov/view.xhtml?req=%28title%3A7+section%3A2014+edition%3Aprelim%29) supports the 50% excess-shelter test and cap exception for an elderly/disabled household. The installed `parameters/gov/usda/snap/income/deductions/excess_shelter_expense/income_share_disregard.yaml` is 0.5. The inherited homeless alternative remains unchanged; neither moved household lists homelessness. +- Lines 132–135 register only the new variable and replacement SNAP shelter deduction. An AST comparison also confirmed that removing only the new interest assignment/addition makes the replacement shelter formula identical to the installed original (`work/r15_scope_ast_check.json`). The separate `housing_cost` variable remains unchanged: its listed components are rent, real-estate taxes, HOA fees, mortgage payments, and homeowners insurance. `mortgage_payments.py` is an annual bare input with no interest fallback. There are **no new hard-coded statutory dollar amounts**; zero is the expense floor and division by 12 is the annual-to-month conversion. R15 alone retains the engine's other parameters and rounding; its recomputed amounts are not a combined r13/r14/r15 result. + +The exact prompt rule is in `/Users/maxghenis/PolicyEngine/policybench-wt/opus55/policybench/prompts.py:10`: unlisted numeric inputs are zero, other unlisted facts/statuses are false, and housing facts are constant through the year. The descriptions list home mortgage interest and first home mortgage interest, but no occupied-property identifier or mortgage-payment total. Counting interest in shelter is a defensible alternative reading, while the missing occupancy fact prevents treating it as uniquely compelled. The existing 056 triage record independently calls this `prompt_ambiguity`. + +### Scope and missed rows + +The complete `scenarios.csv` scan found 11 households listing mortgage interest: 040 AZ, 046 OK, 049 NH, 056 NJ, 078 MD, 081 MA, 088 TX, 092 AL, 110 OH, 118 NY, and 120 CT. Every one has both person-level and tax-unit first-home interest, so the person-only fallback is not used for any of them. Only 046 and 120 additionally list second-home interest. No scenario supplies `mortgage_payments`. The broader mortgage-key inventory finds 61 first-home balance records and four second-home balance records; 50 households have a balance but no listed interest. A balance alone does not supply an annual expense, so those are not missed interest rows under the zero-input rule. The complete inventory is saved in `work/r15_mortgage_input_inventory.json`. `Scenario.to_pe_household()` builds a single tax unit and a single SPM unit in each scenario, so the mixed-tax-unit defect does not affect these 100 households. + +Scenarios 078 and 081 list both `pre_subsidy_rent` ($38,400 and $44,400 respectively) and mortgage interest. This is a concrete reason not to assume that every listed mortgage necessarily describes the occupied home. Their SNAP outputs are zero under the original sweep. The module does not check occupancy, a potential over-application if used as a general engine fix. V2 expressly documents the alternate-reading assumption rather than inventing a property-occupancy input. + +Independent recomputation confirms only 056 and 118 receive SNAP among the mortgage-interest households. The other nine remain SNAP-ineligible in both January and October, even after the interest increment. The 61-household trace also covers all mortgage-balance households. No missed moved row was found. No extra principal, escrow, second-home occupancy, or within-year expense timing was inferred. + +### Moved rows and traces + +The independent original-fix sweep recomputed all **1,984** outputs. Every recomputed value exactly matches the builder CSV; the moved set is exactly 056/snap and 118/snap, with no sub-$1 nonzero deltas. The 056 result agrees with the triage investigator's $1,956.24 to less than one cent. + +| Scenario | Variable | Frozen | Recomputed | Confirmed | Reason | +|---|---|---:|---:|---|---| +| 056 NJ | snap | $1,140.00 | $1,956.24 | Yes | Treating listed $6,773 interest as occupied-shelter cost raises the capped deduction; no other input/parameter changes. | +| 118 NY, unflagged | snap | $2,903.94 | $3,596.04 | Yes | Treating listed $5,551 interest as occupied-shelter cost raises this elderly/disabled household's uncapped deduction enough to reduce SNAP net income to zero. | + +“Confirmed” means the output moves through this input interpretation alone; it does not turn the absent occupancy fact into a known fact. + +The actual household and monthly calculations are: + +- **056:** One adult, age 20, NJ; listed mortgage interest $6,773 and real-estate taxes $11,949.087890625. Baseline annual `mortgage_payments` is $0 and annual `housing_cost` is $11,949.087890625. Both remain unchanged by the fix; its SNAP-only increment is $6,773/12 = $564.416667 monthly. In January, gross income is $1,486.544067, pre-shelter net income $1,205.210693, and utility allowance $0. Shelter deduction rises from $393.151978 to the $744 cap; net income falls from $812.058716 to $461.210693, expected contribution from $243.60 to $138.30, and SNAP from the NJ $95 minimum to $159.70. In October, the same increment raises the deduction from $395.494446 to $760.677429; net income falls from $805.031372 to $439.848389 and SNAP becomes $172.979950. The engine's annual result agrees within floating-point precision with 9 × $159.70 + 3 × $172.979950 = $1,956.239850. +- **118:** One adult, age 74, NY, explicitly blind and disabled; listed mortgage interest $5,551, real-estate taxes $1,633.50, and retirement Social Security $2,800. Baseline `mortgage_payments` is $0; `housing_cost` is $1,633.50. Again, both remain unchanged. The extra SNAP-only shelter amount is $5,551/12 = $462.583333 monthly. The engine has $1,014 gross monthly SNAP income and a $877 utility allowance, unchanged by this fix. In January the pre-shelter net is $805; the deduction rises from $610.625 to $1,073.208374 and net income falls from $194.375 to $0. The elderly/disabled cap exception is essential here. SNAP rises from $239.80 to $298. In October the deduction rises from $612.967468 to $1,075.550781, again leaving zero net income; SNAP rises from $248.579956 to $304.679962. The annual result agrees within floating-point precision with 9 × $298 + 3 × $304.679962 = $3,596.039886. + +In both households the only changed traced monthly variables are shelter deduction, net income, expected contribution, and SNAP; gross/earned/unearned income, earned-income deduction, standard deduction, utility allowance, minimum/maximum allotments, and eligibility are identical. `work/r15_trace_scope_checks.json` records these comparisons and the annual-vs-monthly arithmetic checks. The figures above deliberately retain 1.755.4's existing rounding and October projections, because the r15 sweep isolates mortgage input interpretation. They are not asserted to be the final legal all-fixes benefit amounts. + +### Correction and reproducibility + +The exact original and v2 increment functions were also extracted with Python AST and executed with a small vector/entity adapter, independently of engine startup. A structured $1,000 tax unit plus a person-only $2,000 tax unit produces $1,000 in v1 and $3,000 in v2; with $500 already reported as payments the increments are $500 and $2,500. V2 also passes full-payment deduplication, one joint unit with spouse-paid interest, person-only fallback, and second-home-only omission probes. All 100 real scenarios have identical original/v2 increment results. The real 1.755.4 engine independently reproduces the two v1 failures and the full-payment deduplication success in `work/r15_snap_mortgage_interest_trace.json`. These arithmetic probes are saved in `work/r15_formula_probe.py`, `work/r15_formula_probe.json`, and `work/r15_formula_bundle_comparison.json`; they do not substitute for the full engine sweep. + +V2: `sweep/fixes/r15_snap_mortgage_interest_v2.py`. It preserves the original SNAP shelter formula and payment lower bound; it changes input fallback to operate separately for each tax unit. Its documentation classifies this as an unlisted-input alternative and identifies the occupied-home assumption. + +Verification script: `sweep/verify/work/r15_verify.py`. Original-fix sweep output: `sweep/verify/work/r15_snap_mortgage_interest.csv`. V2 sweep output: `sweep/verify/work/r15_snap_mortgage_interest_v2.csv`. Trace JSON files use the matching module name plus `_trace.json`; prompt/situation dumps are `r15_scenario_056_*` and `r15_scenario_118_*` in the same work directory. + +**V2 result:** Its full 1,984-output sweep has exactly the same numeric values and moved flags as the independently rerun original; no row was added or removed. Real-engine probes now return $3,000 for the mixed-tax-unit case, $2,500 after a $500 existing payment, and $0 after a $4,000 existing payment. All assertions passed. The original harness performs the normal reformed-system construction for this sweep; subsequent traces reuse the default baseline and apply this variable-only reform to its parameter clone. Every traced annual SNAP value agrees exactly with the full-sweep CSV (maximum absolute difference $0). + +Run commands from the assigned workspace (all outputs stay there): + +```bash +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/sweep.py --fix /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/r15_snap_mortgage_interest.py --out sweep/verify/work/r15_snap_mortgage_interest.csv +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python sweep/verify/work/r15_verify.py +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python sweep/verify/work/r15_run_v2.py +python3 sweep/verify/work/r15_formula_probe.py +``` + +Hashes of the original module, original output, scenario bundle, v2 module, and both independently generated sweep files are recorded in `work/r15_input_output_hashes.json`. No original fix/output file was modified. + +Original fix/output files remain unchanged. No issue, external message, push or publication was made. The assigned branch is `triage-verify`. The attempted commit was blocked because the environment makes `.git` read-only (`Unable to create .git/index.lock: Operation not permitted`); the new files remain uncommitted in this workspace. + +| Module | Verdict | Original → revised moved rows | Revision and result | +|---|---|---:|---| +| r13_hold_fy2026 | accept_with_changes | 13 → 13 | [r13_hold_fy2026_v2.py](../fixes/r13_hold_fy2026_v2.py): hold SNAP FPG and published minimum; bound 2026; apply California final-net rounding. Only 023 changes, $420 → $408. MT/PA retained-utility source gaps remain. | +| r14_unlisted_weekly_hours_v2 | accept_with_changes | 2 → 2 | [r14_unlisted_weekly_hours_v2_v2.py](../fixes/r14_unlisted_weekly_hours_v2_v2.py): clarify unlisted-hours sensitivity, without claiming unique legal $0 entitlements. No numeric change. V1-only 012/032/038 are general-work-registration artifacts. | +| r15_snap_mortgage_interest | accept_with_changes | 2 → 2 | [r15_snap_mortgage_interest_v2.py](../fixes/r15_snap_mortgage_interest_v2.py): classify unlisted occupancy input and correct mixed-tax-unit fallback. No bundle numeric change; 056 and unflagged 118 confirmed as mortgage-input sensitivities. | diff --git a/reference_audit/2026-09-22/verification/v5a_projection_irs_wi_id_ca.md b/reference_audit/2026-09-22/verification/v5a_projection_irs_wi_id_ca.md new file mode 100644 index 00000000..ba44d57e --- /dev/null +++ b/reference_audit/2026-09-22/verification/v5a_projection_irs_wi_id_ca.md @@ -0,0 +1,317 @@ +# Projected-parameter audit of PolicyBench references + +Audit date: **2026-09-22**. Reference freeze: **2026-07-03**. Engine: **policyengine-us 1.755.4**, supplied triage environment. Only files in the assigned workspace were written; existing tracked files were preserved. Nothing was filed, pushed, or posted. + +Four full **1,984-output** sweeps and one full no-op control identify **14 causal moved outputs**: three `engine_defect_stale` Wisconsin rows and eleven `held_convention` rows. The JSON also includes Wisconsin 108 as `unchanged`. No additional causal output was found beyond the assigned r18/r12 row sets. All affected outputs are listed and sourced in the family sections below. + +The **Idaho threshold publication finding remains provisional**, and the exact **FTB 2026 indexing publication date is unverified**. These limitations are explicit below and in the relevant JSON entries. The IRS and California calculations preserve unrelated engine formulas, including withholding/local-sales proxies. + +The current reference CSV differs from the saved r18 frozen values on **14 SNAP outputs**: all now equal the earlier r13 held values. The no-op control reproduces those same differences, including nine above $1. These are not effects of the four audited families. Raw sweep CSVs are retained; the shared verification section reconciles every such row. + +| Family | Full outputs | Raw moved rows | Causal moved rows | Convention | +|---|---:|---:|---:|---| +| IRS sales-tax tables | 1,984 | 11 | 2 | Published 2025 tables held | +| Wisconsin | 1,984 | 12 | 3 | Published 2026 schedules | +| Idaho | 1,984 | 12 | 3 | 2025 thresholds held, publication status provisional | +| California | 1,984 | 15 | 6 | Sourced 2025 values held; correct older CalEITC breakpoints | + +Machine-readable results: [r19_summary.json](work/r19_summary.json). Automated verification: [r19_validation.json](work/r19_validation.json). + +## IRS optional state sales-tax tables + +The 2026 convention uses the **published 2025 table**, rather than the engine's extrapolation of its 2023 YAML table. The r18 hold removed projected 2025 and 2026 entries but left projected **2024** entries in place; it therefore did not test the last published table. There are 5,358 projected leaves in this family (47 state/DC codes, six family sizes, 19 income brackets), each with projected 2025 and 2026 entries. + +### Sources, dates, and convention + +| Edition | Verified IRS directory file date | Printed document date | Finding | +|---|---|---|---| +| [2024 Schedule A instructions](https://www.irs.gov/pub/irs-prior/i1040sca--2024.pdf) | **2024-12-20**, 22:10:36 in the [IRS prior-files directory](https://www.irs.gov/downloads/irs-prior?order=uri&page=188&sort=desc) | 2024-12-16, page 1 | Texas table on page 15. | +| [2025 Schedule A instructions](https://www.irs.gov/pub/irs-pdf/i1040sca.pdf) ([archived edition](https://www.irs.gov/pub/irs-prior/i1040sca--2025.pdf)) | **2025-12-18**, 14:10:50 in the [IRS current-files directory](https://www.irs.gov/downloads/irs-pdf?order=uri&page=24&sort=asc); archived file timestamp is three seconds later | 2025-12-08, page 1 | Full state tables on pages 13–17; Texas on page 16. Published before the freeze; these are the held 2026 convention amounts. | +| 2026 | **No publication found as of 2026-09-22** | Not available | The [IRS current-revision page](https://www.irs.gov/forms-pubs/about-schedule-a-form-1040) still links the 2025 instructions, and the IRS directories identify the current edition as 2025. No 2026 table amount or publication date is verified. | + +The directory timestamps are verified file dates in the IRS column labeled “Date”; exact first-publication dates were not separately established. Together with the printed editions they provide pre-freeze publication evidence. The date evidence and exact URLs are saved in [sources.json](work/irs_sales_tax/sources.json) and [posting_dates.txt](work/irs_sales_tax/posting_dates.txt). The absence of a located 2026 edition is explicitly a publication-status finding, not an invented future date. + +The IRS explains on page 17 that its 2025 tables use the 2024 tables adjusted for sales-tax revenue growth per person, plus Louisiana's changed rate. Thus generic IRS chained-CPI uprating is not the published table methodology. The 2025 engine entries themselves must be replaced, not carried forward unchecked. + +### Parameters and complete table coverage + +Every changed leaf is `gov.irs.deductions.itemized.salt_and_real_estate.state_sales_tax_table.tax...`. Family sizes are 1–5 and over 5 (encoded as 6); the table does not have separate filing-status schedules. The 19 bracket lower bounds are $0, $20,000, $30,000, $40,000, $50,000, $60,000, $70,000, $80,000, $90,000, $100,000, $120,000, $140,000, $160,000, $180,000, $200,000, $225,000, $250,000, $275,000, and $300,000. These bounds match the engine; they are not projected. + +The fix sets **all 5,814 cells** across 50 states and DC, all six family sizes, and all 19 income brackets for 2025 and 2026. It uses the 46 printed state/DC tables and zero for the five absent-state tables under the worksheet's line-1 instructions. Four zero-table states are absent from raw YAML and are filled by engine homogenization; the reform's second application covers them. This is broader than the two Texas cells, so a national sweep checks households the hold screen could miss. It does not encode the published 2024 edition or change income definitions. Because reforms also run before uprating, inserting 2025/2026 values can prevent creation of earlier 2024 projections; only 2025/2026 table values and 2026 household outputs are in this audit’s verified scope. + +The full numeric source table is [published_2025_values.csv](work/irs_sales_tax/published_2025_values.csv); the reform's data file is [r19_irs_sales_tax_2025.json](../fixes/r19_irs_sales_tax_2025.json). [extract_tables.py](work/irs_sales_tax/extract_tables.py) reconstructs it from the saved official-PDF extracts and asserts 51 jurisdictions, six columns, 19 ordered brackets, and the two focal Texas values. + +### Texas attribution and local component + +`state_sales_tax.py` selects the state, tax-unit size (capped at six), and `state_sales_tax_income_bracket`. The bracket formula sums federal AGI and the income sources listed in `income_sources.yaml`, including exempt interest, veterans' benefits, and exempt Social Security. Both focal households have family size one. Scenario 000 selects bracket 10 ($100,000–$119,999.99); scenario 020 selects bracket 19 ($300,000 and over). + +| Scenario | Parameter suffix | Engine YAML 2023 | Published 2024 | Published 2025 / held 2026 | +|---|---|---:|---:|---:| +| 000 TX | `tax.TX.1.10` | 1,402 | 917 | **931** | +| 020 TX | `tax.TX.1.19` | 2,440 | 1,571 | **1,595** | + +`variables/gov/local/tax/sales/local_sales_tax.py` computes **0.2 × state_sales_tax**. The coefficient is hardcoded, not a projected parameter; the dollar amount inherits the state-table projection. Holding the published table therefore produces local proxy amounts **$186.20** and **$319.00** and combined state/local sales deductions **$1,117.20** and **$1,914.00** for 000 and 020. + +The [computed parameter and household snapshots](work/irs_sales_tax/diagnostics.json) confirm the projected 2025 values differ sharply from the published edition: + +| Cell | Engine projected 2024 (r18 carries this forward) | Engine projected 2025 | Engine projected 2026 | Published 2025 / convention 2026 | +|---|---:|---:|---:|---:| +| TX, size 1, bracket 10 | 1,482.153451 | 1,524.884946 | 1,559.443280 | 931 | +| TX, size 1, bracket 19 | 2,579.496733 | 2,653.865383 | 2,714.009703 | 1,595 | + +The simulated table-income totals are $111,568.823730 for 000 and $300,150.187500 for 020, confirming the selected brackets. **Both households switch from itemizing to the standard deduction.** For 000, itemized deductions fall from $18,341.332031 to $17,587.199219, below its unchanged $18,150 standard deduction; taxable income rises from $26,095.781250 to $26,287.113281. For 020, itemized deductions fall from $17,090.341797 to $15,747.529297, below its unchanged $16,100 standard deduction; the engine's complete downstream calculation raises taxable income from $283,059.843750 to $283,854.187500. This branch change is why a simple marginal-rate multiplication of the sales-table difference would not reproduce the tax deltas. The diagnostic federal outputs equal the full-sweep values exactly. + +The IRS worksheet instead requires the actual local rate. Texas uses the ratio method: local table deduction = state-table amount × local rate / 6.25%. The engine's 20% proxy corresponds to an assumed 1.25% local rate. Neither household states a locality or local rate, so that rate is **unverified from the stated facts**. This parameter audit preserves the proxy and does not claim that the resulting federal amounts repair that separate input/formula limitation. + +The downstream path is `state_sales_tax` → `local_sales_tax` → `state_and_local_sales_or_income_tax` (greater of income withholding or combined sales taxes) → `salt` (including real-estate taxes) → `salt_deduction` → itemized deductions → taxable income → federal income tax. The traced formulas are in the installed 1.755.4 package under `variables/gov/states/tax/sales/`, `variables/gov/local/tax/sales/`, and `variables/gov/irs/income/taxable_income/deductions/itemizing/`. + +Fix: [r19_irs_sales_tax_convention.py](../fixes/r19_irs_sales_tax_convention.py). Full sweep: [r19_irs_sales_tax_convention.csv](work/r19_irs_sales_tax_convention.csv). The row classification is `held_convention`, because a 2026 edition was not located and the convention carries the published 2025 table; the stale projected 2025 entries are an additional defect in the engine's starting point. + +### Computed output verification + +The full sweep computed **1,984 outputs**. Exactly **two causal outputs move**, both already detected by r18, but with different convention amounts. All **168 outputs across ten Texas households** were scanned: 000, 003, 020, 030, 052, 060, 083, 088, 101, and 112. All other non-SNAP outputs nationwide are unchanged. The raw CSV also has nine unrelated SNAP moves and five SNAP differences within $1: current frozen SNAP references match r13, while the engine still reproduces the original r18 frozen SNAP values. These shared reference differences are accounted for in the audit-wide control section and are not attributed to the IRS table reform. + +| Scenario | Variable | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| scenario_000 TX | `federal_income_tax_before_refundable_credits` | $2,883.493652 | $2,906.453613 | $+22.959961 | `held_convention` | Published 2025 table replaces extrapolated 2023 cells; local proxy inherits the change. | +| scenario_020 TX | `federal_income_tax_before_refundable_credits` | $68,056.710938 | $68,334.734375 | $+278.023438 | `held_convention` | Published 2025 table replaces extrapolated 2023 cells; local proxy inherits the change. | + + + +## Wisconsin — published 2026 standard deductions and tax brackets + +The published 2026 single maximum deduction is **$13,960**, its phase-out starts at **$20,120**, and the first tax bracket ends at **$15,110**. The engine projects $13,870, $19,993.059944, and $15,012.691559, respectively. These are pre-freeze published amounts, so the convention uses the 2026 schedules, not a 2025 hold. + +### Primary sources and chronology + +| Source | Date evidence read | What it establishes | +|---|---|---| +| [2026 Form 1-ES instructions](https://www.revenue.wi.gov/TaxForms2026/2026-Form1-ES-inst.pdf), pp. 2–3 | Printed revision **R. 1-26**; legal currency date **2026-01-16** on p. 3. [DOR's 2026 forms index](https://www.revenue.wi.gov/Pages/Form/2026Individual.aspx) search extraction shows the final 1-ES item dated **2026-02-06 08:08 AM**. | All published 2026 amounts below; the final indexed edition precedes 2026-07-03. The item's timestamp is not claimed to be its first public release date. | +| [2026 WT-4A worksheet](https://www.revenue.wi.gov/TaxForms2017through2019/w-234f.pdf), p. 2 | Printed revision **R. 11-25**; legal currency date **2025-11-03**. | Independent DOR form prints the same complete 2026 schedules. This corroborates the pre-freeze chronology; its exact first posting day was not verified. | +| [2025 Form 1-ES instructions](https://www.revenue.wi.gov/TaxForms2025/2025-Form1-ES-inst.pdf), printed pp. 2–3 (PDF pp. 3–4) | Printed revision **R. 1-25**, later instruction-update cover for **2025 Wisconsin Act 15**, legal currency date **2025-07-08**. | All published 2025 values below. Exact posting day of the updated PDF was not verified. | +| [2025 Form 1 instructions](https://www.revenue.wi.gov/TaxForms2025/2025-Form1-inst.pdf), p. 13 | 2025 annual instructions; exact posting date not independently established. | Wisconsin has no separate qualifying-surviving-spouse schedule: a federal qualifying surviving spouse may file as Wisconsin head of household. The engine's surviving-spouse-to-joint mapping is a separate limitation, described below. | + +The source evidence is the text returned by the web tool: [forms index and combined extracts](work/wi/source_tool_evidence.json), [2025 1-ES](work/wi/source_2025_1es.txt), [2026 1-ES](work/wi/source_2026_1es.txt), and [2026 WT-4A](work/wi/source_2026_wt4a.txt). PDF downloads through the sandbox shell were unavailable. Revision dates and dates through which a document reflects law are distinguished above from publication timestamps. + +### Parameters and convention values + +All paths below are relative to `gov.states.wi.tax.income`. The r18 inventory contains **22 Wisconsin projected entries, all dated 2026-01-01; none is a projected 2025 entry**. The 2025 columns were checked against DOR's 2025 1-ES schedules. The reform replaces all 22 engine entries for 2026, including all four published filing-status schedules and the engine's existing surviving-spouse alias. + +| Parameter suffix | Published 2025 | Engine 2025 | Engine projected 2026 | Published/convention 2026 | +|---|---:|---:|---:|---:| +| `deductions.standard.max.SINGLE` | 13,560 | 13,560 | 13,870 | 13,960 | +| `deductions.standard.max.JOINT` | 25,110 | 25,110 | 25,680 | 25,840 | +| `deductions.standard.max.SEPARATE` | 11,930 | 11,930 | 12,200 | 12,280 | +| `deductions.standard.max.HEAD_OF_HOUSEHOLD` | 17,520 | 17,520 | 17,920 | 18,030 | +| `deductions.standard.max.SURVIVING_SPOUSE` (engine joint alias) | 25,110† | 25,110 | 25,680 | 25,840† | +| `deductions.standard.phase_out.single[1].threshold` | 19,550 | 19,550 | 19,993.059944 | 20,120 | +| `deductions.standard.phase_out.joint[1].threshold` | 28,210 | 28,210 | 28,849.320768 | 29,040 | +| `deductions.standard.phase_out.separate[1].threshold` | 13,390 | 13,390 | 13,693.456402 | 13,780 | +| `deductions.standard.phase_out.head_of_household[1].threshold` | 19,550 | 19,550 | 19,993.059944 | 20,120 | +| `deductions.standard.phase_out.head_of_household[2].threshold` | 57,210 | 57,210 | 58,506.545238 | 58,827 | +| `rates.single[1].threshold`; `rates.head_of_household[1].threshold` | 14,680 | 14,680 | 15,012.691559 | 15,110 | +| `rates.single[2].threshold`; `rates.head_of_household[2].threshold` | **50,480** | **51,130** | 52,288.754729 | 51,950 | +| `rates.single[3].threshold`; `rates.head_of_household[3].threshold` | 323,290 | 323,290 | 330,616.693061 | 332,720 | +| `rates.joint[1].threshold` | 19,580 | 19,580 | 20,023.739832 | 20,150 | +| `rates.joint[2].threshold` | **67,300** | **68,170** | 69,714.930762 | 69,260 | +| `rates.joint[3].threshold` | 431,060 | 431,060 | 440,829.075167 | 443,630 | +| `rates.separate[1].threshold` | 9,790 | 9,790 | 10,011.869916 | 10,080 | +| `rates.separate[2].threshold` | **33,650** | **34,090** | 34,862.578695 | 34,630 | +| `rates.separate[3].threshold` | 215,530 | 215,530 | 220,414.537584 | 221,820 | + +† These amounts are the published **joint** schedule retained for the engine alias, not an independently published Wisconsin surviving-spouse schedule. The reform preserves existing filing-status selection formulas. The 2025 explicit second-bracket errors are visible above; even though they are not projected 2025 entries, their 2026 projections are replaced by the correct published 2026 values. The 2025 simulation itself is not changed. + +The published marginal rates (3.5%, 4.4%, 5.3%, 7.65%) and deduction phase-out rates (single 12%; joint/separate 19.778%; HOH 22.515% then 12%) agree with the engine and are not projected entries. Their formulas remain unchanged. The HOH switch point is encoded in the existing continuous marginal-scale implementation; this audit does not repair independent whole-dollar table/rounding or filing-status defects. + +### Formula attribution and scope + +The installed 1.755.4 code at `variables/gov/states/wi/tax/income/wi_standard_deduction.py` reads the selected maximum and phase-out schedule. `wi_taxable_income.py` subtracts that deduction and exemptions from `wi_agi`; `wi_income_before_credits.py` applies the selected tax-rate scale; `wi_income_tax_before_refundable_credits.py` subtracts nonrefundable credits and floors at zero. Thus these parameters directly reach the benchmarked state-tax output. The deduction also reaches `credits/itemized_deduction/wi_itemized_deduction_credit_potential.py`, which feeds the capped itemized-deduction credit. `wi_withheld_income_tax.py` separately reads the single maximum and single tax-rate scale for the engine's withholding proxy, so a full national output sweep also checks downstream federal effects. + +Fix: [r19_wi_convention.py](../fixes/r19_wi_convention.py). It accepts optional `WI_PREFIXES` for parameter-subset attribution. Full output CSV: [r19_wi_convention.csv](work/r19_wi_convention.csv). Intermediate diagnostic script: [diagnostics.py](work/wi/diagnostics.py). + +### Output verification + +The prescribed full sweep completed **1,984 outputs**. It changed **three Wisconsin state-tax outputs**; no additional Wisconsin household or federal/refundable-credit output moved because of the Wisconsin parameters. All four Wisconsin households (042, 064, 091, 108), comprising **82 outputs**, were scanned. The previously unflagged household 108 still has zero state income tax. The three affected state references were already excluded for other defects, as specified in the assignment; these are isolated parameter-correction values, not combined repairs of those other defects. + +| Scenario | Variable | Frozen | Convention value | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| 042 | `state_income_tax_before_refundable_credits` | 284.740906 | 279.234802 | -5.506104 | `engine_defect_stale` | Published single deduction maximum, phase-out start, and first tax threshold replace the projections. | +| 064 | `state_income_tax_before_refundable_credits` | 4,605.997070 | 4,598.476074 | -7.520996 | `engine_defect_stale` | Published joint deduction and first/second thresholds replace the projections; deduction and first-bracket savings exceed the effect of the lower second bracket. | +| 091 | `state_income_tax_before_refundable_credits` | 843.661499 | 838.155457 | -5.506042 | `engine_defect_stale` | Same single-parameter path as 042. | +| 108 | `state_income_tax_before_refundable_credits` | 0.000000 | 0.000000 | 0.000000 | `unchanged` | Additional Wisconsin household: state tax remains zero after replacing the parameters. | + +For 042 and 091, the causally active parameters are `max.SINGLE`, `phase_out.single[1].threshold`, and `rates.single[1].threshold`. The deduction rises by $105.232807; the 4.4% marginal rate saves $4.630243, and the larger first bracket saves another $0.875776. For 064, active parameters are `max.JOINT`, `phase_out.joint[1].threshold`, and `rates.joint[1,2].threshold`: the deduction increase saves $10.478765, the first bracket saves $1.136342, and the smaller second bracket adds $4.094377. These independently computed deltas agree with the simulation within $0.0003 (float32 rounding). The formulas and [analytic attribution](work/wi/analytic_attribution.json) establish why each row moves without relying on the blanket hold screen. The data files retain full floating-point results. + +The raw CSV also contains **nine unrelated SNAP differences above $1 and five below $1**. A direct file comparison verified for all 14 that its current `frozen` SNAP amount equals the earlier `r13_hold_fy2026_v2.csv` held result, while its recomputed amount equals the original `frozen` amount in `r18_hold_all_projections.csv`. Thus the reference bundle already incorporates the SNAP convention while this isolated Wisconsin reform leaves the engine's SNAP projections in place. These are not Wisconsin effects. The nine above-tolerance SNAP rows are 008, 012, 038, 043, 054, 066, 079, 080, and 109; the five below-tolerance rows are 027, 030, 045, 073, and 108. [Comparison evidence](work/wi/snap_reference_comparison.json) preserves every numeric match. The separate [full empty-reform control](work/r19_noop_control.csv) reproduced all 14 SNAP differences exactly. Comparing all 1,984 Wisconsin-reform results with that control leaves precisely the three Wisconsin state-tax changes above, and every other result is identical; see [control comparison](work/wi/noop_comparison.json). The raw Wisconsin CSV has not been filtered or altered. + +Validation completed: Python syntax; exact static coverage of all 22 r18 Wisconsin parameter names; full prescribed sweep; all 82 Wisconsin outputs checked; direct formula-delta reconciliation; exact reconciliation of the 14 unrelated SNAP differences to the earlier source CSVs; and a second full 1,984-output empty-reform control isolating the three Wisconsin changes. The optional intermediate diagnostic script was interrupted during repeated system construction and did not produce an intermediate-value artifact; its contents remain for reproducibility. Three initial subset sweeps were also interrupted during import and are not counted as completed verification. JSON rows are in [r19_wi_summary.json](work/r19_wi_summary.json). + + +## Idaho + +The Idaho projection screen concerns five income-tax thresholds and five retirement-deduction caps. The tax-threshold convention used here is **held at the last verified Idaho-published amounts, with the publication limitation below**. The retirement caps use amounts derived from the statutory formula and SSA's pre-freeze publication. The retirement choice cannot affect these households because none meets the engine's minimum retirement-deduction age. + +### Parameters, sources, and dates + +In the following table, threshold names abbreviate `gov.states.id.tax.income.main.[1].threshold`; cap names abbreviate `gov.states.id.tax.income.deductions.retirement_benefits.cap.`. + +| Parameter / filing statuses | Verified 2025 amount | Engine projected 2026 | Convention 2026 | Basis | +|---|---:|---:|---:|---| +| Threshold: `single`, `separate` | $4,811 | $4,920.031273 | $4,811 | Last verified Commission-published amount; held, subject to limitation below | +| Threshold: `joint`, `head_of_household`, `surviving_spouse` | $9,622 | $9,840.062546 | $9,622 | Same | +| Cap: `SINGLE`, `HEAD_OF_HOUSEHOLD`, `SURVIVING_SPOUSE` | $48,216 | $49,308.715001 | $49,824 | $4,152/month × 12, derived from statute and SSA publication | +| Cap: `JOINT` | $72,324 | $73,963.072501 | $74,736 | $4,152/month × 12 × 1.5, derived from statute and spouse benefit | +| Cap: `SEPARATE` | $0 | $0 | $0 | Separate filers cannot claim this deduction | + +1. **Idaho 2025 amounts:** [2025 Individual Income Tax Forms and Instructions, EIN00046](https://tax.idaho.gov/wp-content/uploads/forms/EIN00046/EIN00046_03-02-2026.pdf), printed revision **March 2, 2026**, before the freeze. Printed page 9 (PDF page 11), Form 40 line 20, gives both thresholds and all five filing statuses. Printed page 31 (PDF page 45), Form 39R line 8, gives the single/joint caps and excludes married separate filers. The printed revision date is verified; the first date this edition was posted is not separately verified. +2. **2026 threshold publication unresolved:** the [Commission's tax-rate schedule](https://tax.idaho.gov/taxes/income-tax/individual-income/individual-income-tax-rate-schedule/) and [instruction-edition index](https://tax.idaho.gov/taxes/income-tax/individual-income/forms/individual-income-tax-instructions/) showed **2025 as the latest year** when read on September 22, 2026. No official 2026 indexed threshold or its publication date was located. This supports using the last verified publication for the sweep, but **does not prove that no pre-freeze publication exists**. Consequently the three `held_convention` row classifications below are provisional on that publication question. The official 2026 amount itself remains unverified. [Idaho Code 63-3024(3)](https://legislature.idaho.gov/statutesrules/idstat/title63/t63ch30/sect63-3024/) instructs the Commission to prescribe the annual factor and adopt the CPI measure. The official HTML already saved in the prior 053 investigation was read locally; its relevant text is preserved in `work/id/id_63-3024_excerpt.txt`. This audit did not verify the Commission’s 2026 factor or substitute a newly calculated factor for its publication. +3. **Do not confuse withholding with the tax threshold:** the Commission's [July 31, 2026 release](https://tax.idaho.gov/pressrelease/withholding-tables-updated-for-2026/) links [withholding tables printed July 23, 2026](https://tax.idaho.gov/document-mngr/pubs_EPB00744). Their annual wage cutoffs are $16,100/$32,200. Those tables do not publish the indexed taxable-income thresholds audited here, and their post-freeze date is not evidence of the tax thresholds' publication date. +4. **Retirement-cap statutory formula:** [Idaho Code 63-3022A](https://legislature.idaho.gov/statutesrules/idstat/title63/t63ch30/sect63-3022a/) defines the cap through maximum Social Security benefits at full retirement age, including the spouse's benefit for a joint return and the single-equivalent widow(er) benefit; it directs the Commission to publish amounts annually. The retrieved official text's history ends in 2015; the cap-formula text predates the freeze. [SSA's 2026 fact sheet](https://www.ssa.gov/cola/factsheets/2026.html) gives $4,018/month for 2025 and $4,152/month for 2026. The [October 24, 2025 SSA release](https://www.ssa.gov/news/en/press/releases/2025-10-24.html) links that fact sheet, establishing pre-freeze availability. [SSA’s spouse-benefit explanation, July 11, 2024](https://www.ssa.gov/blog/en/posts/2024-07-11.html), confirms the maximum spouse benefit is 50% of the worker’s full-retirement-age benefit. The annual caps above are **our arithmetic from those sources, not a located Idaho 2026 cap publication**. The 2025 arithmetic reproduces the Commission's caps exactly. An Idaho-specific 2026 publication/date remains unverified. + +The engine's only projected **2025** Idaho entry is `cap.SEPARATE`, and it remains zero, consistent with the verified instructions. All other Idaho parameters listed have explicit 2025 values. This was checked against `r18_projected.json` and the installed 1.755.4 YAML, rather than relying on the misleading `last_explicit: 2100-01-01` labels in the supplied post-uprating value dump. + +### Attribution + +Only scenarios 007, 053, and 076 are in Idaho in the 100-household bundle (60 reference outputs total). The assigned output is `state_income_tax_before_refundable_credits` in each case. + +The installed `variables/gov/states/id/tax/income/id_income_tax_before_non_refundable_credits.py` reads `id_taxable_income`, calculates all five status schedules, and selects the household's filing-status result. `id_income_tax_before_refundable_credits.py` subtracts nonrefundable credits and floors at zero. Thus 007 and 053's effective parameter is `main.single[1].threshold`; 076's is `main.head_of_household[1].threshold`. + +The retirement path is `id_retirement_benefits_deduction` → `id_subtractions` → `id_agi` → `id_taxable_income`. The cap is read by `id_retirement_benefits_deduction.py`, but its result is the smaller of the cap less head/spouse Social Security retirement benefits and eligible retirement income. The eligibility formula requires age 65, or age 62 if disabled. Household adults are **56, 25, and 45**, respectively; the two children in 076 are 13 and 8. Therefore eligible retirement income and the deduction are zero for all three households regardless of the cap. + +At the engine's unchanged 5.3% rate, moving the single threshold from 4,920.031273 to 4,811 adds $5.778657; moving the head-of-household threshold from 9,840.062546 to 9,622 adds $11.557315, before engine float32 rounding. This explains the supplied Idaho-only r18 CSV deltas. The projection audit does not apply the separate missing-health-insurance-subtraction repair: 007 and 053 remain excluded for that other defect, as the user specified. + +### Fix and verification + +Fix: `sweep/fixes/r19_id_convention.py`. It updates all ten projected Idaho leaves for 2026 and explicitly preserves the sourced zero for the one projected 2025 leaf. Set `ID_RETIREMENT_CAP_MODE=held` to inspect the alternative of awaiting an Idaho cap publication; the source-based age reasoning above establishes that this alternative cannot change the benchmark values. A separate sensitivity run was not completed. + +Full sweep: **1,984 outputs; 3 Idaho moves; 9 unrelated SNAP moves against the current reference file** (`sweep/verify/work/r19_id_convention.csv`). All 60 Idaho outputs equal the supplied Idaho-only r18 sweep: the same three move, the other 57 are exactly unchanged, and there are no additional Idaho moves missed by r18. Every non-Idaho recomputed value matches, within 1e-9, the original frozen value recorded in the supplied r18 full sweep. The current reference file has since adopted the SNAP hold: for example, 008 SNAP is now frozen at $15,120.00, whereas r18 recorded $15,246.905273, exactly the present engine recomputation. All 14 unrelated deltas above $0.000001 (9 over $1, 5 within tolerance) are SNAP and are recorded in `work/id/raw_sweep_drift.json`. This establishes a reference-file difference, not a consequence of the Idaho parameters. The raw CSV is preserved without rewriting its deltas. Two attempted r18 subset sweeps and an optional trace process were stopped during engine import to reduce shared filesystem contention; their absence must not be counted as successful validation. Parameter attribution above uses the read formulas and the supplied completed `out/r18_attr/gov.states.id.csv`. + +| Scenario | Variable | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| scenario_007 | `state_income_tax_before_refundable_credits` | $755.778320 | $761.557007 | $+5.778687 | `held_convention` (provisional) | Hold the single taxable-income threshold at the last verified 2025 publication; a 2026 publication/date remains unverified. | +| scenario_053 | `state_income_tax_before_refundable_credits` | $2,435.278320 | $2,441.057129 | $+5.778809 | `held_convention` (provisional) | Hold the single taxable-income threshold at the last verified 2025 publication; a 2026 publication/date remains unverified. | +| scenario_076 | `state_income_tax_before_refundable_credits` | $6,806.786621 | $6,818.344238 | $+11.557617 | `held_convention` (provisional) | Hold the head_of_household taxable-income threshold at the last verified 2025 publication; a 2026 publication/date remains unverified. | + + +## California confirmation + +The freeze convention uses published 2025 amounts. The existing `r12_hold_ca_2025.py` reproduces the five nonrefundable/federal changes, but its refundable-credit result for 023 does **not** hold the correct 2025 CalEITC breakpoint. The 2025 engine values also are not uniformly correct: 56 of the 61 projected fiscal parameters checked agree with the sourced 2025 values; two CalEITC breakpoints and three renter-credit caps differ. + +### Sources, publication dates, and convention + +| Primary source | Date established by this audit | What was read / convention | +|---|---|---| +| [FTB Tax News, October 2025](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2025/10.html) | October 2025 edition; exact first-post day unverified | 2025 brackets, standard deductions, exemption credits, renter caps; June 2024–June 2025 CCPI factor 3.0%. These pre-freeze amounts carry into 2026. | +| [FTB 2025 rate schedules](https://www.ftb.ca.gov/forms/2025/2025-540-tax-rate-schedules.pdf) and [2025 Form 540 booklet](https://www.ftb.ca.gov/forms/2025/2025-540-booklet.html) | Tax-year 2025 publications; exact first-post dates unverified. Brackets and listed credits independently appear in October 2025 Tax News. | All filing-status brackets, exemption credits, and renter caps below. | +| [FTB 2025 Form 3514 booklet](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html), [PDF](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.pdf) | Cached official PDF created 2026-01-06, modified 2026-02-19; these are document metadata, **not independently verified publication dates**. | Earned-income / phaseout starts $4,661, $6,998, $9,823; investment-income ceiling $4,814; young-child/foster-youth amounts $1,189 and phaseout start $27,425; YCTC loss ceiling $35,640. | +| [AB 91, section 2, R&TC 17052(o)](https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=201920200AB91), [R&TC 17052](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052.) | AB 91 approved and filed 2019-07-01; annual index factors below were published by October 2025. | $200/$505 credit breakpoints in 2019, indexed with annual rounding. Derived 2025 values are $252/$636, rather than engine $257/$649. They are statutory computations; the booklet does not print these parameters by name. | +| [FTB October 2021 indexing](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/october-2021/indexing.html), [October 2023](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2023/10.html), [October 2024](https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2024/10.html), October 2025 above | Respective October editions; exact first-post days unverified | Published 2020–25 factors: 1.4%, 4.4%, 8.3%, 3.1%, 3.3%, 3.0%. | +| [2026 Form 540-ES instructions](https://www.ftb.ca.gov/forms/2026/2026-540-es-instructions.html) | Tax-year 2026 estimated-tax instructions; exact first-post date unverified | Worksheet explicitly uses 2025 tax tables and exemption credits; standard deductions $5,706 / $11,412. | +| [BLS June 2026 CPI release](https://www.bls.gov/news.release/archives/cpi_07142026.htm) | Embargo/release 2026-07-14 | June observations were unavailable at the July 3 freeze. | +| [DIR California CPI percentage-change table](https://www.dir.ca.gov/OPRL/CPI/PresentCCPIchange.PDF) | PDF says last updated **2026-08-12** | June 2025–June 2026 California all-urban CPI change **3.4%**, after the freeze. This is a DIR date, not an FTB publication date. | + +**Unverified FTB 2026 publication date:** searches located a purported FTB indexing memorandum dated **2026-09-03**, hosted at [CalTax](https://www.caltax.org/regulatory-issues/ftb/2026-PIT-Indexing-Memo.pdf). The PDF could not be retrieved by the web tool; shell networking could not resolve the host. The date and its printed 2026 amounts have **not** been independently confirmed from that primary document. This report does not substitute DIR's August 12 date for FTB's publication date or claim that FTB has not published the memo. The verified July/August release chronology supports the held convention irrespective of this unresolved exact FTB date. + +### Parameter values + +All names below have prefix `gov.states.ca.tax.income.`. The complete 61-name comparison, including engine 2025 and projected 2026 values, is [parameter_comparison.json](work/ca/parameter_comparison.json). The fix sets each to its sourced 2025 amount for 2026; it also repairs both projected 2025 CalEITC breakpoints. + +| Parameter group | Published 2025 / held convention values | Engine 2025 comparison | +|---|---|---| +| `rates.single`, `rates.separate`, thresholds 1–8 | 11,079; 26,264; 41,452; 57,542; 72,724; 371,479; 445,771; 742,953 | All equal | +| `rates.joint`, `rates.surviving_spouse`, thresholds 1–8 | 22,158; 52,528; 82,904; 115,084; 145,448; 742,958; 891,542; 1,485,906 | All equal; use printed schedules, including upper-bracket rounding | +| `rates.head_of_household`, thresholds 1–8 | 22,173; 52,530; 67,716; 83,805; 98,990; 505,208; 606,251; 1,010,417 | All equal | +| `exemptions.amount`, `exemptions.dependent_amount` | 153 per personal/aged/blind exemption; 475 per dependent | Equal | +| `credits.earned_income.earned_income_amount` and `phase_out.start`, 0 / 1 / 2+ children | 4,661 / 6,998 / 9,823 | Equal | +| `credits.earned_income.eligibility.max_investment_income` | 4,814 | Equal | +| `credits.earned_income.phase_out.final.start`, 0 / 1+ children | **252 / 636**, derived from annual statutory rounding | **257 / 649**, both projected in 2025 | +| `credits.foster_youth.amount[1].amount`, `phase_out.start` | 1,189 / 27,425 | Equal | +| `credits.young_child.amount`, `phase_out.start`, `loss_threshold` | 1,189 / 27,425 / 35,640 | Equal | +| `credits.renter.income_cap`, single / separate | 53,994 | Equal | +| Same, joint / surviving spouse / head of household | **107,988** | **107,987**, $1 below published amount | + +The statutory CalEITC sequence (no children / one or more) is 200/505 in 2019, 203/512 in 2020, 212/535 in 2021, 230/579 in 2022, 237/597 in 2023, 245/617 in 2024, and **252/636 in 2025**. `check_caleitc.py` reproduces this using decimal round-half-up each year. FTB's 2025 table is consistent at the transition: no-child credit at earnings $5,451–5,500 is $252, and the one-child credit at $11,801–11,850 is $634 after its final phaseout begins. This checks the breakpoint, not the accuracy of every part of the engine's CalEITC formula. + +The raw CPI series itself is an index, not an independently scored statutory amount. The fix overrides its 61 fiscal descendants explicitly, avoiding changes to unrelated parameters. The six filing-status parameter groups already held at explicit 2025 values do not require a projection override. + +### Fix, attribution, and r12 correction + +Fix: [r19_ca_convention.py](../fixes/r19_ca_convention.py). Full harness output: [r19_ca_convention.csv](work/r19_ca_convention.csv). Attribution script: [diagnose.py](work/ca/diagnose.py). It isolates brackets, exemptions, CalEITC income starts, final-phase breakpoints, and other credits against the same baseline system. + +`ca_income_tax_before_credits` applies the filing-status rate schedule to `ca_taxable_income`; `ca_exemptions` computes personal, aged/blind and dependent credits; `ca_income_tax_before_refundable_credits` subtracts nonrefundable credits. This attributes 005 and 099 to joint brackets plus exemption amounts, and 022/023 to single brackets plus exemption amounts. No changed renter eligibility is asserted merely because the cap parameter changes. + +For **022 federal**, the code path is: + +`ca_withheld_income_tax` (each person's AGI minus the SINGLE standard deduction, passed through `rates.single`) → `state_withheld_income_tax` (CA component in the state's sum) → `state_and_local_sales_or_income_tax` (maximum of income/withholding and sales taxes) → `salt` (adds real-estate taxes) → `salt_deduction` → itemized deductions → federal taxable income and tax. The held single brackets increase this withholding proxy and the SALT deduction, lowering federal tax. This confirms an **engine proxy path**, not a claim about the household's actual withholding or its actual EDD withholding schedule. + +The computed 022 withholding proxy rises from **$5,978.141602 to $6,058.852051**, and SALT from **$14,807.141602 to $14,887.851563**. The bracket-only reform reproduces the entire federal change. Full convention diagnostic outputs exactly match the full harness for all CA scored outputs checked. Parameter and calculation snapshots are saved in [parameter_snapshots.json](work/ca/parameter_snapshots.json) and [attribution.json](work/ca/attribution.json). + +| Scenario/output | Brackets-only delta | Exemptions-only delta | CalEITC income-start-only delta | CalEITC final-breakpoint-only delta | +|---|---:|---:|---:|---:| +| 005 state | +161.421875 | +6.933594 | 0 | 0 | +| 022 federal | −17.756836 | 0 | 0 | 0 | +| 022 state | +59.285156 | +6.934814 | 0 | 0 | +| 023 state | +2.510818 | +3.467422 | 0 | 0 | +| 023 refundable | 0 | 0 | −1.134277 | −5.320709 | +| 099 state | +118.569336 | +28.464355 | 0 | 0 | + +Other-credit overrides have no scored CA effect. The two CalEITC components interact, so their isolated deltas are not expected to sum exactly to the combined change; state-tax differences at the final decimal reflect float32 rounding. + +**The r12 refundable result is not a correct held-2025 value.** `CountryTaxBenefitSystem.__init__` applies reforms before uprating (lines 86–94) and again after it (112–113). In r12's first application, `cpi("2025")` reads raw YAML's last value, 2023's **332.035**, rather than forecast 2025's 361.1377838. The final CalEITC breakpoint YAML has only 2019's 200/505; consequently the r12 raw index produces **236/597**. For 023, the engine's existing continuous formula with breakpoint 236 computes **134.0826733**, matching r12 CSV's **134.0826721** within $0.000002. Holding engine 2025's 257 would instead give **144.3009628**. The sourced 2025 breakpoint 252 gives **141.8898417** before float32 rounding. See [caleitc_checks.json](work/ca/caleitc_checks.json). Thus r12's five other output changes have the intended held-amount reason; its CalEITC change includes this construction error. + +### Sweep results + +The full harness completed with **1,984 outputs**, **six California moves**, nine unrelated SNAP moves, and five unrelated nonzero SNAP deltas below $1. All 14 nonzero SNAP deviations exactly match the common [no-op control](work/r19_noop_control.csv): the current bundle's frozen SNAP entries differ from the original sweep's frozen entries. They are not assigned to this CA reform. The raw CSV is preserved; see [reference drift details](work/r19_reference_drift.json). + +| Scenario | Output | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| 005 | State tax before refundable credits | 41,051.51 | 41,219.87 | +168.36 | held_convention | Hold joint bracket thresholds 1–5 and personal exemptions at 2025 amounts. | +| 022 | Federal tax before refundable credits | 11,131.33 | 11,113.57 | −17.76 | held_convention | Single bracket thresholds 1–5 increase the withholding proxy and SALT deduction. | +| 022 | State tax before refundable credits | 2,439.65 | 2,505.87 | +66.22 | held_convention | Hold single bracket thresholds 1–4 and personal/aged exemption amounts. | +| 023 | State tax before refundable credits | 6.80 | 12.78 | +5.98 | held_convention | Hold first single bracket threshold and personal exemption; renter credit remains $60. | +| 023 | State refundable credits | 148.31 | **141.89** | **−6.42** | held_convention | Hold CalEITC income start at $4,661 and use statutory 2025 final-phase breakpoint $252. | +| 099 | State tax before refundable credits | 4,493.74 | 4,640.78 | +147.03 | held_convention | Hold joint bracket thresholds 1–4 plus personal/dependent exemptions. | + +The 023 refundable row additionally contains an `engine_defect_stale` component: the engine's **2025** projected breakpoint was already inconsistent with the pre-freeze statutory amount. Its single row label remains `held_convention` because the 2026 indexing is held. JSON records the additional defect explicitly. + +All **100 outputs across five CA households** (005, 022, 023, 031, 099) were scanned. The six causal row identities are exactly the six r12 rows and the six CA tax rows in r18; **no additional CA row was missed**. Household 031 remains unchanged. The only r19-versus-r12 numerical difference is 023 refundable credits, +$7.807175 over r12's incorrect $134.082672. Against the original r18 hold, that row is +$0.968246 (within the $1 output tolerance), because r18 preserves an older projected 2024 breakpoint while dropping its 2025–26 projections. + +Other formula defects remain separate: in particular, 023's CalEITC still uses the engine's earned-income-only calculation, and this parameter audit does not repair its missing AGI comparison or other separately audited inputs. + +## Shared verification and reference-file differences + +All four modules were run from the triage sweep directory using the supplied `sweep.py`, `../.venv-pe1755/bin/python`, and `PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55`. `PYTHONDONTWRITEBYTECODE=1` prevented cache writes outside the workspace. Every output was computed; no scenario filter was used for the final family sweeps. Reproduction pattern: + +```bash +cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep +PYTHONDONTWRITEBYTECODE=1 \ +PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ +../.venv-pe1755/bin/python sweep.py \ + --fix /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5a/sweep/fixes/r19__convention.py \ + --out /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5a/sweep/verify/work/r19__convention.csv +``` + +Family names are `irs_sales_tax`, `wi`, `id`, and `ca`. The full [no-op control](work/r19_noop_control.csv) uses [an empty reform](work/wi/noop_reform.py). Every output outside each family's reported causal set is **bit-identical to the control**. In particular, none of the family fixes causes any SNAP movement. + +The table below accounts for all unrelated raw deltas above $0.000001. Two further decimal-serialization differences, each −$0.0000000000000568434, occur at 112 federal refundable credits and payroll tax; both are far below tolerance and identical in every sweep. In every row, the current frozen reference equals the supplied r13 recomputation; the no-op and each family recomputation equal the saved original r18 frozen value. The current reference already contains the held SNAP convention, so these raw differences must not be treated as new exclusions or fixes arising from this audit. They are outside the four-family classification list. [Reference reconciliation](work/r19_reference_drift.json) records all three source CSV paths and exact amounts. + +| Scenario | State | Output | Current frozen / existing held convention | Control / original r18 frozen | Raw delta | Above $1? | +|---|---|---|---:|---:|---:|---| +| scenario_008 | NJ | snap | 15,120.000000 | 15,246.905273 | +126.905273 | Yes | +| scenario_012 | MS | snap | 4,884.000000 | 4,952.089355 | +68.089355 | Yes | +| scenario_027 | CT | snap | 288.000000 | 287.683167 | -0.316833 | No | +| scenario_030 | TX | snap | 288.000000 | 287.683167 | -0.316833 | No | +| scenario_038 | LA | snap | 7,212.000000 | 7,286.944336 | +74.944336 | Yes | +| scenario_043 | CO | snap | 3,576.000000 | 3,596.039795 | +20.039795 | Yes | +| scenario_045 | MI | snap | 288.000000 | 287.683167 | -0.316833 | No | +| scenario_054 | NC | snap | 6,060.000000 | 6,125.688965 | +65.688965 | Yes | +| scenario_066 | VA | snap | 3,576.000000 | 3,596.039795 | +20.039795 | Yes | +| scenario_073 | MI | snap | 288.000000 | 287.683167 | -0.316833 | No | +| scenario_079 | AZ | snap | 2,376.000000 | 2,428.017334 | +52.017334 | Yes | +| scenario_080 | PA | snap | 3,576.000000 | 3,596.039795 | +20.039795 | Yes | +| scenario_108 | WI | snap | 288.000000 | 287.683167 | -0.316833 | No | +| scenario_109 | FL | snap | 7,932.000000 | 8,020.554199 | +88.554199 | Yes | + +Validation also checks all 1,984 keys, frozen amounts, deltas, and tolerance flags for each sweep; summary completeness; and all four modules' Python syntax. The IRS extraction asserts complete table dimensions, the Wisconsin inventory covers all 22 projected leaves, Idaho covers all ten, and California compares all 61 fiscal projected leaves. The initial IRS run failed because this installed `Parameter.update` requires period objects/strings rather than string `start`/`stop`; it was corrected to annual `period` updates and the complete sweep rerun successfully. Canceled exploratory processes are not counted as successful tests. Optional diagnostic scripts and saved source extracts are included as supporting artifacts; the reports distinguish read formulas, computed checks, and unresolved publication dates. + +**Commit limitation:** the assigned branch is `projection-audit`. An authorized commit attempt failed because the sandbox prohibits creating `.git/index.lock` (`Operation not permitted`), and this session has no permission-escalation mechanism. All audit files remain uncommitted in this workspace. History was not rewritten, and nothing was pushed. diff --git a/reference_audit/2026-09-22/verification/v5b_projection_mn_md_mi_mo_il.md b/reference_audit/2026-09-22/verification/v5b_projection_mn_md_mi_mo_il.md new file mode 100644 index 00000000..87e6bdb2 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v5b_projection_mn_md_mi_mo_il.md @@ -0,0 +1,353 @@ +# Projected-parameter audit: Minnesota, Maryland, Michigan, Missouri, Illinois + +Reference freeze: **2026-07-03**. Engine: **policyengine-us 1.755.4**. Audit date: **2026-09-22**. + +Four assigned references have verified stale-parameter defects; two r18 candidates match published 2026 amounts. Maryland 068 is a **provisional** held-convention calculation, not a resolved exclusion recommendation. No additional outputs in the five families move after controlling for the reference-snapshot discrepancy described below. + +| Scenario | State | Output | Frozen | Convention | Delta | Classification | +|---|---|---|---:|---:|---:|---| +| 032 | MN | Refundable credits | 2187.600098 | 2187.600098 | +0.000000 | `unchanged` | +| 045 | MI | State tax before refundable credits | 1288.837524 | 1290.962524 | +2.125000 | `engine_defect_stale` | +| 068 | MD | State tax before refundable credits | 1252.967773 | 1255.342773 | +2.375000 | `held_convention` **(provisional)** | +| 070 | IL | State tax before refundable credits | 1650.561279 | 1650.561279 | +0.000000 | `unchanged` | +| 078 | MD | Federal tax before refundable credits | 24772.693359 | 24780.613281 | +7.919922 | `engine_defect_stale` | +| 093 | MO | State tax before refundable credits | 3389.650879 | 3388.245605 | -1.405273 | `engine_defect_stale` | +| 122 | MN | State tax before refundable credits | 1028.216553 | 1025.541504 | -2.675049 | `engine_defect_stale` | + +Machine-readable assigned-row results: [`work/r19_summary.json`](work/r19_summary.json). Each family section gives sources, dates, full parameter tables or inventories, the fix, and all its attributed movers. `unchanged` uses the benchmark's $1 tolerance; no binary output changes. + +## Verification and reference snapshots + +All five modules completed the unmodified triage harness over **1,984 outputs each**. The full no-fix control also completed 1,984 outputs. Across the assigned states, the scans cover MN 42, MD 32, MI 52, MO 40, and IL 32 outputs, including every household in those states. Full CSVs, diagnostics, source inventories, and [`work/r19_validation.json`](work/r19_validation.json) are retained. + +**The initial five family sweeps and baseline control read a different SNAP reference snapshot from the final Michigan rerun.** In the initial runs, 14 SNAP frozen values differed from the stored r18 frozen column: nine by more than $1 and five by −$0.316833. The engine reproduced the original r18 amounts in every run. The final harness points to the original v1.1 bundle and its frozen inputs now agree with those amounts. All 1,984 Michigan computed values are exactly identical between runs; only the 14 SNAP frozen inputs and resulting comparison fields changed. The seven assigned rows are unaffected. Both raw runs and the exact input changes are retained in [`work/r19_mi_final_convention.csv`](work/r19_mi_final_convention.csv) and [`work/r19_final_sweep_reference_changes.json`](work/r19_final_sweep_reference_changes.json). No file under `triage/` or the publish bundle was modified by this audit; the external change was not attributed to anyone. + +The following table classifies all nine otherwise-unattributed raw movers as **unchanged by these families**. The initially read SNAP convention reference is shown separately from the computed result; no SNAP correction is inferred from these state-family sweeps. The five below-tolerance differences (027, 030, 045, 073, 108) are retained in [`work/r19_reference_snapshot_difference.json`](work/r19_reference_snapshot_difference.json). These SNAP controls are outside the seven assigned rows in the main summary JSON. + +| Scenario | Output | Earlier frozen / SNAP convention | Computed result / final frozen | Raw delta | Family delta | Classification / reason | +|---|---|---:|---:|---:|---:|---| +| 008 | SNAP | 15120.000000 | 15246.905273 | +126.905273 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 012 | SNAP | 4884.000000 | 4952.089355 | +68.089355 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 038 | SNAP | 7212.000000 | 7286.944336 | +74.944336 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 043 | SNAP | 3576.000000 | 3596.039795 | +20.039795 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 054 | SNAP | 6060.000000 | 6125.688965 | +65.688965 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 066 | SNAP | 3576.000000 | 3596.039795 | +20.039795 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 079 | SNAP | 2376.000000 | 2428.017334 | +52.017334 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 080 | SNAP | 3576.000000 | 3596.039795 | +20.039795 | 0 | `unchanged` by assigned families; reference snapshot differs | +| 109 | SNAP | 7932.000000 | 8020.554199 | +88.554199 | 0 | `unchanged` by assigned families; reference snapshot differs | + +Reproduction command, substituting `mn`, `md`, `mi`, `mo`, or `il` for FAMILY: + +```bash +cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep +PYTHONDONTWRITEBYTECODE=1 \ +PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ +../.venv-pe1755/bin/python sweep.py \ + --fix /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5b/sweep/fixes/r19_FAMILY_convention.py \ + --out /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5b/sweep/verify/work/r19_FAMILY_convention.csv +``` + +Publication limitations remain explicit: Maryland annual-return deductions and CDCC timing; Michigan's additional held tables; Minnesota marriage-credit timing; and Missouri's 2026 public-pension-cap publication date. The four verified mover classifications do not depend on those unresolved dates. Maryland 068 is the one assigned row whose convention classification remains provisional. + +The initial workspace had no tracked modifications. All new files are inside the assigned workspace. The requested commit attempt failed because the supplied environment makes `.git` read-only (`.git/index.lock: Operation not permitted`); no commits, history rewrites, remote pushes, filings, or posts were made. + + + + +The final expanded Michigan module was rerun through the original full harness: all 1,984 computed values exactly match its earlier sweep. Its reference inputs differ on 14 SNAP rows, as documented above; only scenario 045 state tax moves against the final run's frozen inputs. Direct Michigan/Illinois traces also verify the final values. A supplementary combined clone-based run was interrupted for runtime overhead; no results from it are used. + + + +## Minnesota + +Minnesota published the 2026 renter schedule before the freeze. The two assigned r18 moves have different dispositions: the child/working-family credit projection matches the published 2026 figures used by scenario 032; the standard deduction used by scenario 122 is $50 too low. The full sweep confirms one Minnesota output changes, by −$2.67505. + +### Sources, publication dates, and convention + +| Source | Document/release date | What was checked | Convention | +| --- | --- | --- | --- | +| [2026 inflation-adjusted amounts](https://www.revenue.state.mn.us/sites/default/files/2025-12/inflation-adjusted-amounts-2026.pdf), pp. 1–3 and 5 | PDF dated 2025-12-01; linked by the [2025-12-16 agency release](https://www.revenue.state.mn.us/press-release/2025-12-16/minnesota-income-tax-brackets-standard-deduction-and-dependent-exemption) | Deductions, exemptions, tax brackets, AMT, child/working-family/dependent-care credits, Social Security/public-pension subtractions, and renter schedule | Published 2026 values | +| [2025 inflation-adjusted amounts](https://www.revenue.state.mn.us/sites/default/files/2024-12/inflation-adjusted-amounts-2025.pdf), pp. 1–3 and 5 | PDF dated 2024-12-11; the [2024-12-16 agency release](https://www.revenue.state.mn.us/press-release/2024-12-16/minnesota-income-tax-brackets-standard-deduction-and-dependent-exemption) announces the 2025 amounts | Corresponding 2025 values, including all 51 projected renter entries | Establishes actual 2025 values, rather than treating the engine's projected 2025 entries as published | +| [2025 M1MA grid final draft](https://www.revenue.state.mn.us/sites/default/files/2025-10/m1ma-25-grid-0.pdf), line 19 | Printed 2025-10-15 | Marriage-credit cap $1,851; engine's explicit 2025 value is $1,853 | Hold $1,851 | +| [2026 M1MA grid near-final draft](https://www.revenue.state.mn.us/sites/default/files/2026-08/m1ma-26-grid.pdf), line 19 | Printed 2026-08-03 | Marriage-credit cap $1,894; projected engine value $1,894.994377 | After freeze; earliest located 2026 publication, with earlier publication unverified | + +The PDF dates establish dated agency documents; exact first-upload dates were not independently verified. The 2026 December release expressly links the complete inflation-adjustment publication, establishing pre-freeze availability. The marriage cap's first publication remains unverified: no pre-freeze 2026 cap publication was located. Its hold is provisional and has no effect on either Minnesota household, because neither qualifies for the marriage credit. The [statute](https://www.revisor.mn.gov/statutes/cite/290.0675), subdivision 3, defines the credit by a joint-versus-single tax comparison and calls for an agency table. + +### Parameters and fix + +Fix: [r19_mn_convention.py](../fixes/r19_mn_convention.py). It sets all **141 projected Minnesota 2026 parameters** and all **51 projected 2025 renter parameters** to the sourced values, covering all filing statuses and all brackets represented in the existing scales. Complete, unrounded parameter-by-parameter 2025, engine-2026, published-2026, convention, source, and date records are in [r19_mn_parameters.json](work/r19_mn_parameters.json). + +All parameter paths below have the prefix `gov.states.mn.tax.income.`. Joint amounts also apply to surviving spouses. + +| Parameter family | Published 2025 | Convention 2026 | +| --- | --- | --- | +| `deductions.standard.base` single/separate; joint; head | 14,950; 29,900; 22,500 | 15,300; 30,600; 23,000 | +| `deductions.standard.extra` unmarried; married | 2,000; 1,550 | 2,000; 1,600 | +| `deductions.{standard,itemized}.reduction.agi_threshold.low` nonseparate; separate | 238,950; 119,475 | 244,400; 122,200 | +| Same, `.high` | 330,300; 165,150 | 337,800; 168,900 | +| `exemptions.amount` | 5,200 | 5,300 | +| `exemptions.agi_threshold` single; joint; head; separate | 239,050; 358,550; 298,800; 179,275 | 244,500; 366,700; 305,600; 183,350 | +| `rates.single` three positive thresholds | 32,570; 106,990; 198,630 | 33,310; 109,430; 203,150 | +| `rates.joint`/`surviving_spouse` | 47,620; 189,180; 330,410 | 48,700; 193,480; 337,930 | +| `rates.separate` | 23,810; 94,590; 165,205 | 24,350; 96,740; 168,965 | +| `rates.head_of_household` | 40,100; 161,130; 264,050 | 41,010; 164,800; 270,060 | +| `amt.fractional_income_threshold` single/head; joint; separate | 71,470; 95,300; 47,660 | 73,100; 97,470; 48,740 | +| `credits.cdcc.phaseout_threshold` | 64,150 | 65,610 | +| `credits.cwfc.ctc.amount` | 1,750 | 1,800 | +| `credits.cwfc.phase_out.threshold` joint; other | 37,910; 31,950 | 38,770; 32,680 | +| `credits.cwfc.wfc.phase_in[1].threshold` | 9,480 | 9,690 | +| `credits.cwfc.wfc.additional.amount` 1; 2; 3+ older children | 1,000; 2,270; 2,710 | 1,020; 2,330; 2,770 | +| `subtractions.pension_income.cap` joint; other | 27,080; 13,540 | 27,690; 13,850 | +| `subtractions.{pension_income,social_security}.reduction.start` single/head; joint; separate | 84,490; 108,320; 54,160 | 86,410; 110,780; 55,390 | +| `subtractions.social_security.alternative_amount` single/head; joint; separate | 4,560; 5,840; 2,920 | Same: source explicitly says not indexed | +| `subtractions.social_security.income_amount` single/head; joint; separate | 69,250; 88,630; 44,315 | Same: source explicitly says not indexed | +| `credits.marriage.maximum_amount` | 1,851 | 1,851, provisional hold described above | + +The 2025 AMT amounts in the publication differ from the engine's explicit YAML values (71,540; 95,390; 47,700). The fix corrects 2026 only for AMT; this audit's 2025 substitutions are confined to the projected renter entries. + +The complete renter table below gives each bracket's lower income bound; the upper bound is one less than the next row. Percentages and credit maxima have the same row alignment in the two years. This table supplies the three existing `credits.renters.{percent_of_income,claimant_share,max_credit}` scales; adjacent repeated amounts are compressed in the module. + +| Income lower bound 2025 | Income lower bound 2026 | Income percentage | Claimant share | Maximum 2025 | Maximum 2026 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 0 | 1.0% | 5% | 2,720 | 2,780 | +| 6,670 | 6,820 | 1.0% | 10% | 2,720 | 2,780 | +| 8,860 | 9,060 | 1.1% | 10% | 2,640 | 2,700 | +| 11,070 | 11,320 | 1.2% | 10% | 2,580 | 2,640 | +| 15,530 | 15,880 | 1.3% | 15% | 2,500 | 2,560 | +| 19,960 | 20,410 | 1.4% | 15% | 2,440 | 2,490 | +| 22,160 | 22,670 | 1.4% | 20% | 2,380 | 2,430 | +| 24,360 | 24,920 | 1.5% | 20% | 2,300 | 2,360 | +| 28,820 | 29,470 | 1.6% | 20% | 2,240 | 2,290 | +| 31,030 | 31,740 | 1.7% | 25% | 2,240 | 2,290 | +| 33,240 | 34,000 | 1.8% | 25% | 2,240 | 2,290 | +| 37,690 | 38,540 | 1.9% | 30% | 2,240 | 2,290 | +| 39,890 | 40,800 | 2.0% | 30% | 2,240 | 2,290 | +| 46,540 | 47,590 | 2.0% | 35% | 2,240 | 2,290 | +| 53,180 | 54,390 | 2.0% | 40% | 2,240 | 2,290 | +| 62,060 | 63,470 | 2.0% | 45% | 2,040 | 2,080 | +| 64,260 | 65,720 | 2.0% | 45% | 1,830 | 1,870 | +| 66,480 | 68,000 | 2.0% | 45% | 1,550 | 1,590 | +| 68,720 | 70,280 | 2.0% | 50% | 1,360 | 1,390 | +| 70,920 | 72,530 | 2.0% | 50% | 1,220 | 1,250 | +| 73,140 | 74,810 | 2.0% | 50% | 680 | 690 | +| 75,350 | 77,070 | 2.0% | 50% | 270 | 270 | +| 77,570 | 79,330 | Ineligible | Ineligible | 0 | 0 | + +Four of the 51 projected 2025 renter entries disagree with the publication: `claimant_share[6].threshold` is 46,530 rather than 46,540; `claimant_share[10].threshold`, `max_credit[15].threshold`, and `percent_of_income[11].threshold` are 77,560 rather than 77,570. The other 47 match. + +### Attribution + +Scenario 032: `state_refundable_credits → mn_refundable_credits → mn_child_and_working_families_credits`. The formula uses `credits.cwfc.ctc.amount` and the 4% working-family phase-in capped at `credits.cwfc.wfc.phase_in[1].threshold`. Holding these explains the r18 delta exactly at arithmetic precision: `(1750−1800) + .04×(9480−9690) = −58.40`. The published 2026 amount is `1800 + .04×9690 = 2187.60`, matching the frozen credit. The formula also reads joint phase-out 38,770, additional older-child amounts, and the other filing-status phase-out, but they do not alter this household's result. The lone child is age 6, earnings are $29,000, and there is no phase-out or older-child add-on. + +Scenario 122: `state_income_tax_before_refundable_credits → mn_income_tax_before_refundable_credits → mn_income_tax_before_credits → mn_basic_tax → mn_taxable_income → mn_deductions → mn_standard_deduction`. The single base deduction is projected at 15,250; its last explicit 2025 value is 14,950. Holding it increases first-bracket tax by `.0535×300 = 16.05`, exactly explaining the r18 move. The published 2026 base is 15,300, implying a convention reduction of `.0535×50 = 2.675`. Two aged/blind additions are $2,000 each in both years. Other projected deductions, exemptions, brackets, Social Security, AMT, and marriage parameters are read in these formulas but do not explain the assigned move. + +Renter schedules are relevant to the parameter audit but do not produce either assigned r18 move. `mn_renters_credit_eligible` requires `mn_renters_credit_qualifying_crp`, whose default is false; neither scenario supplies that input. The fix preserves those household inputs and eligibility formulas. + +### Full sweep and classified rows + +The full sweep completed successfully: **1,984 outputs, 42 Minnesota outputs across scenarios 032 and 122, one Minnesota move, no additional Minnesota movers beyond the r18 candidates**. The other 41 Minnesota outputs match exactly. Result: [r19_mn_convention.csv](work/r19_mn_convention.csv). Machine-readable classified rows: [r19_mn_summary.json](work/r19_mn_summary.json). + +| Scenario | Variable | Frozen | Convention | Delta | Classification | Reason | +| --- | --- | ---: | ---: | ---: | --- | --- | +| 032 MN | `state_refundable_credits` | 2,187.60010 | 2,187.60010 | 0.00000 | `unchanged` | Published child-credit and working-family phase-in amounts equal the projections used by this household. | +| 122 MN | `state_income_tax_before_refundable_credits` | 1,028.21655 | 1,025.54150 | −2.67505 | `engine_defect_stale` | The published single standard deduction is $15,300, $50 above the projection. | + +The raw CSV also contains nine out-of-state SNAP moves. These are a **reference-snapshot discrepancy**, not a Minnesota-parameter effect: the initial sweep read different reference values from the values recorded as frozen in the r18 output, and the Illinois no-op sweep reproduces the old r18 values. The exact 14 changed reference rows (nine above tolerance and five below) are recorded in [r19_reference_snapshot_difference.json](work/r19_reference_snapshot_difference.json). Every SNAP result below equals the no-op control and its old r18 frozen value. The Minnesota fix's effect on each is therefore `unchanged`; the initial-reference deltas are preserved separately and do not enter the Minnesota convention summary. The final Michigan rerun subsequently read the original frozen bundle; the aggregate report documents that input change. + +| Scenario | State | Variable | Frozen | Raw recomputed | Raw delta | +| --- | --- | --- | ---: | ---: | ---: | +| 008 | NJ | `snap` | 15,120.00000 | 15,246.90527 | 126.90527 | +| 012 | MS | `snap` | 4,884.00000 | 4,952.08936 | 68.08936 | +| 038 | LA | `snap` | 7,212.00000 | 7,286.94434 | 74.94434 | +| 043 | CO | `snap` | 3,576.00000 | 3,596.03979 | 20.03979 | +| 054 | NC | `snap` | 6,060.00000 | 6,125.68896 | 65.68896 | +| 066 | VA | `snap` | 3,576.00000 | 3,596.03979 | 20.03979 | +| 079 | AZ | `snap` | 2,376.00000 | 2,428.01733 | 52.01733 | +| 080 | PA | `snap` | 3,576.00000 | 3,596.03979 | 20.03979 | +| 109 | FL | `snap` | 7,932.00000 | 8,020.55420 | 88.55420 | + +Five further SNAP differences of −$0.31683 in scenarios 027, 030, 045, 073, and 108 are below tolerance and also shared across family sweeps. Two scenario-112 float serialization differences are below $10^-12. + +Validation also passed for module syntax, complete 141/141 projected-2026 coverage, and 51/51 projected-2025 coverage. Applying the actual modifier to a core `ParameterNode` populated from the frozen parameter inventory verifies every resulting 2026 amount and all 51 projected 2025 entries against the source inventory. Three subset attribution runs were interrupted during slow initial parameter loading; attribution above uses the formulas and existing r18 family sweep. An initial full attempt used an unsupported start/stop argument type; it was corrected to annual `period` updates and the full sweep was rerun successfully. + +Reproduce from the triage sweep directory, with all output and fix paths inside the assigned workspace: + +```sh +PYTHONDONTWRITEBYTECODE=1 \ +PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ +../.venv-pe1755/bin/python sweep.py \ + --fix /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5b/sweep/fixes/r19_mn_convention.py \ + --out /Users/maxghenis/PolicyEngine/_wk/pb-triage-verify-v5b/sweep/verify/work/r19_mn_convention.csv +``` + + +## Maryland + +The federal mover is a stale withholding-deduction parameter. The state mover is a return-deduction projection whose publication status remains **unverified at the freeze**. The supplied module computes a provisional hold for that return deduction and exposes `MD_VERIFIED_ONLY=1` to retain only the verified withholding correction. + +### Parameters and attribution + +All paths below start with `gov.states.md.tax.income.`. The family contains 25 projected parameters: five return flat amounts, five old maxima, five old minima, and ten CDCC eligibility caps. Ten have projected 2025 entries (the maxima and minima). The complete inventory, engine values, and treatment are in [`work/r19_md_parameters.json`](work/r19_md_parameters.json). + +| Parameter / status | Engine 2025 | Engine 2026 | Convention used for 2026 | +|---|---:|---:|---:| +| `deductions.standard.flat_deduction.amount.SINGLE`, `SEPARATE` | 3,350 | 3,400 | 3,350, **provisional hold** | +| Same, `JOINT`, `HEAD_OF_HOUSEHOLD`, `SURVIVING_SPOUSE` | 6,700 | 6,850 | 6,700, **provisional hold** | +| `deductions.standard.max.SINGLE`, `SEPARATE` | 2,750 projected | 2,800 | 3,400 as withholding allowance | +| Same, other three statuses | 5,600 projected | 5,700 | 3,400 as withholding allowance; not read by current formulas | +| `deductions.standard.min.SINGLE`, `SEPARATE` | 1,850 projected | 1,850 | No operative minimum after 2024; unchanged, unused | +| Same, other three statuses | 3,750 projected | 3,800 | No operative minimum after 2024; unchanged, unused | +| `credits.cdcc.eligibility.agi_cap.JOINT` | 174,300 | 178,250 | 174,300, **provisional hold** | +| Same, all other statuses | 112,100 | 114,600 | 112,100, **provisional hold** | +| `credits.cdcc.eligibility.refundable_agi_cap.JOINT` | 91,400 | 93,450 | 91,400, **provisional hold** | +| Same, all other statuses | 60,900 | 62,250 | 60,900, **provisional hold** | + +`md_standard_deduction.py` first checks `flat_deduction.applies`, which is true from 2025; it returns the flat amount and does not read the old minimum/maximum calculation. Scenario 068 is single and uses the flat amount. Reducing that amount from $3,400 to $3,350 increases taxable income by $50 and state tax by $50 × 4.75% = $2.375. Scenario 078 itemizes and the state deduction is larger than either flat amount. + +The otherwise-obsolete maximum remains live in `md_withheld_income_tax.py`, which reads **only `max.SINGLE` for every person**, irrespective of filing status. The path is: + +`max.SINGLE` → `md_withheld_income_tax` → `state_withheld_income_tax` → `state_and_local_sales_or_income_tax` → `salt` → `salt_deduction` → federal itemized deductions → federal taxable income → federal income tax before refundable credits. + +The state withholding proxy excludes the taxpayer's Maryland liability calculation. This explains why scenario 078's federal output moves while its Maryland output does not. The r18 hold drops both 2025 and 2026 projected maxima and reaches **$2,700 (2024)**, rather than $2,750 (projected 2025) or $3,350 (published 2025). Its federal delta therefore cannot serve as the correction. The published $3,400 withholding allowance reduces the proxy tax by $600 × 5.5% = $33 and, with uncapped marginal SALT and a 24% federal marginal rate, should increase federal tax by approximately $7.92. That arithmetic is an attribution check; exact floating-point output belongs to the sweep below. + +### Sources, dates, and limits + +1. **Published 2026 withholding allowance: $3,400, all payroll statuses.** The [Maryland 2026 Employer Withholding Guide](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/instructions/withholding/2026/withholding-guide.pdf), cover marked **revised December 2025**, explicitly limits this amount to the percentage withholding method (PDF page 2; complete payroll-period table on page 11). A [US Department of the Interior payroll announcement dated April 9, 2026](https://ibc.doi.gov/HRD/Payroll/Announcements/04-09-26) independently records the increase from $3,350 to $3,400 effective January 1, 2026. Thus the amount was public before July 3, even though the guide's exact first upload date is unverified. The guide's $3,400 is a withholding allowance, not a published joint return deduction. + +2. **Published 2025 return amounts: $3,350 single/separate/dependent; $6,700 joint/HOH/surviving spouse.** [Chapter 604 / HB352](https://mgaleg.maryland.gov/2025RS/Chapters_noln/CH_604_hb0352e.pdf) replaced the income-dependent minimum/maximum with flat amounts and inflation adjustment after 2025; the [legislative history](https://mgaleg.maryland.gov/mgawebsite/Legislation/Details/HB0352?ys=2025RS) gives approval on **May 20, 2025**. The Comptroller's [Tax Alert, revised December 22, 2025](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/legal-publications/alerts/tax-alert-changes-to-standard-and-itemized-deductions-and-to-state-and-local-income-tax-rates-from-the-2025-legislative-session.pdf), section IV.A, confirms the amounts and removal of the income phase-in. This proves the engine's projected 2025 $2,750 maximum is stale as a withholding/standard-deduction proxy. There is no operative post-2024 minimum table to replace the five obsolete projected minima with. + +3. **2026 return publication status is not established conclusively.** The official [HB411 fiscal note](https://mgaleg.maryland.gov/2026RS/fnotes/bil_0001/hb0411.pdf), dated **February 11, 2026** on its last page, says the 2026 standard deduction had not yet been announced. The currently accessible [2025 Resident Booklet](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/instructions/2025/resident-booklet.pdf), PDF page 53, gives $3,350/$6,700 in its instructions specifically for estimating **2026** tax. Its first publication and revision dates were not established. The separate [official 2026 estimated-tax worksheet](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/forms/worksheets/2026-pv-worksheet.pdf), page 1, line 4 instructions, also gives $3,350/$6,700, but has no visible publication date. These sources do not prove that no later return-deduction announcement appeared between February 11 and July 3. The source search located no final 2026 return-deduction schedule. The default fix therefore computes the requested last-published hold **provisionally**; scenario 068's `held_convention` label must remain provisional. If a qualifying pre-freeze source publishes the single return deduction at $3,400, that row becomes `unchanged`. + +The statutory measurement window does **not** establish a late-publication hold. [Maryland §10-217(c)](https://mgaleg.maryland.gov/mgawebsite/laws/StatuteText?article=gtg§ion=10-217) uses IRC §1(f)(3), substituting base year 2024. [IRC §1(f)(3) and (6)](https://uscode.house.gov/view.xhtml?req=title%3A26+section%3A1+edition%3Aprelim) use the preceding calendar year and the 12 months ending August 31, with the index vintage fixed when that August index is initially published. For tax year 2026 this points to August 2025, whose [BLS release was September 11, 2025](https://www.bls.gov/news.release/archives/cpi_09112025.htm). Thus the measurement timing permits publication before July 3, 2026; it does not prove whether Maryland announced a return amount by then. A further search of official March–July materials located withholding notices and the [standalone 2026 estimated-tax worksheet](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/forms/worksheets/2025-PV-Worksheet.pdf) carrying $3,350/$6,700, but no conclusive dated return-deduction announcement. The provisional classification is preserved. + +4. **CDCC caps.** [2025 Form 502CR](https://www.marylandcomptroller.gov/content/dam/mdcomp/tax/forms/2025/502cr.pdf), printed **10/25**, PDF page 6, gives AGI caps of $174,300 joint and $112,100 all other statuses; refundable caps are $91,400 joint and $60,900 individual. Exact first-publication day is unverified. No 2026 publication/date was located. The module holds all ten caps provisionally at those published values. Both Maryland households have no children or care expenses, so this unresolved timing cannot affect their scored credit outputs. + +### Fix, coverage, and results + +Fix: [`../fixes/r19_md_convention.py`](../fixes/r19_md_convention.py). It changes parameters only. It covers all five statuses and all ten CDCC caps. It repurposes the obsolete maximum table as the live withholding proxy: $3,350 for 2025 and $3,400 for 2026, without a filing-status split, matching the withholding guide. The old minima remain untouched because they are legally superseded and no longer read. This does not fix the withholding proxy's other simplifications, such as its treatment of withholding exemptions, county taxes, or actual W-2 withholding; those are outside this parameter audit. + +The full unmodified triage sweep completed: **1,984 outputs**, **11 moved** by more than $1, of which **2 are Maryland outputs**, with 5 nonzero differences within tolerance. The scan covered all 32 Maryland outputs in scenario_068, scenario_078. There were 0 additional movers beyond the r18 Maryland-family sweep and 9 movers outside Maryland. All nine out-of-state movers and five small differences are SNAP reference-snapshot differences: the initial sweeps' harness reference file differed from the r18 frozen column on exactly those 14 rows, while the Illinois no-op recomputation reproduces the old r18 frozen values on all 14. They are retained in the raw CSV and excluded from Maryland classifications. See [`r19_reference_snapshot_difference.json`](work/r19_reference_snapshot_difference.json). The Maryland frozen values agree across the snapshots. + +| Scenario | Variable | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| scenario_068 | `state_income_tax_before_refundable_credits` | 1252.96777344 | 1255.34277344 | +2.37500000 | `held_convention` (provisional) | Provisional hold of return deduction from $3,400 to published 2025 $3,350; absence of a later pre-freeze return announcement is unverified. | +| scenario_078 | `federal_income_tax_before_refundable_credits` | 24772.69335938 | 24780.61328125 | +7.91992188 | `engine_defect_stale` | Published 2026 withholding allowance is $3,400, not projected legacy $2,800; lower withholding reduces federal SALT and raises federal tax. | + +Exact results: [`work/r19_md_convention.csv`](work/r19_md_convention.csv); summary: [`work/r19_md_summary.json`](work/r19_md_summary.json). The verified-only setting omits the provisional return/CDCC holds; it was not separately swept. The 2026 withholding correction and default hold have separate parameter readers, as traced above. + +Validation: Python syntax parse passed. The first full attempt exposed the installed core's requirement for `Instant` objects when passing `stop`; the module was corrected to use `update(period=str(year), value=...)`, and the complete sweep above is the successful rerun. Two redundant r18 subset runs were stopped during initial parameter loading to reduce contention; their incomplete results are not used. Attribution is from the installed formulas and existing r18 Maryland-family sweep. + + +## Michigan + +The personal exemption causing scenario 045's r18 move is a verified stale projection: the engine uses $5,950; the published 2026 amount is **$5,900**, not the $5,800 r18 hold. + +### Parameters and amounts + +All paths below start with `gov.states.mi.tax.income.`. The inventory [`work/r19_mi_parameters.json`](work/r19_mi_parameters.json) covers all **28 projected parameters**, including all **17 projected 2025 entries**. Status tables cover SINGLE, SEPARATE, HEAD_OF_HOUSEHOLD, SURVIVING_SPOUSE, and JOINT; as in the installed table, JOINT receives the joint limit and other engine statuses receive the nonjoint limit. + +| Parameter / table | Published 2025 | Convention 2026 | Publication assessment | +|---|---|---|---| +| `exemptions.personal` | 5,800 | 5,900 | Published before freeze | +| `deductions.retirement_benefits.tier_one.amount.*` | 65,897 nonjoint; 131,794 joint | 67,610 nonjoint; 135,220 joint | Published before freeze | +| `exemptions.disabled.amount.base` | 3,400 | 3,400 | Provisional hold; 2026 publication/date unverified | +| `deductions.interest_dividends_capital_gains.amount.*` | 14,688 nonjoint; 29,376 joint | Same published 2025 amounts | Provisional hold; 2026 publication/date unverified | +| `credits.homestead_property_tax.cap` | 1,900 | 1,900 | Provisional hold | +| `.household_resources_limit` | 71,500 | 71,500 | Provisional hold | +| `.property_value_limit` | 165,400 | 165,400 | Provisional hold | +| `.reduction.start` | 62,500 | 62,500 | Provisional hold | +| `credits.home_heating.standard.base[0..5].amount` (0/1, 2, 3, 4, 5, 6 exemptions) | 604; 815; 1,027; 1,239; 1,451; 1,662 | Same published 2025 table | Hold; agency plan schedules 2026 forms for January 2027 | +| `.additional_exemption.amount` | 212 above six exemptions | 212 | Same | +| `.alternate.household_resources.cap[0..3].amount` (0/1, 2, 3, 4+ exemptions) | 18,592; 25,018; 31,449; 34,227 | Same published 2025 table | Same | +| `.alternate.heating_costs.cap` | 3,765 | 3,765 | Same | + +The engine's 2025 senior investment limits are already projected ($14,685.5292/$29,371.0583), rather than the published $14,688/$29,376. Its 2025 heating allowances, additional allowance, fuel cap, and four alternate-income caps are also projections; the inventory gives each engine value. The fix uses actual published 2025 values for both 2025 and held 2026 entries, rather than treating the forecast as a published base. + +### Primary sources and date evidence + +1. [2026 Treasury Form 446](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2026/446_Withholding-Guide_2026.pdf), **revision February 2026**, pages 1–2, supplies the personal exemption and retirement limits. [2025 Form 446](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2025/446_Withholding-Guide_2025.pdf), **revision January 2025**, supplies their 2025 counterparts. A [DOI payroll announcement dated February 12, 2026](https://ibc.doi.gov/HRD/Payroll/Announcements/02-12-26) independently confirms the exemption's change from $5,800 to $5,900. Exact first upload days of the guides are unverified; both revision months precede the freeze. +2. [2025 MI-1040 instructions](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040-Book.pdf), pages 2–3, 14 and 26–33, give disability, senior investment, and homestead amounts. A precise first publication date was not found; this is a 2025 filing-season booklet for returns due April 15, 2026. Its pre-freeze availability is inferred from that context, not independently established by a dated upload. No 2026 table or release date for these amounts was located; the corresponding 2026 holds are provisional. +3. [2025 MI-1040CR-7](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040CR-7.pdf), **revision January 2026**, line 40, gives the $3,765 fuel cap. Its [instruction book](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040CR-7-Book.pdf), Table A/B (PDF page 12), gives the full allowance and alternate-income tables. The [2025 Taxpayer Assistance Manual](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Tax-Professional/2025-Taxpayer-Assistance-Manual.pdf), printed pages 83 and 85, independently agrees with these tables. The linked instruction book's exact publication day was not established. The current summary HTML has conflicting entries ($1,207 instead of $1,027, and $18,595 instead of $18,592); the fix follows the matching instruction PDF and manual. +4. The [FY2027 proposed LIHEAP state plan](https://www.michigan.gov/mdhhs/-/media/Project/Websites/mdhhs/Inside-MDHHS/Reports-and-Statistics---Health-Services/Proposed-LIHEAP-State-Plan-27.pdf), page 10, says tax-year 2026 heating forms will be available in January and final benefits are estimates pending funding. This is evidence for January 2027 availability, not an actual publication date. Its own illustrative old fuel cap is not used in place of the filed 2025 form's amount. + +### Attribution, fix, and coverage + +`exemptions.personal` → `mi_personal_exemptions` → `mi_exemptions` → `mi_taxable_income` → Michigan tax before refundable credits. Scenario 045 is a 44-year-old single filer without dependents, so one $50 reduction from the projected allowance increases tax by $50 × 4.25% = **$2.125**. The r18 increase of $6.375 instead used a $150 reduction. These are arithmetic attribution checks; the table below uses the actual sweep output. + +Fix: [`../fixes/r19_mi_convention.py`](../fixes/r19_mi_convention.py). It updates all sourced statuses/brackets and all projected 2025 entries. `MI_VERIFIED_ONLY=1` isolates the six published 2026 leaves and omits held tables. Provisional amounts are never described as conclusively established at the freeze. The full sweep includes Michigan scenarios **045, 072 and 073**, plus every other reference; the full sweep results are below. + +### Computed results + +| Scenario | Output | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| 045 | State tax before refundable credits | 1288.837524 | 1290.962524 | +2.125000 | `engine_defect_stale` | The published 2026 exemption of $5,900 replaces the projected $5,950; one exemption raises taxable income by $50 at 4.25%. | + +The required full sweep completed **1,984 outputs**, including **52 MI outputs**. No additional state/family movers were found. Raw CSV: [`work/r19_mi_convention.csv`](work/r19_mi_convention.csv). The nine other-state SNAP raw movers and five small SNAP discrepancies are common reference-snapshot differences: their initially read reference amounts differ from r18, and this sweep recomputes the old r18 amounts. They are not effects of this family; see the aggregate report and [`work/r19_reference_snapshot_difference.json`](work/r19_reference_snapshot_difference.json). + +A direct formula trace of the final module confirms scenario 045 taxable income rises from $30,325.59 to $30,375.59 while its refundable credits remain $760.78876. Scenarios 072 and 073 retain zero taxable income; all three retain zero home-heating credit. The tracer recorded variable results but no parameter-access nodes, so parameter attribution rests on the installed formulas and controlled reforms, not on interpreting empty trace lists. Trace: [`work/r19_mi_il_trace.json`](work/r19_mi_il_trace.json). + +Final expanded-module rerun: [`work/r19_mi_final_convention.csv`](work/r19_mi_final_convention.csv) contains **1,984 outputs and exactly one mover**, scenario 045 (+$2.125). Every computed value exactly matches the earlier full run; only 14 SNAP reference inputs changed when the harness selected the original frozen bundle. The earlier CSV is retained unchanged. + + +## Missouri (`gov.states.mo.*`) + +The assigned `scenario_093` change is attributable to the income-tax bracket thresholds. The convention uses the published **2026** thresholds, not a hold of the 2025 schedule. Publication before the freeze is verified: the Missouri Department of Revenue forms index dates its 2026 withholding formula **November 21, 2025**, and a second official publication reproduces the same annual brackets on **May 14, 2026**. The index date is a revision date, not an independently verified first-upload timestamp. [DOR forms index](https://dor.mo.gov/forms/), [2026 formula, annual table on page 2](https://dor.mo.gov/forms/Withholding%20Formula_2026.pdf), [USDA National Finance Center bulletin](https://help.nfc.usda.gov/bulletins/2026/1773783048.htm) + +The complete common schedule for all filing statuses is below. The final infinite threshold is the engine's inactive ninth bracket; the official schedules have no ninth finite boundary. The two thresholds with projected 2025 entries (`rates[0]` and `rates[8]`) remain zero and infinity and therefore match the published schedule. The official 2025 chart is listed with a **December 23, 2025** revision date. [2025 tax chart](https://dor.mo.gov/forms/2025%20Tax%20Chart_2025.pdf), [DOR forms index](https://dor.mo.gov/forms/) + +| Parameter suffix | 2025 published | Engine 2026 projection | 2026 convention | Marginal rate above boundary | +|---|---:|---:|---:|---:| +| `rates[0].threshold` | 0 | 0 | 0 | 0% | +| `rates[1].threshold` | 1,313 | 1,342.756404 | 1,348 | 2% | +| `rates[2].threshold` | 2,626 | 2,685.512809 | 2,696 | 2.5% | +| `rates[3].threshold` | 3,939 | 4,028.269213 | 4,044 | 3% | +| `rates[4].threshold` | 5,252 | 5,371.025618 | 5,392 | 3.5% | +| `rates[5].threshold` | 6,565 | 6,713.782022 | 6,740 | 4% | +| `rates[6].threshold` | 7,878 | 8,056.538427 | 8,088 | 4.5% | +| `rates[7].threshold` | 9,191 | 9,399.294831 | 9,436 | 4.7% | +| `rates[8].threshold` | infinity | infinity | infinity | inactive | + +Every threshold path begins `gov.states.mo.tax.income.`. The fix is [`sweep/fixes/r19_mo_convention.py`](../fixes/r19_mo_convention.py). It supplies every 2025 and 2026 boundary, including both stable projected 2025 boundaries, and leaves the already correct marginal rates unchanged. + +Attribution from installed policyengine-us 1.755.4 formulas: `mo_income_tax_before_credits` reads `gov.states.mo.tax.income.rates` and calls `rates.calc(mo_taxable_income)` per person; state tax aggregates the resulting liability after nonrefundable credits. `scenario_093` has two adult earners and an adult dependent. The supplied r18 Missouri-family attribution CSV changes only its state income-tax output, by +$7.974609375 when the 2025 boundaries are held. + +A separate projected parameter, `gov.states.mo.tax.income.deductions.social_security_and_public_pension.mo_max_social_security_benefit`, has a material source gap that does **not** affect these households. The engine's explicit 2025 value is $48,216; the official 2025 Form MO-A instead specifies **$47,633** (Part 3, Section A, line 2; index revision **December 23, 2025**). The fix repairs only 2025 and explicitly preserves the frozen 2026 projection of $49,308.71500083913. This preservation matters because the harness constructs a system that applies reforms before and after uprating. [2025 MO-A](https://dor.mo.gov/forms/MO-A_2025.pdf), [DOR forms index](https://dor.mo.gov/forms/) + +The current DOR pension FAQ publishes **$48,967 for 2026**, but its publication date is **unverified**; no dated primary record establishing which side of July 3, 2026 it falls on was found. If pre-freeze, $48,967 applies; if first published after the freeze, the held convention is $47,633. The module does not label either value definitive for 2026. RSMo 143.124(5) defines the state cap by a CPI adjustment to a statutory amount; one should not substitute SSA's national maximum benefit merely because the parameter has a similar name. The displayed statutory version is effective August 28, 2023. [DOR pension FAQ](https://dor.mo.gov/faq/taxation/individual/pension.html), [RSMo 143.124](https://revisor.mo.gov/main/OneSection.aspx?section=143.124) + +Neither Missouri household supplies public pension income. Scenario 021 has Social Security and **private** pension income; scenario 093 has earned and interest income. The installed `mo_pension_and_ss_or_ssd_deduction_section_a` applies the projected cap to `taxable_public_pension_income`, which is zero for those facts. + +The required full sweep completed **1,984 outputs**. Its Missouri result is: + +| Scenario | Variable | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| 093 | `state_income_tax_before_refundable_credits` | 3,389.650879 | 3,388.245605 | −1.405273 | `engine_defect_stale` | The pre-freeze 2026 bracket boundaries are larger than the engine projections. | + +All **40 Missouri outputs** (16 for scenario 021 and 24 for scenario 093) were scanned; no additional Missouri outputs move, including none missed by the r18 hold. The full sweep's raw CSV also contains nine out-of-state SNAP moves and five smaller SNAP deltas. A control calculation with the current global baseline and a no-op clone reproduces all fourteen values exactly: these are **reference-snapshot differences, not Missouri reform effects**, and are not assigned a Missouri convention classification. For these same fourteen rows, the initial harness's frozen values differed from the frozen values recorded in the supplied r18 CSV, and the no-op calculation matches the original r18 values. This identifies a changed reference snapshot without assuming an engine change. See [`work/r19_reference_snapshot_difference.json`](work/r19_reference_snapshot_difference.json). Both the raw result and the control are retained rather than silently removing rows. + +Artifacts: [`work/r19_mo_convention.csv`](work/r19_mo_convention.csv), [`work/r19_mo_snap_control.json`](work/r19_mo_snap_control.json), [`work/r19_mo_summary.json`](work/r19_mo_summary.json), and [`work/r19_mo_sources.json`](work/r19_mo_sources.json). The full **1,984-output** [`baseline control`](work/r19_baseline_control.csv) completed and confirms that the Missouri reform has exactly one nonzero effect across the entire bundle; [`attributed effects`](work/r19_mo_attributed_effects.json) records it. Pension-cap invariance is established algebraically in [`work/r19_mo_formula_checks.json`](work/r19_mo_formula_checks.json): all four people have zero public pension income, and `min(public_income, cap)` is zero with either $47,633 or $48,967. This is formula-based evidence, not a completed full-engine sensitivity sweep. An exploratory direct-clone sensitivity method failed in `loss_ald` with a NumPy structured-record type error, so no results from that method are claimed. The full harness sweeps completed successfully. The separate r18 rate-subset run was interrupted during YAML parsing; formula attribution and the supplied family-attribution CSV establish the parameter path instead. + +Validation: full reform sweep 1,984 rows; full baseline control 1,984 rows; one attributed reform effect; all 40 Missouri rows checked; static Python syntax and whitespace checks passed. All writes are confined to the assigned workspace. + + +## Illinois + +The r18 mover is **unchanged under the convention**: the engine projected the published $2,925 exemption correctly. + +### Parameters, sources, and dates + +Both `gov.states.il.tax.income.exemption.personal` and `.dependent` are $2,850 in 2025 and $2,925 in 2026. Neither has a projected 2025 entry. The [IDOR FY 2026-15 bulletin](https://tax.illinois.gov/research/publications/bulletins/fy-2026-15.html), dated **December 2025** (exact day unverified), publishes both annual amounts; its 2025 Schedule IL-E/EITC discussion also identifies the dependent allowance. The [2026 IL-700-T withholding booklet](https://tax.illinois.gov/content/dam/soi/en/web/tax/forms/withholding/documents/currentyear/il-700-t.pdf), **R-12/25**, uses $2,925 for each regular exemption. The convention therefore uses the published 2026 value, not the 2025 hold. The module sets both leaves for all filers and dependents; eligibility and additional aged/blind allowances are unchanged. + +### Attribution and fix + +Scenario 070 is a single filer without dependents. `exemption.personal` → `il_personal_exemption` → `il_total_exemptions` → `il_taxable_income` → Illinois tax. The r18 hold reduces this person's allowance by $75 and raises tax by $75 × 4.95% = $3.7125; `.dependent` multiplies a zero dependent count and contributes nothing. The formula also reads the personal allowance in its eligibility check, but this taxpayer's eligibility does not change. + +Fix: [`../fixes/r19_il_convention.py`](../fixes/r19_il_convention.py). Inventory: [`work/r19_il_parameters.json`](work/r19_il_parameters.json). The other Illinois household is scenario 098; the full sweep includes both, together with every other state. The full sweep results are below. + +### Computed results + +| Scenario | Output | Frozen | Convention | Delta | Classification | Reason | +|---|---|---:|---:|---:|---|---| +| 070 | State tax before refundable credits | 1650.561279 | 1650.561279 | +0.000000 | `unchanged` | The projected 2026 exemption of $2,925 already equals the amount published in December 2025; the r18 hold at $2,850 would be incorrect. | + +The required full sweep completed **1,984 outputs**, including **32 IL outputs**. No additional state/family movers were found. Raw CSV: [`work/r19_il_convention.csv`](work/r19_il_convention.csv). The nine other-state SNAP raw movers and five small SNAP discrepancies are common reference-snapshot differences: their initially read reference amounts differ from r18, and this sweep recomputes the old r18 amounts. They are not effects of this family; see the aggregate report and [`work/r19_reference_snapshot_difference.json`](work/r19_reference_snapshot_difference.json). diff --git a/reference_audit/2026-09-22/verification/v6_flag_067.md b/reference_audit/2026-09-22/verification/v6_flag_067.md new file mode 100644 index 00000000..0a0127f7 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v6_flag_067.md @@ -0,0 +1,97 @@ +**scenario_067 / dependent1_medicaid_eligible — verdict: `prompt_ambiguity`** + +The frozen value is **1**. The relationship-only alternative is **0** if Dependent 1 is the biological, adopted, or step child of the claiming filer. The prompt calls this person “Dependent 1,” age 23, and never establishes that relationship. Another adult tax dependent can properly have a separate Medicaid MAGI household. The judge's parent-child premise is therefore an assumption, not a stated fact. Under the requested scoring standard this is an **unlisted-input exclusion**, not a demonstrated engine defect on fully specified facts. + +This relationship root cause is **not an entry in `triage/root_causes.json`**, which I read without reopening its settled findings. The paper already acknowledges under-specified adult tax-dependent relationships in [its limitations](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/index.qmd:1243). Its statement about historically scoring encoded inputs does not resolve the ambiguity under this investigation's supplied rule. + +**Facts and reproduced mechanism.** The frozen scenario puts Head (60), Spouse (58), and Dependent 1 (23) in `adults`, with an empty `children` list. Dependent 1 has both tax-head and tax-spouse flags false, `is_disabled=true`, and $10,800 of generic `disability_benefits`. There is no parent-child link or `own_children_in_household` input. “All listed people live together” and a shared household group do not specify the relationship needed for a person-specific Medicaid household. The instruction setting unlisted numeric inputs to zero explains the engine's zero parent-count input; it does not tell the reader which family relationship “Dependent 1” represents. + +I ran `pe_case.py` under 1.755.4 against the pristine frozen bundle. Both `frozen_reference` and `recomputed_baseline` were **1.0**; the full situation and calculated intermediates are in [baseline.txt](sweep/work/r067_adult_dependent_relationship/baseline.txt). These findings independently reproduce the earlier investigator's leads. + +The following paths are relative to `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. I read each described formula: + +| Engine file | Observed mechanism | +| --- | --- | +| `variables/household/demographic/person/is_parent.py` and `own_children_in_household.py` | `is_parent` is whether the own-child count exceeds zero; the absent count leaves all three false. | +| `variables/household/demographic/person/is_qualifying_child_dependent.py` | Its age/student test is false for this 23-year-old nonstudent. This variable explicitly excludes the separate disability exception. | +| `variables/gov/hhs/medicaid/income/medicaid_claimed_by_parent_in_tax_unit.py` | Infers a parental claimant from the qualifying-child test or any head/spouse with `is_parent=true`. Neither holds here. | +| `variables/gov/hhs/medicaid/income/medicaid_tax_dependent_exception_other_than_spouse_or_child.py` and `medicaid_uses_non_filer_rules.py` | A dependent without that inferred parental claimant triggers the other-dependent exception and nonfiler household rules. | +| `variables/gov/hhs/medicaid/income/medicaid_household_income.py` and `medicaid_household_size.py` | The adult nonfiler calculation yields Dependent 1's own $0 MAGI and size 1; the tax-household branch instead includes $87,302 and size 3. | +| `variables/gov/hhs/medicaid/income/medicaid_magi_person.py` and `medicaid_income_level.py` | Individual MAGI is `[87302, 0, 0]`; household income divided by the applicable poverty guideline produces the eligibility ratio. The generic disability benefit's taxability is not changed in this investigation. | +| `variables/gov/hhs/medicaid/eligibility/categories/adult/is_adult_for_medicaid_fc.py`, `parameters/gov/hhs/medicaid/eligibility/categories/adult/income_limit.yaml`, and `variables/gov/hhs/medicaid/eligibility/categories/medicaid_category.py` | Indiana's adult threshold is 1.38 FPL. Dependent 1 selects `ADULT` at zero income; under the child-relationship reading no category succeeds. | +| `variables/gov/hhs/chip/is_chip_eligible_child.py` and `parameters/gov/hhs/chip/child/max_age.yaml` | CHIP's child pathway requires age under 19. Changing a parent relationship does not change this person's age. | + +**Law applicable in 2026.** The official [2025 edition of 42 CFR 435.603](https://www.govinfo.gov/content/pkg/CFR-2025-title42-vol4/pdf/CFR-2025-title42-vol4-sec435-603.pdf), printed April 8, 2026, precedes the July 3 freeze. Paragraph (b) defines child by biological/adoptive/step relationship without an age limit. Paragraph (f)(2) uses the claiming taxpayer's household; (f)(2)(i) is the exception for dependents other than spouse/child. Paragraph (f)(3) limits this 23-year-old's nonfiler household to self and relevant resident spouse/children; younger applicants can also include parents/siblings. Paragraph (d)(1) sums household MAGI, subject to dependent-income exclusions in (d)(2). These rules distinguish the two readings; age 23 alone cannot establish the exception. + +[Indiana's Medicaid manual chapter 3200](https://www.in.gov/dA/9fcd49126c/Medicaid_PM_3200.pdf?language_id=1), pages 5–9, corroborates the distinction and specifically discusses adult children claimed by parents. Its current PDF revision date was not verified, so the official CFR edition supplies the prefreeze evidence. + +[42 CFR 435.119](https://www.govinfo.gov/content/pkg/CFR-2025-title42-vol4/pdf/CFR-2025-title42-vol4-sec435-119.pdf) provides the age 19–64 adult group and 133% FPL standard. The five-point disregard in §435.603(d)(4) yields the effective 138% test. [Indiana chapter 3500, §3515](https://www.in.gov/dA/0ca2b535d9/Medicaid_PM_3500.pdf?language_id=1) confirms this HIP standard and permits HIP pending a disability determination. Its current PDF revision date is also unverified. Generic disability does not automatically foreclose MAGI coverage, and failure of the MAGI test alone does not decide all disability-based pathways. + +[HHS's January 15, 2026 notice](https://www.govinfo.gov/content/pkg/FR-2026-01-15/pdf/2026-00755.pdf) publishes the $27,320 three-person poverty guideline. The child's alternative household income is **$82,360 wages + $4,942 pension = $87,302**, giving **$87,302 / $27,320 = 3.195534407 FPL**, above **1.38 × $27,320 = $37,701.60**. The sandbox returns 3.1955345 because of floating-point precision. This is not a projected-parameter dispute. + +[42 CFR 457.10](https://www.ecfr.gov/current/title-42/chapter-IV/subchapter-D/part-457/subpart-A/section-457.10) separately limits CHIP's child definition to under 19. An adult child for MAGI household composition remains an adult for age-based eligibility. + +**Alternative and materiality.** The [2026-only Reform](sweep/fixes/r067_adult_dependent_relationship.py) changes only `medicaid_claimed_by_parent_in_tax_unit`: adult tax dependents are interpreted as children of their claiming filer. It preserves the original formula before 2026 and restores it in 2027. This is a sensitivity assumption for missing relationships, not a proposed universal rule that all adult dependents are children. It does not alter `is_parent`, ages, income definitions, or disability-status inputs. + +| Scenario 067 quantity | Frozen encoding | Adult-child reading | +| --- | ---: | ---: | +| Dependent 1 Medicaid household income | $0 | $87,302 | +| Dependent 1 Medicaid household size | 1 | 3 | +| Dependent 1 income / FPL | 0 | 3.1955345 | +| `head_medicaid_eligible` | 0 | 0 | +| `spouse_medicaid_eligible` | 0 | 0 | +| `dependent1_medicaid_eligible` | 1 | 0 | +| `head_chip_eligible` | 0 | 0 | +| `spouse_chip_eligible` | 0 | 0 | +| `dependent1_chip_eligible` | 0 | 0 | + +The binary flip is material. SSI and the optional senior/disabled and SSI-recipient Medicaid pathways remain as encoded; the baseline calculates no SSI and neither category succeeds. The alternative is conditional on the specified relationship reading while retaining the separate disability-status convention, not a certification of real-world eligibility through every possible pathway. + +**Full sweep.** [The CSV](sweep/out/r067_adult_dependent_relationship.csv) contains all **1,984** frozen outputs across **100** households. Exactly **two** move under the adult-child interpretation; the harness reports **zero** small nonzero deltas above its 0.000001 diagnostic floor. Scenario 112's federal refundable credits and payroll tax each differ by only −5.68×10⁻¹⁴ because of decimal/float representation; they are not relationship effects or moved outputs. No output is omitted because models answered it correctly. + +| Scenario | State | Output | Frozen | Recomputed | Confirmed for this root cause? | +| --- | --- | --- | ---: | ---: | --- | +| scenario_064 | WI | `dependent1_medicaid_eligible` | 1 | 0 | Yes: the same predicate changes MAGI from $0 to $124,366.484375, household size from 3 to 5, and income/FPL from 0 to 3.215266; `ADULT` becomes `NONE`. | +| scenario_067 | IN | `dependent1_medicaid_eligible` | 1 | 0 | Yes: parental-claim predicate brings $87,302 into the size-3 household and removes adult income eligibility. | + +The complete manifest scan finds four adult dependents in three households: + +| Scenario/person | Age | Result of adult-child interpretation | +| --- | ---: | --- | +| 064 / dependent1 | 27 | Confirmed flip above. The prompt states disability and insurance, but no parent relationship. | +| 064 / dependent2 | 18 | Already satisfies the qualifying-child age test; the child-reading module changes nothing for this person. | +| 067 / dependent1 | 23 | Confirmed flag above. | +| 093 / dependent1 | 27 | MAGI changes from $45,000 / size 1 (2.819549 FPL) to $157,408.25 / size 3 (5.761649 FPL). Medicaid remains 0; no requested output changes. | + +The 97 other households contain no adult dependents and the sweep reports no movement in them. All figures above are direct outputs in [diagnostics.json](sweep/work/r067_adult_dependent_relationship/diagnostics.json). The reform changes no Head/Spouse parent flag, minor's relationship predicate, age, disability input, or source of income. Scenario 064's baseline size 3 comes from the engine including the two under-19 family members in its nonfiler calculation; I have not adjudicated that separate family-membership approximation. Its zero-income result and high-income alternative do not depend on treating that size as established fact. + +For scenario 064, [Wisconsin's BadgerCare Plus handbook §16.1](https://www.emhandbooks.wisconsin.gov/bcplus/policyfiles/3/16/16.1.htm) states a 100% FPL adult limit, including disregards; the page identifies its last update as December 18, 2024, effective January 1, 2025. The engine parameter also says 1.00. The audit prompt's generated explanation calling Wisconsin a 138% ACA expansion state is therefore inaccurate; I used the source parameter and run rather than that narrative. Its alternative MAGI is far above the actual adult limit. + +The 18-year-old's opposite relationship reading exposes another affected output. The prompt calls this person **Dependent 2**, separately from **Child 1**, and does not identify a parent. The [nonchild-reading module](sweep/fixes/r067_adult_dependent_nonchild.py) sets the same Medicaid relationship predicate false for adult dependents during 2026 only. It leaves the already-nonchild readings of the other three adult dependents intact. Its [separate full CSV](sweep/out/r067_adult_dependent_nonchild.csv) covers **1,984 outputs / 100 households**, with exactly **one** moved output and no other delta above 0.000001. [The log](sweep/work/r067_adult_dependent_relationship/nonchild_sweep.txt) records the summary. The candidate diagnostics confirm: + +| Scenario | State | Output | Frozen | Recomputed under nonchild reading | Confirmed for this root cause? | +| --- | --- | --- | ---: | ---: | --- | +| scenario_064 | WI | `dependent2_medicaid_eligible` | 0 | 1 | Yes: the relationship exception removes $124,366.484375 of claimant-household MAGI; own/family-child MAGI is $0 and the category becomes `OLDER_CHILD`. | + +The encoded household size changes from 5 to 2; this retains the engine's broad family-child approximation. Whether the 12-year-old belongs in that nonfiler household is unverified, but both younger members have zero MAGI, so size 1 versus 2 would not change the zero-income eligibility result. An 18-year-old nonchild dependent with no resident parent of their own is one consistent alternative. [Wisconsin handbook §7.1](https://www.emhandbooks.wisconsin.gov/bcplus/policyfiles/2/07/7.1.htm), last updated April 9, 2025, says children ages 6–18 at or below 156% FPL are exempt from the current-insurance restriction. Thus the stated employer coverage does not defeat this zero-income reading. The read engine files `variables/gov/hhs/medicaid/eligibility/categories/older_child/is_older_child_for_medicaid.py` and corresponding `parameters/gov/hhs/medicaid/eligibility/categories/older_child/{age_range,income_limit}.yaml` apply age 6 through 18 and Wisconsin's 1.56 threshold. All five scenario 064 CHIP outputs remain zero in this run. + +These are **different conditional readings**, not three outputs from one corrected household file. They expose three distinct Medicaid references that depend on the missing adult relationship. The primary answer for scenario 067 remains 1 under the encoded nonchild reading and 0 under the child reading. No further family-tree reconstruction is claimed. + +**Upstream status.** A fresh run under the supplied **2.8.0** environment still returns 1, household income $0, and size 1 for Dependent 1; see [latest_release.txt](sweep/work/r067_adult_dependent_relationship/latest_release.txt). The relationship, exception, nonfiler, and household-income formulas are byte-identical in 1.755.4, 2.8.0, and the examined local `upstream/main`, commit `2c2e42c08f9c3a163437166c7fed398024ffb892`. Main has a separate California pregnancy-size change that does not resolve this relationship ambiguity. No main execution is claimed. + +Related merged [PR #8169](https://github.com/PolicyEngine/policyengine-us/pull/8169) introduced these MAGI household formulas; this is verified from git commit `b31679c5d561f9f4a0d324049afd346c5b815047` dated April 30, 2026. A local remote-tracking branch, `upstream/medicaid-magi-parent-ids`, contains September 7 commit `875a58982e67c52d1987a6db0c195473cade840c`, adding explicit parent IDs. I read that branch's formula: it retains the current inference when IDs are absent. The branch is not in examined main; execution, live status, and a PR number are unverified. GitHub CLI issue/PR searches failed with API connectivity errors, so no claim is made that no case-specific issue exists. Exact comparisons and limitations are in [upstream.txt](sweep/work/r067_adult_dependent_relationship/upstream.txt). + +**Reproduction and validation.** Run from the triage sweep directory; all outputs remain in the assigned workspace: + +```sh +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ + ../.venv-pe1755/bin/python sweep.py \ + --fix /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-067/sweep/fixes/r067_adult_dependent_relationship.py \ + --out /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-067/sweep/out/r067_adult_dependent_relationship.csv +``` + +[verify.py](sweep/work/r067_adult_dependent_relationship/verify.py) independently scans candidates, records baseline and alternative intermediates, checks all six scenario 067 Medicaid/CHIP outputs, compares the older parent-count perturbation, and checks 2025/2027 boundaries. **All assertions passed**; [verify.txt](sweep/work/r067_adult_dependent_relationship/verify.txt) retains the output. [sweep.txt](sweep/work/r067_adult_dependent_relationship/sweep.txt) records the complete sweep summary. The published sweep CSV has 1,984 unique scenario/output rows. + +[nonchild_verify.py](sweep/work/r067_adult_dependent_relationship/nonchild_verify.py) repeats the candidate calculations for the other relationship reading and checks its 2025/2027 boundaries; all assertions passed. See [nonchild_verify.txt](sweep/work/r067_adult_dependent_relationship/nonchild_verify.txt) and [nonchild_diagnostics.json](sweep/work/r067_adult_dependent_relationship/nonchild_diagnostics.json). Reproduce the second full sweep with the command above substituting `r067_adult_dependent_nonchild.py` and `r067_adult_dependent_nonchild.csv`. + +All changes are confined to this workspace. No external issue, PR, or message was posted, nothing was pushed, and no upstream file was edited. A requested commit on `flag-triage` was attempted, but the session's read-only `.git` protection rejected creation of `.git/index.lock` with `Operation not permitted`; no commit was created. diff --git a/reference_audit/2026-09-22/verification/v6_flag_076.md b/reference_audit/2026-09-22/verification/v6_flag_076.md new file mode 100644 index 00000000..9934e031 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v6_flag_076.md @@ -0,0 +1,57 @@ +**Verdict: `reference_correct` for the Idaho child-credit flag.** Idaho Code § 63-3029L's $205 nonrefundable child credit expired for taxable years beginning January 1, 2026. PolicyEngine 1.755.4 deliberately implements that sunset. Do not subtract $410 or exclude this scored output for the judge's hypothesis. The applicable benchmark value remains **$6,818.34** after the already-settled Idaho publication convention; the pristine frozen value is **$6,806.79**. The category here adjudicates the child-credit allegation, not the superseded frozen threshold projection. + +This is not a new root cause in `triage/root_causes.json`. `c_id_hold_2025` already explains the frozen-to-v1.2 difference. The separate settled `r07_idaho_health_premiums` affects households 007 and 053; their tax values below are isolated baseline/convention checks, not revisions to that finding. Neither settled rule was re-investigated. + +**Law available before the July 3, 2026 freeze.** The following primary documents settle the credit's expiration and the relevant 2025 legislation. Supplemental bill-history checks and access limitations are recorded in [law_notes.md](law_notes.md). + +| Primary source | What it establishes | +| --- | --- | +| [Idaho Legislature, 2020 HB 574, §3, p.4, printed lines 7–17](https://legislature.idaho.gov/wp-content/uploads/sessioninfo/2020/legislation/H0574.pdf#page=4) | Reproduces §63-3029L(1): $205 for each IRC §24(c) qualifying child, nonrefundable, available to Idaho residents, and restricted to taxable years beginning in 2018 through 2025. The January 1, 2026 cutoff predates HB 40. | +| [Idaho Tax Commission decision 200-814-790-656, p.2](https://tax.idaho.gov/wp-content/uploads/decisions/200-814-790-656.pdf#page=2) | Independently reproduces the statutory amount and expiration date. | +| [July 1, 2026 Idaho Administrative Bulletin, pp.87–88, Docket 35-0101-2601](https://files.dfm.idaho.gov/dfm-admin-website/bulletin/2026/07.pdf#page=88) | The Commission's notice, dated May 29, confirms that §63-3029L ended the credit after December 31, 2025. This contemporaneous confirmation was published two days before the freeze. It proposes administrative cleanup following the existing statutory sunset; the notice itself did not cause expiration. | +| [2025 HB 40, all eight pages, legislative document read on LegiScan's mirror](https://legiscan.com/ID/text/H0040/id/3075079/Idaho-2025-H0040-Introduced.pdf) | §§1–4 amend §§63-3022, 63-3022A, 63-3024 and 63-3025, addressing metals, retirement income and tax rates. §5 applies retroactively to January 1, 2025. It does not amend, repeal or replace §63-3029L. The [official PDF URL](https://legislature.idaho.gov/wp-content/uploads/sessioninfo/2025/legislation/H0040.pdf) was inaccessible. | +| [Governor's March 6, 2025 signing announcement](https://gov.idaho.gov/pressrelease/idaho-delivers-largest-income-tax-cut-in-state-history-sending-another-253-million-back-to-idahoans/) | Confirms HB 40's enactment and the reduction from 5.695% to 5.3%. | + +The checked 2026 proposals, [S1450](https://legiscan.com/ID/bill/S1450/2026) and [H0782](https://legiscan.com/ID/bill/H0782/2026), sought to extend the child credit indefinitely. Retrieved secondary legislative histories show neither passed: S1450 stopped after referral to Local Government & Taxation on April 1; H0782 stopped after printing and filing in the Chief Clerk's office on February 27. The [Governor's final April 10 bill-action sheet](https://gov.idaho.gov/wp-content/uploads/2026/04/daily-bill-action_041026_10-AM-MT.pdf) contains neither bill. Official legislative history pages were blocked, so those precise procedural histories were **not independently verified from official journals**. The pre-freeze Commission bulletin independently establishes the operative sunset after the legislative session. The checked 2025 S1057 proposal also retained the cutoff. No comprehensive claim about reading every 2025–2026 session law is made. + +The separate parental-choice education credit does not automatically replace the expired $205 credit. The [Tax Commission's March 4, 2026 announcement](https://tax.idaho.gov/pressrelease/parental-choice-tax-credit-deadline-approaches/) describes a refundable program requiring an application for eligible nonpublic-school expenses. Scenario 076 lists no such expenses or school status. This also concerns a refundable program, while the flagged output is before refundable credits. + +**Mechanism read in 1.755.4.** Paths below are relative to `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`. + +| Engine path | Observed behavior | +| --- | --- | +| `parameters/gov/states/id/tax/income/credits/non_refundable.yaml` | `2018-01-01: [id_ctc]`; `2026-01-01: []`. This explicit dated list implements expiration. | +| `parameters/gov/states/id/tax/income/credits/ctc/amount.yaml` | Retains the raw $205 amount from 2018. | +| `variables/gov/states/id/tax/income/credits/id_ctc.py` | Multiplies `ctc_qualifying_children` by that amount. Scenario 076 computes two children and raw `id_ctc = 410`. This standalone variable is not the credit actually applied to 2026 liability. | +| `variables/gov/states/id/tax/income/id_non_refundable_credits.py` and `variables/gov/states/tax/income/non_refundable_credit_cap.py` | Read the dated list and apply listed credits subject to remaining liability. An empty list produces zero. | +| `variables/gov/states/id/tax/income/id_income_tax_before_refundable_credits.py` | Subtracts applied nonrefundable credits from `id_income_tax_before_non_refundable_credits`, floored at zero. | +| `variables/gov/states/id/tax/income/id_income_tax_before_non_refundable_credits.py` and `parameters/gov/states/id/tax/income/main/head_of_household.yaml` | Apply the filing-status schedule to taxable income; the applicable marginal rate is 5.3%. The threshold parameter is uprated in the frozen engine. | +| `variables/gov/states/id/tax/income/id_taxable_income.py`, `id_agi.py`, and `deductions/id_deductions.py` | Taxable income is Idaho AGI minus deductions and the separate QBI/Schedule 1-A deduction. The run returns $163,220 − $24,150 − $800 = $138,270. | + +The [saved trace and three-household calculations](sweep/work/id_child_credit/idaho_baseline_trace.txt) verify the actual path. The question explicitly states Idaho, tax year 2026, and children ages 13 and 8 living in the tax/benefit household. The engine recognizes both children. This flag does not depend on an unlisted household input or an alternative reading of child eligibility; the decisive input is the stated tax year. + +**Values and reach.** [check_idaho.py](sweep/work/id_child_credit/check_idaho.py) scans the entire scenario table and asserts that the only Idaho households are 007, 053 and 076. It runs all three with the reference engine, then separately with the existing `r19_id_convention.py`; it introduces no child-credit reform. All amounts below are sandbox results rounded to cents. + +| Household | Listed ages | Qualifying children / raw `id_ctc` | Applied 2026 nonrefundable credits | Frozen = recomputed baseline | Existing convention only | +| --- | --- | --- | --- | --- | --- | +| scenario_007 | 56 | 0 / $0 | $0 | $755.78 | $761.56 | +| scenario_053 | 25 | 0 / $0 | $0 | $2,435.28 | $2,441.06 | +| scenario_076 | 45, 13, 8 | 2 / $410 | $0 | $6,806.79 | $6,818.34 | + +For 076 the exact frozen result is `6806.78662109375`. Applying only the settled convention yields `6818.34423828125`, consistent with ($138,270 − $9,622) × 0.053 = $6,818.344 before engine floating-point representation. Hypothetically subtracting the expired credit yields `6408.34423828125` ($6,408.34); this was computed in the diagnostic script and is **not a lawful corrected value**. It would reduce the convention result by $410, well beyond the $1 tolerance. The valid child-credit correction is $0. + +**Sweep and moved outputs.** No fix module is warranted: `sweep/fixes/` has no new module. The unchanged harness was run over all 100 households and 1,984 frozen outputs. The [CSV](sweep/out/id_child_credit_baseline.csv) and [log](sweep/work/id_child_credit/baseline_sweep.log) report zero moved outputs and zero nonzero deltas above `1e-6`. Two unrelated CSV float-parsing residuals are approximately `5.68e-14`; every other delta is exactly zero. No output moves for this root cause, including federal downstream outputs. + +| Moved output for this flag | Frozen value | Recomputed value | Confirmed | +| --- | --- | --- | --- | +| None (all 1,984 checked) | Each original reference | Matches within `5.69e-14` | Yes | + +The three values in the convention column are reconciliation checks for an already-settled rule, not new child-credit movements. Households 007 and 053 lack children; 076's credit is expired. Other states cannot receive this Idaho credit (`defined_for = StateCode.ID`). + +**Upstream.** Installed distribution metadata confirms versions 1.755.4 and 2.8.0. The relevant credit amount, dated list and tax-wiring files are byte-identical in both and in inspected `upstream/main` at `2c2e42c08f9c3a163437166c7fed398024ffb892` (September 22, 2026). There is no baseline correction to make. This latest-version conclusion is from source comparison, not a 2.8.0 simulation. See [upstream_notes.md](upstream_notes.md) for exact paths and commits. + +Related [PR #7911](https://github.com/PolicyEngine/policyengine-us/pull/7911) implements proposed S1450 as an optional reform whose activation defaults to false. [PR #8856](https://github.com/PolicyEngine/policyengine-us/pull/8856) adds another optional Idaho credit revival while preserving the baseline. [Closed issue #8899](https://github.com/PolicyEngine/policyengine-us/issues/8899) concerns activation leakage from contributed reforms, not this baseline sunset. Commit history establishes the PRs; GitHub API searches failed, so the existence of any other specifically matching issue or PR remains unverified. + +Reproduction commands and validation details are in [commands.md](sweep/work/id_child_credit/commands.md). Only this workspace was written. Nothing was filed, posted or pushed. + +**Commit limitation:** the assigned branch is `flag-triage`, initially clean at `c53f54e`. The sandbox marks this workspace's `.git` directory read-only. `git add` failed with `Unable to create .../.git/index.lock: Operation not permitted`; no files could be staged or committed. All report and evidence files remain in the assigned workspace for review and commit when Git metadata is writable. No history was rewritten. diff --git a/reference_audit/2026-09-22/verification/v6_flag_081.md b/reference_audit/2026-09-22/verification/v6_flag_081.md new file mode 100644 index 00000000..ad430561 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v6_flag_081.md @@ -0,0 +1,76 @@ +# scenario_081 / state_income_tax_before_refundable_credits + +**Verdict: `reference_wrong_engine_bug`.** The stated $1,080 short-term capital loss eliminates the household's Part A dividends, but PolicyEngine 1.755.4 taxes the gross dividends. Correcting that defect alone reduces the reference from **$8,238.41 to $8,232.90**, more than the $1 tolerance. There is also an **unlisted interest-source input** (`prompt_ambiguity`): treating the $56 of interest as ordinary Part A interest gives **$8,230.10** after the same loss correction. The judge's proposed $100 Massachusetts-bank-interest exemption is wrong for 2026; it was repealed from 2024. + +This root cause is not among the settled entries in `triage/root_causes.json`. No settled fix or publication convention was reapplied. Under the supplied scoring rule, the confirmed engine defect is sufficient to exclude this output for every model, irrespective of which interest-source reading is chosen. + +The prompt states non-qualified dividends of $110, short-term capital gains of −$1,080, taxable interest of $56, wages of $175,002, and Massachusetts residence. It does **not** say that the interest comes from a Massachusetts bank or a qualifying deposit. Bank-account assets and residence do not establish that fact. The prompt's instruction that unlisted facts are false supports the ordinary Part A reading; the engine instead places all taxable interest in residual Part B. The frozen input contains dividends of $110.11764526367188; the prompt rounds this to $110. This rounding does not explain the material error because all dividends are absorbed under either corrected reading. + +**Law applicable to 2026 and published before the freeze.** [M.G.L. c. 62 § 2(b)(1)–(2)](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2) places ordinary interest and dividends in Part A, except enumerated sources including qualifying Massachusetts-bank deposits, which fall in Part B. Sections 2(c)(2)(a) and 2(c)(4) apply excess short-term capital losses against Part A interest/dividends, with a combined short-/long-term loss limit of $2,000; remaining short-term losses can reach Part C. Section 2(f) bases taxable Part A income on adjusted Part A income. [DOR TIR 02-21](https://www.mass.gov/technical-information-release/tir-02-21-capital-gains-and-losses-massachusetts-tax-law-changes), dated January 2, 2003, describes this ordering and cap, establishing the rule before July 3, 2026. The statute was read directly; the TIR's relevant text and date were read in official search-indexed text because direct retrieval was blocked. + +[Acts of 2024, chapter 140, §§ 101 and 251](https://malegislature.gov/Laws/SessionLaws/Acts/2024/Chapter140), approved July 29, 2024, removes the former c. 62 § 3.B(a)(6) bank-interest deduction for taxable years beginning January 1, 2024. Thus even qualifying Massachusetts-bank interest remains taxable in Part B in 2026. No bank exemption is included in either calculation here. The $8,230.10 alternative results from the **capital-loss offset against Part A interest**, not from the repealed exemption. + +[M.G.L. c. 62 § 3.B](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section3) supplies the $2,000 FICA deduction cap, 50%-of-rent deduction capped at $4,000, and $4,400 statutory maximum single personal exemption. The engine applies $4,400 here. As additional 2026 support, [DOR's 2026 Form 1-ES](https://www.mass.gov/doc/2026-form-1-es-estimated-tax-payment-vouchers-instructions-and-worksheets/download), official search-indexed text, identifies wages and interest/dividends as 5% income; [§ 4](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section4) links the Part A interest/dividend rate to the Part B rate. The exact Form 1-ES publication date was not independently verified; the loss-offset rule and repeal have the separately dated pre-freeze evidence above. No projected parameter is changed. Additional source-access details are in [law_notes.md](law_notes.md). + +**Read engine mechanism.** Paths below are relative to `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/`: + +| File | Read behavior and observed implication | +| --- | --- | +| `variables/gov/states/ma/tax/income/gross_income/ma_part_a_gross_income.py` | Adds dividends and nonnegative short-term gains; omits interest. This case returns $110.117645. | +| `variables/gov/states/ma/tax/income/adjusted_gross_income/ma_part_a_agi.py` | Calculates capital-loss offsets against interest/dividends using a $2,000 cap, then floors Part A AGI at zero. This case already returns zero. | +| `variables/gov/states/ma/tax/income/taxable_income/ma_part_a_taxable_dividend_income.py` | Uses gross `dividend_income` less unused Part B exemption, bypassing Part A AGI. This case still taxes $110.117645 of dividends. | +| `variables/gov/states/ma/tax/income/gross_income/ma_part_b_gross_income.py` | Subtracts Parts A and C from MA gross income. Omitted Part A interest remains in Part B; this case has $175,058. | +| `variables/gov/states/ma/tax/income/taxable_income/ma_part_b_taxable_income_deductions.py` | Applies the FICA/rent deductions. Its bank-interest branch is disabled because `parameters/gov/states/ma/tax/income/exemptions/interest/in_effect.yaml` is false from 2024. | +| `variables/gov/states/ma/tax/income/ma_income_tax_before_credits.py` | Taxes the dividend base at 5%, alongside Part B and other bases. `ma_income_tax_before_refundable_credits.py` then subtracts nonrefundable credits. | + +The baseline `pe_case.py` run matched the frozen value exactly: **8238.40625**. Its optional diagnostic list subsequently requested a nonexistent `ma_personal_exemption` variable and exited with an error; this did not invalidate the already printed baseline. The dedicated check script uses valid variables and records complete results separately. + +**Correction and alternative calculation.** Keeping interest in Part B isolates the certain error: + +`Part A dividends after loss = max(0, 110.117645 − min(1,080, 2,000, 110.117645)) = 0`. + +`Part B tax = (175,002 + 56 − 2,000 − 4,000 − 4,400) × 5% = $8,232.90`. + +Under the ordinary Part A interest reading, the loss covers $166.117645 of dividends plus interest. Part A tax is again zero, while Part B becomes: + +`(175,002 − 2,000 − 4,000 − 4,400) × 5% = $8,230.10`. + +The extra reduction from interest classification is $2.80. Thus $8,232.90 is the correction holding the engine's source assumption constant, and $8,230.10 is the correction under the prompt's no-unlisted-exception reading. Neither value deducts remaining capital loss against wages. Both exceed the $1 materiality threshold relative to the frozen reference. + +**Sandbox artifacts.** [sweep/fixes/ma_part_a_loss_offset.py](sweep/fixes/ma_part_a_loss_offset.py) connects the existing loss-adjusted Part A AGI to the dividend tax base and updates the accompanying dividend/gain exemption allocation. It retains interest in Part B. [sweep/fixes/ma_part_a_ordinary_interest.py](sweep/fixes/ma_part_a_ordinary_interest.py) adds the ordinary Part A interest interpretation. Both use `Reform`, inherit Massachusetts applicability, and return the original formulas outside 2026. These are independent narrow modules, not adoption of the earlier investigator's unverified broader reform. + +The modules preserve existing AGI/netting logic. They are verified for the reached household and simple loss/rate controls, not a complete Massachusetts implementation: mixed positive long-term gains and short-term losses, collectibles, carryovers, and mixed exempt/bank-interest sources require broader handling. None is present in the only Massachusetts household. The ordinary-interest variant explicitly assumes all its taxable interest is ordinary Part A interest. + +**Full sweep and every moved output.** [sweep/out/baseline.csv](sweep/out/baseline.csv) reproduces all **1,984 original frozen CSV values exactly** when both are read with Python's `float`. The two reform sweeps each move **one output**; the other **1,983 recomputed values equal the no-fix run exactly**. Both moved rows are shown below; they are the same output under two treatments, not two different exclusions. + +| Reform / interpretation | Scenario | Output | Frozen | Recomputed | Delta | Confirmed? | +| --- | --- | --- | ---: | ---: | ---: | --- | +| Dividend defect only; retain Part B interest | scenario_081 | state_income_tax_before_refundable_credits | 8238.40625 | 8232.900390625 | −5.505859375 | Yes: Part A dividend base falls from 110.117645 to 0; Part B taxable income remains 164658. | +| Same defect + ordinary Part A interest reading | scenario_081 | state_income_tax_before_refundable_credits | 8238.40625 | 8230.1005859375 | −8.3056640625 | Yes, conditional on that source reading: Part A interest/dividends are absorbed; Part B taxable income falls to 164602. | + +Complete results: [dividend-only CSV](sweep/out/ma_part_a_loss_offset.csv), [ordinary-interest CSV](sweep/out/ma_part_a_ordinary_interest.csv), and [sweep summary](sweep/out/sweep_summary.json). The sub-cent differences from the arithmetic amounts are the engine's floating-point representation. The harness's pandas parsing introduces −5.684341886080802e−14 deltas for `scenario_112` federal refundable credits and payroll tax in **all three** CSVs, including baseline. These are unchanged parsing artifacts, not moved outputs or reform effects; comparison directly with the original frozen CSV confirms this. + +The scan of all 100 scenario records found **only scenario_081 in Massachusetts**. It has no long-term gains/losses, U.S.-government interest, or collectible gains. Every other state's outputs and every other output for scenario_081 remain unchanged, including federal tax and refundable credits. The reform therefore has no unconfirmed spillover in this bundle and no additional Massachusetts household left unexamined. This is a scan of this frozen population, not a claim of general coverage for all Massachusetts households. + +[sweep/check_ma_part_a.py](sweep/check_ma_part_a.py) passed baseline reproduction, both corrected values, five controls (the $2,000 cap, partial short-term loss, no loss, positive short-term gain, and a long-term loss remaining after short-term gains), and unchanged nonzero 2025/2027 results. [sweep/out/checks.json](sweep/out/checks.json) contains the intermediate variables and assertions' results; [checks.log](sweep/out/checks.log) records successful completion. Run it from this workspace with: + +```sh +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python sweep/check_ma_part_a.py +``` + +Reproduction commands, run from the triage sweep directory, with all outputs in this workspace: + +```sh +cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 ../.venv-pe1755/bin/python sweep.py --out /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-081/sweep/out/baseline.csv +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 ../.venv-pe1755/bin/python sweep.py --fix /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-081/sweep/fixes/ma_part_a_loss_offset.py --out /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-081/sweep/out/ma_part_a_loss_offset.csv +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 ../.venv-pe1755/bin/python sweep.py --fix /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-081/sweep/fixes/ma_part_a_ordinary_interest.py --out /Users/maxghenis/PolicyEngine/_wk/pb-flag-v6-081/sweep/out/ma_part_a_ordinary_interest.csv +``` + +All runs use the pristine `results/local/newmodels/publish/us_full_run_20260612_policyengine_4_16_1_populace/us` bundle. The reference CSV SHA-256 is `b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368`. + +**Upstream status.** Release **2.8.0 still returns 8238.40625**, confirmed by a successful run in [latest_081_run.txt](latest_081_run.txt). The four central formulas are byte-identical in 1.755.4, 2.8.0, and read-only `upstream/main` at `2c2e42c08f9c3a163437166c7fed398024ffb892`. Neither inspected release nor main fixes this defect. [upstream_notes.md](upstream_notes.md) records paths, hashes, and commands. + +Related local history identifies [PR #5564](https://github.com/PolicyEngine/policyengine-us/pull/5564) / [issue #5562](https://github.com/PolicyEngine/policyengine-us/issues/5562) for the 2024 Massachusetts parameter update, including repeal of the bank exemption, and [issue #6968](https://github.com/PolicyEngine/policyengine-us/issues/6968) for a separate short-term-loss-to-long-term-gain correction. These references were read in local commit history, not current GitHub discussions. **A current issue/PR for this exact defect is unverified**: the GitHub API search failed and web retrieval did not supply matching coverage. No claim that no issue exists is made. + +Only files in the assigned workspace were written; no issue, PR, comment, or push was made. **Commits are blocked by the environment:** `.git` is read-only and `git add`/`git commit` failed creating `.git/index.lock` with `Operation not permitted`. The assigned branch remains `flag-triage`; history was not rewritten. All deliverables remain available as workspace files. diff --git a/reference_audit/2026-09-22/verification/v7_review_claims.md b/reference_audit/2026-09-22/verification/v7_review_claims.md new file mode 100644 index 00000000..04109e37 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v7_review_claims.md @@ -0,0 +1,96 @@ +SHIP AFTER FIXES + +1. **Blocker — CONFIRMED: SNAP arithmetic defects remain scored under a publication convention.** + [paper/index.qmd:1069](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/index.qmd:1069) treats statutory rounding corrections as grounds to regenerate references. But [r13_hold_fy2026_v2.py:77](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v2.py:77) fixes an existing arithmetic defect, independently of projected parameters. + + **Evidence:** In policyengine-us 1.755.4, ran a `Reform` updating only `snap_expected_contribution` and `snap_net_income`, leaving parameters unchanged: + + | SNAP scenario | Original reference | Rounding-only result | Change | Currently excluded | + |---|---:|---:|---:|---| + | 008 | $15,246.9053 | $15,243.3076 | −$3.5977 | No | + | 012 | $4,952.0894 | $4,942.7896 | −$9.2998 | No | + | 054 | $6,125.6890 | $6,118.7900 | −$6.8989 | No | + | 109 | $8,020.5542 | $8,017.5542 | −$3.0000 | No | + + **Smallest fix:** Separate rounding defects from the parameter convention; sweep the isolated defect across all references; exclude every >$1 mover, restore its original frozen value, and rescore/refreeze the publications. + +2. **Major — CONFIRMED: the CalEITC fix cites the wrong upstream PR.** + [paper/index.qmd:1067](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/index.qmd:1067) and [root_causes.json:148](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/root_causes.json:148) attribute the missing AGI comparison to #9542. + + **Evidence:** Retrieved and read the upstream PR diffs through `github_fetch_pr`. [#9542](https://github.com/PolicyEngine/policyengine-us/pull/9542) changes the **2024 eligibility boundary from $31,950 to $31,951**, plus references and tests. [#9363](https://github.com/PolicyEngine/policyengine-us/pull/9363), merged **September 1**, implements the earned-income/AGI credit comparison. + + The same paper sentence says the remaining defects “are reported,” while committed exclusion entries say “to be filed,” for example [reference_exclusions.json:185](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json:185). That claim lacks recorded issue links. + + **Smallest fix:** Replace #9542 and its date with #9363/September 1 throughout the records; describe remaining reports as pending unless actual links are supplied. + +3. **Minor — CONFIRMED: “most … never flagged” reverses the recorded proportions.** + [paper/index.qmd:1067](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/index.qmd:1067) and [the audit note:7](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/app/src/notes/2026-09-22-reference-audit.json:7) make this claim. + + **Evidence:** Joined engine-defect exclusions to adjudications on `(scenario_id, variable)`, filtering `judge_reference_suspect`: + + ```text + engine-defect outputs: 31 + flagged: 22 + never flagged: 9 + roots touching flagged outputs: 11 of 12 + ``` + + **Smallest fix:** Say “the sweep identified nine additional outputs that had never been flagged.” + +4. **Minor — CONFIRMED: sensitivity prose retains stale values and a wrong scored-failure count.** + Recomputed with `canonical_filtered_scores` using committed predictions and frozen references: + + - [paper/index.qmd:1020](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/index.qmd:1020): Fable 5.1’s uplift is `91.2878185 − 90.4271661 = 0.8606524`, hence **0.9 points**, not 1.2. + - [sensitivity document:51](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/sensitivity/claude-thinking-2026-08.md:51): Sonnet’s all-output score is **80.254155**, rounding to **80.3**, not 80.2. + - [sensitivity document:60](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/sensitivity/claude-thinking-2026-08.md:60): only **55 of 56** parse failures contribute scored misses. `scenario_008/payroll_tax` is excluded. + - [sensitivity document:141](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/sensitivity/claude-thinking-2026-08.md:141): Fable 5.1’s board advantage over Fable 5 is **6.759856**, rounding to **6.8**, not 6.7. + + **Smallest fix:** Correct these values and the failure-count wording; derive the paper’s uplift from the sensitivity data, then rerender. + +5. **Minor — CONFIRMED, pre-existing: cost provenance describes the opposite priority from the code.** + [Methodology.tsx:247](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/app/src/components/Methodology.tsx:247) says provider-reported costs take priority. [eval_no_tools.py:550](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/policybench/eval_no_tools.py:550) selects reconstructed costs whenever available. + + **Evidence:** Summed the frozen prediction cost columns for DeepSeek V4 Pro 0813: + + ```text + provider_reported_cost_usd: $12.378115 + reconstructed_cost_usd: $19.865589 + total_cost_usd: $19.865589 + ``` + + **Smallest fix:** Describe the actual priority consistently in Methodology and the paper’s cost caption. This paragraph predates the branch but lies within the assigned review scope. + +6. **Minor — CONFIRMED: two README reproducibility assertions need narrower wording.** + [README.md:13](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/README.md:13) says every module is limited to 2026. However, [the IRS convention:42](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/fixes/r19_irs_sales_tax_convention.py:42) explicitly updates 2025, as does the California convention. + + [README.md:14](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/README.md:14) says the unmodified harness reproduces every frozen reference. Comparing the saved baseline sweep with HEAD instead returned: + + ```text + baseline/current outputs: 1984 / 1984 + changed: 23 + changes over $1: 18 + ``` + + **Smallest fix:** Say the modules were evaluated on the 2026 bundle, and that the baseline reproduces the **pre-audit July 3 references**. + +7. **Minor — CONFIRMED: the manifest retains an obsolete response-window endpoint.** + [manifest.json:180](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/paper/snapshot/20260501/manifest.json:180) says collection ended September 1; its `model_response_date` correctly ends September 22. Local Sol/Luna run-state timestamps confirm September 22 collection. + + **Smallest fix:** Replace the hard-coded endpoint in [freeze_snapshot.py:1383](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/scripts/freeze_snapshot.py:1383) with the recorded response window and regenerate the manifest. + +8. **Nit — CONFIRMED: changed audit prose uses avoidable passive voice.** + [The audit note:6](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/app/src/notes/2026-09-22-reference-audit.json:6) says “Each flag was settled.” + + **Smallest fix:** “Developers settled each flag.” + +Checks that found no discrepancies: + +- Recomputed the published top-model scores and unrounded ranks; both new notes’ numeric fact dictionaries match the frozen data. +- Verified the recorded counts: **42 models; 55 exclusions = 31 engine defects + 24 unlisted inputs; 12 defect roots; 1,929 scored outputs; 23 regenerated references = 14 SNAP + nine tax; nine conventions, eight nonempty; 61 adjudications**. +- Verified annotation counts: **7,493 scored annotations, 7,489 exact misses, four exact hits, and 1,842 additional unannotated bounded-score misses**. +- Recomputed all **14 paper tables** in memory; their rendered cells match. All **18 rendered-artifact hashes** match, and regenerating the scatter figure in memory produced the identical PNG hash. +- Verified sensitivity headline scores, ranks, per-program tables, costs and latency; the discrepancies above are in prose. +- Read the adjudication flag-clearing/freezer mechanisms and upstream #8839/#9301 diffs; their described behavior is supported. +- **141 Bun tests passed.** Targeted Python runs produced **88 passes**; nine temporary-file-dependent tests were blocked by the read-only sandbox. The full 841-test suite was not reproduced. +- Snapshot date, reference-freeze date and dataset version **1.1** agree. Historical Models API creation dates and live release availability remain independently unverified. +- Working tree remained clean; no files were edited. \ No newline at end of file diff --git a/reference_audit/2026-09-22/verification/v7_review_data.md b/reference_audit/2026-09-22/verification/v7_review_data.md new file mode 100644 index 00000000..cf9ead42 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v7_review_data.md @@ -0,0 +1,57 @@ +SHIP AFTER FIXES + +1. **Major — CONFIRMED: the SNAP convention regenerates outputs affected by an engine defect that the publication rule requires excluding.** + + **Location:** [r13_hold_fy2026_v2.py:77](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/fixes/r13_hold_fy2026_v2.py:77), classified entirely as a convention in [root_causes.json:124](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/reference_audit/2026-09-22/root_causes.json:124). + + The module corrects contribution rounding as well as holding published parameters. The pinned engine computes `floor(net_income) * rate`; line 83 adds the missing upward rounding. That requirement already appears in the [2025 CFR, §273.10(e)(2)(ii)(A)](https://www.govinfo.gov/content/pkg/CFR-2025-title7-vol4/pdf/CFR-2025-title7-vol4-sec273-10.pdf). + + **Evidence:** In the pinned 1.755.4 environment, I ran a reform whose only action was `self.update_variable(m.snap_expected_contribution)`, using the committed module and original parameters. These currently scored SNAP outputs changed: + + | Scenario | Original reference | Rounding-only result | Change | + |---|---:|---:|---:| + | 008 | $15,246.905273 | $15,243.307617 | −$3.60 | + | 012 | $4,952.089355 | $4,942.789551 | −$9.30 | + | 038 | $7,286.944336 | $7,281.843750 | −$5.10 | + | 054 | $6,125.688965 | $6,118.790039 | −$6.90 | + | 079 | $2,428.017334 | $2,418.717041 | −$9.30 | + | 109 | $8,020.554199 | $8,017.554199 | −$3.00 | + + All six appear in the regeneration sidecar and none in exclusions. Thus the defect independently crosses the $1 threshold; combining it with a convention does not satisfy the supplied exclusion rule. + + Excluding these six changes the four headline scores to **94.578, 93.389, 93.102, and 91.869**, respectively. All 42 ranking positions remain unchanged. + + **Smallest fix:** Separate rounding into an `engine_defect` cause, sweep it, exclude its qualifying outputs—including these six—and preserve their original frozen values. Remove their regeneration entries and regenerate the board, sensitivity summaries, publication numbers, and pins. + +2. **Minor — CONFIRMED: the audit test accepts unsupported and misattributed sweep evidence.** + + **Location:** [tests/test_reference_audit.py:85](/Users/maxghenis/PolicyEngine/policybench-wt/opus55/tests/test_reference_audit.py:85). + + The test reduces sweep records to output keys and checks only subset membership. It discards causes, classes, values, and movement thresholds; it also never checks the reverse direction for missed exclusions. + + **Evidence:** I loaded the test module with `runpy.run_path` and patched `csv.DictReader` in memory to change every sweep record to: + + ```python + recomputed = frozen + delta = "0" + moved_over_1 = "False" + root_cause = "r02_ira_219g" + class_ = "engine_defect" + ``` + + All three audit tests passed, including `test_sweep_moves_cover_every_exclusion_and_regeneration`. No files were changed. + + **Smallest fix:** Retain cause-keyed records and assert matching classifications, qualifying movements, complete exclusion coverage, and exact convention ownership. + +Checks that found nothing wrong: + +- **Snapshot arithmetic:** Recomputed **1,984 unique outputs, 55 exclusions, 1,929 scored outputs, 23 unique regenerated references, and 61 unique adjudications**. No regeneration overlaps an exclusion. Every regenerated value matches the CSV; every excluded value retains its recorded frozen value. +- **Recorded exclusion support:** All **44 September 22 additions**—31 defects and 13 unlisted-input exclusions—have qualifying movements of the matching class. No declared defect sweep has an unexcluded qualifying output. Finding 1 concerns a defect bundled into a convention. +- **Sweep integrity:** Compared all **136 committed movement records across 31 modules** with the full local sweep CSVs: no missing records or numerical discrepancies beyond serialization precision. The stored baseline has 1,984 rows, zero qualifying moves, and maximum difference `5.68e-14`. +- **Convention consistency:** All nine conventions’ scored movement sets exactly match their sidecar ownership. Every owned value equals its convention-only sweep result within `1e-6`; no duplicate ownership or module-hash mismatch. +- **Primary-source spot checks:** IRS Texas table values **931/1,595** match the [2025 Schedule A instructions](https://www.irs.gov/pub/irs-prior/i1040sca--2025.pdf). Michigan’s **$5,900 exemption** and **$67,610/$135,220 retirement caps** match its [2026 withholding guide](https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2026/446_Withholding-Guide_2026.pdf). +- **Board recomputation:** Both raw snapshot predictions and dashboard predictions reproduce all 42 scores using the repository scorer. Requested values: **94.505970046, 93.214066253, 92.938541256, 91.616327493**. All 83,328 prediction values, reference values, and scoring flags agree; no duplicate prediction keys. +- **Pins:** All 42 manifest file checks pass. Reconstructed dashboard bytes equal `data-board42.json`: **116,127,282 bytes**, SHA-256 **`a4eadbbc9d329b09a33183117596c2c34cd41df69bad56c767bf60e2a9e8edf8`**. The freezer constant and app pointer agree. +- **Sensitivity:** Independently reproduced all four summaries’ three metrics, ranks, deltas, counts, eight asset hashes, and per-variable rows. Exact scores: Fable 5.1 **91.288**, Fable 5 **91.088**, Opus 5 **89.617**, Sonnet 5 **84.434**. +- **Test changes:** Reviewed every changed Python test and all eight changed app test files against `origin/main`. No literal tautologies or unjustified assertion deletions found. The app headline tolerance changes from four to three decimals alongside its three-decimal pin. +- **Executed tests:** **102 passed** across reference audit, sensitivity evidence, paper results, notes, report costs, and model cards, using `pytest -q -s -p no:cacheprovider` with bytecode disabled. Full Python and app suites were not rerun. Final `git status --short` was clean. \ No newline at end of file diff --git a/reference_audit/2026-09-22/verification/v8_axiom_snap_rounding.md b/reference_audit/2026-09-22/verification/v8_axiom_snap_rounding.md new file mode 100644 index 00000000..545b88cc --- /dev/null +++ b/reference_audit/2026-09-22/verification/v8_axiom_snap_rounding.md @@ -0,0 +1,51 @@ +# SNAP rounding parity (r13) + +Executed the existing §2017(a) encoding for every household with a nonzero delta in the supplied r13 sweep, including scenario_118 below the $1 scoring threshold. The test covers each January–September 2026 month (FY2026). It does not test October–December FY2027 parameters or reproduce a full annual SNAP calculation from raw household facts. + +**19 households, 171 monthly comparisons; 171/171 allotments match the rounding-only sandbox.** + +The comparator runs the supplied r13 reform with `R13_COMPONENTS=round_min,round_allot`. The original annual SNAP calculation is asserted against the frozen sweep reference for each household. All other reforms and FY2027 parameter replacements are excluded from this comparison. + +Axiom receives the household size and eligibility calculated by PolicyEngine, together with its floored net income. That floor is deliberately held constant: the original [PE contribution code](/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/lib/python3.13/site-packages/policyengine_us/variables/gov/usda/snap/snap_expected_contribution.py:20) and the sandbox both use it. The unmodified imported net-income module carries this boundary value through `snap_monthly_household_income` with all further deductions set to zero. This is a test of the allotment provision given net income, not a test of Axiom eligibility or deduction calculation. Every supplied boundary value is retained in `request.json` and `results.csv`. + +The formulas round the allotment down and the small-household minimum to the nearest dollar at [2017/a.yaml:128](/Users/maxghenis/TheAxiomFoundation/rulespec-us/us/statutes/7/2017/a.yaml:128) and [2017/a.yaml:153](/Users/maxghenis/TheAxiomFoundation/rulespec-us/us/statutes/7/2017/a.yaml:153), consistent with [7 USC 2017(a)](https://www.law.cornell.edu/uscode/text/7/2017). The same rules also appear in the existing [7 CFR 273.10 encoding](/Users/maxghenis/TheAxiomFoundation/rulespec-us/us/regulations/7-cfr/273/10.yaml); that second module was read but not executed here. + +All amounts below are monthly dollars. Each row represents nine separately executed months; the displayed amounts were checked identical across January–September. The original and corrected columns come from live PolicyEngine 1.755.4 simulations, not annual totals divided by 12. + +| Household | State | PE minimum | Corrected minimum | Axiom minimum | PE allotment | Corrected allotment | Axiom allotment | Matches rounding law at tested boundary? | +|---|---|---:|---:|---:|---:|---:|---:|---| +| scenario_008 | NJ | 95.00 | 95.00 | 0.00 | 1,260.10 | 1,260.00 | 1,260.00 | Yes; federal minimum only* | +| scenario_012 | MS | 0.00 | 0.00 | 0.00 | 407.90 | 407.00 | 407.00 | Yes | +| scenario_023 | CA | 23.84 | 24.00 | 24.00 | 35.20 | 35.00 | 35.00 | Yes | +| scenario_027 | CT | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_030 | TX | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_038 | LA | 0.00 | 0.00 | 0.00 | 601.30 | 601.00 | 601.00 | Yes | +| scenario_043 | CO | 23.84 | 24.00 | 24.00 | 298.00 | 298.00 | 298.00 | Yes | +| scenario_045 | MI | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_054 | NC | 0.00 | 0.00 | 0.00 | 505.70 | 505.00 | 505.00 | Yes | +| scenario_057 | LA | 23.84 | 24.00 | 24.00 | 219.00 | 219.00 | 219.00 | Yes | +| scenario_066 | VA | 23.84 | 24.00 | 24.00 | 298.00 | 298.00 | 298.00 | Yes | +| scenario_073 | MI | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_079 | AZ | 23.84 | 24.00 | 24.00 | 198.90 | 198.00 | 198.00 | Yes | +| scenario_080 | PA | 23.84 | 24.00 | 24.00 | 298.00 | 298.00 | 298.00 | Yes | +| scenario_100 | MT | 0.00 | 0.00 | 0.00 | 713.90 | 713.00 | 713.00 | Yes | +| scenario_108 | WI | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_109 | FL | 0.00 | 0.00 | 0.00 | 661.30 | 661.00 | 661.00 | Yes | +| scenario_112 | TX | 23.84 | 24.00 | 24.00 | 23.84 | 24.00 | 24.00 | Yes | +| scenario_118 | NY | 23.84 | 24.00 | 24.00 | 239.80 | 239.00 | 239.00 | Yes | + +*New Jersey has a $95 state minimum override in PE. The tested §2017(a) output is the federal minimum; it is not compared as if it implemented the NJ override. For scenario_008 the calculated allotment exceeds both minimums, so the allotment comparison remains meaningful. + +No numerical Axiom defect was found in the tested rounding outputs. Households already at the maximum can match PE before the fix; including them ensures the whole affected r13 household set was tested. FY2027 COLA encodings were not found in the tracked SNAP policy inventory. They are a coverage gap; no FY2027 correctness result is claimed. + +Additional validation: executed four existing §2017(a) companion fixtures and checked all 14 derived-output assertions; all passed. Three parameter-output expectations were skipped because the CLI query interface accepts derived outputs. Exact requests/responses and summary are in `companion_*.json`. + +Reproduce: + +```sh +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python snap/run.py +PYTHONDONTWRITEBYTECODE=1 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python snap/check_companions.py +python3 snap/build_report.py +``` + +`python3 snap/run.py --replay` reruns only the Rust engine from the saved PolicyEngine checkpoint, without importing PolicyEngine. No RuleSpec is created or modified. diff --git a/reference_audit/2026-09-22/verification/v8_snap_state_rounding.md b/reference_audit/2026-09-22/verification/v8_snap_state_rounding.md new file mode 100644 index 00000000..9f6253ed --- /dev/null +++ b/reference_audit/2026-09-22/verification/v8_snap_state_rounding.md @@ -0,0 +1,277 @@ +The clearest net-income findings are **California: retain cents, then round final net income to the nearest dollar; New Jersey: round income and deductions to whole dollars; New York and North Carolina: retain cents.** None is equivalent to flooring otherwise unrounded final net income. + +This report covers **FY2026, October 1, 2025–September 30, 2026**. Sources were read during this run on September 22, 2026. Later revisions, inaccessible documents, and evidence available only as indexed official text are identified below. + +**Allotment methods:** **1** = round the 30% contribution up; **2** = subtract the unrounded contribution and round the benefit down. With an integer maximum allotment, these produce the same result: + +`maximum − ceil(0.30 × state-calculated net income)` + +Neither is generally equivalent to the described PolicyEngine calculation. “No” below means an algorithmic mismatch; individual households can nevertheless receive the same amount. Unknowns remain unknown rather than being forced into yes/no answers. + +| State | Net-income procedure | Allotment method | FY2026 minimum | Does PE’s final-net floor match? | +|---|---|---|---|---| +| California | Keep cents; round final net income nearest dollar | Tables; current method unspecified; historical instructions use **1** | **$24**; qualifying pilot households **$60 total** | **No** | +| New Jersey | Nearest-dollar rounding of income and deductions | **1** | **$95 total**, with state supplement | **No** | +| New York | Keep cents throughout | **1** | **$24** | **No** | +| North Carolina | Nearest-cent calculations | **2** | **$24** | **No** | +| Mississippi | Precision not specified in reviewed provisions | **1** | **$24**, derived from rounded-8% rule | **Unresolved** | +| Louisiana | Repeated income rounding; precision unspecified | Not established | **$24** | **Unresolved** | +| Florida | Nearest-cent averaging/proration; final-net precision not separately stated | **1** | **$24** | **Not established** | +| Montana | Not specified in reviewed provisions | Reviewed provision silent | **$24** | **Unresolved** | +| Arizona | General manual rule not found; FY2026 agency example retains cents | **1** | **$24** | **No in agency example; general manual rule unresolved** | +| Colorado | Not reviewed, as requested | Current official text inaccessible | **$24**, official rulemaking evidence | Not reviewed | +| Connecticut | Not reviewed, as requested | Manual access incomplete | Minimum entitlement confirmed; numeric table inaccessible | Not reviewed | +| Michigan | Not reviewed, as requested | **1**; issuance tables for 1–2 people | **$24** | Not reviewed | +| Pennsylvania | Not reviewed, as requested | **1** | **$24** | Not reviewed | +| Texas | Not reviewed, as requested | **1** | **$24** | Not reviewed | +| Virginia | Not reviewed, as requested | Whole-dollar tables; **1 versus 2 unspecified** | **$24** | Not reviewed | +| Wisconsin | Not reviewed, as requested | **2**, official worksheet | **$24** | Not reviewed | + +Except for the California pilot and New Jersey supplement described below, **no higher state minimum was identified in the sources read**. That is a finding about these sources, not proof that no other supplementary program exists. + +**California — CalFresh** + +- **Net income:** MPP **63-503.31**, immediately preceding .311, requires that “the entire calculation process including individual shelter and medical costs shall include exact dollars and cents.” Only the final net figure is rounded: 1–49 cents down, 50–99 up. The rounding page is **p.283, FS-04-07, effective July 1, 2004**; .311’s calculation sequence is **p.284, FS-06-04, effective November 1, 2006**. [CDSS MPP 63-503](https://www.cdss.ca.gov/ord/entres/getinfo/pdf/fsman06.pdf) +- **Allotment:** **63-503.324**, pp.288–289, effective **July 1, 2004**, directs use of USDA issuance tables. The accessible [Handbook 63-1101](https://www.cdss.ca.gov/ord/entres/getinfo/pdf/fsman11b.pdf) does not specify method 1 versus 2. Historical CDSS **ACL 92-79, September 4, 1992**, instructs: “Round the product up to the next dollar if it ends in 1-99 cents”—**method 1**. That historical instruction was read through indexed official text; direct PDF retrieval failed. It is not independent confirmation of the current calculation procedure. [ACL 92-79](https://www.cdss.ca.gov/lettersnotices/entres/getinfo/acl92/92-79.PDF) +- **Minimum:** **$24**, explicitly in **ACIN I-46-25**, September 3, 2025, p.2, effective October 1, 2025. The older MPP’s $10 language is obsolete. [FY2026 notice](https://www.cdss.ca.gov/Portals/9/Additional-Resources/Letters-and-Notices/ACINs/2025/I-46_25.pdf?ver=KfyywOVXv5fZDkjuw5l9AA%3D%3D) +- **Higher minimum:** A state-funded pilot brings qualifying existing **ESAP households of two or more** to **$60 total**, starting **December 1, 2025**, for up to twelve months subject to funding. It is not a universal CalFresh minimum. [ACL 25-01, January 8, 2025](https://www.cdss.ca.gov/Portals/9/Additional-Resources/Letters-and-Notices/ACLs/2025/25-01.pdf?ver=2025-01-08-120805-737), [ACL 25-59, August 8, 2025](https://www.cdss.ca.gov/Portals/9/Additional-Resources/Letters-and-Notices/ACLs/2025/25-59.pdf?ver=MN8ddDPgX8ngsUpVynDZ4g%3D%3D) +- **PE comparison:** Different whenever final net income has 50–99 cents: California increases it to the next dollar; PE decreases it. The $23.84 minimum also differs. + +**New Jersey** + +The official **N.J.A.C. 10:87 SNAP Manual** compilation includes regulations published through **October 6, 2025**. [Manual](https://www.nj.gov/humanservices/notices/documents/rules-and-regulations/SNAPManual_10.14.25.pdf) + +- **Net income:** **10:87-6.16(b)–(c)**, PDF p.269, rounds income and deduction amounts nearest dollar, including work allowance and shelter deductions: “round-up those ending in $.50 through $.99 to the nearest dollar.” Individual monthly incomes are rounded before totaling. +- **Allotment:** **10:87-12.6(a)(1)**, PDF p.557: “Round the product up to the next dollar if it ends in cents”—**method 1**. +- **Minimum:** **10:87-13.2**, PDF p.564, provides a supplement bringing a full-month federal benefit of at least $1 but below $95 to **$95 total**. It is not restricted to one- and two-person households; prorated months are excluded. Subchapter 13 was adopted **December 18, 2023**. Section 12.6 still contains obsolete **$10 federal-minimum wording**. +- **PE comparison:** Different. Flooring only final net income cannot reproduce rounding income and deductions along the calculation. PE also omits the contribution ceiling and the $95 combined minimum. + +**New York** + +**Access qualification:** The official PDFs below failed direct retrieval. Their relevant provisions, revisions, and tables were read through **indexed official PDF text**. + +- **Net income:** SNAP Source Book **§11 A.3.d, p.11-5, revision July 2025**, says cents “shall be utilized throughout the computation of the net monthly income.” +- **Allotment:** **§11 A.4**, same revision: “Any result ending in cents shall be rounded up to the nearest higher dollar.” This applies after multiplying net income by 30%—**method 1**. +- **Minimum rule:** **§11 A.5.b, pp.11-5–11-6**, specifies 8% of the one-person maximum, rounded nearest whole dollar. [OTDA SNAP Source Book](https://otda.ny.gov/programs/snap/SNAPSB.pdf) +- **FY2026 amount:** **$24**, in **GIS 25DC059, September 10, 2025**, effective October 1, 2025, p.2. [FY2026 update](https://otda.ny.gov/policy/gis/2025/25DC059.pdf) +- **PE comparison:** Different: PE removes cents New York retains, omits the contribution ceiling, and uses $23.84 instead of $24. + +**North Carolina** + +**FNS 360, Determining Benefit Levels**, Change **01-2025**, effective **October 1, 2025**, provides all three rules. [Official manual](https://policies.ncdhhs.gov/wp-content/uploads/FNS-360-Determining-Benefit-Levels_09.30.2025.pdf) + +- **Net income:** **360.04.D**, p.6, says “round to two decimals (cents).” The third decimal rounds down at 0–4 and up at 5–9, unless otherwise specified. +- **Allotment:** **360.04.E.27–28**, pp.7–8, multiplies net income by 30%, subtracts it from the maximum, then says: “Round the result down to the whole dollar.” This is **method 2**. +- **Minimum:** **E.28** expressly gives **$24** for eligible one- and two-person units, before initial-month proration. +- **PE comparison:** Different: PE discards net-income cents instead of retaining them and fails to round the final benefit down. Its minimum is $0.16 lower. + +**Mississippi** + +The current agency-linked **SNAP Policy Manual**, Administrative Code **Title 18, Part 14**, contains **Rule 30.13**, p.141, and **Rule 31.10**, p.146, both marked **revised March 2022**. [Official manual](https://www.sos.ms.gov/adminsearch/ACCode/00000331c.pdf) + +- **Net income:** **30.13.A** gives the calculation sequence but does **not specify rounding precision or direction**. +- **Allotment:** **30.13.B** says “multiply the net monthly income by 30% and round up to the nearest higher dollar amount”—**method 1**. +- **Minimum:** **31.10** specifies 8% of the one-person maximum, “rounded to the nearest whole dollar.” Applying that rule to MDHS’s **$298 maximum effective October 1, 2025** yields **$24**. This is a calculation from the manual’s rule, not a $24 table entry in that code. [MDHS SNAP standards](https://www.mdhs.ms.gov/help/snap/) +- **PE comparison:** Final-net flooring remains **unresolved**. The unrounded contribution and $23.84 minimum nevertheless differ from explicit state rules. + +The older agency manual’s **§§6190–6193**, revised July 1, 2019, also did not resolve net-income precision. [October 2020 manual](https://www.mdhs.ms.gov/wp-content/uploads/2020/10/RevisedSNAPManual-10.20.pdf) + +**Louisiana** + +- **Net income:** **B-640-SNAP, §II, pp.3–5**, says “LITE WILL AUTOMATICALLY ROUND THE INCOME AND APPLY THE CONVERSION FACTOR.” It describes repeated rounding during earned-income aggregation, averaging, and conversion, but specifies **neither precision nor direction**. The live document is effective **September 1, 2026**, so it cannot independently establish the June 30 rule. [B-640](https://public.powerdms.com/LADCFS/documents/399187) +- **Allotment:** **Not established.** **J-300, §II**, effective **October 1, 2025**, points to J-311 issuance tables. **J-310**, dated **January 1, 2010**, identifies those tables without supplying the calculation. J-311 was not reached; the manual tree returned no readable content. [J-300](https://public.powerdms.com/LADCFS/documents/399088), [J-310](https://public.powerdms.com/ladcfs/documents/394420), [manual tree attempted](https://public.powerdms.com/LADCFS/tree) +- **Minimum:** **$24**, explicitly in **E-281-SNAP-PR**, p.4, effective **October 1, 2025**, for qualifying one- and two-person categorically eligible households. The general 8% rule appears in **LAC 67:III.1957.C**, September 2025 compilation, provision amended October 2019. [E-280/E-281](https://public.powerdms.com/LADCFS/documents/398402), [administrative code](https://doa.louisiana.gov/media/tp3lmkyg/67.pdf) +- **PE comparison:** Net-floor and allotment-rounding matches remain **unresolved**. The published $24 minimum differs from $23.84. + +**Florida** + +The relevant pages of the ESS manual’s Chapters 2400 and 2600 display **no revision or effective date**. + +- **Net income:** **2410.0600**, printed p.19, requires division used for averaging or prorating to “round the resulting amount to the nearest cent.” **2610.0105** lists net-income deductions without a separate final-net whole-dollar rounding instruction. Cent-based averaging is explicit; retaining cents through every net-income operation is an **inference**, not an express general rule. [Chapter 2400](https://ffic.myflfamilies.com/manual/2400.pdf), [Chapter 2600](https://ffic.myflfamilies.com/manual/2600.pdf) +- **Allotment:** **2610.0106.01**, printed p.3, specifies that after multiplying by 30%, “the resulting product is rounded up to the next whole dollar”—**method 1**. Section 2410.0601 independently confirms it. [Chapter 2600](https://ffic.myflfamilies.com/manual/2600.pdf) +- **Minimum:** **$24**, explicitly in **Appendix A-1**, effective **October 1, 2025**. [Appendix A-1](https://ffic.myflfamilies.com/manual/essfiles/31176.pdf) +- **PE comparison:** Final-net flooring is **not supported by the reviewed instructions**, but its precise replacement remains unverified. The contribution and minimum calculations clearly differ. + +**Montana** + +- **Net income:** No precision or direction was specified in the reviewed **SNAP 500**, effective **July 1, 2017**, or **601-1, Prospective Budgeting**, effective **October 1, 2022**. This does not establish truncation or cents retention. [SNAP 500](https://dphhs.mt.gov/assets/hcsd/snapmanual/SNAP500.pdf), [601-1](https://dphhs.mt.gov/assets/hcsd/snapmanual/SNAP601-1.pdf) +- **Allotment:** **SNAP 001, Thrifty Food Plan**, effective **October 1, 2025**, describes the maximum as “reduced by 30 percent of the net monthly income.” It is **silent on rounding that contribution or the resulting allotment**. +- **Minimum:** The same FY2026 document explicitly gives **$24**. The main live URL has been replaced with FY2027 standards; the agency’s `prod-dphhs` copy still provides the FY2026 document. [FY2026 SNAP 001](https://prod-dphhs.mt.gov/assets/hcsd/snapmanual/SNAP001.pdf) +- **PE comparison:** Net-floor and allotment-rounding matches remain **unresolved**; $23.84 differs from the explicit $24 minimum. + +**Arizona** + +- **Net income:** CNAP **FAA5.I01A, NA Net Income Test** lists deductions but gives no general rounding precision; the page displays no revision date. **FAA4.H02A** separately floors gross Social Security benefits, excluding SSI—this is not authority to floor total net income. [Benefit determination](https://dbmefaapolicy.azdes.gov/FAA5/NA_Eligibility_and_Benefit_Determination.html), [Income Budgeting Basics](https://dbmefaapolicy.azdes.gov/FAA4/Income_Budgeting_Basics.html) +- **Supplementary agency evidence:** DES’s calculation example, expressly effective **October 1, 2025–September 30, 2026**, carries **$386.46 net income** into the 30% calculation. Thus cents retention is demonstrated by an official example, although the general manual rule remains unstated. [DES FY2026 example](https://des.az.gov/node/4875) +- **Allotment:** CNAP **FAA6, §08, Thrifty Food Plan**, manual-calculation instructions: “Round the product up to the next whole dollar”—**method 1**. The page contains schedules effective October 1, 2025 and October 1, 2026; the rounding paragraph is undated. [Thrifty Food Plan](https://dbmefaapolicy.azdes.gov/FAA6/Thrifty_Food_Plan_(NA).html) +- **Minimum:** **FAA6, §05**, explicitly **$24 effective October 1, 2025**. [Minimum NA Allotment](https://dbmefaapolicy.azdes.gov/FAA6/Minimum_NA_Allotment.html) +- **PE comparison:** Different from the FY2026 example’s treatment of net cents, and from the explicit contribution ceiling and minimum. + +**Colorado — allotment and minimum only** + +- **Access limitation:** The official **10 CCR 2506-1** listing identifies the current compilation as effective **December 30, 2025**, but its PDF was not retrievable through the available interface. The adopted DOCX also could not be read. [Current rule listing](https://www.sos.state.co.us/CCR/DisplayRule.do?action=ruleinfo&ruleId=2818), [DOCX attempted](https://www.sos.state.co.us/CCR/Upload/AGORequest/AdoptedRules02025-00474%20.docx) +- **Allotment:** **Unverified for FY2026.** Historical official versions differ and cannot establish the current rule. +- **Minimum:** Official FY2026 rulemaking text for **4.207.3(D)** specifies **$24 for households of one or two, effective October 1, 2025**. This was read through indexed official text; direct PDF retrieval failed. The permanent docket records adoption November 7, 2025, effective December 30, 2025. [FY2026 rulemaking document](https://www.coloradosos.gov/CCR/Upload/AGORequestEmergency/EmergencyJustificationPathAttach2025-00506.pdf), [adoption docket](https://www.sos.state.co.us/CCR/eDocketDetails.do?trackingNum=2025-00474) +- **PE comparison:** The supported $24 minimum differs; current allotment rounding remains unresolved. + +**Connecticut — allotment and minimum only** + +- **Minimum rule:** The current SNAP manual’s **“Minimum Benefits”** section, **undated**, says: “Eligible EDGs with one or 2 members receive no less than the minimum monthly benefit.” [Manual section](https://portaldir.ct.gov/dss/snap/MinimumBenefits.html) +- **Numeric amount:** The linked **Tables** page returned a bot-request rejection. I therefore cannot report its FY2026 numeric minimum as directly verified. [Table URL attempted](https://portaldir.ct.gov/dss/snap/Tables.html) +- **Allotment:** **Not verified, rather than established silence.** The official **UPM §6005** download could not be parsed because it is an unsupported Word document; its revision date was not readable. [UPM §6005 attempted](https://portal.ct.gov/dss/-/media/departments-and-agencies/dss/upms/upm6---calculation-of-benefits-benefit-issuance/6005.doc?rev=a199a752e00a42d0a87976822c30245c) +- **PE comparison:** No state-manual conclusion about the requested rounding choice from the accessible material. + +**Michigan — allotment and minimum only** + +- **Allotment:** **BEM 556, “Complete Benefit Calculation,” p.6**, **BPB 2025-028, November 1, 2025**, instructs multiplication by .30 followed by “Round up.” Subtraction follows on p.7: **method 1**. One- and two-person groups are directed to RFT 260. [BEM 556](https://dhhs.michigan.gov/olmweb/ex/BP/Public/BEM/556.pdf) +- **Minimum:** **RFT 260**, **RFB 2025-006, October 1, 2025**, explicitly covers FY2026 and has a **$24** minimum in its one- and two-person issuance columns. [RFT 260](https://dhhs.michigan.gov/OLMWEB/EX/RF/Public/RFT/260.pdf) +- **PE comparison:** Different: PE omits the contribution ceiling and uses $23.84 rather than $24. Net-income rounding was not reviewed. + +**Pennsylvania — allotment and minimum only** + +**Access qualification:** The full official page text was read through search indexing; direct retrieval timed out. + +- **Allotment:** SNAP Handbook **Chapter 568, Appendix D**, updated **October 16, 2025**, steps 1–3, specifies **method 1**: “Round the product up to the next whole dollar if it ends in 1 through 99 cents.” +- **Minimum:** The same appendix explicitly gives **$24**, including its categorical-eligibility paragraph. Step 5 retains an inconsistent older less-than-$10 trigger; that should not be copied into a new implementation without resolving the stale wording. [Appendix D](https://services.dpw.state.pa.us/oimpolicymanuals/snap/568_Computing_Eligibility_and_Benefit_Amount/568_Appendix_D.htm) +- **PE comparison:** Different: contribution ceiling and $24 minimum are absent from the described PE calculation. Net-income rounding was not reviewed. + +**Texas — allotment and minimum only** + +- **Allotment:** Texas Works Handbook **C-122**, **Revision 25-4, effective October 1, 2025**, requires “rounding the cents to the next higher whole dollar amount” after multiplying net income by .30: **method 1**. Its example turns $289.20 into $290. [C-122](https://fhb.hhs.texas.gov/handbooks/texas-works-handbook/c-120-supplemental-nutrition-assistance-program) +- **Minimum:** **C-1431**, same revision and effective date, gives **$24** for an unprorated eligible one- or two-person household. [C-1431](https://fhb.hhs.texas.gov/handbooks/texas-works-handbook/c-1430-snap-allotment-charts) +- **PE comparison:** Different: PE omits upward contribution rounding and understates the minimum. Net-income rounding was not reviewed. + +**Virginia — allotment and minimum only** + +Use the **full manual effective October 1, 2025**; the separately linked Part XXIII document still contained older standards. [Full manual](https://www.dss.virginia.gov/media/vdss/benefit-programs/documents/snap/snap-manual/Entire-Manual-eff-10012025.pdf) + +- **Allotment:** **Volume V, Part XXIII.A–B**, **10/25, Transmittal 36**, directs use of whole-dollar issuance tables, without identifying method 1 versus 2. The October 2025 table gives a one-person benefit of **$297 at net income of $1–3**, consistent with both federal methods. +- **Minimum:** **Part XXIII.A, printed p.1/PDF p.398**: “For household sizes 1 and 2, $24 is the minimum allotment for all eligible households”. +- **PE comparison:** Different: at $1 net income, the described PE formula produces **$297.70**, while the table prescribes **$297**. Its minimum also differs. Net-income rounding was not reviewed. + +**Wisconsin — allotment and minimum only** + +- **Allotment:** FoodShare Handbook **§4.2.2**, release **25-03, December 10, 2025**, directs use of official worksheet **F-16033**. Its **revision 10/2023**, p.2, line 29, subtracts 30% of adjusted net income and says “round cents down to the nearest whole dollar”—**method 2**. [Handbook](https://www.dhs.wisconsin.gov/publications/p16001-25-03.pdf), [official worksheet](https://www.dhs.wisconsin.gov/forms/f1/f16033.pdf) +- **Minimum:** Handbook **§8.1.1.1**, pp.349–350, expressly covers FY2026: “The minimum allotment for one- and two-person assistance groups is $24.” [Handbook](https://www.dhs.wisconsin.gov/publications/p16001-25-03.pdf) +- **PE comparison:** Different: PE floors before multiplication instead of rounding the final allotment down, and uses $23.84 rather than $24. The broader net-income procedure was outside the requested scope. + +```json +{ + "states": { + "CA": { + "net_income": "Keep cents throughout; round final net income to nearest dollar, half up.", + "allotment": "Current accessible MPP uses issuance tables without specifying method; historical CDSS instructions use method 1.", + "minimum": { + "ordinary": 24, + "qualifying_pilot_total": 60, + "pilot_scope": "Qualifying existing ESAP households of two or more; implemented 2025-12-01." + }, + "source": "https://www.cdss.ca.gov/ord/entres/getinfo/pdf/fsman06.pdf; https://www.cdss.ca.gov/Portals/9/Additional-Resources/Letters-and-Notices/ACINs/2025/I-46_25.pdf; https://www.cdss.ca.gov/Portals/9/Additional-Resources/Letters-and-Notices/ACLs/2025/25-59.pdf" + }, + "NJ": { + "net_income": "Nearest-dollar rounding of income and deductions; not flooring final net income.", + "allotment": "method_1", + "minimum": { + "state_total": 95, + "conditions": "Full-month federal benefit at least $1 and below $95; excludes prorated months.", + "caveat": "Manual federal-minimum paragraph retains obsolete $10 wording." + }, + "source": "https://www.nj.gov/humanservices/notices/documents/rules-and-regulations/SNAPManual_10.14.25.pdf" + }, + "NY": { + "net_income": "Keep cents throughout.", + "allotment": "method_1", + "minimum": 24, + "source": "https://otda.ny.gov/programs/snap/SNAPSB.pdf; https://otda.ny.gov/policy/gis/2025/25DC059.pdf", + "access": "Indexed official text read; direct retrieval failed." + }, + "NC": { + "net_income": "Nearest-cent calculations; third decimal 5 or greater rounds up.", + "allotment": "method_2", + "minimum": 24, + "source": "https://policies.ncdhhs.gov/wp-content/uploads/FNS-360-Determining-Benefit-Levels_09.30.2025.pdf" + }, + "MS": { + "net_income": "Precision and direction unresolved in reviewed manual provisions.", + "allotment": "method_1", + "minimum": 24, + "minimum_basis": "Calculated from manual's rounded-8-percent rule and official FY2026 one-person maximum.", + "source": "https://www.sos.ms.gov/adminsearch/ACCode/00000331c.pdf; https://www.mdhs.ms.gov/help/snap/" + }, + "LA": { + "net_income": "Repeated income rounding described, but precision and direction unspecified; live B-640 effective 2026-09-01.", + "allotment": "Unresolved; referenced J-311 issuance tables not reached.", + "minimum": 24, + "source": "https://public.powerdms.com/LADCFS/documents/399187; https://public.powerdms.com/LADCFS/documents/399088; https://public.powerdms.com/LADCFS/documents/398402" + }, + "FL": { + "net_income": "Nearest-cent averaging and proration explicit; final-net rounding precision not separately specified.", + "allotment": "method_1", + "minimum": 24, + "source": "https://ffic.myflfamilies.com/manual/2400.pdf; https://ffic.myflfamilies.com/manual/2600.pdf; https://ffic.myflfamilies.com/manual/essfiles/31176.pdf" + }, + "MT": { + "net_income": "Not specified in reviewed provisions; unresolved.", + "allotment": "Reviewed SNAP 001 provision silent on rounding.", + "minimum": 24, + "source": "https://dphhs.mt.gov/assets/hcsd/snapmanual/SNAP500.pdf; https://dphhs.mt.gov/assets/hcsd/snapmanual/SNAP601-1.pdf; https://prod-dphhs.mt.gov/assets/hcsd/snapmanual/SNAP001.pdf" + }, + "AZ": { + "net_income": "General manual precision unresolved; official FY2026 agency example retains cents.", + "allotment": "method_1", + "minimum": 24, + "source": "https://dbmefaapolicy.azdes.gov/FAA5/NA_Eligibility_and_Benefit_Determination.html; https://dbmefaapolicy.azdes.gov/FAA6/Thrifty_Food_Plan_(NA).html; https://dbmefaapolicy.azdes.gov/FAA6/Minimum_NA_Allotment.html; https://des.az.gov/node/4875" + }, + "CO": { + "net_income": "Not reviewed.", + "allotment": "Current official compiled rule inaccessible; FY2026 method unresolved.", + "minimum": { + "amount": 24, + "evidence": "Indexed official FY2026 rulemaking text; current consolidated document not read." + }, + "source": "https://www.sos.state.co.us/CCR/DisplayRule.do?action=ruleinfo&ruleId=2818; https://www.coloradosos.gov/CCR/Upload/AGORequestEmergency/EmergencyJustificationPathAttach2025-00506.pdf; https://www.sos.state.co.us/CCR/eDocketDetails.do?trackingNum=2025-00474" + }, + "CT": { + "net_income": "Not reviewed.", + "allotment": "Not verified; manual access incomplete.", + "minimum": { + "amount": null, + "rule": "Eligible one- and two-member groups receive the minimum monthly benefit, subject to initial-month exception.", + "caveat": "Linked numeric table inaccessible." + }, + "source": "https://portaldir.ct.gov/dss/snap/MinimumBenefits.html; https://portaldir.ct.gov/dss/snap/Tables.html" + }, + "MI": { + "net_income": "Not reviewed.", + "allotment": "method_1; issuance tables used for one- and two-person groups", + "minimum": 24, + "source": "https://dhhs.michigan.gov/olmweb/ex/BP/Public/BEM/556.pdf; https://dhhs.michigan.gov/OLMWEB/EX/RF/Public/RFT/260.pdf" + }, + "PA": { + "net_income": "Not reviewed.", + "allotment": "method_1", + "minimum": 24, + "source": "https://services.dpw.state.pa.us/oimpolicymanuals/snap/568_Computing_Eligibility_and_Benefit_Amount/568_Appendix_D.htm", + "access": "Indexed official page read; direct retrieval timed out." + }, + "TX": { + "net_income": "Not reviewed.", + "allotment": "method_1", + "minimum": 24, + "source": "https://fhb.hhs.texas.gov/handbooks/texas-works-handbook/c-120-supplemental-nutrition-assistance-program; https://fhb.hhs.texas.gov/handbooks/texas-works-handbook/c-1430-snap-allotment-charts" + }, + "VA": { + "net_income": "Not reviewed.", + "allotment": "Whole-dollar issuance tables; method 1 versus method 2 not specified.", + "minimum": 24, + "source": "https://www.dss.virginia.gov/media/vdss/benefit-programs/documents/snap/snap-manual/Entire-Manual-eff-10012025.pdf" + }, + "WI": { + "net_income": "Not reviewed.", + "allotment": "method_2, specified in official worksheet F-16033", + "minimum": 24, + "source": "https://www.dhs.wisconsin.gov/forms/f1/f16033.pdf; https://www.dhs.wisconsin.gov/publications/p16001-25-03.pdf" + } + } +} +``` \ No newline at end of file diff --git a/reference_audit/2026-09-22/verification/v9_axiom_ca-17052.md b/reference_audit/2026-09-22/verification/v9_axiom_ca-17052.md new file mode 100644 index 00000000..29494ff6 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_ca-17052.md @@ -0,0 +1,171 @@ +# ca-17052 — Axiom encoding preparation + +Prepared 2026-09-22. Read-only investigation; only this workspace's scratch artifacts were written. No RuleSpec or module tests were authored, no local encode ran, and no commit, PR, corpus change, or signed run was made. The specific read-only assignment overrides the initial generic commit instruction. + +## 1. Verdict + +**BLOCKED.** The statute is present, but a source-complete, admissible signed encoding of the requested 2026 CalEITC calculation cannot yet be specified from the verified dependencies. The missing items are corpus-backed FTB 3514 instructions/table and applicable annual factor, authorization and indexed parameters; a supported way to supply the incorporated federal mechanics; and a fresh current-main/collision check. The planned repin chain alone does not supply the missing California sources. + +The independent adjudication is narrower and clear: **the statute supports the sandbox's AGI-comparison mechanism.** Holding the frozen projected schedule fixed, the independently recomputed value is **$99.4600676051**, matching r17's **$99.460068**, versus frozen **$148.310867**. This is conditional on that schedule. Neither number equals the **$96** obtained using the published **2025** FTB worksheet/table. No definitive 2026 annual amount was established from the sources retrieved here. Do not describe $96 as final 2026 law or $99.46 as an independently verified 2026 legal amount. + +**Attribution:** PR **#9363** introduced the comparison. PR **#9542** changed the 2024 exclusive income boundary and references. Exact local commit/diff evidence appears in section 6. This package records Axiom coverage and acceptance criteria, not an external-oracle finding. + +The required fetch was attempted and failed with `cannot open '.git/FETCH_HEAD': Operation not permitted`. All requested `gh` reads failed with `error connecting to api.github.com`. Local immutable objects were used; freshness of remote main, PR status and today's runs remains unverified. Binding [Axiom issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was read through the browser; the displayed copy was crawled three weeks earlier. + +## 2. Source + +Official controlling text: [California RTC §17052](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052), incorporating [IRC §32(a)](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapA-partIV-subpartC-sec32.htm). The operational worksheet is [FTB 3514, 2025 booklet](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.pdf), worksheet on printed page 8 and relevant table on pages 26–27. The 2025 material establishes the comparison and dated controls; it is not a 2026 annual table. + +Canonical citation: **`us-ca/statute/rtc/17052`**, the entire section. Neither inspected California source scope contains a separate `us-ca/statute/rtc/17052/a` record. Use the whole section as the encoder's source unit. + +Canonical corpus local origin/main read: **`942e138e7a8250c9814e774ac9b8e63008148106`**. Two tracked versions contain the section: + +| Version | Tracked provision file | Source vintage | +|---|---|---| +| July source used by the pinned release | `data/corpus/provisions/us-ca/statute/2026-07-06-ca-rtc-pit-core-us-ca-sections-rtc-17041-rtc-17043-rtc-17045-rtc-17052-rtc-17054-rtc-17073.5.jsonl` | expression/source-as-of 2026-07-06 | +| Later corpus-main snapshot | `data/corpus/provisions/us-ca/statute/2026-09-14-income-tax-chapter-us-ca-sections-4e26e6efabc3f0c7.jsonl` | expression/source-as-of 2026-09-14 | + +Both identify the same AB 1766 amendment, Stats. 2022 ch. 482 §5, effective January 1, 2023. Their normalized bodies are byte-identical after extraction (`diff -u` produced no differences). The retained July HTML is `data/corpus/sources/us-ca/statute/2026-07-06-ca-rtc-pit-core-us-ca-sections-rtc-17041-rtc-17043-rtc-17045-rtc-17052-rtc-17054-rtc-17073.5/california-leginfo-sections/RTC-17052.html`, SHA-256 **`b0109e3cbb14a528423c83e6a41ad5145e711f702baff33ae24c84204601427d`**. Its ingest manifest records that path/digest and generation at 2026-07-15T19:51:39.937073+00:00. The September HTML digest is `1d75484931ec99eef7dd05dbedbb1e5c21a5d09727200042f101d2c89eb51768`. + +**In pinned release: yes, for §17052.** On inspected rulespec-us main, `.axiom/toolchain.toml` pins `us-rulespec-2026-08-08-obbb-alien-snap`, content digest **`0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`**. The tracked release manifest explicitly includes the July California statute scope. The same source record exists at the pinned corpus checkout commit `8f7d60aaced28ee4252b9237f9d6e02360dc34bc`. This verifies scope membership from Git; `axiom-locate release` found no installed release bundle, so signed bundle bytes/signature were not independently downloaded or checked. + +IRC 32 also exists in corpus at `data/corpus/provisions/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup.jsonl`, citation `us/statute/26/32`, expression date 2026-07-13, retained official XML `sources/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup/official-documents/usc26-section-32.xml`. Its allowance and limitation text was read, independently of the PolicyEngine fix docstring. + +**Supporting source gap:** the tracked California form/guidance inventories and provisions inspected contain no FTB 3514 worksheet or credit table. The purported 2025 tax-materials scope contains only Form 540 tax-table/rate-schedule records. Ingest the official [2025 booklet PDF](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.pdf), preserving worksheet guards, rows and columns, URL, retrieval/source date and SHA provenance, for dated controls. For a final 2026 monetary encoding, obtain and ingest the actual applicable 2026 annual FTB/indexing authority and enacted Budget Act factor/resource provision first. Direct attempted 2026 booklet/instruction URLs did not resolve; this is not proof that all 2026 authority is unpublished. No new citation for an uningested worksheet is invented here. The planned August23 union release retains the same California scopes and does not itself fill these gaps. + +Source extracts: [§17052](statute-17052.txt), [IRC 32](irc32-source.json), [release manifest](pinned-release-manifest.json). + +## 3. Existing rulespec-us coverage + +Inspected local **origin/main `f43dec520dd392bc5333934f56dad7498363e704`**. The repository tree and targeted content search found the following relevant coverage; no separate subsection(a) or FTB 3514 computation module was found. + +| Module | Actual coverage and relevant quoted lines | AGI-comparison status | +|---|---|---| +| `us-ca/statutes/rtc/17052.yaml` | Lines 13–16: `output: us-ca:statutes/rtc/17052/a#california_earned_income_tax_credit`; `reason: The final credit depends on IRC section 32, RTC section 17039 net tax,` followed by annual Budget Act and other dependencies. Lines 36–410 provide base/alternative statutory parameters; lines 413–450 provide two ITIN duties. | Final credit expressly deferred; no earned-income-only monetary computation to compare. This is missing coverage, not the same computed defect. | +| Its companion `17052.test.yaml` | Three cases test the requested-document and SSN-notice duties, with all three local Boolean inputs assigned. | No monetary or AGI-comparison cases. | +| `us-ca/policies/income_tax/2026_resident_liability_source_hold.yaml` | Lines 36–43 state missing annual materials and Budget Act factor. Lines 261–287 name `ca_pit_2026_credit_surface_source_hold_applies` and set `formula: true`. Lines403–421 define refundable credits with `source: Fail-closed annual refundable-credit sentinel while source held` and `formula: '0'`. | Explicit unavailable-source sentinel, not an adjudicated zero or an implementation of the disputed mechanism. Lines5–15 also contain the rejected plural source key. | +| `us-ca/policies/income_tax/pilot_liability_pipeline.yaml` | Summary describes the estimated-tax rate-schedule branch above $100,000 and explicitly excludes `all other worksheet taxes and credits` and `final-return liability`. | No CalEITC computation; plural source key at lines 5–10. | +| `us-ca/statutes/rtc/17041.yaml` | Lines 59–72 explicitly defer `#inflation_adjustment_factor` and `#recomputed_income_tax_bracket`. Base brackets and the non-surtax characterization output do not supply the required annual CalEITC indexing. | Not a credit implementation or a usable annual-parameter dependency. | +| `us-ca/statutes/rtc/17054.yaml` | Lines 9–10 defer `#personal_exemption_credit`; statutory base credits, blindness and another-taxpayer dependency logic are encoded. | Nearby personal-exemption provision, outside CalEITC. | +| Federal `us/statutes/26/32.yaml` | Imports lines 2–7; `eitc_phase_out_income` uses `max(adjusted_gross_income, earned_income)` and `eitc_before_eligibility` uses `max(0, min(eitc_phased_in, eitc_maximum - eitc_reduction))`. | Correct comparison structure for the federal module; federal schedules/eligibility cannot become California parameters unchanged. Closure is inadmissible here and rejected by the pinned engine as detailed below. | + +The CA 17052/17041 manifests inspected are legacy `axiom-encode/applied-rulespec/v1`, not signed-v5 imports. The target may still be the existing replacement path; do not mistake that for import eligibility. The old target's ITIN proof excerpts contain literal ellipses; a new run must ground fresh exact spans rather than copy these excerpts. + +## 4. Import closure + +Candidate **`existing_signed_imports_json=[]`**. This is the only supported candidate established here, and is **not** a complete dependency plan. No usable signed-v5 California module supplying the incorporated comparison plus annual schedule was identified. The inspected workflow helper rejects supplied imports from a different exact jurisdiction; `us/...` cannot be supplied for `us-ca/...`. This is a workflow input restriction, not a general claim about RuleSpec import syntax. + +For completeness, the natural federal candidate's entire ten-module closure was read: + +| Module | Direct imports | Located legacy binary | Pinned engine `af6e4ea…` | +|---|---|---|---| +| `us:statutes/26/32` | 32/c/2,152/c,7703,IRS EIC | Compiles | Rejected through IRS EIC, established by code | +| `us:statutes/26/32/c/2` | 112 | Compiles | Runtime unverified | +| `us:statutes/26/112` | None | Compiles | Runtime unverified | +| `us:statutes/26/152/c` | None | Compiles | Runtime unverified | +| `us:statutes/26/7703` | 151 | Compiles | Runtime unverified | +| `us:statutes/26/151` | 911/a,931,933 | Compiles | Runtime unverified | +| `us:statutes/26/911/a` | None | Compiles | Runtime unverified | +| `us:statutes/26/931` | None | Compiles | Runtime unverified | +| `us:statutes/26/933` | None | Compiles | Runtime unverified | +| `us:policies/irs/rev-proc-2025-32/earned-income-credit` | None | Compiles | Rejected: plural `corpus_citation_paths`, lines 8–11 | + +Pinned engine `src/rulespec.rs:819–822` rejects `corpus_citation_paths`; lines 1086 and 1093–1095 enforce that recursively on imports. This is direct code evidence, not a claim to have reproduced a failure with a current executable. The same parser rule excludes the two California policy modules above. CA 17052 and 17041 have no imports, use the singular key, and compile with the located binary; current-engine runtime/proof validation was not completed. + +`axiom-locate engine` returned `/Users/maxghenis/TheAxiomFoundation/_tariff-parity/axiom-rules-engine/target/release/axiom-rules-engine`, SHA-256 `674ca6e70afdccb59c3d6847933bc24b4590105e49db54790f2dcd0bdbbe32d7`. It reports version 0.1.0 and accepts the removed plural key, so its 12 successful existing-module compiles cannot clear signed-run compatibility. No engine build was attempted. Exact per-module commands, results, manifests and pinned-source excerpts are in [closure notes](closure/notes.md), [compile results](closure/compile-results.json) and [parser evidence](closure/pinned-engine-plural-contract.txt). + +A prerequisite is an approved source-complete California dependency route, or supported cross-jurisdiction context with its dependency failures resolved. Merely removing the IRS import, copying federal annual amounts, or accepting precomputed credit inputs would not establish a lawful complete California encoding. + +## 5. Dispatch inputs + +**No pass-ready dispatch command can be certified. Do not execute the command below.** It is fully populated against the inspected snapshot for review; its corpus lacks required supporting sources and its rulespec SHA is not verified as today's remote tip. After prerequisites land, refs and finding must be refreshed and Max must approve the signed run. This lane grants no dispatch approval. + +Workflow inspected: axiom-encode local origin/main **`5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`**. The [saved workflow](workflow/targeted-signed-reencode.yml) declares the inputs at lines 4–115. The [helper](workflow/prepare_signed_backfill_impl.py) requires exact current remote main for `open_pr=true` at lines 267–331, canonical citation paths at 334–353, and same-jurisdiction tracked signed-v5 import paths at 1368–1502. Workflow 362–401 verifies full 40-character checkout identities and corpus/engine main ancestry. Replacement validation was also read, saved in [replacement extract](workflow/replacement-target-validation.txt). + +| Input | Verified snapshot value and provenance | +|---|---| +| rulespec_ref | `f43dec520dd392bc5333934f56dad7498363e704`, canonical checkout's local origin/main | +| corpus_ref | `8f7d60aaced28ee4252b9237f9d6e02360dc34bc`, that rulespec commit's `.axiom/workflow-toolchain.toml`; commit object and local-main ancestry verified | +| rules_engine_ref | `af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca`, same workflow-toolchain pin; commit object and local-main ancestry verified | +| citation / replacement | `us-ca/statute/rtc/17052` / `us-ca/statutes/rtc/17052.yaml`, verified corpus record and existing module | +| imports / source bundle | `[]` / `[]`, admissible empty shapes only; unresolved legal dependencies remain | +| review_finding | Exact accompanying [review_finding.md](review_finding.md) | +| country / PR base / open_pr | `us` / `main` / `true`, per workflow schema; the future action would create a draft PR | + +```sh +gh workflow run targeted-signed-reencode.yml \ + -R TheAxiomFoundation/axiom-encode \ + --ref main \ + -f citation=us-ca/statute/rtc/17052 \ + -f country=us \ + -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \ + -f pr_base_branch=main \ + -f corpus_ref=8f7d60aaced28ee4252b9237f9d6e02360dc34bc \ + -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \ + -F review_finding=@scratch/ca-17052/review_finding.md \ + -f source_bundle_json='[]' \ + -f existing_signed_imports_json='[]' \ + -f replace_rulespec_path=us-ca/statutes/rtc/17052.yaml \ + -f open_pr=true +``` + +All remaining optional repair/dependent/legacy/queue inputs are intentionally absent and retain declared empty/default values; no such run IDs or dependent paths were established. The mandatory `--ref main` is a moving workflow branch: the actual workflow SHA must be recorded again at dispatch. Neither the current nor a future signed run was executed. + +**Repin chain:** inspected main is the merge of waiver renewal #1384. Local #1387 activation head `500d9df58be4e17f8572e51f665a28241b039823` and workflow-pin branch `a9dd1fb7985ccfdaf9f98548edde94ad48cd227c` are not ancestors of it. The latter's mapping to #1386 is task-supplied, not verified live. Waiver activation #1387, workflow pin #1386, drift staging, then activation/release repin remain the named coordination chain; later PR numbers and current statuses are unverified. Reconcile that chain before the orchestrator freezes its eventual exact-main ref. §17052 already belongs to today's pinned release, so that chain is not needed to add this statute; it also cannot cure the missing worksheet/annual authority. Waiting only for the chain is insufficient for READY-AFTER. + +## 6. Review finding text and independent worked cases + +The complete proposed `review_finding` payload is [review_finding.md](review_finding.md), including the assignment's encoding regime verbatim. It is held pending the prerequisites, not offered as a workaround for missing sources. Its required source coverage is: + +- Preserve IRC 32(a)(1)'s earned-income cap and (a)(2)(B)'s greater-of-AGI-or-earnings phase-out cap; apply the applicable California schedule and annual adjustment. +- Cover §17052(a) authorization/default-factor gates, (b) California substitutions/no joint increment, (c) California eligibility and earned-income modifications, (d)-(e) investment income/indexing, (f) refundable excess, (o) alternative schedule and inflation rules, and (p)-(q) ITIN treatment/duties. +- Precisely defer administrative (g), (i), (j) and declaratory (r) outputs; represent (k) as naming metadata or precisely defer its noncomputational naming output; identify (h)'s cross-program benefit-disregard execution and (l)'s effective dates. Identify expired years for historical (m), (n), (c)(3) branches if limiting the module to 2026. Historical (e)(2)-(3) CPI floors may be deferred only when sourced annual values already incorporate their cumulative effects; recomputation from base amounts must preserve them. Do not use a broad deferral to omit the requested comparison or final credit while calling the assignment complete. +- Pair positive and blocking tests for authorization, schedule boundaries, child groups, ages, abode/withholding, self-employment dates, investment-income limit, separated-spouse conditions, identification and ITIN duties. Explicitly set every local input, including false Booleans. Test AGI above/below/equal earnings and phase-out start, the earned-income phase-in cap, and zero/negative AGI. + +The ordinary incorporated branch, for nonnegative earned income E and applicable parameters, is: + +`k × max(0, min(c × min(E,M), c × M − p × max(0,max(A,E)−P)))`. + +Here A is federal AGI; c/p are the applicable credit/phase-out rates; M/P are the earned-income/phase-out amounts; k is the authorized adjustment factor. This expresses the ordinary statutory limitation, **not** a complete substitute for California(o)'s alternative schedule. IRC 32(a)(2)(B) says **“adjusted gross income (or, if greater, the earned income)”**. The source text—not the fix docstring—therefore decides the ordering. + +**All moved benchmark households:** `sweep_moves.csv` contains exactly one r17 row, scenario_023, state_refundable_credits. Facts from the supplied scenario: tax year 2026, California, single adult age 28, no children; wages **17,442.646484375**, traditional 401(k) contribution **2,778.47998046875**, taxable 403(b) distribution **8,000**, traditional IRA contribution **129.8303985595703**. Residency/identification conditions not explicit in the record are held eligible, as in the isolated comparison. Observed `ca_eitc_eligible=true`, `filer_adjusted_earnings=17,442.646484375`, federal AGI=CA AGI=**22,534.3359375**. Exact arithmetic on the supplied inputs gives AGI 22,534.33610534668; the tiny difference is float precision. California and federal AGI happen to agree here; the worksheet calls for federal AGI. + +The independent calculation read projected parameters from baseline without importing the fix: credit/phase-out rate 0.0765, factor 0.85, first threshold 4,766.6318362973125, final-segment starting credit K=263 and endpoint U=32,901. Derived transition L=5,488.6654411451825. The final segment `C(x)=max(0,263×(32901−x)/(32901−L))` gives: + +| Case | Inputs | Expected amount and adjudication | +|---|---|---| +| **1. scenario_023, 2026; comparison with inherited schedule held** | E=17,442.646484375; A=22,534.3359375; child count 0 | Frozen **148.310867**; r17 **99.460068**. Independent C(E)=148.310862246357; C(A)=**99.460067605107**. **Supports the corrected comparison/value conditional on the inherited schedule**, not an independent endorsement of its 2026 annual parameters. | +| Same household, published 2025 table sensitivity | Earned lookup 143; AGI lookup 96 | **96**, so **neither** audit amount is the result under the explicitly held 2025 table convention. Removing the 401(k) deferral from taxable earnings changes the first lookup to 168; the minimum remains96. This is not a 2026-law conclusion. | +| **2. 2025 equality control** | Otherwise eligible/no children; E=A=17,442.65 | **143**. Equality must not alter the earned-income lookup. | +| **3. 2025 lower-AGI control** | Otherwise eligible/no children; E=17,442.65; A=0 | **143**. Do not replace the phase-out base with low AGI or perform a spurious zero-credit AGI phase-in lookup. | +| **4. Statutory factor-default control** | No overriding annual Budget Act factor; otherwise eligible with positive pre-factor credit | **0**, because(a)(2)(B) sets the default factor to zero. Pair with a sourced positive-factor/resource-authorization case after ingest. | + +The `$96` sensitivity is part of case 1, not another benchmark household. It arises from the dated worksheet, not by rounding `$99.46`. The projected arithmetic reproductions passed absolute tolerance `$0.0001`; worksheet minimum checks passed exactly. [Successful calculation](household/household-calculation.json), [raw household](household/scenario_023.json), [calculation script](household/check_household.py), and [detailed adjudication](household/adjudication.md) preserve the evidence. This is one household, not a population simulation. + +**Upstream attribution resolved from code:** required `gh pr view --json title,mergedAt,files` and `gh pr diff` requests for both PRs were attempted but failed to connect. Local PE Git merge/commit objects provide the fallback; do not describe the timestamps below as retrieved GitHub mergedAt fields. + +- [#9363](https://github.com/PolicyEngine/policyengine-us/pull/9363), merge **`7d19aa355671b69fcfb96d18ede3549ea2897d5f`**, commit time **2026-09-01T18:37:55-04:00**, title **“Take smaller of earned-income and AGI credits for CalEITC”**. At that merge, `policyengine_us/variables/gov/states/ca/tax/income/credits/earned_income/ca_eitc.py:64` adds `agi = tax_unit("adjusted_gross_income", period)`; lines 73–74 add `higher_income = max_(earned_income, agi)` and `return min_(credit_for(earned_income), credit_for(higher_income))`. [Saved full diff](household/pr9363-local.diff), lines 252–262, shows these additions. +- [#9542](https://github.com/PolicyEngine/policyengine-us/pull/9542), commit **`7a7ddd7cb5523a1e6e58e1bba46bc37b22a62b79`**, commit time **2026-09-20T14:26:01-04:00**, title **“Fix 2024 CalEITC income boundary and California credit references”**. `parameters/gov/states/ca/tax/income/credits/earned_income/phase_out/final/end.yaml:5` changes **31,950→31,951 for 2024**. Its eligibility module already had a federal-AGI upper-bound check; the PR updates comments/references. It does not change `ca_eitc.py` or add the two-credit comparison. [Saved full diff](household/pr9542-local.diff). + +Thus root_causes.json's attribution of the comparison to #9542 should not be carried into the encoder brief; **9363** is the implementing PR. + +## 7. Risks + +| Risk | Evidence and preparation response | +|---|---| +| Incomplete source unit | Workflow defaults `require_complete_source_unit=True`; its encode step supplies the flag. Finding accounts for the whole section, precise historical/administrative deferrals, and paired exceptions. It forbids deferring the central comparison while calling the task done. | +| Missing annual authority / unsupported test answers | Corpus inventory plus existing source-hold identify the gap. Keep projected 2026, held 2025 and final-law amounts distinct; ingest authoritative annual material before dollar certification. | +| Invalid import input | Helper requires exact jurisdiction and signed-v5 manifests. Empty array is only an admissible placeholder for a still-unresolved dependency strategy. | +| Recursive compile rejection | Pinned-engine code rejects the IRS EIC plural key and the California policy plural keys. No waiver or protected-pin edit is proposed. | +| False assurance from local compiler | Located binary accepts removed syntax. Report both its successful checks and the limits of those checks; rerun with the approved current engine before dispatch. | +| Proof/table grounding rejection | Existing ITIN excerpts contain ellipses. Finding requires exact resolved substrings. July normalized(b) tables collapse repeated cells; compare retained HTML/table identity before asserting missing columns. | +| Wrong baseline or moving ref | `open_pr=true` requires the exact current main tip. Refresh after prerequisite merges; the fixed snapshot command is inspection-only. | +| Collisions / attempt budget | Required open-PR searches in rulespec-us and axiom-encode and the latest 30 targeted runs were attempted but inaccessible. Browser fallback listings were stale. **No collision clearance is claimed.** Repeat same-citation searches and inspect today's runs and failed-attempt budget immediately before dispatch. | + +The task's example failed run 35789753522/artifact could not be downloaded; its rejection details are **task-supplied and unverified here**, not cited as an inspected log. The claim of up to 3.5-hour strict CI is likewise task-supplied; no duration guarantee is made. Actual current PR statuses, fresh main tips, workflow budget and production-engine runtime compatibility remain unverified. + +Checks completed: corpus membership and version comparison; existing module/import/manifest inventory; pinned-workflow and pinned-engine code inspection; twelve existing-module compiles with the legacy binary; one-household baseline diagnostics and independent Decimal arithmetic; published 2025 worksheet row checks; local diffs for both attribution PRs. Initial household helper invocation used a nonexistent diagnostic after valid outputs and failed; it was not counted as a passing check. The independent valid-diagnostic script and a clean rerun of the requested household helper both passed (exit 0); the latter log is [pe-case-valid.log](household/pe-case-valid.log). No new RuleSpec/tests, full microsimulation, commit or external mutation was performed. + +```json +{"lane":"ca-17052","verdict":"BLOCKED","citation":"us-ca/statute/rtc/17052","in_corpus":true,"in_pinned_release":true,"existing_modules":["us-ca/statutes/rtc/17052.yaml","us-ca/policies/income_tax/2026_resident_liability_source_hold.yaml","us-ca/policies/income_tax/pilot_liability_pipeline.yaml","us-ca/statutes/rtc/17041.yaml","us-ca/statutes/rtc/17054.yaml","us/statutes/26/32.yaml"],"blocking_imports":["us:policies/irs/rev-proc-2025-32/earned-income-credit"],"dispatch_command":"gh workflow run targeted-signed-reencode.yml -R TheAxiomFoundation/axiom-encode --ref main -f citation=us-ca/statute/rtc/17052 -f country=us -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 -f pr_base_branch=main -f corpus_ref=8f7d60aaced28ee4252b9237f9d6e02360dc34bc -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca -F review_finding=@scratch/ca-17052/review_finding.md -f source_bundle_json='[]' -f existing_signed_imports_json='[]' -f replace_rulespec_path=us-ca/statutes/rtc/17052.yaml -f open_pr=true","prerequisites":["DO NOT EXECUTE the snapshot command: it is inspection-only, not a certified or approved dispatch.","Ingest official FTB 3514 worksheet/table and applicable 2026 factor, resource authorization and indexed parameters with URL/date/SHA provenance; publish and consume an approved release containing them.","Establish a complete admissible California dependency/source route; direct federal imports are rejected by the inspected workflow and the federal 32 closure contains removed plural syntax.","Reconcile waiver activation #1387, workflow pin #1386, drift stage, and activation/release repin chain; that chain alone does not fill the California source gap.","Refresh exact remote main and all immutable refs after prerequisites; inspect current workflow, same-citation PRs/runs and attempt budget; obtain Max's approval for the signed run."]} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_ca-17076.md b/reference_audit/2026-09-22/verification/v9_axiom_ca-17076.md new file mode 100644 index 00000000..24501f6f --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_ca-17076.md @@ -0,0 +1,222 @@ +# ca-17076 — read-only Axiom preparation + +## 1. Verdict + +**BLOCKED.** No passing signed dispatch can be specified against today's main. The California provisions exist on corpus main, but their September 14 scope is absent from both the currently pinned August 8 release and the planned August 23 successor. Amount-ready encoding also needs the federal IRC text applicable on January 1, 2025; the inspected current corpus roots contain post-OBBB law. Completing the named serial re-pin chain alone does not resolve either issue. + +The statute supports the sandbox's isolated deduction changes for all three households, subject to the stated qualifying-expense/gift assumptions and unchanged benchmark tax context. Independently derived comparison taxes are **$40,522.71 (005), $1,902.59 (022), and $4,429.91 (099)**. Household 099 is entirely a charitable-deduction case; encoding §17076 alone cannot explain its movement. + +Inspected rulespec-us main: `f43dec520dd392bc5333934f56dad7498363e704`; corpus main: `942e138e7a8250c9814e774ac9b8e63008148106`. Both were confirmed against live GitHub. Checks occurred on September 22, 2026 America/New_York (through September 23 UTC). The required fetch was attempted first but denied access to `.git/FETCH_HEAD`; live read-only GitHub checks confirmed the cached identity. + +This lane followed the task's specific read-only prohibition: **no commits, repository edits, new RuleSpec or module-test YAML, encodes, dispatches, PRs, or external messages**. Scratch contains source copies, exact existing-module compile snapshots, arithmetic, and this report. No orchestrator `-o` destination was exposed in the lane context; the report is retained at `scratch/ca-17076/report.md` for collection. + +## 2. Source + +All three target rows exist at corpus main in: + +`data/corpus/provisions/us-ca/statute/2026-09-14-income-tax-chapter-us-ca-sections-4e26e6efabc3f0c7.jsonl` + +| Official provision | Exact canonical citation | JSONL line | Retained source vintage | In pinned release | +| --- | --- | ---: | --- | --- | +| [R&TC 17024.5](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17024.5) | `us-ca/statute/rtc/17024.5` | 41 | Captured 2026-09-14; SB 711 §1, effective 2025-10-01 | No | +| [R&TC 17076](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17076) | `us-ca/statute/rtc/17076` | 120 | Captured 2026-09-14; SB 711 §11, effective 2025-10-01 | No | +| [R&TC 17201](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17201) | `us-ca/statute/rtc/17201` | 227 | Captured 2026-09-14; Stats. 1993 ch. 873 §12 | No | + +Raw official HTML is tracked under `data/corpus/sources/us-ca/statute/2026-09-14-income-tax-chapter-us-ca-sections-4e26e6efabc3f0c7/california-leginfo-sections/RTC-
    .html`. Exact bodies, inventory hashes, and histories are preserved in [source notes](source/notes.md) and [selected rows](source/selected-provisions.json). Collection in September 2026 does not make the relevant SB 711 rules post-freeze law: their effective date was October 1, 2025. + +The current pin is `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`. I checked its tracked scope manifest and the manifest for `us-rulespec-2026-08-23-canada-338-suspension-union`. Neither includes the September California income-tax scope. An exact-citation scan of all tracked California statute JSONL files found no older copy of these target roots. This is release-manifest/inventory evidence; published release-object signatures were not independently reverified. `axiom-locate release` found neither object locally. + +Necessary adjoining authority in the same September scope includes `us-ca/statute/rtc/17049` (line 69), `17250.1` (265), `17250.2` (266), and `17275.5` (282). In particular, [§17250.1(b)](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17250.1) expressly disapplies the increased federal cash-contribution limit. All were checked as source text, not presumed from the fix. + +**Historical federal-source prerequisite:** canonical roots `us/statute/26/67` and `us/statute/26/170` were found only in the July 13 recovery scope, with post-OBBB text. The former now places educator expenses in (g) and permanent suspension in (h); the latter includes the 0.5% charitable floor. They cannot silently stand in for California's January 1, 2025 incorporated law. + +Ingest the official [2024 USC §67 HTML](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partI-sec67.htm) and [2024 USC §170 HTML](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partVI-sec170.htm), or their official annual USC XML, with retrieval date, raw SHA256 and a distinct dated expression. Preserve current federal law. Those historical dependency encodings stop at this ingest prerequisite here; no historical corpus citation or future release/ref is invented. + +## 3. Existing rulespec-us coverage + +There is **no module at 17024.5, 17076, or 17201**, including dotted and slash-normalized variants. These would be new encodings, so `replace_rulespec_path` is empty. No California miscellaneous or charitable deduction computation was found in the scoped main-tree search. + +The entire adjacent `us-ca/statutes/rtc/` inventory was read; none implements this disputed stage: + +| Existing YAML path beneath that directory | What it encodes | +| --- | --- | +| `17014.yaml`, `17016.yaml` | Residence and residence presumptions | +| `17017.yaml` | Geographic United States definition | +| `17029.yaml`, `17034.yaml` | Repeal survival and default effective dates | +| `17038.yaml` | California CPI series | +| `17041.yaml` | Tax-rate/statutory structure; final liability and inflation outputs deferred | +| `17052.yaml` | CalEITC modifications/parameters; final credit deferred | +| `17053/6.yaml` | Prisoner joint-venture wage credit | +| `17054.yaml`, `17054/7.yaml` | Exemption and senior-head-of-household credit rules; final amounts/qualification deferred | +| `17061.yaml` | Unemployment-insurance refund credit | +| `17062/1.yaml` | AMT incorporation, explicitly deferred: `rules: []` | + +The relevant annual and pilot modules are: + +- `us-ca/policies/income_tax/2026_resident_liability_source_hold.yaml`, lines 81–85: “The pinned corpus lacks the FTB-published 2026 indexed standard deduction, complete itemized-deduction surface, and indexed exemption credit inputs required for the annual return.” The named hold starts at line 208 and ends in `formula: true` at line 230. Lines 49–51 say its zero sentinel “is not a claim that the taxpayer owes zero California tax.” +- `us-ca/policies/income_tax/pilot_liability_pipeline.yaml`, lines 30–32: “The caller supplies completed worksheet line 3 taxable income after all deductions”; it uses 2025 estimated-tax schedules and excludes taxable-income construction/final-return liability. +- Nearby federal `us/statutes/26/67/h.yaml` says “no miscellaneous itemized deduction is allowed for any taxable year beginning after December 31, 2017”; its allowance judgment is `formula: false`. This is federal denial, not California coverage. `us/statutes/26/67/e.yaml` computes estate/trust AGI only. There is no 67(a)/(b) individual-floor module in this tree. + +Thus the California modules do not carry an implemented equivalent of the disputed federal-inheritance behavior; they lack the calculation and expose holds. Full inventory, exact quotes and manifest reads are in [coverage notes](coverage/coverage-notes.md). + +**What would lift the hold:** signed California conformity/modifier modules plus applicable-vintage expense classification and the 2% computation address this part of the deduction gap. Charity needs its own §17201/§17250.1 and historical §170 companion. Removing the entire line-208 deduction/exemption hold additionally requires the remaining deduction surface, authoritative 2026 indexed standard-deduction and exemption amounts, and composition from ordinary facts/upstream outputs. The separate CA AGI, rates, credits and ordering holds remain until their own dependencies are complete. Existing annual provenance also requires authorized repair before modern compilation. + +## 4. Import closure + +Prospective source-local §17076 modifier candidate: **`existing_signed_imports_json=[]`**. There are no suitable existing signed-v5 CA deduction modules to name. The empty existing-import closure has no compile blocker; this does **not** make the missing amount calculation ready. + +| Inspected candidate/context | Local current-engine result | Reason to exclude from proposed imports | +| --- | --- | --- | +| Annual CA source-hold module | Fails: removed plural `corpus_citation_paths` | Manual applied-rulespec/v1 manifest; incomplete stage | +| CA estimated-tax pilot | Same failure | Manual v1 manifest; caller-supplied taxable income | +| Federal 67(h) | Compiles, one derived output | Manual v1 manifest; federal denial and wrong jurisdiction | +| Federal 67(e) | Not compiled in this lane | Estate/trust AGI, not this individual's CA deduction; not proposed | +| Federal 26/32 → IRS earned-income-credit module | Imported plural field inspected; no compile run needed for this excluded closure | Unrelated to target and known incompatible provenance | + +Verbatim failures are in [annual compile log](coverage/annual_hold.current.compile.log) and [pilot compile log](coverage/pilot.current.compile.log). Existing files were copied byte-for-byte with `git archive` into a minimal scratch tree, not authored or repaired. + +The locator returned an old 0.1.0 binary that accepted plural fields; its successful compiles are not current readiness evidence. A canonical 0.2.0 binary rejected both modules and accepted 67(h); its SHA256 is `faf4383622f63c64b861e5772b78b00df97efef4a8315b792b25219033bee75e`. Exact binary-to-commit provenance is unverified. Current engine main `6e709eb1ca7ea686263293d932c759d9dee48a4a`, `src/rulespec.rs:810–824`, independently corroborates recursive plural-field rejection. The protected-run log discussed below also demonstrates rejection with the pinned engine SHA. + +The signed workflow helper `src/axiom_encode/prepare_signed_backfill.py:1368–1500` requires same-jurisdiction tracked primary paths, excludes the target/source/dependent paths, and requires applied-rulespec/v5 manifests; the workflow then verifies their signatures. Hence federal modules cannot be casually added to the CA `existing_signed_imports_json`. Plan separate applicable-vintage dependencies and California adapters; final signed paths/symbols must be inspected after they exist. Modifier-only completeness acceptance remains unverified. + +## 5. Dispatch inputs + +**Do not dispatch the following candidate.** It records exact, observed inputs, but fails the active-release prerequisite and has no ready dollar-computation dependencies. There is no honest complete command that can be promised to pass today. The final JSON deliberately leaves `dispatch_command` empty. + +```sh +gh workflow run targeted-signed-reencode.yml \ + -R TheAxiomFoundation/axiom-encode --ref main \ + -f country=us \ + -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \ + -f pr_base_branch=main \ + -f corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106 \ + -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \ + -f citation=us-ca/statute/rtc/17076 \ + -f replace_rulespec_path= \ + -f 'existing_signed_imports_json=[]' \ + -f 'source_bundle_json=[]' \ + -F 'review_finding=@scratch/ca-17076/review-finding.txt' \ + -f open_pr=true +``` + +Ref provenance: + +| Input | Where verified | +| --- | --- | +| `rulespec_ref=f43dec…` | Local origin/main and live GitHub branch endpoint | +| `corpus_ref=942e138…` | Canonical corpus origin/main, exact tracked rows, live branch endpoint | +| `rules_engine_ref=af6e4e…` | rulespec-us `.axiom/workflow-toolchain.toml` and protected run 35789753522 log | +| Workflow `--ref main` | Required workflow branch gate; inspected encoder main `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9` | +| Citation/replacement/imports | Exact canonical row; absent target module; inspected import validator | + +Optional repair, legacy-replacement, dependent-cascade and queue fields remain their empty defaults; retained-successor paths default to `[]`. `source_bundle_json=[]` is deliberately empty because a valid fresh-source bundle with the historical dependencies has not yet been established. This candidate is for a new primary, not replacement of the annual hold. + +The workflow's input block and validation steps were read in this run. Full SHA identities are checked at lines 373–401; corpus/engine must be main ancestors. With `open_pr=true`, `src/axiom_encode/prepare_signed_backfill.py:267–330` requires the exact current PR-base tip, so this recorded rulespec SHA becomes invalid once main advances. Lines 830–876 materialize the release selected by the **rulespec toolchain**, not unrestricted corpus HEAD. The resolver filters active release scopes (`corpus_resolver.py:553–667`). Supplying a newer `corpus_ref` therefore cannot bypass the pin. Encoding runs in `production-signing` and invokes the protected encoder with `--backend openai` (workflow lines 184, 1249–1270). + +The immutable future rulespec SHA cannot be filled before the missing release/activation/re-pin lands. Recheck all refs and collisions, validate actual signed dependency closure, and obtain Max's approval for each eventual signed run. This lane does not request or provide that approval. + +## 6. Review finding text + +The exact prepared `review_finding` payload is also saved in [review-finding.txt](review-finding.txt). It is a prospective brief, not a claim of dispatch readiness. + +Encoding regime (verbatim task requirement): + +RuleSpec content in any `rulespec-*` repository is produced ONLY by the supervised +encoder: `axiom-encode encode --backend codex --apply` (local +supervised runtime, subscription Codex auth via a lane CODEX_HOME; never +OPENAI_API_KEY). Every atomic module carries the encoder's apply manifest under +`.axiom/encoding-manifests/`. Hand-written YAML is never a module — not for a +pilot, not for a demo, not "to avoid API spend". The only hand edits allowed are +repair rounds on the encoder's output (findings file + replay) on repos whose +`run-generated-guard` is off, and composed `module.kind: composition` pipelines, +which are assembled, not encoded. Briefs to lanes must say this verbatim; a brief +that says "hand-author" is wrong. New repos set `run-generated-guard: true`. + +For rulespec-us, landable content comes from the signed path only: the axiom-encode workflow `targeted-signed-reencode.yml` (workflow_dispatch; the `encode` job runs in environment `production-signing`, and its encode step runs `/opt/axiom-verification/axiom-encode encode --backend openai` with the org key). You never write RuleSpec YAML, never write test YAML for a module, and never run a local encode. + +Encode the complete California R&TC 17076 source unit, with California-specific meaning and explicitly identified incorporated-source dependencies. This is a prospective brief: do not dispatch before the source-release and historical-source prerequisites in report.md are satisfied. A modifier-only encoding is not an amount-ready adjudication and must not be represented as clearing the annual liability hold. + +Source coverage: +1. Cover 17076(a): apply the incorporated IRC 67 two-percent floor, subject to California modifications. For an ordinary individual with positive applicable federal AGI A and otherwise deductible miscellaneous expenses E, the amount is max(0, E - 0.02*A). Expense deductibility and membership in the miscellaneous pool must be established from ordinary facts and applicable-source rules, not assumed from an input labeled final allowed deduction. Use the January 1, 2025 IRC vintage selected by 17024.5(a)(1)(Q). The ordinary benchmark employee costs are treated as otherwise qualifying; undocumented deductibility/valuation is an explicit case assumption. +2. Cover every condition of 17076(b): a deduction otherwise allowable under California law, described in 17049, and strictly greater than $3,000 must be excluded from the miscellaneous pool. Exclusion from the floor is not denial of the deduction. Do not infer 17049 qualification merely from the amount. The earlier unrestricted-right inclusion, repayment circumstances, and inventory exception belong to 17049; its alternative tax comparison and carryover mechanics may be precisely deferred to that dependency. +3. Cover 17076(c): the referenced vintage IRC 67(g) suspension does not apply. The incorporated pre-OBBB suspension already ends before 2026. Current federal corpus text instead puts educator expenses in (g) and permanent suspension in (h); never bind the California cross-reference to those current subsection meanings. +4. The related conformity encoding must cover 17024.5(a)(1)(Q), (h)(2)(A), the registered-domestic-partner exception (h)(2)(B), (h)(5)-(6), and (i). Federal AGI, not California AGI, governs these percentage limits. Do not imply this narrow slice completes all of 17024.5: if targeting its root, account for each earlier-date branch (a)(1)(A)-(P), uncodified/technical amendment rules (a)(2)-(3), excluded federal references (b), historic debt rules (c), regulations (d), elections/consents (e)-(f), limitations period (g), and remaining terminology/construction rules (h)(1),(3)-(4),(7)-(8), with implementation or precise source-specific deferrals. +5. A separate charity companion is essential for the supplied household comparisons. 17201(a) incorporates Part VI; (b) and (c) separately incorporate Parts VII and IX and must not disappear when its root is encoded. Use pre-OBBB IRC 170 with no 0.5% floor. Include 17250.1(b)'s express exclusion of the increased cash-gift limit; ordinary qualified gifts remain subject to the applicable 50%/30%/20% categories. If encoding that root, also cover or precisely defer its agricultural-research recipient and conservation termination branches. Do not promise the simplified sandbox charity formula covers all noncash gifts, carryovers, substantiation, or donee categories. + +Permissible precise deferrals: +- To applicable-vintage incorporated IRC 67 dependencies: the full (b) exclusion classification and (d) impairment definition; (c) pass-through regulation mechanics; (e) estate/trust AGI; and (f) coordination. Classification and the floor cannot remain deferred if the module claims final miscellaneous-dollar outputs. +- To 17049: claim-of-right eligibility details, inventory exception, annual tax comparison and carryovers, while preserving the local 17076(b) strict-dollar and exclusion rule. +- To separate incorporated-source/California modules: other deductions and prohibitions under 17201; qualified charitable recipients, payment, valuation, substantiation including 17275.5, carryovers and special gift classes. No final charity amount is available outside an explicitly supported domain. +- To annual composition: 17077 high-income limitation, standard-versus-itemized election, CA AMT add-back, tax brackets, exemptions, credits and ordering. Final tax amounts below are integration expectations holding other benchmark lines fixed, not outputs that a 17076 atomic module can independently certify. The existing annual hold must remain until its other source requirements and provenance repair are complete. + +Required paired positive and blocking tests (request only; this lane authors no test YAML): +- Expenses above, exactly at, and below 2% of applicable federal AGI; zero qualifying expenses. +- 17076(b) qualifying claim-of-right deduction greater than $3,000 versus exactly $3,000; qualifying versus nonqualifying repayment at the same amount, including the 17049 inventory exception. Assert classification as well as dollar results where the dependency supports them. +- Each encoded IRC 67(b) exception against an ordinary cost failing that exception. At minimum pair qualifying impairment-related workplace expenses with otherwise ordinary work expenses. Keep charity outside the miscellaneous pool. Do not import the post-OBBB educator exception into the 2025 vintage. +- Federal AGI different from CA AGI (household 022 supplies a regression), and RDP as-if-spouse AGI versus the ordinary federal-return branch if that exception is encoded. +- Suspension nonapplication in 2025 and 2026 within supported effective periods. No current-federal permanent-denial alias. +- For the charity companion: gifts below the would-be federal floor remain deductible, provided eligibility is met; missing payment/qualification/substantiation blocks the appropriate output. Include cash ceiling and relevant noncash-category boundaries if claimed. +Every companion case must assign every local #input, including all false facts. Proof excerpts must be verbatim substrings of the precisely resolved source. Never alter toolchain pins, workflow pins, CODEOWNERS or waivers to make this run pass. + +Worked cases: all are California TY2026. Expenses/gifts below are accepted as otherwise deductible, qualified, paid and substantiated; unlisted tax-preparation fees are zero. All relevant gift ceilings are nonbinding. Federal AGI and other unchanged tax context come from the existing independently verified benchmark traces. Independent decimal arithmetic applies the statute-derived deduction changes to that context; it does not certify unrelated IRA, indexing, filing-status or AMT rules. + +Case 1, scenario_005: joint, ages 35/33, no children. Federal and CA AGI $536,764.50. Employee expenses $13,847.8232421875; cash gifts $1,069.7646484375; noncash gifts $1,503.7646484375. Floor $10,735.29; CA miscellaneous deduction $3,112.5332421875; CA charity $2,573.529296875 (frozen charity and misc both zero). Total restored deductions $5,686.0625390625. Existing $1,941.21 high-income limitation remains unchanged; marginal rate is 9.3%. Frozen state tax before refundable credits $41,051.511719; sandbox $40,522.707031. Independent result $40,522.7079028671875, or $40,522.71. Statute supports the corrected treatment and isolated corrected value, not the frozen treatment. Miscellaneous deduction alone would yield approximately $40,762.05; charity is needed for the whole change. + +Case 2, scenario_022: single, age 77, no children; preserve explicit single filing status despite a separate surviving-spouse flag. Federal AGI $109,149.1640625; CA AGI $91,524.4140625. Employee expenses $8,350.5439453125; cash gifts $14,693.412109375; noncash gifts $700. Floor $2,182.98328125; CA miscellaneous deduction $6,167.5606640625. CA charity $15,393.412109375; removing the federal floor restores another $545.7458203125. Restored deductions $6,713.306484375; unchanged marginal rate 8%. Frozen $2,439.650146; sandbox $1,902.585449; independent $1,902.58562725, or $1,902.59. Statute supports corrected treatment and isolated corrected value. Applying the floor to CA AGI would be wrong. Miscellaneous alone yields approximately $1,946.25. + +Case 3, scenario_099: joint, adults 42/41, children 9/4. Federal/CA AGI $159,585.046875; employee expenses $1,262.6470947265625; cash gifts $11,932.94140625; noncash gifts $688.058837890625. Floor $3,191.7009375 exceeds expenses, so both the frozen and corrected miscellaneous deduction are correctly zero. CA charity $12,621.000244140625; restored charity floor $797.925234375. At unchanged 8%, tax falls $63.83401875. Frozen $4,493.743164; sandbox $4,429.909180; independent $4,429.90914525, or $4,429.91. Statute supports the bundled corrected treatment and isolated corrected tax, but 17076 by itself changes nothing here: this is a charity conformity case. + +Case 4, boundary family: ordinary individual, federal AGI $100,000, qualifying ordinary miscellaneous costs $5,000 => $3,000 after the floor; costs $2,000 => $0; costs $1,999 => $0. An otherwise allowable, 17049-qualified $4,000 repayment is excluded from this floor; an otherwise allowable ordinary $4,000 cost without a qualifying exception yields $2,000. At exactly $3,000 the specific 17076(b) exclusion predicate is false; determine any actual deduction under its remaining applicable rules, rather than inventing a denial. + +These cases adjudicate the isolated provision changes. Neither set of full benchmark returns is independently certified by this atomic encoding brief. Axiom currently has no runnable CA miscellaneous/charitable deduction result for them; held zero sentinels are not statutory tax values. + +## 7. Risks + +| Rejection or review risk | How this package addresses it | +| --- | --- | +| Target exists only outside active corpus release | Names exact absent scope and requires a release plus separate gated re-pin; newer corpus checkout alone is insufficient | +| Wrong federal vintage/subsection | Requires retained pre-OBBB sources; distinguishes vintage 67(g) from current 67(g)/(h) | +| Incomplete source unit | Explicitly covers all three 17076 branches and names clause-specific deferrals/dependencies for larger related roots | +| Missing positive/blocking exception tests | Specifies floor, claim-of-right, impairment, AGI/RDP, suspension, and charity boundary pairs; all local inputs, including false ones, assigned | +| Import compile/signature failure | Empty existing-import candidate; excludes manual-v1 CA/federal modules and identifies plural-field failures | +| Misstated household scope | Includes all three moved cases and separates miscellaneous-only effects from charity; no full-return or held-zero parity claim | +| Annual hold prematurely cleared | Identifies remaining indexed amounts, deduction coverage and separate AGI/rate/credit holds | +| Stale PR base | Exact-base-tip rule means refs must be rebuilt after prerequisites land | +| Oversized successor release | Current workflow caps release downloads at 16 MiB. Open encoder #1675 proposes 64 MiB; whether that is required depends on the eventual release object size, not merely its name | + +**Collisions and serial chain.** Live exact-citation open-PR searches across rulespec-us and axiom-encode returned no match for 17076, 17024.5 or 17201. I read 30 recent targeted-run records and all fetched runs covering September 22 EDT through the check; none targeted these citations. This is a snapshot, not a reservation. Saved evidence: [collision check](workflow/collision-check.json). + +[Rulespec-us #1387](https://github.com/TheAxiomFoundation/rulespec-us/pull/1387) waiver activation and [#1386](https://github.com/TheAxiomFoundation/rulespec-us/pull/1386) workflow-pin update were open; #1384 was already merged at the inspected main. Their PR descriptions confirm the serial order and intended August 23 re-pin. Later drift staging/activation remains prospective in the supplied plan, not a verified merged SHA. That entire chain still lacks this September California scope. [Encoder #1675](https://github.com/TheAxiomFoundation/axiom-encode/pull/1675) is relevant if a larger union is selected. [Encoder #1665](https://github.com/TheAxiomFoundation/axiom-encode/pull/1665) concerns the unrelated 26/32 legacy-import repair; this proposed empty closure does not depend on it. + +**Observed failure mode.** Protected [run 35789753522](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522) failed with the verbatim log message that imported `us:statutes/42/416/l` “declares removed plural `corpus_citation_paths`; every source/proof node must declare exactly one singular `corpus_citation_path`.” Its actual immutable input values and failure lines were read and saved in [run excerpts](workflow/run-35789753522-excerpts.log). The artifact listing was read; download produced a remote file reference, but local DNS prevented retrieving the ZIP. Its `issues.json` was therefore **not inspected**. Additional complete-source-unit rejection details from the task are not independently verified from that artifact here; the complete-source requirement itself is verified in workflow/helper code. + +**Checks and limitations.** Three independent Decimal-arithmetic comparisons passed, each within $0.001 of the supplied sandbox sweep; [script](households/hand_arithmetic.py) and [results](households/hand_arithmetic.json) are retained. Sweep rows 75–77, scenario JSON, root-cause record, fix, earlier coverage, and independent verification report were read. An attempted new three-household PE trace stalled in system initialization and was canceled; no new PE simulation result is claimed. AGIs, existing limiter/rates and AMT-zero context are explicitly from the supplied verifier's traces, while this lane's legal deductions and decimal deltas were independently derived. The annual and pilot existing-module compile failures and federal 67(h) compile success were observed in this run. No population computation ran. + +[Binding issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was read through the GitHub connector after shell `gh` networking failed. Its proof, companion-test, signed-generation and decrement-only-waiver rules are reflected in the brief; no waiver, authority-file change or manual encoding is proposed. Remaining uncertainties are final historical-source representation, future release/ref identities, actual signed dependency outputs, encoder acceptance, and complete annual-return authority. + +```json +{ + "lane": "ca-17076", + "verdict": "BLOCKED", + "citation": "us-ca/statute/rtc/17076", + "in_corpus": true, + "in_pinned_release": false, + "existing_modules": [ + "us-ca/policies/income_tax/2026_resident_liability_source_hold.yaml", + "us-ca/policies/income_tax/pilot_liability_pipeline.yaml", + "us/statutes/26/67/h.yaml", + "us/statutes/26/67/e.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Ingest provenance-preserving historical IRC 67 and 170 expressions applicable on 2025-01-01; retain current federal expressions.", + "Publish a signed corpus release including the September 14 California income-tax scope and required historical federal authority; current August 8 and planned August 23 releases omit the CA targets.", + "Complete applicable serial waiver/workflow activation steps and a dedicated rulespec-us re-pin to the source-complete release; evaluate the release-size cap before selecting a large union.", + "Establish signed applicable-vintage classification/calculation dependencies and California adapters; inspect actual signed paths and full closure before specifying an amount-ready dispatch.", + "Include a separate charity conformity companion for all r11 movements, especially scenario_099.", + "Refresh immutable refs after main advances, verify exact PR base, recheck collisions, and obtain Max's approval for every signed run.", + "To lift the entire annual deduction/exemption hold, also complete remaining deductions and authoritative 2026 indexed amounts and repair the annual module's legacy provenance." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_ca-mpp-snap.md b/reference_audit/2026-09-22/verification/v9_axiom_ca-mpp-snap.md new file mode 100644 index 00000000..f16bee01 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_ca-mpp-snap.md @@ -0,0 +1,133 @@ +# ca-mpp-snap: existing Axiom artifact adjudication + +## 1. Verdict + +**NOT-NEEDED for a new numerical encoding of the three assigned rounding rules.** California net-income rounding is already encoded at **MPP 63-503.31**, the parent of the assignment's .311. The existing .31 rule returns **$877** for scenario_023; feeding that output into existing 7 USC 2017(a) returns **$34/month**. Existing federal statute and regulation modules both return the FY2026 small-household minimum of **$24**. + +This verdict concerns the provision encodings. The broader California benefit composition does not import .31 or .311 and is **not certified here as a complete CalFresh calculator**. It also has current-engine schema blockers described below. No duplicate atomic encode is warranted; integration and legacy-provenance migration are separate work. + +Read rulespec-us `origin/main` **f43dec520dd392bc5333934f56dad7498363e704**, independently confirmed as live GitHub main. The requested fetch failed with `cannot open '.git/FETCH_HEAD': Operation not permitted`; the connector supplied read-only remote verification. Standalone rulespec-us-ca local `origin/main` is **321ed395ac076b580f20af351eb476056e018779**. Its .31/.311 files are byte-identical to the tested monorepo files. No commit, push, branch, external finding, workflow dispatch, local encode, or new RuleSpec/test YAML was made. Scratch includes byte-for-byte existing-source copies and JSON execution requests. + +## 2. Source + +Official source: [CDSS Food Stamp Manual, chapter 63-503, fsman06.docx](https://www.cdss.ca.gov/Portals/9/Regs/Man/Fsman/fsman06.docx?ver=wrx9nKY4BHRH9SGFqVVMkg%3D%3D), linked by the [current official manual index](https://www.cdss.ca.gov/inforesources/letters-regulations/legislation-and-regulations/calworks-calfresh-regulations/calfresh-regulations). The DOCX retained in corpus was read directly, extracted into `official-fsman06.txt`, and its SHA-256 verified: + +`eb0eb8a58a8371c8f56e80a3bf49d561e01eb0ffcbae2567a2475d7abb8f4e75`. + +The controlling rounding provision is **`us-ca/regulation/mpp/63-503.31`**. The ordinary net-income arithmetic is **`us-ca/regulation/mpp/63-503.311`**. These are state regulations, not statutes. Both exist in canonical axiom-corpus `origin/main` **942e138e7a8250c9814e774ac9b8e63008148106**: + +- Provision file: `data/corpus/provisions/us-ca/regulation/2026-07-13-recovery.jsonl`, records 527–528. +- Retained source: `data/corpus/sources/us-ca/regulation/2026-07-13-recovery/official-documents/us-ca-mpp-63-503`. +- Provenance: sibling `provenance/us-ca-mpp-63-503.json`, fetched `2026-07-13T23:53:00Z`; provision `source_as_of=2026-07-13`. +- Printed vintage: .31 appears on page 283, Manual Letter FS-04-07 effective July 1, 2004; .311 appears on page 284, FS-06-04 effective November 1, 2006. These dates differ from the 2026 snapshot date. + +MPP .31 retains cents through the calculation, then rounds the final net income down below fifty cents and up at fifty cents. CDSS independently reiterates that reading in [ACIN I-25-11, April 29, 2011](https://cdss.ca.gov/lettersnotices/entres/getinfo/acin/2011/I-25_11.pdf). This derivation uses the official regulation itself, not the sandbox docstring or a ParaRegs summary. + +**In pinned release: yes, by verified release selection.** rulespec-us pins `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA-256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`. Its tracked selector includes `us-ca/regulation` version `2026-07-13-recovery`; the two records also exist at the selector's historical corpus commit **1f87488a9fa7def07988cafc6de254811a5b1d28**. No ingest or release expansion is needed. The release artifact was not locally materialized, so this is selector-and-historical-content verification, not fresh cryptographic verification of its distributed bytes. See `dispatch/report.md`. + +## 3. Existing rulespec-us coverage + +`git ls-files` found .31 and .311 in standalone rulespec-us-ca; `git ls-tree origin/main` confirmed both under rulespec-us/us-ca. Relevant immutable source copies are under `rulespec-us/` and `federal/rulespec-us/` in this scratch directory. + +| Module | What was read and tested | +|---|---| +| `us-ca/regulations/mpp/63-503/31.yaml` | Correct nearest-dollar net-income rule, line 125: `floor(net_monthly_income_calculated_with_exact_dollars_and_cents_including_shelter_and_medical_costs + net_monthly_income_round_up_cent_floor)`. Parameter is `0.50` at line 77. No imports. Four rounding cases pass. | +| `us-ca/regulations/mpp/63-503/311.yaml` | Ordinary income/deduction arithmetic. Line 294: `if net_monthly_income_determination_steps_apply: max(0, monthly_income_after_homeless_shelter_deduction - excess_shelter_deduction) else: 0`. It deliberately has no final rounding; .31 supplies that separate rule. Household arithmetic and elderly/disabled blocking case pass. | +| `us-ca/regulations/mpp/63-503/312.yaml` | Neighboring elderly/disabled route, including medical and uncapped excess-shelter deductions. Final formula is `if household_elderly_or_disabled_net_monthly_income_calculation_applies: max(0, monthly_income_after_standard_dependent_care_medical_and_homeless_deductions - excess_shelter_cost_for_elderly_or_disabled_household) else: 0`. Read, not executed; scenario_023 is outside this USDA classification. | +| `us-ca/regulations/mpp/63-503/321.yaml` | Gross/net eligibility comparisons, including monthly and quarterly reporting. Does not implement net-income rounding. Read, not executed. | +| `us-ca/regulations/mpp/63-503/324.yaml` | Table-allotment interface: `max(0, coupon_allotment_table_monthly_allotment_for_household_size_and_net_income)`. It accepts an allotment boundary value; it does not compute .31. | +| `us-ca/policies/cdss/snap/fy-2026-benefit-calculation.yaml` | Composition for eligibility, deductions and federal allotments; imports .324 but neither .31 nor .311. Its 18-module closure contains no .31 rounding rule. Successful local compilation does not establish a complete household result or current-runtime compatibility. | +| `us/statutes/7/2017/a.yaml` | Line 128: `floor(max(0, snap_maximum_allotment - snap_household_food_contribution))`. Lines 153–156 apply `floor((snap_one_person_thrifty_food_plan_cost * snap_minimum_allotment_rate) + 0.5)` to household sizes ≤2, otherwise zero. | +| `us/regulations/7-cfr/273/10.yaml` | Line 425 uses the same nearest-dollar minimum. Line 445 uses either `ceil(snap_net_monthly_income * snap_allotment_net_income_reduction_rate)` before subtraction or `floor(snap_maximum_allotment - (snap_net_monthly_income * snap_allotment_net_income_reduction_rate))`. Line 465 separates initial-month issuance/minimum rules. | + +Both federal modules return `floor(298 × .08 + .5) = 24`, consistent with the [official USC §2017(a)](https://www.govinfo.gov/content/pkg/USCODE-2024-title7/pdf/USCODE-2024-title7-chap51-sec2017.pdf), [CFR §273.10(e)(2)(ii)](https://www.govinfo.gov/content/pkg/CFR-2025-title7-vol4/pdf/CFR-2025-title7-vol4-sec273-10.pdf), and [USDA FY2026 minimum table](https://fns-prod.azureedge.us/sites/default/files/resource-files/snap-fy26MinimumAllotments.pdf). The 2024 USC and 2025 CFR publication vintages were actually read; live preliminary USC/eCFR retrieval failed. No later statutory change is inferred from that failure. + +These atomic formulas do not contain the three questioned numerical behaviors. This is an Axiom coverage assessment and requested comparator adjudication, not an external-oracle finding. + +## 4. Import closure + +No new target is proposed; candidate `existing_signed_imports_json` is **`[]`**. MPP .31 and .311 have no imports and both compile and execute locally. Neither depends on 26/32 or its EIC policy import. + +The federal closure union comprises `us:statutes/7/2017/a`, `us:statutes/7/2014/e/6/A`, `us:statutes/7/2014/e/2`, `us:statutes/7/2014/e/2/B`, `us:policies/usda/snap/fy-2026-cola/maximum-allotments`, `us:regulations/7-cfr/273/10`, `us:policies/usda/snap/fy-2026-cola/deductions`, and `us:statutes/7/2012/j`. Every module loads in the two successful local compiles, and none declares plural citation paths. Full sources, hashes and closure are retained in `federal/summary.json`. + +The separate California composition closure is enumerated in `composition_closure.json` (18 modules). All load together in the **located local binary**, yielding 114 outputs. However, five modules declare removed `corpus_citation_paths`: + +- `us-ca:policies/cdss/snap/fy-2026-benefit-calculation` +- `us-ca:policies/cdss/snap/modified-categorical-eligibility` +- `us:policies/usda/snap/fy-2026-cola/income-eligibility-standards` +- `us:regulations/7-cfr/273/7` +- `us:regulations/7-cfr/273/24` + +At verified current engine main **6e709eb1ca7ea686263293d932c759d9dee48a4a**, `src/rulespec.rs:819–823` rejects that key recursively, and line 1086 applies the validator to loaded imports. Thus these are **code-derived current-engine blockers**, not a reproduced failure of the older local executable. We did not build a new engine. Local binary SHA-256 is `674ca6e70afdccb59c3d6847933bc24b4590105e49db54790f2dcd0bdbbe32d7`; its exact build commit is unverified. + +Existing apply provenance is not signed-v5 eligibility: .31's manifest is `axiom-encode/applied-rulespec/v1`, generated May 14, 2026, and its source hash matches `1af19100ca58a0983e81cb624f7361e535400324670318d10ce757d1d8e9785e`. The two federal manifests are also v1. The inspected workflow import validator requires v5 manifests, tracked atomic YAML, and the target's primary jurisdiction; a California target cannot simply list federal `us/` leaves in `existing_signed_imports_json`. Local numerical validity does not establish signed-import admissibility. + +## 5. Dispatch inputs + +**Not applicable: no encode command or workflow dispatch is proposed.** Providing a runnable duplicate-encode command would contradict the tested NOT-NEEDED verdict. `dispatch_command` is empty below. + +For traceability, the workflow and validation code were read at encoder main **5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9**; source snapshot is corpus **942e138e7a8250c9814e774ac9b8e63008148106**; rulespec main and current engine identities are recorded above. These are observed immutable identities, not an approved dependency set or a future dispatch recipe. The workflow requires full SHA inputs, origin/main ancestry for corpus/engine, and current main for `open_pr=true`; saved refs must be refreshed before any future approved run. + +The serial re-pin chain is **not needed to make these provisions available**: they are already in the pinned release. #1387 activation and #1386 workflow pin remain open. A separate integration/signing repair must coordinate with that chain, resolve the named schema/provenance blockers and recheck main. This lane neither changes pins nor requests approval for a signed run. + +## 6. Review finding text and worked adjudication + +No `review_finding` is submitted because no new atom is needed. The following review text is suitable for the orchestrator's disposition and any later, separately scoped integration work: + +> Preserve and use the existing MPP 63-503.31 final-net-income rounding rule. Resolve the official .31 source unit, not .311 alone: .311 computes the ordinary net amount, while .31 carries cents through the calculation and rounds the final net amount half up. Route the resulting net income into federal allotment computation before rounding the contribution up or the allotment down. Preserve the nearest-dollar 8% federal minimum. Defer eligibility, quarterly averaging, individual income exclusions, deduction amounts, proration and FY2027 parameter determination to their separately identified provisions; these boundary checks do not adjudicate those inputs. Include paired .49/.50 and whole-dollar tests, ordinary versus elderly/disabled .311 applicability, homeless deduction versus excess-shelter treatment if that arithmetic is changed, and regular versus initial-month minimum cases if the benefit composition is changed. Assign all local input facts, including false. A .311 re-encode must not claim complete-source-unit coverage while silently dropping its QR/PB or exception branches. Use verbatim proof excerpts from each resolved unit and existing supervised artifacts; do not hand-author a new atomic module. + +**Complete California reference inventory.** Reading every SNAP row in `reference_outputs.csv` and matching `scenarios.csv` gives: + +| California household | Frozen annual SNAP | +|---|---:| +| scenario_005 | $0 | +| scenario_022 | $0 | +| **scenario_023** | **$461.3397216796875** | +| scenario_031 | $0 | +| scenario_099 | $0 | + +Thus **scenario_023 is the only California household with nonzero reference SNAP and the only r27 moved household**. Selection and facts are retained in `households/selection.json`; audit sweep comparison and fresh household execution are in `households/`. + +**Worked case 1 — scenario_023.** One person, age 28; annual employment $17,442.646484375 and taxable 403(b) distributions $8,000; rent $2,000/month; utility allowance $663/month. For the tested boundary, USDA elderly/disabled classification is false (the separate generic disability fact is true), so .311 applies. Earned-income deduction is 20%; standard deduction $209; shelter deduction capped at $744; dependent-care, child-support, medical and homeless deductions zero. Eligibility is held at the benchmark's true value; this is not an independent eligibility determination. + +Using exact annual facts, `17442.646484375/12 × .8 + 8000/12 − 209 − 744 = 876.509765625`, or $876.51 to cents. MPP .31 yields **$877**, not $876. Then `ceil(.30 × 877)=264`; `298−264=34`. Equivalently `floor(298−263.10)=34`. The $24 minimum does not bind. The existing Axiom .31 result fed into the existing federal module returns **$34**, recorded in `joined-ca-federal.response.json`. + +| Quantity, January–September FY2026 | PolicyEngine 1.755.4 | Isolated r27 sandbox | Independent legal calculation / Axiom | +|---|---:|---:|---:| +| Raw net-income calculation | $876.5096435546875 | Same before new rounding | About $876.51 | +| Net used for 30% contribution | $876 after floor | $877 | **$877** | +| Monthly benefit | $35.199981689453125 | $34.899993896484375 | **$34.00** after federal whole-dollar rule | +| Federal one/two-person minimum | $23.84000015258789 | Unchanged by r27 | **$24.00** | + +**Adjudication:** MPP supports the corrected r27 **net income**, not the original floored net. For the final benefit, the governing rules support **neither $35.20 nor isolated-r27 $34.90**: the complete calculation is $34. The combined r26+r27 calculation therefore matches the law at this FY2026 boundary; r28's minimum correction does not bind for this household. + +The frozen annual $461.3397216796875 is reproduced exactly and combines nine $35.199981689453125 months with three $48.179962158203125 months under original FY2027 extrapolated parameters. Do not divide it by 12 to infer a FY2026 month. The audit's FY2026-held annual baseline is $422.3997802734375 and isolated-r27 comparator $418.7999267578125; the complete corrected FY2026-held calculation is **$408**. That $408 is the benchmark parameter-holding convention, **not a claim about actual October–December 2026 entitlement**. January–September under the tested law totals $306; actual fourth-quarter amounts require FY2027 parameters. + +Existing .311 with extracted decimal intermediates returns $876.5097351074219 rather than PE's $876.5096435546875 because the latter uses floating-point calculation/summation. Both round to $877. All 13 local .311 inputs were supplied, including false exception flags; the zero care-cap placeholder cannot affect zero care expenses. + +**Worked case 2 — half-dollar boundary.** Exact calculated net $123.49 → .31 output **$123**; $123.50 → **$124**; exactly $123 → **$123**. All three JSON cases executed successfully. Changing the .311 elderly/disabled fact to true makes that ordinary-route output zero; that is an applicability sentinel, not a finding that an elderly household has no income. + +**Worked case 3 — minimum binds.** Eligible one-person regular-month household, FY2026 one-person maximum $298 and net $971: ordinary benefit `floor(298−.30×971)=6`; minimum `floor(.08×298+.5)=24`; issued regular-month benefit **$24**. Both federal modules execute to $24; both permitted contribution/allotment rounding elections agree. The statute supports the rounded comparator, not $23.84. Initial-month rules are a separate branch. + +**Executed checks:** 784 federal derived assertions pass (9 existing companion cases, 15 additional JSON boundary requests, and 171 saved household-month replays for all 19 households from the earlier parity check). Four California rounding assertions, two .311 arithmetic/applicability assertions and the joined $34 benefit assertion pass. The 171 replays use the earlier saved PE boundary facts; scenario_023's fresh facts are independently reproduced here. These are provision-boundary checks, not population runs or full eligibility validation. Reproduction commands: + +```sh +ruby scratch/ca-mpp-snap/check_ca.rb +PYTHONDONTWRITEBYTECODE=1 /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/.venv-pe1755/bin/python scratch/ca-mpp-snap/federal/check.py +``` + +The household reproduction command and exact original/held/fixed values are retained in `households/evidence.md`. No PolicyBench source file was modified. + +## 7. Risks + +- **Wrong source unit:** dispatching .311 to add rounding misses controlling .31 and duplicates existing coverage. The source correction and tested module prevent that mistake. +- **Composition coverage:** existing atomic correctness does not imply the broad CalFresh composition applies .31. Its missing import/connection and five removed-field modules require separate integration/provenance work. +- **Runtime mismatch:** older local engine accepts plural metadata; current source rejects it. Current-engine behavior here is code-derived, not executed. No general current signed-CI pass is claimed. +- **Legacy manifests:** correct existing output is not proof of signed-v5 import admissibility. Do not use the v1 CA/federal artifacts as if the workflow import validator accepted them. +- **Complete-source-unit and grounding:** future edits must cover or precisely defer all source branches and retain paired exception tests and verbatim path-anchored proof text. No waiver, pin, workflow or CODEOWNERS change is proposed. Binding [issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was read and saved as `issue39.md`. +- **Collisions:** searched all 102 open rulespec-us and 24 open encoder PR titles/bodies, plus 82 encoder runs created September 22 UTC (16 targeted signed runs). No direct MPP .31/.311 collision found. Related [rulespec-us #1363](https://github.com/TheAxiomFoundation/rulespec-us/pull/1363) changes the federal SNAP modules, parameter leaves and California categorical-eligibility dependency; it does not change .31/.311. Its proposed repairs are not on main. This was not a changed-file audit of every open PR. +- **Annual interpretation:** the isolated r27 sandbox alone still produces cents; the federal rule must also apply. The FY2026-held annual comparator cannot validate actual FY2027 benefits. Generic disability and USDA disability were kept distinct; eligibility remains an input boundary. + +```json +{"lane":"ca-mpp-snap","verdict":"NOT-NEEDED","citation":"us-ca/regulation/mpp/63-503.31","in_corpus":true,"in_pinned_release":true,"existing_modules":["us-ca/regulations/mpp/63-503/31.yaml","us-ca/regulations/mpp/63-503/311.yaml","us/statutes/7/2017/a.yaml","us/regulations/7-cfr/273/10.yaml"],"blocking_imports":["us-ca:policies/cdss/snap/fy-2026-benefit-calculation","us-ca:policies/cdss/snap/modified-categorical-eligibility","us:policies/usda/snap/fy-2026-cola/income-eligibility-standards","us:regulations/7-cfr/273/7","us:regulations/7-cfr/273/24"],"dispatch_command":"","prerequisites":["No new encode is needed for the tested rounding atoms.","For separate CalFresh integration: connect existing MPP .31, repair current-engine plural-field blockers, establish admissible signed provenance, coordinate with the serial re-pin chain, and reverify live main before any Max-approved signed dispatch."]} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_id-63-3022p.md b/reference_audit/2026-09-22/verification/v9_axiom_id-63-3022p.md new file mode 100644 index 00000000..a93ce391 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_id-63-3022p.md @@ -0,0 +1,215 @@ +# Idaho health-insurance deduction: signed encoding preparation + +Lane `id-63-3022p`; September 22, 2026. Read-only preparation for Max Ghenis. This is an Axiom coverage package, not a finding against an oracle. No repository content was edited, committed, pushed, or dispatched. Only scratch evidence/report files were written, including an unchanged existing module snapshot for a compile check. + +## 1. Verdict + +**BLOCKED.** The statute is in corpus main, but neither rulespec-us's current signed-release selection nor the specified Canada re-pin endpoint contains it. A checked-in September 14 release manifest selects the needed source; its published signed archive, release-content SHA256, and activation could not be verified. The external prerequisite is to verify/activate an immutable containing release and land its approved rulespec-us pin, then freeze the resulting exact main SHA. The named #1387/#1386/drift/Canada chain alone is insufficient. + +The independent reading supports **the corrected value for both households**, conditional on the supplied personal premiums being after-tax taxpayer payments and other return components remaining fixed: scenario_007 **$645.54**, scenario_053 **$2,170.28**. The independently established deductions are $2,080 and $5,000; this is not certification of every component of either return. + +Inspected rulespec-us local `origin/main`: **`f43dec520dd392bc5333934f56dad7498363e704`**. The requested fetch failed because `.git/FETCH_HEAD` was not writable. GitHub CLI reads also failed to connect. Therefore this report identifies verified local snapshots, not a refreshed claim about today's live main or active runs. See [access checks](access-checks.md). No passable immutable future dispatch command can truthfully be supplied yet; section 5 preserves the complete current input set as explicitly ineligible. + +## 2. Source + +Official text: [Idaho Code §63-3022P, Health insurance costs](https://legislature.idaho.gov/statutesrules/idstat/Title63/T63CH30/SECT63-3022P/). + +- Exact canonical corpus citation: **`us-id/statute/63-3022P`**, including capital `P`. +- Canonical corpus local `origin/main`: **`942e138e7a8250c9814e774ac9b8e63008148106`**. +- Tracked provision: `data/corpus/provisions/us-id/statute/2026-09-14-income-tax-chapter-us-id-title-63-chapter-30.jsonl`. +- Retained official artifact: `data/corpus/sources/us-id/statute/2026-09-14-income-tax-chapter-us-id-title-63-chapter-30/idaho-statutes-section-html/title-63/chapter-30/63-3022P.html`; substantive text at line 529, history at line 532. +- Format: `idaho-statutes-section-html`; `source_as_of` and `expression_date`: **2026-09-14**. Retained amendment history ends with 2003 chapter 10, section 2. The snapshot date is not a new effective date. +- Source artifact SHA256: **`c413e98e9cb2e5a63dcb20a351046310327c1d5d4a0ee8c54203e9373ca0d0f6`**, independently recomputed and matching the inventory/ingest manifest. See [provision](provision.json), [official HTML snapshot](source-63-3022P.html), and [provenance](corpus-63-3022P-provenance.json). + +No statute ingest is needed. `axiom-locate corpus-file 63-3022P` missed it; `git ls-tree`/`git show` in the canonical checkout established its presence. The live Legislature fetch failed, so the retained official artifact supplies the exact text read here. + +Release evidence, read from `manifests/releases/` in corpus main: + +| Release | Idaho statute scope selected | Contains §63-3022P? | +|---|---|---| +| `us-rulespec-2026-08-08-obbb-alien-snap` — current consumer pin | `2026-07-31-id-title-63-chapter-30-successor` | No | +| `us-rulespec-2026-08-23-canada-338-suspension-union` — specified chain endpoint | Same July 31 scope | No | +| `us-rulespec-2026-09-14-wave4-r2-union` — checked-in later manifest | `2026-09-14-income-tax-chapter-us-id-title-63-chapter-30` (lines 1861–1864) | Its selected source includes P; published signed release unverified | + +The July scope has only the five substantive sections 63-3022D, 63-3022E, 63-3024, 63-3024A, and 63-3025D. Current `.axiom/toolchain.toml` pins release-content SHA256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`. The September manifest was introduced at corpus commit `8d77a9d2aa7edbab056365273a20bd884a90adb2`; its file hash is not a release-content hash. Both `axiom-locate release` queries for the September wave4 variants returned no local artifact. + +Administrative corroboration actually read: [March 2, 2026 Form 40/39R instructions](https://tax.idaho.gov/wp-content/uploads/forms/EIN00046/EIN00046_03-02-2026.pdf), Form 39R line 18 and worksheet, and [current IDAPA 35.01.01](https://proddfmmainsa.blob.core.windows.net/dfm-admin-website/rules/current/35/350101.pdf), Rule 193. These explain excluded prior-accounting amounts and the Idaho standard-deduction treatment. The instructions are for tax year 2025; they supply no independently verified 2026 indexed constants. Their corpus/release inclusion was not established, so they must not be invented as proof sources for the atomic statute encoding. + +## 3. Existing rulespec-us coverage + +At the inspected rulespec-us SHA, the target primary `us-id/statutes/63-3022P.yaml`, companion, and manifest are absent. The Idaho statute tree has only the following five statute modules and their tests. The two nearby income-tax policies are also listed. None computes this premium deduction; no existing monetary implementation of P was found to compare with the audit's deduction omission/election issue. + +| Module | Actual coverage and relevant source excerpt | +|---|---| +| `us-id/statutes/63-3022D.yaml` | Child/dependent care. Summary lines 7–8: “employment-related expenses paid during the taxable year, not to exceed $12,000.” | +| `us-id/statutes/63-3022E.yaml` | Elderly/developmental-disability household deduction. Summary: “$1,000 for each qualifying individual age 65 or older or with developmental disabilities”. | +| `us-id/statutes/63-3024.yaml` | Ordinary tax from supplied taxable income, filing facts, and inflation factor. Lines 7–10: “The tax is 5.3% of taxable income over $2,500, or over $5,000 for joint returns”. No premium calculation. | +| `us-id/statutes/63-3024A.yaml` | Grocery credit and election. Lines 7–13: “refundable grocery tax credits for the taxpayer, spouse, and dependents.” No premium calculation. | +| `us-id/statutes/63-3025D.yaml` | Elderly/developmental-disability household payment and self-filer credit. Lines 7–11: “$100 for each such member, capped at three payments per calendar year.” | +| `us-id/policies/income_tax/pilot_liability_pipeline.yaml` | Caller-supplied income, subtraction, rate, threshold and credit. Lines 32–39 describe an “unverified caller-supplied subtraction”; line 56: `max(0, id_pit_pilot_adjusted_gross_income - id_pit_pilot_supplied_personal_exemption)`. It does not derive the premium deduction. | +| `us-id/policies/income_tax/2026_full_year_resident_source_hold.yaml` | Explicit unavailable computations. Lines 42–44 describe zero sentinels, not legal zero claims. Lines 151–167 declare the income/modification hold below. | + +Relevant source-hold lines, unchanged on main: + +```text +151 - name: id_pit_2026_income_and_modification_source_hold_applies +156 source: Missing substantive official 2026 Idaho income-base, addition, and subtraction authority in the pinned corpus +165 - effective_from: '2026-01-01' +166 effective_to: '2026-12-31' +167 formula: true +``` + +**What would lift it:** a signed P module would fill the health-premium component. It would not justify setting this broad hold false. The income-stage hold also requires the federal starting point and complete operative additions/subtractions (lines 64–66), then a source-backed composition wiring them. Full liability additionally requires the deduction/exemption, indexed schedule, surtax, and credit stages (lines 101–103). Resolve those through their own governed encodings/composition work; do not replace this policy using P as its citation. The older [coverage report](/Users/maxghenis/PolicyEngine/_wk/axiom-pb-parity/state/coverage.md) agrees that the target calculation was unavailable. + +## 4. Import closure + +**Candidate `existing_signed_imports_json`: `[]`.** There are zero proposed signed-v5 imports and thus no imported module whose compilation needs to be assumed. The atomic source can consume payment/coverage/timing/person facts and actual prior-accounting amounts, then derive the deduction. Federal qualification and deduction calculations remain precise external boundaries, not supplied final Idaho answers. No path to 26/32 or its EITC imports is needed. + +The inspected workflow helper accepts canonical checkout-relative primary YAML paths, not module IDs, and requires same-jurisdiction tracked signed-v5 manifests (`prepare_signed_backfill.py:1368–1501`). A direct federal `us/statutes/26/*` import is not valid for this `us-id` target. Adjacent 63-3022D's old manifest is v1, not a verified v5 candidate; none of the neighboring modules is needed here. + +Two nearby modules are explicitly unsuitable imports: + +| Noncandidate module | Compatibility evidence | +|---|---| +| `us-id:policies/income_tax/2026_full_year_resident_source_hold` | Removed plural `corpus_citation_paths` at line 6. Direct compile of a byte-identical main snapshot fails with that exact schema error. [Compile log](compile-checks/source-hold-current.log). | +| `us-id:policies/income_tax/pilot_liability_pipeline` | Same plural declaration at line 6. Pinned engine code rejects that key, independently of its formulas. | + +Engine `af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca`, `src/rulespec.rs:819–822`, returns `PluralCorpusCitationPaths` whenever it encounters that key. The direct compile used the available canonical binary; its exact build commit was not established, so the pinned source code is the immutable compatibility evidence. An older binary returned by `axiom-locate engine` did compile the hold, but that legacy result is **not** current compatibility evidence. Compiling against the complete canonical checkout also encountered an unrelated forbidden root-level `programs/` layout; the isolated unchanged snapshot avoided that confounder. No new module was compiled or executed. Details: [coverage and corpus notes](coverage-corpus-notes.md). + +## 5. Dispatch inputs + +**No eligible command is authorized or available.** The complete command below records verified current inputs and is **WITHHELD / DO NOT RUN**: the current consumer release cannot resolve P. Choosing a newer `corpus_ref` does not fix this. The workflow reads the target RuleSpec checkout's release pin and materializes that release (`targeted-signed-reencode.yml:827–875`); the encoder resolves only its verified release inventory (`corpus_resolver.py:552–578, 991–1003`). + +Ref provenance: + +| Input | Verified value and origin | +|---|---| +| `rulespec_ref` | `f43dec520dd392bc5333934f56dad7498363e704`, local rulespec-us origin/main; ineligible until additional pin lands | +| `corpus_ref` | `942e138e7a8250c9814e774ac9b8e63008148106`, local canonical corpus origin/main containing P | +| `rules_engine_ref` | `af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca`, rulespec-us `.axiom/workflow-toolchain.toml:7`, and commit/source read in canonical engine | +| Workflow inspected | axiom-encode local origin/main `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`; mandated command uses `--ref main`, whose live value could not be refreshed | +| Citation | Exact canonical row `us-id/statute/63-3022P` | +| Replacement / imports / source bundle | Empty replacement (new absent statute target); `[]` imports; `[]` bundle | + +All 25 workflow inputs are explicit below. `-F review_finding=@...` reads the exact accompanying text file; `-F open_pr=true` sets the workflow's Boolean-typed input to true. This command was not executed. + +```sh +gh workflow run targeted-signed-reencode.yml \ + -R TheAxiomFoundation/axiom-encode --ref main \ + -f citation=us-id/statute/63-3022P \ + -f country=us \ + -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \ + -f pr_base_branch=main \ + -f corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106 \ + -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \ + -F review_finding=@scratch/id-63-3022p/review-finding.txt \ + -f repair_run_id= \ + -f source_bundle_json='[]' \ + -f existing_signed_imports_json='[]' \ + -f replace_rulespec_path= \ + -f replace_legacy_rulespec_path= \ + -f legacy_exact_dependent_rulespec_path= \ + -f second_legacy_exact_dependent_rulespec_path= \ + -f legacy_retained_successor_rulespec_paths_json='[]' \ + -f dependent_citation= \ + -f dependent_review_finding= \ + -f second_dependent_citation= \ + -f second_dependent_review_finding= \ + -F open_pr=true \ + -f queue_id= \ + -f queue_item_id= \ + -f queue_manifest_sha256= \ + -f queue_item_generation_sha256= \ + -f queue_dispatcher_run_id= +``` + +After release activation and the approved pin change, re-read all refs, inspect current workflow changes, validate release provenance against `corpus_ref`, and replace the recorded RuleSpec SHA with the then-current exact main tip. The workflow checks exact base equality at entry and again before PR publication (`prepare_signed_backfill.py:267–331`; workflow lines 3589–3593). Serialize main merges during the run; the ongoing chain can invalidate an otherwise valid base. Recheck collision/attempt budget and obtain Max's approval. No waiver, workflow-pin, CODEOWNERS, or toolchain edits belong in this feature encoding. + +The protected `production-signing` job and `/opt/axiom-verification/axiom-encode encode --backend openai ... --apply --require-complete-source-unit` invocation were read in workflow lines 169–185 and 1268–1284. Local encoding is not a substitute. All workflow constraints and defaults: [dispatch contract](dispatch-contract.md). + +## 6. Review finding text + +The exact proposed `review_finding` is [review-finding.txt](review-finding.txt), reproduced below; SHA256 `9299baef4a8e5a64e0d8f7ac43129696dd114ae06ccd4e8e135249362a3bdf1d`. It covers four worked cases and the required paired tests; it is prose for the supervised encoder, not RuleSpec or test YAML. + +The two moved sweep rows and independent arithmetic, in dollars: + +| Household | Frozen 1.755.4 | Sandbox corrected | Statutory premium deduction | Independent conditional tax | Reading | +|---|---:|---:|---:|---:|---| +| scenario_007 | 755.77832 | 645.53833 | 2,080 | 645.5383203125 → **645.54** | Supports corrected value | +| scenario_053 | 2,435.27832 | 2,170.27832 | 5,000 | 2,170.2783203125 → **2,170.28** | Supports corrected value | + +This conclusion comes from the retained official statute's taxpayer-payment and anti-duplication rule, not the fix docstring. Both households use Idaho standard deductions; none of the personal premium is already deducted/accounted for under the explicit case assumptions. The separately read §63-3024 rate is 5.3%, giving exact marginal changes $110.24/$265.00. Baseline household reruns reproduced the frozen references exactly. The 2026 indexed threshold and unrelated credits were held fixed, not independently certified. See [full adjudication and source/fact evidence](adjudication.md), and [rate provision](rate-provision.json). + +```text +Encode the complete source unit us-id/statute/63-3022P, Idaho Code section 63-3022P, from the verified signed corpus release selected by rulespec-us. This is an Axiom coverage request and independent statutory expectation, not an oracle finding. Do not run until the containing Idaho scope is in the consumer's verified release. Use the signed workflow only. No local encoding or hand-authored RuleSpec or companion YAML is authorized by this preparation package. + +Cover the individual taxpayer's amount actually paid during the taxable year for medical-care insurance covering the taxpayer, spouse, or dependents. Compute the additional Idaho deduction only for dollars not otherwise deducted or accounted for for Idaho income tax. Do not accept a supplied final Idaho premium subtraction as an input. Use underlying payment amounts, timing, payer, insured relationship, coverage facts, and the nonoverlapping portion of those same payments already deducted/accounted for. Reject invalid negative payment/accounting inputs or make the valid nonnegative input domain explicit; never deduct a dollar twice. + +Cover the entire statutory insurance definition: hospital or medical policy/certificate, subscriber contract, specific-disease insurance, hospital-confinement indemnity, accident-only, dental, vision, single-employer self-funded retained-risk coverage, student health benefits, and medical-care/treatment coverage supplementing liability insurance. The word includes is not authority to exclude all other medical-care insurance. Do not transform employer-paid retained risk into an amount paid by the taxpayer. An employee's after-tax contribution to qualifying employer coverage can qualify. + +Account explicitly for the third sentence's employer duty to disclose whether employee health contributions were excluded from taxable income. Prefer an obligation output. If administrative compliance is outside the individual's monetary interface, precisely defer only that named statement-compliance output, grounded to that sentence and with the missing employer/employee compliance interface stated. Do not silently omit the sentence; receipt of a statement is not an express statutory condition for the deduction. + +Precise external boundaries: this section consumes facts about spouse/dependent status and amounts already deducted/accounted for; federal AGI, federal dependency, business/SEHI and Schedule A calculations require separate governing units and are not recreated here. An itemized amount counts only if actually retained for Idaho. Standard deduction selection does not itself account for the premium. Honor the actual Idaho election; optimizing the whole state return is a separate composition. Idaho MSA and long-term-care computations (63-3022K/Q), nonresident allocation, tax rates/thresholds, credits, and complete liability are also separate. State clearly that the atomic deduction does not clear the entire 2026 income/modification source hold. + +Administrative corroboration was read in official tax-year-2025 instructions EIN00046_03-02-2026.pdf, Form 39R line 18/worksheet and Form 40 deduction election, and current IDAPA 35.01.01 Rule 193. They explain pretax, MSA, business, and actual Idaho itemization accounting. Do not cite them as corpus proof unless separately resolved inside the consumer's verified release. Do not import their 2025 dollar constants into 2026. This statutory encoding can use actual prior-accounting facts without encoding the external worksheet formula or election optimizer. No imports are requested: existing_signed_imports_json=[]. + +Worked case 1 -- scenario_007, tax year 2026: single Idaho resident, age 56, no spouse/dependents; personal qualified insurance paid during the year 2080; wholly taxpayer-paid after tax; employer-paid premiums 6309.27587890625 are separate and excluded; pretax personal premiums 0; SEHI/business deduction of the same premiums 0; no MSA funding or other prior accounting; Idaho standard deduction selected. Expected section 63-3022P deduction: 2080. The two input aliases health_insurance_premiums_without_medicare_part_b and other_health_insurance_premiums both describe these same 2080 dollars; do not sum them. Benchmark baseline Idaho AGI 35280, standard deduction 16100, taxable income 19180; premium deduction makes these 33200 and 17100. Frozen PolicyEngine 1.755.4 tax before refundable credits: 755.77832; sandbox corrected: 645.53833. Statutory premium delta at the separately read section 63-3024 rate of 5.3% is 110.24; holding unrelated return components fixed gives 645.5383203125, or 645.54. The statute supports the corrected value under the explicit after-tax facts, not the frozen omission. + +Worked case 2 -- scenario_053, tax year 2026: single Idaho resident, age 25, no spouse/dependents; employment income 66968.6796875; qualified personal medical insurance paid in year 5000; after-tax taxpayer payment; separate employer-paid premiums 3389.27587890625 excluded; pretax amount 0; SEHI/business deduction 0; no MSA or other prior accounting; Idaho standard deduction selected, 16100. The potential medical/Idaho itemized deduction is 377.34863, but is not used on this Idaho standard-deduction return. Expected premium deduction: 5000, not 4622.65137. Frozen tax: 2435.27832; sandbox corrected: 2170.27832. Statutory tax reduction: 265; conditional resulting tax: 2170.2783203125, or 2170.28. The statute supports the corrected value. As in case 1, the two premium aliases are one payment. The premium input's after-tax character must be explicit in generated tests; has_esi=true alone establishes neither pretax treatment nor disqualification. + +Worked case 3 -- partial prior accounting: individual pays 8000 of otherwise qualifying own/spouse/dependent medical insurance during the year; 2000 of those same dollars is already deducted in AGI and retained for Idaho; another nonoverlapping 1000 is actually used in Idaho itemized medical deductions; no other exclusions. Expected additional deduction: 5000. A paired blocking test with all 8000 already accounted for must produce zero. Account for overlap before aggregating exclusions; never subtract the same payment twice. + +Worked case 4 -- election integration boundary: given otherwise valid AGI 60000, eligible premiums 10000, supplied medical deduction 5500, Idaho itemized deductions 20200, allowed standard deduction 16100, and no mandatory-itemization restriction. Under actual Idaho itemization, premium deduction is 4500 and combined reductions are 24700. Under actual Idaho standard selection, premium deduction is 10000 and combined reductions are 26100. The standard route gives taxable income 33900 rather than 35300. These are supplied integration amounts, not constants to encode under this section. The atomic module must honor the election/accounting facts; the external optimizer and worksheet allocation require their own authority. + +The benchmark full-tax expectations certify only this subtraction's effect with other components held fixed. They are not outputs to invent in the atomic premium module. The primary numerical companion assertions are the statutory deductions. Cent-level tax comparisons are integration evidence; floating-point tails in the sweep are not legal rounding rules. + +Paired positive and blocking tests: taxpayer/spouse/dependent versus unrelated nondependent insured; paid this year versus another year; taxpayer-paid versus wholly employer-paid; medical versus nonmedical insurance; after-tax employee contribution versus pretax amount; ordinary funds versus already-accounted-for Idaho MSA funds; zero versus positive overlapping business/SEHI deduction; Idaho standard election versus actual Idaho itemized use; no, partial, and full prior accounting; and employer statement addressing both excluded and nonexcluded contributions without inventing a receipt gate. Exercise each enumerated insurance category, and both zero and positive monetary amounts. Every generated companion case must explicitly assign every local input, including all false Booleans. Ground proof atoms to exact substrings of the resolved statutory unit; source dates are snapshot dates, not statutory effective-date amendments. Use the singular corpus_citation_path schema. Do not import the held annual Idaho policy, its pilot, or the federal EITC closure. +``` + +## 7. Risks + +| Risk / likely rejection | How this package addresses it | +|---|---| +| Citation outside the pinned release | Blocks dispatch. Requires a verified signed release containing the September Idaho scope and an approved consumer pin; later corpus checkout alone cannot help. | +| Moving main, serial re-pin chain | Records immutable local refs and explains both base checks. Future refs cannot be invented; orchestrator must refresh after prerequisites and serialize merges. | +| Incomplete source unit | Finding explicitly covers all insurance categories, taxpayer/insured/timing conditions, anti-duplication, and the employer-statement duty; only named external computations or administrative output may be precisely deferred. | +| Unassigned false facts / untested exception | Finding requires every local input in every companion and paired positive/blocking cases. It distinguishes actual Idaho itemization from an unused federal medical amount. | +| Invalid proofs / invented administrative authority | Exact resolved statutory substrings only. Form and Rule 193 are corroboration unless separately verified in the consumer release. No 2025-to-2026 dollar extrapolation. | +| Broken imports | Empty requested closure avoids both Idaho plural-path policies and the federal EITC closure. No unverified signed-v5 dependency is offered. | +| Duplicate premiums / unspecified after-tax status | Cases separate employer-paid amounts, identify duplicate premium input aliases, and explicitly state taxpayer payment, after-tax treatment, and zero prior accounting. | +| Overclaiming full annual output | Primary assertions are premium deductions; full-tax values are conditional comparisons. P alone does not lift every annual return hold. | + +**Collisions and live run state remain unverified.** Required `gh pr list` calls for both repositories and the last-30 targeted runs call all failed. Public fallback did not provide today's target-specific PRs or runs. Cached local branch names include old Idaho annual-return and oracle-workflow work but establish neither open status nor a current P encoding. The orchestrator must repeat searches for `us-id/statute/63-3022P`, `63-3022P`, and the canonical target path on both repositories and today's workflow runs before approval. No assertion of “no collisions” is made. Run 35789753522 and its artifact were inaccessible; its supplied failure account remains user-provided and unverified here. The workflow's attempt-budget code was read; citation-specific remaining attempts were not observable. + +The binding [agent-PR rules, issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) were read through the public browser fallback after the requested CLI read failed. They require signed generation, exact proof grounding, complete companion inputs, source provenance, and separate governed pin changes; this lane made no external oracle filing or repository change. The issue page was cached, so newer edits remain unverified. + +Checks completed: corpus row/path and official artifact hash; current/next/later release scope comparison; seven adjacent coverage artifacts; workflow inputs/validators/resolver/pins; pinned engine rejection code and unchanged-source-hold compile; both household baseline reproductions; independent decimal premium/tax arithmetic and final report structure. The arithmetic/provenance/structure checks [passed](checks.json); reproduce with `PYTHONDONTWRITEBYTECODE=1 python3 scratch/id-63-3022p/verify_package.py` from the assigned workspace. No new RuleSpec/test YAML, signed run, population simulation, or passing whole-return Axiom result was produced. The generic commit instruction is superseded by this assignment's explicit read-only/no-commit rule. + +The empty `dispatch_command` below means **withheld because no verified passing immutable input set exists**; section 5 preserves the complete ineligible command without representing it as dispatchable. + +```json +{ + "lane": "id-63-3022p", + "verdict": "BLOCKED", + "citation": "us-id/statute/63-3022P", + "in_corpus": true, + "in_pinned_release": false, + "existing_modules": [ + "us-id/statutes/63-3022D.yaml", + "us-id/statutes/63-3022E.yaml", + "us-id/statutes/63-3024.yaml", + "us-id/statutes/63-3024A.yaml", + "us-id/statutes/63-3025D.yaml", + "us-id/policies/income_tax/pilot_liability_pipeline.yaml", + "us-id/policies/income_tax/2026_full_year_resident_source_hold.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Verify and activate an immutable signed corpus release containing 2026-09-14-income-tax-chapter-us-id-title-63-chapter-30, with archive provenance and release-content SHA256; the wave4-r2-union manifest alone is insufficient evidence.", + "Land the separately approved rulespec-us pin to that containing release after coordinating the current serial chain; the Canada 2026-08-23 endpoint lacks this provision.", + "Refresh and freeze exact rulespec main, corpus, engine and workflow evidence; verify base stability, live PR/run collisions and remaining attempt budget.", + "Obtain Max Ghenis's approval for the concrete signed workflow run." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_ma-62-2.md b/reference_audit/2026-09-22/verification/v9_axiom_ma-62-2.md new file mode 100644 index 00000000..bef1f86e --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_ma-62-2.md @@ -0,0 +1,137 @@ +# ma-62-2 — Axiom preparation report + +Observed September 22, 2026 EDT (September 23 UTC). Read-only lane; no RuleSpec, module tests, encoding, dispatch, repository changes, or commits. Evidence is retained beside this report. The subfleet manifest names `/Users/maxghenis/PolicyEngine/_wk/axenc-pb/report-ma-62-2.md` as the output destination; this workspace copy and the final response are supplied for the orchestrator to collect because that destination is outside this lane's writable workspace. + +## 1. Verdict + +**BLOCKED — official-source ingestion and a signed release containing M.G.L. c. 62 §2 are required.** The provision is absent from canonical corpus main and rulespec-us's pinned release. Following the instruction to stop encoding preparation when the source is missing, no dispatch command or encoder finding is proposed. + +The ongoing re-pin chain is insufficient: both the currently pinned August 8 release manifest and the planned August 23 Canada suspension-union manifest select the same Massachusetts statute snapshot, which lacks §2. This lane cannot dispatch against today's main or merely wait for that chain and then dispatch. A subsequent source-bearing release and dedicated activation/pin must land, followed by a fresh preparation check and Max's signed-run approval. + +Independent household conclusion: **the statute supports the r22 corrected $8,232.90 when the benchmark's Part B treatment of the $56 interest is retained as an explicit qualifying-bank-interest assumption. Ordinary Part A interest instead gives $8,230.10.** The stated household does not resolve that distinction; the dividend-loss offset is supported in either case. + +## 2. Source + +Official source: [Massachusetts General Court, chapter 62 section 2](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2), HTML, particularly (b), (c)(2)(a), (c)(4), and (f). Corroboration: [DOR TIR 02-21](https://www.mass.gov/technical-information-release/tir-02-21-capital-gains-and-losses-massachusetts-tax-law-changes), dated January 2, 2003, section II.C. The TIR's historical rates are not used for 2026. + +The intended canonical citation is **`us-ma/statute/62/2`**, consistent with the neighboring section citations. **It is not an existing resolved corpus citation.** No subsection or block citation for this missing source has been invented. + +Verified revisions: + +| Repository / evidence | Immutable value | +|---|---| +| rulespec-us local `origin/main`, equal to remotely read main | `f43dec520dd392bc5333934f56dad7498363e704` | +| canonical axiom-corpus `origin/main`, equal to remotely read main | `942e138e7a8250c9814e774ac9b8e63008148106` | +| rulespec-us pinned corpus release | `us-rulespec-2026-08-08-obbb-alien-snap` | +| pinned content SHA-256 | `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc` | + +The requested `git fetch -q origin` was attempted and failed because the sandbox denied writing `.git/FETCH_HEAD`. Read-only GitHub ref queries independently confirmed both local remote-tracking SHAs; no checkout was changed. + +The sole tracked Massachusetts statute provision file is [`data/corpus/provisions/us-ma/statute/2026-07-13-recovery.jsonl`](https://github.com/TheAxiomFoundation/axiom-corpus/blob/942e138e7a8250c9814e774ac9b8e63008148106/data/corpus/provisions/us-ma/statute/2026-07-13-recovery.jsonl). Its 36 records cover 12 sections: 3, 4, 6, 6l, 10a, 11a, 14, 16, 42, 54, 62, and 64, each with a document row and two blocks. **None covers §2.** The snapshot's `source_as_of` and `expression_date` are July 13, 2026; retained metadata records fetching July 14. A tracked provision search for `us-ma/statute/62/2` returned no match. The coverage file's `complete: true` describes that recovered inventory; it does not establish that every chapter section exists. + +Both release manifests select `us-ma / statute / 2026-07-13-recovery`; thus §2 is absent from both selected statute scopes. The July 6 chapter-level ingest manifest is a source plan, not evidence of retained §2 text. Evidence: [citation inventory](evidence/corpus-ma-citations.txt), [current release scope](evidence/pinned-ma-scopes.json), [planned release scope](evidence/planned-ma-statute-scope.json), [planned release manifest](https://github.com/TheAxiomFoundation/axiom-corpus/blob/942e138e7a8250c9814e774ac9b8e63008148106/manifests/releases/us-rulespec-2026-08-23-canada-338-suspension-union.json), and [remote refs](evidence/corpus-remote-ref.json). Release inclusion was checked against these tracked manifests and their selected provision snapshot; the signed release object itself was not downloaded. + +**Required ingest:** retain the complete official §2 HTML from the URL above, with retrieval date, effective/source vintage, SHA-256 of the retained bytes, extraction, and signed provenance. Preserve the dated amendment alternatives and subsection boundaries. If TIR 02-21 will supply proof evidence, ingest its official HTML separately with the same provenance; its exact citation must be assigned and verified through that ingest. This lane's web extracts are research evidence, not signed corpus artifacts. Publish a new immutable signed release containing the provision, then activate/pin it through the dedicated gated process. No ingest or corpus change was attempted. + +## 3. Existing rulespec-us coverage + +There is no `us-ma/statutes/62/2.yaml` on the verified main. The relevant neighboring modules are: + +| Existing path | Observed scope and relevant verbatim excerpt | +|---|---| +| `us-ma/statutes/62/3.yaml` | Part B exemption scalars; summary: “Part B(b) allows exemptions against Part B income”. It defers the final personal-exemption computation. | +| `us-ma/statutes/62/4.yaml` | Rate and surtax parameters; includes `part_a_short_term_capital_gains_tax_rate` and `part_a_interest_and_dividends_stated_tax_rate`. It does not compute the §2 loss-adjusted base. | +| `us-ma/statutes/62/6.yaml` | Lead-paint and earned-income credit provisions; summary: “the credit equals 40 per cent of the federal credit”. | +| `us-ma/policies/income_tax/pilot_liability_pipeline.yaml` | Estimated-tax slice using completed 5% taxable income; explicitly excludes “cross-Part deductions upstream of completed taxable income”. | +| `us-ma/policies/income_tax/2026_full_year_resident_source_hold.yaml` | Explicit coverage hold: “every public Massachusetts Money stage is a fail-closed zero sentinel”; these are “not legal claims that Massachusetts income, credits, or tax are zero.” | + +The remaining chapter-62 modules are `10a.yaml` (qualified funeral trust), `11a.yaml` (trust withholding), `14.yaml` (corporate fiduciaries), `16.yaml` (fiduciary settlement), `42.yaml` (fiduciary liability), `54.yaml` (severability), `62.yaml` (accounting methods), `64.yaml` (tax tables/rounding), and `6l.yaml` (credit refund election). Their summaries were read and retained in [nearby-summaries.txt](evidence/nearby-summaries.txt); the full tracked Massachusetts inventory is [rulespec-ma-paths.txt](evidence/rulespec-ma-paths.txt). + +These modules supply adjacent parameters or explicit holds, not the assigned computation. No existing §2 calculation was found that could carry the same loss-offset behavior or adjudicate the household. Held monetary zeros are not an Axiom tax answer. + +## 4. Import closure + +**Stopped at the missing-source gate.** No signed-v5 import set or closure was selected, and no existing module was compiled in this lane. The empty `blocking_imports` list below means no proposed closure, not a clean compilation result. No claim is made that this target reaches federal §32. + +Both nearby income-tax policy modules visibly declare plural `corpus_citation_paths`. A [signed-run log](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522) read in this lane rejects that field in imported `us:statutes/42/416/l`. By that observed rule, importing either nearby MA policy module would risk the same rejection; they are not proposed imports. Future preparation must check the actual complete closure, signatures, and engine loads after ingestion. + +## 5. Dispatch inputs + +**No runnable command.** A command with `citation=us-ma/statute/62/2` cannot satisfy source resolution against the inspected release. Supplying a newer `corpus_ref` alone does not fix this: the [workflow at verified encoder SHA `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/.github/workflows/targeted-signed-reencode.yml), lines 831–879, materializes the release selected by the rulespec toolchain. + +The required input block and validation were read. The same commit's helper requires `open_pr=true` runs to use the exact remote PR-base tip; today's rulespec SHA cannot be carried unchanged through future merges. Remotely verified engine main was `6e709eb1ca7ea686263293d932c759d9dee48a4a`, but it is not presented as a tested dispatch ref. No replacement path, imports JSON, review finding, or future ref was fabricated. See [workflow notes](workflow/notes.md). + +## 6. Review finding text + +**No `review_finding` dispatch input is supplied while the source is absent.** The following is the independently requested household adjudication, not an encoder brief or a filed finding against an oracle. + +Section 2(b) classifies ordinary interest/dividends in Part A, subject to specified bank-interest and other exceptions. Section 2(c)(2)(a) applies excess short-term losses to Part A interest/dividends before Part C gains and carryforward. Section 2(c)(4) provides one combined $2,000 limit for short- and long-term losses against that interest/dividend income. Section 2(f) starts taxable Part A income from adjusted Part A income. [Official §2](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section2) and [TIR 02-21, II.C](https://www.mass.gov/technical-information-release/tir-02-21-capital-gains-and-losses-massachusetts-tax-law-changes) support this ordering. + +**Every moved r22 household: scenario_081 only.** Facts directly extracted from `scenario_json`: MA, 2026, single, age 30, no children; wages $175,002; dividends $110.11764526367188; interest $56; short-term capital gains −$1,080; annual rent $44,400. No long-term or collectible gain/loss or carryforward is supplied. Bank assets of $129,000 do not establish the deposit type or institution qualifying for the bank exception. + +| Interpretation of the same household | PolicyEngine 1.755.4 | Supplied corrected sweep | Independent amount | +|---|---:|---:|---:| +| r22: retain qualifying Part B bank interest | $8,238.40625 | $8,232.900391 | **$8,232.90** | +| r23 context only: ordinary Part A interest | $8,238.40625 | $8,230.100586 | **$8,230.10** | + +For the r22 comparison, the allowable loss is `min(1080, 2000, 110.11764526367188) = 110.11764526367188`. Part A adjusted income and taxable dividends are zero. With no Part C gains, unused short-term loss is $969.88235473632812. The unchanged Part B calculation is: + +```text +175,002 wages + 56 bank interest − 2,000 FICA deduction +− 4,000 rent deduction − 4,400 single exemption = 164,658 +164,658 × 5% = 8,232.90 +``` + +The [§3 deductions and exemption](https://malegislature.gov/Laws/GeneralLaws/PartI/TitleIX/Chapter62/Section3) and [DOR 2026 rates](https://www.mass.gov/info-details/massachusetts-tax-rates) were checked. Rent gives `min(44,400 × 50%, 4,000) = 4,000`; wages exhaust the $2,000 FICA deduction cap. The observed household has no nonrefundable credit and is below the surtax threshold. The dividend tax removed is $5.505882263183594 before numeric representation differences. + +For ordinary Part A interest, Part A interest/dividends are $166.11764526367188, all absorbed by the loss. Part B taxable income becomes $164,602 and tax is $8,230.10; unused short-term loss is $913.88235473632812. Neither branch deducts the remaining loss from wages or restores the repealed bank-interest exemption. + +**Adjudication:** the statute supports the corrected r22 amount under its retained interest-classification assumption. It supports the r23 contextual amount under ordinary-interest facts. The frozen $8,238.41 is supported by neither branch of this supplied household comparison. A unique full-household answer remains undetermined by the prompt's interest facts; the zero taxable-dividend result does not. + +Verification: the authorized household-level `pe_case.py` run reproduced frozen `8238.40625` exactly. It recorded Part A AGI zero, taxable dividends `110.117645`, Part B taxable income `164658`, and nonrefundable credits zero. Checks passed for household identity across the supplied and helper bundles, exactly one moved r22 output, only one MA scenario, and both decimal calculations. Fixed simulations were not rerun; their values above come from the supplied sweep. [Facts/checks](adjudication/facts_and_math.json), [calculation log](adjudication/pe_case_081.txt), and [full legal reading](adjudication/adjudication.md) preserve the evidence. + +## 7. Risks + +- **Missing retained source is the immediate rejection risk.** Ingest and release activation/pinning are prerequisites; no waiver or finding wording can substitute for them. Binding [agent rules #39](https://github.com/TheAxiomFoundation/.github/issues/39) were read. +- **Serial-chain timing:** [#1384](https://github.com/TheAxiomFoundation/rulespec-us/pull/1384) is merged; [#1387](https://github.com/TheAxiomFoundation/rulespec-us/pull/1387) and [#1386](https://github.com/TheAxiomFoundation/rulespec-us/pull/1386) remained open when checked. Later drift/activation stages were not verified as landed. Even their planned August 23 release lacks this statute section. +- **Future encoding scope:** the resolved source unit will determine the required branch coverage. Complete-source-unit enforcement, verbatim corpus proof excerpts, every local input assigned in every test, and positive/blocking exception cases must be addressed after ingest. A finding cannot yet be grounded in a nonexistent corpus extraction. The bank exception and shared loss cap must not be inferred from the household's residence or handled as separate $2,000 allowances. +- **Import validity:** singular-source compatibility, signed-v5 admission, and actual closure compilation are separate checks. They remain unperformed for a future §2 module. +- **Collisions:** open PR searches for `"62/2"` and `"chapter 62"` in both repositories returned none. `"us-ma"` matches were unrelated SNAP/provenance/waiver work. All 16 September 22 UTC dispatches and the latest 30 targeted runs, including September 22 EDT's later UTC runs, contained no MA target. Searches are a dated observation, not a guarantee against later work. +- **Evidence limits:** direct TIR retrieval returned 403; official search-indexed text supplied its content and date. Current official text was read, but no signed source vintage or complete legal change-history validation is claimed. No Axiom-generated household result exists for this provision. + +Only scratch evidence and this report were written. No commits were made because the lane's explicit read-only/no-commit rule governs this preparation task. + +```json +{ + "lane": "ma-62-2", + "verdict": "BLOCKED", + "citation": "us-ma/statute/62/2", + "in_corpus": false, + "in_pinned_release": false, + "existing_modules": [ + "us-ma/statutes/62/3.yaml", + "us-ma/statutes/62/4.yaml", + "us-ma/statutes/62/6.yaml", + "us-ma/statutes/62/6l.yaml", + "us-ma/statutes/62/10a.yaml", + "us-ma/statutes/62/11a.yaml", + "us-ma/statutes/62/14.yaml", + "us-ma/statutes/62/16.yaml", + "us-ma/statutes/62/42.yaml", + "us-ma/statutes/62/54.yaml", + "us-ma/statutes/62/62.yaml", + "us-ma/statutes/62/64.yaml", + "us-ma/policies/income_tax/pilot_liability_pipeline.yaml", + "us-ma/policies/income_tax/2026_full_year_resident_source_hold.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Ingest official M.G.L. c.62 section 2 HTML with retained bytes, URL, date, SHA-256 and signed provenance; verify its resolved canonical citation and source unit.", + "Ingest official TIR 02-21 with provenance if it will supply proof evidence.", + "Publish and activate/pin a new immutable signed corpus release containing section 2; the current and planned Canada-338 releases lack it.", + "Repeat source-scope, signed-v5 import-closure, engine-load and collision checks after the source-bearing pin lands; select the then-current exact rulespec main SHA for open_pr=true.", + "Obtain Max's approval before any signed run; the orchestrator controls dispatch." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_nj-43-21-7.md b/reference_audit/2026-09-22/verification/v9_axiom_nj-43-21-7.md new file mode 100644 index 00000000..4328755b --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_nj-43-21-7.md @@ -0,0 +1,9 @@ +# nj-43-21-7 + +**BLOCKED.** The statutory sections are retained in official bulk-source bytes but absent as normalized corpus provisions; the NJDOL 2026 schedule is also absent. Neither the current release nor the planned Canada-union release supplies them. Source admission and a dedicated re-pin are required before dispatch. + +Scenario_008: independent statutory arithmetic supports the corrected **$2,276.66**, conditional on ordinary covered NJ employment, State DI/FLI coverage and timely withholding. The additional $113.90 consists of compensation $97.15, administration $5.36, workforce $6.70 and supplemental workforce $4.69. Missing employer/coverage facts prevent an unconditional adjudication. + +Full seven-section report and dispatch-status JSON: [reports/axiom-nj-43-21-7.md](reports/axiom-nj-43-21-7.md). + +Checks: corpus and release inventories, statutory source hash, live refs/PRs/runs, one reproduced household baseline, independent arithmetic, and a qualified compile check of unchanged existing content. No encoding, external changes, commits or dispatch occurred, per the assignment's read-only rule. diff --git a/reference_audit/2026-09-22/verification/v9_axiom_ny-606e.md b/reference_audit/2026-09-22/verification/v9_axiom_ny-606e.md new file mode 100644 index 00000000..4861e138 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_ny-606e.md @@ -0,0 +1,156 @@ +# ny-606e — Axiom encoding preparation + +Prepared 2026-09-23. Read-only lane; no RuleSpec, module test YAML, commits, repository edits, or signed runs were produced. All new files are under `scratch/ny-606e/`. + +## 1. Verdict + +**BLOCKED.** The statute supports scenario_104's corrected **$0**, subject to the explicit assumption that the supplied rent needs no service-charge adjustment. A passing signed dispatch cannot yet be specified. + +The whole of section 606 exists in the corpus and is selected by the current release, but the proposed narrow citation **`us-ny/statute/TAX/606/e/7/D` does not resolve**. The retained July body has literal backslash-n sequences instead of structural line breaks. The newer September body also lacks structural line breaks. The inspected encoder rejects the narrow target with `CorpusSourceSliceError`. A source repair/structured child ingestion or an approved resolver correction must make the exact source unit resolve before dispatch. Form IT-214 and its instructions also need official ingestion if their operational rules are included in the encoding. + +The named serial re-pin chain is **insufficient by itself**: both the August 8 and planned August 23 release selectors select the same July NY statute scope. Source admission/resolution must be addressed, followed by fresh immutable refs and live PR/run checks. No claim of READY-AFTER merely upon #1387/#1386 merging is justified. + +Snapshot limitation: the prescribed `git fetch -q origin` failed because `.git/FETCH_HEAD` is outside this lane's writable roots. GitHub CLI reads failed to connect. Thus “origin/main” below means the local remote-tracking snapshot, not independently refreshed remote state: + +| Repository | Snapshot read | +|---|---| +| rulespec-us | `f43dec520dd392bc5333934f56dad7498363e704` — merge #1384 | +| axiom-corpus | `942e138e7a8250c9814e774ac9b8e63008148106` | +| axiom-encode | `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9` | + +## 2. Source + +Official sources read: [N.Y. Tax Law §606](https://www.nysenate.gov/legislation/laws/TAX/606), displayed revision September 4, 2026; [2025 IT-214](https://www.tax.ny.gov/pdf/current_forms/it/it214_fill_in.pdf); [2025 IT-214-I](https://www.tax.ny.gov/pdf/current_forms/it/it214i.pdf); and [S3009C, Part RR, printed pages 134–141](https://legislation.nysenate.gov/pdf/bills/2025/S3009C). The [enacted bill page](https://www.nysenate.gov/legislation/bills/2025/S3009/amendment/C) records Chapter 59, signed May 9, 2025. A 2026 IT-214 was not verified. Part RR changes surrounding eligibility, definitions, and amounts from 2025; its §7 amends (e)(7)(A), **not the $450 text of (e)(7)(D)**. + +The exact **stored** canonical citation is **`us-ny/statute/TAX/606`**. The proposed child citation is `us-ny/statute/TAX/606/e/7/D`; no exact child row exists. Distinguish retained legal text from an admissible resolved source unit: + +| Source unit | In inspected corpus main | Selected by pinned release | +|---|---|---| +| Whole §606, July vintage | Yes | Yes, by tracked release selector | +| Exact §606(e)(7)(D) child | No stored row; parent slicing fails | No usable resolved child established | +| IT-214 / IT-214-I | No retained source unit found | No | + +Pinned provision file: `data/corpus/provisions/us-ny/statute/2026-07-06-ny-tax-article22-core-us-ny-sections-tax-601-tax-606-tax-614-tax-615-tax-616.jsonl`. Its `source_as_of` and `expression_date` are `2026-07-06`, with `metadata.active_date=2026-06-05`. Format: `new-york-openleg-json`; official acquisition URL: `https://legislation.nysenate.gov/api/3/laws/TAX/606?full=true`. Raw retained file: `data/corpus/sources/us-ny/statute/2026-07-06-ny-tax-article22-core-us-ny-sections-tax-601-tax-606-tax-614-tax-615-tax-616/new-york-openleg-json/TAX/606.json`. + +The raw-source SHA256 is `fda308972641a125f95dc0b233c2f79e1f26507b524bf23f7153dd060f5cfafd`; provisions-file SHA256 is `c9513c936e31200b318b40fa5f3f73dfef6e805284668fd42b3848263e916247`. Both independently match the recorded ingest manifest. This lane did not cryptographically verify that manifest's signature. + +Newer file: `data/corpus/provisions/us-ny/statute/2026-09-14-income-tax-chapter.jsonl`, dated September 14, 2026. It contains an empty-body §606 parent and a body-bearing `us-ny/statute/TAX/606/block-1`. The entire subsection (e) matches July after whitespace normalization. Those normalized scratch excerpts are reading aids, **not admitted source replacements**. + +rulespec-us `.axiom/toolchain.toml` pins `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`. Corpus `manifests/releases/us-rulespec-2026-08-08-obbb-alien-snap.json:1062` selects the July NY statute scope. `manifests/releases/us-rulespec-2026-08-23-canada-338-suspension-union.json` selects the same scope. The downloaded signed release was unavailable through `axiom-locate release`, so release membership here is verified from the tracked selector; the published object/content hash was not independently reverified. + +**Required source work:** retain an official JSON/HTML source with URL, acquisition date and SHA provenance and expose a correctly structured (e)/(7)/(D) unit, including the controlling paragraph (7) context. If repairing the July source is the chosen route, publish a new immutable release and obtain the dedicated consumer re-pin; do not edit the current immutable release. Admit the official IT-214 and IT-214-I PDFs above, explicitly recording their 2025 vintage, before relying on their form-specific rules. No form dispatch is proposed while those source units are absent. + +Evidence: [corpus inventory and hashes](corpus-coverage-note.md), [exact July row](corpus-606-july-full.json), [resolver failures](workflow-resolver-test.txt). + +## 3. Existing rulespec-us coverage + +The inspected origin/main contains no §606(e), IT-214, RPTC, or real-property-tax-credit module. The bounded inventory covered tracked NY modules and their §606/IT-214 references, and target-path searches across rulespec-us. The relevant nearby modules are: + +| Module | Existing behavior and source evidence | +|---|---| +| `us-ny/policies/income_tax/pilot_liability_pipeline.yaml` | §601 resident main-tax schedule only. Lines 56–57: “Section 606 credits and final New York income tax are outside the / narrow non-circular `ny_main_income_tax` comparison target.” | +| `us-ny/statutes/TAX/601.yaml` | Broad income-tax source deferred: line 3 `status: deferred`; line 11 `ny_income_tax_before_credits`; line 13 `rules: []`. Its deferred reason identifies unresolved status, residency, source-fraction, tax-base and §606-credit boundaries. | +| `us-ny/statutes/NYC/11-1706.yaml` | Encodes city pass-through entity tax credit. Lines 20–21 defer `household_and_dependent_care_credit`, referring to “credits allowed under Tax Law § 606(c) and (c-2)”. This is a different credit. | + +`programs/us-ny/income-tax/fy-2026.yaml:7` exposes only `ny_pit_pilot_taxable_income` and `ny_pit_pilot_main_income_tax`; it marks the latter incomplete. The adjacent §614 module is a standard-deduction module, not a credit computation. + +These files demonstrate **missing Axiom coverage**, not an encoded instance of the numerical behavior in the supplied comparison. There is no existing target to replace. The existing pilot must not be passed as `replace_rulespec_path`. + +## 4. Import closure + +Candidate **`existing_signed_imports_json=[]`**. No existing signed-v5 module was identified that supplies the target rent-cap computation. The three nearby modules above are unnecessary for a narrow cap rule. This candidate closure contains no modules, so it has **no blocking import and no per-module compile claim**. No RuleSpec compile was run; there is no target module to compile. + +Adjusted-rent derivation, averaging, and any included-service/subsidy treatment must be sourced. If separate companion encodings are chosen, they must be produced by the signed path and their completed import closure rechecked before generating the final command. An empty existing-import list is not permission to substitute unsupported legal conclusions for facts. Avoid importing federal earned-income-credit or general NY liability machinery merely to encode this restriction; the proposed empty closure cannot reach 26/32. + +Source-resolution check, distinct from an engine compile: the exact encoder snapshot was extracted to scratch and its existing resolver executed without encoding. These were direct private and public slicing checks on retained body strings, not a signed-release-backed `resolve_local_corpus_source` invocation. The record-loading/encode call chain was inspected separately and passes the July body unchanged. All six body/citation combinations failed through both APIs for `/e`, `/e/7`, and `/e/7/D`. The July row contains **0 actual line breaks and 6,363 literal `\\n` sequences**; September's block contains neither actual line breaks nor literal `\\n`. Code evidence is `src/axiom_encode/corpus_resolver.py`: record-body handling, selected body to slicing at lines 621–654, parent fallback at 1405–1430, and generic state-source slicing at 1519–1529. The exact-target path maps to `us-ny/statutes/TAX/606/e/7/D.yaml`, but correct path routing does not establish successful source resolution. [Call-chain evidence and reproduction](workflow-findings.md). + +## 5. Dispatch inputs + +**No passing dispatch command is supplied.** `dispatch_command` is empty in the final JSON. The verified current source fails before encoding, while future corrected source/release/base SHAs do not exist in this lane's evidence. Presenting the current refs as a READY command, or inventing future refs, would misstate readiness. + +The following are verified preparation values, **not a run authorization**: + +| Input | Verified value or disposition | +|---|---| +| workflow/repository/ref | `targeted-signed-reencode.yml`, `TheAxiomFoundation/axiom-encode`, `main`; inspected workflow snapshot `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9` | +| `country` | `us`, which routes `us-ny` to rulespec-us | +| `citation` | Proposed `us-ny/statute/TAX/606/e/7/D`; rejected by present source resolution | +| `rulespec_ref` | Inspected `f43dec520dd392bc5333934f56dad7498363e704`; must refresh against actual dispatch-time main | +| `corpus_ref` | Inspected corpus `942e138e7a8250c9814e774ac9b8e63008148106`; insufficient for the proposed narrow target | +| `rules_engine_ref` | Protected rulespec workflow pin `af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca`, read from `.axiom/workflow-toolchain.toml`; ancestry/build and continued approval were not reverified | +| `replace_rulespec_path` | Empty: new target, no existing module | +| `existing_signed_imports_json` | `[]`, subject to the companion-source decision | +| `source_bundle_json` | No admitted transaction specified yet; default `[]` does not solve missing adjusted-rent/form sources | +| `review_finding` | [Prepared text](review_finding.txt), provisional until the final resolved source boundary is known | +| `pr_base_branch`, `open_pr` | `main`, `true` for a future approved dispatch | +| Other repair/dependent/legacy/queue inputs | Empty/default; this lane is not a repair replay or queue dispatch | + +The workflow's input block was read in full. Its checkout gate at lines 372–403 requires lowercase 40-character SHAs, exact checkout identity, an admissible rulespec base, and corpus/engine ancestry on main. The signed job is gated on `refs/heads/main`, uses `production-signing`, and runs the installed protected encoder. Import parsing requires tracked same-jurisdiction signed-v5 modules; target/replacement validation is not bypassed by supplying a path. These checks and the live-base requirement are why a static pre-chain SHA must not be silently reused after the chain advances. + +The workflow loads the signed corpus release selected by rulespec's `.axiom/toolchain.toml` (lines 827–876); a newer `corpus_ref` alone does not admit newer source scopes. After source admission, one possible bounded transaction would encode the complete adjusted-rent definition `/e/1/G` as a fresh companion before the D target. The source-bundle code supports this and makes the companion a required import. Those companion lanes receive empty review findings and complete-source validation, so a broad `/e/7` companion cannot be described as merely a chapeau. The D unit must retain its operative parent context, or the primary must become the whole bounded paragraph (7), with all its branches reviewed. Neither alternative is yet a verified dispatch package. [Workflow/source-bundle evidence](workflow-findings.md). + +The full issue [TheAxiomFoundation/.github#39](https://github.com/TheAxiomFoundation/.github/issues/39) was read through the public web after CLI access failed. This package follows its source-provenance, protected-pin, generated-content, exact-proof, exhaustive-local-input and no-new-waiver rules. Max's approval remains required for any future signed run; none was requested or consumed here. + +## 6. Review finding text and independent adjudication + +The complete provisional `review_finding` is [review_finding.txt](review_finding.txt). It requires a strict tenant-specific `$450` monthly **adjusted-rent** prohibition, accurate source context, adjustments and rental-month treatment, paired positive/blocking cases, and precise deferrals. It does not claim that escaping this one restriction establishes positive-credit eligibility. + +The retained subsection (e)(7) provides the no-credit context for (D). Its threshold is adjusted rent; subsection (e)(1)(F)(ii)'s 25% tax equivalent is a separate calculation. Subsection (e)(1)(G) supplies the included-service exclusions. For 2025 onward, the qualified-taxpayer test uses federal AGI and positive tax-equivalent excess; the senior flat-credit table gives $375 at AGI up to $3,000. The 2025 form independently performs the monthly rent test at line 13 before the 25% operation at line 14. These conclusions were derived from official/retained source text, not the sandbox docstring. [Statutory reading and source locations](statute-adjudication.md). + +**Every moved benchmark output:** the sweep has exactly one r09 row, scenario_104, NY, `state_refundable_credits`, frozen **$375**, corrected **$0**. The supplied household JSON is TY2026, one single adult age 72, wages $0, Social Security retirement $14,096, veterans benefits $20,520, annual pre-subsidy rent **$6,483.529296875**. The new household-only baseline run reproduced total refundable credits $375 and RPTC $375, with federal AGI $0, housing assistance $0, actual rent equal to the supplied rent, and real-estate tax $0. [Input](household-source.json), [sweep row](sweep-moved-rows.json), [successful rerun](household-pe1755-success.log). + +With no included-service adjustment and 12 rental months, $6,483.529296875 / 12 = **$540.2941080729**, above $450. Therefore the statute independently gives **$0 RPTC**. Holding the other benchmark components fixed gives **$0 state refundable credits**. **The statute supports the corrected value under that explicit input interpretation.** The annual tax equivalent, $1,620.88232421875, cannot be substituted in the cap comparison. + +This conclusion is conditional on sparse inputs: the JSON does not state whether heat, utilities, furnishings, or board are included. Heat included without a separately stated charge would leave $459.24999 monthly and still fail; an unitemized heat/gas/electricity bundle would reduce monthly adjusted rent to $432.23529 and escape the cap. Accordingly neither full real-world entitlement nor universal correctness of the entire sandbox reform is established by this one household. No external-oracle finding is filed or proposed. + +Four worked cases follow. Positive amounts additionally assume TY2026, full-year NY residence, age 72, AGI $0, the same residence throughout, not another person's dependent, property not wholly tax-exempt, no subsidy, and all other restrictions satisfied. All included-service facts are false except as stated. + +| Case | Rent facts | Monthly adjusted rent | Cap prohibits? | Wider statutory credit; benchmark comparison | +|---|---|---:|---|---| +| **scenario_104** | $6,483.529296875; 12 months; no adjustment supplied | $540.2941080729 | Yes | **$0**; frozen **$375**, corrected **$0** | +| Equality | $5,400; 12 months; no services | $450 | No | $375; tax equivalent $1,350 exceeds zero | +| Paired blocking case | $5,412; 12 months; no services | $451 | Yes | $0 despite positive tax equivalent | +| Included heat | $6,000; 12 months; unitemized heat, 15% exclusion | $425 | No | $375; adjusted annual rent $5,100, tax equivalent $1,275 | + +The finding additionally calls for 6-versus-12-month and tenant-versus-nontenant test pairs, all local inputs including false facts, and every operative service adjustment if that companion is encoded. The isolated cap should return only its restriction; the contextual positive dollar amounts above do not expand its output claim to a complete credit. + +## 7. Risks and checks + +- **Source-resolution rejection — demonstrated.** Fix the admitted source structure or approved resolver behavior, verify the exact resolved text/hash, and rerun resolution before proposing a command. Do not use the 400,000-character whole §606 as an opportunistic substitute target. +- **Missing form authority.** IT-214 references elsewhere in the corpus are not a retained form. Ingest the official PDFs with vintage and provenance before a form-based encoding; a pure statutory cap must explicitly limit its claims. +- **Completeness and input-boundary rejection.** The finding identifies the controlling chapeau, adjusted-rent definition, service exceptions, paired tests, and specific neighboring computations that a narrow module defers. A broader resolved unit needs a fresh completeness review. A final-credit claim requires all remaining eligibility and amount rules. +- **Proof mismatch.** Human-normalized excerpts are not the resolved provision. Generate proofs against exact admitted text; do not repair proofs by paraphrase or add a waiver. +- **Import risk.** Current candidate closure is empty. Any later companion/import changes require signed-v5 validation and an actual compile of that closure at the chosen engine. No existing-module load success is claimed. +- **Serial chain and collisions — live status unverified.** The cached main contains #1384. The supplied brief names #1387, #1386 and later drift/activation/re-pin steps; their current state could not be verified. Searches for open rulespec-us and axiom-encode PRs and the latest 30 targeted workflow runs were attempted through `gh`; all failed network access. Public-web PR/action fallbacks also failed. Consequently absence of a competing §606 run/PR is **not established**. The example failed run 35789753522 was not downloaded in this lane and is not presented as a newly inspected log. [Read-attempt logs](workflow-recent-runs.error), [rulespec PR query](workflow-open-rulespec-prs.error), [encoder PR query](workflow-open-encode-prs.error). +- **Upstream status.** The supplied root-causes record attributes a later change to policyengine-us#9301. `gh pr view 9301` failed, so current PR metadata is unverified and does not support this legal conclusion. + +Completed checks: tracked corpus/module inventory; July/September subsection comparison; source/provision SHA comparisons with recorded ingest manifest; exact-origin encoder source-resolution checks; extraction of the sole moved row; one successful PE1.755.4 household baseline reproduction; four Decimal arithmetic checks. Results are in [adjudication-checks.json](adjudication-checks.json). An initial baseline command used an unavailable `social_security_taxable` variable; it failed after printing the relevant values, and the corrected command completed with exit 0. The sandbox correction was checked against the supplied sweep and prior verifier, **not rerun**. No population simulation, RuleSpec compilation, local encoding, workflow dispatch, or repository mutation was performed. + +The JSON flags below refer to the **exact proposed child source unit**. Its whole-section parent is present and selected, as separately recorded. + +```json +{ + "lane": "ny-606e", + "verdict": "BLOCKED", + "citation": "us-ny/statute/TAX/606/e/7/D", + "in_corpus": false, + "in_pinned_release": false, + "source_parent_citation": "us-ny/statute/TAX/606", + "parent_in_corpus": true, + "parent_in_pinned_release": true, + "existing_modules": [ + "us-ny/policies/income_tax/pilot_liability_pipeline.yaml", + "us-ny/statutes/TAX/601.yaml", + "us-ny/statutes/NYC/11-1706.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Make the exact section 606(e)(7)(D) source unit and its controlling context resolve through approved source ingestion/repair or an approved resolver correction.", + "If the source changes, publish an immutable signed corpus release and complete a dedicated rulespec-us re-pin; the named August 23 re-pin alone retains the unusable July source.", + "Ingest official IT-214 and IT-214-I with URL/date/SHA provenance before encoding their form-specific rules; verify any adjusted-rent companion source units.", + "Refresh rulespec-us main, corpus and engine identities, verify the signed release object, resolve the target and compile any final signed import closure.", + "Check live serial-chain status, open PRs and same-citation runs, finalize the exact command and review finding, and obtain Max's approval for the signed run." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_us-2014-sua.md b/reference_audit/2026-09-22/verification/v9_axiom_us-2014-sua.md new file mode 100644 index 00000000..f0b6a87b --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_us-2014-sua.md @@ -0,0 +1,258 @@ +# 1. Verdict + +**READY-AFTER — align the protected validator with the encoder that will sign the run, then refresh the immutable dispatch refs.** The amended source is already in the pinned corpus. No new ingestion is needed. Prepare two separate signed encodes: `us/statute/7/2014/e/6` for the SUA/excess-shelter source unit and `us/statute/7/2014/k/4` for the companion state-law energy-assistance treatment. Both can have empty import closures. + +Today's rulespec-us main is **`f43dec520dd392bc5333934f56dad7498363e704`**. Its `.axiom/workflow-toolchain.toml` pins axiom-encode `f856cfcb886d9bd050b228aa60aeb4b96939f739`, version `0.2.2006`; the protected signed workflow on main uses `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`, version `0.2.2015`. The pinned verifier compares manifest execution repository, commit and version and rejects a mismatch with **`.axiom_encode does not match the running pinned encoder`**. The shared CI workflow passes the expected encoder checkout to that check. This is a code-confirmed prerequisite, not an inferred concern. See [verifier excerpt](scratch/us-2014-sua/workflow/pinned-encoder-identity-check.py.txt) and [workflow evidence](scratch/us-2014-sua/workflow/notes.md). + +A dedicated protected validation-toolchain alignment, including its exact-pin fixtures, must land outside this content lane. The current serial chain has #1384 merged, #1387 activation open, and #1386 workflow pin open; its stated changes and the planned Canada corpus re-pin **do not establish encoder identity alignment**. Coordinate the new prerequisite with that chain. The statutory source itself needs neither the Canadian release nor those merges. Do not spend a signed run against today's main expecting landable CI. After any merge, obtain a new exact main SHA; `open_pr=true` requires the current main tip. Max must approve each signed run. + +No RuleSpec or test YAML was authored, no local encode ran, and no commit, branch, push, PR, issue or workflow dispatch was made. The later explicit read-only rule governs over the generic initial commit instruction. All new files are report/scratch artifacts inside the assigned workspace. The requested fetch was attempted and denied at `.git/FETCH_HEAD`; GitHub's read-only connector independently confirmed that local `origin/main` equals live main. [Issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was read through that connector after shell `gh` network access failed. + +# 2. Source + +Official authorities: [7 USC 2014](https://uscode.house.gov/view.xhtml?req=%28title%3A7+section%3A2014+edition%3Aprelim%29), [7 USC 2012](https://uscode.house.gov/view.xhtml?req=%28title%3A7+section%3A2012+edition%3Aprelim%29), and [P.L. 119-21, §10103](https://www.govinfo.gov/content/pkg/PLAW-119publ21/html/PLAW-119publ21.htm), approved July 4, 2025. + +Corpus `origin/main`, independently checked: **`942e138e7a8250c9814e774ac9b8e63008148106`**. The tracked file `data/corpus/provisions/us/statute/2026-07-22-rulespec-title-7-consolidated.jsonl` contains: + +| Canonical citation | JSONL line | Resolved source | +|---|---:|---| +| `us/statute/7/2014/e/6` | 135 | Entire excess-shelter paragraph, including amended C(iv)(I) | +| `us/statute/7/2014/k/4` | 162 | Both amended state-law energy-payment clauses | +| `us/statute/7/2012/j` | 50 | All seven elderly/disabled routes | + +**`us/statute/7/2014/e/6/C/iv/I` does not exist as a separate corpus record.** Dispatch the real paragraph citation, with complete-source-unit coverage; do not invent a narrower leaf. + +Vintage is OLRC **Online@119-100**, expression/source-as-of **2026-06-26**, USLM XML from [the official releasepoint ZIP](https://uscode.house.gov/download/releasepoints/us/pl/119/100/xml_usc07@119-100.zip). Tracked source: `data/corpus/sources/us/statute/2026-07-22-rulespec-title-7-consolidated/2026-07-21-snap-chapter-51-title-7-title-7/uslm/usc7.xml`; retained source SHA-256 `1e85a2d4a9e3068671d2f444ad7faeccc9ed0e5f232ca6f1504dbdfc0a25a3d3`. Exact resolved bodies are in [statutory-text.txt](scratch/us-2014-sua/corpus/statutory-text.txt). + +The rulespec pin is **`us-rulespec-2026-08-08-obbb-alien-snap`**, content SHA-256 **`0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`**. Its manifest includes the consolidated Title 7 scope. Both amendments are present; the planned August 23 union also contains them. No ingest prerequisite applies. + +The resolved C(iv)(I) says the standard shall be available to **“households with an elderly or disabled member”**, with a qualifying payment greater than $20 in the current month or immediately preceding 12 months. In k(4), (A) applies the direct-payment treatment to households **without** such a member; (B) deems qualifying third-party-paid expenses household-paid for households **with** one. The gates run in opposite directions. + +For [7 CFR 273.9](https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/section-273.9), canonical corpus citation is `us/regulation/7/273/9`. The pinned `2026-05-10-snap-7-cfr-273-r2026-07-15-self-contained.jsonl` has source date April 29, 2026; main also holds `2026-07-15-title-7-part-273.jsonl`, source date July 9. Both retain the unrestricted LIHEAA shortcut at **(d)(6)(iii)(D)(3)** after regulatory restructuring. Neither contains the conforming elderly/disabled gate. Live eCFR access failed, so conformance after July 9 is **unverified**. Encode the amended statute rather than repeating the unrestricted regulatory branch. + +USDA's [August 29, 2025 implementation memo](https://www.usda.gov/sites/default/files/guidance-documents/fns.SNAP-admin-energy-assistance-payments.pdf) preserves the actual-heating/cooling-expense route and addresses ongoing-case recertification. Its [May 8, 2026 Q&A](https://www.usda.gov/sites/default/files/guidance-documents/fns.snap-obbb-restrictionsInternetExpEnergyAssistPayments-QAs1.pdf) distinguishes federally authorized LIHEAP/HUD payments from k(4)'s state-law payments. These guidance documents exist in corpus main's September 13 SNAP-guidance scope, **outside both the pinned and planned August releases**. They inform this reading; the proposed modules do not import them or use their text as pinned-source proofs. + +# 3. Existing rulespec-us coverage + +Read only the verified `origin/main` snapshot. There is no root `us/statutes/7/2014.yaml`. These are all seven modules beneath `us/statutes/7/2014/`; each has an existing companion test: + +| Module suffix | What it encodes; relevant source lines | +|---|---| +| `a.yaml` | Participation basis; lines 13–18 defer `household_eligible_for_snap_participation` subject to the remaining eligibility rules. | +| `c.yaml` | Gross/net income standards; lines 107–109 apply `not household_includes_elderly_or_disabled_member` to the gross-income test. Imports e/6/A. | +| `d.yaml` | Selected income exclusions; lines 10–11 defer aggregate excluded income: `this file encodes selected source-backed leaves only`. | +| `e/2.yaml` | Earned-income deduction; line 24: `max(0, snap_countable_earned_income - work_supplementation_earned_income)`; line 42 multiplies by the deduction rate. | +| `e/2/B.yaml` | Earned-income deduction rate; lines 28–30 set `0.20`. | +| `e/6/A.yaml` | Income after externally supplied shelter deduction; line 43: `max(0, snap_net_income_pre_shelter - snap_excess_shelter_deduction)`. No SUA eligibility rule. | +| `g.yaml` | Resource limits and selected rules; lines 10–11 defer complete `countable_household_financial_resources`. | + +None encodes either §10103 amendment. New `e/6.yaml` and `k/4.yaml`, their `.test.yaml` companions, and their ownership manifests are absent. Existing e/6/A is a separate legacy interface; this package does not overwrite it. + +`us/regulations/7-cfr/273/9.yaml` contains income tests and utility hooks. Lines 77–128 set each standard/limited/individual utility option to `formula: '0'` and sum the hooks. It does not implement the amendment. + +The more specific **`us/regulations/7-cfr/273/9/d/6/iii.yaml` does encode the old unrestricted receipt route**. Lines 89–102 include: + +```text +or ( + liheaa_or_similar_energy_assistance_payment_received_or_made_on_household_behalf_in_current_month_or_immediately_preceding_twelve_months + and liheaa_or_similar_energy_assistance_annual_payment_amount > liheaa_or_similar_energy_assistance_annual_payment_threshold +) +``` + +Its threshold is 20; the only version starts October 1, 2008. No elderly/disabled input exists. The existing test `liheaa_payment_above_threshold_makes_heating_cooling_standard_available` expects `holds` at $21 without separately incurred costs. Thus this module carries the same **missing elderly/disabled restriction**, although it is not the state-flag implementation used in the supplied sandbox comparison. + +Executed that unchanged existing module for a scenario_080-style January 2026 case, assigning all seven local inputs. Assuming the generic $2,000 subsidy is qualifying timely LIHEAP, with no actual utility costs, the result was **`holds`**. That assumption tests the most favorable receipt-route case; its program/timing are not independently stated in the benchmark. [Request](scratch/us-2014-sua/coverage/scenario080-old-gate-request.json) and [explain response](scratch/us-2014-sua/coverage/scenario080-old-gate-response.json) show the unrestricted branch. This is an existing gate execution, not an amended signed encoding or annual SNAP calculation. + +`us/statutes/7/2012/j.yaml` also does not adjudicate disability: lines 14–23 merely aggregate an external classification with `count_where(member_of_household, snap_member_is_elderly_or_disabled) > 0`. + +# 4. Import closure + +**Candidate `existing_signed_imports_json='[]'` for both new modules.** Define an explicit input for the independently resolved §2012(j) status and precise external boundaries for State amounts, State choices and other statutory classifications. Do not silently import the legacy disability aggregator or utility gate. A future module that derives §2012(j) from individual facts needs its own signed encoding. + +All relevant manifests found are `axiom-encode/applied-rulespec/v1` with `hmac-sha256`; some modules have no manifest. None qualifies as an existing signed-v5 import. The workflow expects canonical module **file paths**, not corpus citation strings, in a nonempty import list. + +The existing root SNAP regulation has this closure: + +```text +us:regulations/7-cfr/273/9 + -> us:statutes/7/2012/j + -> us:regulations/7-cfr/273/10 + -> us:policies/usda/snap/fy-2026-cola/deductions -> 2012/j + -> us:policies/usda/snap/fy-2026-cola/maximum-allotments + -> us:statutes/7/2012/j + -> us:policies/usda/snap/fy-2026-cola/income-eligibility-standards +``` + +| Existing module | Observed local compile result | +|---|---| +| 273/9/d/6/iii; 2012/j | Load | +| All seven 2014 modules, including c → e/6/A → e/2 → e/2/B | Load | +| FY2026 income-eligibility-standards | Fails: removed plural `corpus_citation_paths` | +| FY2026 deductions; maximum-allotments | Fail: unknown `module.source_verification` field `values` | +| 273/10 | Fails through deductions | +| 273/9 | Fails through 273/10; also contains the independently failing income standards | + +No module in this closure reaches 26/32. The proposed empty closure has no blocking imports. The three COLA modules above would block reuse of the broader SNAP composition. + +Compile limitation: `axiom-locate engine` returned an older permissive binary. Checks were repeated with the canonical checkout's stricter executable, version `0.2.0`, SHA-256 `faf4383622f63c64b861e5772b78b00df97efef4a8315b792b25219033bee75e`. Its build commit is **unverified**, so these are observed local checks, not proof of compilation at the dispatch engine SHA. Current engine source independently contains the plural-field rejection. Exact commands, copied existing sources, manifest inventory and logs are in [coverage notes](scratch/us-2014-sua/coverage/coverage-notes.md). + +# 5. Dispatch inputs + +**Reference commands only — do not dispatch before the prerequisite in section 1.** Every current input is populated below; empty strings and empty arrays are intentional. No future SHA has been invented. After protected alignment lands, regenerate both commands against then-current main and its compatible toolchain. If one resulting PR merges before the other run, refresh the second command's `rulespec_ref` too. + +Verified ref provenance: + +- `rulespec_ref=f43dec…`: local `origin/main`, confirmed by live GitHub `branches/main`. +- `corpus_ref=942e138…`: canonical corpus main, confirmed by GitHub. Actual source content comes from the signed release pinned by rulespec-us, not arbitrarily from this newer checkout. +- `rules_engine_ref=af6e4ea…`: rulespec-us `.axiom/workflow-toolchain.toml`; GitHub compare confirms ancestry of current engine main `6e709eb1ca7ea686263293d932c759d9dee48a4a`. +- Workflow `--ref main`: inspected at encoder `5d80d753…`. The protected encode job requires main, so substituting an old encoder SHA is not a bypass. +- Citations and absent destination/module/test/manifest paths: verified against corpus and rulespec snapshots above. Both replacement fields are empty because these are new source units. + +Read the full [workflow](scratch/us-2014-sua/workflow/targeted-signed-reencode.yml.txt), its [validation helper](scratch/us-2014-sua/workflow/prepare_signed_backfill.py.txt), and a real [failed signed-run log excerpt](scratch/us-2014-sua/workflow/run35789753522-excerpts.log). Validation requires full immutable SHAs, exact main for PR creation, existing pinned source, and complete source-unit coverage. `gh workflow run --help` confirms `-F name=@file` sends the file contents. Shell syntax was checked; neither command was executed. + +```sh +# CURRENT-BASE REFERENCE ONLY. BLOCKED FROM LANDABLE CI BY ENCODER IDENTITY MISMATCH. +# Do not execute until Max approves and protected validation pins align; refresh exact main refs. +gh workflow run targeted-signed-reencode.yml -R TheAxiomFoundation/axiom-encode --ref main \ + -f 'citation=us/statute/7/2014/e/6' \ + -f 'country=us' \ + -f 'rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704' \ + -f 'pr_base_branch=main' \ + -f 'corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106' \ + -f 'rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca' \ + -F 'review_finding=@scratch/us-2014-sua/review_finding.txt' \ + -f 'repair_run_id=' \ + -f 'source_bundle_json=[]' \ + -f 'existing_signed_imports_json=[]' \ + -f 'replace_rulespec_path=' \ + -f 'replace_legacy_rulespec_path=' \ + -f 'legacy_exact_dependent_rulespec_path=' \ + -f 'second_legacy_exact_dependent_rulespec_path=' \ + -f 'legacy_retained_successor_rulespec_paths_json=[]' \ + -f 'dependent_citation=' \ + -f 'dependent_review_finding=' \ + -f 'second_dependent_citation=' \ + -f 'second_dependent_review_finding=' \ + -f 'open_pr=true' \ + -f 'queue_id=' \ + -f 'queue_item_id=' \ + -f 'queue_manifest_sha256=' \ + -f 'queue_item_generation_sha256=' \ + -f 'queue_dispatcher_run_id=' + +# CURRENT-BASE REFERENCE ONLY. BLOCKED FROM LANDABLE CI BY ENCODER IDENTITY MISMATCH. +# Do not execute until Max approves and protected validation pins align; refresh exact main refs. +gh workflow run targeted-signed-reencode.yml -R TheAxiomFoundation/axiom-encode --ref main \ + -f 'citation=us/statute/7/2014/k/4' \ + -f 'country=us' \ + -f 'rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704' \ + -f 'pr_base_branch=main' \ + -f 'corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106' \ + -f 'rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca' \ + -F 'review_finding=@scratch/us-2014-sua/review_finding_k4.txt' \ + -f 'repair_run_id=' \ + -f 'source_bundle_json=[]' \ + -f 'existing_signed_imports_json=[]' \ + -f 'replace_rulespec_path=' \ + -f 'replace_legacy_rulespec_path=' \ + -f 'legacy_exact_dependent_rulespec_path=' \ + -f 'second_legacy_exact_dependent_rulespec_path=' \ + -f 'legacy_retained_successor_rulespec_paths_json=[]' \ + -f 'dependent_citation=' \ + -f 'dependent_review_finding=' \ + -f 'second_dependent_citation=' \ + -f 'second_dependent_review_finding=' \ + -f 'open_pr=true' \ + -f 'queue_id=' \ + -f 'queue_item_id=' \ + -f 'queue_manifest_sha256=' \ + -f 'queue_item_generation_sha256=' \ + -f 'queue_dispatcher_run_id=' +``` + +# 6. Review finding text + +The exact multiline inputs are [review_finding.txt](scratch/us-2014-sua/review_finding.txt) for e/6, SHA-256 `76e476f8218d206f3efd9990910346ad1766933810e8993fd1e2b6286d1b0ebe`, and [review_finding_k4.txt](scratch/us-2014-sua/review_finding_k4.txt) for k/4, SHA-256 `6af43df28b7a84917add398e63abc3dca3a613fa0c2ffc0059a809513885ffc2`. They carry the required encoding regime verbatim. They request signed generation only and contain no RuleSpec or test YAML. + +The e/6 finding requires coverage of **A–E**, not just the desired gate: the 50% excess-shelter calculation and third-party restriction; indexed caps and elderly/disabled exception; optional and mandatory State standards, qualifying costs, central-meter and shared-cost restrictions with their precise exceptions; amended receipt eligibility, separate-standard option, no compelled reduction and season proration; homeless alternative and its exclusions; and internet-fee exclusion. Historical schedules must be encoded or named precisely as deferred historical outputs. CPI series, published State amounts, State elections and separately resolved legal classifications may be explicit boundaries; missing values must not become invented constants or zero legal entitlements. + +The k/4 finding separately covers both opposing gates, distinguishes payment amounts from expenses paid, preserves the k(2)(G) exception, and precisely defers other income exclusions and total-benefit calculation. It does not infer that an unspecified energy subsidy is State-law income. The federal LIHEAP and one-time weatherization exclusions remain relevant. + +Both findings specify July 4, 2025 amendment applicability, an effective-date boundary test, and a separate deferral of administrative recertification transitions. Both findings require verbatim path-anchored proof excerpts and every local input assigned in every test, including false facts. Paired positive/blocking tests cover $20 versus $21; qualifying member versus generic disability only; payment/timing/State election; actual-cost eligibility without an elderly/disabled member; mandatory-standard exceptions; no-cost households; homeless conditions and anti-stacking; cap exceptions; internet costs; and both k/4 age/disability directions. Broad blanket deferrals are prohibited. + +**Disability determination.** The head in scenario_080 is 41, and scenario_100's head is 46. Generic `is_disabled=true` establishes none of §2012(j)'s enumerated routes. The record states no SSI/SSDI or other qualifying assistance, governmental disability retirement, qualifying VA/survivor status, or Railroad Retirement facts. Therefore **neither head is established as a SNAP disabled member on the stated facts**; using the benchmark's absent-input defaults, the resolved predicate is false. This does not mean benefit receipt is always necessary: some statutory VA-status routes independently qualify. Nor does generic Medicaid eligibility or model-generated Medicaid spending establish receipt of disability-related assistance meeting the specified standards. [Official definition](https://uscode.house.gov/view.xhtml?req=%28title%3A7+section%3A2012+edition%3Aprelim%29). + +The supplied general audit files stop before r30; the actual affected-row evidence is `triage/sweep/out/r30_on_c13v3.csv`, retained locally as [affected_outputs.csv](scratch/us-2014-sua/affected_outputs.csv). Exactly two rows move. The specified and frozen-bundle household JSONs were checked equal. Household-level reruns of the supplied c13+r30 reform reproduced both corrected annual values exactly; [results](scratch/us-2014-sua/household_results.json) and [arithmetic checks](scratch/us-2014-sua/arithmetic_checks.json) preserve the distinction between reproduced values and independently derived law. + +| Worked benchmark | PE 1.755.4 frozen | PE with FY2026 convention | Supplied utility correction | Independent source result | +|---|---:|---:|---:|---| +| scenario_080, PA | $3,596.039794921875 | $3,576.00 | $3,244.799560546875 | SUA receipt route false; utility and excess-shelter deductions $0. With other benchmark assumptions held, whole-dollar allotment is $270/month, **$3,240/year**. | +| scenario_100, MT | $8,625.8896484375 | $8,566.7998046875 | $6,936.00 | SUA false; utility and excess-shelter deductions $0. $6,936 follows the benchmark net-income floor; retaining net-income cents gives **$6,924/year**. MT's election remains unverified. | + +**Case 1 — scenario_080.** One adult, age 41; no stated utilities, rent or mortgage payments. A $93,000 mortgage balance is not a shelter payment. Financial assistance is $3,600/year, dividends $16, long-term capital gains $924, energy subsidy $2,000, bank assets about $19,828. Even assuming all $2,000 is qualifying timely LIHEAP, C(iv)(I)'s elderly/disabled condition is unmet. C(ii)(I) also supplies no actual-cost route. The independent provision result is **zero SUA and zero excess-shelter deduction**, supporting the corrected treatment. + +Holding the benchmark's income classification, eligibility and published FY2026 schedule fixed: `(3600 + 16)/12 − 209 = 92⅓` monthly net income. The supplied comparator floors this to $92, yielding `298 − .30 × 92 = 270.40`. The separate federal whole-dollar benefit rule yields $270, or $3,240 over twelve identical months; Pennsylvania's own rounding rule produces the same result here. Thus **the statute supports removal of the allowance, but neither $3,576 nor $3,244.80 is the fully rounded annual amount under these held assumptions**. This is not a finding filed against any oracle. [Federal rounding](https://www.govinfo.gov/content/pkg/CFR-2025-title7-vol4/pdf/CFR-2025-title7-vol4-part273.pdf), [Pennsylvania rule](https://www.pacodeandbulletin.gov/secure/pacode/data/055/chapter501/s501.9.html), [FY2026 amounts](https://fns-prod.azureedge.us/sites/default/files/resource-files/snap-fy26maximumAllotments-deductions.pdf). + +**Case 2 — scenario_100.** Head age 46 and children ages 6 and 5; earnings $5,914.720703125/year; no energy subsidy, utility bills or housing costs. Generic head disability supplies no §2012(j) route. No payment means the receipt route fails even before considering the new age/disability limitation: this case demonstrates why a State mandatory-standard flag cannot create eligibility without qualifying costs. The independently derived deductions are again **zero**, supporting the corrected treatment. + +The rerun supplies annual TANF of $6,064.958984375 as an upstream computed boundary, not a household-stated payment. Holding it and other benchmark rules fixed gives monthly net income `5914.720703125/12 × .8 + 6064.958984375/12 − 209 = 690.727962…`. Flooring net income gives $578/month and the supplied $6,936/year. Preserving cents and rounding the 30% contribution upward gives `785 − ceil(207.218388…) = 577`, or $6,924/year. The scoped corpus/manual and [official Montana budgeting policy](https://dphhs.mt.gov/assets/hcsd/snapmanual/SNAP601-1.pdf) did not establish Montana's cents election. **The statute supports the corrected no-SUA result; it does not independently establish the exact $6,936 total. Neither annual figure can be certified unconditionally from this provision alone.** + +**Case 3 — qualifying-member contrast.** State elects a heating/cooling standard; a member is 60, or independently satisfies a qualifying §2012(j) disability route; timely qualifying payment is $21; actual utility costs are zero. C(iv)(I) requires the allowance to be available. Flip only qualifying-member status to false: no receipt entitlement. Set payment to exactly $20 or outside the prior-12-month window: no receipt entitlement. No State dollar amount is invented. + +**Case 4 — actual-cost and k/4 contrasts.** An under-60 household with no qualifying disabled member, no energy payment, and $100 of qualifying actual heating expense retains the cost route when the State lawfully uses the standard; removing the expense removes that route. Separately, for a State-law energy payment of $100 and covered expense of $80, no other exceptions, k(4)(A) deems $100 directly payable when qualifying-member status is false, while k(4)(B) deems $0 household-paid. Flip that status to true: the amounts are $0 under (A) and $80 under (B). These are provision-specific deeming amounts, not a complete income-inclusion or allotment calculation. + +All annual illustrations use PolicyBench's twelve-month FY2026 freeze. They do not establish actual October–December FY2027 amounts, every other eligibility rule, the subsidy program/timing, ongoing-case recertification history, or upstream TANF entitlement. The new source units should return the legal gates/deductions, not pretend to compute a complete annual SNAP award. + +# 7. Risks + +| Rejection or interpretation risk | Preparation that addresses it | +|---|---| +| Signing/validator identity mismatch | Named protected pin-alignment prerequisite; current commands withheld. Recheck main and compatible refs after it lands. | +| Citation does not resolve | Use actual e/6 and k/4 corpus records, already in signed release; do not dispatch invented C/iv/I. | +| Incomplete source unit | Finding maps all e/6 A–E branches and both k/4 branches, with specific external dependencies for allowed deferrals. | +| Encoder imports broken legacy modules despite `[]` | Finding explicitly requires no existing-module imports and an empty sequence, not null. Review generated closure before acceptance. | +| Old regulatory rule overrides amended statute | The existing gate run documents missing restriction; encode controlling amended statute. | +| Generic disability substituted for §2012(j) | Explicit legal-status boundary and contrast tests; no invented benefit receipt or VA status. | +| Proof/test rejection | Exact corpus excerpts and every local input explicitly set, including false; paired exception tests. No new waiver exit. | +| Overclaiming full household adjudication | Separate statutory deduction result from comparator arithmetic, benefit rounding, MT uncertainty and the freeze convention. | + +Actual failure evidence was read for [run 35789753522](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522): a null-import attempt failed, and later attempts imported `us:statutes/42/416/l`, whose plural citation field prevented compilation. An empty requested import list alone is insufficient protection. This supports the explicit standalone constraint in the finding. + +Open PR searches in rulespec-us and axiom-encode for 2014/273/9 found no exact e/6, k/4 or utility-leaf signed encoding collision. Nearby open work includes rulespec-us #1138, #850, #853/#854 and #891. #1307 and #1363 concern shared metadata; checked file lists do not touch these target paths, but #1363 touches 273/10, 2017/a, COLA, 26/32 and shared compositions. The 30 most recent dispatch runs contained no target collision; today's three were Canada runs 35855033586, 35807158166 and 35807156409. These are point-in-time checks and must be repeated before dispatch. [Saved PR/run evidence](scratch/us-2014-sua/workflow/evidence-summary.json). + +Remaining uncertainty is explicit: exact build provenance of the local strict binary, live eCFR conformance after the retained vintage, MT income rounding, unstated statutory disability/program facts, and future main/toolchain SHAs. A signed run and its generated artifacts do not yet exist. Source readiness is established; guaranteed future encoder success is not claimed. + +Validation completed: affected-row inventory and household-fact equality checks; corrected household reruns matching both sweep values; independent decimal arithmetic; existing-module compile checks and one all-input Axiom gate execution; read-only workflow/ref/collision checks; reference-command shell syntax. No population simulation or external mutation occurred. + +```json +{ + "lane": "us-2014-sua", + "verdict": "READY-AFTER", + "citation": "us/statute/7/2014/e/6", + "in_corpus": true, + "in_pinned_release": true, + "existing_modules": [ + "us/regulations/7-cfr/273/9.yaml", + "us/regulations/7-cfr/273/9/d/6/iii.yaml", + "us/statutes/7/2012/j.yaml", + "us/statutes/7/2014/a.yaml", + "us/statutes/7/2014/c.yaml", + "us/statutes/7/2014/d.yaml", + "us/statutes/7/2014/e/2.yaml", + "us/statutes/7/2014/e/2/B.yaml", + "us/statutes/7/2014/e/6/A.yaml", + "us/statutes/7/2014/g.yaml", + "us/regulations/7-cfr/273/10.yaml", + "us/policies/usda/snap/fy-2026-cola/income-eligibility-standards.yaml", + "us/policies/usda/snap/fy-2026-cola/deductions.yaml", + "us/policies/usda/snap/fy-2026-cola/maximum-allotments.yaml" + ], + "blocking_imports": [], + "dispatch_command": "gh workflow run targeted-signed-reencode.yml -R TheAxiomFoundation/axiom-encode --ref main \\\n -f 'citation=us/statute/7/2014/e/6' \\\n -f 'country=us' \\\n -f 'rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704' \\\n -f 'pr_base_branch=main' \\\n -f 'corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106' \\\n -f 'rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca' \\\n -F 'review_finding=@scratch/us-2014-sua/review_finding.txt' \\\n -f 'repair_run_id=' \\\n -f 'source_bundle_json=[]' \\\n -f 'existing_signed_imports_json=[]' \\\n -f 'replace_rulespec_path=' \\\n -f 'replace_legacy_rulespec_path=' \\\n -f 'legacy_exact_dependent_rulespec_path=' \\\n -f 'second_legacy_exact_dependent_rulespec_path=' \\\n -f 'legacy_retained_successor_rulespec_paths_json=[]' \\\n -f 'dependent_citation=' \\\n -f 'dependent_review_finding=' \\\n -f 'second_dependent_citation=' \\\n -f 'second_dependent_review_finding=' \\\n -f 'open_pr=true' \\\n -f 'queue_id=' \\\n -f 'queue_item_id=' \\\n -f 'queue_manifest_sha256=' \\\n -f 'queue_item_generation_sha256=' \\\n -f 'queue_dispatcher_run_id='", + "prerequisites": [ + "Land a dedicated protected validation-toolchain and exact-fixture alignment to the encoder identity that will sign the run; current f856cfcb/0.2.2006 validator does not match 5d80d753/0.2.2015 signing workflow.", + "After prerequisite and serial-chain merges, refresh exact rulespec main and compatible corpus/engine refs, verify source and absent destinations, and repeat collision checks; displayed command is current-base reference only.", + "Obtain Max approval for each signed e/6 and k/4 run; no dispatch was made." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_us-219.md b/reference_audit/2026-09-22/verification/v9_axiom_us-219.md new file mode 100644 index 00000000..7748dac4 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_us-219.md @@ -0,0 +1,273 @@ +# us-219 — Axiom signed-encoding preparation + +## 1. Verdict + +**READY — dispatch the source-bundle command below against the verified current main.** Max's approval is required before the orchestrator executes it. This lane did not dispatch, encode, author RuleSpec/test YAML, change external repositories, or commit. + +The package adds a new whole-section `us:statutes/26/219` module and first signs the annual parameters in Notice 2025-67 page 4. It supplies the section 219 arithmetic needed for the ten benchmark households, with precisely identified external calculation/classification boundaries. It does **not** claim to independently derive every possible filing status, plan qualification, or upstream income component. In particular, legal surviving-spouse classification remains an explicit upstream dependency, rather than a fabricated local input. + +**The statute supports the sandbox-corrected IRA deduction in all ten households.** Nine corrected return deductions are zero; scenario_064 retains the head's $108.19200134277344 and removes the dependent's $18.031999588012695. This adjudication concerns the IRA deduction; section 219 alone does not independently establish an entire federal or state tax liability. + +No source-membership prerequisite requires waiting for the serial re-pin chain. Both needed source scopes are already in today's pinned release selector and in the proposed successor. The command is valid only while rulespec-us main remains `f43dec520dd392bc5333934f56dad7498363e704`, rechecked through the GitHub connector at **2026-09-23 02:33 UTC / September 22 ET**. The workflow requires an exact current main tip for `open_pr=true`. If orchestration waits for #1387, #1386, and the subsequent drift-stage/activation/re-pin, refresh the immutable refs and repeat the collision check; do not dispatch the stale command. Waiting is an operational sequencing choice, not a demonstrated dependency of this provision. + +The required `git fetch` was attempted and denied at the external checkout's `.git/FETCH_HEAD`. The local `origin/main` SHA matched the independently read remote main. All repository inspection used immutable `git show`/`git ls-tree` snapshots or read-only GitHub GETs. The subfleet manifest names an external `-o` destination, `/Users/maxghenis/PolicyEngine/_wk/axenc-pb/report-us-219.md`; direct writes there are outside this lane's allowed workspace. The report and its dispatch attachments are supplied in this workspace for orchestration to collect. + +## 2. Source + +Official statutory locations: [OLRC current section 219](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section219&num=0&edition=prelim), [OLRC retained release-point XML archive](https://uscode.house.gov/download/releasepoints/us/pl/119/100/xml_usc26@119-100.zip), and the independently readable [GovInfo section 219 text](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partVII-sec219.htm). OLRC live HTML timed out/returned 403; the retained corpus XML and GovInfo text were read. The latter includes the 2022 catch-up-indexing amendment. + +Official annual authority: [IRS Notice 2025-67, page 4](https://www.irs.gov/pub/irs-drop/n-25-67.pdf). Its 2026 IRA ceiling is $7,500, with $1,100 catch-up at age 50; own-active phase-outs are $81,000–$91,000 for single/HOH and $129,000–$149,000 for joint returns, spouse-only coverage is $242,000–$252,000, and married-separate coverage remains $0–$10,000 absent the living-apart exception. + +Canonical corpus `origin/main`: **`942e138e7a8250c9814e774ac9b8e63008148106`**, verified locally and remotely. + +| Source | Exact canonical citation | Tracked provision file and vintage | +|---|---|---| +| Whole statute, selected primary | `us/statute/26/219` | `data/corpus/provisions/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup.jsonl`; `source_as_of` and expression date `2026-07-13`; publication `Online@119-100`, source metadata created `2026-04-17` | +| Required statutory descendants | `us/statute/26/219/b/1`, `/b/5`, `/c`, `/f/1`, `/g` | All present in that same file, together with the rest of (a)–(g) and their paragraph records | +| Annual parameter source | `us/guidance/irs/notice-2025-67/page-4` | `data/corpus/provisions/us/guidance/2026-07-23-irs-notice-2025-67.jsonl`; `source_as_of=2025-11-13`, expression date `2026-01-01`, version `2026-07-23-irs-notice-2025-67` | + +The statute's retained artifact is `data/corpus/sources/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup/official-documents/usc26-section-219.xml`, with adjacent `provenance/usc26-section-219.xml.json`. Its measured SHA-256 is `b11cd5cea06f2621c32c65274aac5ebb799c89ef0e1405be391e2c6f06f120df`, exactly matching provenance. Notice's retained artifact is `data/corpus/sources/us/guidance/2026-07-23-irs-notice-2025-67/official-documents/irs-notice-2025-67.pdf`. + +**In corpus: yes. In pinned release: yes, verified through its tracked scope selector.** The rulespec-us toolchain pins `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA-256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`. `manifests/releases/us-rulespec-2026-08-08-obbb-alien-snap.json` contains both exact source versions above. The successor `manifests/releases/us-rulespec-2026-08-23-canada-338-suspension-union.json` also contains both; the lane asserted these memberships. No ingest is needed. + +Limitation: `axiom-locate release` found no materialized local release, and the public R2 release object was inaccessible through shell DNS and web. Membership is established from the canonical tracked selector, not a fresh verification of the public object's signature/content hash. The signed workflow itself retrieves and verifies/materializes that object before encoding (workflow lines 827–876). + +## 3. Existing rulespec-us coverage + +All claims below concern **`f43dec520dd392bc5333934f56dad7498363e704`**. The relevant primary modules are exactly: + +| Module | Current coverage | +|---|---| +| `us/statutes/26/219/b.yaml` | Compensation minimum, age catch-up, statutory base figures, SEP/SIMPLE exclusions, and the separate 501(c)(18) limitation. Does not derive compensation/spousal capacity or apply annual indexing. | +| `us/statutes/26/219/g.yaml` | Participation categories, exception/applicability predicates, and a forwarded MAGI input. Monetary threshold selection and phase-out outputs are explicitly deferred. | +| `us/policies/irs/notice-2025-67/savers-credit.yaml` | Nine section 25B Saver's Credit parameters; no section 219 annual ceiling/phase-out parameters. | + +There is no parent `219.yaml`, no (c)/(f) module, and no Notice `page-4.yaml`. Fresh parent/source destinations, companions, and canonical manifest destinations were checked absent. + +The earlier compensation assessment needs correction: Axiom **already has** the minimum at the immediate input boundary. At [219/b line 200](https://github.com/TheAxiomFoundation/rulespec-us/blob/f43dec520dd392bc5333934f56dad7498363e704/us/statutes/26/219/b.yaml#L200): + +```text +formula: min(deductible_amount_before_compensation_limit, max(0, compensation_includible_in_gross_income)) +``` + +But lines 27–29 and 58–60 assign `formula: '5000'` and `formula: '1000'`, each effective from `'1990-01-01'`; lines 177–180 add the latter at age 50. Existing-module execution for 2026 returned maximum general deduction limits of **0, 3,000, 5,000, and 6,000** for `(age, compensation)` `(40,0)`, `(40,3000)`, `(40,10000)`, `(50,10000)`. Thus the stale-ceiling observation is confirmed. These are existing maximum-limit outputs, not complete IRA deduction computations. + +At [219/g lines 15–39](https://github.com/TheAxiomFoundation/rulespec-us/blob/f43dec520dd392bc5333934f56dad7498363e704/us/statutes/26/219/g.yaml#L15), deferred outputs include: + +```text +us:statutes/26/219/g#applicable_dollar_amount +us:statutes/26/219/g#dollar_limitation_after_subsection_reduction +us:statutes/26/219/g#limitation_reduction +us:statutes/26/219/g#limitation_reduction_before_rounding +us:statutes/26/219/g#phaseout_denominator +``` + +The recorded reason is: + +> Generated rule treated a filing-status or marital-status legal classification as a local fact. This output is deferred until the upstream status source can be encoded or imported without inventing local tax-status component inputs. + +Tests with phase-out names assert participation/applicability rather than dollar results. The new finding requires actual monetary assertions. The existing active-participant expression also globally excludes participation when a reserve/firefighter exception holds; new tests must preserve independently qualifying participation in another plan, as required by the statutory “solely because” limitation. + +Evidence: `scratch/us-219/coverage/coverage-notes.md`, exact extracted existing module bytes, and saved compiled requests/results. No extracted RuleSpec bytes were authored or repaired. + +## 4. Import closure + +**Candidate `existing_signed_imports_json`: `[]`.** No relevant signed-v5 dependency is presently available. The two 219 modules have legacy `applied-rulespec/v1` HMAC manifests; the Saver's Credit module has v1/manual provenance. A repository-wide manifest search found exactly two v5 manifests, for unrelated Rev. Proc. 2025-32 page 15 and section 7/2015/f. + +The intended new closure is: + +```text +us:statutes/26/219 + -> us:policies/irs/notice-2025-67/page-4 [fresh signed source in this run] +``` + +The workflow signs the fresh source first and requires its direct import into the target. No existing module is in that proposed closure. Both new modules await generation and compilation; this lane cannot claim an execution result for nonexistent artifacts. + +| Existing nearby module | Imports | Actual local load check | Current signed-runtime significance | +|---|---|---|---| +| 219/b | None | Compiled; 4 derived outputs; four 2026 execution cases passed | Not a v5 import candidate; current pinned-engine load not independently run | +| 219/g | None | Compiled; 7 derived outputs | Not a v5 import candidate; current pinned-engine load not independently run | +| Notice savers-credit | None | Compiled with located older binary | Declares removed plural `corpus_citation_paths`; unsafe for current signed runtime and irrelevant to IRA values | + +`axiom-locate engine` selected `/Users/maxghenis/TheAxiomFoundation/_tariff-parity/axiom-rules-engine/target/release/axiom-rules-engine`, binary SHA-256 `674ca6e70afdccb59c3d6847933bc24b4590105e49db54790f2dcd0bdbbe32d7`; its checkout is at `ffd8213271947b0189a9dd61a055c1e0e78908a0`. It accepts the removed plural field, so its successful checks are explicitly **not** proof of compatibility with the current signed runtime. + +The current-runtime rejection is evidenced by the actual log of [run 35789753522](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522): an automatically selected `us:statutes/42/416/l` import caused `declares removed plural corpus_citation_paths`. That run also specified `existing_signed_imports_json=[]`: an empty explicit list does not disable automatic repo-augmented imports. Therefore the finding explicitly excludes broken/legacy closures. Neither current 219 module reaches 26/32. The proposed closure has **no known blocking import**; the known 26/32 → legacy EIC closure must not be added automatically. + +## 5. Dispatch inputs + +Inspected workflow/helper revision: **`5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`**, [targeted signed workflow](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/.github/workflows/targeted-signed-reencode.yml). The required `--ref main` is the workflow's protected dispatch branch; it is not an immutable workflow selector. Recheck that workflow revision if main changes. + +| Input identity | Verified origin | +|---|---| +| `rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704` | Local `origin/main` and remote branch GET; includes merged #1384 | +| `corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106` | Canonical corpus `origin/main` and remote main GET | +| `rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca` | Today's rulespec-us `.axiom/workflow-toolchain.toml`; the same ref successfully checked out/built in run 35789753522 | +| `citation=us/statute/26/219` | Exact resolved corpus parent record | +| `replace_rulespec_path=` | Empty intentionally: new parent, not replacement of (b) or (g) | +| `source_bundle_json=["us/guidance/irs/notice-2025-67/page-4"]` | Existing canonical source, absent canonical module destination | +| `existing_signed_imports_json=[]` | Fresh source supplied atomically; no suitable existing v5 module | +| `review_finding` | Complete file `scratch/us-219/review-finding.txt`, SHA-256 `aefa0a37f8033656147c4a058448f2d6e5c51959acb4fffed76d56a45624d7ac` | + +Run from this assigned workspace so the findings attachment resolves. **This is a prepared command, not an executed command.** All 25 workflow inputs are explicitly supplied, including empty optional values. `gh workflow run --help` confirmed `-F key=@file` reads the file's contents. + +```sh +gh workflow run targeted-signed-reencode.yml \ + -R TheAxiomFoundation/axiom-encode --ref main \ + -f citation=us/statute/26/219 \ + -f country=us \ + -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \ + -f pr_base_branch=main \ + -f corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106 \ + -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \ + -F review_finding=@scratch/us-219/review-finding.txt \ + -f repair_run_id= \ + -f 'source_bundle_json=["us/guidance/irs/notice-2025-67/page-4"]' \ + -f 'existing_signed_imports_json=[]' \ + -f replace_rulespec_path= \ + -f replace_legacy_rulespec_path= \ + -f legacy_exact_dependent_rulespec_path= \ + -f second_legacy_exact_dependent_rulespec_path= \ + -f 'legacy_retained_successor_rulespec_paths_json=[]' \ + -f dependent_citation= \ + -f dependent_review_finding= \ + -f second_dependent_citation= \ + -f second_dependent_review_finding= \ + -f open_pr=true \ + -f queue_id= \ + -f queue_item_id= \ + -f queue_manifest_sha256= \ + -f queue_item_generation_sha256= \ + -f queue_dispatcher_run_id= +``` + +Verified constraints: full lowercase 40-character refs; corpus/engine main ancestry; exact current rulespec main when opening a PR; canonical same-jurisdiction source citations; fresh source/target destinations absent; existing imports expressed as file paths and verified v5 if used. The source bundle executes before the parent (workflow 1927–1937, 1970–1976) and enforces direct imports (1357–1360, 1664–1677, 1737–1739). Source lanes receive an empty review finding; this matters for the Notice-page risk below. + +The encoding regime for any downstream lane remains verbatim: + +> RuleSpec content in any `rulespec-*` repository is produced ONLY by the supervised +> encoder: `axiom-encode encode --backend codex --apply` (local +> supervised runtime, subscription Codex auth via a lane CODEX_HOME; never +> OPENAI_API_KEY). Every atomic module carries the encoder's apply manifest under +> `.axiom/encoding-manifests/`. Hand-written YAML is never a module — not for a +> pilot, not for a demo, not "to avoid API spend". The only hand edits allowed are +> repair rounds on the encoder's output (findings file + replay) on repos whose +> `run-generated-guard` is off, and composed `module.kind: composition` pipelines, +> which are assembled, not encoded. Briefs to lanes must say this verbatim; a brief +> that says "hand-author" is wrong. New repos set `run-generated-guard: true`. + +For rulespec-us specifically, use only the protected signed workflow: its job is in `production-signing`, and the encode step invokes `/opt/axiom-verification/axiom-encode encode --backend openai`. No local encode is authorized. [Binding agent rules, issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) were read and retained in scratch. + +## 6. Review finding text + +**The exact `review_finding` input is the attached [review-finding.txt](/Users/maxghenis/.subfleet/worktrees/20260922-222028-axpb-us-219/scratch/us-219/review-finding.txt).** It contains the complete prose specification, four case families, all ten household adjudications, conservative MAGI bounds, and all 16 affected output rows. The following is the reviewable substance. + +Cover the complete parent source (a)–(g), not only the two benchmark triggers: qualifying contributions; compensation and indexed dollar minima; lower-earning spousal capacity; all compensation inclusions/exclusions, including the special section 1402(c)(6) service rule in (f)(1); per-person treatment; complete active-participant phase-out with its $10 rounding, positive $200 floor and zero endpoint; all plan categories and exceptions; SEP/SIMPLE/501(c)(18); rollover/inherited/endowment restrictions; timely prior-year payments; excess carryforwards and closed-year reductions; election cross-reference; and repealed paragraphs. Import the new annual Notice outputs. Do not hardcode ungrounded Notice amounts under a statutory proof or carry forward unindexed base amounts. + +Precisely defer only external determinations: section 401(c)(2) earned income, section 3401(h)(2) qualification, referenced plan/account qualification, section 415(c)(3) compensation, section 4973 excess history, general income computations under sections 86/469 and the specified disregards, actual-return legal eligibility, account/election administration, and surviving-spouse classification through the complete section 2(a) dependency chain. Keep section 219 arithmetic and its explicit blocking effects runnable on established underlying amounts/facts. Do not defer the monetary phase-out, compensation minimum, or 2026 ceiling. + +Current encoder source prohibits local numeric/string `filing_status` and legal-status aliases: `cli.py:55930` invokes that hard apply check. Its `US_TAX_PACK` prompt expressly permits source-backed facts about whether a joint/separate return was actually made. The finding uses that factual boundary, distinguishes it from filing eligibility, and prohibits renaming a classification to evade the guard. It specifically requires a precise unresolved survivor-selection contract rather than silently applying the single threshold. This is conditional statutory arithmetic, not universal upstream tax-status adjudication. + +Require paired positive/blocking tests for each exception, including reserve 90/91 days, firefighter $1,800/more, and an excepted plan plus a separately qualifying plan; 457(b)-only/other-plan; own/spouse-only/no participation; all-year apart/any cohabitation; valid contribution/rollover/inherited/life-insurance; qualifying compensation/pension/deferred; spouse/nonspouse; timely/late payment; and carryforward/no carryforward. Every local `#input` must be assigned, including false ones. Proof excerpts must be verbatim, singular-path anchored. No new failure has a waiver exit. + +The deduction derivation is independent of the sandbox docstrings: section (a) supplies qualified contributions; (b)(1) supplies the lesser of dollar amount and compensation; (c) only extends capacity to a lower-earning joint spouse; (f)(1) excludes retirement receipts; and (g) reduces the dollar ceiling to zero at the relevant upper endpoint. For r02, actual modeled positive 401(k) allocations establish the represented qualifying-plan activity; they are not a universal proxy for participation. [IRS coverage guidance](https://www.irs.gov/retirement-plans/are-you-covered-by-an-employers-retirement-plan) confirms the allocation interpretation. Desired deferrals alone are not independent real-world evidence of completed allocations; this is the explicit benchmark-fixture interpretation. + +| Household | Relevant facts | Baseline IRA deduction | Corrected IRA deduction | Independent §219 deduction and adjudication | +|---|---|---:|---:|---| +| scenario_003 (TX, joint) | Head age 64, spouse 61; both wages and every represented SE source zero. Head requests traditional $1,117.9840087890625 and Roth $1,734.0159912109375; receipts are investment/retirement income. | $1,117.9840087890625 | $0 | $0. Both compensation amounts are zero, so §219(c) supplies no spousal capacity. Supports corrected deduction. Desired 401(k) inputs do not change the compensation result. | +| scenario_064 (WI, joint) | Head age 48 has wages $97,295, pretax 401(k) $2,315.39990234375, taxable wages $94,979.6015625, net represented farm/partnership SE sources −$2,800; head traditional IRA $108.19200134277344. Disabled dependent age 27 has no wages/SE earnings and traditional IRA request $18.031999588012695. | $126.22400093078613 (head $108.19200134277344 + dependent $18.031999588012695) | $108.19200134277344 | Head $108.19200134277344; dependent $0 on parents' return. Dependent has no compensation and is not a joint spouse; the head's compensation easily covers their own contribution. Joint income is below the active-contributor starting threshold. Supports corrected deduction. | +| scenario_085 (PA, single) | Age 67; no wages/SE earnings; Social Security, pensions, distributions and interest only. Traditional request $0.3967039883136749; Roth $0.6152960062026978. | $0.3967039883136749 | $0 | $0. None of the represented retirement/investment receipts supplies compensation. Supports corrected deduction. The sub-dollar contribution is still subject to the compensation limit. | +| scenario_005 (CA, joint) | Ages 35/33, wages $180,000/$250,000; actual modeled traditional 401(k) $20,825 each (requests $23,154 each), Roth deferrals positive; traditional IRA $1,081.9200439453125 each. Joint §219 MAGI $538,928.375. | $2,163.840087890625 | $0 | $0 for each spouse. Each contributor is active and joint MAGI exceeds $149,000. Supports corrected deduction. | +| scenario_049 (NH, joint) | Ages 53/52; head wages $0/no IRA request; spouse wages $253,000, traditional 401(k) $15,436 and Roth 401(k) $2,724; spouse traditional IRA $721.280029296875. Joint §219 MAGI $247,385. | $721.280029296875 | $0 | $0. The contributor is the active spouse, so the $149,000 upper endpoint applies. The higher spouse-only-coverage threshold does not apply to that contributor. Supports corrected deduction. | +| scenario_052 (TX, joint) | Ages 59/55; head wages $500,000, traditional/Roth 401(k) $926.1599731445312/$163.44000244140625, traditional IRA $43.276798248291016; spouse wages $0/no IRA request. Joint §219 MAGI $504,821.84375. | $43.276798248291016 | $0 | $0. Active contributing head exceeds joint $149,000 endpoint. Supports corrected deduction. | +| scenario_082 (NY, HOH) | Age 23, wages $100,195; traditional/Roth 401(k) $181.3730010986328/$32.00699996948242; traditional IRA $8.475040435791016. §219 MAGI $117,593.625. | $8.475040435791016 | $0 | $0. Active HOH exceeds $91,000 endpoint. Supports corrected deduction. | +| scenario_099 (CA, joint) | Ages 42/41, wages $85,000/$80,000; each traditional/Roth 401(k) $1,543.5999755859375/$272.3999938964844; each traditional IRA $72.12799835205078. | $144.25599670410156 | $0 | $0 each. Taxable wages alone are about $161,912.80; even subtracting $3,000 capital loss and $337.50 requested educator expense leaves $158,575.30, above $149,000. Both contributors active. Supports corrected deduction. | +| scenario_110 (OH, single) | Age 51, wages $100,000; traditional/Roth 401(k) $4,244.89990234375/$749.0999755859375; traditional IRA $198.3520050048828; substantial additional investment income. | $198.3520050048828 | $0 | $0. Taxable wages alone are $95,755.10, above $91,000; no other represented above-line deduction reduces that below the endpoint. Active contributor. Supports corrected deduction. | +| scenario_120 (CT, single) | Age 76, wages $165,597; traditional/Roth 401(k) $617.4400024414062/$108.95999908447266; traditional IRA $28.851200103759766; additional pension/SS income. | $28.851200103759766 | $0 | $0. Taxable wages approximately $164,979.56 remain far above $91,000 even allowing the full represented $2,446.588134765625 farm-rent loss; active contributor. The source has no age-70½ bar: §219(d)(1) is repealed. Supports corrected deduction. | + +The conservative MAGI arguments deliberately do not depend on adjudicating unrelated positive-income treatments. The lower bounds use modeled taxable wages minus every represented non-IRA/non-disregarded above-line deduction, leaving out all other positive income. All seven bounds independently exceed their phase-out endpoints: + +| Scenario | Conservative MAGI lower bound | Sandbox §219 MAGI | Complete-phase-out endpoint | +|---|---:|---:|---:| +| 005 | $385,350 | $538,928.375 | $149,000 | +| 049 | $237,564 | $247,385 | $149,000 | +| 052 | $496,073.84375 | $504,821.84375 | $149,000 | +| 082 | $97,013.625 | $117,593.625 | $91,000 | +| 099 | $158,575.296875 | $159,729.296875 | $149,000 | +| 110 | $95,755.1015625 | $163,620 | $91,000 | +| 120 | $162,532.97436523438 | $232,663.96875 | $91,000 | + +All gross/ALD components are retained in `intermediates.json`. None of the IRA requests approaches the old or new statutory dollar ceiling, and all seven r02 cases are entirely phased out, so changing the base/catch-up figures cannot change these results. + +**Observed downstream outputs from the complete sweep** + +F = `federal_income_tax_before_refundable_credits`; S = `state_income_tax_before_refundable_credits`. The statute supports the corrected IRA treatment in each row. These are phase-only/compensation-only sweep totals, not claims that all other policy components are independently correct. + +| Scenario | Output | Frozen | Sandbox corrected | Delta | +|---|---|---:|---:|---:| +| 003 | F | 22,154.699219 | 22,400.654297 | 245.955078 | +| 064 | F | 4,439.289551 | 4,441.455078 | 2.165527 | +| 064 | S | 4,605.997070 | 4,607.142090 | 1.145020 | +| 085 | F | 2,322.760010 | 2,322.848633 | 0.088623 | +| 005 | F | 106,505.898438 | 107,198.335938 | 692.437500 | +| 005 | S | 41,051.511719 | 41,276.824219 | 225.312500 | +| 049 | F | 30,543.908203 | 30,702.589844 | 158.681641 | +| 052 | F | 104,211.406250 | 104,225.257812 | 13.851562 | +| 082 | F | 9,563.052734 | 9,564.916992 | 1.864258 | +| 082 | S | 5,598.552246 | 5,599.143555 | 0.591309 | +| 099 | F | 10,679.750000 | 10,711.484375 | 31.734375 | +| 099 | S | 4,493.743164 | 4,505.339844 | 11.596680 | +| 110 | F | 23,897.443359 | 23,941.082031 | 43.638672 | +| 110 | S | 4,057.472168 | 4,062.927002 | 5.454834 | +| 120 | F | 40,021.816406 | 40,028.320312 | 6.503906 | +| 120 | S | 11,917.183594 | 11,919.057617 | 1.874023 | + +Scenario_085 F and scenario_082 S are below the sweep's $1 cutoff but are included because the assignment requires every moved household/output. + +Four worked-case families beyond the affected households: + +1. **2026 dollar and compensation limits:** qualifying traditional contributions $9,000, compensation $20,000, no active plan for taxpayer/spouse: age 49 allows $7,500; age 50 allows $8,600. With compensation $600, the deduction is $600 at either age. A dependent's personal deduction never enters the parent's return. +2. **Spousal compensation:** joint lower earner compensation $0; higher earner compensation $8,000 and valid deductible+nondeductible+Roth contributions totaling $2,000; neither active: lower-earner qualifying contribution $7,500 allows $6,000. Change to separate filing and the deduction is $0; a dependent relationship also cannot activate §219(c). +3. **Rounding, floor and endpoint:** single active contributor, under 50, compensation/qualifying contribution at least $7,500. MAGI $86,001 yields raw reduction $3,750.75, rounded down to $3,750, deduction $3,750; MAGI $90,999 yields $200 due to the positive-limit floor; MAGI $91,000 yields $0. If contribution is only $80 at MAGI $90,999, deduction remains $80, not $200. Compensation $1,000 at MAGI $86,001 allows $1,000, proving the phase-out reduces the dollar ceiling, not the compensation ceiling. +4. **Spouse-only coverage and living apart:** under-50 nonactive joint contributor with active spouse, qualifying contribution/compensation $7,500, MAGI $247,000 → $3,750; if neither spouse active → $7,500. Separately filing active contributor with MAGI $85,000: lived apart all year → $4,500 using the single range; lived together any time → $0 using the MFS range. + +The federal/state totals in the table are **sweep observations**, not independently derived total-tax amounts. The independent statutory answer is the IRA deduction column. Accordingly, every household supports the corrected IRA treatment; neither set of whole-tax totals is fully adjudicated from section 219 alone. The distinction prevents overstating the evidence while providing every moved benchmark output. + +## 7. Risks + +- **Actual-return facts versus legal classifiers.** The new encoder must honor the current input rules. A local enum is rejected even though an older local encoder had a repair path. A source-backed actual filing event is allowed; a renamed legal status is not. Survivor status remains a named external dependency. Full raw-household, universal filing-status coverage needs subsequent signed upstream work. +- **Automatic legacy imports.** An empty explicit imports list does not prevent auto-selection, as the observed failed run demonstrates. The finding excludes legacy 219 modules, the irrelevant plural-metadata Saver's Credit module, and broken closures including 26/32. The signed run must compile its actual generated closure; this preparation cannot certify future imports. +- **Complete-source-unit coverage.** Parent 219 includes more than the two benchmark triggers. The finding enumerates all branches and exact external deferrals, plus paired tests. For fresh Notice page 4, the source lane receives no parent review finding. That page begins in the middle of section 25B's HOH paragraph and ends in the middle of a section 408A paragraph. The source encoder must encode fully supported amounts and precisely defer incomplete boundary clauses. This residual source-lane rejection risk cannot be eliminated by the parent finding; no rejection-free run is promised without running it. +- **Annual grounding and numerical boundaries.** Keep Notice proofs on the Notice path. Test age 49/50, exact thresholds, $10 reduction rounding, $200 positive floor, and actual contributions below $200. Do not apply phase-out fractions to compensation or contributions. Do not equate represented IRA contributions with automatically deductible amounts. +- **Source snapshot/engine limitations.** Public release-object verification could not be repeated here; tracked membership and retained source hashes were checked. The located binary is older than the selected signed-runtime engine. Its compilation evidence is narrowly labeled. The failure artifact's `issues.json` was not downloaded because shell networking was unavailable; only the actual GitHub job log rejection is asserted as independently read. +- **Serial-chain collisions.** #1384 is merged; [#1387 activation](https://github.com/TheAxiomFoundation/rulespec-us/pull/1387) and [#1386 workflow pin](https://github.com/TheAxiomFoundation/rulespec-us/pull/1386) remain open at `500d9df58be4e17f8572e51f665a28241b039823` and `a9dd1fb7985ccfdaf9f98548edde94ad48cd227c`. #1386 specifies ordering after activation. The measurement-only #1385 drift probe was closed; the later drift/activation/re-pin remains separate orchestration work. Neither pin file, workflow pin, CODEOWNERS nor waiver ledger is part of this feature package. +- **Same-provision collision search.** Exact open-PR searches for `26/219` and `2025-67` in both repositories found none; all 102 open rulespec-us PR metadata records were read. The latest 30 targeted signed runs and 100 recent workflow-dispatch runs, including September 22 UTC and the September 23 UTC portion of September 22 ET, contained no matching target. This is a point-in-time observation; repeat immediately before dispatch. + +**Changes/tests:** only this Markdown report and lane scratch evidence were written. Existing changes were preserved. No commit was made because this lane's specific read-only rule prohibits commits. Three existing modules compiled with the qualified older binary; four existing section 219(b) evaluations passed. Sequential household-only baseline/reform calculations verified all ten corrected IRA deductions and all seven conservative MAGI bounds; six state tax pairs reproduced the sweep within $0.000001. Ten federal tax rows were extracted from the sweep rather than recalculated under their benchmark label. Both release-selector membership checks and the retained statutory XML provenance hash passed. No population/microsimulation run, local encode, or new RuleSpec/test module was produced. + +Machine-readable evidence and complete logs are under `scratch/us-219/households/`, `scratch/us-219/coverage/`, and `scratch/us-219/workflow/`. The review finding is a prose instruction file, not RuleSpec. + +```json +{ + "lane": "us-219", + "verdict": "READY", + "citation": "us/statute/26/219", + "in_corpus": true, + "in_pinned_release": true, + "existing_modules": [ + "us/statutes/26/219/b.yaml", + "us/statutes/26/219/g.yaml", + "us/policies/irs/notice-2025-67/savers-credit.yaml" + ], + "blocking_imports": [], + "dispatch_command": "gh workflow run targeted-signed-reencode.yml \\\n -R TheAxiomFoundation/axiom-encode --ref main \\\n -f citation=us/statute/26/219 \\\n -f country=us \\\n -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \\\n -f pr_base_branch=main \\\n -f corpus_ref=942e138e7a8250c9814e774ac9b8e63008148106 \\\n -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \\\n -F review_finding=@scratch/us-219/review-finding.txt \\\n -f repair_run_id= \\\n -f 'source_bundle_json=[\"us/guidance/irs/notice-2025-67/page-4\"]' \\\n -f 'existing_signed_imports_json=[]' \\\n -f replace_rulespec_path= \\\n -f replace_legacy_rulespec_path= \\\n -f legacy_exact_dependent_rulespec_path= \\\n -f second_legacy_exact_dependent_rulespec_path= \\\n -f 'legacy_retained_successor_rulespec_paths_json=[]' \\\n -f dependent_citation= \\\n -f dependent_review_finding= \\\n -f second_dependent_citation= \\\n -f second_dependent_review_finding= \\\n -f open_pr=true \\\n -f queue_id= \\\n -f queue_item_id= \\\n -f queue_manifest_sha256= \\\n -f queue_item_generation_sha256= \\\n -f queue_dispatcher_run_id=", + "prerequisites": [ + "Max approval before any signed run; orchestrator alone dispatches.", + "At dispatch, rulespec_ref must equal current main; reverify and refresh immutable refs after any serial-chain merge." + ], + "scope": "Section 219 economic rules with signed 2026 Notice parameters; explicit external income, plan-qualification, return-validity, and survivor-classification boundaries.", + "source_bundle_json": [ + "us/guidance/irs/notice-2025-67/page-4" + ], + "review_finding_sha256": "aefa0a37f8033656147c4a058448f2d6e5c51959acb4fffed76d56a45624d7ac" +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_us-32c2.md b/reference_audit/2026-09-22/verification/v9_axiom_us-32c2.md new file mode 100644 index 00000000..284cd6b3 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_us-32c2.md @@ -0,0 +1,253 @@ +# us-32c2 — Axiom encoding preparation + +## 1. Verdict + +**BLOCKED. Do not dispatch a signed run from this package today.** The independent statutory adjudication is complete for the provision effects: the Montana and Virginia corrections are supported within the benchmark's continuous-dollar convention. California's corrected earned-income base is supported, but **neither reported California credit amount is established by the law; holding AGI and the schedule fixed, the isolated lawful credit change is $0 because AGI remains binding**. + +The following external prerequisites prevent a passing, narrow signed replacement: + +1. **Source-identity admission:** `us/statutes/26/32/c/2.yaml` currently declares `us/statute/26/32`. The inspected encoder accepts exact identity or one direct-child refinement, so a request for `us/statute/26/32/c/2` fails before generation. A supported signed identity migration or reviewed encoder change is required. The same problem affects requesting `us/statute/26/24/d/1` against existing `24/d.yaml`, which declares `us/statute/26/24`. +2. **Dependency strategy needs verification:** the current §112 and earned-income/credit dependencies have legacy v1 manifests, so they cannot be explicitly reused through `existing_signed_imports_json`. An empty array neither attests nor forbids inherited §112 in repo-augmented generation. Its current compile/proof closure remains unverified. Signed refresh is a proposed dependency strategy, not a demonstrated universal requirement for inherited imports. +3. **Downstream compile failure:** §32 imports `us:policies/irs/rev-proc-2025-32/earned-income-credit`, which declares removed plural `corpus_citation_paths`; §24(d) reaches it through §32. A signed successor/import migration must land before this credit closure can pass the pinned engine. Encoder [PR #1665](https://github.com/TheAxiomFoundation/axiom-encode/pull/1665) proposes the relevant migration, but remains open. +4. **Direct deferral classification:** §402(e)(3) is in corpus main but outside both the current release and the planned August 23 release. A bounded §32 definition can consume already-classified includible compensation; independently calculating that classification from gross wages requires a release including §402 and appropriate signed dependencies. The named serial re-pin chain alone does not satisfy this requirement. +5. **Coordinate overlapping work and regenerate exact refs:** rulespec-us #1160 directly touches `32/c/2`, `112`, and `24/d`; #1158, #1159 and #1363 overlap the consumer closure. Complete or coordinate the pending serial pin changes, then obtain a fresh exact-main SHA. No future commit SHA is invented here. + +This is not **NOT-NEEDED**: existing Axiom arithmetic can consume correct taxable wages, but does not derive the exclusion from raw pay, has a pension input-contract hazard, and lacks a runnable current signed credit closure. It is not **READY-AFTER** on the named re-pin chain alone: source admission and dependency migrations also need resolution. + +Read-only scope was preserved. Only scratch evidence, calculation/check scripts, this report and a proposed finding were written. No RuleSpec or module test YAML, encodes, commits, branches, pushes, PRs, comments, dispatches or approvals were created. The assignment's specific no-commit rule controls over the generic opening commit instruction. No `-o` destination was exposed in the supplied context, so the report is saved at `scratch/us-32c2/report.md` for the orchestrator. + +## 2. Source + +Inspected rulespec-us `origin/main`: **`f43dec520dd392bc5333934f56dad7498363e704`**. The required `git fetch -q origin` was attempted and denied because `.git/FETCH_HEAD` is outside writable scope. A read-only live GitHub ref query independently matched that local SHA. Canonical corpus main was likewise verified as **`942e138e7a8250c9814e774ac9b8e63008148106`**. Reads use immutable Git objects, not the older checkout HEAD. + +| Provision | Official text | Exact available canonical corpus unit | Corpus main | Current pinned release | +|---|---|---|---|---| +| 26 USC 32(c)(2)(A)(i) | [OLRC §32](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section32&num=0&edition=prelim) | `us/statute/26/32/c/2` | Yes | Yes | +| 26 USC 24(d)(1)(B)(i) | [OLRC §24](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section24&num=0&edition=prelim) | `us/statute/26/24/d/1` | Yes | Yes | +| 26 USC 24(h)(6), 2026 threshold substitution | [OLRC §24](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section24&num=0&edition=prelim) | `us/statute/26/24/h/6` | Yes | Yes | +| 26 USC 402(e)(3), relevant upstream exclusion | [OLRC §402](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section402&num=0&edition=prelim) | `us/statute/26/402/e/3` | Yes | **No** | +| Cal. R&TC 17052(c)(4)(A) | [California Legislature §17052](https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17052) | `us-ca/statute/rtc/17052` | Yes | Yes | + +The corpus has no separate `/32/c/2/A/i`, `/24/d/1/B/i` or `/17052/c/4/A` node. Do not invent these as dispatch citations. + +Exact tracked provision files and vintages: + +- §§24 and 32: `data/corpus/provisions/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup.jsonl`. `source_as_of` and `expression_date` are **2026-07-13**; metadata records OLRC `Online@119-100`, created **2026-04-17**. Source XML files are `data/corpus/sources/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup/official-documents/usc26-section-32.xml` and `usc26-section-24.xml`; the official source archive is [OLRC title 26, PL 119-100](https://uscode.house.gov/download/releasepoints/us/pl/119/100/xml_usc26@119-100.zip). +- §402: `data/corpus/provisions/us/statute/2026-09-13-tax-statute-closure-31-title-26.jsonl`. `source_as_of`/expression date **2026-09-02**, metadata created **2026-09-09**, `Online@119-103`; captured [OLRC title 26, PL 119-103 XML archive](https://uscode.house.gov/download/releasepoints/us/pl/119/103/xml_usc26@119-103.zip), member `usc26.xml`. +- California: `data/corpus/provisions/us-ca/statute/2026-07-06-ca-rtc-pit-core-us-ca-sections-rtc-17041-rtc-17043-rtc-17045-rtc-17052-rtc-17054-rtc-17073.5.jsonl`, source/expression date **2026-07-06**, effective **2023-01-01**, AB 1766, Stats. 2022 ch. 482 §5. Captured official HTML provenance digest: `b0109e3cbb14a528423c83e6a41ad5145e711f702baff33ae24c84204601427d`. + +`.axiom/toolchain.toml` pins **`us-rulespec-2026-08-08-obbb-alien-snap`**, content digest **`0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`**. Membership was checked against canonical tracked release manifests. `axiom-locate release` found no local materialization, so this is manifest membership verification, not signature verification of a downloaded release. The planned `us-rulespec-2026-08-23-canada-338-suspension-union` also excludes the September §402 scope; `us-rulespec-2026-09-13-federal-and-plans-union` includes it. No new ingest is needed for the listed provisions; a release change is needed for §402 classification. + +California **does follow the federal includible-gross-income condition**. Section 17052(c)(4)(A) changes the end of federal clause (A)(i) to add the California withholding condition, preserving the includibility words; (c)(4)(B) retains self-employment earnings. Do not confuse that conclusion with automatic federal/California conformity for every other payroll exclusion. The live official California page was read; live OLRC requests encountered 403/timeouts, so the exact official corpus captures govern proof excerpts. Full source records and release evidence are in [source-audit.md](corpus/source-audit.md). + +For an expanded **Roth-classification encoding**, the bounded corpus citation search did not locate §402A itself. Before preparing that separate provision, ingest [official OLRC §402A](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section402A&num=0&edition=prelim) from title-26 USLM XML with source URL, retrieval/expression date and SHA provenance, then include it in the signed release. No §402A dispatch is prepared here. The household Roth distinction is corroborated by official IRS guidance; it is not represented as an existing corpus encoding. + +## 3. Existing rulespec-us coverage + +The following inventory is at the immutable rulespec SHA above. Quoted line numbers refer to those Git objects; numbered copies are retained under `coverage/`. + +| Existing module | Coverage and relevance | +|---|---| +| `us/statutes/26/32/c/2.yaml` | Earned-income definition, exclusions and §112 election. Uses **already includible** compensation; does not itself classify elective deferrals. Thus no demonstrated gross-wage inclusion error under its stated input contract, but no independent raw-wage exclusion either. Literal pension input can subtract a pension twice. Legacy v1 provenance. | +| `us/statutes/26/32.yaml` | EITC eligibility, rates, imported annual amounts, AGI/earnings phaseout and restrictions. Imports the definition and the legacy IRS module. Not runnable with the pinned modern citation contract. | +| `us/statutes/26/24/d.yaml` | Refundable CTC arithmetic, §32 pre-election earnings plus §112 pay, social-security alternative and §911 gate. It follows the definition, rather than independently classifying wages. Its 2026 formula still uses $3,000; this is a separate Axiom composition gap. Imports §32 and inherits its loader blocker. | +| `us/statutes/26/24/h.yaml` | Special post-2017 amounts, thresholds, dependents and SSNs. Records $2,500 but explicitly defers composing it into §24(d). | +| `us/statutes/26/24.yaml` | Parent CTC eligibility/nonrefundable and advance-payment surface; imports §24(h), explicitly defers refundable computation to §24(d). Not an independent earned-income classifier. | +| `us/statutes/26/112.yaml` | Combat-pay exclusion amount, used by both earned-income consumers. Legacy v1 manifest, not a reusable signed-v5 dependency. | +| `us/policies/irs/rev-proc-2025-32/earned-income-credit.yaml` | Legacy annual EITC table imported by §32; removed plural source field. | +| `us/policies/irs/rev-proc-2025-32/page-15.yaml` | Existing signed-v5 annual EITC successor, plus other page content. Not yet the import used by §32. | +| `us-ca/statutes/rtc/17052.yaml` | California base tables/parameters; explicitly defers final CalEITC and indexed amounts. Does not compute the wage definition or the disputed 2026 refund. | +| `us/statutes/26/86.yaml` | Nearby cross-reference: line 576 treats Social Security benefits as pension/annuity for §32(c)(2), among other provisions. Does not compute wage deferral exclusion; not in this import closure. | + +No tracked `us/statutes/26/402.yaml` or `us/statutes/26/402/...` module exists on the inspected main. The bounded inventory and citation searches are reproducible from `git ls-tree`/`git grep`; no new RuleSpec was made. + +Relevant excerpts: + +```text +32/c/2.yaml:3 - us:statutes/26/112#amount_excluded_from_gross_income_by_reason_of_section_112 +32/c/2.yaml:8 corpus_citation_path: us/statute/26/32 +32/c/2.yaml:57 employee_compensation_includible_in_gross_income +32/c/2.yaml:58 + net_earnings_from_self_employment_after_self_employment_tax_deduction +32/c/2.yaml:59 - pension_or_annuity_amount +32.yaml:7 - us:policies/irs/rev-proc-2025-32/earned-income-credit +32.yaml:444 max(adjusted_gross_income, earned_income) +24/d.yaml:47 3000 +24/d.yaml:114 earned_income_before_section_112_election +24/d.yaml:115 + amount_excluded_from_gross_income_by_reason_of_section_112 +24/h.yaml:137 2500 +``` + +The §24(h) deferral explains that its substituted source value is recorded but composition into the subsection (d) formula remains deferred. The California module, lines 13–19, defers `california_earned_income_tax_credit` because federal/state dependencies are not copied, and `inflation_adjusted_earned_income_credit_amounts` because §17041(h) is not supplied. + +Four fresh executions using the discovered older Rust artifact verified the existing definition's limited contract: with already-classified wages and pension input zero, earned income was **14,664.166015625**, **2,055.720703125**, and **51,912.80078125** for scenarios 023/100/119. Giving scenario 023 its separate $8,000 pension in `pension_or_annuity_amount` returned **6,664.166015625**. This demonstrates the double-subtraction risk under the literal mapping; it does not establish that every caller maps the field that way. Requests/responses: [earned-income-checks.json](coverage/earned-income-checks.json). + +## 4. Import closure + +The actual graph is: + +```text +32/c/2 -> 112 +32 -> 32/c/2, 152/c, 7703, IRS/earned-income-credit +7703 -> 151 -> 911/a, 931, 933 +24/d -> 32/c/2, 112, 32 +24 -> 24/h +CA/17052 -> no imports (final credit deferred) +IRS/page-15 -> no imports +``` + +**The narrow definition does not itself import §32.** Its broader consumers do, so it would be incorrect to say every module in this assignment fails for the plural field. + +| Module ID (prefix `us:` except California) | Legacy discovered Rust compile | Pinned-engine assessment today | Signed-v5 reuse | +|---|---|---|---| +| `statutes/26/32/c/2` | Pass | Standalone current compile unverified; no plural field in its two-node closure | No, v1 | +| `statutes/26/112` | Pass | Current compile unverified; no plural field | No, v1 | +| `statutes/26/32` | Pass | **Cannot load: reaches legacy IRS plural field** | No, v1 | +| `statutes/26/152/c` | Pass | Current compile unverified; no plural field | No, v1 | +| `statutes/26/7703` | Pass | Current compile unverified; no plural field in closure | No, v1 | +| `statutes/26/151` | Pass | Current compile unverified; no plural field in closure | No, v1 | +| `statutes/26/911/a` | Pass | Current compile unverified; no plural field | No, v1 | +| `statutes/26/931` | Pass | Current compile unverified; no plural field | No, v1 | +| `statutes/26/933` | Pass | Current compile unverified; no plural field | No, v1 | +| `policies/irs/rev-proc-2025-32/earned-income-credit` | Pass | **Cannot load: directly declares plural field** | No, v1 | +| `statutes/26/24/d` | Pass | **Cannot load: transitive through §32** | No, v1 | +| `statutes/26/24` | Pass | Current compile unverified; no plural field in closure | No, v1 | +| `statutes/26/24/h` | Pass | Current compile unverified; no plural field | No, v1 | +| `us-ca:statutes/rtc/17052` | Pass | Current compile unverified; no plural field | No, v1 | +| `policies/irs/rev-proc-2025-32/page-15` | Not present in older checkout | No imports/plural field; exact current compile not executed | **v5 manifest present, primary digest matches** | + +Engine evidence matters here. `axiom-locate engine` selects the artifact under `_tariff-parity`, checkout `ffd8213271947b0189a9dd61a055c1e0e78908a0`; it compiled 14 existing modules whose bytes matched the inspected immutable source. It is permissive about the removed field and **cannot certify current signed validation**. A second newer local binary (`sha256 faf4383622f63c64b861e5772b78b00df97efef4a8315b792b25219033bee75e`) stopped at an outdated repository-root check rejecting `programs/`; its build commit is unverified. Neither failure nor success from that binary is attributed to the modules' current legal semantics. No rebuild or RuleSpec-copy workaround was attempted. + +The pinned engine **`af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca`** was inspected directly: `src/rulespec.rs` lines 802–833 recursively reject any key equal to `corpus_citation_paths`; `src/compile.rs` also rejects it. The pinned root validator explicitly permits declarative `programs/`, unlike the second local binary. The actual protected run [35789753522](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522) built that pin and failed on another imported module with the exact same removed-field error. Therefore the §32/§24(d) conclusion is a deterministic inference from inspected source plus the actual imported bytes, corroborated by a real pinned-engine log, **not a claimed fresh pinned compile of this closure**. All local commands/statuses are retained in [inventory.json](coverage/inventory.json), [modern-inventory.json](coverage/modern-inventory.json), and [pinned-engine-contract.txt](coverage/pinned-engine-contract.txt). + +Candidate import inputs: + +- **Current narrow run:** no verified complete signed-v5 import set exists. `[]` is syntactically admissible but does not certify §112's current compile/proof closure; it is not a readiness claim. +- **After signed §112 refresh:** intended direct-import path is `["us/statutes/26/112.yaml"]`, conditional on its new full-path manifest, hash, signature and closure checks. This array is **not valid as a signed-v5 claim today**. +- **Future downstream §32 annual-table repair:** `["us/policies/irs/rev-proc-2025-32/page-15.yaml"]` is the existing candidate. Its full-path manifest declares v5/Ed25519 and its primary SHA-256 matches `b032822be996093985f5d04fb64477d613fefd239c133e4c3c45192cc281f58e`. Protected signature/inventory verification still must run. It does not automatically replace legacy output names or repair the remaining v1 closure. + +`existing_signed_imports_json` takes tracked **file paths**, not module IDs. The helper requires `.axiom/encoding-manifests/.json` with v5 schema; the old manifests are often at legacy shortened paths as well as being v1. They cannot be reused by relabeling the JSON. Evidence: [import helper](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/src/axiom_encode/prepare_signed_backfill.py#L1364). + +## 5. Dispatch inputs + +**There is no verified passing dispatch command today.** The following complete command records observed immutable values and the intended narrow target. It is a **known-blocked diagnostic snapshot**, not a recommendation to run: its citation/replacement pairing fails current source admission, and `[]` does not supply protected signed-import verification of the inherited closure. Do not spend a signed run demonstrating the already-read preflight error. + +```sh +gh workflow run targeted-signed-reencode.yml \ + -R TheAxiomFoundation/axiom-encode --ref main \ + -f country=us \ + -f rulespec_ref=f43dec520dd392bc5333934f56dad7498363e704 \ + -f pr_base_branch=main \ + -f corpus_ref=8f7d60aaced28ee4252b9237f9d6e02360dc34bc \ + -f rules_engine_ref=af6e4ea2920b0c0a97bf6a6f45b0c6643e93c0ca \ + -f citation=us/statute/26/32/c/2 \ + -f replace_rulespec_path=us/statutes/26/32/c/2.yaml \ + -f existing_signed_imports_json='[]' \ + -f source_bundle_json='[]' \ + -F review_finding=@scratch/us-32c2/review-finding.txt \ + -f open_pr=true +``` + +The three SHA inputs come from live rulespec main and the inspected `origin/main:.axiom/workflow-toolchain.toml`; corpus/engine pins also appear in the actual protected-run checkout logs. Current corpus main `942e138e…` and current engine main `6e709eb1ca7ea686263293d932c759d9dee48a4a` were read, but selecting newer checkout refs alone does not change rulespec-us's signed corpus release. + +The workflow was read at encoder main **`5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`**. Its required `--ref main` condition means the workflow itself is mutable between preparation and dispatch; re-read it at authorization time. All three checkout refs require full lowercase 40-character SHAs. Corpus/engine SHAs must be ancestors of their main branches. With `open_pr=true`, rulespec must equal main **both initially and immediately before push**. The encoded run operates in `production-signing` and uses `--backend openai --apply --mode repo-augmented --require-complete-source-unit`. This preparation lane does not execute that command. [Workflow and validation source](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/.github/workflows/targeted-signed-reencode.yml). + +Using `citation=us/statute/26/32` would satisfy the existing target's source identity, but requests the **whole section**. It is not an equivalent narrow run: every other branch must be encoded or genuinely and precisely deferred, the existing `32.yaml` surface must not be duplicated, and dependent compatibility must hold. That route has not been demonstrated to pass. Changing only the citation to hide the admission error is not the recommended package. + +No separate ACTC command is ready: its identity refinement, threshold composition and §32 import closure must be resolved first. After prerequisites land, the orchestrator must produce new exact-main commands with refreshed verified import arrays and obtain Max's approval. The JSON at the end therefore leaves `dispatch_command` empty rather than presenting a known-failing command as runnable. + +## 6. Review finding text + +The full proposed `review_finding` input is [review-finding.txt](review-finding.txt). It includes the assignment's encoding regime verbatim, the narrow-source admission condition, whole-source coverage, precise deferrals, paired tests, and all four worked cases below. It is preparation text for a future corrected narrow transaction; it does not cure present workflow admission or provenance blockers. + +The source requires wages, salaries, tips and employee compensation **“but only if such amounts are includible in gross income for the taxable year”**. Section 402(e)(3), read with the inclusion rules and §402(g), supports excluding qualified traditional deferrals while retaining excess amounts required includible. The retained designated-Roth treatment is separately corroborated by the [official IRS Roth comparison](https://www.irs.gov/retirement-plans/roth-comparison-chart); no §402A corpus proof is claimed. Section 32(c)(2) also covers adjusted net self-employment income and all six B rules: disregard community-property allocation; exclude pensions/annuities, §871(a) amounts, inmate-service compensation, and only the subsidized portion of specified state work activities; permit the §112 election. The source does not direct subtracting separate pension receipts from otherwise qualifying wages. + +Precise deferral boundaries are upstream classification under §§401(k), 402, 402A and 403(b), limits/excess/catch-up rules, §1402/164(f) net earnings, and §112 excluded amounts; downstream §32 credit eligibility/rates/AGI/tables and state credit formulas are separate. A classified-compensation input is acceptable for a bounded definition, but must not be advertised as independent raw-pay classification. No vague deferral of a local §32(c)(2) branch or opaque earned-income answer input is acceptable. Existing public pre-election/final earned-income outputs must be preserved or their consumers migrated through signed transactions. + +For a separate §24(d) consumer, apply the section-32 definition and **§24(h)(6)'s $2,500 substitution for 2026**. Distinguish its mandatory inclusion of §112 pay from the optional EITC election. A replacement of the full §24(d)(1) source also needs the three-or-more-child alternative and lesser/greater-of limitations, not merely clause (B)(i); replacing all (d) also needs its §911 gate. [Official §24](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section24&num=0&edition=prelim). + +Every operative exception/election needs paired positive and blocking witnesses in the same period, differing in exactly one controlling input and asserting the main amount. Cover each controlling conjunct separately, partial subsidies, pension classification without double subtraction, combat election true/false, and community-property invariance. Assign every local `#input`, including false Booleans. Include zero/positive boundaries and traditional/Roth/excess classification integration tests. Exact proof excerpts must be substrings of the resolved source, not stitched or paraphrased assertions. These requirements come from [binding issue #39](https://github.com/TheAxiomFoundation/.github/issues/39), read via the GitHub connector after `gh issue view` could not reach the network, and the inspected completeness validator/workflow. + +**Worked cases and adjudication.** These use actual scenario facts and independently calculated legal formulas, not the fix docstring. Qualification, residency, valid IDs and absence of other exceptions are held at benchmark facts/defaults; the supplied situations do not prove every eligibility condition. All three cases have no self-employment or supplied combat/inmate/§871/state-work income. Employer-paid health insurance is not subtracted again as an employee payroll deduction. Calculations follow the continuous-dollar benchmark convention; §32(f)'s Treasury bracket-table filing amounts are not certified here. Float32 differences below one cent do not change the conclusions. [Detailed facts, method and evidence](households/adjudication.md). + +| Household / moved benchmark output | PE 1.755.4 frozen | Sandbox corrected | Independently derived provision result | Adjudication | +|---|---:|---:|---:|---| +| 023 CA — state refundable credits | 148.310867 | 174.968216 | **Legal isolated change $0**; exact 2026 table amount unresolved; same-schedule control **99.460066** | **Neither credit amount established**; corrected wage base supported | +| 100 MT — federal refundable credits | 2,878.096680 | 822.288269 | **822.288281** | Corrected supported | +| 100 MT — state refundable credits | 473.177704 | 164.457657 | **164.457656** | Corrected supported | +| 119 VA — federal refundable credits | 2,854.008789 | 3,366.073730 | **3,366.073525**, including unchanged ACTC balance | Corrected provision effect supported | +| 119 VA — state income tax before refundable credits | 1,859.094360 | 1,756.681396 | **1,756.681436**, holding unrelated components fixed | Corrected provision effect supported | + +**Case 1 — scenario_023 (California).** Single adult, age 28, no children. Gross wages $17,442.646484375; traditional 401(k) $2,778.47998046875; Roth 401(k) $490.32000732421875; taxable 403(b) distribution $8,000; IRA deduction $129.8303985595703. Statutory earned income is **$14,664.16650390625**, compared with frozen **$17,442.646484375** and sandbox **$14,664.166015625**. The Roth amount stays in taxable wages. The separate pension is excluded from earned income, not subtracted from these pension-free wages. + +AGI under the unchanged IRA assumption is **$22,534.33610534668**, above both wage measures. California's incorporated AGI comparison therefore remains binding. The legal credit is `min(C2026(14664.16650390625), C2026(22534.33610534668))`, with the latter binding under the schedule, **unchanged by this wage correction**. The available official [FTB 3514 instructions](https://www.ftb.ca.gov/forms/2025/2025-3514-booklet.html) corroborate the second AGI lookup and smaller-credit rule; they are 2025 guidance, not a verified 2026 table. + +No final 2026 table amount is certified. Holding the saved schedule fixed and independently applying it to AGI yields **$99.46006599474843**, matching the separately supplied r17 diagnostic. That is a conditional numerical control, not a statutory 2026 result. Thus the statute supports the corrected **wages**, supports **neither** the frozen $148.31 nor corrected $174.97 as an adjudicated credit, and does not support the claimed +$26.657349 change. The audit's `not_confirmed` entry and verification report corroborate this distinction; the statutory interpretation was derived separately. + +**Case 2 — scenario_100 (Montana).** Head of household, adult 46, children 6 and 5. Wages $5,914.720703125; traditional 401(k) $3,859; Roth 401(k) $681; IRA $180.32000732421875. Statutory earned income is **$2,055.720703125** (frozen $5,914.720703125; sandbox $2,055.720703125). The IRA deduction lowers AGI, not earned wages; AGI is $1,875.4006958007812, so there is no phaseout. + +EITC is `0.40 × 2055.720703125 = $822.28828125`. ACTC is `0.15 × max(2055.720703125 − 2500, 0) = $0`; two children do not qualify for the three-child alternative. Federal refundable credits are therefore **$822.28828125**. Montana's 2026 refundable match is **20%**, yielding **$164.45765625**. Both corrected outputs are supported. The two-child rate comes from §32(b)(1); Montana's change is enacted in [2025 chapter 227/HB 337 §3, effective under §§5–6](https://archive.legmt.gov/content/Sessions/69th/Contractor_index/CH0227.pdf). + +**Case 3 — scenario_119 (Virginia).** Head of household, adult 45, children 14 and 11. Wages $55,000; traditional 401(k) $3,087.199951171875; Roth $544.7999877929688; taxable interest $800; IRA $144.25599670410156. Earned income is **$51,912.800048828125** (frozen $55,000; sandbox $51,912.80078125). AGI is **$52,568.54405212402** and controls the phaseout. Using the 2026 two-child maximum $7,316 and unmarried threshold $23,890 from [IRS Rev. Proc. 2025-32 §4.06](https://www.irs.gov/pub/irs-drop/rp-25-32.pdf): + +`7316 − 0.2106 × (52568.54405212402 − 23890) = $1,276.2986226226807` EITC. + +Corrected ACTC phase-in capacity remains $7,411.920007324219, above the unchanged refundable balance **$2,089.77490234375**. That balance is derived by the supplied benchmark from $4,400 child credit less $2,310.22509765625 limiting tax; this lane does not independently audit that unrelated tax liability. Adding it gives **$3,366.073524966431** federal refundable credits. The independently derived EITC increase is **$512.0646226226807**. + +Virginia's selected nonrefundable 20% branch reduces pre-refund tax by **$102.4129245245361**, giving **$1,756.681435827026** while holding unrelated components fixed. [Virginia Code §58.1-339.8 B(2)–(3)](https://law.lis.virginia.gov/vacode/title58.1/chapter3/section58.1-339.8/) supports the 20% match; its 2025–2026 refundable alternative is also 20%. Household output confirms the benchmark selects the nonrefundable branch; the read selection code favors that branch on ties, but this lane did not calculate both hypothetical liabilities to prove a tie here. The law permits that choice; it does not require that accounting branch. Corrected provision effects are supported, without claiming a complete independent federal/Virginia return calculation. + +**Case 4 — classification boundary.** $10,000 pay with $2,000 qualifying excluded traditional deferral gives $8,000 includible wages. Changing only its classification to taxable designated Roth gives $10,000. A separate $3,000 pension must leave those earned-income amounts unchanged. Set unrelated local facts explicitly zero/false; include a separate excess-deferral case retaining amounts required includible by §402(g). If the narrow module receives classified wages, these are upstream integration witnesses, not proof that the module itself encodes the missing classification. [Official §402](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section402&num=0&edition=prelim), [IRS Roth treatment](https://www.irs.gov/retirement-plans/roth-comparison-chart). + +## 7. Risks + +**Likeliest rejection reasons and mitigation:** source-identity mismatch and non-v5 manifests for paths explicitly listed in `existing_signed_imports_json` are pre-generation blockers; prose cannot cure them. Complete-source validation rejects unaccounted branches and generic deferrals; the proposed finding enumerates the whole narrow source and named boundaries. Paired-exception tests must flip one controlling input and assert the affected output. Literal pension subtraction can corrupt already-net wages; the finding supplies a concrete counterexample. A consumer run must resolve the annual IRS import and the $2,500 ACTC substitution. Do not admit the California +$26.66 move as an expected result. Use exact proof substrings and all local facts, and preserve the reviewed source/manifest chain; no new waiver or manual metadata patch is proposed. + +**Serial chain:** live #1384 is merged at the inspected main; [#1387](https://github.com/TheAxiomFoundation/rulespec-us/pull/1387) activation and [#1386](https://github.com/TheAxiomFoundation/rulespec-us/pull/1386) workflow pin remain open. The later drift-stage/activation/re-pin PRs were not found in the inspected complete open-PR collection. The chain's existence is not itself a legal requirement for encoding §32(c)(2), which is already in the pinned release. Operationally a long `open_pr=true` run should wait for coordinated main stability because exact-base validation can fail after any intervening merge. The chain neither fixes deep source refinement nor adds §402(e)(3). The brief's “up to 3.5 hours” per strict CI was not independently measured in this lane and remains unverified. + +**Verified open collisions:** + +| PR | Observed overlapping files/status | +|---|---| +| [rulespec-us #1160](https://github.com/TheAxiomFoundation/rulespec-us/pull/1160) | Open draft; `32/c/2.yaml` + tests, `112.yaml` + tests, `24/d.yaml` + tests | +| [rulespec-us #1158](https://github.com/TheAxiomFoundation/rulespec-us/pull/1158) | Open draft; `32.yaml` + tests and `24/d.yaml` + tests | +| [rulespec-us #1159](https://github.com/TheAxiomFoundation/rulespec-us/pull/1159) | Open draft; `32.yaml` + tests | +| [rulespec-us #1363](https://github.com/TheAxiomFoundation/rulespec-us/pull/1363) | Open draft; legacy IRS annual module, `32.yaml`, `24/d.yaml`; body reports provenance blocks. Not an approved signed repair. | +| [axiom-encode #1665](https://github.com/TheAxiomFoundation/axiom-encode/pull/1665) | Open proposed legacy-to-signed IRS successor repoint. Proposal read; implementation/landing success unverified. | + +REST collections were read because connector search omitted open PRs; exact changed-file APIs verified overlaps. The latest **30 targeted runs** across the inspected 200 repository runs contained no §32 or §24 citation/descendant. This is a bounded check, not a promise that no one will dispatch later. Full refs, PR heads, workflow contracts and run evidence: [workflow/report.md](workflow/report.md). + +**Validation performed:** all five baseline and r08 moved outputs reproduced their CSV values within **$0.000001** using only the three assigned households. Independent decimal arithmetic agreed with all four supported MT/VA outputs within **$0.01**; CA's same-schedule AGI diagnostic and zero isolated effect were checked separately. Four existing Axiom earned-income executions passed their intended input-contract probes, including the pension counterexample; 14 legacy-binary compile checks passed with immutable-byte comparisons. Current pinned-engine compile acceptance remains unverified beyond the definite plural-field blocker established from source. No population simulation, local encode or new RuleSpec test was run. Evidence: [arithmetic-checks.log](households/arithmetic-checks.log), [household-checks.json](households/household-checks.json), [independent-arithmetic.json](households/independent-arithmetic.json), [coverage inventory](coverage/inventory.json). + +```json +{ + "lane": "us-32c2", + "verdict": "BLOCKED", + "citation": "us/statute/26/32/c/2", + "in_corpus": true, + "in_pinned_release": true, + "existing_modules": [ + "us:statutes/26/32/c/2", + "us:statutes/26/32", + "us:statutes/26/24/d", + "us:statutes/26/24/h", + "us:statutes/26/24", + "us:statutes/26/112", + "us:policies/irs/rev-proc-2025-32/earned-income-credit", + "us:policies/irs/rev-proc-2025-32/page-15", + "us-ca:statutes/rtc/17052", + "us:statutes/26/86" + ], + "blocking_imports": [ + "us:policies/irs/rev-proc-2025-32/earned-income-credit", + "us:statutes/26/32" + ], + "dispatch_command": "", + "prerequisites": [ + "Supported signed source-identity refinement for existing 32/c/2 and ACTC target; current two-level refinement fails admission", + "Verify the dependency strategy: signed-v5 explicit imports, or a separately validated inherited closure and precise bounded source contracts; section 112 cannot currently be listed as signed-v5", + "Signed legacy IRS successor/import migration and successful current-engine recheck of 32 and 24/d closure; coordinate encoder PR 1665", + "For independent raw-pay deferral classification, release-bind section 402 sources beyond both 08-08 and planned 08-23 release, then encode/import appropriate signed classification", + "Compose section 24(h)(6) 2500-dollar threshold and preserve mandatory combat-pay inclusion in ACTC consumer", + "Coordinate overlapping rulespec PRs 1160, 1158, 1159, 1363 and the serial 1387/1386/later repin chain; refresh exact-main refs and obtain Max approval" + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_us-662.md b/reference_audit/2026-09-22/verification/v9_axiom_us-662.md new file mode 100644 index 00000000..59eb3320 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_us-662.md @@ -0,0 +1,199 @@ +# us-662 — Axiom encoding preparation + +## 1. Verdict + +**BLOCKED.** `us/statute/26/662/a` has no normalized provision in the inspected canonical corpus `origin/main`. Its official text is retained inside a full-title source archive, but the signed encoder resolves normalized provision records from the pinned release; it does not encode directly from that archive. An official-source extraction/ingest, signed release publication, and separately approved rulespec-us re-pin are required before dispatch. + +The named serial chain ending at `us-rulespec-2026-08-23-canada-338-suspension-union` is **insufficient by itself**: that release recipe also lacks §662. Do not dispatch against the inspected main. No passing, immutable future command can be specified before the prerequisite commits exist. + +Independent household adjudication supports including **$25,950** for scenario_039 and **$6,322.94140625** for scenario_110, treating the supplied amounts as already classified taxable beneficiary estate income. Zero estate QBI follows from the benchmark's explicit unlisted-status=false convention, not from a statutory rule that estate income is always nonbusiness. The households lack the distribution, DNI, and character facts needed to compute a raw §662 amount. Full federal/state liabilities therefore remain conditional, as detailed below. + +Read-only boundary: only this workspace's `scratch/us-662/` was written. No encoding, authored RuleSpec/test YAML, commits, branches, dispatches, PRs, or external comments. The task-specific no-commit rule controls over the generic commit instruction. + +## 2. Source + +Canonical corpus snapshot inspected: **`942e138e7a8250c9814e774ac9b8e63008148106`**. Rulespec snapshot: **`f43dec520dd392bc5333934f56dad7498363e704`**. These are verified local `origin/main` objects, not confirmed live remote tips. The requested rulespec fetch failed with `cannot open '.git/FETCH_HEAD': Operation not permitted`; GitHub CLI reads failed to connect. + +| Provision | Canonical citation and corpus status | Vintage and release membership | +|---|---|---| +| §662(a) | Intended citation `us/statute/26/662/a`; neither it nor parent `us/statute/26/662` exists as a normalized record | Not in the pinned release or named Aug23 successor recipe | +| §61(a)(14) | **`us/statute/26/61/a/14`** exists, as do parent `/61` and `/61/a` | Recovery scope dated **2026-07-13**, present in both release recipes | +| §199A(c)(3)(A)(ii) | **`us/statute/26/199A/c/3`** is the stored source unit containing (A)(ii); parent `/199A` and `/199A/c` also exist. Do not mistake an unverified deeper path for a stored record | Same 2026-07-13 recovery scope, present in both release recipes | + +The supporting normalized file is `data/corpus/provisions/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup.jsonl`. Its §61/§199A records and provenance were read from canonical `origin/main`; excerpts are saved in [supporting-corpus-provisions.json](supporting-corpus-provisions.json). The retained source excerpts come from OLRC archive `xml_usc26@119-100.zip`; their provenance records supply archive/member/excerpt SHA-256 values. + +Official §662 text was independently read at [GovInfo §662](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapJ-partI-subpartC-sec662.htm), and in the newer retained OLRC release **Online@119-103**. Related official locations are [§61](https://www.govinfo.gov/link/uscode/26/61), [current §199A](https://uscode.house.gov/view.xhtml?req=%28title%3A26+section%3A199a+edition%3Aprelim%29), and [26 CFR 1.199A-6(d)](https://www.govinfo.gov/content/pkg/CFR-2025-title26-vol4/pdf/CFR-2025-title26-vol4-sec1-199A-6.pdf). + +**Required ingest:** extract §662, including (a)(1)–(2), (b), and (c), from the official OLRC **USLM XML** archive [Title 26, release 119-103](https://uscode.house.gov/download/releasepoints/us/pl/119/103/xml_usc26@119-103.zip), retain source URL/date/hash provenance, and create resolvable provision records through the authorized corpus ingestion process. The source archive already exists on canonical main at: + +`data/corpus/sources/us/statute/2026-09-13-tax-statute-closure-31-title-26/olrc/xml_usc26@119-103.zip` + +Its recorded source-as-of is **2026-09-02**; the ingest manifest is dated **2026-09-13**. This lane verified archive SHA-256 **`285b9862808f26055c3eed16c2aca1d8de1fae96b581f44a55cfedb907a46eff`** and extracted member `usc26.xml` SHA-256 **`ab999da948658a2265f762abfedd72413a3f7828d34659ff27ed8240fcd956e4`**. The actual §662 text is retained in [official-usc26-662.xml](official-usc26-662.xml). Presence of the raw source does **not** mean the citation is currently resolvable. + +Exact `citation_path` searches, allowing JSON whitespace and checking all tracked federal statute provision files plus the legacy provision directory, returned no §662 record. A broad substring search instead found a §6621 cross-reference; that was excluded. The corpus-tree inventory and empty exact-match output are retained here. + +The rulespec pin read from `.axiom/toolchain.toml` is `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA-256 **`0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`**. [Pinned](pinned-release.json) and [successor](chain-release.json) release recipes were read from canonical corpus main. `axiom-locate release` did not locate a materialized signed object, so this lane does not claim fresh signature verification of the published release object. + +Per the missing-source instruction, executable dispatch preparation stops here for §662. The remaining sections preserve requested coverage evidence and independent review material for the orchestrator; they do not authorize or represent a runnable encode. + +## 3. Existing rulespec-us coverage + +At rulespec SHA `f43dec520dd392bc5333934f56dad7498363e704`, no primary module exists at §662, §661, §663, or §643. The directly relevant existing modules are: + +| Module | What the inspected code encodes | Relation to this task | +|---|---|---| +| `us/statutes/26/61.yaml` | General gross-income definition and enumerated positive income sum | Includes the estate/trust category; does not compute beneficiary distributions/DNI | +| `us/statutes/26/62.yaml` | AGI arithmetic consuming a supplied gross-income input and deduction inputs; no imports | Does not construct estate/trust beneficiary income | +| `us/statutes/26/199A.yaml` | QBI deduction arithmetic using resolved QBI as an input | Explicitly defers QBI composition and trade/business classification | +| `us/statutes/26/67/e.yaml` | The estate or trust entity's own AGI, subtracting resolved deduction inputs from its gross income | Does not calculate a beneficiary's income; read, not compiled | + +The §61 enumerated formula contains the literal line: + +```text ++ max(0, income_from_an_interest_in_an_estate_or_trust) +``` + +Its separate `gross_income` formula is: + +```text +max( + 0, + all_income_from_whatever_source_derived + - amounts_otherwise_provided_excluded_from_gross_income_under_this_subtitle +) +``` + +Thus the category is present, but the module does not connect the enumerated sum to the aggregate input. Two existing-module engine queries demonstrate the distinction: estate income 7,800 with aggregate input zero gives enumerated income **7,800** and gross income **0**; supplying aggregate income 7,800 gives **7,800** for both. This is a composition boundary, not evidence of a categorical omission in §61. It also does not make §662 **NOT-NEEDED**. + +The §199A deferral says: + +```text +Subsection (c) defines qualified business income, qualified items, and +exclusions. This module consumes resolved qualified business income as +an input while encoding the deduction arithmetic; QBI composition is +deferred. +``` + +The following deferral similarly covers subsection (d)'s trade/business classification. Neither this module nor §61 establishes a universal true or false estate-QBI classification. Exact copied modules, manifests, compile logs, and query evidence are in this scratch directory; [coverage-analysis.md](coverage-analysis.md) supplies line references and command details. + +## 4. Import closure + +Candidate future **`existing_signed_imports_json=[]`**. A standalone §662(a) unit can consume explicit, resolved DNI/applicability/character facts; no existing module import is necessary to calculate its two distribution tiers. This choice must be reviewed after the missing corpus unit is ingested. It must not be used to hide unresolved legal boundary facts. + +The nearby existing closures were actually compiled: + +| Existing closure | Local engine result | Direct signed-workflow eligibility | +|---|---|---| +| §61 | Compiles; two outputs, including the estate traces above | Manifest is `applied-rulespec/v1`, not v5 | +| §62 (no imports; gross income is a local input) | Compiles | Manifest v1 | +| §199A → §1(h) → IRS Rev. Proc. 2025-32 capital-gains module | Compiles; §199A closure has 34 outputs | All inspected manifests v1 | + +The workflow's direct-import validator requires tracked same-jurisdiction primary **file paths** with signed-v5 manifests. It rejects these v1 modules as supplied direct imports even though the local engine compiles them. Therefore they are not candidates for `existing_signed_imports_json`. + +**Compile-blocking imports in the proposed empty closure: none.** The inspected nearby closures do not reach §32 or its earned-income-credit import, and do not contain the removed plural citation field. The brief's §32 failure is not a blocker for this standalone target. This lane did not independently compile §32. + +The local engine was located with `axiom-locate engine`; binary SHA-256 is **`674ca6e70afdccb59c3d6847933bc24b4590105e49db54790f2dcd0bdbbe32d7`**. These are local load checks of existing bytes, not a claim that the binary exactly matches the prospective signed-run engine commit. + +## 5. Dispatch inputs + +**No dispatch command is supplied: there is no valid verified command for an absent corpus unit.** Freezing today's rulespec SHA would also fail after the required re-pin: the workflow requires `open_pr=true` to use the exact then-current main tip. A command with placeholder refs or a known-invalid current pin would not meet the requested dispatch standard. + +Verified local immutable objects, to preserve the inspection provenance: + +| Input/component | SHA | Source | +|---|---|---| +| `rulespec_ref` inspected | `f43dec520dd392bc5333934f56dad7498363e704` | rulespec-us local `origin/main`, merge #1384 | +| `corpus_ref` inspected | `942e138e7a8250c9814e774ac9b8e63008148106` | canonical axiom-corpus local `origin/main` | +| `rules_engine_ref` candidate only | `6e709eb1ca7ea686263293d932c759d9dee48a4a` | canonical engine local `origin/main`; not the identified local binary build | +| Workflow inspected | `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9` | axiom-encode local `origin/main` | + +After the prerequisites, refresh and verify immutable refs and exact canonical citation. The eventual workflow is `targeted-signed-reencode.yml`, repository `TheAxiomFoundation/axiom-encode`, workflow ref `main`; candidate values are `country=us`, `pr_base_branch=main`, `citation=us/statute/26/662/a` **only if ingested at that path**, empty `replace_rulespec_path` for a still-new module, `existing_signed_imports_json=[]`, `source_bundle_json=[]`, `open_pr=true`, and the reviewed finding below. Max's approval remains required for every signed run. No new approval is requested by this read-only lane. + +Verified mechanisms: [workflow lines 827–875](workflow-evidence/targeted-signed-reencode.yml) materialize the toolchain-pinned signed release; [resolver lines 551–580 and 937–976](workflow-evidence/corpus_resolver.py) restrict source lookup to that release's provision artifacts. [Base validation lines 267–332](workflow-evidence/prepare_signed_backfill_impl.py) enforce exact main for PR creation; direct imports at lines 1368–1502 require v5. [workflow-analysis.md](workflow-analysis.md) records the remaining input validations and protected signing path. + +## 6. Review finding text + +**Prepared finding for review after ingest, not currently dispatchable:** + +Encode the entire resolved §662(a) source unit. Cover the §661 applicability gate, currently required distributions whether or not paid, the first-tier proportional DNI limitation computed without the §642(c) deduction, the rule for income-or-corpus obligations to the extent satisfied from current income, other proper distributions, and their residual-DNI proportional limitation. Guard zero denominators and exhausted residual DNI. Do not replace this source unit with a single unconditional addition of an estate-income input. + +Precisely defer external determinations to explicit local inputs: §643 DNI construction, §642(c) charitable deductions and allocation, §661 entity classification, §663 exclusions/timing/separate shares, and §662(b) character classification. Supply separate DNI inputs where the statutory computations differ. §662(a)'s reference to (b) means a gross cash distribution cannot silently become taxable ordinary income. A deferred or unresolved classification must remain identifiable; it must not masquerade as a legally determined zero. If the resolved citation is the whole §662 rather than (a), cover or precisely defer (b)'s character proportions, governing-instrument exception and deduction allocations, and (c)'s taxable-year alignment. + +Treat the §199A issue as a separate classification boundary. Taxable-income inclusion under §199A(c)(3)(A)(ii) is necessary but not sufficient: clause (i), the qualified-business definition, and (B)'s exclusions still apply. Preserve carried-out character for a separate signed QBI-composition unit. Do not encode an automatic estate-QBI=true or estate-QBI=false rule, and do not label a positive-only current amount as a complete treatment of qualified-business losses. + +Require paired positive/blocking cases for: entity/applicability eligibility; mandatory income despite no cash payment; binding versus nonbinding first-tier cap; positive versus exhausted residual DNI; zero other distributions; income-or-corpus obligation satisfied from income versus solely corpus; resolved §663 exclusion versus eligible distribution; taxable versus tax-exempt character; and qualified taxable business items versus nonbusiness or excluded items at any QBI interface. For any implemented (b)/(c) branches, also pair governing-instrument special allocation and mismatched taxable years with their controls. Every companion case must explicitly assign every local input, including false facts. Proof excerpts must come verbatim from the eventual resolved provision, not from this prose or the sandbox docstring. + +**Case 1 — scenario_039 (VA).** Single, age 61, no spouse/children; 2026 estate income **25,950**, self-employment loss **6,260.029296875**, Social Security **36,105**, taxable IRA distributions **26,800**, taxable pension **2,030**, wages zero. Estate-QBI status is unlisted. Disability and surviving-spouse flags are listed; filing status is expressly single. + +| Measure | PolicyEngine 1.755.4 | Assigned sandbox correction | Independent reading | +|---|---:|---:|---| +| Estate contribution to gross income | 0 | 25,950 | **25,950** under the classified taxable-beneficiary-income input contract | +| Estate contribution to QBI before other-item netting | 25,950 | 0 | **0 under the benchmark convention**; actual business character unknown | +| Federal tax before refundable credits | 1,345.510498 | 8,596.028320 | Corrected provision-level treatment supported; full amount not independently established by the assigned provisions | +| VA tax before refundable credits | 514.498413 | 1,975.798218 | Same inclusion conclusion; complete VA liability not independently derived here | + +The positive taxable-beneficiary amount belongs in gross income under §61(a)(14), with §662(a) supplying its upstream distribution calculation and (b) preserving character. Therefore the statute supports the **corrected inclusion**, not zero inclusion of established taxable beneficiary income. Under the benchmark's expressly supplied unlisted-status=false convention, estate QBI is zero. Neither exact final liability is certified as an unconditional statutory result: missing DNI/character facts and additional federal/state provisions prevent that conclusion. + +Fresh two-household reproduction observes taxable Social Security increasing from **10,129.0986328125** to **30,689.25**, AGI from **32,699.068359375** to **79,209.21875**, and QBI deduction from **3,319.813720703125** to zero. These explain why the federal movement exceeds a simple marginal-rate multiplication of 25,950; they are simulator observations, not a separate signed Axiom adjudication. + +**Case 2 — scenario_110 (OH).** Single, age 51, no spouse/children; estate income **6,322.94140625**, wages **100,000**, long-term gains **30,851.765625**, short-term gains **18**, qualified dividends **10,032**, other dividends **5,662.11767578125**, taxable interest **21,301**, tax-exempt interest **954**. Listed desired traditional contributions are 401(k) **4,244.89990234375** and IRA **198.3520050048828**. Estate-QBI status is unlisted. Full facts are preserved in [households.json](households.json). + +| Measure | PolicyEngine 1.755.4 | Assigned sandbox correction | Independent reading | +|---|---:|---:|---| +| Estate contribution to gross income | 0 | 6,322.94140625 | **6,322.94140625** under the same taxable-income input contract | +| Estate contribution to QBI | 6,322.94140625 | 0 | **0 under the benchmark convention**; actual business character unknown | +| Federal tax before refundable credits | 23,897.443359 | 25,700.166016 | Corrected provision-level treatment supported; full amount not independently established | +| OH tax before refundable credits | 4,057.472168 | 4,231.353027 | Inclusion supported; state result additionally depends on Ohio business-income classification | + +Again the statute supports the **corrected inclusion**. A false federal QBI qualification input alone does not establish Ohio nonbusiness classification. The supplied verification report conditions Ohio's corrected amount on that additional interpretation, and the audit records a separate IRA-related change for this household. Neither frozen nor single-reform final tax is certified here as the complete legally correct return. Observed AGI is **163,421.640625 → 169,744.578125** and QBI deduction **1,264.5882568359375 → 0**. + +**Why these are conditional findings:** neither household provides estate DNI, aggregate/current distribution requirements, all-beneficiary amounts, governing-instrument allocations, or income classes. A raw §662 calculation is therefore underdetermined. The independently derived dollar inclusions use the supplied `estate_income` as an already classified beneficiary-income amount. QBI status from economic facts alone is **unknown** in both cases. `policybench/prompts.py:19–21`, read in this run, expressly supplies the additional convention that unlisted status inputs are false. That benchmark convention produces the zero-QBI test fact; it is not a universal rule of law. + +**Case 3 — current-distribution cap.** Domestic nongrantor complex trust, same tax year, ordinary taxable nonbusiness income, no charity deduction, §663 exclusion, or separate shares. A's current entitlement is 30,000; all beneficiaries' current entitlements total 60,000; first-tier DNI is 40,000; other distributions zero; cash paid to A zero. First-tier inclusion = **40,000 × 30,000 / 60,000 = 20,000**. Mandatory entitlement is included despite unpaid cash. Paired nonbinding-cap control with DNI 80,000 gives **30,000**. These amounts follow directly from the first-tier ratio in the retained official §662 text. + +**Case 4 — other distributions and residual DNI.** Same classifications. A's current entitlement is 10,000, total current entitlements 40,000; A's other distribution is 30,000, total other distributions 120,000; DNI is 100,000 with no charity deduction. First tier **10,000**; residual DNI **60,000**; second tier **60,000 × 30,000 / 120,000 = 15,000**; total **25,000**. Nonbinding-cap control with DNI 200,000 gives **40,000**. Exhausted-residual control with DNI 40,000 gives **10,000**. All are ordinary taxable nonbusiness amounts by explicit facts, not by inference from estate origin. + +The legal reading above uses the independently read [official §662 text](official-usc26-662.txt), [§61 text](official-usc26-61.txt), and [§199A text](official-usc26-199A.txt), extracted from the retained current OLRC archive. [household-analysis.md](household-analysis.md) provides the complete household evidence and qualifications. + +## 7. Risks + +| Risk | Preparation response | +|---|---| +| Missing active source | Block dispatch; ingest normalized §662 records, publish an authenticated successor release, and land a dedicated pin update | +| Treating the announced chain as sufficient | Both inspected release recipes omit §662; recheck the actual eventual release scope | +| Stale base or unverifiable immutable refs | The workflow requires exact main for `open_pr=true`; refresh after prerequisites, not before | +| Import signature rejection | Candidate import list is empty; nearby compiling modules have v1 manifests and cannot be admitted as direct v5 imports | +| Incomplete source-unit coverage | Finding covers both tiers and cap branches; names external calculations and their input contracts precisely | +| Missing exception tests or invented proof text | Finding requires paired positive/blocking cases and all local inputs; proof text must be extracted from resolved corpus text after ingest | +| Overclaiming household adjudication | Separate taxable-income input interpretation, benchmark QBI convention, actual unknown economic classification, and observed full-tax outputs | +| Confusing local compile with signed-run success | Existing bytes were compiled locally; no new encoding, v5 attestation verification, or complete target CI run occurred | + +Binding [Axiom issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was read successfully through the web fallback after the requested `gh issue view` failed. Its restrictions on provenance, protected pins, proof excerpts, complete input assignments, and oracle reporting were followed. The page was cached; its historical tool pins are not asserted to be today's pins. This package makes no upstream oracle finding. + +Collision searches were **attempted but could not be completed**: GitHub CLI reads of open rulespec-us/axiom-encode PRs matching 662/61/199A and the latest 30 targeted runs failed to connect. Web fallback for #1387 and #1386 returned cache misses. No absence-of-collisions claim is made. Current statuses of #1387/#1386, subsequent drift/activation PRs, today's runs, and the example failure artifact remain unverified. Local refs confirm inspection of the #1384 merge and cached chain branches only. Before any approval request, the orchestrator must repeat the live citation/run/PR checks and inspect actual failure history. + +**Checks completed:** all eight baseline/corrected assertions for the four moved outputs passed within one cent using the exact assigned sandbox variant under policyengine-us **1.755.4**, for these two households only. Five nearby existing-module compile checks passed; two §61 queries demonstrated the composition boundary. No population run was performed. An optional nonexistent `traditional_ira_deduction` lookup is recorded in the reproduction artifact; it did not affect the assertions. No signed encoding was possible, so there is no Axiom §662 monetary test result. + +```json +{ + "lane": "us-662", + "verdict": "BLOCKED", + "citation": "us/statute/26/662/a", + "in_corpus": false, + "in_pinned_release": false, + "existing_modules": [ + "us/statutes/26/61.yaml", + "us/statutes/26/62.yaml", + "us/statutes/26/199A.yaml", + "us/statutes/26/67/e.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Ingest normalized, resolvable section 662 source units from the retained official OLRC XML with URL/date/SHA provenance; confirm the final canonical subsection citation.", + "Publish an authenticated corpus release containing section 662 and land its dedicated approved rulespec-us pin update; the named Aug23 chain alone is insufficient.", + "Recheck current remote refs, open PRs and targeted signed runs; obtain the exact rulespec main SHA plus compatible immutable corpus and engine refs after prerequisites land.", + "Review the explicit DNI, income-character and benchmark-QBI input contracts, then obtain Max's approval for the concrete signed-run command." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_us-852-capgain.md b/reference_audit/2026-09-22/verification/v9_axiom_us-852-capgain.md new file mode 100644 index 00000000..a29fce44 --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_us-852-capgain.md @@ -0,0 +1,193 @@ +# us-852-capgain — Axiom preparation report + +Prepared 2026-09-23. This is an Axiom coverage and source-preparation package, not a finding filed against an oracle. No encoding, signed run, repository change, commit, PR, or approval was performed. Scratch evidence is in `scratch/us-852-capgain/` in the assigned workspace. + +## 1. Verdict + +**BLOCKED — ingest §852 before dispatch.** Neither `us/statute/26/852` nor any descendant has a normalized provision row in axiom-corpus main `942e138e7a8250c9814e774ac9b8e63008148106`. Consequently the intended §852(b)(3)(B) and (C) citations cannot currently resolve in the pinned release. The existing §61 module does not supply the missing distribution classification. + +The currently planned re-pin to `us-rulespec-2026-08-23-canada-338-suspension-union` is **insufficient**: its manifest does not introduce a §852 source scope. This package cannot dispatch against today's main, or merely after that chain. It needs an official §852 ingest, publication in a signed release, and a dedicated approved rulespec-us pin to that release. A separate source decision is needed if the encoding includes Form 1040 reporting: the available instructions are for **2025**, not a verified final 2026 form. + +The independent provision-level reading supports including **$3,753, $180 and $1,170** as long-term capital gains for scenarios **042, 051 and 091**, respectively. Scenario 051 is an additional moved household present in the sweep but omitted from the assignment's short summary. All four moved benchmark outputs were reproduced. This conclusion does not certify complete state returns. + +Rulespec main inspected: `f43dec520dd392bc5333934f56dad7498363e704`. The required local fetch was attempted and denied at `.git/FETCH_HEAD` by workspace permissions; no escalation was attempted. Read-only GitHub ref queries independently confirmed that both local `origin/main` hashes above matched remote main. All repository reads used those immutable trees. + +## 2. Source + +| Authority | Official source and canonical citation | Corpus main and pinned release | +|---|---|---| +| §852(b)(3)(B), shareholder treatment | [OLRC §852](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section852&num=0&edition=prelim); text successfully read from the [GovInfo 2024 edition](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/pdf/USCODE-2024-title26-subtitleA-chap1-subchapM-partI-sec852.pdf), PDF pp. 1–2. Intended citation: `us/statute/26/852/b/3/B`. | **Absent.** This is the intended canonical spelling, not an existing resolvable citation. No tracked normalized provision file or corpus source vintage can be supplied for it. | +| §852(b)(3)(C), definition and excess allocations | Same official section. Intended citation: `us/statute/26/852/b/3/C`, with descendants preserving uppercase subparagraph and subclause letters. | **Absent**, including descendants; absent from the current and planned Canada-union release scope inventories. | +| §61(a)(3), inclusion of property gains | [OLRC §61](https://uscode.house.gov/view.xhtml?req=granuleid:USC-prelim-title26-section61&num=0&edition=prelim); [GovInfo 2024 edition](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partI-sec61.htm). Exact citation `us/statute/26/61/a/3`; section citation `us/statute/26/61`. | **Present and in the pinned scope.** `data/corpus/provisions/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup.jsonl`. Source-as-of and expression date `2026-07-13`; publication `Online@119-100`, source created `2026-04-17`. Retained source is `data/corpus/sources/us/statute/2026-07-13-recovery-r2026-07-15-self-contained-r2026-07-17-dedup/official-documents/usc26-section-61.xml`. | +| Form 1040 capital gains without Schedule D | [IRS Form 1040 instructions](https://www.irs.gov/instructions/i1040gi). Exact corpus citation `us/form/irs/ty2025/i1040gi/block-83`, “Capital Gain or (Loss).” | **Present in main, outside both releases examined.** `data/corpus/provisions/us/form/2026-09-10-tax-irs-forms-ty2025.jsonl`; HTML source-as-of `2026-09-10`, expression date `2025-01-01`, tax year **2025**. Source snapshot `data/corpus/sources/us/form/2026-09-10-tax-irs-forms-ty2025/official-documents/irs-i1040gi-ty2025.html`. It uses line **7a** and the line **7b** checkbox. Do not label it final 2026 instructions. | + +The pinned release is `us-rulespec-2026-08-08-obbb-alien-snap`, content SHA-256 `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc`, read from `.axiom/toolchain.toml`. Its manifest is `manifests/releases/us-rulespec-2026-08-08-obbb-alien-snap.json`. Release-scope membership was checked against tracked manifests and normalized rows; the published release object was not downloaded or cryptographically re-verified in this lane. `axiom-locate release ` found no local materialization. + +**Required ingest:** retain an official, dated §852 source, preferably OLRC USLM XML, or the official GovInfo section PDF with a reproducible extraction. Record the exact download URL, retrieval date, source SHA-256, legislative release point and effective-date provenance. Normalize §852(b)(3)(B), all of (C)(i)–(vi), and the context/cross-reference material needed to establish applicability and exceptions; preserve hierarchy and verbatim text. The 2024-edition PDF was read; the current OLRC search result also reproduced the relevant B/C text, but direct retrieval of its complete page failed. Confirm the release point and law applicable to 2026 during ingest rather than treating retrieval date or the older edition as a 2026 consolidation. If form-routing behavior is included, use a separately dated IRS source and an explicit tax-year contract. + +Evidence: [saved §61 rows](../scratch/us-852-capgain/61-corpus.jsonl), [IRS matches](../scratch/us-852-capgain/form1040-corpus-matches.jsonl), [pinned manifest](../scratch/us-852-capgain/pinned-release-manifest.json), [Canada-union US scopes](../scratch/us-852-capgain/canada-union-us-scope.json), and the empty exact-citation searches `852-citation-search.txt` / `852-all-federal-provision-search.txt`. The latter checked all 289 tracked federal normalized-provision files across document classes. This establishes absence of an encodable row, not absence of every possible mention in retained raw source files. + +Under the assignment's “If the corpus lacks it … stop there” instruction, **dispatch preparation stops at this ingest prerequisite**. The remaining sections preserve the requested coverage, independent household reading, and future review requirements; they are not a dispatch-ready encoding brief. + +## 3. Existing rulespec-us coverage + +The tracked tree and federal module text at the stated rulespec SHA contain no §852 module or module implementing capital-gain-distribution/Schedule D routing. Relevant nearby coverage is: + +| Module | What the inspected code does; relationship to the missing provision | +|---|---| +| `us/statutes/26/61.yaml` | Lines 27–41 sum enumerated income categories, including `+ max(0, gains_derived_from_dealings_in_property)`. Lines 66–70 define `gross_income` independently as `max(0, all_income_from_whatever_source_derived - amounts_otherwise_provided_excluded_from_gross_income_under_this_subtitle)`. It includes a gain supplied by the caller, but neither classifies a RIC dividend nor connects that classified amount automatically to the aggregate. No Schedule D gate was found here. | +| `us/statutes/26/62.yaml` | Line 56: `formula: max(0, gross_income - deductions_described_in_subsection_a)`. It consumes externally supplied gross income and deduction amounts, without identifying §852 dividends or form-routing amounts. | +| `us/statutes/26/1222.yaml` | Defines capital-gain and loss categories and netting. Its long-term-gain rule consumes `gains_from_sales_or_exchanges_of_capital_assets_held_more_than_one_year_taken_into_account_in_gross_income`. The missing §852 deemed long-term treatment is not derived from a distribution input. | +| `us/statutes/26/1/h.yaml` | Lines 118–125 compute the preferential base from `max(0, long_term_capital_gains)`, net short-term losses, the investment-income election and `+ qualified_dividend_income`. This consumes an already classified gain. | +| `us/policies/irs/rev-proc-2025-32/capital-gains.yaml` | Supplies 2026 zero-rate and 15-percent thresholds; e.g. `capital_gains_zero_rate_threshold_single: 49450`. It does not classify distributions. | +| `us/statutes/26/1411.yaml` | Lines 180–188 include `taxable_net_gain_from_dispositions_after_active_partnership_s_corporation_exception` alongside other investment-income inputs. It does not supply the missing §852 classification or general gross-income connection. | + +These are **coverage and composition boundaries**, not a demonstrated repetition of the benchmark omission within those formulas. An isolated §61 inclusion test with a caller-supplied amount does not make §852 already encoded, and does not warrant NOT-NEEDED. + +The §61 and §62 manifests exist under the legacy nested `us/.axiom/encoding-manifests/` directory and declare `axiom-encode/applied-rulespec/v1`, HMAC signatures. The other nearby manifests inspected are also v1. None is an eligible signed-v5 existing import under the present workflow parser. Immutable copies and the full inventory are retained in [coverage evidence](../scratch/us-852-capgain/coverage/module-inventory.json). + +## 4. Import closure + +**Candidate `existing_signed_imports_json`: `[]`.** No existing signed-v5 module was established as a suitable required direct import for the narrow classification task. This is a future candidate only; no new module exists whose actual closure could be tested. A §61 legal citation is not automatically a computational import of the old §61 module. + +After ingest, an atomic §852(C) definition and §852(B) shareholder-treatment dependency, or a supported fresh-source bundle covering both, should be reviewed before choosing the final target. Do not select the whole §852(b)(3) source unit and silently omit (A), (D) or (E). Do not import §32 merely to create a household-wide tax pipeline. + +The neighboring existing closures were inspected and compiled **without modifying their bytes**: + +| Existing root and complete additional closure | Local discovered engine | Current engine / dispatch conclusion | +|---|---|---| +| §61; §62 (each has no imports) | Both compile. | Current binary not executed; legacy v1 manifests prevent signed-v5 selection. | +| §1222 → §1211, §1212(a)(1) → §172(c) | All compile. | No plural-key barrier found in this closure; current binary not executed, all inspected manifests v1. | +| §1(h) → IRS capital-gains thresholds | Both compile. | **Current schema load barrier:** the imported threshold module contains `source_verification.values`; current `SourceVerification` rejects unknown fields and has no `values` field. This is a source-code inference, not a current-binary test. | +| §1411 → §67(e), §911(a)(1), §911(d)(6) | All compile. | No plural-key barrier found; current binary not executed, all inspected manifests v1. | +| §32 → §32(c)(2) → §112; §152(c); §7703 → §151 → §911(a), §931, §933; IRS earned-income-credit thresholds | All compile in the old binary. | **Current load failure:** `us:policies/irs/rev-proc-2025-32/earned-income-credit` declares removed `corpus_citation_paths`. Any closure reaching it fails the current recursive loader. Exclude this entire unrelated dependency path. | + +`axiom-locate engine` resolved `/Users/maxghenis/TheAxiomFoundation/_tariff-parity/axiom-rules-engine/target/release/axiom-rules-engine`. Its checkout HEAD is `ffd8213271947b0189a9dd61a055c1e0e78908a0`, and it accepted the retired plural field. That is why its 22 successful module compile checks **do not establish compatibility with today's signed runtime**. The binary's build commit was not independently attested. + +Current engine main `6e709eb1ca7ea686263293d932c759d9dee48a4a`, [`src/rulespec.rs`](https://github.com/TheAxiomFoundation/axiom-rules-engine/blob/6e709eb1ca7ea686263293d932c759d9dee48a4a/src/rulespec.rs), rejects plural keys at lines 819–823; lines 1086–1095 validate every loaded module and recurse into imports. The typed source-verification schema also rejects unknown fields. [Compile commands/results](../scratch/us-852-capgain/coverage/compile-results.json) and [current source](../scratch/us-852-capgain/workflow/engine-rulespec.txt) retain the distinction. + +There are **no blocking imports in the proposed empty selection**. There are unresolved source and dependency-design prerequisites, and the generated closure must be checked again: an empty dispatch import array does not prevent the encoder from introducing other imports. Protected [run 35789753522](https://github.com/TheAxiomFoundation/axiom-encode/actions/runs/35789753522) did exactly that and failed on §416(l)'s plural field. + +## 5. Dispatch inputs + +**No executable `gh workflow run` command is supplied.** A command claiming an existing §852 citation or a release containing it would be false. The assignment's source-missing stop rule controls this exception to the requested command format. No placeholder or knowingly failing command should be dispatched. + +The following immutable references were verified by GitHub GET queries during this run; they are evidence of current state, **not future approved dispatch inputs**: + +| Input / implementation | Verified value | Origin | +|---|---|---| +| `rulespec_ref` | `f43dec520dd392bc5333934f56dad7498363e704` | rulespec-us remote `refs/heads/main`, matching local `origin/main` | +| `corpus_ref` | `942e138e7a8250c9814e774ac9b8e63008148106` | axiom-corpus remote `refs/heads/main`, matching local `origin/main` | +| `rules_engine_ref` | `6e709eb1ca7ea686263293d932c759d9dee48a4a` | axiom-rules-engine remote `refs/heads/main` | +| workflow code read | `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9` | axiom-encode remote main; workflow blob `b3ef651c6167342e366ddcc749193ff7e2efa45b` | +| `citation` | Intended `us/statute/26/852/b/3/B`, paired with (C); **unresolvable today** | Intended OLRC hierarchy; ingest must establish actual normalized rows | +| `country`, `pr_base_branch`, `open_pr` | Intended `us`, `main`, `true` | Present workflow input schema; no action taken | +| `replace_rulespec_path` | Omit for a genuinely new target | No existing target found | +| `existing_signed_imports_json` | Candidate `[]` | No eligible existing signed-v5 dependency established | + +The [workflow](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/.github/workflows/targeted-signed-reencode.yml) and [helper implementation](https://github.com/TheAxiomFoundation/axiom-encode/blob/5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9/src/axiom_encode/prepare_signed_backfill.py) were read in full. Relevant constraints: + +- Inputs require lowercase 40-character immutable repository SHAs. For ordinary `open_pr=true` dispatch, `rulespec_ref` must equal the exact remote main tip when validated; today's SHA will become unsuitable after prerequisite merges. +- Workflow lines 826–875 materialize the release from rulespec-us's pin. Moving only `corpus_ref` cannot widen that release. +- Existing-import entries are tracked same-jurisdiction **file paths**, not module IDs. Helper lines 1368–1510 require their canonical manifests to be signed-v5. +- The protected encode job uses environment `production-signing` and `/opt/axiom-verification/axiom-encode encode --backend openai ... --apply`; complete-source-unit is enabled by default. No local encode is permitted for this task. + +After ingest and dedicated re-pin, the orchestrator must refresh these identities, verify the resolved B/C source units and actual closure, finalize the fresh-source strategy, assemble the complete command, and obtain Max's approval for that specific signed run. + +## 6. Review finding text + +**Independent review memo retained for the post-ingest brief; not a ready `review_finding` submission.** + +Encode the shareholder treatment in §852(b)(3)(B) and the entire capital-gain-dividend definition in (C). The statute treats a qualifying capital gain dividend as gain on a capital asset held more than one year. Together with §61(a)(3), this requires inclusion of the gain; filing Schedule D is not a condition of that treatment. IRS instructions expressly provide a direct Form 1040 reporting route. A zero preferential tax rate does not remove the income from gross income or AGI. Keep the amount distinct from ordinary and qualified dividends to avoid double counting. [Official §852](https://www.govinfo.gov/link/uscode/26/852), [§61](https://www.govinfo.gov/content/pkg/USCODE-2024-title26/html/USCODE-2024-title26-subtitleA-chap1-subchapB-partI-sec61.htm), [IRS instructions](https://www.irs.gov/instructions/i1040gi). + +Coverage checklist for the eventual resolved source units: + +- (C)(i): written shareholder reporting; (ii)–(iv): excess reporting, proportional reduction, fiscal-year post-December exception, and all defined amounts including §855 distributions. The exception must protect pre-January dividends only when its stated conditions hold. +- (C)(v): increased reported amounts after a qualifying §860(e) determination remain subject to the subparagraph's limits. (C)(vi): account explicitly for the reference to (b)(8)'s late-year losses; do not treat pre-adjustment company gain as final when an election affects it. +- Establish the input contract for qualifying RIC/part-I applicability. External company net capital gain, adjusted reporting and election determinations may be separately computed inputs only with precise legal definitions and explicit responsibility for their upstream derivation. +- Defer company-level tax under (3)(A), undistributed-gain treatment and credits under (D), and unrelated loss-netting, preferential-rate and full-return calculations only by naming their source clauses and outputs. A narrow B/C encoding must still address the overriding classification in (E) when applicable; an explicit supported-profile exclusion must block a claim of complete shareholder treatment for that case. +- The 2025 IRS block can ground a separately scoped reporting rule, not a fabricated 2026 form. For the direct route, the source requires no capital losses, only box-2a distributions, no boxes 2b/2c/2d, and no QOF deferral. Nominee amounts belong to the actual owner. Failure of a reporting exception routes the amount elsewhere; it does not make the distribution exempt. + +Companion cases must assign **every** declared local input, including false booleans. Pair ordinary qualifying reporting with absent/incorrect reporting; no excess with positive excess; fiscal-year post-December relief with its blocking conditions; qualifying determination with no qualifying determination; applicability with its exclusion. Demonstrate that short ownership of the fund does not negate (B)'s deemed long-term character. Pair direct-reporting eligibility with each applicable blocking fact if that reporting rule is in scope. Use verbatim proof excerpts from the **resolved ingested row**, not this memo or an oracle docstring. + +The benchmark's `non_sch_d_capital_gains` input is treated as the already reportable shareholder amount. The households do not contain fund-level aggregate reporting, net gain, fiscal-year allocation or determination facts. Do not fabricate zero values for those corporate inputs to claim that these households test all of (C). Synthetic cases must test that definition independently. + +**Worked case 1 — scenario_042, Wisconsin, single, age 79, year 2026.** Relevant facts: distribution **$3,753**; Social Security $29,580; taxable IRA $19,200; pension $4,886; interest $175; ordinary dividends $5,920; qualified dividends $928; farm loss $1,127.2940673828125. Baseline and sandbox both show zero separately entered long- and short-term sale gains and no investment-income election. + +The independent §852/§61 contribution is **$3,753**, agreeing with the corrected inclusion; the omitted contribution of $0 is not supported. The additional AGI increase is larger because §86 changes taxable Social Security. Including the farm loss, other income before this distribution is $29,981.7059326171875. Provisional income becomes $48,524.7059326171875. Applying the single §86 upper-tier formula gives `min(0.85 × 29,580, 0.85 × (48,524.7059326171875 − 34,000) + 4,500) = 16,846.000042724609375`. AGI is therefore **$50,580.705975341796875**, or **$50,580.71**. The frozen AGI is $43,637.65234375; corrected AGI is $50,580.70703125. The small difference from exact arithmetic is floating-point precision. [Official §86](https://www.govinfo.gov/link/uscode/26/86). + +The $3,753 plus $928 qualified dividends produces a $4,681 preferential base. Holding the benchmark's single-filer deduction treatment fixed, use the $16,100 standard deduction, $2,050 age addition, $6,000 senior deduction and 2026 ordinary brackets. Taxable income is $26,430.705975341796875, ordinary taxable income $21,749.705975341796875, and federal tax is `$1,240 + 12% × ($21,749.705975341796875 − $12,400)` = **$2,361.964717041015625**, matching corrected **$2,361.96** to cents. Rate/deduction amounts were checked against [Rev. Proc. 2025-32](https://www.irs.gov/pub/irs-drop/rp-25-32.pdf) §§4.01, 4.03 and 4.14; the senior deduction against [IRS 2026 guidance](https://www.irs.gov/newsroom/2026-filing-season-updates-and-resources-for-seniors). This is conditional independent arithmetic, not an Axiom execution. The assigned provisions do not themselves fix Wisconsin deductions, rates or the separate retirement election, so the Wisconsin total below remains a conditional downstream comparison. + +**Worked case 2 — scenario_051, Louisiana, single, age 25, year 2026.** Wages $40,000; distribution **$180**; no separately entered sale gain/loss. The law-derived distribution inclusion is **$180**, so AGI is **$40,180**, matching corrected $40,180 rather than frozen $40,000. Federal tax remains $2,620 in both runs because the extra gain is in the zero-rate band; unchanged federal tax does not justify excluding the gain. The Louisiana state output increases by $5.40. The assigned federal provisions establish the $180 income correction; they do not alone establish Louisiana's full liability. + +**Worked case 3 — scenario_091, Wisconsin, single, age 22, year 2026.** Wages $27,000; traditional 401(k) contribution $926.1599731445312; interest $4,806; ordinary dividends $208; qualified dividends $3,968; traditional IRA contribution $43.276798248291016; distribution **$1,170**. No separately entered sale gain/loss or investment-income election. Given the stated deductible retirement treatment, AGI from the facts is `27,000 − 926.1599731445312 + 4,806 + 208 + 3,968 + 1,170 − 43.276798248291016 = 36,182.563228607177784`, or **$36,182.56**. Frozen AGI is $35,012.5625; corrected AGI $36,182.5625. **The statute supports the corrected $1,170 inclusion**, not the omitted $0 contribution. The preferential base rises from $3,968 to $5,138; the federal output is unchanged. Wisconsin's final total again requires separate state law. + +**Worked case 4 — synthetic (C) allocation and blocking pair.** A calendar-year RIC reports aggregate capital gain dividends $10,000 but has $8,000 net capital gain; the shareholder's reported amount is $1,000. Excess is $2,000, allocated share `$2,000 × $1,000 / $10,000 = $200`; qualifying capital gain dividend is **$800**. With net capital gain $10,000, the same shareholder amount qualifies in full: **$1,000**. For a noncalendar year with post-December reporting $4,000 and the same $2,000 excess, a $1,000 post-December dividend bears $500 excess and qualifies for **$500**, while an otherwise identical pre-January dividend bears zero excess and qualifies for **$1,000**. Reducing post-December reporting below $2,000 blocks that special allocation and restores the general proportional allocation. These are direct arithmetic applications of (C)(ii)–(iv), not household facts. + +**Every moved benchmark output** from `sweep_moves.csv`, `fix=r04_capital_gain_distributions`: + +| Household | Benchmark output | PE 1.755.4 frozen | Sandbox corrected | Independent adjudication | +|---|---|---:|---:|---| +| scenario_042 | Federal tax before refundable credits | 1,979.158325 | 2,361.964844 | Corrected direct inclusion $3,753 supported; federal calculation above gives **2,361.96**. | +| scenario_042 | State tax before refundable credits | 284.740906 | 414.204346 | Corrected inclusion supported; **neither full state total is established by §§852/61 alone**. Other Wisconsin provisions remain material. | +| scenario_051 | State tax before refundable credits | 814.950012 | 820.349976 | Corrected inclusion $180 supported; full state total needs Louisiana law. | +| scenario_091 | State tax before refundable credits | 843.661499 | 884.021851 | Corrected inclusion $1,170 supported; full state total needs Wisconsin law. | + +Conditional checks explain the state propagation without upgrading it to independent full-return certification: Wisconsin's retained 30% capital-gain subtraction leaves $2,627.10 / $819 additional state income in 042 / 091; the retained 12% deduction taper and 4.4% marginal rate give increases **$129.463488 / $40.360320**, matching the sweep to cents. Louisiana's retained 3% rate gives **$5.40**. These state parameters are engine-context checks, not conclusions derived from the assigned federal provisions. Scenario 042 also has a separately audited Wisconsin retirement-election interaction, so the r04-only corrected state value must not be advertised as the uniquely lawful final return. + +Household evidence: [fresh calculations and complete facts](../scratch/us-852-capgain/households/household_check.jsonl), [reproduction script](../scratch/us-852-capgain/households/household_check.py), [independent exact arithmetic](../scratch/us-852-capgain/households/independent_arithmetic.json), [four successful sweep comparisons](../scratch/us-852-capgain/households/reproduction_checks.json), and [source/command notes](../scratch/us-852-capgain/households/notes.md). The baseline/corrected script ran household-level calculations only, sequentially, with bytecode writes disabled; all four sweep rows matched to the CSV's six-decimal precision. No population simulation was run. The supplied fix, root-cause record, verification material and earlier coverage report were read as context, not substituted for the statutory reading. + +## 7. Risks + +- **Source resolution is the present blocker.** Issue [TheAxiomFoundation/.github#39](https://github.com/TheAxiomFoundation/.github/issues/39), read in this run, requires existing corpus provisions or official snapshots with URL/date/hash provenance. Do not encode from this report, broaden a release through `corpus_ref`, or edit the pin in a feature PR. +- **Source-unit completeness.** B alone requires a defined qualifying amount; C has excess-reporting, fiscal-year and determination branches. The proposed checklist retains those branches and names external computations and exclusions. A broad parent citation cannot silently drop corporate tax or undistributed gains. The workflow's complete-source-unit requirement is verified; the example failure artifact's additional `issues.json` findings were not downloaded, so their detailed contents remain briefing-only. +- **Input/proof discipline.** Household facts do not prove the fund-level prerequisites. Separate household inclusion checks from synthetic definition tests; assign every local input and ground every excerpt in the ingested row. The memo contains no RuleSpec or test YAML. +- **Closure/schema incompatibility.** The current loader rejects plural corpus paths, and the nearby capital-gains threshold file has an unsupported source-verification field. Avoid unneeded legacy imports, require eligible signatures for selected imports, and test the actual generated closure in the signed engine. Old local compile success is insufficient. +- **AGI composition.** §61's external aggregate is not wired to its enumerated-income output. A correct classification module alone does not establish end-to-end AGI or tax parity. Ensure future composition consumes the distribution exactly once and permits §86 feedback. +- **Timing and collisions.** As of the live checks, [#1384](https://github.com/TheAxiomFoundation/rulespec-us/pull/1384) is merged; [#1387](https://github.com/TheAxiomFoundation/rulespec-us/pull/1387) is open with successful strict run [35791296078](https://github.com/TheAxiomFoundation/rulespec-us/actions/runs/35791296078); [#1386](https://github.com/TheAxiomFoundation/rulespec-us/pull/1386) remains open and its inspected run was cancelled. [#1385](https://github.com/TheAxiomFoundation/rulespec-us/pull/1385) closed unmerged. No drift-stage or final activation/re-pin PR appeared among the open PRs examined. Coordinate the additional source-release pin with that serial chain rather than assuming its eventual result includes §852. +- **No target collision found, within the inspected scope.** The full open collections contained 21 rulespec-us and 15 axiom-encode PRs. Titles/bodies and specific searches found no §852/capital-gain-distribution work. The latest 30 targeted runs spanned September 21–23; today's three concerned Canada and none targeted this provision. Refresh immediately before dispatch. Encoder [#1675](https://github.com/TheAxiomFoundation/axiom-encode/pull/1675) concerns release-object size limits; current workflow enforces 16 MiB, but no size/publication blocker is asserted for the not-yet-created §852 release. +- **Limits of verification.** The current engine was inspected, not built or executed. Published release bytes/signatures and final 2026 Form 1040 instructions were not verified. No signed Axiom numerical result exists for this target. No oracle issue or other external write was made. + +Read-only checks completed: fresh remote ref/PR/run queries; canonical normalized-source and release-scope inventories; immutable module/manifest/closure inspection; 22 existing-module compiles using the discovered older binary; current-engine schema inspection; three household baseline/corrected runs reproducing all four moved outputs; independent statutory and exact-arithmetic checks. Supporting workflow, helper, source and protected failure log are in [workflow notes](../scratch/us-852-capgain/workflow/notes.md). + +Encoding regime retained verbatim for any subsequent lane brief: + +> RuleSpec content in any `rulespec-*` repository is produced ONLY by the supervised +> encoder: `axiom-encode encode --backend codex --apply` (local +> supervised runtime, subscription Codex auth via a lane CODEX_HOME; never +> OPENAI_API_KEY). Every atomic module carries the encoder's apply manifest under +> `.axiom/encoding-manifests/`. Hand-written YAML is never a module — not for a +> pilot, not for a demo, not "to avoid API spend". The only hand edits allowed are +> repair rounds on the encoder's output (findings file + replay) on repos whose +> `run-generated-guard` is off, and composed `module.kind: composition` pipelines, +> which are assembled, not encoded. Briefs to lanes must say this verbatim; a brief +> that says "hand-author" is wrong. New repos set `run-generated-guard: true`. + +For rulespec-us, landable content comes from the signed path only: the axiom-encode workflow `targeted-signed-reencode.yml` (workflow_dispatch; the `encode` job runs in environment `production-signing`, and its encode step runs `/opt/axiom-verification/axiom-encode encode --backend openai` with the org key). You never write RuleSpec YAML, never write test YAML for a module, and never run a local encode. + +```json +{ + "lane": "us-852-capgain", + "verdict": "BLOCKED", + "citation": "us/statute/26/852/b/3/B", + "in_corpus": false, + "in_pinned_release": false, + "existing_modules": [ + "us/statutes/26/61.yaml", + "us/statutes/26/62.yaml", + "us/statutes/26/1222.yaml", + "us/statutes/26/1/h.yaml", + "us/policies/irs/rev-proc-2025-32/capital-gains.yaml", + "us/statutes/26/1411.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Ingest official 26 USC 852(b)(3)(B), (C) and necessary context with retained URL/date/SHA provenance and canonical normalized citations.", + "Publish a signed corpus release containing those rows; the planned Canada-union release is insufficient.", + "Land a dedicated approved rulespec-us corpus pin coordinated with the serial re-pin chain.", + "If Form 1040 routing is encoded, resolve its tax-year source contract and include the appropriate IRS source in the signed release.", + "Finalize the B/C source-unit and fresh dependency strategy, check the actual closure in the current signed engine, refresh immutable refs at exact main, and obtain Max's approval for the resulting command." + ] +} +``` diff --git a/reference_audit/2026-09-22/verification/v9_axiom_wi-71-05-54m.md b/reference_audit/2026-09-22/verification/v9_axiom_wi-71-05-54m.md new file mode 100644 index 00000000..36134fde --- /dev/null +++ b/reference_audit/2026-09-22/verification/v9_axiom_wi-71-05-54m.md @@ -0,0 +1,224 @@ +# Wisconsin retirement subtraction: signed encoding preparation + +Lane `wi-71-05-54m`; inspected September 23, 2026. This is an Axiom coverage and dispatch-readiness package, not a finding against an external oracle. No encoding, repository commit, external write, or workflow dispatch was performed. Evidence is under [scratch/wi-71-05-54m](scratch/wi-71-05-54m/). + +The subfleet manifest specifies `/Users/maxghenis/PolicyEngine/_wk/axenc-pb/report-wi-71-05-54m.md`. That path is outside this lane's writable workspace. This file is its same-relative-path worktree counterpart; the orchestrator can collect it. Existing workspace files were preserved. + +## 1. Verdict + +**BLOCKED.** The statute's substantive text exists in the corpus and is selected by the pinned release manifest, but the requested narrow source unit is not resolvable. The encoder snapshot's own slicing function rejects `us-wi/statute/71.05/6/b/54m`. Separately, the 2025 Schedule SB instructions have discovery entries but no ingested provision or retained PDF in the inspected corpus tree. A source-unit ingest/normalization or resolver fix, the Schedule SB ingest, and an approved containing release/pin are prerequisites. This lane stops dispatch preparation at those source blockers; it supplies no executable dispatch command or invented canonical leaf. + +**Household adjudication: the statute supports the corrected $0 for scenario_042 when the retirement subtraction is elected.** The frozen $284.74 is the reproduced unelected-path value. Election is optional in the statute; selecting the lower-tax permitted path is the benchmark/model convention, not a statutory mandate. Section 6 provides the independent arithmetic and its assumptions. + +This cannot be certified for dispatch against today's main. The requested `git fetch -q origin` failed with `cannot open '.git/FETCH_HEAD': Operation not permitted`; GitHub CLI reads failed connecting to `api.github.com`. The verified local rulespec-us `origin/main` is **`f43dec520dd392bc5333934f56dad7498363e704`**, the merge of waiver-renewal #1384. The #1387 activation, #1386 workflow-pin, drift stage, and Canada activation/re-pin chain were not verified live. Their named endpoint retains the same Wisconsin scopes and does not remedy either source blocker. After the source prerequisites and approved pin changes, re-check the chain and choose a stable exact main: the workflow checks that base both before encoding and before publishing a PR. + +## 2. Source + +Official statute: [Wis. Stat. 71.05(6)(b)54m](https://docs.legis.wisconsin.gov/statutes/statutes/71/i/05/6/b/54m). The live Legislature endpoint was unavailable, so the retained official HTML and its normalized corpus record were read. + +| Item | Verified evidence | +|---|---| +| Canonical corpus checkout `origin/main` | `942e138e7a8250c9814e774ac9b8e63008148106` | +| Actual canonical provision citation | **`us-wi/statute/71.05`**; one whole-section record, including 54m.a–e | +| Tracked provision file | `data/corpus/provisions/us-wi/statute/2026-07-16-pit-west-chapter-71.jsonl` | +| Retained source | `data/corpus/sources/us-wi/statute/2026-07-16-pit-west-chapter-71/wisconsin-statutes-html/statutes/statutes/71.html` | +| Source URL recorded by corpus | `https://docs.legis.wisconsin.gov/statutes/statutes/71?view=section` | +| Vintage | `source_as_of` and `expression_date`: `2026-04-03`; source version: `2026-07-16-pit-west-chapter-71`; ingest manifest generated July 21 | +| Source HTML SHA256 | `662f8ba8b5410f11f34b4c8a3d0b6767ce778845287e9fa84ddfd3b6f6e50e98`, recomputed from the tracked blob and matching the ingest manifest | +| Consumer release | `us-rulespec-2026-08-08-obbb-alien-snap`, content pin `0d69a0cdbe024fc2276f3c261c00402bb0a47488ac5f482cc20bd3404980adbc` | + +The [pinned release manifest](scratch/wi-71-05-54m/pinned-release.json) selects jurisdiction `us-wi`, document class `statute`, version `2026-07-16-pit-west-chapter-71`, and the 2026 Form 1-ES source. Thus the whole-section statute is **in corpus main and within the pinned release's declared selection**. `axiom-locate release` found no local published archive; the archive's bytes/signature were not independently verified. The [Canada endpoint manifest](scratch/wi-71-05-54m/chain-endpoint-release.json) selects those same Wisconsin sources. + +The intended child path `us-wi/statute/71.05/6/b/54m` is **not an exact corpus record and fails parent slicing**. The [resolver probe](scratch/wi-71-05-54m/workflow/resolver-probe.log), using unmodified resolver code from encoder commit `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`, reports: + +```text +Could not isolate 'us-wi/statute/71.05/6/b/54m' from active parent 'us-wi/statute/71.05'; missing structural marker path '6/b/54m' +``` + +The source labels this subdivision `54m.`; the generic slicer looks for parenthesized markers. `/6/b` is also unsafe: it starts at a cross-reference inside paragraph (a), rather than the subtraction paragraph. `/6` returns over 55,000 characters. Substituting the 100,371-character whole section would expand the source-coverage obligation well beyond this assignment. No such substitution is proposed. + +**Required statutory source work:** either normalize/ingest a canonical 54m source unit retaining the subtraction chapeau, all a–e branches, cross-references, official URL, retrieval date, source hash, and effective-date provenance; or land a tested resolver change that isolates that same complete unit from the already retained HTML. An ingestion that creates a new corpus record also needs an approved release and consumer pin selecting it. A resolver-only repair may reuse the already selected statute, subject to signed-runtime validation. + +**Schedule SB:** [2025 Schedule SB instructions](https://www.revenue.wi.gov/TaxForms2025/2025-ScheduleSB-Inst.pdf), revision October 2025, page 7 line 16 and page 8 line 17, were read from the official PDF. In corpus main, `manifests/us-wi-tax-forms.yaml` has two case-variant discovery entries, source IDs `taxforms2025-2025-schedulesb-inst-1e6e63914c` and `taxforms2025-2025-schedulesb-inst-7163d493ed`, dated May 23, 2026. These are **not** ingested source records. Neither the Wisconsin provision inventory nor retained-source inventory contains that PDF; neither named release selects a Schedule SB source scope. There is no verified canonical ingested citation for its line 16. + +**Required Schedule SB ingest:** the official PDF URL above, format PDF, retaining the original artifact and URL/date/SHA provenance; normalize line 16 with its credit caution and line 17's no-double-counting worksheet as addressable source units. Resolve the case-variant discovery duplicates against the actual artifact. Publish/select an approved signed release. Per the task's stop rule, no Schedule SB dispatch citation or command is supplied before that ingest. + +Temporal caution: [DOR Tax Bulletin 233, page 4](https://www.revenue.wi.gov/WisconsinTaxBulletin/233-04-30-WTB.pdf) describes Act 174 amendments effective April 4, 2026. The retained statute includes those amendments despite its April 3 expression label. Source snapshot date and legal effective date must be distinguished. The 2025 instructions also describe a broader Form 1 credit restriction than the retained statute's literal §71.07 cross-reference; resolve that scope/vintage question before encoding a universal credit ban. It does not change this household's zero result. + +## 3. Existing rulespec-us coverage + +The full tracked tree and targeted text search on the rulespec-us SHA above found **two income-tax modules**, no `us-wi/statutes/71.05...` module, and no 54m subtraction/election implementation. The other Wisconsin primaries are CMS eligibility modules, unrelated to this citation. A historical `pilot_liability_pipeline` manifest remains, but no corresponding primary module exists in this tree; it is not runnable coverage. + +| Module | What it encodes and the relevant boundary | +|---|---| +| `us-wi/policies/income_tax/2026_full_year_resident_core.yaml` | Bounded full-year AGI, deductions, exemptions and selected credits. Explicitly excludes other retirement subtractions. Public complete tax outputs are held zero sentinels, not a legal-value computation. | +| `us-wi/policies/income_tax/2026_form1es_estimated_schedule.yaml` | Estimated tax from supplied taxable income and filing selectors. Does not calculate retirement income, an election, or annual liability. | + +Relevant verbatim source from the [core snapshot](scratch/wi-71-05-54m/existing-wi-core.txt): + +```text +509: and not wi_pit_2026_has_retirement_subtraction_other_than_encoded_military +1630: - name: wi_pit_2026_has_retirement_subtraction_other_than_encoded_military +1634: description: Whether another retirement-system or age-based retirement subtraction applies. +``` + +Its direct-subtraction formula, lines 647–652, comprises U.S. obligation interest, federally taxable Social Security, military/uniformed-services retirement, and specified insurance subtractions. It has no ordinary age-67 retirement subtraction. Lines 1320–1335 name `wi_pit_2026_complete_tax_after_nonrefundable_credits`, describe it as held pending the annual Tax Table, and set `formula: 0`. + +The [estimated-schedule snapshot](scratch/wi-71-05-54m/existing-wi-estimated.txt), lines 23–29, states: + +```text +It does not +construct deductions, exemptions, or annual-return taxable income; +apply credits or return rounding; prorate nonresident or part-year- +resident brackets; map surviving-spouse status; compute estimated +payments or final annual liability; or claim exact PolicyEngine +parity. +``` + +**Axiom conclusion:** missing coverage, not the same demonstrated numerical defect. Neither module implements the two election paths needed to adjudicate the benchmark output. A held zero cannot be counted as an Axiom match to the independently derived zero. A new atomic subtraction module would fill one component; a later governed composition would still need to wire its elected income and credit consequences consistently into the return outputs. + +## 4. Import closure + +**Candidate `existing_signed_imports_json`: `[]`.** A narrow atomic subtraction module should use retirement receipts, federal taxability/qualification, age, filing/election, already-exempt amounts, and residency/allocation facts. It need not import federal EITC or a whole state return. Its proposed import closure is empty, so no import reaches 26/32 or the known EITC dependency. + +The workflow helper accepts tracked primary YAML **paths**, not module IDs, and requires signed-v5 manifests in the target's exact `us-wi` jurisdiction. Both adjacent tax modules have `axiom-encode/applied-rulespec/v1`, `backend: manual` manifests, so neither is an eligible signed-v5 import. + +| Nearby module, excluded from proposed closure | Compile evidence | +|---|---| +| `us-wi:policies/income_tax/2026_full_year_resident_core` | **Fails** with the canonical local v0.2.0 binary: removed plural `corpus_citation_paths` at module line 6. [Actual diagnostic](scratch/wi-71-05-54m/wi-core-current-compile.log). | +| `us-wi:policies/income_tax/2026_form1es_estimated_schedule` | **Loads**, compiling five derived outputs. Still not a signed-v5 import and not needed. [Compile log](scratch/wi-71-05-54m/wi-estimated-current-compile.log). | + +The compile inputs were byte-identical `git show` snapshots under an isolated scratch `rulespec-us` root; no RuleSpec was authored or repaired. Binary: `/Users/maxghenis/TheAxiomFoundation/axiom-rules-engine/target/release/axiom-rules-engine`, SHA256 `faf4383622f63c64b861e5772b78b00df97efef4a8315b792b25219033bee75e`. Its exact build commit is unverified. Independently, engine source at `6e709eb1ca7ea686263293d932c759d9dee48a4a`, `src/rulespec.rs:813–823`, explicitly rejects the plural key. + +`axiom-locate engine` initially returned an older v0.1.0 binary that compiled both modules. That legacy result is retained in the scratch logs but is not used to assert current signed-runtime compatibility. + +## 5. Dispatch inputs + +**No eligible `gh workflow run` command exists for this package yet.** Supplying the unresolved child, a discovery-only Schedule SB citation, or an unapproved broader whole-section target would not meet the requested pass-ready standard. This is the task's source-ingestion stop condition, not a request for additional permission. `dispatch_command` is therefore empty below. + +Verified values for the orchestrator's eventual reconstruction, **not a dispatch authorization**: + +| Input / workflow selector | Verified value or disposition | +|---|---| +| Repository / workflow / ref | `TheAxiomFoundation/axiom-encode`, `targeted-signed-reencode.yml`, `main` | +| Inspected workflow commit | `5d80d753f1ad54bc4b6a0686522ae9aa69e47bb9`, local encoder `origin/main` | +| `country`, `pr_base_branch` | `us`, `main` | +| `rulespec_ref` | Observed `f43dec520dd392bc5333934f56dad7498363e704`; must be refreshed to the exact stable main after prerequisites | +| `corpus_ref` | Observed `942e138e7a8250c9814e774ac9b8e63008148106`; a source ingest will require a newly verified containing commit | +| `rules_engine_ref` | Inspected source `6e709eb1ca7ea686263293d932c759d9dee48a4a`; verify approved runtime/ancestry before use | +| `citation` | **Unset:** actual record is `us-wi/statute/71.05`; desired child `/6/b/54m` fails resolution | +| `replace_rulespec_path` | Empty for a new atomic module; do not replace the broad annual core | +| `existing_signed_imports_json` | `[]` | +| `source_bundle_json` | `[]`; no guessed Schedule SB source citation | +| `review_finding` | Section 6, revised after source-unit and vintage resolution | +| `open_pr` | `true` for the eventual Max-approved run | +| Repair/dependent/legacy/queue inputs | Empty/default; no repair run or dependent transaction proposed | + +Mechanisms read in the [workflow](scratch/wi-71-05-54m/workflow/targeted-signed-reencode.yml) and [helper](scratch/wi-71-05-54m/workflow/prepare_signed_backfill_impl.py): input block lines 4–115; immutable lowercase 40-character checkout checks and corpus/engine ancestry at workflow lines 365–401; exact main requirement in helper lines 267–332; main re-check before publication at workflow lines 3590–3593. The workflow uses `production-signing`, `/opt/axiom-verification/axiom-encode encode --backend openai --apply`, and complete-source-unit validation. Release materialization at workflow lines 827–877 follows the RuleSpec checkout's pin; changing `corpus_ref` alone does not make an unselected new source usable. + +The named serial chain is therefore a scheduling prerequisite to re-check, **not a sufficient source fix**. Once that chain and required source/release work settle, re-read live main, workflow, imports, and same-citation run history, fill the new immutable values, and let Max approve the concrete signed dispatch. + +## 6. Review finding text + +The following is a prospective finding brief. It must be finalized against the resolved source unit after the blockers in section 2 are removed. + +**Required coverage.** Encode the complete 54m.a–e unit, including the applicable taxable-year start, qualified-plan/IRC 408 receipt restriction, previously exempt government/military/uniformed-services/railroad payments, age at year-end, individual $24,000 cap, joint $48,000 pooling when both spouses qualify, §71.07 credit/carryover prohibition upon claiming, part-year allocation and nonresident exclusion. The provision has no general AGI phaseout; do not borrow subdivision 54's income thresholds. Derive the permitted subtraction and the consequences of a supplied election fact. Do not import a minimum-tax/tie-breaking policy into statutory eligibility. Use one singular canonical source path per source/proof node and verbatim excerpts from the actual resolved text. + +**Precisely deferred boundaries.** Defer computing federal plan qualification and federal taxability to externally established facts, while applying those conditions explicitly. Defer separate exemption computations under 71.05(1)(a)/(am)/(an) and railroad law, while removing their already-exempt amounts from the base. Defer the sibling 54/line-17 entitlement, full AGI construction, indexed deduction, annual tax-table computation and individual credit formulas to separate source units/composition. Expose the subtraction and credit restriction so that composition can recompute income/deduction/tax and report one consistent elected return. Do not claim complete annual tax from this atom. If part-year mixed-spouse or zero-denominator cases cannot be supported from the resolved authority, identify those cases precisely and fail closed with a coverage hold; do not return a normal-looking final amount or silently omit paragraph e. + +**Instruction boundary.** Once ingested, account for line 16's federally taxable and previously subtracted income limits and credit caution. Preserve the line-17 worksheet's prevention of double subtraction. Its federal-AGI eligibility is determined before the Wisconsin subtraction; a reduced Wisconsin AGI cannot create federal-AGI eligibility. Resolve the 2025 instructions versus amended statutory credit scope before making all Form 1 credits zero. Do not disable tax withholding or estimated payments as though they were prohibited credits. + +**Required paired tests.** Age 67 versus 66; taxable-year beginning after 2024 versus in 2024; election true versus false; qualifying taxable IRA/plan receipts versus unqualified/already-exempt receipts; both joint spouses age 67 versus one younger, including all distributions paid to one spouse; full-year/part-year eligibility versus nonresident; ordinary permitted credit and prior-year carryover without an election versus their §71.07 prohibition when the subtraction is claimed. Assign every local `#input`, including all false flags, in every companion case. Include allocation caps and no-double-subtraction boundaries after the relevant sources are resolved. New failures have no waiver route. + +**Worked case 1 — the entire moved benchmark set: scenario_042.** The [exact root-cause filter](scratch/wi-71-05-54m/benchmark-moves.json) returns one moved output, `state_income_tax_before_refundable_credits`: + +| Household facts / result | Value | +|---|---:| +| Tax year / state / modeled filing status | 2026 / WI / single | +| Adult age; spouse/dependents | 79; none | +| Taxable IRA / private pension | $19,200 / $4,886 | +| Already tax-exempt IRA, not included in subtraction base | $4,700 | +| Frozen 1.755.4 output, freshly reproduced | $284.74090576171875 | +| Supplied sandbox-corrected output | $0 | +| Independently derived elected-path output | **$0** | + +The [scenario record](scratch/wi-71-05-54m/scenario_042.json) also supplies interest $175, ordinary dividends $5,920, qualified dividends $928, farm loss $1,127.2940673828125, Social Security dependent benefits $29,580, capital-gain distributions $3,753 and qualified BDC income $70.93914031982422. `is_surviving_spouse=true` is present, but the household calculation returns `SINGLE`; no $48,000 cap is assumed. Full-year resident treatment and an elected subtraction are explicit case assumptions. + +On the r06-isolated income base, ordinary Wisconsin income excluding Social Security is independently summed as: + +```text +5,920 + 928 + 175 + 19,200 + 4,886 - 1,127.2940673828125 + = 29,981.7059326171875 +Eligible retirement receipts = 19,200 + 4,886 = 24,086 +Elected subtraction = min(24,086, 24,000) = 24,000 +Post-subtraction income = 5,981.7059326171875 +Taxable income = max(0, 5,981.7059326171875 - 13,960 - 950) = 0 +``` + +The $24,000 amount assumes the private pension is from a qualifying plan; the **zero-tax conclusion does not require that assumption**. Using only the explicit taxable IRA gives $10,781.7059326171875 remaining income. Even adding the full $3,753 capital distribution and $70.93914031982422 BDC amount conservatively gives $14,605.64507293701172, still below $13,960 + $950 = $14,910. This upper bound is not an assertion about the exact tax treatment of those additional items. + +The $13,960 deduction is the official 2026 single schedule for income through $20,119; the $950 exemption combines the $700 personal amount and $250 age-65 amount. [2026 Form 1-ES instructions, page 2](https://www.revenue.wi.gov/TaxForms2026/2026-Form1-ES-inst.pdf). The statute supplies the Social Security subtraction and personal exemptions in 71.05. Zero taxable income yields zero pre-credit tax; a barred credit cannot create positive tax. The only observed standard-path credit was the $300 school-property credit; observed refundable credits were zero. Federal AGI was $43,637.652, so the single-filer line-17 $15,000 ceiling was not met. + +**Adjudication: supports the corrected $0 on the elected return.** The frozen $284.74 is the reproduced model's unelected-path calculation. The statute permits leaving the subtraction unclaimed; this review does not certify that model amount as the lawful unelected liability. It does not represent the elected lower-tax path being benchmarked. No signed Axiom numeric result exists yet. + +**Worked case 2 — individual threshold.** Full-year resident, nonjoint, 2026, age 67, $30,000 taxable IRC 408 IRA, no previously exempt amount, election true: subtraction **$24,000**, and §71.07 credits/carryovers barred. Change only age to 66: subtraction **$0**. Change only election to false: subtraction **$0**, and this provision's credit bar does not apply. A year beginning in 2024 also gives **$0** under this provision. + +**Worked case 3 — joint pooling.** Full-year joint filers aged 67/67, qualifying taxable IRA receipts $48,000/$0, election true: **$48,000** subtraction. Ages 67/66 with the same receipts: **$24,000**. Ages 66/67 with the same receipts: **$0**, because only the younger spouse received the income. The first result tests pooling independent of receipt allocation; it must not be capped at $24,000 merely because only one spouse received the distribution. + +**Worked case 4 — exclusions and residency.** Age 70, single, full-year resident, election true, receipts consisting of $20,000 independently exempt military retirement plus $10,000 taxable IRC 408 IRA: subtraction **$10,000**. A separate part-year variant with $30,000 qualifying IRA and statutory allocation numerator $20,000 / denominator $40,000 has limit **$12,000**. A nonresident otherwise eligible has subtraction **$0**. These test subtraction outputs, not invented complete tax liabilities. + +The binding preparation regime supplied for encoder handoff is reproduced verbatim: + +> RuleSpec content in any `rulespec-*` repository is produced ONLY by the supervised +> encoder: `axiom-encode encode --backend codex --apply` (local +> supervised runtime, subscription Codex auth via a lane CODEX_HOME; never +> OPENAI_API_KEY). Every atomic module carries the encoder's apply manifest under +> `.axiom/encoding-manifests/`. Hand-written YAML is never a module — not for a +> pilot, not for a demo, not "to avoid API spend". The only hand edits allowed are +> repair rounds on the encoder's output (findings file + replay) on repos whose +> `run-generated-guard` is off, and composed `module.kind: composition` pipelines, +> which are assembled, not encoded. Briefs to lanes must say this verbatim; a brief +> that says "hand-author" is wrong. New repos set `run-generated-guard: true`. + +For rulespec-us, landable content comes from the signed path only: the axiom-encode workflow `targeted-signed-reencode.yml` (workflow_dispatch; the `encode` job runs in environment `production-signing`, and its encode step runs `/opt/axiom-verification/axiom-encode encode --backend openai` with the org key). You never write RuleSpec YAML, never write test YAML for a module, and never run a local encode. + +## 7. Risks + +- **Source resolution and release selection:** verified leaf-slicing failure and missing Schedule SB ingest are hard blockers. Whole-section substitution would create a much larger complete-source-unit obligation. Require the exact source unit and validate its bytes before a paid/signed run. +- **Incomplete source coverage:** an age/cap-only formula would omit exclusions, optional election, pooling, credit carryovers and residence. The finding covers a–e and identifies external computation boundaries; any unresolved part-year subcase must be precisely deferred and held. +- **Unsuitable imports:** the annual core fails current-engine compilation and both neighboring tax manifests are v1/manual. The proposed empty closure avoids these and the federal EITC path. It does not repair the annual core. +- **Proof/test rejection:** use substrings of the resolved provision, singular source paths, and complete local input assignments with positive/blocking pairs. Do not alter toolchain, workflow pins, CODEOWNERS or waivers in this feature. [Binding issue #39](https://github.com/TheAxiomFoundation/.github/issues/39) was successfully read through the web tool after the requested CLI read failed. +- **Temporal/credit scope:** April 2026 amendments and October 2025 instructions need explicit version treatment. Do not infer that every refundable credit outside §71.07 is barred solely from the current 54m.d cross-reference. The benchmark case's result is insensitive to that unresolved general scope. +- **Annual liability claim:** the existing core's missing annual tax table is a separate coverage hold. An atomic subtraction encoding does not establish complete Wisconsin liability coverage; the zero household derivation is independently supported by zero taxable income. +- **Live collisions and moving main — unverified:** attempted the required recent signed-run list and open-PR searches for `71.05` in rulespec-us and axiom-encode; CLI requests failed. Web attempts for current filtered PRs, #1387/#1386, the workflow runs page and run 35789753522 were unavailable; a cached August PR page cannot clear September 23 collisions. No claim of “no collisions” or inspection of that failure artifact is made. Re-check open PRs and today's runs for `71.05`, `54m`, and Schedule SB before dispatch; exact-main validation can invalidate a run during the serial chain. [Workflow-readiness evidence](scratch/wi-71-05-54m/workflow/notes.md). + +Validation completed: exact affected-row extraction (one household); one pinned 1.755.4 household reproduction; independent decimal arithmetic, including the IRA-only upper bound; exact corpus/release/module inventory checks; resolver-owned child-slicing probe; and compilation of the two unchanged existing modules with the current local binary (one expected schema failure, one success). The reproduction command and recorded intermediates are in the [household research notes](scratch/wi-71-05-54m/statute-household-notes.md). No population run, new RuleSpec, module test YAML, local encode, signed run, or external repository mutation was performed. + +In the JSON below, `citation` and the two presence booleans refer to the **existing whole-section record** and the release manifest's source selection. They do not assert that the requested 54m child or Schedule SB is resolver-ready. `blocking_imports` describes the proposed empty closure; the incompatible neighboring core is documented above. + +```json +{ + "lane": "wi-71-05-54m", + "verdict": "BLOCKED", + "citation": "us-wi/statute/71.05", + "in_corpus": true, + "in_pinned_release": true, + "requested_leaf": "us-wi/statute/71.05/6/b/54m", + "requested_leaf_resolvable": false, + "schedule_sb_in_corpus": false, + "release_archive_verified": false, + "existing_modules": [ + "us-wi/policies/income_tax/2026_full_year_resident_core.yaml", + "us-wi/policies/income_tax/2026_form1es_estimated_schedule.yaml" + ], + "blocking_imports": [], + "dispatch_command": "", + "prerequisites": [ + "Make the complete 71.05(6)(b)54m source unit canonically resolvable through governed ingestion/normalization or a tested resolver repair.", + "Ingest the official 2025 Schedule SB PDF with URL/date/SHA provenance and addressable line 16 and line 17 context; reconcile effective dates and credit scope.", + "Publish and activate an approved containing release and consumer pin for any newly ingested source; the named Canada chain alone does not add Schedule SB.", + "Re-verify the serial re-pin chain, live exact rulespec-us main, approved corpus/engine/workflow refs, signed-release bytes, and same-citation open PRs/runs.", + "Finalize the immutable dispatch command for the verified narrow citation and obtain Max's signed-run approval." + ] +} +``` diff --git a/reference_audit/2026-09-28/README.md b/reference_audit/2026-09-28/README.md new file mode 100644 index 00000000..1a3b719f --- /dev/null +++ b/reference_audit/2026-09-28/README.md @@ -0,0 +1,64 @@ +# Reference upgrade, September 29, 2026 + +This directory records how PolicyBench's US references moved from policyengine-us 1.755.4 to policyengine-us 2.15.17, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded 00:23 UTC). PolicyBench built the references at 11:57 UTC that day, and rebuilt them at 15:04 UTC to correct the records' dates and wording; no reference value changed. Later that day it rewrote the records without recomputing any output, to carry the reviewers' full reasons and the one output excluded on review (below). policyengine-us 2.17.0, the newest release when PolicyBench checked PyPI on 2026-09-29 at 14:58 UTC, gives the same value as 2.15.17 for all 1,984 outputs under the same conventions and adapter. The reference sidecar's `engine_upgrade` revision lists every change. + +Of the 1,984 outputs: +- 4 scored references change value; +- 3 outputs are newly excluded from scoring; +- 2 references change by less than the $1 exact-match tolerance; +- 19 already-excluded outputs were recomputed, re-reviewed and stay excluded. + +On review of the release, PolicyBench also excluded one output whose reference did not move ("Also excluded on review", below). + +The record now has 1,928 scored outputs and 56 exclusions: 28 engine-defect and 28 unlisted-input. + +The records date the upgrade by its UTC day. The directory's name, 2026-09-28, is the US Eastern date on which the wave began; the sweep began at 01:42 UTC on 2026-09-29 (`verification/sweep_timing.json`), so the upgrade is dated 2026-09-29. The one decision dated 2026-09-28 is Max's ruling, labeled as his, and `clusters.json` labels each review with its local date. + +## The rules + +1. References come from the newest policyengine-us release when PolicyBench begins the reference sweep. Before publishing, PolicyBench checks that the newest release on PyPI gives the same values, and records when it checked (step 6 of the method). The rule follows Max's ruling of 2026-09-28: "we should be using the latest pe for this always!" policyengine.py 6.1.2 is recorded for provenance. Its certified US bundle is policyengine-us 2.2.1, which still carries SNAP rounding defects fixed since, and `import policyengine` 6.1.2 refuses to load next to 2.15.17. PolicyBench computes references with `policyengine_us.Simulation` directly, as it always has, so the sidecar records `model_matches_policyengine_bundle: false`. +2. A scored reference follows from the stated facts and from law published before the 2026-07-03 reference freeze (Max, 2026-09-22). The nine publication conventions of the September 22 audit are re-expressed for 2.15.17 in `fixes/latest_c_*.py`. The SNAP convention needed a rewrite: 2.15.17 applies reforms after uprating and carries USDA's FY2027 figures as published values, so holding the uprating index alone changes nothing, and each FY2026 figure is held directly for October to December 2026. The eight upstream fixes the September 22 audit regenerated (r04, r09, r17, r26, r27, r28, r31, r33) are all in 2.15.17 and need no module. +3. The benchmark's output definitions hold. policyengine-us #8888 folded Maryland county income tax into its state income tax aggregate, but PolicyBench's state income tax output excludes local tax. `fixes/latest_md_local_output_scope.py` restores that scope; the county tax stays in the federal SALT deduction. +4. An output whose reference turns on a fact the prompt does not state is excluded, as before. +5. Excluded outputs recorded before this wave keep the values they were decided on. They are not scored. + +## What changed and why + +| Output | 20260922c | Now | Why | +|---|---:|---:|---| +| scenario_008 state refundable credits (NJ) | 5,342.40 | 5,842.40 | New Jersey child tax credit schedule for 2026-2028, P.L.2026, c.26, approved June 30, 2026 (upstream #8971) | +| scenario_013 SNAP (AZ) | 0 | 240 | Arizona raised its expanded categorical eligibility gross limit from 185% to 200% of poverty from benefit month 03/2026 | +| scenario_028 reduced-price school meals (PA) | 1 | 0 | Child support received counts as income for school meals (7 CFR 245.6(a)(5)(ii)); 1.755.4 left it out | +| scenario_082 state refundable credits (NY) | 650.50 | 667.00 | Empire State child credit phase-out rounding (upstream #9425) | +| scenario_033, 078 and 117 federal income tax | scored | excluded | Whether a listed state and local tax refund is income depends on whether the refunded tax reduced federal tax in the prior year, which the prompt does not say (26 U.S.C. 111(a)). 2.15.17 counts the whole refund (upstream #9422, fixing issue #9122) | + +Also: +- **Stated weekly hours.** Upstream #9261 changed the default of `weekly_hours_worked_before_lsr`, the input SNAP's work rules read, from 40 to 0. The prompt's "usual weekly hours worked" is `hours_worked_last_week`, so `policybench.scenarios.PE_INPUT_ALIASES` now passes the stated value under both names. Without it, scenario_066 (40 stated hours) would fail SNAP's ABAWD test. Its reference stays at 3,576. +- **Changes under $1.** scenario_078 and scenario_117 state income tax each move by under $1 through the refund. +- **The 19 excluded outputs that move on 2.15.17.** Each was re-reviewed and stays excluded. Their defects (r01, r02, r07, r11, r30, r32) are still in 2.15.17, or they depend on an unlisted input that no engine version resolves. The sidecar's `excluded_outputs_rechecked` gives each one's 2.15.17 value and reason. + +## Also excluded on review + +The investigation of cluster `excl_snap_ssi_disability` flagged one scored output outside its cluster, scenario_023's head Medicaid eligibility, and the cluster's independent review agreed. The prompt says only that the head is disabled. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition of disability (42 CFR 435.540(a)), which is the unlisted input `meets_ssi_disability_criteria` that already excludes this household's SNAP. policyengine-us 2.15.17 tests the broad `is_disabled` flag instead, so its reference of 1 follows the reading under which the head meets that definition. With the program's disability test reading `meets_ssi_disability_criteria`, the other reading gives 0 (`scripts/probe_023_medicaid.py`, `verification/probe_023_medicaid.json`). Rule 4 excludes the output. + +The reference did not move, so the sidecar's `engine_upgrade` revision, which lists the engine's changes, does not name it. `final_actions.json` lists it under `audit_exclusions`, `clusters.json` records the disposition under `reconciliations`, and `verification/reviews/pr182_review_023.md` gives the review and the computed values. + +## Method + +1. **Sweep.** `scripts/sweep_latest.py` recomputed all 1,984 outputs on 2.15.17, reusing the household builder of the September 22 harness (`scripts/sweep.py`). It compares each output with the 20260922c reference and with raw 1.755.4. `verification/sweep_timing.json` records when the sweep began and each release's PyPI upload time. On 2.15.17, PolicyBench re-ran four of the September 22 sweeps over every output: the IRA deduction limit fix (r02), the net investment income tax definition (r25), and the readings for mortgage residence and 40 unlisted weekly hours. It also ran a new sweep for the state and local tax refund reading (`fixes/latest_alt_*.py`; the r25 sweep composes `fixes/alt_conventions_r25.py`). Each sweep adds its fix or reading to the conventions sweep (`latest_conventions`), except the 40-hour sweep, which adds its reading to the conventions with the stated-hours alias (`latest_map_stated_hours`). Against those baselines, no sweep moves a scored output by more than the $1 exact-match tolerance. Three scored state income tax outputs move by less: scenario_082's under the IRA fix, and scenario_078's and scenario_117's under the refund reading. Against the conventions sweep alone, the 40-hour sweep also moves scenario_066's SNAP, whose prompt states 40 hours; the stated-hours alias, which the references apply, gives it the same value. `verification/rerun_sweeps.json`, written by `scripts/summarize_rerun_sweeps.py`, lists every output each sweep moves. The other defect fixes and readings of the September 22 audit ran on 1.755.4 only. +2. **Port.** Two agents ported the conventions (reports in `verification/port_*`). `fixes/latest_conventions.py` composes them, and `fixes/latest_final.py` adds the Maryland output-scope adapter. +3. **Explain.** Every output that still moved was grouped into 16 root-cause clusters (`verification/compose_result.json`). An investigator settled each cluster from the upstream commits, the engine code in both versions, and primary legal sources with their publication dates. An independent adversarial reviewer then tried to refute each settlement. `clusters.json` holds each cluster's investigation and review. Six reviews ran in a Claude Code workflow begun on the evening of 2026-09-28, US Eastern time (early on 2026-09-29 UTC), and ten on subfleet review lanes later on 2026-09-29 (`verification/reviews/`). Where two reviews conflicted (scenario_078 federal), `clusters.json` records the reconciliation. It also records the disposition of the one output an investigation flagged outside its cluster (scenario_023's head Medicaid eligibility). +4. **Build.** `scripts/build_references_latest.py` recomputes every output with `latest_final` and writes the references and sidecar. It refuses any scored output that moves without a reviewed action in `final_actions.json`, any listed action the engine does not reproduce, and any excluded output that moves without a recheck. It records each rechecked output's reason as its reviewer's full text in `clusters.json`. Its docstring gives the rebuild command. `scripts/rewrite_reference_records.py` applies the builder's record functions to the installed records when no output needs recomputing. `scripts/install_adds0929_references.py` installs either script's output only if the reference CSV is byte-identical and nothing changed but record text and the audit exclusions `final_actions.json` lists. +5. **Record.** `sweep_moves.csv` lists every output with these values: + - raw 1.755.4 (`v11_1755`); + - the 20260922c reference (`board_20260922c`); + - raw 2.15.17 (`raw_2_15_17`); + - 2.15.17 with the conventions (`conventions`); + - 2.15.17 with the conventions and the adapter (`final`); + - the published reference (`reference`). + + Each moved output also carries its cluster and action, as does the output excluded on review. +6. **Verify.** Before publishing, PolicyBench read PyPI on 2026-09-29 at 14:58 UTC, when policyengine-us 2.17.0 was the newest release (`verification/sweep_timing.json`). `verification/latest_final_2170.csv` and `verification/latest_final_2170.log` hold `scripts/sweep_latest.py --fix fixes/latest_final.py` run on policyengine-us 2.17.0 with policyengine.py 6.1.2. It reproduces every scored reference, and it gives each of the 19 rechecked excluded outputs the 2.15.17 value the sidecar records. A second check before publishing, at 18:28 UTC, found policyengine-us 2.17.1 (uploaded 17:24 UTC); it also gives the same value for all 1,984 outputs (`verification/latest_final_2171.csv`). +7. **Date the judge verdicts.** `scripts/date_adds0928_judge_verdicts.py` dates each judge verdict the adjudication record keeps from its sha256-bound sidecar, and writes what it read to `verification/judge_verdicts.json`. `verification/flagged_sept22_wave.json` lists the cases the September 22 wave's judge runs flagged; a kept flag that the dated verdict does not raise names that wave in `judge_reference_suspect_source`. + +`tests/test_reference_upgrade.py` checks that every module the revision names is committed unchanged, that the table is the committed reference, and that every change is listed and backed by its cluster's review. The scripts use the paths of the machine that ran them, as the September 22 scripts do. diff --git a/reference_audit/2026-09-28/clusters.json b/reference_audit/2026-09-28/clusters.json new file mode 100644 index 00000000..8efbbf12 --- /dev/null +++ b/reference_audit/2026-09-28/clusters.json @@ -0,0 +1,1699 @@ +{ + "wave": "2026-09-28", + "engine": "policyengine-us 2.15.17", + "clusters": [ + { + "id": "md_county_tax_8888", + "title": "Maryland county income tax wired into the state output and SALT (scored)", + "outputs": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "state": "MD", + "board": 1255.342773, + "latest": 2089.608887, + "v11_1755": 1252.967773, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "hold_output_scope" + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "state": "MD", + "board": 6936.337402, + "latest": 11295.388672, + "v11_1755": 6936.337402, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "hold_output_scope" + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "state": "MD", + "board": 24780.613281, + "latest": 24164.457031, + "v11_1755": 24772.693359, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "exclude_unlisted_input" + } + ], + "investigation": { + "cluster_id": "md_county_tax_8888", + "classification": "mixed", + "summary": "Upstream #8888 (6b0bca0b9f, 2026-07-05; in 2.15.17, not in 1.755.4) did two separate things, and they need different rulings. First, it redefined PolicyEngine's state_income_tax_before_refundable_credits to include Maryland county income tax. That is a change of output scope, not of law. The benchmark defines this output as \"excluding local income and payroll taxes\" (benchmark_specs.json), and the county is not an input: the county variable falls back to first_county_in_state, which is ALLEGANY_COUNTY_MD at 3.03%. So the county tax adds 834.27 to 068 state and 4,358.51 to 078 state. The existing adapter latest_md_local_output_scope.py (sha 53a6de3c) fixes this. Swept on top of latest_conventions (out/latest_md_county_tax_8888_conv_scope.csv), it changes exactly two outputs: 068 state becomes 1,255.342773 (equal to the board) and 078 state becomes 6,936.876465 (+0.539 against the board, all of it from #9122 through the MD itemized phase-out, so within tolerance). The moved count drops from 29 to 27. Second, #8888 put county withholding into md_withheld_income_tax, which feeds federal SALT. That is an upstream fix of law published before the freeze: IRC 164(a)(3) makes local income taxes deductible, Md. Tax-Gen. 10-103(a) puts a county tax on every resident, and 10-106(a)(1) sets that tax at 2.25% to 3.30%. It does not depend on the unlisted county. Under the board's withholding proxy, SALT reaches the $40,400 cap at any county rate of 2.09% or more with #9122, or 2.15% or more without it. Both are below the 2.25% statutory floor, and pinning Worcester, Talbot, Garrett, Cecil, Anne Arundel, Frederick, Montgomery or Dorchester gives the same 24,164.457031 as the Allegany default. Engine-free recomputation (recompute.py) reproduces every value to within cents. Removing both #8888's SALT piece and #9122 reproduces the board exactly: 24,780.613281 federal and 6,936.337402 state. So 078 federal should adopt 2.15.17: 24,164.457031 if the salt_refund_gross_income_9122 cluster adopts #9122, or 23,772.800781 if it holds #9122.", + "upstream_changes": [ + { + "commit": "6b0bca0b9f", + "date": "2026-07-05", + "what": "#8888: adds md_local_income_tax_before_refundable_credits to parameters/gov/states/household/state_income_tax_before_refundable_credits.yaml (the only difference in that list between 1.755.4 and 2.15.17). Adds the county share, flat county rate times (AGI - 3,400), to md_withheld_income_tax, which feeds state_withheld_income_tax and then SALT. Adds the MD local EITC and poverty-line credits (both 0 here). Effect: 068 state +834.27; 078 state +4,358.51; 078 federal -986.27 with #9122 present (SALT 36,290.53 -> 40,400 cap) or -1,007.81 without #9122 (36,200.77 -> 40,400)." + }, + { + "commit": "97137280f7", + "date": "2026-07-22", + "what": "Adds an UNKNOWN-county fallback to Allegany in md_withheld_income_tax. It has no effect here: an unlisted county already resolves through county.py -> first_county_in_state to ALLEGANY_COUNTY_MD (decomp shows county_str = ALLEGANY_COUNTY_MD). That resolution is where the 3.03% default comes from." + }, + { + "commit": "b711aa56b7", + "date": "2026-07-23", + "what": "Adds the same UNKNOWN-county guard to md_applicable_local_tax_rate and md_flat_rate_county_tax. No effect on these scenarios." + }, + { + "commit": "316e7832a1", + "date": "2026-09-08", + "what": "#9122 (belongs to cluster salt_refund_gross_income_9122): salt_refund_income enters federal gross income and is subtracted from MD AGI. For 078, AGI goes 198,505 -> 200,136.91, which gives federal +391.66 with the SALT cap binding. The MD itemized phase-out (Tax-Gen 10-218(c): 7.5% of federal AGI over 200,000) then cuts MD itemized by 10.27, so state +0.539." + } + ], + "law": "Primary sources read 2026-09-28; copies are in triage/latest/md_county_tax_8888/.\n\n(1) Md. Code, Tax-General 10-103(a) (mgaleg.maryland.gov): \"Each county shall have a county income tax on the Maryland taxable income of each resident\" domiciled in the county. 10-106(a)(1): the county rate is \"at least 2.25% but not more than 3.30%\", for taxable years beginning after December 31, 2001. 10-106(b)(2): rate changes must be noticed to the Comptroller by July 1 before they take effect, so 2026 rates were fixed by 2025-07-01. 10-106(c)(2)(iii): bracket counties also have a 2.25% minimum. All of this is pre-freeze.\n\n(2) Comptroller of Maryland 2026 Employer Withholding Guide, \"Revised December 2025\" (pre-freeze): \"Withholding is a combination of the state income tax rate and local taxes\". Residents are withheld at their county's rate, and the lowest calculated rate is 2.25%. It gives the $3,400 standard deduction the board's c_md_2026 uses.\n\n(3) IRC 164(a)(3) (uscode.house.gov) allows State and local income taxes. 164(b)(6)-(7) sets the cap at $40,400 for 2026, phased down above a $505,000 MAGI threshold. Source: P.L. 119-21 sec. 70120, July 4, 2025, pre-freeze.\n\n(4) IRS 2025 Instructions for Schedule A, dated Dec 8, 2025: line 5a includes \"State and local income taxes withheld from your salary\".\n\n(5) Md. Tax-Gen 10-105(a)(1) brackets, 10-211 exemption, and 10-218(b)-(c) itemized deductions: used in the independent recomputation. On the 10-218 addback when SALT is capped, the Comptroller accepts \"any reasonable interpretation\" and gives a property-tax-first example (quoted by Thomson Reuters, 2024-02-20; I could not render the Comptroller's own JS page). PolicyEngine's md_itemized_deductions (itemized_deductions_less_salt + capped_property_taxes) matches that example, so MD itemized is 54,670 whether or not the cap binds.\n\n(6) Output definition: policybench/benchmark_specs.json defines state_income_tax_before_refundable_credits as \"excluding local income and payroll taxes\". local_income_tax covers only NYC, Philadelphia, Kansas City and St. Louis.\n\nConclusion: county tax is pre-freeze law and belongs in federal SALT. It must not appear in the state output.", + "per_output": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "board": 1255.342773, + "latest": 2089.608887, + "proposed_reference": 1255.342773, + "action": "hold_by_convention", + "reason": "The convention here is the output definition, not a law hold. 2.15.17 adds Maryland county tax at the default Allegany rate (27,533.53 x 3.03% = 834.27) to an output the benchmark defines as excluding local income taxes. The MD state piece is 1,255.342773, recomputed without the engine: 20 + 30 + 40 + 4.75% x 24,533.53 on MD taxable income of 34,083.53 - 3,350 - 3,200. The existing adapter latest_md_local_output_scope.py, applied with latest_conventions, gives exactly the board value. Federal 068 is unchanged because the filer takes the standard deduction." + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "board": 6936.337402, + "latest": 11295.388672, + "proposed_reference": 6936.876465, + "action": "hold_by_convention", + "reason": "Same cause as 068: 4,358.51 of county tax (143,845.27 x 3.03%) sits inside the state aggregate. With the adapter, the value is 6,936.876465, recomputed engine-free as 6,936.8766. The +0.539 against the board comes entirely from #9122: federal AGI of 200,136.91 triggers the 10-218(c) phase-out (7.5% x 136.91 = 10.27). If #9122 is held, the value is 6,936.337402, equal to the board. Both are within $1. MD itemized stays 54,670 before the phase-out even though SALT is now capped, because PolicyEngine subtracts cap minus property tax, which is the Comptroller's own addback example." + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "board": 24780.613281, + "latest": 24164.457031, + "proposed_reference": 24164.457031, + "action": "adopt_latest", + "reason": "The #8888 SALT piece is an upstream fix of pre-freeze law (IRC 164(a)(3); Tax-Gen 10-103/10-106). Under the board's withholding proxy, the county share is 3.03% x 196,736.91 = 5,961.13, which lifts SALT from 36,290.53 to the $40,400 cap. The value does not depend on the county: the cap binds at any rate of 2.089% or more, below the 2.25% statutory floor, and eight pinned counties all give 24,164.457031. 24,164.457031 also includes #9122 (+391.66). Adopt it if cluster salt_refund_gross_income_9122 adopts #9122. If #9122 is held, the reference is 23,772.800781 (#8888 only). Both were recomputed without the engine: taxable income 131,526.91 or 129,895, with 24% above 105,700. Sensitivity: the Comptroller's literal withholding formula (wages, one $3,200 exemption) needs a rate of 2.28% or more to cap. At Worcester's 2.25% only, federal tax would be about $14 higher. The board has used PolicyEngine's proxy convention throughout." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "hold_module is empty, but both state holds depend on latest_md_local_output_scope.py (sha256 53a6de3cd9b1...), and the composed latest_conventions.py deliberately leaves that module out. If the fold uses latest_conventions alone, it publishes 2,089.608887 (068 state) and 11,295.388672 (078 state). Both include Maryland county tax, which the output definition excludes ('excluding local income and payroll taxes', benchmark_specs.json). The settlement should name the adapter as the module, and the lead must add it to the composed module. My harness runs confirm it changes exactly 2 of 1,984 outputs (moved 29 -> 27).", + "The settlement's county-tax figures describe an engine artifact, not the law. 2.15.17 resolves the unlisted county to the alphabetically first MD county (first_county_in_state.py -> ALLEGANY_COUNTY_MD). Its gov.local.md.flat_rate still carries Allegany at 3.03% (set 2023) and Kent at 3.20% for 2026. The Comptroller's Central Payroll Bureau memo '2026 Maryland State and Local Income Tax Withholding Information' (Feb 4, 2026, pre-freeze), Attachment 1, lists Allegany at 3.20% and Kent at 3.30%, and gives 3.30% as the default for an unknown county. So 834.27 and 4,358.51 are county tax at a stale rate for an arbitrarily chosen county. This is an unfixed 2.15.17 parameter defect. It changes no proposed reference, because the adapter removes county tax from the state output and SALT caps at every 2026 rate. It would matter if the adapter were ever dropped.", + "The claim that 078 federal 'does not depend on the unlisted county' holds only under the board's withholding-proxy convention. That convention is c_md_2026: 'federal tax through the Maryland withholding proxy'. Under a liability reading, where a solver deducts 2026 MD plus county liability, SALT stays below the $40,400 cap and the answer depends on the county. Using the Comptroller's 2026 rates I get 24,705.95 to 25,068.44 with #9122, and 24,314.51 to 24,676.97 without. The proposed 24,164.457031 is 2.2% to 3.7% below that range, while the old board value (24,780.61) fell inside it. I still accept adoption. IRC 164 deducts taxes paid (withheld) in the year, the Comptroller says MD withholding 'is a combination of the State income tax ... and local taxes', and the board already scores this output through the proxy. The settlement should still state the dependence on this convention explicitly.", + "Under the Comptroller's literal formula, the federal value depends slightly on the county. The formula applies the rate to wages less the $3,400 allowance and one $3,200 exemption. At Worcester's 2.25% (verified as the 2026 rate), SALT is 40,342.64, which is $57.36 under the cap, so federal is 24,178.23 (+13.77). That misses the $1 exact band but is within 1%. The settlement discloses this (as 'about $14'). It does not affect the proxy-based reference.", + "Adopting #8888's SALT piece creates a new interpretive sensitivity in 078 state. Once SALT is capped, the Md. Tax-Gen 10-218(b)(3) addback ('the amount claimed as taxes on income paid to a state or political subdivision') has several readings, and the Comptroller 'will accept any reasonable interpretation'. Property-first gives 6,936.88; that is the engine's result and the Comptroller's own example. Pro-rata gives 7,008.19 (+71.31, 1.03%), and income-first gives 7,051.65 (+114.77). The proposed 6,936.876465 is the right reference: it is the Comptroller's example and also the value under every reading where the cap does not bind. But the example rests only on a secondary source (Thomson Reuters, 2024-02-20). The Comptroller's KB page needs authentication, so neither the investigator nor I could read the primary text. The primary 2025 resident booklet says only that line 17b 'is capped at $40,000'.", + "Minor labelling point. For the two state outputs, 'hold_by_convention' is really an output-definition mapping: the 2.15.17 variable is correct Maryland law, just outside the output's scope. It is not a hold of pre-freeze law. The sidecar revision should record it as a permanent mapping grounded in benchmark_specs.json, not as a law convention." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "action": "hold_by_convention", + "proposed_reference": 1255.342773, + "reason": "This is an output-scope mapping (latest_md_local_output_scope.py, which must be added to the composed module), not a hold of pre-freeze law. 2.15.17 adds md_local_income_tax_before_refundable_credits to the state list (#8888, 6b0bca0b9f; that one line is the only change to the list between 06665727d8 and 79be99f671). The output is defined as excluding local income taxes, and Maryland county tax is a local income tax (Tax-Gen 10-103, 10-106). Harness (latest_conventions + adapter) gives 1,255.342773, equal to the board. Pinning Worcester gives the same value, so with the adapter the output does not depend on the unlisted county. Engine-free: MD taxable income 34,083.53 - 3,350 - 3,200 = 27,533.53; tax 90 + 4.75% x 24,533.53 = 1,255.34." + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "action": "hold_by_convention", + "proposed_reference": 6936.876465, + "reason": "Same output-scope mapping as 068. Harness with the adapter gives 6,936.876465 (+0.539 against the board), and the value is the same at all 11 counties I pinned. The +0.539 comes from #9122: federal AGI 200,136.91 triggers Tax-Gen 10-218(c), 7.5% x 136.91 = 10.27 less MD itemized, x 5.25%. With salt_refund_income zeroed the value is 6,936.337402 (= board), so this output should follow the salt_refund_gross_income_9122 ruling; both values are within $1. Engine-free: MD itemized 28,210 + 26,460 - 10.27 = 54,659.73, exemption $0 (10-211(c)(1)(iii)), taxable 143,845.27, tax 6,936.88. This is the property-first addback, which matches the Comptroller's example; pro-rata or income-first readings would give +71 or +115." + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 24164.457031, + "reason": "#8888's county withholding in md_withheld_income_tax fixes the SALT treatment of pre-freeze law: IRC 164(a)(3) allows local income taxes, Tax-Gen 10-103(a) and 10-106(a)(1) set a 2.25% to 3.30% county tax, and the Comptroller's 2026 memo (Feb 4, 2026) says withholding combines state and local tax. Under the board's c_md_2026 withholding proxy, the cap binds at any county rate of 2.089% or more (2.152% without #9122). The lowest 2026 rate is Worcester's 2.25%, so the value holds for every county. Harness: 24,164.457031 at all 11 counties pinned. Engine-free: AGI 200,136.91 - 28,210 - 40,400 = 131,526.91; 17,966 + 24% x 25,826.91 = 24,164.46 (Rev. Proc. 2025-32 Table 3; cap per 164(b)(7)(A)(ii)); no AMT (TMT 21,275). This value assumes #9122 is adopted. If salt_refund_gross_income_9122 holds #9122 or adopts the no-prior-benefit reading, the reference is 23,772.800781 (harness with the refund zeroed; engine-free 23,772.80). Caveat: the value depends on the proxy convention; under a liability reading it would be county-dependent, 24,706 to 25,068." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv): (1) latest_conventions on 068/078 reproduces all three 'latest' values exactly, with no other output of the two scenarios moving (rv_conv_068_078.csv). (2) latest_conventions + latest_md_local_output_scope, both unmodified and loaded by rv_fix.py, gives 068 state 1,255.342773 (= board), 078 state 6,936.876465 and 078 federal 24,164.457031 (rv_conv_scope_068_078.csv). (3) The same pair over the full sweep changes exactly 2 of 1,984 outputs against out/latest_conventions.csv and lowers moved from 29 to 27. My output is identical to the investigator's out/latest_md_county_tax_8888_conv_scope.csv (rv_conv_scope_full.csv). These are the only two MD scenarios. One-process variants (rv_variants.py/.csv): 11 pinned counties, including Worcester 2.25%, Talbot 2.40%, Anne Arundel, Frederick and Dorchester, plus salt_refund_income zeroed. With the adapter, federal and state are county-invariant, federal is 23,772.800781 and state 6,936.337402 with the refund zeroed, and local_income_tax and state_refundable_credits stay 0. Code read with git show, read-only: 6b0bca0b9f (#8888) diff, md_withheld_income_tax.py at 06665727d8 and 79be99f671, state_and_local_sales_or_income_tax.py, local_income_tax.py, MD local tax variables, county.py and first_county_in_state.py, the gov.local.md parameters, and commits 97137280f7, b711aa56b7 and 316e7832a1 (#9122). The list parameter has a single 0000-01-01 value, so the adapter's 2015-2034 update is safe. Prompt: I rendered the exact batch prompt with prompts.make_no_tools_batch_prompt (rendered_prompts.txt). It lists no county and no withholding. The state output text says 'excluding local income and payroll taxes', and local_income_tax names only NYC, Philadelphia, Kansas City and St. Louis. Primary sources fetched this session: Md. Tax-Gen 10-103, 10-106, 10-211 and 10-218 (mgaleg.maryland.gov, current text); 26 USC 164(a)(3) and (b)(6)-(7) ($40,400 for 2026, $505,000 threshold); Rev. Proc. 2025-32 Table 3 and the AMT exemption (IRS, as of Oct 9, 2025); and the Comptroller CPB memo on 2026 MD state and local withholding (Feb 4, 2026), including its county rate table. Secondary: Thomson Reuters, 2024-02-20, quoting the Comptroller's addback example. The Comptroller's KB API returned 'User is not authenticated'. Board provenance: c_md_2026 in reference_outputs.csv.meta.json and root_causes.json ('federal tax through the Maryland withholding proxy'). Scoring (analysis.py): exact within $1, plus 1%, 5% and 10% bands. Independent engine-free recomputation (rv_recompute.py/.txt) reproduces 1,255.34, 6,936.88/6,936.34 and 24,164.46/23,772.80. It covers every 2026 county under the proxy, the literal Comptroller formula (Worcester +13.77) and the liability reading (24,705.95 to 25,068.44), plus the addback alternatives (+71.31 pro-rata, +114.77 income-first)." + } + }, + { + "id": "salt_refund_gross_income_9122", + "title": "Listed SALT refund now in federal gross income (scored)", + "outputs": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "state": "SD", + "board": 3788.988037, + "latest": 3818.148193, + "v11_1755": 3788.988037, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "exclude_unlisted_input" + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "state": "AR", + "board": 24391.796875, + "latest": 24961.339844, + "v11_1755": 24391.796875, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "exclude_unlisted_input" + } + ], + "investigation": { + "cluster_id": "salt_refund_gross_income_9122", + "classification": "mixed", + "summary": "Upstream #9122 alone moves both outputs. It landed as commit 316e7832a1 on 2026-09-08 and was merged through PR #9422 as e990b4a6f8 on 2026-09-16, adding the listed salt_refund_income to gov.irs.gross_income.sources. With only that entry removed, 2.15.17 plus the conventions reproduces the board to the cent (033: 3788.988037; 117: 24391.796875). A recompute that uses no engine gives 3818.15 vs 3788.99 and 24961.34 vs 24391.80: the refund at the 12% and 22% 2026 MFJ rates (Rev. Proc. 2025-32).\n\nThe law is pre-freeze: 26 USC 61 and 111(a), whose text dates to 1986, and the 2025 Form 1040 instructions, whose cover is dated Feb 25, 2026. There is nothing to hold. 2.15.17 is right if the listed amount is the taxable Schedule 1 line 1 figure, which is how the engine now documents the input and how the data built it (policyengine-us-data puf.py: salt_refund_income = PUF E00700). 1.755.4 was also inconsistent: Idaho, South Carolina and West Virginia subtracted a refund that federal AGI never contained.\n\nThe prompt, however, shows \"state and local tax refund income\" without the word \"taxable\" and gives no prior-year facts. Under 111(a) and the IRS line 1 instructions, none of the refund is taxable if the household did not itemize, or deducted sales tax instead, in the year it paid the tax. The prompt also says to treat unlisted status facts as false. Both households take the standard deduction in 2026. South Dakota has no income tax, so 033's $243 cannot be a line 1 income-tax refund at all.\n\nModel explanations on the 22c board split both ways. On 033, 7 models, including claude-opus-5.5, gpt-6-astra and gpt-5.6-sol, hit the 2.15.17 value by including the refund. claude-opus-4.7, claude-fable-5 and claude-sonnet-5 excluded it as untaxable after a prior-year standard deduction.\n\nRecommended ruling: adopt the 2.15.17 value as the reference, but exclude both outputs from scoring as reference_depends_on_unlisted_input, the same class as r16 (survivor-benefit taxability) and r24 (disability-benefit taxability). The alternative value is the board value.\n\nThe alternative-reading module fixes/latest_alt_salt_refund_no_prior_benefit.py zeroes the listed refund on the conventions stack. Swept over all 1,984 outputs (out/latest_alt_salt_refund_no_prior_benefit.csv), it moves exactly 7 outputs against latest_conventions:\n- 033 federal: -29.16\n- 117 federal: -569.54\n- 078 federal: -391.66\n- 005 federal: -1,327.26\n- 005 CA state: -47.14\n- 082 federal: -14.85\n- 120 federal: -394.85\n\nThe same ruling covers the other outputs. 078 MD federal becomes an unlisted-input exclusion whatever md_county_tax_8888 decides. 005, 082 and 120 federal stay excluded, with the SALT-refund input added to their records; on the refund reading alone their alternative equals the board exactly. 005 CA state also moves, and it is already excluded.", + "upstream_changes": [ + { + "commit": "316e7832a1", + "date": "2026-09-08", + "what": "Include state tax refund income in federal gross income (#9122): adds salt_refund_income to parameters/gov/irs/gross_income/sources.yaml (comment 'Schedule 1 line 1'), adds it to many state subtraction lists (CA, DC, MD, MN, ND, NY, PA, WI and others), documents salt_refund_income as 'Taxable state and local income tax refunds, credits, or offsets reported on Form 1040, Schedule 1, line 1' (reference 26 USC 111), and makes salt_refund_last_year add salt_refund_income" + }, + { + "commit": "5581f5be1a", + "date": "2026-09-09", + "what": "Standardizes state refund inputs on salt_refund_income; the sources.yaml comment becomes 'Taxable state and local income tax refunds (Schedule 1 line 1 / IRC \u00a7 111)'" + }, + { + "commit": "56d7361c31", + "date": "2026-09-14", + "what": "Adds IRC \u00a7 111 and state statutory citations to the gross income sources metadata" + }, + { + "commit": "e990b4a6f8", + "date": "2026-09-16", + "what": "Merge of PR #9422 (fix-9122-salt-refund-gross-income) into main; in 2.15.17 (79be99f671)" + } + ], + "law": "- 26 U.S.C. 111(a), read at uscode.house.gov (laws in effect 2026-09-27): \"Gross income does not include income attributable to the recovery during the taxable year of any amount deducted in any prior taxable year to the extent such amount did not reduce the amount of tax imposed by this chapter.\" The text was last amended by Pub. L. 99-514 (1986-10-22), with 26 U.S.C. 61(a) alongside it.\n- IRS 2025 Instructions for Form 1040, Schedule 1 line 1 (PDF cover dated Feb 25, 2026): \"None of your refund is taxable if, in the year you paid the tax, you either (a) didn't itemize deductions, or (b) elected to deduct state and local general sales taxes instead of state and local income taxes.\" The State and Local Income Tax Refund Worksheet limits the taxable amount further.\n- IRS Pub. 525 (2025), PDF created 2026-02-25, \"Deductions not itemized\": a recovery is not included if the taxpayer did not itemize.\n- Rev. Rul. 2019-11, 2019-17 I.R.B. (2019-04-22): how much of a recovery is taxable when the SALT cap applied.\n- South Dakota DOR (page undated): \"South Dakota is one of seven states that does not impose a state income tax.\"\n- Recompute parameters: Rev. Proc. 2025-32 (PDF 2025-10-17) for 2026 MFJ brackets $24,800/$100,800/$211,400, the $32,200 standard deduction, the $1,650 additional amount and the $2,200 CTC. From uscode.house.gov: 26 U.S.C. 151(d)(5)(C) (senior deduction of $6,000, 6% phase-out over $150,000 MFJ), 225(b) (overtime deduction capped at $25,000 MFJ, phase-out over $300,000) and 170(p) (non-itemizer cash charity up to $2,000 MFJ).\n- Every source is dated before 2026-07-03, so hold vs adopt is decided by adopt. Scoring is decided by the prompt, which lists no prior-year itemization.", + "per_output": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "board": 3788.988037, + "latest": 3818.148193, + "proposed_reference": 3818.148193, + "action": "adopt_latest", + "reason": "The reference value adopts 2.15.17: pre-freeze IRC 61/111, with the listed $243 read as taxable Schedule 1 line 1 income. The independent recompute gives AGI 83,451.24, taxable income 35,951.24 and tax 3,818.15; taxable Social Security stays at the 85% cap of 16,320 under both readings. RECOMMEND EXCLUDING it from scoring as reference_depends_on_unlisted_input. The unlisted input is whether the prior-year deduction of the refunded tax reduced federal tax (IRC 111(a); 2025 Form 1040 instructions, Schedule 1 line 1). The prompt says unlisted status facts are false. The couple takes the $35,500 + $12,000 standard and senior deductions against $1,169 of itemizable tax. South Dakota has no income tax, so this cannot be a line 1 income-tax refund. alternative_value = 3788.988037, which equals the board, 1.755.4, 2.15.17 + conventions with #9122's federal entry removed, and the recompute (3,788.99). Models split on this output: 7 included the refund and hit 3818.12; claude-opus-4.7, claude-fable-5 and claude-sonnet-5 excluded it as untaxable." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "board": 24391.796875, + "latest": 24961.339844, + "proposed_reference": 24961.339844, + "action": "adopt_latest", + "reason": "The reference value adopts 2.15.17, with the listed $2,588.82 as taxable Schedule 1 line 1 income. The independent recompute gives AGI 250,733.35, less the $32,200 standard deduction, the $2,000 IRC 170(p) charity deduction and the $25,000 IRC 225 overtime deduction, for taxable income of 191,533.35. Tax before credits is 31,561.34, less a $6,600 CTC (the CDCC is $0 because the head has no earnings), giving 24,961.34. RECOMMEND EXCLUDING it from scoring as reference_depends_on_unlisted_input: prior-year itemization and tax benefit under IRC 111(a) are not stated. The household takes the standard deduction in 2026; itemized deductions of 33,840 are below 34,200 under either reading, a choice taken from the engine's SALT figure. alternative_value = 24391.796875, which equals the board, 1.755.4, 2.15.17 + conventions with #9122 removed, and the recompute (24,391.80). The AR state output moves only $0.48 under the alternative, below the $1 tolerance. Models split on this output too: claude-opus-4.7, claude-fable-5 and qwen-3.7-max excluded the refund; claude-opus-4.8, claude-sonnet-5 and glm-5.2 included it." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": false, + "problems": [ + "The per_output action is wrong: both outputs are coded adopt_latest, but the reason text says to exclude them. If the lead applies the actions as written, both outputs get scored at the 2.15.17 value. The review standard says an output that depends on an unlisted fact is excluded, not adopted. I agree with the substance, which is exclusion. Both outputs need an exclusion record with reason_code reference_depends_on_unlisted_input, the same class as the board's r16 and r24 records (e.g. scenario_002, 062, 107 and 121 in reference_exclusions.json). Fields: frozen_value = the 2.15.17 value; alternative_value = the board value; unlisted_input = whether the prior-year deduction of the refunded tax reduced federal tax. That turns on three facts: prior-year itemization, the income-versus-sales-tax election, and SALT-cap headroom.", + "The settlement leaves out evidence that settles whether the prompt states the engine's input. PolicyBench renders the input with a fixed label map. policybench/prompts.py:130 maps salt_refund_income to 'state and local tax refund income'. Inputs that are already the taxable amount get an explicit 'taxable' in the same map: 'taxable interest income', 'taxable IRA distributions', 'taxable private pension income' and 'taxable 401(k) distributions'. In 2.15.17 the engine label is still 'State and local tax refund income'. #9122 (316e7832a1) only added documentation calling it the 'Taxable ... Form 1040, Schedule 1, line 1' amount. So the prompt does not state the engine's input contract, and a careful reader could treat the $243 and the $2,589 as gross refunds whose taxability they must work out.", + "This overstates: 'South Dakota has no income tax, so 033's $243 cannot be a line 1 income-tax refund at all.' The prompt says nothing about prior-year residence or out-of-state income. A household living in SD in 2026 can get a refund of another state's or a locality's 2025 income tax, and that refund is Schedule 1 line 1. Refunds of real-property or general sales tax are also section 111 recoveries (Pub. 525 (2025), recoveries worksheet line 2). SD itself pays sales and property tax refunds to seniors, per dor.sd.gov, but its income limits ($17,215 alone, $23,265 household) rule out this household. The conclusion is unchanged, because taxability still turns on unlisted prior-year facts. The SD fact is at most a plausibility hint.", + "The ambiguity runs both ways more than the settlement says, and no single default settles the six outputs. That is why exclusion is right and hold_by_convention on either value is not.\n- 117: if the 2026 facts also held in 2025, the household would likely have itemized in 2025. Using the engine's 2026 SALT proxy of 9,193.71 plus the full $25,900 of cash charity gives about $35,094, against the 2025 MFJ standard deduction of $31,500 (2025 Form 1040 instructions). The 0.5% charitable floor and the $2,000 170(p) non-itemizer deduction only apply to tax years beginning after 2025 (Pub. L. 119-21, sections 70424(b) and 70425(c)). The tax benefit of about $3,594 exceeds the $2,589 refund, so the whole refund is taxable, which gives the 2.15.17 value. The prompt's rule that unlisted status is false gives the board value instead.\n- 033 under the same constant-facts reading: itemizable tax of 1,168.80 is far below the standard deduction, so none of the refund is taxable, which gives the board value.\n- 078: 2.15.17's own SALT paid is 15,791.66 + 26,460 = 42,251.66, which is 1,851.66 over the $40,400 cap and more than the $1,631.91 refund. By Rev. Rul. 2019-11 Situation 2 (IRB 2019-17, 2019-04-22; refund 'not includable'), the refund would be wholly nontaxable if 2025 looked like 2026, since the 2025 cap was $40,000. That holds even though this filer itemizes.", + "The two cluster settlements conflict on 078. The md_county_tax_8888 settlement proposes adopt_latest for 078 federal (24,164.457031, or 23,772.800781 if #9122 is held). This settlement makes 078 federal an unlisted-input exclusion. The lead must reconcile them. Under this review, 078 federal is excluded with frozen_value 24,164.457031 (2.15.17 + conventions) and alternative_value 23,772.800781 (the alt module; I verified it). That alternative assumes md_county_tax_8888 keeps #8888's county-SALT piece. If md holds it, the alternative must be recomputed.", + "Outputs already excluded: I verified that the refund reading alone makes the alt module equal the board exactly for 005 federal (106,505.898438), 082 federal (9,563.052734) and 120 federal (40,021.816406). But their r02_ira_219g records carry 1.755.4 frozen values. On 2.15.17 those become 107,833.156250, 9,577.902344 and 40,416.667969. Their r02 alternative values were computed when the refund was never income, so each refreshed record has to state which refund reading its alternative uses. 005 CA state also moves under the alt: 41,267.011719 becomes 41,219.867188, a change of -47.144531. It is already excluded (r02+r11) and should name the refund input too. The excl_r02_ira_219g_federal and excl_r11 reviews own those records.", + "The model tallies are incomplete, though the ruling is unchanged.\n- 033: exactly 7 models land within $1 of the 2.15.17 value: claude-opus-5.5, gpt-5.6-luna, gpt-5.6-sol, gpt-5.6-terra, gpt-6-astra, gpt-6-luna and ox-alpha. None land within $1 of the board. Besides the three named, qwen-3.7-max explicitly excluded the refund, and gemini-3.7-flash implicitly did (its stated AGI of 83,208 leaves it out).\n- 117: no model is within $1 of either value. qwen-3.7-max's explanation stops mid-recalculation, so calling it an excluder is not supported." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "action": "exclude_unfixed_defect", + "proposed_reference": 3818.148193, + "reason": "Exclude from scoring. The enum has no unlisted-input code, so this is recorded as reason_code reference_depends_on_unlisted_input; it is not an engine defect.\n\nThe record: frozen_value 3818.148193 (2.15.17 + latest_conventions; I reproduced it bit-for-bit, and it equals out/latest_conventions.csv). alternative_value 3788.988037, which equals the board, v11_1755, the alt module, and 2.15.17 + conventions with only salt_refund_income dropped from gov.irs.gross_income.sources (my rv_undo9122_federal.py). Engine-free recompute: 3,818.15 vs 3,788.99. That is AGI 83,451.24 vs 83,208.24, with taxable Social Security capped at 16,320 either way, $47,500 of standard plus senior deductions, and a difference of 0.12 x 243 = 29.16.\n\nThe law is pre-freeze: 26 USC 111(a), last amended by Pub. L. 99-514 section 1812(a)(1), 1986-10-22. The 2025 Form 1040 instructions (created 2026-02-25) say 'None of your refund is taxable if ... you ... didn't itemize'. Pub. 525 (2025) says the same.\n\nThe prompt labels the input 'state and local tax refund income', without the 'taxable' the prompt uses for already-taxable inputs, and gives no prior-year facts. So the value depends on an unlisted fact." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "action": "exclude_unfixed_defect", + "proposed_reference": 24961.339844, + "reason": "Exclude from scoring, recorded as reason_code reference_depends_on_unlisted_input; it is not an engine defect.\n\nThe record: frozen_value 24961.339844 (2.15.17 + latest_conventions, reproduced). alternative_value 24391.796875, which equals the board, v11_1755, the alt module, and the undo-#9122-federal run.\n\nEngine-free recompute: AGI 250,733.35 (248,675 - 530.47 + 2,588.82). Taxable income is 191,533.35 after the $32,200 standard deduction, the $2,000 170(p) charity deduction and the $25,000 225(b) overtime deduction (MAGI is under $300,000). Tax is 31,561.34, less the $6,600 CTC, for 24,961.34. CDCC is 0 because the head has no earned income (21(d)(1)(B)). Without the refund the result is 24,391.80.\n\nThe engine probe confirms the household does not itemize under either reading: itemized deductions are 33,840.04 or 33,757.20, against 34,200 of standard plus 170(p). AR state moves only 0.48 under the alternative, below tolerance, so it stays scored.\n\nThe ambiguity runs both ways here. The 'unlisted status = false' default gives the board value. Carrying the 2026 facts into 2025 implies the household itemized in 2025 and got a full tax benefit, which gives the 2.15.17 value. The prompt resolves neither, so neither value can be scored." + } + ], + "what_i_checked": "Harness, run myself on policyengine-us 2.15.17 via sweep_latest.py. Outputs are in this dir.\n- conv_full.csv: full 1,984-output sweep with fixes/latest_conventions.py. It is 29 moved and identical to out/latest_conventions.csv to 0.0.\n- alt_full.csv: full sweep with fixes/latest_alt_salt_refund_no_prior_benefit.py. Against conventions, exactly 7 outputs move by more than $1: 005 fed -1327.26, 005 CA -47.14, 033 -29.16, 078 fed -391.66, 082 fed -14.85, 117 fed -569.54 and 120 fed -394.85. Two move by less than $1: 117 AR -0.48 and 078 MD -0.85. Only 8 scenarios list salt_refund_income (005, 033, 074, 078, 082, 088, 117, 120). 074 LA and 088 TX have zero federal tax under every reading.\n- undo_fed8.csv: my own rv_undo9122_federal.py, which is conventions minus the #9122 federal entry. 033 and 117 federal equal the board bit-for-bit. The module had to be idempotent because CountryTaxBenefitSystem applies a reform twice at init (system.py).\n- rv_probe.json: an engine probe of AGI, deductions, itemization and SALT for 033, 117 and 078 under both readings.\n- rv_recompute.py/.json: an engine-free recompute matching both readings to the cent.\n\nUpstream (read-only git): 316e7832a1 (2026-09-08), 5581f5be1a (2026-09-09), 56d7361c31 (2026-09-14) and merge e990b4a6f8 (2026-09-16) are all ancestors of 79be99f671 and absent from 06665727d8. The 316e diff adds salt_refund_income to sources.yaml and documents the input as the taxable line 1 amount. The label is unchanged. At 06665727d8 only ID, SC and WV (and UT via salt_refund_last_year) used the refund, confirming the old inconsistency. The installed 2.15.17 sources.yaml and variable match. policyengine-us-data puf.py:375 has salt_refund_income = puf.E00700.\n\nPrimary sources, fetched this session:\n- 26 USC 111(a), 151(d)(5)(C), 225(b), 170(p) and 63(b) from uscode.house.gov, with amendment and effective-date notes: 111 was last amended in 1986; the 170 amendments by 70424 and 70425 apply to tax years after 2025.\n- 2025 i1040gi: PDF CreationDate 2026-02-25, with the Schedule 1 line 1 'None of your refund is taxable' text and the 2025 MFJ standard deduction of $31,500.\n- Pub. 525 (2025): CreationDate 2026-02-25, 'Deductions not itemized' and Example 29.\n- Rev. Proc. 2025-32: CreationDate 2025-10-17. Checked the 2026 MFJ brackets, the $32,200 standard deduction, the $1,650 aged amount and the $2,200 CTC.\n- Rev. Rul. 2019-11, IRB 2019-17 (2019-04-22): Situations 1 and 2.\n- SD DOR (search results only; direct page fetches returned 404): no state income tax, and the senior sales and property tax refund program with its income limits.\nAll of these predate 2026-07-03.\n\nPrompt and precedent: I read the 033 and 117 prompts, the label map at policybench/prompts.py:130, and the board's reference_exclusions.json, including the r16/r24 unlisted-input precedents and the r02 records for 005, 082 and 120.\n\nModel answers: tabulated from predictions.csv.gz in model_readings_rv.txt.\n\nCross-cluster: I read the md_county_tax_8888 review prompt and found the conflict over 078 federal." + } + }, + { + "id": "snap_abawd_hours_default_9261", + "title": "SNAP work rules read weekly_hours_worked_before_lsr, default now 0 (scored 066, excluded 112 and 056)", + "outputs": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "state": "VA", + "board": 3576, + "latest": 0, + "v11_1755": 3596.039795, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "hold_output_scope" + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "state": "TX", + "board": 287.683167, + "latest": 0, + "v11_1755": 287.683167, + "excluded": true, + "exclusion_root_cause": "unlisted:weekly_hours_worked_before_lsr", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "state": "NJ", + "board": 1140, + "latest": 95, + "v11_1755": 1140, + "excluded": true, + "exclusion_root_cause": "unlisted:weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "snap_abawd_hours_default_9261", + "classification": "mixed", + "summary": "All three outputs move for one reason. Upstream 82745ca239 (PR #9261, merged a112cc5a0a on 2026-08-12) changed the default of weekly_hours_worked_before_lsr from 40 to 0, and that is the only input the SNAP ABAWD and general work tests read. Restoring the hours in 2.15.17 plus latest_conventions gives back the board's own readings: 066 = 3,576, 056 = 1,140, and 112 = 288. For 112 the board's 287.68 differs only by the minimum-allotment rounding fix cf165464a6.\n\nFor 066 (scored), the prompt states 40 usual weekly hours. The builder passes that only as hours_worked_last_week, so the engine sees 0 hours and returns $0. That is wrong in law: 40 hours meets the 20-hour test of 7 U.S.C. 2015(o)(2) and exempts the person under 2015(d)(2)(E) and (o)(3)(D). I wrote fixes/latest_map_stated_hours.py, which is latest_conventions plus a builder mapping that copies stated usual hours into weekly_hours_worked_before_lsr. Swept over all 1,984 outputs (out/latest_map_stated_hours.csv), it changes exactly one output: 066 snap, from 0 to 3,576. That equals the board and my engine-independent calculation (12 \u00d7 $298 on the FY2026 schedule). Patch invariants hold for all 177 people in 100 scenarios.\n\n112 and 056 should stay excluded. Their hours are unlisted, so the reading is a prompt ambiguity, and no engine version resolves it. At 0 hours the legal value also depends on facts the prompt does not give: prior countable months under the 3-in-36 rule, and for 056 the county. Both exclusion records need new values on 2.15.17:\n- 112: frozen 0, alternative 288. If the frozen value is set to 0 and the alternative stays at 0.0, load_reference_exclusions rejects the record as 'not excludable'.\n- 056: frozen 95, alternative 1,908 (40 hours and mortgage counted). The $95 is January only: 2.15.17 models the NJ 20-county waiver through 2026-01-31, and a household with no county falls back to Atlantic County. So the 0.0 computed on 1.755.4 is out of date.\n\nfixes/latest_alt_unlisted_hours_40.py, swept over all outputs (out/latest_alt_unlisted_hours_40.csv), moves only these two, which confirms no scored output depends on unlisted hours. The only moves come from engine and builder defaults, not new law. Every figure used is pre-freeze law, so no hold module was needed.", + "upstream_changes": [ + { + "commit": "82745ca239 (merged a112cc5a0a, PR #9261, closes #9254)", + "date": "2026-08-10 (merged 2026-08-12)", + "what": "weekly_hours_worked_before_lsr default_value 40 -> 0 (variables/household/income/person/weekly_hours_worked.py). This variable is the input the SNAP ABAWD 20-hour and general 30-hour tests read. It moves all three outputs: 066 3576->0, 112 287.68->0, 056 1140->95. It is a modeling default, not law." + }, + { + "commit": "d325c7a9af (merged 7193611d84, PR #9030)", + "date": "2026-08-07", + "what": "Adds parameters/gov/usda/snap/work_requirements/abawd/waived_counties/nj.yaml: 20 NJ counties (all except Morris) waived 2025-02-01 to 2026-01-31, then empty. This produces 056's January-only $95 at 0 hours." + }, + { + "commit": "74b0a75e5f (PR #8872) and adea8827e5 (merged 51af6c14e5, PR #9216)", + "date": "2026-08-04", + "what": "Area-waiver machinery: is_in_snap_abawd_waived_area checks the statewide list plus county_str matching, and a household with no county falls back to the first county alphabetically. 056 therefore resolves to ATLANTIC_COUNTY_NJ (waived) and 066 to ACCOMACK_COUNTY_VA (not waived)." + }, + { + "commit": "cf165464a6 (merged 3c41c31457, PR #9162)", + "date": "2026-07-23 (merged 2026-07-28)", + "what": "snap_min_allotment rounded to the nearest dollar. Under the 40-hour reading, 112 becomes 12 x $24 = $288 instead of 1.755.4's unrounded $287.68. It is not the cause of 112's move to 0." + }, + { + "commit": "48a10d4d43 (merged 7fc69a35b6, PR #8961)", + "date": "2026-07-08 (merged 2026-07-27)", + "what": "Non-exempt work registrants are assumed compliant, so at 0 hours only the ABAWD test fails, not the general work rule. This is why the default change hits only these three outputs. The Sept 22 r14_v2 sandbox backported the same rule." + } + ], + "law": "All sources are pre-freeze and were read this session. (1) 7 U.S.C. 2015(o)(2): a person who does not work 20 or more hours a week (averaged monthly) is limited to 3 months of benefits in 36. (o)(3)(D) exempts anyone exempt under (d)(2), and (d)(2)(E) exempts anyone employed at least 30 hours a week. Text is from uscode.house.gov prelim, as amended by Pub. L. 119-21 \u00a710102 (enacted 2025-07-04). All three people (ages 40, 31 and 20) are inside both the pre- and post-OBBBA age ranges. (2) 7 CFR 273.24(a)(1)(i) (20 hours a week averaged monthly = 80 hours a month), 273.24(b)(1) (countable months; months covered by a waiver do not count) and 273.7(b)(1)(vii) (30-hour exemption). Text is eCFR as of 2026-07-01; the versioner shows the last amendment on 2025-01-17. (3) USDA FNS 'SNAP \u2013 Fiscal Year 2026 Cost-of-Living Adjustments', digitally signed 2025-08-14 (Wayback copy of fns.usda.gov snap-cola-fy26memo.pdf): 48 States/DC one-person maximum $298, standard deduction $209, excess shelter cap $744, minimum $24. (4) FNS NJ ABAWD waiver response dated 2024-12-16: 20 counties (every county except Morris), 2025-02-01 to 2026-01-31. FNS NJ response dated 2026-02-10: Cape May County and Camden City only, from 2026-02-01. The NJ DFD 'Federal Changes' page confirms both. 2.15.17 omits the FY2026 Cape May and Camden City waiver; that does not affect these outputs. The USDA 2026-02-26 reinstatement memo returned 403 and had no Wayback capture. (5) FNA BBCE page, updated 2026-06-29: VA no asset limit, 200%; TX $5,000 asset limit (one vehicle up to $22,000 excluded), 165%; NJ no asset limit, 185%. (6) HHS 2026 poverty guideline for 1 person: $15,960 (ASPE). The Oct\u2013Dec 2026 FY2026 hold is the existing c_snap_hold_fy2026 convention. Engine-independent arithmetic is in hand_calc.py in the scratch folder: 066 3,576; 112 at 20+ hours 288; 056 95/month (1,140 a year), or 159/month (1,908 a year) with the mortgage counted.", + "per_output": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "board": 3576, + "latest": 0, + "proposed_reference": 3576, + "action": "adopt_latest", + "reason": "Adopt 2.15.17's value with the input mapping, not the unmapped 0. The prompt states 'usual weekly hours worked: 40'. The builder sends that only to hours_worked_last_week, which no SNAP rule reads, so after #9261 the ABAWD test sees 0 hours and fails every month. That result is wrong in law: 40 hours meets 7 U.S.C. 2015(o)(2) and is exempt under (d)(2)(E)/(o)(3)(D). The fix belongs in the builder. With fixes/latest_map_stated_hours.py (conventions plus stated usual hours copied to weekly_hours_worked_before_lsr), 2.15.17 returns 3,576, and that is the only output of 1,984 the module changes. 3,576 equals the board and the hand calculation: gross $43.33 less the 20% earned-income deduction and the $209 standard deduction gives $0 net, so 12 x $298 under the FY2026 schedule. VA BBCE has no asset limit, so the $14,000 in the bank does not matter. If the lead declines the mapping, the fallback is exclude_unfixed_defect with corrected value 3,576." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "board": 287.683167, + "latest": 0, + "proposed_reference": 0, + "action": "keep_excluded", + "reason": "The exclusion is for an unlisted input (weekly_hours_worked_before_lsr). It is a prompt ambiguity, not an engine defect. The 'treat unlisted numbers as 0' rule conflicts with the $5,923 in listed wages. Even at 0 hours, the law gives $0 to $72 depending on prior countable months, which the prompt does not state; the engine's documented shortcut ends benefits immediately. #9261 only swapped which reading the engine takes: the 2.15.17 reference is now the zero-hours reading (0, TX has no waiver), and the old alternative is now the reference. Refresh the record to frozen_value 0 and alternative_value 288 (the 40-hour or 20+-hour reading: net $1,269, so the 1-person minimum $24 x 12; TX BBCE at 165% of FPL with a $5,000 asset limit is met). Also flip the alternative_reading text. Leaving the alternative at 0.0 with the frozen value at 0 fails load_reference_exclusions ('not excludable'). The swept source is out/latest_alt_unlisted_hours_40.csv." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "board": 1140, + "latest": 95, + "proposed_reference": 95, + "action": "keep_excluded", + "reason": "Keep it excluded: both unlisted inputs are still unlisted (hours, and whether the mortgage is on the residence). The 2.15.17 value of 95 is the zero-hours reading. It pays only January 2026, through the FNS NJ 20-county waiver that expired 2026-01-31, and only because a household with no county falls back to Atlantic County (Morris County would give 0). Under the law it also depends on prior countable months (3-in-36). The record's alternative_value of 0.0, computed on 1.755.4, is out of date. Refresh to frozen_value 95 and alternative_value 1,908: 40 hours with the mortgage counted is 12 x $159 (net $461, contribution $139). The one-input alternatives are 1,140 (40 hours only, equal to the old board) and 159 (mortgage only, the sibling latest_alt_snap_mortgage_residence result). The engine and the hand calculation agree on every one of these values." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "066's 3,576 is not a raw 2.15.17 value. It exists only with the stated-hours mapping; latest_conventions gives 0. Labelling the action adopt_latest invites adopting 0. The adoption holds only if the mapping (hours_worked_last_week copied to weekly_hours_worked_before_lsr) ships in the builder that generates the regenerated references (policybench Scenario.to_pe_household or ground_truth) and is recorded as a revision in the sidecar. Keeping it only in triage/sweep/fixes/latest_map_stated_hours.py is not enough.", + "066's prompt contradicts itself. It says 'usual weekly hours worked: 40' and that work facts are constant all year, yet wages are $520 a year at $32/h, about 16 hours a year. I agree the stated hours govern the 7 U.S.C. 2015(o)(2) test, because the prompt states them and states that work facts are constant. In predictions.csv the models that reasoned about hours used 40: fable-5, fable-5.1, gpt-5.5, gpt-5.6-sol, gpt-6-astra, kimi-k3 and ox-alpha all answered 3,576, and fable-5.1 wrote 'Work hours (40/week) satisfy ABAWD rules'. All 10 zero answers cite assets or income; none cites hours. Record this as a residual risk, not a reason to exclude.", + "The investigator's 056 figures leave the source of the $95 unstated. It is New Jersey's state-funded minimum benefit, not a formula result: the federal amount is 298 - 244 = $54, and 2.15.17's snap_min_allotment reads 95 from gov.states.nj.snap.amount. The law section never cites NJ law, and hand_calc.py cites only the engine parameter ('board value'). Primary sources: the NJ DHS SNAP eligibility page (live 2026-09-29: 'New Jersey has a minimum monthly SNAP benefit amount of $95') and the Governor's release of 2023-02-08 (Wayback capture 2023-12-30). Both predate the freeze, and the parameter is unchanged between 06665727d8 and 79be99f671. The refreshed 056 record should cite them, because 95, 1,140 and 1,908 all rest on NJ's minimum or the federal formula floor.", + "112 at 0 hours: the frozen 0 comes from the engine's documented steady-state shortcut (meets_snap_abawd_work_requirements docstring), not from the law. Under 7 U.S.C. 2015(o)(2) and 7 CFR 273.24(b), with no prior receipt listed, the person has 3 countable months: 3 x $24 = $72. The stated $5,923 in wages also imply at most 15.7 hours a week at the $7.25 federal minimum wage, so the 40-hour alternative is itself in tension with the stated facts. Both points strengthen keep_excluded. The new alternative_reading text should not present 0 as the law's answer under the zero-hours reading.", + "112 income: 2.15.17's SNAP unearned list omits farm_rent_income ($1,920). 7 CFR 273.9(b)(2)(ii) counts rental income net of costs as unearned (as earned under (b)(1)(ii) when the owner manages the property 20+ hours a week). So 'net $1,269' is the engine's figure; the legal net is about $1,429. The benefit is the $24 one-person minimum either way (273.10(e)(2)(ii)(C)), so 288 stands. This is a wording fix only.", + "056 at 0 hours also turns on an unlisted county: the engine falls back to ATLANTIC_COUNTY_NJ, which was waived in January, while Morris would give 0. It also turns on 3-in-36 history; the law gives up to about 4 x $95 = $380 if January is waived and 3 countable months are unused. And housing_cost counts the $11,949 property tax as a shelter cost with no check on whether it is the occupied home, the same question the record raises for the mortgage. None of this changes 95, 1,140, 159 or 1,908, but the record should name the county and prior-months dependence.", + "The citation of the HHS 2026 poverty guideline ($15,960) is not the FY2026 SNAP standard. FY2026 limits come from the FNS memo table ($1,305 at 100%, $1,696 at 130%, $2,152 at 165% for one person). It does not matter here: every gross test passes with a wide margin (112: $1,737 including farm rent vs $2,152).", + "Required record edits, confirmed against policybench/reference_exclusions.py: both records need engine_version 'policyengine-us 2.15.17', the new frozen_value and alternative_value, and flipped alternative_reading text. Otherwise load_reference_exclusions rejects 112 (0 == 0, 'not excludable'), and verify_exclusions_against_reference rejects both (recorded 287.68 and 1,140 vs regenerated 0 and 95). The sibling excl_snap_mortgage_residence settlement defers 056's combined alternative to this cluster ('95 to 159; a different cluster whose recorded alternative also combines r14'), so 1,908 does not conflict with it." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "action": "adopt_latest", + "proposed_reference": 3576, + "reason": "Adopt 2.15.17 plus the stated-hours builder mapping: 3,576 = 12 x $298. The prompt states 40 usual weekly hours. That meets 7 U.S.C. 2015(o)(2)(A) and 7 CFR 273.24(a)(1)(i) (80 hours a month) and exempts the person under 2015(d)(2)(E), (o)(3)(D) and 7 CFR 273.7(b)(1)(vii) and 273.24(c)(5). All were verified from uscode.house.gov and from eCFR as of 2026-07-01 (last amended 2025-01-17). Net income is 0: 43.33 - 8.67 - 209 < 0. VA BBCE has no asset limit (FNA chart updated 2026-06-29), and the FY2026 figures come from the FNS memo signed 2025-08-14. My sweep shows the mapping moves only this output of 1,984, and forcing 40 hours on every person moves only this output and the two excluded ones, so no other output depends on #9261. Condition: the mapping must land in the reference-generating builder and be recorded in the sidecar. If it does not, use exclude_unfixed_defect with corrected value 3,576, naming the builder defect: a prompt-visible 'usual weekly hours' fed only to hours_worked_last_week, which no SNAP rule reads." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 0, + "reason": "Hours are unlisted, and at 0 hours the law also depends on unlisted 3-in-36 history, so no engine version resolves this. Refresh the record: frozen_value 0 (the 2.15.17 plus conventions reference), alternative_value 288, engine_version 2.15.17, and flip the alternative_reading text. At 20 or more hours, TX BBCE (165%; $5,000 assets with one vehicle up to $22,000 excluded) is met, and the net income of about $1,269 (engine) or $1,429 (farm rent counted) leaves only the $24 one-person minimum: 12 x 24 = 288. The text should say the 0 comes from the engine's no-history shortcut (the law gives up to $72), and that the stated wages imply at most 15.7 hours a week at the federal minimum wage." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 95, + "reason": "Two inputs are still unlisted: hours, and whether the mortgage (and the property tax) is on the occupied home. At 0 hours the value also depends on the county and on 3-in-36 history. Refresh the record: frozen_value 95, alternative_value 1,908, engine_version 2.15.17, flipped alternative_reading. The 95 is January 2026 only, from the FNS-approved 20-county waiver (letter 2024-12-16, through 2026-01-31; NJ DFD confirms) applied to the engine's Atlantic County fallback. The amount is the NJ state minimum $95 (law signed 2023-02-08; federal formula $54). The alternative of 40 hours with the mortgage counted gives shelter capped at $744, net $461, contribution $139, so 12 x 159 = 1,908. I reproduced it with my own composition (latest_alt_snap_mortgage_residence reform plus the latest_alt_unlisted_hours_40 patch) and by hand. The one-input alternatives are 1,140 (40 hours only) and 159 (mortgage only)." + } + ], + "what_i_checked": "Harness, all 1,984 outputs, my own runs in this folder: no-fix matches out/pepy612_us21517.csv exactly (41 moved). latest_conventions (29 moved), latest_map_stated_hours (28) and latest_alt_unlisted_hours_40 (26) each match the investigator's CSVs to 0.0. conventions -> map changes only 066 snap (0 -> 3,576). map -> alt40 changes only 056 (95 -> 1,140) and 112 (0 -> 288). My extra sweep rv_all40.py sets 40 hours for every person, undoing #9261: it changes exactly these 3 outputs versus conventions, and 0 outputs versus alt40, so stated hours below 20/30 matter nowhere. Monthly traces (trace_*.out) show the ABAWD test is the only failing rule; the county fallbacks are ACCOMACK_COUNTY_VA, ANDERSON_COUNTY_TX and ATLANTIC_COUNTY_NJ (waived January only); and snap_min_allotment is 95 in NJ. combo_alt40_mortgage.py gives 1,908 for 056 and combo_alt0_mortgage.py gives 159. Code: read git show 82745ca239 (default 40 -> 0; merged a112cc5a0a on 2026-08-12; in 79be99f671, not in 06665727d8), and checked the ancestry and dates of d325c7a9af/7193611d84, 74b0a75e5f/adea8827e5/51af6c14e5, cf165464a6/3c41c31457 and 48a10d4d43/7fc69a35b6. Read the 2.15.17 SNAP work-rule, snap_min_allotment and housing_cost variables and the NJ minimum parameters; git grep lists every reader of the hours variables. Prompt: rendered make_no_tools_prompt for all three scenarios; the hours label and the 'unlisted numeric = 0' rule have been in prompts.py since 2026-05-01, before the 2026-06-12 run. Read every model's snap answer and explanation for all three. Law, fetched this session: eCFR 7 CFR 273.24 and 273.7 as of 2026-07-01 plus the version list (last amended 2025-01-17); 7 U.S.C. 2015 from uscode.house.gov (OBBBA amendments); a Wayback copy of the FNS FY2026 COLA memo, byte-identical to the investigator's (sha256 67ecdba8), signed 2025-08-14; the NJ DHS $95 minimum page and the Governor's release of 2023-02-08. Read the investigator's copies of the NJ FNS waiver letter (2024-12-16), the NJ DFD Federal Changes page, the FNA BBCE chart (updated 2026-06-29) and 7 CFR 273.10(e)(2)(ii)(C) and 273.9(b). Checked policybench/reference_exclusions.py validation, and read the sibling mortgage-residence settlement for 056. Independent arithmetic is in hand_calc_rv.py/.out: 066 3,576; 112 288 with or without farm rent; 056 95, 1,140, 159 and 1,908." + } + }, + { + "id": "nj_ctc_fy2027_budget", + "title": "New Jersey child tax credit raised 25% for TY2026-2028 (scored)", + "outputs": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "state": "NJ", + "board": 5342.399902, + "latest": 5842.399902, + "v11_1755": 5342.399902, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "adopt_latest" + } + ], + "investigation": { + "cluster_id": "nj_ctc_fy2027_budget", + "classification": "upstream_fix_of_pre_freeze_law", + "summary": "scenario_008 (NJ, married filing jointly, six dependents aged 17/14/12/10/5/1, NJ taxable income $18,915) moves on state_refundable_credits from 5,342.40 (board and raw 1.755.4) to 5,842.40 on 2.15.17 with the conventions. Decomposing nj_refundable_credits in both venvs shows that only nj_ctc moves, from 2,000 to 2,500. nj_eitc stays at 3,292.40 (0.40 x federal max $8,231; AGI 30,632 is below the $31,160 MFJ phase-out start), and nj_property_tax_credit stays at 50. The nj_refundable_credits adds list and the nj_ctc/nj_ctc_eligible formulas are byte-identical between the versions. The only change is the ctc/amount.yaml brackets from upstream 0319635b6e (2026-07-08), merged as PR #8971 on 2026-07-09. A scratch attribution reform puts the 2026-2028 CTC schedule back to $1,000/$800/$600/$400/$200 on top of latest_conventions. In the full 1,984-output sweep it changes exactly one output, this one, back to the board's 5,342.40, and leaves the other 28 moved outputs unchanged. The law is P.L.2026, c.26 (S-4531). NJ Legislature records say it was introduced 2026-06-26, reported 2026-06-28, passed both houses and approved on 2026-06-30. The chaptered text ends 'Approved June 30, 2026', and the Governor's release posted 06/30/2026 lists S-4531 as signed. All of that is before the 2026-07-03 freeze. The enacted text sets $1,250 per child under 6 at taxable income of $30,000 or less for tax years 2026-2028, effective for tax years beginning on or after 2026-01-01. Worked by hand: 2 children x $1,250 = $2,500, plus 3,292.40 plus 50, gives 5,842.40. So adopt 2.15.17's value. No hold module is needed.", + "upstream_changes": [ + { + "commit": "0319635b6e", + "date": "2026-07-08", + "what": "'Update New Jersey for the enacted FY2027 budget': gov.states.nj.tax.income.credits.ctc.amount brackets set to 1,250/1,000/750/500/250 from 2026-01-01, reverting to 1,000/800/600/400/200 on 2029-01-01, citing P.L.2026 c.26 (S-4531). This is the only change that moves the output: nj_ctc goes from 2,000 to 2,500 for scenario_008." + }, + { + "commit": "115c5a8e36", + "date": "2026-07-09", + "what": "Review follow-up. The CTC reference title now reads 'P.L.2026, c.26 (S-4531), enacted 2026-06-30' and the host is harmonized to pub.njleg.state.nj.us. It adds CTC tests for 2026 and the 2029 revert. The CTC values do not change." + }, + { + "commit": "b011a7ef48", + "date": "2026-07-09", + "what": "Merge of PR #8971 (DTrim99/nj-fy2027-budget) into upstream main, which carries both commits above into 2.15.17 (79be99f671)." + } + ], + "law": "P.L.2026, c.26 (Senate No. 4531), amending P.L.2022, c.24 (C.54A:4-17.1). In taxable years 2026, 2027 and 2028 the credit per child under age six is $1,250 at taxable income of $30,000 or under, then $1,000/$750/$500/$250 through $80,000. Other years stay at $1,000/$800/$600/$400/$200. Section 2: 'take effect immediately and apply to taxable years beginning on and after January 1, 2026.' Chaptered text: pub.njleg.state.nj.us/Bills/2026/PL26/26_.HTM and .PDF, ending 'Approved June 30, 2026.' The bill-format copy AL26/26_.PDF is headed 'P.L. 2026, CHAPTER 26, approved June 30, 2026'. NJ Legislature bill history (njleg.state.nj.us/api/billDetail/billHistory/S4531/2026): introduced 6/26/2026; reported from Senate Budget and Appropriations 6/28/2026 with no amendment action; passed Senate 39-1, substituted for A5329 and passed Assembly 79-0-0 on 6/30/2026; 'Approved P.L.2026, c.26.' 6/30/2026. The introduced text (4531_I1.HTM, dated June 26, 2026) and the committee statement dated June 28, 2026 (4531_S1.PDF; server last-modified 2026-06-29) state the same amounts as the chaptered law. The Governor's release 'Governor Sherrill Takes Action on Legislation' (nj.gov/governor/news/2026/20260630e.shtml, posted 06/30/2026) lists S-4531/A-5329 as signed. Enactment and publication both fall before the 2026-07-03 freeze. One caveat: the chaptered PL26 files on the legislature site carry a server last-modified date of 2026-09-17. The bill text itself was public from introduction on 6/26, and it passed unamended. Copies of every source are saved in triage/latest/nj_ctc_fy2027_budget/.", + "per_output": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "board": 5342.399902, + "latest": 5842.399902, + "proposed_reference": 5842.4, + "action": "adopt_latest", + "reason": "The whole +500 comes from nj_ctc (2,000 to 2,500), and a scratch revert of the 2026-2028 CTC schedule shows it is the only moved output. P.L.2026 c.26 was approved 2026-06-30, before the freeze, and sets $1,250 per child under 6 at NJ taxable income of $30,000 or under for TY2026. By hand: NJ taxable income 30,915 - 12,000 exemptions = 18,915; MFJ; 2 children under 6 (ages 5 and 1) x $1,250 = $2,500. Add NJ EITC 3,292.40 (0.40 x 8,231) and the property tax credit of 50 to get 5,842.40. 1.755.4 and the board predate the encoding." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "Minor, not outcome-changing: the investigator says the bill text 'was public from introduction on 6/26'. The server does not show that. Fetched today, pub.njleg.state.nj.us/Bills/2026/S5000/4531_I1.HTM has Last-Modified 2026-07-01 14:27 GMT. The pre-freeze publication evidence is instead: the committee statement 4531_S1.PDF (dated June 28, 2026, Last-Modified 2026-06-29 15:17 GMT, same $1,250/$1,000/$750/$500/$250 schedule); the NJ Legislature bill history ('Approved P.L.2026, c.26.' 6/30/2026); and the Governor's release 'Posted on 06/30/2026'. All of these, and the 7/1 date too, fall before 2026-07-03.", + "Minor omission: the investigator gave the post-freeze server date only for PL26 (2026-09-17). The AL26/26_.PDF bill-format copy also carries a post-freeze Last-Modified (2026-08-13). Neither matters, because enactment on 6/30 and the 6/29 committee statement are pre-freeze.", + "Could not corroborate with the Wayback Machine. CDX and availability queries for the Governor's release, 4531_I1.HTM and 4531_S1.PDF returned no captures up to 2026-07-10, or timed out. Dating therefore rests on official records (bill history, 'Approved June 30, 2026' in the chaptered text, the Governor's release) and server Last-Modified headers, which I judge sufficient." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 5842.4, + "reason": "Confirmed. Re-running sweep_latest.py with fixes/latest_conventions.py on 2.15.17 gives 5,842.40 against a board value of 5,342.40, and it is the only moved output for scenario_008. Decomposition in both venvs shows nj_ctc going from 2,000 to 2,500 while nj_eitc (3,292.40), nj_property_tax_credit (50), nj_cdcc (0) and nj_taxable_income (18,915) are identical. nj_refundable_credits adds [property_tax_credit, eitc, cdcc, ctc], and nj_anchor (450) is not in it. The cause is commit 0319635b6e (2026-07-08, ctc/amount.yaml 2026-01-01: 1,250/1,000/750/500/250, reverting in 2029). It is an ancestor of 79be99f671 and not of 06665727d8, and was merged via b011a7ef48 (PR #8971, 2026-07-09); 115c5a8e36 changes only the CTC reference. My own attribution reform takes a different route: an nj_ctc override that multiplies the per-child amount by 0.8 for 2026-2028, which reproduces the pre-c.26 schedule exactly. On the full 1,984-output sweep it differs from latest_conventions in exactly this one output (back to 5,342.40), and 28 moved outputs remain. My sweep CSVs match the investigator's two CSVs row for row. Law: P.L.2026 c.26 (S-4531) sets $1,250 per child under 6 at NJ taxable income of $30,000 or under for TY2026-2028, effective for tax years beginning on or after 2026-01-01. It was approved 6/30/2026, per the bill history API, the chaptered text and the Governor's release posted 06/30/2026, and the committee statement dated 6/28 was served by 6/29. All of that is before the 2026-07-03 freeze, so this is adopt, not hold. Independent calculation: NJ gross income 26,800 + 4,000 + 115 = 30,915, less exemptions 12,000 (2x1,000 + 1,000 blind/disabled + 6x1,500) = 18,915. Two children under 6 (ages 5 and 1) x $1,250 = $2,500. NJ EITC is 0.40 x 8,231 = 3,292.40: Rev. Proc. 2025-32 (IRS, Last-Modified 2025-10-17) gives the 3+ child maximum of $8,231 and the MFJ threshold of $31,160, and AGI 30,632.41 and earned income 30,517.41 are both below the threshold. Adding the property tax credit of 50 gives 5,842.40." + } + ], + "what_i_checked": "I read LATEST_BRIEF.md and the investigator's saved sources and scripts. All scratch work is in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/nj_ctc_fy2027_budget/review/; I modified no existing file.\n\n(1) Harness. I ran sweep_latest.py with fixes/latest_conventions.py on 2.15.17 for scenario_008 (review/rerun_s008_conventions.csv). Only state_refundable_credits moves (5,342.40 to 5,842.40). I also ran the full 1,984-output sweep (review/review_latest_conventions_all.csv): 29 moved. latest_conventions has no NJ component.\n\n(2) Decomposition. review/decompose_review.py, run under .venv-pe1755 (1.755.4) and .venv-pepy612-us21517 (2.15.17), shows that only nj_ctc changes (2,000 to 2,500). The CTC brackets read 1,000/800/\u2026 in 1.755.4 and 1,250/1,000/750/500/250 in 2.15.17. nj_ctc.py, nj_ctc_eligible.py and nj_refundable_credits.py are identical between the venvs (diff -rq). The NJ files that do differ (EITC investment income test, 529 deduction, exemption metadata, Stay NJ, ANCHOR, CCAP) leave every intermediate for this scenario unchanged.\n\n(3) Independent attribution. review/review_attr_nj_ctc_x08.py overrides the nj_ctc variable with 0.8x the per-child amount for 2026-2028. Full sweep (review/review_attr_nj_ctc_x08_all.csv): 28 moved, and it differs from latest_conventions only on scenario_008 state_refundable_credits (back to 5,342.40). Both of my CSVs match the investigator's verify/attribution CSVs exactly.\n\n(4) Upstream. I ran git log 06665727d8..79be99f671 on the NJ credits paths and read 0319635b6e (the ctc/amount.yaml diff and its commit message citing S-4531) and 115c5a8e36 (reference-only CTC change). merge-base confirms 0319635b6e is in 2.15.17 and not in 1.755.4; b011a7ef48 is the 2026-07-09 merge of PR #8971.\n\n(5) Primary sources, fetched fresh:\n- NJ Legislature API billHistory/S4531/2026: introduced 6/26, reported 6/28, Senate 39-1, Assembly 79-0-0, 'Approved P.L.2026, c.26.' all dated as the investigator states.\n- 4531_I1.HTM and 4531_S1.PDF are SHA-256 identical to the investigator's copies. Last-Modified is 2026-07-01 and 2026-06-29. The committee statement is dated June 28, 2026, reports the bill favorably without amendments, and gives the same amounts.\n- The chaptered PL26 HTM/PDF read 'Approved June 30, 2026'; the AL26 PDF header reads 'P.L. 2026, CHAPTER 26, approved June 30, 2026'. Section 2 makes the act effective for tax years beginning on or after 1/1/2026.\n- The Governor's release nj.gov/governor/news/2026/20260630e.shtml is 'Posted on 06/30/2026' and lists S-4531/A-5329 among the bills 'signed \u2026 into law'.\n- Wayback returned no pre-freeze captures.\n\n(6) Independent calculation. From the scenario facts and Rev. Proc. 2025-32 (fetched from irs.gov; 2026 EITC table: 3+ child maximum $8,231, MFJ threshold $31,160, investment income limit $12,200) I get 5,842.40. scenario_008 state_refundable_credits is not in reference_exclusions.json; the only scenario_008 exclusion is payroll_tax (r05), and no board revision touches this output.\n\nNo hold module is needed. The un-exclusion checks do not apply." + } + }, + { + "id": "ny_ctc_phaseout_rounding_9425", + "title": "NY Empire State child credit phase-out increment (scored)", + "outputs": [ + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "state": "NY", + "board": 650.5, + "latest": 667, + "v11_1755": 650.5, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "adopt_latest" + } + ], + "investigation": { + "cluster_id": "ny_ctc_phaseout_rounding_9425", + "classification": "upstream_fix_of_pre_freeze_law", + "summary": "scenario_082 files as head of household in tax year 2026 with one child aged 1. The move from 650.50 to 667.00 is entirely the NY Empire State child credit (ny_ctc), which goes from 290.50 to 307.00. ny_cdcc is 360 in both engines and every other NY refundable credit is 0. The threshold ($75,000 for head of household) and the base credit ($1,000) are the same in both engines. The only thing that differs is how the phase-out counts increments. 1.755.4 rounds up with (excess + 999)//1000, giving 43 x 16.50 = 709.50. 2.15.17, after d8f06f394c (#9425), rounds down with excess//1000, giving 42 x 16.50 = 693. Tax Law 606(c-1)(1-a)(C), enacted by Chapter 59 of 2025 (signed 2025-05-09), reduces the credit by $16.50 \"for each one thousand dollars by which\" federal AGI exceeds the threshold. Unlike IRC 24(b)(1), it has no \"or fraction thereof\", so only whole $1,000s count. The 2025 Form IT-213 line 6 (federal AGI rounded down to the nearest $1,000) and the IT-213-I instructions (created 2025-10-27) apply the same round-down. I computed federal AGI by hand from the scenario inputs and it matches both engines exactly: 117,585.15 in 1.755.4 and 117,652.65 in 2.15.17. The 67.50 difference comes from #9122 (316e7832a1), which now includes the SALT refund in federal AGI. Neither that change nor the still-excluded IRA-deduction defect (r02) changes the answer: every AGI reading gives an excess between 42,585 and 42,661, which is 42 whole thousands, so the credit is 307 and state_refundable_credits is 667. A 2.15.17 counterfactual sweep (conventions plus the old round-up) moves exactly one output out of 1,984, this one, back to 650.5. The board value was therefore a 1.755.4 defect against pre-freeze law, and 2.15.17's 667 should be adopted. No hold module is needed. The same scenario's federal_income_tax_before_refundable_credits move (+14.85) is a separate cluster (SALT refund and the excluded r02) and is outside this task. Scratch files are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/ny_ctc_phaseout_rounding_9425/: decompose.py, recompute.py, counterfactual_ceil_rounding.py, cf_ceil_rounding.csv, s3009.txt, it213i.pdf/.txt and form_it213_fill_in.pdf/form_it213.txt.", + "upstream_changes": [ + { + "commit": "d8f06f394c", + "date": "2026-09-10", + "what": "#9425: ny_ctc_post_2024_phase_out now counts whole $1,000 increments by rounding down (excess_income // increment) instead of rounding up ((excess + increment - 1) // increment). It also moves the SURVIVING_SPOUSE threshold from 110,000 to 75,000, which does not apply here because the filer is head of household. This change alone causes the +16.50 on scenario_082 state_refundable_credits (ny_ctc 290.50 -> 307.00). A counterfactual sweep restoring the round-up moves only this output back to 650.5." + }, + { + "commit": "316e7832a1", + "date": "2026-09-08", + "what": "#9122 (merged via e990b4a6f8 on 2026-09-16): includes state tax refund income in federal gross income and excludes it in conforming states. Federal AGI rises by 67.50 (117,585.15 -> 117,652.65) and NY AGI is unchanged. The ESCC does not change, because the excess over $75,000 is 42 whole thousands either way." + } + ], + "law": "NY Tax Law 606(c-1)(1-a) was added by S.3009-C/A.3009-C Part C section 2 (Chapter 59 of the Laws of 2025). The NY Assembly bill status shows it passed 2025-05-08 and was signed as chap.59 on 2025-05-09. I read the enacted text from nyassembly.gov. (B)(i) gives $1,000 per qualifying child aged three or younger for tax years 2026-2027. (C) says the credit \"shall be reduced (but not below zero) by sixteen dollars and fifty cents for each one thousand dollars by which the taxpayer's federal adjusted gross income exceeds the threshold amount\", and sets the threshold at $75,000 for single, head of household or qualified surviving spouse. It has no \"or fraction thereof\" clause (compare IRC 24(b)(1)), so partial thousands do not count. The NY Tax Department's 2025 Instructions for Form IT-213 (tax.ny.gov it213i.pdf, PDF CreationDate 2025-10-27, Last-Modified 2025-10-27) say \"Line 6: Round down the amount from Form IT-201, line 19 to the nearest $1,000\". The 2025 Form IT-213 (it213_fill_in.pdf, barcode 213001250094, server Last-Modified 2026-06-16) has line 6 as federal AGI rounded down to the nearest $1,000, line 7 as the excess over $75,000 for head of household, and line 8 as line 7 x 1.65%. All three sources were published before the 2026-07-03 freeze.", + "per_output": [ + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "board": 650.5, + "latest": 667, + "proposed_reference": 667, + "action": "adopt_latest", + "reason": "Independent recompute: 2026 ESCC base is $1,000 (one child aged 1, (1-a)(B)(i)). Federal AGI is 117,585.15 to 117,661.13 across every reading of the SALT refund (#9122) and the excluded IRA deduction (r02), so the excess over the $75,000 head-of-household threshold is 42 whole thousands. The reduction is 42 x $16.50 = $693, giving ESCC $307. Adding ny_cdcc of $360 (unchanged in both engines) gives $667. 1.755.4's round-up (43 increments, ESCC 290.50) contradicts the whole-thousands wording of Tax Law 606(c-1)(1-a)(C) (Ch. 59 of 2025, signed 2025-05-09) and IT-213 line 6. 2.15.17 fixes this in d8f06f394c, which implements pre-freeze law." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "Minor labeling only, and it does not change the verdict: 316e7832a1's title says '(#9122)', but #9122 is the issue. The change reached main as PR #9422 (merge e990b4a6f8, 2026-09-16), and I confirmed 316e7832a1 is an ancestor of that PR's branch head fc4a2d706c. Calling it '#9122' in the upstream_changes entry is ambiguous; 'issue #9122 / PR #9422' would be exact.", + "Also minor, not relied on: the investigator's scratch tax606.txt/tax606.html is a Cloudflare 'Just a moment...' challenge page from nysenate.gov, not statute text. The legal claim instead rests on the nyassembly.gov S.3009-C text, which I re-fetched independently and which does contain the quoted (1-a)(C) language. So this does not undermine the finding." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 667, + "reason": "Confirmed independently. I re-fetched the S.3009-C bill text and actions from nyassembly.gov: passed both houses 05/08/2025 and signed as chap.59 on 05/09/2025. Part C section 2 adds Tax Law 606(c-1)(1-a). (B)(i) gives $1,000 per qualifying child aged three or younger for tax years 2026-2027. (C) reduces the credit by $16.50 'for each one thousand dollars by which' federal AGI exceeds $75,000 (head of household), with no 'or fraction thereof'. Part C section 3 says the act takes effect immediately. The NY Tax Department reads it the same way. The 2025 IT-213-I (PDF CreationDate and Last-Modified 2025-10-27; my download is sha1-identical to the investigator's) has line 6 'Round down the amount from Form IT-201, line 19 to the nearest $1,000'. The 2025 IT-201-I (2025-11-20) Note 4 confirms that line 19 is federal AGI. The 2025 IT-213 fill-in form (Last-Modified 2026-06-16, sha1-identical) has line 8 = line 7 x 1.65%. All of these predate 2026-07-03. The 2025 IT-201-I page 26 that 1.755.4 cited only points to IT-213 and contains no round-up. Engine side: I diffed the installed sources. The only relevant difference is ny_ctc_post_2024_phase_out, where (excess+increment-1)//increment became excess//increment (d8f06f394c, 2026-09-10, #9425; it comes after 06665727d8 and is an ancestor of 79be99f671). The only parameter change is SURVIVING_SPOUSE 110k->75k, which does not apply to this head-of-household filer. My decomposition in both venvs: the base is $1,000 in both, the phase-out is 709.50 in 1.755.4 and 693 in 2.15.17, and ny_cdcc is 360 in both. Federal AGI is 117,585.15 in 1.755.4 and 117,652.65 in 2.15.17. I recomputed everything without the engine: federal AGI is 117,585.15 to 117,661.13 under every SALT-refund and IRA (r02) reading, which is 42 whole thousands over the threshold, so ESCC = 1000 - 693 = 307. NY CDCC = 3,000 x 0.20 x 0.600 = 360; the 0.600 factor for NYAGI 65k-150k comes from the 2025 IT-216-I (2025-10-22). Total 667. Harness with latest_conventions: 667. My full conventions sweep vs the investigator's round-up counterfactual sweep: exactly 1 of 1,984 outputs differs (this one, 667 vs 650.5), and my counterfactual CSV matches theirs to 0.0. The output is not in reference_exclusions.json; only scenario_082 federal_income_tax_before_refundable_credits is excluded (r02). No hold module is needed." + } + ], + "what_i_checked": "What I read first: LATEST_BRIEF.md, all of the investigator's scratch files, and fixes/latest_conventions.py.\n\nHarness runs: sweep_latest.py with latest_conventions on scenario_082 moves 2 outputs, state_refundable_credits 650.5->667 and the separate-cluster federal tax +14.85. I then ran full 1,984-output sweeps for (a) latest_conventions and (b) the investigator's counterfactual_ceil_rounding.py. They differ only in scenario_082 state_refundable_credits (667 vs 650.5), and (b) reproduces cf_ceil_rounding.csv exactly.\n\nEngine code: I diffed the installed NY CTC variables and parameters between the 1.755.4 and 2.15.17 venvs. The only relevant change is the increment rounding in ny_ctc_post_2024_phase_out, plus the threshold for surviving spouses, which this filer is not. d7fc17abfe touches ny_ctc_pre_2024 only, which does not apply in 2026.\n\nGit (read-only):\n- d8f06f394c: 2026-09-10, #9425, lies inside 06665727d8..79be99f671, and its diff matches the claim.\n- 316e7832a1: 2026-09-08, reached main through PR #9422 (merge e990b4a6f8, 2026-09-16).\n\nDecomposition: I wrote review/rev_decompose.py and ran it in both venvs. It shows NY refundable credits = ny_ctc + ny_cdcc (360) with every other credit 0, federal AGI and NY AGI as claimed, and the base credit $1,000 in both.\n\nPrimary sources, re-fetched myself:\n- nyassembly.gov S03009 bill text and actions, signed chap.59 on 05/09/2025. I read the (1-a)(A)-(F) text and Part C section 3 (effective immediately) myself.\n- tax.ny.gov it213i.pdf (Last-Modified and CreationDate 2025-10-27) and it213_fill_in.pdf (Last-Modified 2026-06-16, barcode 213001250094), both sha1-identical to the investigator's copies.\n- it201i.pdf (2025-11-20): line 63 defers to IT-213, and Note 4 says line 19 is federal AGI.\n- it216i.pdf (2025-10-22): 0.600 NY factor for NYAGI 65k-150k and the .20 federal decimal.\n\nIndependent recompute (review/rev_recompute.py): all four AGI readings (SALT refund in or out, IRA deducted or not) fall in 42 whole thousands over $75,000. Rounding down gives ESCC 307 and a total of 667. The round-up gives 650.5, and a continuous 1.65% reading would give about 657, but the statute and IT-213 support neither.\n\nOther checks:\n- The rendered prompt for scenario_082 lists 'employer sponsored insurance premiums: $21,208'. The engine documents that input as employer-paid, and IRC 106 excludes such premiums from income, so there is no AGI effect. This is a scenario-wide reading question outside this cluster, and it would not flip the rounding verdict.\n- reference_exclusions.json has only scenario_082's federal tax output (r02), not this output.\n\nScratch outputs are in triage/latest/ny_ctc_phaseout_rounding_9425/review/: rev_conv_082.csv, rev_conv_full.csv, rev_cf_ceil_full.csv, the scripts, and the fetched PDFs and texts. I modified no existing file." + } + }, + { + "id": "school_meals_child_support_income", + "title": "Child support now counted in school-meal income (scored)", + "outputs": [ + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "state": "PA", + "board": 1, + "latest": 0, + "v11_1755": 1, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "adopt_latest" + } + ], + "investigation": { + "cluster_id": "school_meals_child_support_income", + "classification": "upstream_fix_of_pre_freeze_law", + "summary": "One output moves: scenario_028 (PA) reduced_price_school_meals_eligible, from 1 on the board to 0 on 2.15.17. The cause is upstream commit fe1cb9ab45 (2026-08-14), the only commit in 06665727d8..79be99f671 that adds child_support_received to gov.usda.school_meals.income.sources (confirmed with git log -S). With that change, school_meal_countable_income rises from 60,010 on 1.755.4 to 61,277 on 2.15.17 with the conventions. The FPG ratio goes from 1.818 to 1.857 against spm_unit_fpg 33,000, and school_meal_tier goes from 1 (reduced) to 2 (paid). I also ran a diagnostic reform: latest_conventions with child_support_received removed from the sources list, swept over all 1,984 outputs. It differs from the latest_conventions sweep in exactly this one output (back to 1), so nothing else in 2.15.17 moves it. The law is pre-freeze. 7 CFR 245.6(a)(5)(ii) names \"support payments\" as a source of household income, and eCFR shows the section's last substantive amendment on 2016-12-22. The FNS income eligibility notices published 2025-03-13 (SY2025-26) and 2026-04-09 (SY2026-27) define income to include \"(11) alimony or child support payments\". The prompt states child support received of $1,267, so this is a listed fact. The independent recompute gives 0 under either school year: income is 60,000 + 10 + 1,267 = 61,277. That exceeds the SY2026-27 reduced-price limit (1.85 x 33,000 = 61,050) and the SY2025-26 limit (1.85 x 32,150, rounded up to 59,478). The action is to adopt 2.15.17's value of 0; no hold module is needed.", + "upstream_changes": [ + { + "commit": "fe1cb9ab45", + "date": "2026-08-14", + "what": "Adds child_support_received to gov/usda/school_meals/income/sources.yaml (part of the CSFP partner-issue fix, #9278-#9280), with a test that school_meal_countable_income counts child support received. This is the sole cause of the move." + }, + { + "commit": "0b558fd335", + "date": "2026-09-03", + "what": "Changes only the citations in sources.yaml to 7 CFR 245.6(a)(5)(ii), along with the general_assistance comment (general_assistance was added in 1de01086ae on 2026-09-02). It has no effect on scenario_028, which has no general assistance." + }, + { + "commit": "f0282be843 / 14596edddb", + "date": "2026-08-06", + "what": "Add receives_snap and receives_tanf to school-meal categorical eligibility. These are not relevant here: snap and tanf are 0 and not reported as received, and free_school_meals_eligible stays 0." + } + ], + "law": "7 CFR 245.6(a)(5)(ii) (eCFR text as of 2026-06-30; section last substantively amended 2016-12-22 per the eCFR versions API) and 7 CFR 245.2 (\"Current income\" and \"Documentation\") both list \"support payments\" among the sources of household income for free and reduced-price meals. USDA FNS \"Child Nutrition Programs: Income Eligibility Guidelines\" (FR Doc 2025-03821, published 2025-03-13, SY2025-26; FR Doc 2026-06842, published 2026-04-09, effective July 1, 2026 to June 30, 2027) defines income to include \"(11) alimony or child support payments\". It sets the reduced-price limit as 1.85 times the HHS poverty guideline, rounded up to the next whole dollar. The HHS 2026 poverty guideline for a household of 4 is $33,000 (FR Doc 2026-00755, published 2026-01-15); the 2025 guideline is $32,150 (FR Doc 2025-01377, published 2025-01-17). All of these were published before the 2026-07-03 freeze. Saved copies are in the scratch folder: ecfr_245_6_20260630.xml, ecfr_245_2_20260630.xml, and the fr_*.txt files.", + "per_output": [ + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "board": 1, + "latest": 0, + "proposed_reference": 0, + "action": "adopt_latest", + "reason": "Child support received ($1,267, stated in the prompt) counts as income under 7 CFR 245.6(a)(5)(ii) and FNS IEG item (11), both pre-freeze. Countable income is 61,277, above the SY2026-27 reduced-price limit of 61,050 (1.85 x $33,000) and the SY2025-26 limit of 59,478, so the household falls in the paid tier. The board's value of 1 came from 1.755.4 leaving child support out of the income sources; fe1cb9ab45 fixed that. A diagnostic counterfactual sweep confirms this is the sole cause." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "Minor, and it does not change the verdict: the list of upstream changes leaves out two commits in 06665727d8..79be99f671 that touch school-meal parameter files. f5ee39ff10 (2026-08-14) edits only the reference metadata in sources.yaml: it relabels the 273.9(b) citation as CFR and adds 7 CFR 247.9(d)(1). d7fc17abfe (2026-09-02) adds propagate_metadata_to_children to state_universal_free_meals.yaml. Neither changes any value. My own diagnostic restored every 1.755.4 school-meal list at once: sources without child_support_received and general_assistance, and categorical eligibility without receives_snap and receives_tanf. It still moves only this one output.", + "Minor imprecision: the finding says 7 CFR 245.2 'Current income' lists support payments. It does not. 'Current income' is defined as 'income, as defined in \u00a7 245.6(a)', and only the 245.2 'Documentation' definition lists 'support payments' explicitly. The legal conclusion is unaffected.", + "Minor: fe1cb9ab45 justifies its own change with the CSFP rule, 7 CFR 247.9(d) (gross income, child support not excluded), not the school-meal rule. The school-meal basis (7 CFR 245.6(a)(5)(ii) and FNS IEG item (11)) comes from the investigator's research and from the later 0b558fd335 citation. I verified that basis independently from primary sources.", + "Minor: in the Federal Register text of the SY2026-27 IEG (FR 2026-06842), the table is an embedded image. So the $61,050 reduced-price limit for a household of 4 is computed from the notice's stated rule (the 2026 guideline x 1.85, rounded up), not read from the published table. The notice's '2.6 percent increase' for a family of four matches $32,150 rising to $33,000. The SY2025-26 table (FR 2025-03821) is text and shows 59,478 for 4, as the investigator stated." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "action": "adopt_latest", + "proposed_reference": 0, + "reason": "Reproduced on 2.15.17 with latest_conventions: school_meal_countable_income is 61,277, spm_unit_fpg is 33,000, the ratio is 1.8569, the tier is PAID and reduced_price_school_meals is 0. On 1.755.4 the income is 60,010, the tier is REDUCED and the value is 3,009.82. A full 1,984-output sweep with latest_conventions plus the 1.755.4 school-meal lists differs from the latest_conventions sweep only in this output. The investigator's drop-child-support diagnostic gives the same result when I re-run it. The law is pre-freeze. eCFR shows 7 CFR 245.6 as of 2026-06-30, with 'support payments' in (a)(5)(ii) and 245.2 'Current income' pointing to 245.6(a). Its last substantive version is 2016-12-22. The FNS IEG notices were published 2025-03-13 and 2026-04-09, and item (11) of each is 'alimony or child support payments'. The HHS guidelines were published 2025-01-17 ($32,150) and 2026-01-15 ($33,000). The independent recompute is 60,000 + 10 + 1,267 = 61,277. That is above the reduced-price limit in both school years (61,050 for SY2026-27, 59,478 for SY2025-26), so the value is 0. It stays 0 if the wage is annualized from the stated $29/hr x 40 hours as well." + } + ], + "what_i_checked": "I read LATEST_BRIEF.md, the investigator's scratch files, sweep_latest.py, benchmark_specs.json and the 2.15.17 school-meal code. The output maps to reduced_price_school_meals, where a positive value counts as 1. That value is the REDUCED tier times net subsidy. The tier compares school_meal_countable_income / spm_unit_fpg with 1.30 and 1.85, and PA is false in state_universal_free_meals.\n\nI diffed the school-meal variables and parameters between the 1.755.4 and 2.15.17 venvs. The only value changes are the two added income sources and the two added categorical-eligibility flags. With read-only git, I listed every commit in 06665727d8..79be99f671 that touches school_meals. Running git log -S child_support_received on sources.yaml returns only fe1cb9ab45 (2026-08-14). I read fe1cb9ab45, f5ee39ff10, 0b558fd335, 1de01086ae, 14596edddb, f0282be843 and d7fc17abfe.\n\nI re-ran the harness three times:\n- A full latest_conventions sweep, which matches the existing out/latest_conventions.csv exactly, with 29 moved including scenario_028.\n- The investigator's diag_drop_child_support.py.\n- My own diagnostic that restores all 1.755.4 school-meal lists.\n\nBoth diagnostics differ from latest_conventions only in scenario_028 reduced_price_school_meals_eligible. A trace of the chain on both engines through the harness situation builder confirms the investigator's numbers.\n\nI checked the board's revisions and reference_exclusions.json: none touches this output, so it was never excluded, and no input is left unlisted. The relevant facts (wages, interest, child support, household of 4, PA, no SNAP/TANF) are all stated.\n\nFrom the publishers, I fetched fresh copies of the Federal Register API metadata and full texts for FR 2025-03821, 2026-06842, 2026-00755 and 2025-01377. Publication dates, the income definition item (11), the rounding rule and the guideline tables all match the investigator's copies byte for byte. I also fetched eCFR 7 CFR 245.2 and 245.6 as of 2026-06-30 (again byte-identical) and the part 245 versions API. Everything cited was published before 2026-07-03. I wrote an independent recompute.\n\nScratch files are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/school_meals_child_support_income/review/: rev_latest_conventions_full.csv, rev_diag_revert_school_meal_params.py/.csv, rev_diag_drop_child_support_rerun.csv, rev_trace.py, rev_trace_21517_conventions.txt, rev_trace_1755.txt, rev_recompute.py/.txt, and the fresh fr_*/ecfr_* copies. I modified no existing file." + } + }, + { + "id": "az_snap_bbce_200", + "title": "Arizona SNAP BBCE gross limit 200% FPL from March 2026 (scored)", + "outputs": [ + { + "scenario_id": "scenario_013", + "variable": "snap", + "state": "AZ", + "board": 0, + "latest": 240, + "v11_1755": 0, + "excluded": false, + "exclusion_root_cause": "", + "final_action": "adopt_latest" + } + ], + "investigation": { + "cluster_id": "az_snap_bbce_200", + "classification": "upstream_fix_of_pre_freeze_law", + "summary": "scenario_013 is a single 80-year-old disabled Arizonan with $58,700 in the bank and SNAP gross income of $2,539.33 a month ((23,736 + 6,720 + 16) / 12). Under 185% FPL (limit $2,412.71) the household fails Arizona's expanded categorical eligibility (ECE). Under regular rules it also fails the net-income test (net about $2,138 against a $1,304 limit) and the $4,500 asset test, so 1.755.4 and the board give 0. At 200% FPL (limit $2,608.33, then $2,660 from October) it is ECE from March, which counts as meeting the resource, gross and net tests. Its computed benefit is negative (30% of $2,138 is $642, above the $298 maximum), so it gets the 1-person minimum: $24 x 10 months = $240. Raw 2.15.17 gives 243 because the FY2027 memo (2026-08-21, after the freeze) adds $1 in Oct-Dec; c_snap_hold_fy2026 already removes that. Arizona published the change before the freeze: DES CNAP \"What's Changed on 03/23/2026\" (effective benefit month 03/2026), then USDA's BBCE chart, page updated June 29, 2026 (Wayback capture 2026-06-30). The CNAP ECE rule (FAA5.I01B) has one gross standard for every budgetary unit; elderly/disabled units only get \"special considerations\" once they meet it. USDA lists Arizona as \"All households are eligible\". So upstream is right to raise gross_hheod too. An attribution sweep on top of latest_conventions, setting AZ gross and gross_hheod back to 185% for 2026, moves exactly one of the 1,984 outputs: scenario_013 snap, 240 to 0. An engine-independent recomputation (latest/az_snap_bbce_200/recompute.py) gives 240.", + "upstream_changes": [ + { + "commit": "caccdcd30b", + "date": "2026-08-05", + "what": "Added state BBCE poverty-guideline vintages (fpg_year_start_month; AZ stays on the October cycle) and first dated AZ's general BBCE gross standard 185% -> 200%, from 2026-04-13, inferred from a CNAP 'What's Changed on 04/13/2026' entry. The elderly/disabled standard stayed at 185%, so this alone would not move scenario_013." + }, + { + "commit": "0881eff4b8", + "date": "2026-08-06", + "what": "Moved AZ gross.yaml's 200% to 2026-03-01, quoting the CNAP manual ('Starting the benefit month of 03/2026 ... 185% ... to 200%'), and added 2026-03-01: 2 to gross_hheod.yaml (elderly/disabled). This is the change that moves scenario_013, which is an elderly/disabled household: tanf_non_cash_gross_income_limit goes from 2,412.71 to 2,608.33 from March." + }, + { + "commit": "6e996f17c8", + "date": "2026-08-11", + "what": "AZ: comments and metadata only (SNAP Screener corroboration of 200%). No value change." + }, + { + "commit": "d27d6c3cc3", + "date": "2026-09-27", + "what": "Encoded USDA's FY2027 SNAP COLA (#9623), including min_allotment.published_adjustment CONTIGUOUS_US = 1 from 2026-10-01 (the $25 FY2027 minimum). This gives raw 2.15.17 its 243 (7x24 + 3x25). It is post-freeze, and c_snap_hold_fy2026 in latest_conventions already holds it back to $24." + } + ], + "law": "(1) Arizona DES, Cash and Nutrition Assistance Policy (CNAP) Manual, \"What's Changed on 03/23/2026\" (PDF CreationDate 2026-03-24; Wayback capture 20260417195741 of dbmefaapolicy.azdes.gov/Archived_Policy/baggage/2026-03-23_What'sChanged.pdf; sha256 f8173335...). It says, under \"Change: NA Expanded Categorical Eligibility\", \"EFFECTIVE DATE: For the benefit month of 03/2026 and ongoing ... The gross income limit for the NA ECE has changed from 185% of the FPL to 200% of the FPL.\" It also says a 02/23/2026-announced change to 130% \"has been recalled\", and it revises FAA5.I01B, FAA5.I01A, FAA6.A03J, FAA3.D06D.01 and FAA6.J02B.04 (\"Updated as of 03/23/2026; Effective 03/01/2026\"). (2) The archived CNAP FAA6.J02B.04 \"NA Expanded Categorical Eligibility Standard (185% FPL)\" (Wayback 20260417210829; read by OCR) shows 1-person $2,413, marked \"Effective 10/01/25 to 02/28/26\". (3) CNAP FAA5.I01B \"NA Categorical Eligibility\", the archived pre-update version (Wayback 20260417203432; PDF created 2026-02-17; read by OCR). ECE exists when gross income does not exceed the percentage of FPL, with no separate standard for elderly or disabled units: those units \"receive special considerations\" (SEC) when they meet the ECE requirements. AZTECS treats ECE units as meeting \"Resources\" and \"Gross and net income tests\". \"NA categorically eligible budgetary units are eligible for at least the minimum NA allotment\", except units of three or more. (4) USDA FNA, \"Broad-Based Categorical Eligibility (BBCE)\" page and chart, \"Page updated: June 29, 2026\" (chart PDF BBCE-States-Chart-June2026.pdf, CreationDate 2026-06-29). The Wayback capture at 2026-06-30 22:08 UTC, before the freeze, shows \"Arizona | All households are eligible (referral on application) | No limit on assets | 200%\". The 2026-06-01 capture, page updated Dec 29, 2025, showed 185%. (5) USDA FY2026 SNAP COLA (Aug 2025): 1-person maximum $298, minimum $24, standard deduction $209, elderly/disabled asset limit $4,500. c_snap_hold_fy2026 holds these for Oct-Dec 2026, because the FY2027 memo of 2026-08-21 is post-freeze. Both Arizona publications (2026-03-23/24 and 2026-06-29) predate the 2026-07-03 freeze. The live dbmefaapolicy.azdes.gov and des.az.gov still serve a Cloudflare challenge, which I did not bypass, so the current page text is unverified. The evidence files are saved under triage/latest/az_snap_bbce_200/ (cnap/, BBCE-States-Chart-June2026.pdf, wb_fna_20260630220846.html, wb_fns_20260601100556.html).", + "per_output": [ + { + "scenario_id": "scenario_013", + "variable": "snap", + "board": 0, + "latest": 240, + "proposed_reference": 240, + "action": "adopt_latest", + "reason": "Arizona raised its expanded categorical eligibility (ECE) gross limit to 200% FPL from benefit month 03/2026. DES published it 2026-03-23 and USDA's BBCE chart 2026-06-29, both pre-freeze. The limit applies to all households, including elderly/disabled ones. Gross income of $2,539.33 fails 185% ($2,412.71) in Jan-Feb and passes 200% ($2,608.33, then $2,660 from Oct) in Mar-Dec. ECE waives the net-income test and the $58,700 asset balance. The computed benefit is negative (30% x ~$2,138 = $642 > $298), so the 1-person minimum of $24 (FY2026, held Oct-Dec by c_snap_hold_fy2026) applies: 10 x $24 = $240. 1.755.4 and the board kept 185% all year and were wrong. The attribution sweep (AZ back to 185%) returns this output to 0, and it is the only one of 1,984 that moves. An engine-independent recompute gives 240. Raw 2.15.17's 243 includes the post-freeze FY2027 $25 minimum, which the existing convention removes." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "Non-blocking: the upstream_changes list omits e8326c59ed (2026-08-06, 'Return the exact BBCE gross income standard'), which replaced caccdcd30b's published-chart rounding with the exact percentage x guideline. It is immaterial here: the household has $63.00/month of headroom under 200% (limit 2,608.33 vs gross 2,539.33) and $114.67 from October (2,660), so the exact $2,608.33, the rounded-up $2,609 and CNAP's published $2,610 all give the same result.", + "Non-blocking: the summary gives the regular-rules net limit as '$1,304'. 100% of the 2025 guideline is $1,304.17/month and is published as a whole dollar. Net income of $2,138 fails either way, and the $58,700 bank balance fails the $4,500 elderly/disabled asset test on its own, so Jan-Feb stay at 0.", + "Observation, not a reason to hold: all 42 model predictions on the board for scenario_013 snap are 0 (several cite a 185% Arizona BBCE limit). Adopting 240 makes every model miss this output. That follows from the publication-date rule, since Arizona published the change on 2026-03-23/24 and USDA on 2026-06-29, both before the 2026-07-03 freeze. The lead should know about it anyway.", + "Settlement gap, now closed: the settlement did not look for Arizona legislation that could have capped the ECE limit after March. The House FY2027 human-services BRB, HB4147, would have added A.R.S. 46-233 ('may not establish a gross income limit of more than one hundred eighty-five percent of the federal poverty level' for BBCE). Governor Hobbs vetoed it on 2026-05-05 (azleg API). The enacted BRB, HB4162 (Laws 2026 ch. 134, signed and filed 2026-06-13), has no gross-limit cap. Neither do chapters 126, 128, 138, 140 or 254. No other 2026 bill whose title or NOW title mentions SNAP, nutrition, DES or human services was chaptered. So no pre-freeze law returns Oct-Dec to 185%, and the settlement stands." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_013", + "variable": "snap", + "action": "adopt_latest", + "proposed_reference": 240, + "reason": "Arizona DES CNAP 'What's Changed on 03/23/2026' (Wayback 20260417195741; sha256 f8173335..., matching the investigator's copy; PDF CreationDate 2026-03-24) raised the NA Expanded Categorical Eligibility gross limit from 185% to 200% FPL 'for the benefit month of 03/2026 and ongoing', and recalled a 02/23/2026 cut to 130%. CNAP FAA5.I01B has one ECE gross standard. Elderly/disabled units get special considerations (SEC) only when they meet ECE, and ECE units are treated as meeting resources and the gross and net tests. The USDA FNA BBCE chart (live PDF sha256 3ebb8ab0... matches; CreationDate 2026-06-29; page capture 2026-06-30 22:08 UTC) lists Arizona as 'All households are eligible', no asset limit, 200%. It has no separate elderly/disabled row, whereas Arkansas has one. All of this was published before the 2026-07-03 freeze, and no enacted 2026 Arizona law overrides it (HB4147's 185% cap was vetoed 2026-05-05). The household has stated gross income of $2,539.33/month (30,472/12). That fails 185% ($2,412.71) in Jan-Feb, and the $58,700 bank balance rules out regular eligibility, so Jan-Feb pay 0. It passes 200% in Mar-Sep ($2,608.33) and Oct-Dec ($2,660; it would also pass on the FY2026 guideline). Net income is about $2,138 (at least about $2,096 even with every possible medical deduction), so 30% exceeds the $298 maximum. Under 7 CFR 273.10(e)(2)(ii)(C) an eligible 1-person household gets the minimum, round(8% x $298) = $24, held for Oct-Dec by c_snap_hold_fy2026 (raw 2.15.17 gives $25 there, from the post-freeze FY2027 memo). Total 10 x $24 = $240. The prompt states every input the result depends on and says not to infer unlisted expenses or rent. The result is also robust to other readings: a benefit above $24 needs net income under about $913, which would take more than about $2,230/month of unstated shelter costs (the prompt lists a $43,000 mortgage balance but no payment). The engine's imputed medical deduction ($192.73) doesn't matter either." + } + ], + "what_i_checked": "Harness (2.15.17 venv, sweep_latest.py) for scenario_013. With fixes/latest_conventions.py, snap = 240 (board 0, v11 0) and none of the other 15 outputs move (conv_s013.csv). With no fix, 243 (raw_s013.csv). My month-by-month trace (rv_trace.py -> trace_conv.log) shows tanf_non_cash_gross_income_limit at 2,412.71 in Jan-Feb, 2,608.33 in Mar-Sep and 2,660 in Oct-Dec. It also shows is_tanf_non_cash_hheod = 1, BBCE eligibility from March, net income 2,138, expected contribution 642, snap_min_allotment 24 in all months under the hold, and snap 0,0 then 24 x 10 = 240. My own attribution module (rv_attr_az_185.py: AZ gross and gross_hheod at 1.85 for 2026 on top of latest_conventions), swept over all 1,984 outputs, differs from out/latest_conventions.csv in exactly one output, scenario_013 snap 240 -> 0. The other 1,983 are identical. A second attribution holding only gross_hheod at 185% (rv_attr_hheod_only.py) also returns scenario_013 snap to 0, so upstream's reading that 200% applies to the elderly/disabled standard is load-bearing. I checked that reading against CNAP FAA5.I01B (OCR of the archived pre-update version) and the USDA chart. Upstream (read-only): 0881eff4b8 (AZ gross.yaml and gross_hheod.yaml 2026-03-01: 2), caccdcd30b, e8326c59ed, 6e996f17c8 and d27d6c3cc3 (min_allotment.published_adjustment CONTIGUOUS_US +1 from 2026-10-01). I read the 2.15.17 engine code for tanf_non_cash_gross_income_limit, tanf_non_cash_fpg, meets_tanf_non_cash_* (AZ net_applies false, asset_limit inf, fpg_year_start_month 10), is_snap_eligible, meets_snap_categorical_eligibility, snap_normal_allotment and snap_min_allotment. Primary sources fetched this session: the Wayback DES What's Changed 03/23/2026 PDF (pdfinfo and text; hash matches the investigator's file), the live USDA BBCE-States-Chart-June2026.pdf (hash matches; Arizona row read), the Wayback FNA page 20260630220846 ('Page updated: June 29, 2026'; AZ 200%), the investigator's saved 2026-06-01 capture (Dec 29, 2025 revision, AZ 185%) and 2026-07-13/09-13 captures (still 200%), the eCFR 7 CFR 273.10(e)(2)(ii)(C) minimum-benefit text, and the azleg.gov API and chaptered texts (HB4147 vetoed 2026-05-05; HB4162 = ch. 134 signed 2026-06-13, no BBCE cap; the 2026 session's bill index screened for SNAP/DES/human-services bills; ch. 126/128/138/140/254 have no BBCE clause). An engine-independent recompute (rv_recompute.py) gives 240, with $126.62/month over 185% and $63.00 headroom under 200%, so capital-gain treatment and CNAP's whole-dollar rounding can't change the result. I rebuilt the prompt the models saw (prompt_s013.txt). It lists every input used and says not to infer expenses, and all 42 models answered 0. I did not bypass the Cloudflare challenge on dbmefaapolicy.azdes.gov, so the live CNAP text after April 2026 is unverified, but USDA's June 29 chart and the enacted-law screen cover the period to the freeze." + } + }, + { + "id": "excl_r02_ira_219g_federal", + "title": "Excluded r02_ira_219g federal outputs moved by #9122", + "outputs": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "state": "CA", + "board": 106505.898438, + "latest": 107833.15625, + "v11_1755": 106505.898438, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "state": "NY", + "board": 9563.052734, + "latest": 9577.902344, + "v11_1755": 9563.052734, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "state": "CT", + "board": 40021.816406, + "latest": 40416.667969, + "v11_1755": 40021.816406, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r02_ira_219g_federal", + "classification": "excluded_defect_still_present", + "summary": "All three outputs stay excluded. The r02 defect is still present in policyengine-us 2.15.17. Its gov/irs/ald/deductions.yaml 2026 list still names traditional_ira_contributions as the deduction. No 219(g) or active-participant logic exists anywhere in the package. No commit in 06665727d8..79be99f671 touches that list or the IRA deduction, and gh search finds no upstream issue or PR for it. The engine still deducts the full contribution: $2,163.84 for 005 (joint MAGI $543,076), $8.48 for 082 (HoH MAGI $117,661) and $28.85 for 120 (single MAGI $234,414). All three filers are 401(k) participants well above the Notice 2025-67 ranges, so 219(g) allows no deduction for any of them. The only thing that moves these outputs is PR #9422 (issue #9122), which adds salt_refund_income to federal gross income. With salt_refund_income set to 0, 2.15.17 reproduces the board exactly (106505.8984375, 9563.052734375, 40021.81640625). With the ported r02_ira_219g_v2 and the refund set to 0, it reproduces the Sept 22 alternative values exactly. The alternative values therefore need recomputing only if #9122's counting of the listed refund in AGI is adopted; the sibling cluster salt_refund_gross_income_9122 owns that ruling under IRC 111 (the prior-year tax benefit is not stated). If it is adopted, the 2.15.17 alternatives are 108525.578125, 9579.767578 and 40423.175781. If not, the Sept 22 values stand. An engine-free recomputation (indep.py, using Rev. Proc. 2025-32 and Notice 2025-67) matches all 12 engine values (board, latest, Sept 22 alternative and new alternative for each scenario) to within $0.01. One caveat for scenario_120: $27.30 of its $394.85 move comes from the engine's SALT proxy, not from CT tax. ct_withheld_income_tax applies CT rates to federal AGI, which now includes the refund, so the federal SALT deduction rises $113.77. CT's own tax is unchanged because CT subtracts the refund. The #9122 cluster should weigh this. Files: fixes/latest_alt_r02_ira_219g.py (an alternative-value module, not a hold); sweep out/latest_alt_r02_ira_219g.csv (35 moved; relative to latest_conventions, r02 changes all 11 r02-excluded outputs plus scenario_082 state_income_tax by $0.59, which is under the $1 tolerance); scratch in triage/latest/excl_r02_ira_219g_federal/ (probe.py, probe_21517.json, probe_1755.json, indep.py, indep.json, sweep_alt_r02.log, and the source PDFs and pages).", + "upstream_changes": [ + { + "commit": "316e7832a1", + "date": "2026-09-08", + "what": "Include state tax refund income in federal gross income and exclude it in conforming states (#9122): adds salt_refund_income to gov.irs.gross_income.sources, so AGI rises by exactly the listed refund (005 +4,147.68, 082 +67.50, 120 +1,750.24)." + }, + { + "commit": "5581f5be1a", + "date": "2026-09-09", + "what": "Standardize state refund inputs on salt_refund_income (PR #9422 follow-up)." + }, + { + "commit": "5d3d1312fd", + "date": "2026-09-14", + "what": "Review feedback on PR #9422: state conformity subtractions (CT among them: ct subtractions.yaml line 38 salt_refund_income), so CT tax is unchanged while federal AGI rises." + }, + { + "commit": "e990b4a6f8", + "date": "2026-09-16", + "what": "Merge of PR #9422 (closes #9122) into main; in 2.15.17 (79be99f671)." + }, + { + "commit": "none", + "date": "2026-07-02..2026-09-28", + "what": "No commit in 06665727d8..79be99f671 touches policyengine_us/parameters/gov/irs/ald/deductions.yaml or adds any 219(g) or active-participant logic (git log -S '219(g)' and -S active_participant return nothing; the retirement variables changed only in 44001a4c24, an uprating change). r02 is unfixed." + } + ], + "law": "26 U.S.C. 219(g)(1)-(3), (5), (8) was read at uscode.house.gov (prelim, current through Pub. L. 119-111, 2026-09-18). Section 219 was last amended by Pub. L. 117-328 on 2022-12-29, before the freeze. Under 219(g)(2), when AGI exceeds the applicable amount by the full $10,000 ($20,000 joint) width, the IRA dollar limit falls to zero. IRS Notice 2025-67 (PDF last-modified 2025-11-13, before the freeze) sets the 2026 active-participant phase-out ranges: $81,000-$91,000 single or HoH and $129,000-$149,000 joint. It also sets the 402(g) limit at $24,500 and the 219(b)(5)(A) limit at $7,500. For #9122: 26 U.S.C. 111(a) (1954 code, read at uscode.house.gov) excludes a recovered deduction only to the extent it did not reduce prior-year tax. Whether the listed refund is taxable is left to the salt_refund_gross_income_9122 cluster. The engine-free recomputation used Rev. Proc. 2025-32 (IRS PDF last-modified 2025-10-17, before the freeze): 2026 Tables 1-3, the 0%/15% breakpoints, the standard deduction, AMT figures and CTC $2,200. It also used 26 U.S.C. 170(b)(1)(I) (0.5% floor) and 170(p) (non-itemizer $1,000/$2,000). All of these were published before 2026-07-03.", + "per_output": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "board": 106505.898438, + "latest": 107833.15625, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "The r02 defect is unfixed in 2.15.17: it still deducts $2,163.84 of IRA contributions for two active participants whose joint 219(g) MAGI is $543,076, above $149,000. The +$1,327.26 move is #9122 alone: 4,147.68 of refund at a 32% ordinary rate. Zeroing the refund on 2.15.17 returns exactly 106505.8984375. On 2.15.17 the alternative value is 108525.578125 if the refund belongs in AGI (latest plus 2,163.84 \u00d7 32%). If #9122 is not adopted, the Sept 22 value 107198.335938 stands. The engine-free recomputation gives 108,525.58 and 107,198.33. NIIT (r25) is included, as in the board." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "board": 9563.052734, + "latest": 9577.902344, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "The r02 defect is unfixed in 2.15.17: it still deducts $8.48 for an HoH active participant (401(k) deferral, wages $100,195) with 219(g) MAGI of $117,661, above $91,000. The +$14.85 move is #9122 alone: 67.50 at 22%. Zeroing the refund returns exactly 9563.052734375. On 2.15.17 the alternative value is 9579.767578 if the refund belongs in AGI. If #9122 is not adopted, the Sept 22 value 9564.916992 stands. The engine-free recomputation gives 9,579.77 and 9,564.92." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "board": 40021.816406, + "latest": 40416.667969, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "The r02 defect is unfixed in 2.15.17: it still deducts $28.85 for a single active participant with 219(g) MAGI of $234,414, above $91,000. The +$394.85 move is #9122 alone. Taxable income rises by 1,750.24 \u00d7 0.94 at 24%. The 0.94 nets the 0.5% charitable floor against a +$113.77 SALT deduction, which comes from the engine's ct_withheld_income_tax proxy applying CT rates to the higher federal AGI; CT's own tax does not change. Zeroing the refund returns exactly 40021.81640625. On 2.15.17 the alternative value is 40423.175781 if the refund belongs in AGI. If #9122 is not adopted, the Sept 22 value 40028.320312 stands. The engine-free recomputation gives 40,423.18 and 40,028.32." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "Alternative-value bookkeeping is framed incompletely. The settlement says the Sept 22 alternatives stand 'if #9122 is not adopted', but that holds only if a hold/convention removes the refund from the scored reference. The sibling cluster's own module (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py) is written as 'Alternative reading (not a reference fix, not a hold)', i.e. an unlisted-input exclusion; under that outcome the reference stays raw 2.15.17 with the refund in AGI. The mapping should be three-way. (a) Refund inclusion adopted: frozen_value becomes latest (107833.15625 / 9577.90234375 / 40416.66796875) and alternative_value becomes 108525.578125 / 9579.767578125 / 40423.17578125. (b) Refund reading excluded as an unlisted input: same frozen_value, and the root cause becomes r02_ira_219g plus the refund cause. On the precedent of r02_ira_219g+r11_ca_itemized_conformity (both readings applied together), the combined alternative is 107198.3359375 / 9564.9169921875 / 40028.3203125, which I reproduced exactly on 2.15.17; the r02-only alternative is the (a) value. (c) A hold keeps the refund out: frozen_value stays the board and the Sept 22 values stand. In (a) and (b) the records' frozen_value and engine_version ('policyengine-us 1.755.4') must also be updated; the settlement is silent on those fields.", + "scenario_005 federal has a second, unlisted-input dependence the settlement does not mention. Each spouse lists 'traditional 401k contributions desired: $23,154' and 'roth 401k contributions desired: $4,086' ($27,240) against the 2026 402(g) limit of $24,500 (Notice 2025-67). The engine scales both pro rata (elective_deferral_contribution_scale: traditional $20,825, Roth $3,675). The prompt does not say which designation absorbs the $2,740 excess, and a careful reader could keep the $23,154 traditional deferral and cut the Roth, or the reverse. On 2.15.17 with conventions (harness-equivalent probe, confirmed by my engine-free calc): Roth cut first 106,342.59; pro rata 107,833.16; traditional cut first 108,096.20, a $1,753.60 spread. The alternative on the same three readings is 107,035.02 / 108,525.58 / 108,788.62. scenario_005 is the only one of the 100 scenarios above the 402(g) limit. This does not change keep_excluded. It does mean 005 federal must not be un-excluded merely because r02 is later fixed upstream, and 005's alternative value is not unique. The exclusion record should carry this second cause. The investigator's indep.py hard-codes the engine's pro rata split as a 'convention reproduced on purpose' without flagging it as a prompt ambiguity.", + "The 'engine-free recomputation' is less independent than described for scenario_120. Without disclosing it, it repeats the engine's absence of IRC 469. loss_ald deducts the full $2,446.59 farm-rental loss against $2,000 of rental income, although rental activities are passive under 469(c)(2) and the 469(i) allowance is zero at MAGI of about $234k. It also applies the $750k 163(h)(3)(F) cap to $1,000,000 of listed mortgage debt whose origination date is unlisted (debt from before 2017-12-16 keeps the $1M limit). My recomputation gives 40,441.42 latest and 40,447.93 alternative with 469 applied (+24.75), and 40,192.03 and 40,198.54 with grandfathered debt (-224.64). So 'matches all 12 engine values' validates engine arithmetic, not law-correct values. These effects hit board, latest and every alternative alike, so keep_excluded is unaffected. But 120 federal carries dependences beyond r02 (the mortgage-debt date, and whether the listed farm rent is already 469-limited), and its alternative value is engine-relative." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17. gov.irs.ald.deductions (2026) still lists traditional_ira_contributions; the list and the retirement variables are identical to 1.755.4 apart from uprating attributes (44001a4c24). No 219(g) or active-participant code exists, and upstream #8388 scoped 219(g) out with no follow-up. Both spouses list 401(k) deferrals, so both are active participants under 219(g)(5)(A)(i); ira_219g_magi is $543,076, far above $149,000, so the deductible limit is $0 and the engine's $2,163.84 deduction is wrong. The +$1,327.26 move is #9122 alone: salt_refund_income=0 on 2.15.17 gives exactly 106505.8984375. The alternative on 2.15.17 is 108525.578125 with the refund in AGI; 107198.3359375 applies with both readings or under a hold. Both were reproduced by the harness and by my engine-free calculation to $0.01. It also depends on the unlisted 402(g) excess allocation (106,342.59 to 108,096.20 on latest), so it should stay excluded even after an upstream r02 fix unless that allocation is ruled. NIIT is included (r25), as in the board." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17 (same evidence as 005). The HoH head lists a $181 traditional 401(k) deferral with $100,195 of wages, so is an active participant; ira_219g_magi is $117,661, above $91,000, so the limit is $0 but the engine deducts $8.48. The +$14.85 move is #9122 alone (67.50 at 22%); refund=0 on 2.15.17 gives exactly 9563.052734375. The alternative on 2.15.17 is 9579.767578125 with the refund in AGI, or 9564.9169921875 with both readings or under a hold. My engine-free calc reproduces these (standard deduction $24,150 + 170(p) $700; CTC $2,200; CDCC 20% of $3,000). No other unlisted dependence was found. The only non-excluded output r02 moves is 082 state, by $0.59, under the $1 tolerance." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17. The single head (76) lists a $617 traditional 401(k) deferral with $165,597 of wages, so is an active participant; ira_219g_magi is $234,414, above $91,000, so the limit is $0 but the engine deducts $28.85. The +$394.85 move is #9122 alone. Taxable income rises $1,645.22, because the engine's ct_withheld_income_tax proxy (CT single rates on federal AGI less $15,000) raises SALT by $113.77 and the 0.5% charitable floor rises $8.75; CT's own tax is unchanged at 11917.18 because CT subtracts salt_refund_income. Refund=0 gives exactly 40021.81640625. The alternative on 2.15.17 is 40423.17578125 with the refund in AGI, or 40028.3203125 with both readings or under a hold. Beyond r02 the output also depends on the engine's missing 469 limit (+24.75) and the unlisted mortgage origination date (-224.64 if grandfathered), so it should not be un-excluded on an r02 fix alone." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv) on 005/082/120 with no fix, latest_conventions and latest_alt_r02_ira_219g (rv_none.csv, rv_latest_conventions.csv, rv_latest_alt_r02_ira_219g.csv). It reproduces board, latest and alternative values exactly. Full 1,984-output sweeps of both modules (rv_full_*.csv; 29 and 35 moved) match the investigator's CSVs bit for bit. r02 changes exactly the 11 r02-excluded outputs plus 082 state by $0.59. My own probe (rv_probe.py \u2192 rv_probe_21517.json) shows refund=0 reproduces the board exactly and r02 with refund=0 reproduces the Sept 22 alternatives exactly. It also shows ira_active_participant=1, ira_219g_deductible_limit=0 and ira_219g_magi of 543,076 / 117,661.125 / 234,414.203, plus the 402(g) allocation variants for 005. Engine code read in both installed packages: gov/irs/ald/deductions.yaml is identical in 1.755.4 and 2.15.17; traditional_ira_contributions = desired \u00d7 ira_contribution_scale; ira_contribution_limit; loss_ald (no 469); ct_withheld_income_tax; NII sources; CT subtractions (salt_refund_income); gross_income sources diff. A grep of 2.15.17 for 219(g)/active participant/ira_deduction found only an MO TANF comment. Upstream (read-only): git log 06665727d8..79be99f671 on the ALD list and retirement dir shows only 44001a4c24 (uprating attributes); -S '219(g)', active_participant and ira_deduction return nothing. 316e7832a1, 5581f5be1a, 5d3d1312fd and e990b4a6f8 exist with the stated dates and are ancestors of 79be99f671. gh search finds only #8388 (closed 2026-05-24, 219(g) explicitly out of scope). Primary sources fetched this session: Notice 2025-67 (IRS PDF, Last-Modified 2025-11-13: 402(g) $24,500; IRA $7,500/$1,100; 219(g) $81k-$91k, $129k-$149k, $242k-$252k); Rev. Proc. 2025-32 (IRS PDF, Last-Modified 2025-10-17: Tables 1-3, CG breakpoints, standard deduction, aged $2,050, AMT, CTC $2,200); and 26 U.S.C. 219(g)(1)-(8) (last amended Pub. L. 117-328), 164(b)(6)-(7), 170(b)(1)(I), 170(p), 63(b), 21(a)(2)/(c), 24(h), 1411(b) at uscode.house.gov. All of these are pre-freeze. I could not locate the uscode 'current through' line in the fetched HTML; it is immaterial. The prompt the models saw (data.json, June bundle) lists '... 401k contributions desired' for every IRA contributor, so active participation is stated and r02 is a genuine engine defect, not an unlisted input. I wrote an independent engine-free calculator (rv_indep.py \u2192 rv_indep.json). It matches all 12 board/latest/alternative values and the 005 allocation variants to within $0.01, and quantifies the 469 and grandfathered-mortgage variants for 120. A 402(g) scan of all 100 scenarios found only 005 above the limit. The 005 wage equals $69.20 \u00d7 50 h \u00d7 52, so no overtime premium is implied." + } + }, + { + "id": "excl_r11_ca_itemized_conformity", + "title": "Excluded CA state outputs (r11_ca_itemized_conformity, +r02) moved by c_ca_hold_2025", + "outputs": [ + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 41051.511719, + "latest": 41267.011719, + "v11_1755": 41051.511719, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g+r11_ca_itemized_conformity", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 2439.650146, + "latest": 2505.870117, + "v11_1755": 2439.650146, + "excluded": true, + "exclusion_root_cause": "r11_ca_itemized_conformity", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 4493.743164, + "latest": 4640.777344, + "v11_1755": 4493.743164, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g+r11_ca_itemized_conformity", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r11_ca_itemized_conformity", + "classification": "excluded_defect_still_present", + "summary": "Both defects are still in policyengine-us 2.15.17, so all three exclusions stay.\n\n- **r11:** `ca_itemized_deductions_pre_limitation.py`, `itemized_deductions_less_salt.py`, `charitable_deduction.py`, `misc_deduction.py` and the charity-floor and misc-applies parameters are byte-identical to 1.755.4. No commit in 06665727d8..79be99f671 adds California rules for charity or misc. Upstream gave Arkansas (#9625) and Hawaii (#9597) their own itemized rules, but not California.\n- **r02:** `traditional_ira_contributions.py` and `gov/irs/ald/deductions.yaml` are identical, and there is still no 219(g) or active-participant logic.\n\nFull sweeps of latest_conventions plus r11_v2 or plus r02_v2 on 2.15.17 move exactly the outputs already excluded for those root causes, and no others. r11 moves only 005, 022 and 099 state. r02 moves only outputs whose exclusions name r02.\n\nHow far each output moves from the board:\n- **022 and 099:** only c_ca_hold_2025 moves them, because the board keeps raw v1.1 values for excluded outputs. Raw 2.15.17 equals the board, and 2.15.17 plus conventions equals 1.755.4 plus r19 exactly.\n- **005:** the gap to the board is +215.50. Of that, +168.36 is the convention and +47.14 is upstream #9122 (316e7832a1, SALT refund now in federal AGI). The +47.14 splits into 6% \u00d7 4,147.68 \u00d7 9.3% = 23.14 through the Schedule CA line 29 limitation, and $24 through the exemption-credit AGI phase-out. Undoing #9122 on top of the conventions gives 41,219.867188, which is 1.755.4 plus r19 to the cent. So r04 and r32 do not touch this output.\n\nLaw-correct values on 2.15.17 plus conventions plus r11_v2 plus r02_v2:\n- **022:** 1,968.81, equal to the Sept 22 alternative.\n- **099:** 4,588.48, equal to the Sept 22 alternative.\n- **005:** 40,975.25. This is 40,920.40, the Sept 22 alternative, plus #9122's +54.85.\n\nI reproduced all of these, and the three latest values, to the cent in `independent_calc.py`, which does not use the engine.\n\nNew finding: 099 also carries a defect nobody has recorded. California does not conform to the federal educator-expense deduction (2025 Schedule CA instructions, Part I line 11), but 2.15.17's `ca_additions` adds back only HSA contributions. That understates 099 by $27.00; the fully law-correct value is 4,615.48. The Sept 22 record also flagged that the 401(k) over-limit deferrals in 005 can be read two ways (about \u2212$471).\n\nScratch files are in `/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/excl_r11_ca_itemized_conformity/`: the diag_*.py modules, out/*.csv, compare_full.py, independent_calc.py and .out, and the sources in src/. No hold module is needed, and scoring is unaffected.", + "upstream_changes": [ + { + "commit": "316e7832a1 (merged e990b4a6f8)", + "date": "2026-09-08 (merge 2026-09-16)", + "what": "#9122: adds salt_refund_income to gov.irs.gross_income.sources and to the CA AGI subtractions. For 005 CA this raises federal AGI by 4,147.68, and federal AGI drives the Schedule CA line 29 limitation (+248.86) and the exemption-credit phase-out (2 more $6 steps \u00d7 2 credits). That is +47.14 on the convention basis and the only upstream change that moves 005 state. Reverting it on top of latest_conventions reproduces 1.755.4 + r19 (41,219.867188) exactly." + }, + { + "commit": "df3482f4ef", + "date": "2026-07-21", + "what": "#9059: makes the CA standard deduction and itemized-limit AGI threshold upratable (they were frozen at 2025 values). It moves raw 005 (40,986.87) but is neutralized by c_ca_hold_2025. It does not touch r11 logic." + }, + { + "commit": "2572674b97", + "date": "2026-09-16", + "what": "#9429: propagate_metadata_to_children for uprating on the CA itemized-limit threshold and standard deduction, also held by c_ca_hold_2025. It does not touch r11 logic." + }, + { + "commit": "44001a4c24", + "date": "2026-09-18", + "what": "#9527: one-line uprating metadata on the *_contributions_desired retirement inputs. It adds no 219(g) or active-participant logic (git log -S active_participant and -S '219(g)' over the range find nothing)." + }, + { + "commit": "a21cd0adc5", + "date": "2026-09-26", + "what": "#9625: Arkansas gets its own misc and casualty itemized rules. This is the analogous fix for another state; California's itemized deductions still inherit the federal charitable_deduction and misc_deduction." + }, + { + "commit": "9a3c7670c6", + "date": "2026-09-28", + "what": "#9597: Hawaii itemized limits. Again another state only; no California change." + } + ], + "law": "All sources were read this session and published before the 2026-07-03 freeze unless noted. Copies are in excl_r11_ca_itemized_conformity/src/ or the Sept 22 src/ folders.\n\n**California conformity and itemized deductions (r11)**\n- **Conformity date:** SB 711 (Stats. 2025 ch. 231) was approved and filed on 2025-10-01 (leginfo chaptered text). Sec. 1 sets R&TC 17024.5(a)(1)(Q): for taxable years beginning on or after 2025-01-01, the Internal Revenue Code is read as of January 1, 2025.\n- **Miscellaneous deductions:** SB 711 Sec. 11 set R&TC 17076(c), effective 2025-10-01: \"Section 67(g) ... relating to suspension for taxable years 2018 to 2025, shall not apply.\" R&TC 17076(a) keeps the IRC 67 2% floor.\n- **Current text of 17024.5:** SB 1435 (Stats. 2026 ch. 236, effective 2026-09-14) amended it after the freeze. It leaves (Q) at January 1, 2025, so it does not change the result.\n- **Itemized deductions generally:** R&TC 17201(a) and (b) (Stats. 1993) bring in IRC Parts VI (which includes \u00a7170) and VII (which includes \u00a7219) \"except as otherwise provided\".\n- **Federal rules California does not follow:**\n - IRC 170(b)(1)(I), the 0.5% charitable floor, was added by P.L. 119-21 \u00a770425 on 2025-07-04, after California's specified date.\n - The federal misc suspension is now IRC 67(h), made permanent by P.L. 119-21 \u00a770110 (OLRC and Cornell text).\n- **FTB 2025 Schedule CA (540) instructions (revised 04/2026):**\n - California does not conform to the OBBBA.\n - Charity is capped at 50% of federal AGI.\n - The misc suspension does not apply (lines 19-25, 2% of federal AGI).\n - The line 29 limitation uses federal AGI from Form 540 line 13.\n - Line 11: \"California law does not conform ... regarding educator expenses\".\n - Part I line 1: California does not tax the state income tax refund.\n- **2025 Form 540 booklet, line 32:** the exemption-credit phase-out uses federal AGI over $252,203 (single) or $504,411 (joint), at $6 per $2,500.\n- **FTB Tax News, October 2025 (\"2025 Indexing\", dated 2025-09-12):** exemption credits of $153 per person and $475 per dependent, and the 2025 rate schedules.\n\n**IRA deduction phase-out (r02)**\n- IRC 219(g)(1), (2) and (5) (Cornell).\n- IRS Notice 2025-67 (irs.gov PDF, last-modified 2025-11-13): the 2026 joint active-participant range is $129,000 to $149,000. 005's MAGI (543,076) and 099's (159,729) exceed both this range and the 2025 one ($126,000 to $146,000), so both are fully phased out under either year.", + "per_output": [ + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "board": 41051.511719, + "latest": 41267.011719, + "proposed_reference": 41267.011719, + "action": "keep_excluded", + "reason": "Both r11 and r02 are still present in 2.15.17. On 2.15.17 + conventions, r11_v2 moves the output \u2212521.09, r02_v2 +225.31, and both together \u2212291.76, giving a law-correct 40,975.25 (hand check 40,975.26).\n\nIf the lead does not adopt #9122, the law-correct value is 40,920.40, the Sept 22 alternative, reproduced exactly by reverting #9122.\n\nThe latest \u2212 board gap is +215.50:\n- +168.36 from c_ca_hold_2025, which the board never applied to this excluded output.\n- +47.14 from #9122: 23.14 through the line 29 limitation and 24.00 through the exemption-credit phase-out.\n\nThe proposed value is the unscored stored reference (2.15.17 + conventions). The exclusion record's alternative_value should become 40,975.25 if #9122 is adopted. The Sept 22 record also notes a two-way reading of the over-limit 401(k) deferrals (about \u2212$471)." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "board": 2439.650146, + "latest": 2505.870117, + "proposed_reference": 2505.870117, + "action": "keep_excluded", + "reason": "r11 is still present. The engine applies the federal 0.5% floor, cutting charity from 15,393.41 to 14,847.67, and drops misc of 8,350.54 \u2212 2% \u00d7 109,149.16 = 6,167.56. Correcting both removes 6,713.31 \u00d7 8% = 537.06, giving 1,968.81. That equals the Sept 22 alternative exactly and matches the engine-independent hand calculation (Schedule X; 306 of exemption credits). r02 has no effect here.\n\nThe move from the board is c_ca_hold_2025 alone: raw 2.15.17 equals the board, and 2.15.17 + conventions equals 1.755.4 + r19. The proposed value is the unscored stored reference.\n\nThe value is computed on the rate schedule, as the board does for every CA output. FTB's tax table for taxable income of $100,000 or less could differ by a few dollars." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "board": 4493.743164, + "latest": 4640.777344, + "proposed_reference": 4640.777344, + "action": "keep_excluded", + "reason": "r11 and r02 are still present.\n- r11: the federal floor removes 797.93 of charity (\u221263.83).\n- r02: both spouses defer into a 401(k), and MAGI of 159,729 exceeds $149,000, so the $144.26 IRA deduction is fully phased out (+11.54 once r11 is also corrected).\n- Together: 4,588.48, equal to the Sept 22 alternative and to the hand calculation.\n\nThe move from the board is c_ca_hold_2025 alone: raw 2.15.17 equals the board, and 2.15.17 + conventions equals 1.755.4 + r19.\n\nThere is also an unrecorded defect that 2.15.17 still has. California does not allow the federal $337.50 educator-expense deduction (Schedule CA Part I line 11), but ca_additions adds back only HSA contributions. That adds +27.00, so the fully law-correct value is 4,615.48. The exclusion reason could cite it.\n\nThe proposed value is the unscored stored reference." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "005 state: the settlement calls 40,975.25 'law-correct' and says the exclusion's alternative_value 'should become 40,975.25 if #9122 is adopted'. But #9122 is already in 2.15.17, so the real question is what the prompt says, not whether to adopt it. The prompt shows 'state and local tax refund income: $4,148' and gives no prior-year facts. Under IRC 111 the refund is federal gross income only to the extent the prior-year deduction produced a tax benefit. The sibling cluster salt_refund_gross_income_9122 (fixes/latest_alt_salt_refund_no_prior_benefit.py) treats this as an unlisted-input ambiguity. The stated facts therefore support two values: 40,975.25 (refund taxable, the engine's reading) and 40,920.40 (refund not income). Neither is 'the' law-correct value. The exclusion record should keep both readings and add an unlisted-input reason. It should not overwrite alternative_value with 40,975.25 alone. Both numbers reproduce exactly (my rv_c_r11_r02 and rv_c_r11_r02_rev sweeps).", + "005 state, new ambiguity: the prompt lists 'tax-exempt interest income: $8,961' without saying who issued the bonds. California taxes federally exempt interest from non-California state and municipal bonds (FTB 2025 Schedule CA (540) instructions, rev. 04/2026, Part I line 2, column C). 2.15.17's ca_additions adds back only ca_hsa_addition, so the engine treats all of it as California-exempt. Reading it as non-California interest moves the output by +833.41: harness rv_c_muni.py gives 42,100.417969 on latest_conventions vs 41,267.011719, and my hand calculation gives 8,961.35 x 9.3%. No engine fix can resolve this, so 005 state must stay excluded even after r02 and r11 are fixed upstream. The settlement does not mention it.", + "005 state: the settlement cites the Sept 22 figure 'about -$471' for the two-way 401(k) reading (cut Roth deferrals first rather than the engine's proportional scaling to the $24,500 limit). That figure was computed on the 1.755.4 basis. On the current basis (conventions + r11 + r02) the reading moves the output by -491.85, under either #9122 reading (rv_independent_calc.py). This is minor, but it adds to the list of unlisted or two-way inputs behind 005.", + "upstream_changes leaves out the California AMT rework in the range: 2dd9762506 (#8873; drops the mortgage-interest line from amti/sources.yaml) and d11f711826 (#8925; ca_pre_exemption_amti no longer adds back the full pre-limitation itemized deductions, and the new ca_itemized_deductions_limitation variable is subtracted). The rework touches the path of r11_v2's Schedule P line-5 swap, and that module's docstring still describes the 1.755.4 AMTI. It changes no number here: ca_amt = 0 for all three outputs under both latest_conventions and conventions + r11 + r02. For example, 005 AMTI is 536,354.81 with a tentative minimum tax far below the regular tax. Reverting #9122 on conventions + r11 + r02 also gives the Sept 22 alternative 40,920.402344 exactly. So the settlement's attribution holds; only the list is incomplete.", + "The full-sweep claim that r02 moves only outputs whose exclusions name r02 holds only under the $1 tolerance. My full rv_c_r02 sweep and the investigator's own full_diag_conv_r02.csv both also move scenario_082 NY state_income_tax_before_refundable_credits by +0.59, and that output is not excluded. This does not affect scoring and is outside this cluster, but the claim as worded is not exact.", + "099, possible second prompt-reading issue (low confidence, benchmark-wide): each adult lists both 'health insurance premiums excluding Medicare Part B: $2,500' and 'other health insurance premiums: $2,500'. The engine counts only the first (medical_expense_health_insurance_premiums uses health_insurance_premiums_without_medicare_part_b), so medical expenses of 10,000 fall below the 7.5% floor of 11,979.70. A reader who adds both lines gets 15,000, a 3,020.30 medical deduction and -241.62 of CA tax. It does not change the action, since 099 stays excluded. The same double listing in 005 and 022 is harmless because they stay under the floor either way.", + "Citation detail: the settlement dates FTB Tax News October 2025 '2025 Indexing' to 2025-09-12, but that date belongs to a separate Tax News Flash on the same page (market-based sourcing). The issue is still pre-freeze, so nothing changes." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 41267.011719, + "reason": "Both r11 and r02 are still in 2.15.17. The CA itemized, charity and misc variables and parameters, traditional_ira_contributions and gov/irs/ald/deductions.yaml are identical over 06665727d8..79be99f671, and git grep finds no active_participant or 219(g) logic. The stored, unscored value is 2.15.17 + conventions = 41,267.011719; I reproduced it in the harness and by hand. The +215.50 gap to the board splits as +168.36 from c_ca_hold_2025 (the board kept raw v1.1 for this excluded output) and +47.14 from #9122: 23.14 through the line 29 limitation (248.86 x 9.3%) and 24.00 through the exemption credits (13 to 15 phase-out steps x $6 x 2). With #9122 reverted on the conventions, the value is 41,219.867188 = 1.755.4 + r19. The output also depends on three stated-but-unresolved or unlisted facts: whether the SALT refund was taxable (40,975.25 vs 40,920.40 after r11 + r02), who issued the tax-exempt interest (+833.41 if non-California), and how the over-limit 401(k) deferrals are cut (-491.85). The exclusion should therefore also carry an unlisted-input reason, and alternative_value should record both SALT-refund readings rather than a single 'law-correct' 40,975.25." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 2505.870117, + "reason": "r11 is still present. ca_itemized_deductions_pre_limitation carries the federal charitable_deduction (0.5% floor from 2026: 14,847.67 instead of 15,393.41) and the federal misc_deduction (0 since 2018). R&TC 17076(c) (Stats. 2025 ch. 231, SB 711 Sec. 11, effective 2025-10-01) and the conformity date in 17024.5(a)(1)(Q) (IRC as of 2025-01-01) mean California allows both. The law-correct value is 1,968.81; I reproduced it by harness and by hand. Federal AGI is 109,149.16 and CA AGI is 91,524.41; itemized deductions are 8,829 + 15,393.41 + 6,167.56 = 30,389.97; CA taxable income 61,134.44 gives tax of 2,274.81, less 306 of exemption credits. That equals the Sept 22 alternative. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 2,505.870117 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. I found no unlisted-input dependence: medical stays under the floor under any reading, and the prompt lists no refund and no tax-exempt interest. This is a pure engine defect, so the output could be un-excluded once r11 is fixed upstream. The value uses the rate schedule, as for every CA output (FTB's tax table for taxable income of $100,000 or less could differ by about a dollar)." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 4640.777344, + "reason": "r11 and r02 are still present. The engine's federal floor removes 797.93 of charity (-63.83). Both spouses defer into a 401(k), which makes them active participants, and MAGI of 159,729.30 exceeds the $149,000 top of the 2026 joint range (IRS Notice 2025-67, posted 2025-11-13), so the $144.26 IRA deduction is fully phased out (+11.54 after r11). Together these give 4,588.48, the Sept 22 alternative. I confirm the new educator-expense defect: FTB 2025 Schedule CA line 11 says 'California law does not conform to federal law regarding educator expenses', but 2.15.17's ca_additions adds only ca_hsa_addition. Adding back $337.50 x 8% = +27.00 gives a fully law-correct 4,615.48 (harness and hand calculation agree). The exclusion reason should cite this third defect. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 4,640.777344 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. Possible extra reading: if both premium lines are summed, the output moves -241.62." + } + ], + "what_i_checked": "Every scratch file is in /Users/maxghenis/PolicyEngine/pb-refs-reviews-0929/excl_r11_ca_itemized_conformity. (1) I re-ran sweep_latest.py on 2.15.17 for the three scenarios under my own composition modules (rv_*.py, loading the fix modules by path): raw; latest_conventions; + r11_v2; + r02_v2; + r11_v2 + r02_v2; + educator add-back; my own idempotent #9122 revert on conventions and on conventions + r11 + r02; and a non-California muni-interest reading. Results: raw 40,986.87 / board / board. Conventions: 41,267.011719 / 2,505.870117 / 4,640.777344. + r11: -521.09 / -537.06 / -63.83. + r02: +225.31 / 0 / +11.60. Both: 40,975.253906 / 1,968.805420 / 4,588.483887. Educator: 099 4,615.483887. Revert: 41,219.867188 and 40,920.402344. Muni: 42,100.417969. (2) I ran full 1,984-output sweeps of conventions + r11 and conventions + r02 and diffed them against latest_conventions.csv. r11 moves only the three cluster outputs. r02 moves 11 outputs by more than $1, all with exclusions naming r02, plus 082 NY state by +0.59. (3) A probe dumped the engine's intermediate values (AGI, CA AGI, itemized deductions pre and post the limit, taxable income, exemptions, AMTI and AMT, 401(k) and IRA amounts). (4) rv_independent_calc.py is pure Python with no engine and uses only scenario inputs and FTB/IRS parameters. It reproduces all 16 values above to the cent. (5) I rendered the exact prompts with policybench.prompts.describe_household (rv_prompts.txt). (6) Upstream, read-only: git diff 06665727d8..79be99f671. charitable_deduction.py, misc_deduction.py, total_misc_deductions.py, itemized_deductions_less_salt.py, ca_itemized_deductions(_pre_limitation).py, ca_deductions.py, the charity floor and ceiling and misc applies and floor parameters, itemized_deductions.yaml, ald/deductions.yaml, traditional_ira_contributions.py and educator_expense.py are all identical. git grep finds no active_participant or 219(g) logic, and no California charity or misc code. I also read git show 316e7832a1 (#9122) and the CA AMT diffs from #8873 and #8925. (7) Primary sources fetched this session, in src/: leginfo SB 711 status and history (Chapter 231, Statutes of 2025, approved and chaptered 10/01/25, urgency) and chaptered text (SECTION 1 amends 17024.5 with (Q) January 1, 2025; SEC. 11 amends 17076 to read with (c) 'Section 67(g) ... shall not apply'); current R&TC 17076 (effective 2025-10-01) and 17024.5 (amended by Stats. 2026 ch. 236, SB 1435, effective 2026-09-14, (Q) unchanged; (h)(2)(A) makes AGI-based limits use federal AGI); FTB 2025 Schedule CA (540) instructions, rev. 04/2026 (OBBBA nonconformity; Part I line 1 refund subtracted, line 2 column C non-California bond interest, line 11 educator; Part II charity at 50% of federal AGI, lines 19-22 misc nonconformity, line 29 worksheet at $504,411/$252,203); the 2025 Form 540 booklet (AGI limitation worksheet, $6 per $2,500); FTB Tax News October 2025 (2025 exemption credits $153/$475); IRS Notice 2025-67 PDF (last-modified 2025-11-13; joint active-participant range $129,000-$149,000); the Cornell text of 26 USC 67 (current (h) suspension, P.L. 119-21 section 70110, July 4, 2025); and, from the salt_refund_gross_income_9122 folder's copy (not fetched by me), Rev. Proc. 2025-32 (2026 educator limit $350, so $337.50 is within the federal cap). Every source the calls rest on was published before 2026-07-03, except SB 1435, which does not change the result." + } + }, + { + "id": "excl_r07_idaho_health_premiums", + "title": "Excluded ID state outputs (r07_idaho_health_premiums) moved by c_id_hold_2025", + "outputs": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "state": "ID", + "board": 755.77832, + "latest": 761.557007, + "v11_1755": 755.77832, + "excluded": true, + "exclusion_root_cause": "r07_idaho_health_premiums", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "state": "ID", + "board": 2435.27832, + "latest": 2441.057129, + "v11_1755": 2435.27832, + "excluded": true, + "exclusion_root_cause": "r07_idaho_health_premiums", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r07_idaho_health_premiums", + "classification": "excluded_defect_still_present", + "summary": "No upstream change moves these two outputs. Raw 2.15.17 equals raw 1.755.4: 755.7783 for scenario_007 and 2435.2783 for scenario_053. Both engines carry the same uprated zero-rate threshold for 2026 ($4,920.03 single), and the Idaho subtraction list, id_deductions and main/*.yaml are byte-identical between them. The +$5.78 comes only from latest_c_id_hold_2025, which sets the threshold to $4,811: 0.053 \u00d7 (4,920.03 \u2212 4,811) = 5.78. On 1.755.4 the board's r19_id_convention gives the same 761.557 and 2441.057, but its revision entry says \"excluded_outputs_untouched\": true, so the board kept the raw values for these excluded rows. The exclusion's root cause, r07 (the engine never applies Idaho's \u00a763-3022P subtraction for health insurance premiums), is still present in 2.15.17. subtractions.yaml is unchanged and has no premium item. No Idaho commit in 06665727d8..79be99f671 touches premiums (git log -S \"3022P\" and -S health_insurance return nothing). upstream/main is at 79be99f671, and gh search finds no issue or PR on it. Running r07_idaho_health_premiums_v2 on top of latest_conventions on 2.15.17 gives exactly the exclusion record's alternative_value: 651.317017 and 2176.057129. A full 1,984-output sweep shows r07 moves only these two outputs, by \u2212110.24 (0.053 \u00d7 2,080) and \u2212265.00 (0.053 \u00d7 5,000). I recomputed all values by hand, without the engine, and got the same numbers. Both outputs stay excluded, and scoring does not change.", + "upstream_changes": [ + { + "commit": "84b94fc350", + "date": "2026-07-05", + "what": "Adds disabled-adult care expenses to Idaho's dependent-care subtraction base. Not premium-related; both households have $0 care expenses." + }, + { + "commit": "7cd885f72b", + "date": "2026-07-07", + "what": "CDCC coupling: gates the Idaho dependent-care subtraction and cap under reforms (#8933). Not premium-related." + }, + { + "commit": "7d4e0d1376", + "date": "2026-07-15", + "what": "Counts reported SNAP/TANF receipt in the Idaho grocery credit and PBF liability. Does not affect the tax before refundable credits for these households." + }, + { + "commit": "e537364e5e", + "date": "2026-07-15", + "what": "Idaho AABD SSI-receipt eligibility. Not an income tax change." + }, + { + "commit": "9f9d646923", + "date": "2026-07-22", + "what": "Idaho aged/disabled credit vs deduction now picks the better option. Neither household claims either, since both have id_non_refundable_credits = 0." + }, + { + "commit": "d7fc17abfe", + "date": "2026-09-02", + "what": "Adds metadata (propagate_metadata_to_children) to Idaho parameters. No value change." + }, + { + "commit": "2d21a968d8", + "date": "2026-09-24", + "what": "Idaho ICCP court-related child-care eligibility. Not an income tax change." + }, + { + "commit": "(none)", + "date": "2026-09-28", + "what": "No commit in 06665727d8..79be99f671 adds the Idaho Code 63-3022P / Form 39R line 18 health insurance premium subtraction. The zero-rate threshold files and the subtractions list are unchanged, and upstream/main equals 79be99f671." + } + ], + "law": "Premium subtraction: Idaho Code \u00a763-3022P (\"Health insurance costs\"). It allows a deduction for premiums the taxpayer paid for medical-care insurance for self, spouse or dependents that are \"not otherwise deducted or accounted for\". History: added 2001, ch. 386; amended 2003, ch. 10. I fetched the text live from legislature.idaho.gov on 2026-09-28; it matches the saved copy triage/scenario_053__.../id_63-3022P.html. The Tax Commission's 2025 instructions (EIN00046, revision dated 03-02-2026, before the 2026-07-03 freeze) put this on Form 39R Part B line 18. The instructions on pp. 34-35 of 52 allow premiums not already deducted or excluded, bar cafeteria-plan pre-tax premiums and self-employed premiums already deducted, and say to enter zero on worksheet line 8 when the filer is not itemizing for Idaho. Both households take the $16,100 standard deduction and have no self-employed ALD. Scenario_007 has no wages, so no pre-tax plan is possible. Their full premiums ($2,080 and $5,000) are therefore subtractable. Zero-rate threshold (the source of the +$5.78): EIN00046 03-02-2026 p. 9 of 52 sets $4,811 single / $9,622 joint and HOH at 5.3%. The Tax Commission rate-schedule page (tax.idaho.gov, fetched 2026-09-28, \"last updated December 29, 2025\") still shows no 2026 row. Holding $4,811 is therefore the pre-freeze convention c_id_hold_2025, and 2.15.17's $4,920.03 is an uprated projection, not a published figure. Rate: 5.3% under Idaho Code \u00a763-3024(2)(a) (2025 HB 40).", + "per_output": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "board": 755.77832, + "latest": 761.557007, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "r07 is still present in 2.15.17. id_subtractions is 13,784.125 (taxable Social Security only, no $2,080 premium subtraction), so id_taxable_income is 19,180. Raw 2.15.17 equals raw 1.755.4 (755.7783). The move to 761.557 comes only from c_id_hold_2025 (0.053 \u00d7 (19,180 \u2212 4,811)), which the board did not apply to excluded outputs. Law-correct value on 2.15.17 plus conventions plus r07_v2: 651.317 = 0.053 \u00d7 (17,100 \u2212 4,811). I recomputed it by hand, and it equals the exclusion record's alternative_value of 651.317017, so the record's alternative needs no update. If the regenerated file stores conventions on excluded rows, the stored (unscored) value is 761.557, and the record's frozen_value and engine_version fields would then need to change to 761.557 and 2.15.17." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "board": 2435.27832, + "latest": 2441.057129, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "r07 is still present in 2.15.17. id_subtractions is 0, so the $5,000 taxpayer-paid premium is never subtracted, and id_taxable_income is 50,868.68. Raw 2.15.17 equals raw 1.755.4 (2435.2783). The move to 2441.057 comes only from c_id_hold_2025 (0.053 \u00d7 (50,868.68 \u2212 4,811)), which the board did not apply to excluded outputs. Law-correct value on 2.15.17 plus conventions plus r07_v2: 2176.057 = 0.053 \u00d7 (45,868.68 \u2212 4,811). I recomputed it by hand, and it equals the exclusion record's alternative_value of 2176.057129. It is also the Sept 22 verdict's 2170.28 moved to the held threshold. If the regenerated file stores conventions on excluded rows, the stored (unscored) value is 2441.057." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "Minor gap, and it does not change the action. For scenario_053, the investigator says the full $5,000 is subtractable but never addresses the salary-reduction bar. The 2025 Form 39R line 18 instructions (EIN00046 03-02-2026, printed p. 34) say premiums 'paid through a cafeteria plan or other salary-reduction arrangement' do not qualify. The household the model sees has gross wages of $66,969 and 'has employer-sponsored insurance', and nothing says whether the $5,000 is withheld pre-tax. So the 2176.057 alternative value holds only under an unstated reading that the premium is paid after tax. Under the pre-tax reading, the engine's no-subtraction value (2441.057 with the hold) would be the correct law. The exclusion therefore also has an unlisted-input side, and the investigator's line 'their full premiums ($2,080 and $5,000) are therefore subtractable' overstates the case for 053. The pre-tax argument holds for scenario_007, which has no wages. Either way the output stays excluded.", + "Minor imprecision. The investigator calls 651.317 and 2176.057 'law-correct'. They are correct only under the stated alternative reading: after-tax premiums, no Medicare Part B enrollment for scenario_007 (the engine has medicare_enrolled = 0, and a separate exclusion covers head_medicare_eligible), and the $950 auto-loan interest in scenario_053 not qualifying for Schedule 1-A (the engine has qualified_passenger_vehicle_loan_interest = 0, consistent with the benchmark's annotation convention). These are unscored alternatives, so nothing turns on it." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 755.7783203125, identical to v11_1755. latest_conventions gives 761.5570068359375 (+5.7787 = 0.053 x (4920.0313 - 4811)). latest_conventions + r07_v2 gives 651.3170166. The subtractions.yaml and main/single.yaml files are identical between 1.755.4 and 2.15.17, and the premium subtraction is still absent, so the r07 defect is still present. The board row is the raw 755.7783 because the c_id_hold_2025 revision has excluded_outputs_untouched: true. By hand: taxable SS 13,784.125, Idaho TI 19,180, and 0.053 x (19,180 - 4,811) = 761.557. With the premium subtraction, 0.053 x (17,100 - 4,811) = 651.317, which matches the exclusion record's alternative_value. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 2435.2783203125, identical to v11_1755. latest_conventions gives 2441.05712890625. latest_conventions + r07_v2 gives 2176.05712890625 (-265.00 = 0.053 x 5,000). id_subtractions is 0 and the r07 defect is still present. By hand: 0.053 x (50,868.68 - 4,811) = 2441.057, and 0.053 x (45,868.68 - 4,811) = 2176.057. The alternative also assumes the $5,000 is not paid through a pre-tax salary reduction. The prompt leaves that unstated (the household has wages and ESI), which gives an additional unlisted-input reason to keep this output excluded. Keep excluded; scoring does not change." + } + ], + "what_i_checked": "**Harness runs.** I used the brief's interpreter (policyengine-us 2.15.17) with sweep_latest.py. Scratch is in triage/latest/excl_r07_idaho_health_premiums/review/.\n- Baseline and latest_conventions on scenario_007 and scenario_053: raw equals v11_1755 (755.7783 and 2435.2783). Conventions give 761.557007 and 2441.057129, both moved +5.78 vs the board.\n- Full 1,984-output sweeps of latest_conventions alone and of my own composition (rev_conv_r07v2.py = latest_conventions + r07_idaho_health_premiums_v2): diffing them, r07 moves exactly two outputs, 007 by -110.24 and 053 by -265.00.\n\n**Board and sidecar.** reference_outputs.csv holds 755.7783203125 and 2435.2783203125. In the meta.json, the c_id_hold_2025 revision (fix r19_id_convention.py) has excluded_outputs_untouched: true and changed only scenario_076. reference_exclusions.json has frozen_value and alternative_value 651.317017 / 2176.057129 for both rows.\n\n**Engine.** In 2.15.17, subtractions.yaml has no premium item. single.yaml has the 2025 value 4,811 at 5.3% with gov.irs.uprating metadata, which gives 4,920.03 in 2026. Recursive diffs of parameters/ and variables/gov/states/id/tax/income between 1.755.4 and 2.15.17 show only the grocery, PBF, aged/disabled, CDCC, 529 and seasonal-rebate files differ. main/, subtractions.yaml and deductions/ are identical. In 2.15.17, medical_expense_health_insurance_premiums is 2,080 and 5,000. employer_sponsored_insurance_premiums is documented as employer-paid. id_pbf is outside this output.\n\n**Upstream.** git log 06665727d8..79be99f671 on the id parameter and variable paths returns exactly the 7 commits listed. -S 3022P and -S health_insurance return nothing. upstream/main is 79be99f671. gh search finds no Idaho premium-subtraction issue or PR.\n\n**Primary sources, fetched 2026-09-28.**\n- Idaho Code 63-3022P: text as quoted; history 2001 ch. 386, amended 2003 ch. 10.\n- 63-3024: 5.3%, last amended 2025 ch. 13, with the CPI indexing in (3).\n- 2026 H0589 (rate bill): only introduced and printed (bill history page).\n- Governor's release dated 2025-03-06: HB 40 cut the rate to 5.3%.\n- EIN00046_03-02-2026.pdf: Last-Modified 2026-03-05, CreationDate 2026-03-02. Printed p. 9 worksheet gives $4,811 single and $9,622 MFJ/HOH/QSS at 5.3%. Printed pp. 34-35 line 18 has the cafeteria-plan bar, the standard-deduction rule, and 'enter zeros on lines 8, 12, and 13' for non-itemizers.\n- tax.idaho.gov rate schedule: 'last updated December 29, 2025', with 2025 as the newest row.\n\n**Independent hand calculation.** Scenario_007: provisional income 44,922.5 gives taxable SS 13,784.125 and Idaho TI 19,180, so tax is 761.557 held and 651.317 with r07. Scenario_053: TI 50,868.68 gives 2441.057 held and 2176.057 with r07.\n\n**Rendered prompts.** The prompt shows ESI, wages and the premium lines, with no pre-tax indicator for 053." + } + }, + { + "id": "excl_r32_wi_capital_gain_distributions", + "title": "Excluded WI outputs (r32_wi_capital_gain_distributions, 042 also r06) moved mostly upstream", + "outputs": [ + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 284.740906, + "latest": 464.1828, + "v11_1755": 284.740906, + "excluded": true, + "exclusion_root_cause": "r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 843.661499, + "latest": 895.81311, + "v11_1755": 843.661499, + "excluded": true, + "exclusion_root_cause": "r32_wi_capital_gain_distributions", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r32_wi_capital_gain_distributions", + "classification": "excluded_defect_still_present", + "summary": "Both exclusions should stay: policyengine-us 2.15.17 still has the r32 defect in both outputs and the r06 defect in scenario_042. Two engine files are byte-identical in 1.755.4 and 2.15.17, and no commit in 06665727d8..79be99f671 touches either one. The first is wi_capital_gain_loss_subtraction.py, which still reads only short- and long-term capital gains, so non_sch_d_capital_gains gets no 30% exclusion. The second is wi_income_tax_before_refundable_credits.py, which is still always the no-subtraction path; 2.15.17 itself returns wi_income_tax 0 but a before-refundable value of 468.83 for 042. The move from the board comes entirely from r04 (#8839, 491642087f) plus the c_wi convention. #8839 raises WI income by the distributions (+184.95 for 042, +57.66 for 091 on 1.755.4), and c_wi subtracts 5.51, which exactly reproduces 1.755.4+c_wi+r04 (464.18, 895.81). #9122 plays no part here, because salt_refund_income is 0 in both households. I re-derived the Sept 22 alternatives on 2.15.17: latest_conventions plus the r32 module as-is plus a minimal r06 elected-path port gives 0.0 for 042 and 878.515747 for 091, with r32 alone giving 408.70 for 042. An engine-free recompute from the stated facts and published 2026 WI parameters matches to the cent. The full-bundle sweep of that verification module differs from latest_conventions on exactly these 2 of 1,984 outputs. Scratch files: /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/excl_r32_wi_capital_gain_distributions/ (check_r32_r06_on_21517.py, check_r32_only_on_21517.py, probe.py, probe_21517.json, recompute.py, sweep_check_conv_r32_r06.csv, sweep_check_conv_r32.csv, fetched sources).", + "upstream_changes": [ + { + "commit": "491642087f", + "date": "2026-07-05", + "what": "#8839: adds non_sch_d_capital_gains to federal gross income, NII and net_capital_gain. Federal AGI, and so WI income, rises by the distributions (3,753 and 1,170). This accounts for the whole move apart from the conventions, and it is the r04 upstream fix of pre-freeze law (IRC 61, 852(b)(3)(B))." + }, + { + "commit": "94e40f1471", + "date": "2026-09-01", + "what": "Projects the WI 2026 standard deduction by uprating rounded to $10 (single 13,870 / 19,990). Raw 2.15.17 is therefore 468.83 / 900.46 instead of 1.755.4+r04's 469.69 / 901.32. The c_wi_published_2026 convention overrides it to the published 13,960 / 20,120." + }, + { + "commit": "35bbe1ba0c", + "date": "2026-07-05", + "what": "#8818: recomputes the WI standard deduction on post-line-16 income in wi_retirement_income_exclusion_tax. Only wi_income_tax reads it, not the benchmarked before-refundable output, so r06 is not fixed." + }, + { + "commit": "757437e4f2", + "date": "2026-09-08", + "what": "Switches WI line 16/17 retirement income to a sources list. The line 16 amount for 042 stays 24,000, and the output does not change." + }, + { + "commit": "316e7832a1", + "date": "2026-09-08", + "what": "#9122 salt_refund_income: 0 in both households, so it has no effect here. The state port's note naming #9122 for 042/091 does not apply." + }, + { + "commit": "f3eec2d09e", + "date": "2026-09-23", + "what": "An unmerged r06 fix draft (refs/subfleet-salvage/pb-fix-wi-act15-20260923T154003Z-a1). It is not an ancestor of 79be99f671, so it is not in 2.15.17. No commit in the range touches wi_capital_gain_loss_subtraction.py or wi_income_tax_before_refundable_credits.py." + } + ], + "law": "r32:\n- Wis. Stat. 71.05(6)(b)9: subtract 30% of net capital gain on assets held more than one year. I fetched the text fresh, and it is identical in the 2023 archival database and the current database (published 2026-09-04).\n- 26 U.S.C. 852(b)(3)(B): a capital gain dividend is treated as long-term gain (Cornell LII, fetched).\n- 2025 WI Schedule SB instructions (I-0104; PDF created and Last-Modified 2025-11-07, pre-freeze), line 5 exception: when the only capital gain on federal line 7 is a capital gain distribution, enter 30% of it on line 5 without filing Schedule WD.\n\nr06:\n- Wis. Stat. 71.05(6)(b)54m, created by 2025 Wis. Act 15 section 220; the Act PDF shows enactment and publication on 2025-07-03, pre-freeze. It lets a filer aged 67 or older subtract up to $24,000 of qualified-plan and IRA distributions, and a filer who claims it forfeits the credits.\n- 2025 Schedule SB instructions, line 16: same age, cap and credit restriction.\n\n2026 parameters for the recompute: WI 2026 Form 1-ES instructions (D-101a, PDF created 2026-01-16, pre-freeze).\n- Single standard deduction: $13,960, less 12% of income over $20,120.\n- Exemptions: $700, plus $250 at age 65 or older.\n- Rates: 3.5% up to $15,110, then $528.85 plus 4.4% up to $51,950.\n\nBase amounts: IRC 86 single base amounts of $25,000 / $34,000, which are not indexed.\n\nEvery rule here was published before 2026-07-03.", + "per_output": [ + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "board": 284.740906, + "latest": 464.1828, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "Both root causes are still in 2.15.17 (identical engine files; 2.15.17's own wi_income_tax is 0 against 468.83 before refundable credits). The 464.18 is 1.755.4+c_wi+r04 exactly. The law-correct value is unchanged at 0.0.\n- Standard path with r32: WI income 32,608.81, which is AGI 50,580.71, less taxable SS 16,846, less 30% of 3,753. Taxable income is 19,197.46, tax is 708.70, less the 300 school property tax credit, giving 408.70.\n- The filer elects line 16 (24,000): income 8,608.81 is below the 13,960 standard deduction plus 950 exemptions, so taxable income is 0, tax is 0 and no credits apply.\nThe engine port and the engine-free recompute agree. If the record is refreshed, its engine value becomes 464.1828 (2.15.17 + conventions) and the alternative stays 0.0." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "board": 843.661499, + "latest": 895.81311, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "The r32 defect is still in 2.15.17 (wi_capital_gain_loss_subtraction.py is unchanged and ignores the 1,170 of distributions). The 895.81 is 1.755.4+c_wi+r04 exactly. The law-correct value is unchanged at 878.515747:\n- WI income is 36,182.56 less 351 = 35,831.56.\n- The standard deduction is 12,074.61 and the exemption 700, so taxable income is 23,056.95.\n- Tax is 528.85 + 4.4% x 7,946.95 = 878.52.\nThe engine and the recompute agree. One caveat on the alternative only: the engine does not model the WI Schedule SB line 6 medical care insurance subtraction. The listed $1,800 of premiums (has_esi) would qualify only if paid after tax, which the facts do not say." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "scenario_091: the claim that 'the law-correct value is unchanged at 878.515747' is overstated, and so is confidence 'high' for it. Wis. Stat. 71.05(6)(b)42, which covers tax years after 2012 and so is pre-freeze, lets an employee whose employer pays part of the cost subtract 100% of the medical care insurance premiums the employee paid, less any amount already deducted in federal AGI. The 2025 Schedule SB instructions (I-0104, PDF created 2025-11-07), line 6, Worksheet 2, implement this for employees and exclude only premiums paid pre-tax or through salary reduction. The prompt lists 'other health insurance premiums: $1,800' and, as a second line, 'health insurance premiums excluding Medicare Part B: $1,800'. Whether the premiums were paid pre-tax is not stated. The prompt's rule to treat any unlisted fact or boolean as false points toward after-tax. The engine's own AGI (36,182.56) does not reduce wages for any premium, so it too treats them as after-tax. policyengine-us 2.15.17 does not model this subtraction at all: the WI subtractions sources.yaml has no medical insurance entry, and no WI variable or parameter mentions medical care insurance. By engine-free recompute on the published 2026 parameters, r32 plus a 1,800 subtraction gives 789.81 (my_recompute.out), 88.70 below 878.52, and summing both premium lines (3,600) gives 701.11. So 878.515747 is correct only if the premiums were pre-tax, which is a fact no engine version can supply. The exclusion should stay, but the record should add this unlisted-input ambiguity (and the unmodeled provision) to the root cause, and it should not present 878.515747 as the unique law-correct value. The investigator raised this only as a caveat 'on the alternative only' and misstated the condition as depending only on after-tax payment, without citing 71.05(6)(b)42.", + "Minor, no outcome effect: in upstream_changes, 94e40f1471 is described as projecting the 2026 WI standard deduction (13,870). In fact the commit only restructures the uprating-rounding metadata on the WI standard deduction phase-out thresholds and the rate bracket thresholds. deductions/standard/max.yaml is byte-identical between 1.755.4 and 2.15.17 and no commit in 06665727d8..79be99f671 touches it. The raw-engine numbers the investigator quotes (468.83 / 900.46; std ded 12,220.63 = 13,870 - 12% x (33,734.71 - 19,990)) are right. c_wi_published_2026 overrides these values anyway.", + "Minor, no outcome effect: the current text of 71.05(6)(b)54m.d (database published 9-4-26) forfeits 'any credit, including any eligible carryover of such credit, listed under s. 71.07'. The text created by 2025 Act 15 section 220 (enacted and published 2025-07-03) forfeited 'any credit listed under s. 71.10 (4)'. The provision was amended after Act 15, and I did not pin down which act did it. For scenario_042 the elected path gives tax 0 before any credit, and standard-path refundable credits are 0, so the choice of forfeited credits does not change the 0.0 alternative." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Both defects are still present in policyengine-us 2.15.17, and I confirmed each directly.\n- diff -rq shows wi_capital_gain_loss_subtraction.py and wi_income_tax_before_refundable_credits.py byte-identical to 1.755.4, and git log 06665727d8..79be99f671 shows no commit touching either file.\n- My probe of 2.15.17 plus latest_conventions shows non_sch_d_capital_gains 3,753 in federal AGI (50,580.71; net_capital_gain 4,681) but wi_capital_gain_loss_subtraction 0.\n- The probe also shows wi_income_tax 0 (the min with the line-16 exclusion path, since wi_retirement_income_exclusion_amount is 24,000 and wi_retirement_income_exclusion_tax is 0) while wi_income_tax_before_refundable_credits is 464.1828.\nHarness runs on 042/091:\n- latest_conventions: 464.1828.\n- Plus r32: 408.698425.\n- Plus r32 and the r06 elected-path port: 0.0.\nAn engine-free recompute on the 2026 1-ES parameters reproduces 464.1827 (standard path, no r32), 408.6984 (with r32) and 0.0 (elected line 16). 71.05(6)(b)54m lets a filer aged 67 or older subtract up to $24,000 of qualified-plan and IRA distributions, and 54m.d forfeits the credits. On the elected path, WI income 8,608.81 is below the 13,960 standard deduction plus 950 exemptions, so tax is 0, and 0 is below the standard path's net 408.70, so the filer elects it. The 0.0 holds under every alternative reading I tried:\n- only the 19,200 IRA counts as line 16 (WI income 14,534.71 or 13,408.81, both below 14,910);\n- the farm loss is double-counted;\n- qualified_bdc_income is added.\nr06 alone suffices; r32 does not change the alternative. #9122 does not apply (salt_refund_income is 0). If the record is refreshed, engine value 464.1828 and alternative 0.0." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "The r32 defect is still present in 2.15.17. The subtraction file is byte-identical to 1.755.4 and no commit in the range touches it. My probe on 2.15.17 plus conventions shows non_sch_d_capital_gains 1,170 in AGI 36,182.56 but wi_capital_gain_loss_subtraction 0.\nThe harness gives 895.81311 with latest_conventions and 878.515747 with r32 added; the engine-free recompute gives 895.8131 and 878.5158. 71.05(6)(b)9 subtracts 30% of long-term gain, IRC 852(b)(3)(B) treats capital gain dividends as long-term, and 2025 SB line 5 applies the 30% exclusion to distributions reported without Schedule WD.\nThere is a second, independent reason to keep this excluded. 2.15.17 does not model the WI medical care insurance subtraction: 71.05(6)(b)42, which covers an employee whose employer pays part of the cost, and 2025 SB line 6, Worksheet 2. The prompt lists $1,800 of other health insurance premiums (repeated as 'health insurance premiums excluding Medicare Part B') without saying whether they were paid pre-tax. The law-correct value is:\n- 878.52 if the premiums were pre-tax;\n- 789.81 if the $1,800 was paid after tax, the reading the prompt's unlisted-boolean-false rule and the engine's own AGI point to;\n- 701.11 if a reader sums both lines.\nThe exclusion record should list root causes r32_wi_capital_gain_distributions plus this unlisted pre-tax-premium ambiguity, and it should label 878.515747 as the r32-only correction, not the unique law-correct value." + } + ], + "what_i_checked": "Sandbox and evidence standard from LATEST_BRIEF.md; all scratch is in this workspace.\n\n(1) Scenario JSON from the bundle, and the exact no-tools prompts rendered with policybench.prompts.make_no_tools_prompt (prompts_rendered.txt): input listing, labels, the unlisted-input rules, and the output definition.\n\n(2) Harness, sweep_latest.py on scenario_042 and scenario_091 with the 2.15.17 venv:\n- no fix: 468.829163 / 900.459473 (rv_raw.csv);\n- fixes/latest_conventions.py: 464.1828 / 895.81311 (rv_conv.csv);\n- the investigator's check_r32_only_on_21517.py: 408.698425 / 878.515747;\n- check_r32_r06_on_21517.py: 0.0 / 878.515747 (rv_conv_r32*.csv).\nOnly these 2 of the 32 outputs for these scenarios move. I also diffed the investigator's full-bundle sweep_check_conv_r32_r06.csv against sweep/out/latest_conventions.csv: exactly these 2 of 1,984 outputs differ.\n\n(3) A probe of engine intermediates on 2.15.17, raw and with conventions (my_probe.py, my_probe_*_21517.json): AGI, taxable SS, non_sch_d_capital_gains, net_capital_gain, salt_refund_income (0 in both), wi_capital_gain_loss_subtraction (0 in both), wi_agi, standard deduction, taxable income, credits, wi_income_tax (0 for 042), and the line-16 amount (24,000) and exclusion-path tax (0).\n\n(4) Engine source:\n- diff -rq of variables/gov/states/wi and parameters/gov/states/wi between the 1.755.4 and 2.15.17 venvs; read 2.15.17's wi_capital_gain_loss_subtraction, wi_income_tax_before_refundable_credits, wi_income_tax, wi_retirement_income_exclusion_tax, wi_standard_deduction, wi_retirement_income_exclusion_amount, the subtractions sources.yaml and state_income_tax_before_refundable_credits;\n- git log 06665727d8..79be99f671 on both defect files (no commits) and on WI tax code;\n- git show --name-only for 491642087f (#8839: gross income sources plus net_capital_gain), 35bbe1ba0c, 757437e4f2, 316e7832a1, 5b44285433, b45bd5d5d5, and 94e40f1471 (read its phase-out diff);\n- 491642087f is in 2.15.17 and not in 1.755.4; f3eec2d09e is not an ancestor of 79be99f671.\nThe existing 1.755.4 sweep CSVs (r04 469.69/901.32; r19 -5.51; cwi_plus_r04 464.18/895.81) support the decomposition. I did not rerun 1.755.4.\n\n(5) Primary sources fetched this session (src/):\n- Wis. Stat. 71.05(6)(b)9, 54m and 35 pages, and the chapter 71 PDF (database published 9-4-26), including (b)42;\n- 2025 Wis. Act 15 PDF: enacted and published July 3, 2025; SECTION 220 creates 54m;\n- 2025 Wis. Act 118 header: enacted 2026-03-27;\n- 2025 Schedule SB instructions I-0104 (PDF created 2025-11-07; Last-Modified 2025-11-07): line 5 exception, line 6, line 16;\n- 2026 Form 1-ES instructions D-101a (PDF created 2026-01-16): standard deduction table 13,960 less 12% over 20,120; exemptions 700 plus 250; brackets 15,110 / 51,950;\n- 26 U.S.C. 852(b)(3)(B) (Cornell LII).\nAll the operative law predates 2026-07-03.\n\n(6) Engine-free recompute (my_recompute.py/.out) of both outputs under the engine path, r32 and r06, plus alternative readings: IRA-only line 16, double-counted farm loss, qualified_bdc_income included, and the WI medical care insurance subtraction at 1,800 / 3,600 / with ESI." + } + }, + { + "id": "excl_r01_ira_compensation", + "title": "Excluded WI output (r01_ira_compensation) moved by c_wi_published_2026", + "outputs": [ + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 4605.99707, + "latest": 4598.476074, + "v11_1755": 4605.99707, + "excluded": true, + "exclusion_root_cause": "r01_ira_compensation", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r01_ira_compensation", + "classification": "excluded_defect_still_present", + "summary": "2.15.17 still has the r01 defect. traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, gov/irs/ald/deductions.yaml and above_the_line_deductions.py are byte-identical to 1.755.4, and no commit in 06665727d8..79be99f671 changes their formulas; upstream/main is 79be99f671 itself. On 2.15.17 the dependent's $18.032 traditional IRA contribution is still deducted on the parents' joint return, the dependent has $0 compensation but a $7,000 limit, and ALD (19,926.22) and AGI (114,403.37) match 1.755.4 to float precision. The raw +2.95 comes entirely from Wisconsin parameters: cd4d339667 adds DOR's published 2026 brackets (+2.958) and 94e40f1471 rounds the projected standard-deduction phase-out start to $10 (-0.007). c_wi_published_2026 overrides both, so 2.15.17 plus the conventions gives 4,598.476074, exactly what 1.755.4 plus r19_wi gives. Adding r01_ira_compensation_v2 to latest_conventions on 2.15.17 reproduces all three r01 exclusions' alternative values exactly (scenario_064 state 4,599.621582, 064 federal 4,441.455078, 003 federal 22,400.654297) and moves no other output by more than $0.09. An engine-free recomputation with DOR's 2026 MFJ tables gives 4,598.48 with the defect and 4,599.62 without it. 2.15.17's value is $1.15 off the law-correct value, beyond the $1 tolerance, so the output stays excluded. Scoring is unaffected. The exclusion record's frozen_value and engine_version should follow the regenerated reference, and its alternative_value stays 4,599.621582.", + "upstream_changes": [ + { + "commit": "cd4d339667", + "date": "2026-09-03", + "what": "Validate state uprating projections: adds explicit 2026 Wisconsin bracket thresholds (MFJ 20,150 / 69,260 / 443,630, citing the 2026 Form 1-ES instructions) in place of 1.755.4's CPI projections (20,023.74 / 69,714.93 / 440,829.08). This is +2.958 of the raw move. The values equal DOR's pre-freeze publication, and latest_c_wi_published_2026 asserts them." + }, + { + "commit": "94e40f1471", + "date": "2026-09-01", + "what": "Fix state parameter uprating rounding metadata: the $10 rounding now applies to Wisconsin's projected standard-deduction phase-out start (MFJ 28,849.32 becomes 28,850). This is -0.007 of the raw move. latest_c_wi_published_2026 overrides it with DOR's published 29,040 and the 25,840 maximum." + }, + { + "commit": "5d40347f7c", + "date": "2026-09-03", + "what": "Fix Wisconsin 2026 tax form links (reference URLs only; no value change)." + }, + { + "commit": "328b34d273", + "date": "2026-09-12", + "what": "Sorts the other-ALD list in adjusted_gross_income_person and student_loan_interest_ald_magi. It does not change the tax-unit above_the_line_deductions (adds = gov.irs.ald.deductions) and does not fix r01." + }, + { + "commit": "44001a4c24", + "date": "2026-09-18", + "what": "Adds an uprating series to traditional/roth_ira_contributions_desired. This only affects dataset uprating, not a household with stated 2026 inputs, and does not fix r01." + }, + { + "commit": "none (06665727d8..79be99f671)", + "date": "2026-07-02..2026-09-28", + "what": "No commit changes traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, limit/ira.yaml, catch_up/limit/ira.yaml or the ALD deductions.yaml IRA entries. git log -S ira_compensation is empty. 2.15.17 still projects the 2026 IRA limit at 7,000 with a 1,000 catch-up, has no compensation cap, and has no head/spouse mask." + } + ], + "law": "Federal (root cause r01). 26 U.S.C. 219(a), 219(b)(1)(B), 219(c) and 219(f)(1) come from LII (law.cornell.edu/uscode/text/26/219), fetched 2026-09-29 UTC. The deduction is for \"the qualified retirement contributions of the individual\". It is capped at the lesser of the deductible amount and the compensation includible in the individual's gross income. The spousal rule in (c) covers only a spouse filing a joint return. The latest amendment LII lists is Pub. L. 117-328 (Dec 2022), so the text predates the 2026-07-03 freeze. IRS Notice 2025-67 (irs.gov/pub/irs-drop/n-25-67.pdf; PDF created 2025-11-13, server Last-Modified 2025-11-13) raises the 2026 219(b)(5)(A) amount to $7,500 and the catch-up to $1,100. 2.15.17 still uses 7,000/1,000, though the dollar limit does not bind here. Under these provisions a dependent with $0 compensation has no deduction, and in any case not on the parents' joint return.\n\nWisconsin. Wis. Stat. 71.01(13) defines Wisconsin adjusted gross income as federal AGI with the 71.05 modifications. That was read on docs.legis.wisconsin.gov, which serves the 2023-24 consolidation published 2026-09-04; the definition itself is long-standing. The 2025 Form 1 instructions (PDF created 2025-10-17) say line 1 is federal AGI from Form 1040 line 11, so the federal IRA error flows into Wisconsin income.\n\nThe Wisconsin 2026 figures come from DOR's 2026 Form 1-ES instructions, D-101A (R. 1-26), revenue.wi.gov/TaxForms2026/2026-Form1-ES-Inst.pdf, fetched this session. The PDF was created 2026-01-16 and modified 2026-01-20, and the server Last-Modified is 2026-02-06, all before the freeze. Its figures:\n- MFJ standard deduction: $25,840, less 19.778% of income over $29,040.\n- Exemptions: $700 each.\n- Schedule B: 3.5% up to $20,150; $705.25 + 4.4% up to $69,260; $2,866.09 + 5.3% up to $443,630.", + "per_output": [ + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "board": 4605.99707, + "latest": 4598.476074, + "proposed_reference": 4598.476074, + "action": "keep_excluded", + "reason": "Root cause r01_ira_compensation is still present in 2.15.17: the dependent's $18.032 IRA contribution is deducted on the parents' joint return, and the IRA code is unchanged in range. The 4,598.476074 comes from c_wi_published_2026 (pre-freeze DOR tables) applied to the still-defective AGI, and it equals 1.755.4 + r19_wi exactly. The law-correct value is 4,599.621582. It is reproduced on 2.15.17 by latest_conventions + r01_ira_compensation_v2, matches the exclusion's alternative_value exactly, and the engine-free check gives 4,599.62. The gap is $1.15, above tolerance. The proposed reference is what regeneration (2.15.17 + conventions) stores for this unscored output. Update the exclusion record's frozen_value to 4598.476074 and engine_version to 2.15.17. Leave alternative_value at 4599.621582. Keeping the board's v1.1 value (4,605.99707) instead would also leave scoring unchanged." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "The law claim is overstated. The investigator calls 4,599.62 the law-correct value and says 2.15.17 is $1.15 off it, beyond tolerance. That is only true under the board's formula convention. Wis. Stat. 71.05(22)(dp)2 (docs.legis.wisconsin.gov, fetched this session, review/wis_71_05_22.html) says the standard deduction \"shall be determined\" under a DOR table published in the instructional booklets. DOR's 2025 Form 1 table (booklet PDF created 2025-10-17, pp. 35-37) equals round(formula at the bracket midpoint) on all 242 MFJ $500 rows (checked in the review). Building the 2026 table the same way from the pre-freeze 1-ES parameters (25,840 less 19.778% over 29,040) puts the defective AGI (114,403.38) and the law-correct AGI (114,421.41) in the same 114,000-114,500 row, with a standard deduction of 8,987. On that reading: r01's effect on this output is $0.96, inside tolerance. The table-law value is about 4,597.83. The regenerated reference 4,598.476 is $0.64 from it. The exclusion's alternative_value 4,599.62 is $1.79 from it (review/table_variant.py and .txt). The 2026 Form 1 table was not published before the 2026-07-03 freeze, so this reading is an inference. It does not overturn keep_excluded, because the defect is unfixed and a reference that differs by reading favors exclusion. But the exclusion record should say that the $1.15 gap and the alternative_value hold under the board's formula convention (c_wi_published_2026), not under statute. Whether Wisconsin's table or formula governs is a board-wide convention question that touches every WI output. It is not specific to this cluster.", + "The hand check is described as \"engine-free ... with DOR's 2026 MFJ tables\". In fact hand_check.py takes AGI from the engine (114,403.367188), and it uses the 1-ES formula schedules, not a Form 1 table. I derived AGI from the stated facts without the engine: wages 97,295 - 401(k) 2,315.40 + partnership 19,350 + 401(k) distributions 20,000 - farm loss 16,800 - capital loss capped at 3,000 - IRA 108.192 - 18.032 = 114,403.376. It matches, so the numbers stand: formula tax 4,598.4769 with the defect and 4,599.6216 without it (review/indep_calc.py).", + "upstream_changes leaves out several commits in range that touch Wisconsin tax or ALD code: 35bbe1ba0c, 84b94fc350 and 372228938d (2026-07-05), f73485d33c (2026-09-14) and 6d60c7944a (the ALD loss-threshold change of 2026-09-28). None affects this household. AGI, WI subtractions and WI credits (both 0) are identical on the two engines, and the raw +2.9507 move breaks down fully into brackets (+2.9580) and rounding (-0.0071), which I recomputed. The omission is harmless but the list is incomplete.", + "Which value to store is a convention choice. The Sept 22 revisions set excluded_outputs_untouched=true, so the board kept raw engine values for excluded outputs (4,605.997 here). The investigator proposes 2.15.17 + conventions (4,598.476074) rather than raw 2.15.17 (4,608.947754). That matches the brief (new references = 2.15.17 + conventions), and scoring is unaffected either way, but the regeneration must treat every excluded output the same way." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 4598.476074, + "reason": "Root cause r01 is still present in 2.15.17. traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, above_the_line_deductions.py and gov/irs/ald/deductions.yaml have identical blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. The dependent's $18.03 IRA contribution (stated in the prompt; $0 wages) is still in the joint return's ALD (19,926.22 = 19,800 loss ALD + 108.19 + 18.03). Under 26 U.S.C. 219(a), (b)(1)(B) and (c) (LII, re-fetched this session) it is not deductible there. The review harness gives 2.15.17 + latest_conventions = 4,598.476074, which equals 1.755.4 + r19_wi_convention (re-run this session). Adding r01_v2 gives 4,599.621582, and also reproduces the 064 federal (4,441.455078) and 003 federal (22,400.654297) alternative values exactly; the only other output it moves is scenario_085 federal, by +0.089. Under the board's formula convention the defect moves this output by $1.15, beyond tolerance, so it stays excluded. Caveat: Wis. Stat. 71.05(22)(dp)2 makes DOR's $500-bracket table determinative. On that reading r01's effect is $0.96, and 4,598.476 is $0.64 from a table-law value of about 4,597.83, but the 2026 table was not published before the freeze. Exclusion is the safe call under either reading. Update frozen_value to 4598.476074 and engine_version to 2.15.17. Keep alternative_value at 4599.621582, and note in the exclusion record that it is formula-convention based." + } + ], + "what_i_checked": "What I checked:\n- Harness runs. I ran the harness (sweep_latest.py, 2.15.17 venv) three times over all 1,984 outputs: baseline (41 moved), latest_conventions (29 moved), and latest_conventions + r01_ira_compensation_v2 (31 moved). Outputs are in triage/latest/excl_r01_ira_compensation/review/rv_*.csv.\n - The results match sweep/out/latest_conventions.csv and the investigator's probe CSV to 0.0.\n - scenario_064 state: raw 4,608.947754, conventions 4,598.476074, conventions + r01_v2 4,599.621582.\n - Collateral moves from r01_v2: scenario_003 federal +245.96, scenario_064 federal +2.17, scenario_085 federal +0.089.\n - I also re-ran 1.755.4 + r19_wi_convention (sweep.py) and got 4,598.476074.\n- Upstream code. The IRA and ALD files have identical git blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. `git log -S ira_compensation` over the range is empty. I read the installed 2.15.17 source:\n - the IRA limit is 7,000 + 1,000 with no compensation cap;\n - the ALD list includes traditional_ira_contributions;\n - above_the_line_deductions adds over the whole tax unit.\n- Wisconsin commits. I read cd4d339667 (WI brackets 20,150 / 69,260 / 443,630, citing the 2026 1-ES) and 94e40f1471 (rounding metadata). Recomputed, the raw move splits into +2.9580 from brackets and -0.0071 from rounding.\n- Scenario and prompt. I read scenario_064's JSON and rendered its prompt; the dependent's $18 traditional IRA contribution is stated, and no wages are listed.\n- Primary sources, re-fetched this session:\n - Wisconsin 2026 1-ES instructions: sha256 identical to the investigator's copy. PDF created 2026-01-16, modified 2026-01-20, server Last-Modified 2026-02-06. The MFJ standard deduction (25,840 less 19.778% over 29,040), Schedule B and the $700 exemptions match.\n - 26 U.S.C. 219 on LII: (a), (b)(1)(B), (c) and (f)(1) text confirmed; the latest amendment listed is Pub. L. 117-328.\n - IRS Notice 2025-67: PDF created 2025-11-13; it gives $7,500 / $1,100 and a 2026 joint 219(g) range of $129k-$149k, which does not bind here.\n - Wis. Stat. 71.01(13): Wisconsin AGI is federal AGI with modifications.\n - Wis. Stat. 71.05(22)(dp)2: the standard deduction is determined under DOR's published table. This is new evidence from this review.\n- Independent calculation. I recomputed AGI from the stated facts without the engine (114,403.376), then the WI tax: 4,598.4769 with the defect and 4,599.6216 without it (formula).\n- Table check. I verified that DOR's 2025 MFJ standard-deduction table equals the formula at each bracket midpoint on 242 rows. Applied to 2026, the table reading gives a $0.96 r01 effect, and the reference is $0.64 from a table-law value of about 4,597.83.\n- Verdict. keep_excluded and 4,598.476074 are sound. The investigator's statement of the Wisconsin law needs the table caveat.\n\nFiles are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/excl_r01_ira_compensation/review/: indep_calc.py and .txt, table_variant.py and .txt, rv_baseline.csv, rv_conventions.csv, rv_conv_r01v2.csv, rv_1755_r19wi_064.csv, wi1es.pdf, lii219.html, n2567.pdf, and wis_71_05_22.html." + } + }, + { + "id": "excl_niit_scope_020", + "title": "Excluded TX federal output (NIIT-scope reading) moved by the sales tax deduction", + "outputs": [ + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "state": "TX", + "board": 68056.710938, + "latest": 68334.734375, + "v11_1755": 68056.710938, + "excluded": true, + "exclusion_root_cause": "unlisted:whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_niit_scope_020", + "classification": "unlisted_input_unchanged", + "summary": "The move comes from the IRS sales tax table. The NIIT question is untouched. 1.755.4 gave a single Texas filer at $300,000 or more a 2026 state amount of 2,714.01. That is 2,440 projected by CPI, and 2,440 is Tennessee's 2023 row, the column shift that #9616 (3f029abf5b, 2026-09-27) fixed. 2.15.17 carries the IRS 2025 cell of 1,595, uprated to 1,631.15 for 2026; the c_irs_sales_tax_2025 convention holds it at 1,595. Local tax is still 20% of the state amount for TX (Texas is not one of #9616's ten zeroed jurisdictions). Sales tax therefore falls from 3,256.81 to 1,957.38 raw, or 1,914.00 with the conventions, and SALT drops from 17,090.34 to 15,790.91 or 15,747.53. Both are below the $16,100 standard deduction plus the $196 non-itemizer charity deduction under 170(p), so the filer stops itemizing and taxable income goes from 283,059.84 to 283,854.19. The same convention already produced 68,334.73 on 1.755.4 (out/r19_irs_sales_tax_convention.csv), but the board skipped it because the output is excluded (meta.json c_irs_sales_tax_2025: \"excluded_outputs_untouched\": true). income_tax_before_refundable_credits.py is byte-identical in the two engines and still adds net_investment_income_tax. On 2.15.17, conventions plus r25 give an alternative of 68,112.09375, exactly the exclusion record's alternative_value, 222.64 of NIIT below the reference, so the exclusion still bites and stays. An engine-free recompute from the Rev. Proc. 2025-32 single schedule, IRC 1411 and the IRS 2025 TX table gives 68,334.74 and 68,112.09. The standard deduction wins for any Texas local rate from 0% to the 2% cap in Tex. Tax Code 321.101(f). (Aside: the reference also takes $5,708.82 of alimony above the line even though no instrument date is stated. That is moot here, because the output stays excluded.)", + "upstream_changes": [ + { + "commit": "3f029abf5b", + "date": "2026-09-27", + "what": "#9616 (Fixes #9595) corrected the three-column shift in the Optional State Sales Tax Table and added the IRS 2022-2025 tables. For TX at size 1 and $300k+, 1.755.4 had 2,440 in 2023, which is Tennessee's 2023 row; IRS 2023 TX is 1,992 and IRS 2025 TX is 1,595. 2026 is uprated from 2025 (1,631.15), and c_irs_sales_tax_2025 holds it at 1,595. It also zeroed local_sales_tax in CT, DC, IN, KY, MA, MD, ME, MI, NJ and RI (not TX, which keeps local = 0.2 x state) and replaced the income-bracket metadata with the thresholds unchanged. This is the cause of the move: sales tax 3,256.81 -> 1,957.38 raw (1,914.00 with the conventions), and the filer stops itemizing." + }, + { + "commit": "81fd50a98a", + "date": "2026-08-19", + "what": "#9306 added passive partnership/S-corp income to the NIIT base. No effect here: net_investment_income is 5,859 and NIIT is 222.64 in both engines." + }, + { + "commit": "none (no change)", + "date": "2026-07-02..2026-09-28", + "what": "variables/gov/irs/tax/federal_income/income_tax_before_refundable_credits.py is byte-identical in 1.755.4 and 2.15.17 and still adds net_investment_income_tax, so the r25 NIIT-scope ambiguity is unchanged." + } + ], + "law": "Sales tax figure: IRS, 2025 Instructions for Schedule A (Form 1040), https://www.irs.gov/pub/irs-pdf/i1040sca.pdf. The copy fetched 2026-09-28 is byte-identical (sha256 b0999b12...) to the 2026-09-25 copy; PDF created 2025-12-08, modified 2025-12-18, so it predates the freeze. In the Optional State Sales Tax Table (p.15), Texas is 6.25% and '$300,000 or more', family size 1, is $1,595. In the worksheet, TX is neither one of the ten no-local jurisdictions named after line 1 nor a line-2 local-table state, so local = line 1 x (local rate / 6.25). Table income is line 11b AGI (300,150.19) plus nontaxable items (none), which puts the filer in the $300,000+ row. Tex. Tax Code 321.101(f) (via texas.public.law) caps combined local tax at 2%, so the worksheet total is at most 2,105.40. Rates and deductions: Rev. Proc. 2025-32 (IRB 2025-45, 2025-11-03) gives the 2026 single Table 3 (35% bracket from 256,225, base 58,448), a $16,100 standard deduction and a $49,450 0% capital gains maximum. 26 U.S.C. 1411 (P.L. 111-152, 2010-03-30): 3.8% of the lesser of NII (5,859) and MAGI over $200,000, which is 222.64. P.L. 119-21 (2025-07-04): 26 U.S.C. 170(p), a $1,000 non-itemizer cash charity deduction (196 used); 170(b)(1)(I), the 0.5% floor for itemizers; 164(b)(7), the 2026 SALT cap of $40,400 (not binding); 225, the overtime deduction, phased to 0 at MAGI of about 300k; 67, miscellaneous itemized deductions permanently denied. The NIIT-scope question is the prompt's output definition (tax after nonrefundable credits, Form 1040 line 22, vs Schedule 2 Part II), which no statute resolves, so it stays an unlisted input.", + "per_output": [ + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "board": 68056.7109375, + "latest": 68334.734375, + "proposed_reference": 68334.734375, + "action": "keep_excluded", + "reason": "The exclusion root cause, r25_niit_in_federal_output, is an unlisted input: whether the output includes the IRC 1411 NIIT. It is unchanged in 2.15.17, and the NIIT-excluded alternative recomputed on 2.15.17 with latest_conventions plus r25 is 68,112.09375, identical to the exclusion record's alternative_value and 222.64 below the reference. The row stays excluded. Its stored reference value should become 68,334.734375: 2.15.17 carries the IRS 2025 sales tax table and fixes the column-shift defect (#9616), both pre-freeze, and the filer stops itemizing. 1.755.4 plus c_irs_sales_tax_2025 gives the same value; the board left it out only because the row was excluded. Raw 2.15.17 equals 2.15.17 with the conventions, so no hold is needed. An engine-free recompute gives 68,334.74 for any Texas local rate from 0% to 2%." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "Claude Code Workflow reviewer (Opus 5.5), 2026-09-28", + "agree": true, + "problems": [ + "Not a problem with the verdict: the investigator's note that alimony is moot understates how close this row is to a second unlisted input. The prompt lists 'first home mortgage balance: $367,000' but no mortgage interest. On 2.15.17 the filer takes the standard deduction by only $548.47 (itemized 15,747.53 vs 16,296 standard). A reader who imputes interest over about $549 would itemize. Like the undated alimony instrument (worth +1,998.09 if it is not deductible; reviewer recompute 70,332.82), this is moot only because the row stays excluded. No other scored scenario_020 output depends on AGI or itemizing: TX state tax is 0, payroll is 16,689, and every benefit is 0.", + "A third reading exists that the exclusion text does not name. PolicyEngine's income_tax_before_refundable_credits adds NIIT but not the 0.9% Additional Medicare Tax (900; it sits in payroll_tax, reference 16,689). So the reference (68,334.73) is neither Form 1040 line 22 (68,112.09) nor 'all Schedule 2 income-type taxes' (69,234.73). This supports keeping the exclusion and does not change it.", + "Minor citation detail left unverified: 'IRB 2025-45, 2025-11-03' for Rev. Proc. 2025-32. The irs.gov PDF was created 2025-10-17 and refers to amendments 'after October 9, 2025', so it predates the 2026-07-03 freeze either way. This does not affect the outcome." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 68334.734375, + "reason": "Confirmed. On 2.15.17, sweep_latest.py gives 68334.734375 both raw and with latest_conventions.py. latest_conventions plus r25_niit_excluded gives 68112.09375, exactly the exclusion record's alternative_value, so the two readings still differ by 222.64 (more than the $1 tolerance). The r25 root cause is the benchmark_specs.json output definition ('federal individual income tax after nonrefundable credits and before refundable credits'), which is unchanged. The engine formula (income_tax_before_refundable_credits.py; git diff 06665727d8..79be99f671 is empty; cmp of the installed files is identical) still adds net_investment_income_tax, and net_investment_income_tax.py has the same sha1 in both engines. The move comes only from #9616 (3f029abf5b, 2026-09-27), which fixed the column-shifted sales tax table. The IRS figure it carries, the 2025 TX size-1 $300k+ cell of 1,595, was published in December 2025, before the freeze. No hold module is needed, because raw and conventions agree." + } + ], + "what_i_checked": "Harness: I re-ran sweep_latest.py on scenario_020 (2.15.17 venv) four ways: no fix, fixes/latest_conventions.py, the investigator's alt_conventions_r25.py, and r25_niit_excluded.py alone. Results were 68334.734375, 68334.734375, 68112.09375 and 68112.09375. The other 15 scenario_020 outputs did not move. CSVs are in latest/excl_niit_scope_020/review/sweep_*.csv. The existing out/r19_irs_sales_tax_convention.csv (1.755.4 plus r19) also gives 68334.734375.\n\nRecords: reference_exclusions.json has scenario_020 with reason reference_depends_on_unlisted_input, frozen 68056.71 and alternative 68112.09375. In meta.json, c_irs_sales_tax_2025 has excluded_outputs_untouched=true and changed only scenario_000. root_causes.json lists r25 as class unlisted_input. The prompt definition in benchmark_specs.json does not mention NIIT. I rendered the scenario_020 prompt with describe_household.\n\nEngine: I read the parameters in both venvs. 1.755.4 has TX size 1, bracket 19 at 2440 for 2022-23, and 2440 \u00d7 176.713/158.872 = 2714.01 in 2026. 2.15.17 has TX 2023 = 1992, TN 2023 = 2440, TX 2025 = 1595, and 2026 = 1631.15, uprated from 2025. The no-local list is CT, DC, IN, KY, MA, MD, ME, MI, NJ, RI, so TX is not on it.\n\nUpstream: I read the git show for 3f029abf5b (#9616, 2026-09-27, Fixes #9595) and 81fd50a98a (#9306, 2026-08-19).\n\nPrimary sources:\n- IRS 2025 Instructions for Schedule A, local copy with sha256 b0999b12\u2026 (identical to a second copy fetched separately). Created 2025-12-08, modified 2025-12-18. The table row TX '$300,000 or more' has size 1 = 1595. In the worksheet, TX is neither a line-2 local-table state nor one of the ten jurisdictions sent to -0-, so local = line 1 \u00d7 rate/6.25. Table income is line 11b.\n- IRS 2023 i1040sca, fetched from irs.gov/pub/irs-prior (created 2024-01-03). TN size 1 $300k+ = 2440 and TX = 1992, which confirms the column shift.\n- Rev. Proc. 2025-32 from irs.gov (PDF created 2025-10-17). Table 3 single: $58,448 plus 35% over $256,225. Standard deduction $16,100. Maximum 0% rate amount $49,450; 15% up to $545,500. AMT exemption $90,100.\n- 26 U.S.C. 170(p): $1,000 for non-itemizers, from uscode.house.gov. 170(b)(1)(I): the 0.5% floor.\n- 26 U.S.C. 225(b): $12,500, reduced $100 per $1,000 of MAGI over $150,000, so it is 0 here.\n- Tex. Tax Code 321.101(f): 2% combined local cap.\n\nIndependent calculation: I wrote review/indep.py (base plus rate times excess, and a QDCG worksheet). It gives 68,334.736 and 68,112.094 at local rates of 0%, 1.25% and 2%. Itemizing would need about a 3.40% local rate. The 1.755.4 projection gives 68,056.72, which reproduces the board value. AMT is clearly 0: tentative minimum tax is about 54.6k against 68.1k regular tax." + } + }, + { + "id": "excl_snap_ssi_disability", + "title": "Excluded SNAP outputs (meets_ssi_disability_criteria unlisted): board keeps fractional v1.1", + "outputs": [ + { + "scenario_id": "scenario_023", + "variable": "snap", + "state": "CA", + "board": 461.339722, + "latest": 408, + "v11_1755": 461.339722, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "state": "LA", + "board": 2669.217041, + "latest": 2628, + "v11_1755": 2669.217041, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria", + "final_action": "keep_excluded" + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "state": "MT", + "board": 8625.889648, + "latest": 8556, + "v11_1755": 8625.889648, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_snap_ssi_disability", + "classification": "mixed", + "summary": "All three exclusions should stay, and 408, 2,628 and 8,556 are the right values for the reference file. The moves from the board's fractional values have two causes: the SNAP rounding fixes in #9318 (merged 2026-08-25), and the FY2026 hold on the FY2027 schedule that 2.15.17 encodes in #9623 (USDA memo 2026-08-21). An engine-independent recomputation reproduces all three values. One change affects the exclusions: PR #9345 (commit 339a1f1b03, merged 2026-09-08) now tests SNAP elderly-or-disabled status by SSI receipt (7 CFR 271.2) instead of by the SSI disability criteria. So on 2.15.17 plus the conventions, flipping meets_ssi_disability_criteria gives 408 and 8,556, not 3,576 and 8,844; 057 still gives 840. For 023 the unlisted input still matters under law, through a different route: the reading-B head gets no SSI but can only get Medi-Cal through a disability pathway (MAGI is 141.2% of FPL, above the 138% adult limit). Receiving that makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), which gives 3,576. 2.15.17 does not model this route, so the 3,576 comes from probe p_title19_readingB. For 100 the unlisted input no longer matters under either the engine or the law (no SSI because resources exceed the limit, and a MAGI parent Medicaid category), but r30 is still unfixed in 2.15.17: MT still gets the heat-and-eat SUA without an elderly or disabled member. The port gives 6,924 and moves only 080 (already excluded under r30, 3,240 reproduced) and 100, so 100's exclusion should be recorded under r30. Out of cluster: the scored scenario_023 head_medicaid_eligible (1) follows reading B, because ca_wdp_disability_eligible reads the broad is_disabled flag (same code in 1.755.4). Under reading A the law gives 0 (42 CFR 435.540(a)), so the lead should review it.", + "upstream_changes": [ + { + "commit": "5d88007d90 (merge of #9318: 8df8c5043a, 949546af53)", + "date": "2026-08-25", + "what": "snap_net_income rounded to the nearest dollar and snap_expected_contribution = ceil(30% of net), per 7 CFR 273.10(e)(1)(ii)(A) and (e)(2)(ii)(A). Moves 023 Jan-Sep from 35.20 to 34 (net 876.51 to 877, 263.1 to 264) and 100 from 713.90 to 713 (net 237, 71.1 to 72). No effect on 057 (net 1,090, 30% = 327 exactly)." + }, + { + "commit": "3c41c31457 (merge of #9162: cf165464a6)", + "date": "2026-07-28", + "what": "Minimum allotment rounded to $24 and income standards rounded up (r28, r31). Neither binds for 023, 057 or 100." + }, + { + "commit": "fa27adbffc (#9587)", + "date": "2026-09-28", + "what": "Half-up net income rounding and income standards for households over eight. No .5 ties and no large households here, so no effect." + }, + { + "commit": "d27d6c3cc3 (#9623)", + "date": "2026-09-27", + "what": "Encodes USDA's FY2027 SNAP COLA (memo dated 2026-08-21, after the freeze). Raw 2.15.17 gives Oct-Dec monthly SNAP of 52 (023), 237 (057) and 740 (100), for annual totals of 462, 2,682 and 8,637. The existing c_snap_hold_fy2026 in latest_conventions holds FY2026, giving 34, 219 and 713, so no new hold is needed. 1.755.4 had CPI-uprated projections for those months (48.18, 232.74, 733.60), which is where the board's fractional values come from." + }, + { + "commit": "f28aa73eff (#9090)", + "date": "2026-07-20", + "what": "SNAP uprating index updated with CPI-U actuals through June 2026. It affects only projections from October 2026, which #9623 supersedes and the hold covers." + }, + { + "commit": "eceb61bb6c (merge of #9345: 339a1f1b03 2026-08-24, e5433454d1 2026-09-03)", + "date": "2026-09-08", + "what": "disabled_programs changed from is_ssi_disabled (the criteria flag) to receives_ssi / ssi > 0 (7 CFR 271.2 receipt test). Stated-facts values are unchanged. 057's blind SSI-recipient head becomes an elderly-or-disabled member, with no numeric effect (medical expenses $3.33/mo, which is under $35, and no shelter costs). The reading-B values on 2.15.17 change from 3,576 to 408 for 023 and from 8,844 to 8,556 for 100. 2.15.17 still omits the 2012(j)(2)(B) title XIX route." + } + ], + "law": "All sources are pre-freeze unless noted.\n- 7 CFR 273.10(e)(1)(ii)(A), (e)(2)(ii)(A), (e)(2)(ii)(C): eCFR current text checked 2026-09-28. The section's source note dates to 43 FR 47889 (1978).\n- FNS \"SNAP FY 2026 Maximum Allotments and Deductions\" PDF, effective 2025-10-01 to 2026-09-30, read this session (saved as fns_snap_fy26_maximum_allotments_deductions.pdf): $298/$546/$785 maximum allotments, $209 standard deduction, $744 shelter cap. FNS FY2026 COLA memo dated 2025-08-13 (captured page): $24 minimum. FY2027 memo dated 2026-08-21 (#9623; FNS page updated 2026-08-28): after the freeze, so it is held.\n- 7 U.S.C. 2012(j)(2) (uscode.house.gov, current through P.L. 119-21) and 7 CFR 271.2 \"Elderly or disabled member\" (eCFR, Amdt. 132, 43 FR 47882, 1978): status comes from receiving SSI, a 1616(a) supplement, or disability-related medical assistance under title XIX when that eligibility rests on criteria at least as stringent as title XVI. 42 CFR 435.540(a) (eCFR) makes Medicaid use the SSI definition of disability.\n- 7 U.S.C. 2014(e)(6)(C)(iv)(I), as amended by P.L. 119-21 sec. 10103(a) (congress.gov text; approved 2025-07-04; no separate effective-date clause): a LIHEAP payment confers the SUA only on households with an elderly or disabled member.\n- SSA \"Cost-of-Living Increase and Other Determinations for 2026\", FR 2025-19763, published 2025-11-03 (govinfo): SSI federal benefit rate $994 individual and $1,491 couple.", + "per_output": [ + { + "scenario_id": "scenario_023", + "variable": "snap", + "board": 461.339722, + "latest": 408, + "proposed_reference": 408, + "action": "keep_excluded", + "reason": "The stated facts (reading A) give $34/mo x 12 = 408, confirmed by hand: gross $2,120.22, net $876.51 rounds to $877, 30% rounds up to $264, and $298 - $264 = $34. The $744 cap binds with or without the SUA. The unlisted input still decides the output under law. Under reading B, SSI is $0 (countable income $1,340.94 exceeds the $994 FBR), and the head's only Medi-Cal is a disability pathway (engine category SENIOR_OR_DISABLED; MAGI is 141.2% of FPL, above the 138% adult limit). Receiving it makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), so shelter is uncapped, net income is $0, and the benefit is $298 x 12 = 3,576. 2.15.17 omits that route, so flipping meets_ssi_disability_criteria alone gives 408. Keep the alternative value at 3,576, but reword the record: SSI-disability criteria no longer confer SNAP status; the route is Medi-Cal receipt. The 3,576 needs a SUA of at least $384 (engine CA SUA: $663)." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "board": 2669.217041, + "latest": 2628, + "proposed_reference": 2628, + "action": "keep_excluded", + "reason": "Reading A gives $219 x 12 = 2,628, recomputed independently. The head's SSI is $994 - $391.42 = $602.58/mo, gross is $1,472.42, net is $1,090, 30% is $327, and $546 - $327 = $219. Under reading B the spouse is an eligible spouse and the couple gets $1,491 - $392.42 = $1,098.58/mo. SNAP counts that income: net is $1,586, 30% is $476, and $546 - $476 = $70, which gives 840. That matches the record's alternative value and the 2.15.17 sweep. The unlisted input still moves the output through actual SSI receipt, so the exclusion and its alternative value (840) stand unchanged." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "board": 8625.889648, + "latest": 8556, + "proposed_reference": 8556, + "action": "keep_excluded", + "reason": "2.15.17 plus the conventions gives $713 x 12 = 8,556, with the $799 MT SUA as the only shelter cost. The unlisted input no longer moves the output. Under reading B, SSI is $0 because $2,800 in resources exceeds the $2,000 limit, and the Medicaid category is PARENT (MAGI), so there is no 7 CFR 271.2 receipt route. 1.755.4's 8,844 was an artifact of the criteria-based definition that #9345 removed. The r30 defect is still unfixed in 2.15.17: snap_state_using_standard_utility_allowance.py has no elderly-or-disabled gate, and always_standard is still True for MT. The household has no elderly or disabled member and no stated utility or rent costs, so under 7 U.S.C. 2014(e)(6)(C)(iv)(I) as amended by P.L. 119-21 sec. 10103(a) it gets no SUA. That gives net $691, 30% of $208, $785 - $208 = $577, and $577 x 12 = 6,924 under both readings, matching the r30 port on 2.15.17 and the hand computation. Keep the output excluded, but re-record its root cause as r30_snap_heat_and_eat_sua (reference_engine_defect), with alternative value 6,924." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": false, + "problems": [ + "scenario_100: the claim that the unlisted input 'no longer matters under either the engine or the law' is refuted for the law. It holds only in the engine. The PARENT category that the settlement relies on is 2.15.17's own category hierarchy, and 2.15.17 does not model Montana's buy-in: is_working_disabled_buy_in_for_medicaid covers only CA WDP, IL HBWD and MS WD. Montana runs Medicaid for Workers with Disabilities (MWD), a title XIX group under 42 USC 1396a(a)(10)(A)(ii)(XIII). Per DPHHS manual ABD 201-6 (effective 2025-05-01, pre-freeze), it needs SSA disability criteria, age 16+ and paid employment, with countable income at or below 250% FPL. Per CMA 001 (effective 2026-01-01), the MWD resource limit is $15,000, not the $2,000 ABD limit. Under reading B the 100 head qualifies: she is 46, has wages and holds $2,800. 42 CFR 435.404 lets an applicant pick among the categories she qualifies for. MWD receipt makes her an elderly-or-disabled member under 7 CFR 271.2(11) and 7 U.S.C. 2012(j)(2)(B). With that status forced on 2.15.17 plus the conventions (rv_probe2), SNAP is 8,844 with the engine's MT heat-and-eat SUA and 7,116 without it. 8,844 is exactly the existing record's alternative value. Free ACA parent/caretaker coverage against a cost share of at least $35/mo makes MWD enrollment the less natural reading, but category choice is another unlisted fact. The dependence is therefore not settled away.", + "scenario_100: the proposed record change goes too far. It re-roots the output to r30 alone, changes the alternative value to 6,924, and describes 8,844 as 'an artifact of the criteria-based definition that #9345 removed'. r30 is real, still unfixed, and confirmed: always_standard.yaml has MT true with no elderly-or-disabled gate, and no change to it or its consumers since 06665727d8 addresses P.L. 119-21 sec. 10103. My own r30 port moves only scenario_080 (3,576 to 3,240) and scenario_100 (8,556 to 6,924) across all 1,984 outputs. But it should be recorded alongside the unlisted-input basis, not in place of it.", + "scenario_023: the reason says '3,576 needs a SUA of at least $384'. That is wrong on the engine it cites. With elderly-or-disabled status under reading B, 2.15.17 plus the conventions gives an excess medical deduction of $150.00/mo and a pre-shelter net of $1,470.51. So net = 205.76 - SUA, and a SUA of about $206 suffices. The $384 figure implies a medical deduction of about $31.67.", + "scenario_023: the settlement does not disclose what the 3,576 SUA rests on. It comes from CA's always_standard heat-and-eat flag: has_heating_cooling_expense = 0, and the prompt states no utility cost and no LIHEAP payment. For an elderly-or-disabled household, 7 U.S.C. 2014(e)(6)(C)(iv)(I) confers the SUA only on receipt of a LIHEAP payment over $20. The prompt says 'Do not infer unlisted ... benefit receipt'. With the heat-and-eat SUA switched off, the reading-B value is 2,832, not 3,576 (rv_probe2 conv_B_ED_noHE). The record should give the alternative as 3,576 with that caveat, or give both values.", + "scenario_023 (supports exclusion; for the lead): under the stated facts, 2.15.17 plus the conventions puts the head in medicaid_category WORKING_DISABLED_BUY_IN, with medicaid of $9,236.48. The engine's own Medicaid module therefore already treats her as receiving disability-related Medi-Cal. Adding the 271.2(11) title XIX route to SNAP moves stated-facts SNAP to 3,576 (rv_probe conv_t19_A, and the investigator's out_title19_full.csv). So 408 is the reading-A law value only if 'is disabled' confers no disability Medi-Cal, which contradicts the scored head_medicaid_eligible = 1. This strengthens the investigator's out-of-cluster flag. It does not change the SNAP action." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_023", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 408, + "reason": "Reproduced on the harness. latest_conventions gives 408 ($34/mo); raw 2.15.17 gives 462. Hand check: gross $2,120.22 (wages $1,453.55 plus 403(b) $666.67); minus 20% earned ($290.71) and the $209 standard deduction gives $1,620.51; minus the $744 cap (binds with or without the SUA) gives $876.51, rounded to $877; 30% is $263.10, rounded up to $264; $298 - $264 = $34. The unlisted input still decides the output under law. Under reading B, federal SSI is $0 (countable $1,340.94 > $994) and CA SSP is $0 (countable income over the $1,335.81 standard). She has no MAGI route (141.19% FPL > 138%). Her Medi-Cal is a disability pathway (engine: SENIOR_OR_DISABLED), and receiving it makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), both text-verified. 2.15.17 does not model that route, so flipping meets_ssi_disability_criteria alone gives 408 (rv_probe conv_B). Record the alternative as 3,576 with the engine's CA heat-and-eat SUA, and 2,832 if no LIHEAP payment is inferred (a SUA of about $206 suffices, not $384). Reword the record: in 2.15.17 SSI criteria no longer confer SNAP status (#9345); the route is Medi-Cal receipt." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 2628, + "reason": "Reproduced. latest_conventions gives 2,628 ($219/mo); raw gives 2,682. Hand and engine agree. Head SSI is $994 - $391.42 = $602.58/mo; gross is $1,472.42; minus $173.33 (20% of earned) and $209 gives $1,090.08, which rounds to $1,090; 30% is $327; $546 - $327 = $219. Under reading B (rv_probe conv_B, and with the r30 port and the title XIX route), both spouses get SSI at the couple rate: $1,491 - $392.42 = $1,098.58/mo in total ($6,591.50 per person per year). Net is $1,586, 30% is $476, $546 - $476 = $70, and 840 per year, so the recorded alternative of 840 stands on 2.15.17 plus the hold. LA always_standard is false, so there is no SUA and the r30 port cannot move it. The dependence runs through actual SSI receipt, an unlisted-input ambiguity that no engine version resolves." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 8556, + "reason": "Reproduced. latest_conventions gives 8,556 ($713/mo); raw gives 8,637. Pre-shelter net is $690.73 (earned $492.89 plus the engine's MT TANF of $505.41/mo, minus $98.58 and $209). With the $799 SUA (FNS FY2026 SUA table: MT HCSUA $799), net is $237, 30% rounds up to $72, and $785 - $72 = $713. r30 is confirmed still present in 2.15.17. The engine grants the always_standard SUA with has_heating_cooling_expense = 0 and no elderly-or-disabled member. My r30 port (rv_fix_r30.py, full sweep rv_r30_full.csv) moves only 080 and 100, giving 100 = 6,924: net $691, $208, $785 - $208 = $577. Keep the output excluded under both bases. Add r30_snap_heat_and_eat_sua with the stated-facts corrected value 6,924. Do not drop the unlisted-input basis. Under reading B, the working head qualifies for Montana MWD (ABD 201-6; CMA 001 $15,000 resource limit), and 42 CFR 435.404 lets her select it. Receipt makes her elderly or disabled under 7 CFR 271.2(11), giving 8,844 with the engine's SUA (the existing alternative) or 7,116 without. The mechanism text should change from 'SSI criteria' to 'MWD (title XIX) receipt', which 2.15.17 does not model." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv), outputs in my workspace. Raw run (rv_raw.csv): 462, 2,682, 8,637. latest_conventions (rv_conv.csv): 408, 2,628, 8,556, identical to out/latest_conventions.csv. The probe driver (rv_probe.py, my own modules, not the investigator's) ran reading B, the r30 port, the title XIX route under readings A and B, and their combinations. Reading B gives 408, 840, 8,556; r30 gives 080 = 3,240 and 100 = 6,924; the title XIX route under reading A gives 023 = 3,576. rv_probe2.py forces elderly-or-disabled status with and without the heat-and-eat SUA: 023 gives 3,576 or 2,832, 100 gives 8,844 or 7,116. A full 1,984-output sweep with my r30 port moved only 080 and 100 against latest_conventions. The investigator's out_r30_full and out_title19_full CSVs diff the same way. Engine code read at 79be99f671 and in the venv: disabled_programs.yaml and is_usda_disabled.py (#9345: receives_ssi and ssi receipt), has_snap_elderly_disabled_member, meets_snap_non_parent_student_exception, snap_net_income (half-up), snap_expected_contribution (ceil), snap_utility_allowance_type, snap_state_using_standard_utility_allowance, always_standard.yaml (MT, CA, PA true; LA false; no elderly-or-disabled gate), medicaid_category hierarchy, is_working_disabled_buy_in_for_medicaid (CA, IL and MS only), and SUA values (MT $799, CA $663). Commit dates and ancestry verified with git log and merge-base: #9318, #9162, #9587, #9623, #9090 and #9345 are all in 2.15.17. No commit in 06665727d8..79be99f671 gates the heat-and-eat SUA. Primary law read this session: eCFR 7 CFR 271.2 (2026-07-01; paragraph (11) title XIX route); 7 CFR 273.10(e) rounding options; 42 CFR 435.540(a); 42 CFR 435.404 (applicant's choice of category); uscode.house.gov 7 U.S.C. 2012(j)(2)(B) and 2014(e)(6)(C)(iv)(I) and (k)(4) as amended by P.L. 119-21 secs. 10103 and 10104 (July 4, 2025); FR 2025-19763 (published 2025-11-03: $994 and $1,491); the FNS FY2026 maximum allotments and deductions PDF; the FY26 COLA memo dated 2025-08-13 (Wayback 2025-09-02); the FY27 page updated 2026-08-28 (post-freeze, so held); the FNS FY2026 SUA table (MT 799, CA 663, LA 465, PA 857); and Montana DPHHS ABD 201-6 (effective 2025-05-01), ABD 010 (2024) and CMA 001 (effective 2026-01-01). I also rendered the exact prompts (prompts_dump.txt) to check what readers saw: the prompts say 'is disabled' with no SSI or SSDI receipt, no utility costs and no LIHEAP payment, and tell readers not to infer benefit receipt. Not independently checked: the MT TANF amount the engine feeds into 100's SNAP, and CA's Medi-Cal program income limits beyond the engine's output." + } + }, + { + "id": "excl_snap_mortgage_residence", + "title": "Excluded NY SNAP output (mortgage-residence reading): board keeps fractional v1.1", + "outputs": [ + { + "scenario_id": "scenario_118", + "variable": "snap", + "state": "NY", + "board": 2903.94043, + "latest": 2868, + "v11_1755": 2903.94043, + "excluded": true, + "exclusion_root_cause": "unlisted:whether the listed home mortgage interest is on the home the SNAP household occupies", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_snap_mortgage_residence", + "classification": "unlisted_input_unchanged", + "summary": "The output stays excluded (keep_excluded). Its root cause, r15_snap_mortgage_interest, is a prompt ambiguity: the prompt never says whether the listed mortgage interest is on the home the household lives in. No engine version can resolve that. The board's 2,903.94 is raw 1.755.4, because the Sept 22 regeneration left excluded outputs untouched (every revision in the sidecar has excluded_outputs_untouched: true).\n\nTwo things move the value from the board's 2,903.94 to 2.15.17's 2,868:\n- **Rounding fix (pre-freeze law, adopted).** PR #9318 rounds net income half-up and rounds 30% of net income up to the next dollar, as 7 CFR 273.10(e) requires. That moves Jan-Sep from 239.80 to 239 a month.\n- **FY2027 figures (post-freeze, already held).** 1.755.4 projected FY2027 values for Oct-Dec, giving 248.58 a month. 2.15.17 encodes the published FY2027 COLA (#9623: $306 maximum, $217 standard deduction; USDA memo dated 2026-08-21, after the freeze), giving 251 a month. latest_c_snap_hold_fy2026 already holds these to FY2026, giving 239 a month.\n\nThe two combine to 12 x 239 = 2,868. On 1.755.4, adding the FY2026 hold alone gives 2,877.60; adding r26/r27/r28/r31/r33 as well gives 2,868. So 2,868 is the value the board's own stack would have produced had the output been regenerated.\n\nThe alternative value, 3,576, needs no change:\n- I ported r15 v2 to 2.15.17's shelter formula (fixes/latest_alt_snap_mortgage_residence.py) and swept all 1,984 outputs (out/latest_alt_snap_mortgage_residence.csv).\n- Compared with latest_conventions, it changes exactly two outputs: scenario_118 snap (2,868 to 3,576) and scenario_056 snap (95 to 159; a different cluster whose recorded alternative also combines r14). Against the board, both sweeps move the same 29 outputs.\n- Under the alternative reading, net income is 0 in every month, so the benefit is the FY2026 maximum of 298 x 12 = 3,576 regardless of rounding.\n\nAn engine-free recomputation reproduces 2,868 and 3,576 under both rounding procedures 7 CFR 273.10(e) allows (latest/excl_snap_mortgage_residence/recompute_by_hand.py).\n\nSide observation, not a move between versions: in both versions SNAP gross income is Social Security plus federal SSI only. No New York state SSI supplement is in 2.15.17's SNAP unearned-income list. I could not reach OTDA or SSA (bot blocks), so I make no claim about the law. Scoring is unaffected because the output stays excluded.", + "upstream_changes": [ + { + "commit": "5d88007d90 (PR #9318; commits 8df8c5043a 2026-08-19, 949546af53 2026-08-25)", + "date": "2026-08-25", + "what": "Two rounding changes. snap_net_income now rounds to the nearest dollar, and snap_expected_contribution rounds 30% of net income up to the next dollar (7 CFR 273.10(e)(1)(ii)(A) and (e)(2)(ii)(A)(1)). Effect on scenario_118: net income 194.38 becomes 194, the contribution 58.20 becomes 59, and the monthly benefit falls from 239.80 to 239. That is -9.60 a year on the held FY2026 schedule. This is the board's r26/r27, pre-freeze law." + }, + { + "commit": "fa27adbffc (PR #9587)", + "date": "2026-09-28", + "what": "Net income is now rounded to cents before half-up rounding, and income standards are fixed for households over eight. No further effect here: 194.375 still rounds to 194." + }, + { + "commit": "d27d6c3cc3 (PR #9623)", + "date": "2026-09-27", + "what": "Encodes USDA's FY2027 SNAP COLA from 2026-10-01: one-person maximum $306, standard deduction $217, from the FNA memo dated 2026-08-21. With these, raw 2.15.17 pays 251 a month in Oct-Dec (total 2,904) instead of 1.755.4's CPI-projected 248.58 (304.68 max, 213.68 standard deduction). This is post-freeze, so the existing convention latest_c_snap_hold_fy2026 (in latest_conventions) holds FY2026: 298 max, 209 standard deduction, 239 a month." + }, + { + "commit": "3c41c31457 (PR #9162; cf165464a6)", + "date": "2026-07-28", + "what": "Rounds the SNAP minimum allotment and the income standards (r28/r31). No effect: the minimum allotment does not bind, and the net income test passes by a wide margin." + }, + { + "commit": "none (housing_cost.py, mortgage_payments.py unchanged; snap_excess_shelter_expense_deduction diff checked)", + "date": "2026-09-28", + "what": "In 2.15.17 the shelter formula changed in only two ways, and neither affects this household. Housing costs are multiplied by snap_expense_counted_share, which is 1 here. The uncapped branch now reads has_snap_elderly_disabled_member, which is true here. Mortgage interest is still not a SNAP shelter cost, so the unlisted-input ambiguity is unchanged." + } + ], + "law": "Pre-freeze law:\n- **7 CFR 273.10(e)(1)(ii)(A) and (e)(2)(ii)(A).** Net income is rounded half-up to whole dollars (or by the state's TANF procedure). 30% of net income is either rounded up to the next dollar, or left unrounded with the allotment rounded down. Read from the eCFR version in force on 2026-07-01; the eCFR versioner lists the last amendment as 2025-01-17.\n- **7 CFR 273.9(d)(6)(ii) (same eCFR version).** (A) allows \"continuing charges for the shelter occupied by the household\", including mortgage payments and \"interest on such payments\". The shelter deduction is uncapped for households with an elderly or disabled member.\n- **USDA FNS FY2026 SNAP COLA memo** (dated 2025-08-13, digitally signed 2025-08-14; snap-cola-fy26memo.pdf from Wayback): one-person maximum $298 and standard deduction $209 for the 48 states and DC, effective 2025-10-01.\n- **FNS FY2026 SUA table** (2026-05-21-SUA-Table-FY26.xlsx): New York Rest of State HCSUA $877, effective 2025-10-01 to 2026-09-30.\n- **SSA OACT, SSI Federal Payment Amounts** (Wayback capture 2026-09-26): $994 a month for an individual in 2026, effective January 2026. After the $20 general exclusion on $233.33 of monthly Social Security, SSI is 780.67 a month (9,368 a year), matching the engine.\n\nPost-freeze figure:\n- **USDA FNA FY2027 SNAP COLA memo** (dated 2026-08-21; fna.snap-cola2027.pdf, Wayback capture 2026-09-08): one-person maximum $306 and standard deduction $217, effective 2026-10-01. This is after the 2026-07-03 freeze, so the existing FY2026 convention holds it.\n\nRecomputation: monthly gross income is 1,014 and income after the standard deduction is 805. Shelter costs are 877 + 136.125 = 1,013.125, so the excess shelter deduction is 610.625 and net income is 194.375. That gives 239 a month and 2,868 a year. Adding mortgage interest of 462.58 a month makes the excess shelter deduction 1,073.21 and net income 0, so the benefit is 298 a month and 3,576 a year.", + "per_output": [ + { + "scenario_id": "scenario_118", + "variable": "snap", + "board": 2903.94043, + "latest": 2868, + "proposed_reference": 2868, + "action": "keep_excluded", + "reason": "The exclusion's root cause is an unlisted input: whether the listed mortgage interest is on the occupied home (7 CFR 273.9(d)(6)(ii)(A)). This does not depend on the engine, and 2.15.17 still leaves mortgage interest out of SNAP shelter costs. The stated-facts value on 2.15.17 plus conventions is 2,868 (12 x 239), and it replaces the stale frozen value of 2,903.94, which is raw 1.755.4 kept because excluded outputs were untouched. The move comes from the #9318 rounding fix (pre-freeze law; Jan-Sep 239.80 to 239) and the FY2026 hold of the post-freeze FY2027 COLA (Oct-Dec). The 1.755.4 board stack (r13 v3 plus r26/r27/r28/r31/r33) also gives 2,868, and an engine-free recomputation reproduces it. The alternative value stays 3,576, recomputed on 2.15.17 plus conventions with the ported r15 reading (fixes/latest_alt_snap_mortgage_residence.py; net income 0, 298 x 12)." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": false, + "problems": [ + "Scope of the disagreement: the action (keep_excluded) and both numbers reproduce. 2.15.17 + latest_conventions gives 2,868, the ported alternative gives 3,576, raw 2.15.17 gives 2,904, and the 1.755.4 board stack (c13v3_upstream_plus_r33) gives 2,868. What does not hold is the settlement's description of 2,868 as \"the stated-facts value\" and its classification unlisted_input_unchanged. Both omit a second unlisted input and an unfixed engine defect that each move this output. Neither changes scoring, because the output stays excluded.", + "Second unlisted input (county / SUA region), missed by the settlement and the exclusion record. The prompt states only \"state: NY\". county_fips is in EXCLUDED_INPUT_VARIABLES (policybench/scenarios.py), so 2.15.17 falls back to first_county_in_state (ALBANY_COUNTY_NY), which maps to SNAP utility region NY_ONY and the $877 HCSUA. FNS's FY2026 SUA table (2026-05-21-SUA-Table-FY26.xlsx, effective 2025-10-01 to 2026-09-30, pre-freeze) gives $1,062 for New York City and $988 for Nassau/Suffolk. On 2.15.17 + latest_conventions the output is 2,868 for Albany, 3,276 for Nassau County and 3,540 for New York County. I verified this in the engine (probe.txt) and by hand (hand_calc.txt). So 2,868 is the engine's value under a default county the prompt never states, not the stated-facts value. The alternative value, 3,576, is the same in all three regions.", + "Unfixed engine defect, left unresolved in the settlement: 2.15.17 omits New York's State Supplement Program (SSP). No NY SSP variable exists anywhere in the 79be99f671 tree. gov.usda.snap.income.sources.unearned lists SSI supplements for 25 other jurisdictions, and unearned_spm_unit lists California's, but New York appears in neither.\nThe law, all published before 2026-07-03:\n- OTDA's \"2026 SSI and SSP Maximum Monthly Benefit Levels Chart\" (revised 2025-10-27; Wayback capture 2026-01-02; statutory reference Chapter 56 of the Laws of 2025) gives an individual living alone $994 federal plus $87 state.\n- NY Social Services Law 209(2)(a) (Wayback capture 2025-07-17) sets the 2025 living-alone standard at $1,054, which is the $967 federal amount plus $87.\n- OTDA's SSP page (Wayback capture 2026-02-13) says SSP \"is part of the monthly benefit paid to most SSI recipients\" and that the SSI application \"serves as your application for SSP benefits\".\n- 7 CFR 273.9(b)(2)(i) counts SSI and other need-based assistance as SNAP unearned income.\nThe harness already sets takes_up_ssi_if_eligible true and the engine pays $9,368 of SSI, so the $87 a month follows from the same stated facts and take-up.\nWith $87 a month added to SNAP income, the engine gives 2,400 for the default county, 2,796 for Nassau and 3,072 for New York County (probe2.txt). A hand calculation matches.\nThe settlement says OTDA and SSA were unreachable. Both are reachable through Wayback. Under the brief, an unfixed 2.15.17 defect that moves an output belongs in the exclusion record as a root cause. It does not change the action or the alternative value, because under the alternative reading net income stays 0.", + "The engine's value comes from a hybrid reading, not from either reading of the fact the exclusion names. housing_cost adds real_estate_taxes, so the $1,633.50 of property taxes counts as a shelter cost for the occupied home (7 CFR 273.9(d)(6)(ii)(B)). It adds mortgage_payments, an input with no formula, and never reads home_mortgage_interest or first_home_mortgage_interest. The interest on the same home's first mortgage therefore does not count. If the mortgaged home is read as not the residence, and the property taxes are dropped for the same reason, the result is 2,376 (Rest of State, no SSP; hand_calc.txt). The exclusion record should describe 2,868 as \"2.15.17 + conventions under the engine's default county, without NY SSP\". It should not say this is the value under the non-residence reading.", + "Minor: the settlement paraphrases 7 CFR 273.9(d)(6)(ii)(A) as covering \"mortgage payments\". The eCFR text in force on 2026-07-01 reads \"Continuing charges for the shelter occupied by the household, including rent, mortgage, condo and association fees, or other continuing charges leading to the ownership of the shelter ..., including interest on such payments.\" The meaning is the same, but quote it verbatim in the record." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_118", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 2868, + "reason": "Keep excluded, now for three independent reasons.\n1. The mortgage-residence ambiguity (r15_snap_mortgage_interest), which no engine version resolves. 2.15.17 still counts only mortgage_payments, not mortgage interest, as a SNAP shelter cost.\n2. A second unlisted input, the county. The prompt gives only NY, and New York's FY2026 HCSUA is $877, $988 or $1,062 by region. That gives 2,868, 3,276 or 3,540 on 2.15.17 + conventions.\n3. An unfixed 2.15.17 defect: the NY State Supplement Program ($87 a month for an individual living alone in 2026, per the OTDA chart revised 2025-10-27; pre-freeze law) is missing from SNAP income. With it, the value would be 2,400, 2,796 or 3,072 by region.\nRecord 2,868 as the regenerated engine value, replacing the stale raw-1.755.4 2,903.94. Label it as 2.15.17 + latest_conventions under the engine's default county (Albany, Rest of State SUA) without NY SSP, not as the law-correct stated-facts value. Add the NY SSP omission as an unfixed-defect root cause and the county/SUA region as a second unlisted input. The alternative value stays 3,576 (298 x 12). It is the same across all three regions, with or without SSP, and under every 7 CFR 273.10(e) rounding option. Scoring is unaffected." + } + ], + "what_i_checked": "Everything below was run or read this session. Scratch files are in pb-refs-reviews-0929/excl_snap_mortgage_residence.\n\n(1) Harness (sweep_latest.py, 2.15.17) on scenario_118 and scenario_056:\n- no fix: 118 gives 2,904; 056 gives 95.\n- latest_conventions: 2,868 and 95.\n- latest_c_snap_hold_fy2026 alone: 2,868 and 95.\n- latest_alt_snap_mortgage_residence: 3,576 and 159.\nFiles: rv_*.csv.\n\n(2) 1.755.4 venv (sweep.py) on scenario_118: c13v3_upstream_plus_r33 gives 2,868, r13_hold_fy2026_v3 gives 2,877.60, and r15_snap_mortgage_interest_v2 alone gives 3,596.04 (rv1755_*.csv). The board and raw 1.755.4 are both 2,903.94. Every one of the 17 sidecar revisions has excluded_outputs_untouched true.\n\n(3) The investigator's full sweeps: latest_alt vs latest_conventions differ in exactly 2 of 1,984 outputs (056 and 118), and 29 outputs move against the board in both. The other NY snap outputs (004, 071, 082, 104) are 0 and not excluded.\n\n(4) Monthly engine trace (probe.txt).\n- Values: SS 233.33 and SSI 780.67, so gross income is 1,014. The standard deduction of 209 leaves 805. The SUA is 877 plus 136.125 of property tax. The excess shelter deduction is 610.625 (uncapped; has_snap_elderly_disabled_member true). Net income rounds to 194, the contribution is 59, and the benefit is 298 - 59 = 239 a month.\n- Raw 2.15.17 pays 251 a month in Oct-Dec (std 217, max 306).\n- The alternative gives an excess shelter deduction of 1,073.21 and net income 0, so 298 a month.\n- Variants: county, and mortgage_payments=5551 as an input, which equals the alt module at 3,576.\n- probe2.txt adds the $87 SSP.\n\n(5) Code:\n- housing_cost and mortgage_payments have no diff between 06665727d8 and 79be99f671.\n- snap_excess_shelter_expense_deduction changed only in expense_share and has_snap_elderly_disabled_member.\n- Diffs read for PR #9318 (5d88007d90: half-up net income, ceil of 30%), #9587 (fa27adbffc: round to cents first), #9162 (3c41c31457) and #9623 (d27d6c3cc3: FY2027 COLA, memo dated 2026-08-21).\n- Also read snap_utility_region (NY county mapping), county, first_county_in_state, always_standard (NY true), and the SNAP unearned-income source lists.\n\n(6) Primary sources:\n- eCFR 7 CFR 273.10 and 273.9 as of 2026-07-01 (versioner API; last amended 2025-01-17): (e)(1)(ii)(A) and (B), (e)(2)(ii)(A)(1) and (2), 273.9(d)(6)(ii)(A) and (B), 273.9(b)(2)(i).\n- FNS FY2026 COLA memo, dated August 13, 2025 (Wayback page 20250902215244 plus PDF): 1-person maximum $298, standard deduction $209 for sizes 1-3.\n- FNA FY2027 COLA memo, dated August 21, 2026 (PDF via Wayback; earliest page capture 2026-08-28): $306 and $217, post-freeze.\n- FNS FY2026 SUA xlsx (live): NY NYC 1,062, Nassau/Suffolk 988, Rest of State 877.\n- SSA OACT SSIamts (Wayback 2026-05-10): 2026 individual $994.\n- OTDA 2026 SSI/SSP chart (Wayback 2026-01-02; revised 2025-10-27): NY SSP living alone $87.\n- OTDA SSP page (Wayback 2026-02-13).\n- NY SSL 209 (Wayback 2025-07-17).\n\n(7) Engine-free recomputation (hand_calc.py): 3 readings x 3 regions x SSP on/off x 3 rounding procedures. Every engine value reproduces. The alternative is 3,576 in every case.\n\n(8) The prompt the models saw (make_no_tools_prompt) lists home mortgage interest, the first home mortgage balance and interest, and real estate taxes. It lists no county, rent or utilities, and it says to assume program take-up.\n\nNot checked: any New York-specific SNAP procedure for SSI recipients. I retrieved no OTDA source for one." + } + }, + { + "id": "excl_r30_snap_heat_and_eat_sua", + "title": "Excluded PA SNAP output (r30 heat-and-eat SUA, still unfixed)", + "outputs": [ + { + "scenario_id": "scenario_080", + "variable": "snap", + "state": "PA", + "board": 3596.039795, + "latest": 3576, + "v11_1755": 3596.039795, + "excluded": true, + "exclusion_root_cause": "r30_snap_heat_and_eat_sua", + "final_action": "keep_excluded" + } + ], + "investigation": { + "cluster_id": "excl_r30_snap_heat_and_eat_sua", + "classification": "excluded_defect_still_present", + "summary": "The r30 defect is still in policyengine-us 2.15.17. The file snap_state_using_standard_utility_allowance.py is identical in 1.755.4 and 2.15.17, and it still returns p.always_standard[state]. The only change to always_standard.yaml in 06665727d8..79be99f671 is metadata (d7fc17abfe). snap_utility_allowance_type still picks the SUA on has_heating_cooling | always_sua, with no elderly or disabled gate. upstream/main is at 79be99f671, and no later commit touches heat-and-eat. A monthly trace on 2.15.17 plus latest_conventions shows always_sua=True, has_heating_cooling_expense=False, has_snap_elderly_disabled_member=False and is_usda_disabled=False (SSI 0, SSDI 0). The engine still grants PA's $857 SUA, caps shelter at $744, gets net income of 0 and pays $298 a month, for 3,576. The move against the board does not come from r30. Raw 2.15.17 gives 3,600 = 9x298 + 3x306, the FY2027 COLA from commit d27d6c3cc3 (USDA memo of 2026-08-21, after the freeze). latest_c_snap_hold_fy2026 holds $298 for all 12 months, giving 3,576, the same value 1.755.4 gave with that convention (c13v3_plus_upstream_snap.csv). Under the record's reading, the prompt states no 7 U.S.C. 2012(j) route. The law-correct value is then 12 x (298 - ceil(0.3 x 92)) = 3,240, and the harness probe latest_conventions+r30 reproduces 3,240. That probe moves only two outputs: scenario_080 snap and scenario_100 snap (8,556 to 6,924), and both are already excluded. The 3,576 from 2.15.17 equals the value under the other reading (head counted as disabled), so the output stays indeterminate. Keep it excluded. If the record is regenerated, only frozen_value and engine_version change (3,596.04/1.755.4 become 3,576/2.15.17). alternative_value 3,240 and the note stand.", + "upstream_changes": [ + { + "commit": "d7fc17abfe", + "date": "2026-09-02", + "what": "Fix breakdown metadata propagation: adds only 'propagate_metadata_to_children: true' to parameters/gov/usda/snap/income/deductions/utility/always_standard.yaml. PA stays true from 2015-10-01. No value or logic change, and snap_state_using_standard_utility_allowance.py has no commits in the range." + }, + { + "commit": "e3b46f1714, 16220c837b, e5cdd36468", + "date": "2026-08-30 to 2026-09-12", + "what": "snap_utility_allowance_type adds the LUA telephone option and reference comments. The SUA branch is still 'has_heating_cooling | always_sua', with no elderly or disabled gate, so the heat-and-eat defect is unchanged." + }, + { + "commit": "02b2b3186a, 50f940eebf, ed8fd93ff9", + "date": "2026-09-05 to 2026-09-12", + "what": "has_heating_cooling_expense now reads the canonical heating/cooling facts, with a legacy adapter for heating_cooling_expense. It is False for scenario_080 (no heating or cooling cost stated), in both engines." + }, + { + "commit": "339a1f1b03, e5433454d1, e998515b75, 857d7c76e5", + "date": "2026-08-24 to 2026-09-04", + "what": "is_usda_disabled and disabled_programs change to a benefit-receipt test per 7 CFR 271.2 (receives_ssi, ssi, social_security_disability, veteran flags), replacing is_ssi_disabled. It is still False for this head (SSI $0 because $19,828 in the bank exceeds the resource limit, SSDI $0). is_disabled is not read, as in 1.755.4." + }, + { + "commit": "0d3d6bee0d, 7802ffd433", + "date": "2026-07-14", + "what": "Adds has_snap_elderly_disabled_member (excludes ineligible members) and uses it for the excess-shelter cap. It is False here, so the $744 cap still applies. No effect on the value." + }, + { + "commit": "d27d6c3cc3", + "date": "2026-09-27", + "what": "Encodes USDA's FY2027 SNAP COLA (memo dated 2026-08-21, after the freeze): 1-person maximum allotment $306 and standard deduction $217 from 2026-10. This moves raw 2.15.17 to 3,600 (9x298 + 3x306) from 1.755.4's 3,596.04, whose Oct-Dec $304.68 was a CPI projection. latest_c_snap_hold_fy2026, part of latest_conventions and a different root cause, already holds it at $298, giving 3,576." + } + ], + "law": "P.L. 119-21 (H.R. 1), sec. 10103(a), approved July 4, 2025 (govinfo PLAW-119publ21, 139 Stat. 83; text read this session). It amends 7 U.S.C. 2014(e)(6)(C)(iv)(I) by inserting 'with an elderly or disabled member' after 'households' and sets no separate effective date. I checked the amended codified clause in the OLRC prelim text of 7 U.S.C. 2014: the SUA 'shall be made available to households with an elderly or disabled member' that received a qualifying LIHEAP or similar payment over $20. Sec. 10103(b) also amends 2014(k)(4): a state-law energy payment to a household without such a member counts as income. 7 U.S.C. 2012(j) (OLRC) defines an elderly or disabled member by age 60 or over or by receipt of listed benefits (SSI, Title II disability, veterans' ratings, and others). The prompt states only 'is disabled' and no such route. The FNS memo 'SNAP Implementation of the One Big Beautiful Bill Act of 2025 - Treatment of Energy Assistance Payments', dated August 29, 2025 (Wayback capture of fns.usda.gov, 2026-02-21), says sec. 10103 was effective on enactment, July 4, 2025. It also says states 'must no longer automatically confer HCSUA eligibility to households without an elderly or disabled member based on receipt of a qualifying LIHEAP' payment, and that such households still qualify only if they incur heating or cooling costs. I took the recomputation's FY2026 figures from the FNS FY2026 COLA memo dated August 13, 2025 (PDF fetched): 1-person maximum allotment $298, standard deduction $209 for sizes 1-3, shelter cap $744. All of this was published before the 2026-07-03 freeze.", + "per_output": [ + { + "scenario_id": "scenario_080", + "variable": "snap", + "board": 3596.039795, + "latest": 3576, + "proposed_reference": null, + "action": "keep_excluded", + "reason": "The r30 engine defect is unfixed in 2.15.17. The engine still gives PA's heat-and-eat SUA ($857/month) to a household with no 7 U.S.C. 2012(j) elderly or disabled member and no heating or cooling cost (trace: always_sua=True, has_heating_cooling_expense=False, has_snap_elderly_disabled_member=False). The shelter deduction is capped at $744, net income is 0 and the benefit is $298 x 12 = 3,576. The -$20.04 move against the board comes from the FY2026 hold convention, not from r30. Independent recomputation under the record's reading, where P.L. 119-21 sec. 10103 bars the SUA: gross $301.33 ($3,600 financial assistance + $16 dividends, /12), minus the $209 standard deduction, gives net $92. 30% of that, rounded up, is $28, so $298 - $28 = $270/month and 3,240/year. The harness probe latest_conventions+r30 gives 3,240. Read the other way (head counts as disabled), LIHEAP over $20 confers the SUA, the shelter cap is lifted, net income is 0, and the value is 3,576. 2.15.17 lands on that value, but for the wrong reason under the record's reading. The prompt also does not say whether the $2,000 energy subsidy is LIHEAP or a state-law payment, which 2014(k)(4)(A) would count as income. The reference is indeterminate on the stated facts, so it stays excluded. If the exclusion record is regenerated, frozen_value 3,596.04 / engine 1.755.4 become 3,576 / 2.15.17. alternative_value 3,240 and the note, which already cites 3,576, are unchanged." + } + ], + "hold_module": "", + "confidence": "high" + }, + "review": { + "runner": "subfleet review lane (Claude Opus 5.5), 2026-09-29", + "agree": true, + "problems": [ + "Minor and stale: the settlement says upstream/main is at 79be99f671. It is now d81bbf1b2c (2026-09-29 03:32 UTC), two commits later: ae69e84376 (child-care court-supervision routes, #9555) and d81bbf1b2c (version bump). Neither touches SNAP, so the conclusion stands.", + "Omitted upstream context: policyengine-us issue #9374 (open) reports this exact defect (the P.L. 119-21 sec. 10103 elderly/disabled gate on the always_standard SUA) for California. Draft PR #9610 (opened 2026-09-25, still a draft) adds an is_snap_sua_hr1_in_effect gate for California only. #7745 (the OBBBA heat-and-eat tracker) and #7853 (always_standard concept) were closed NOT_PLANNED. PA's always_standard=true from 2015-10-01 came from #8132 (merged 2026-05-01, before 1.755.4). No merged or in-flight fix covers PA (or MT, scenario_100). The record's upstream field ('to be filed') should cite #9374 and #9610 and say PA is not covered.", + "Overstated: 'the prompt states no 7 U.S.C. 2012(j) route.' 2012(j)(2)(B) (OLRC text read this session) also counts receipt of 'disability-based State general assistance benefits' that meet SSI-stringent criteria. The prompt lists an unexplained 'financial assistance: $3,600' next to 'is disabled'. The engine defines financial_assistance as help from friends or relatives, but the prompt does not say that. This is one more path to the disabled-member reading (3,576), so it strengthens the indeterminacy. It does not change the action.", + "Omitted from the law section: the FNS memo of 2025-08-29 (earliest Wayback capture 2025-09-06, before the freeze) says sec. 10103 is effective on enactment for new applicants, but lets states apply it to ongoing households 'at a minimum ... at the household's next recertification.' The prompt gives no certification timing. So even under the record's reading, some 2026 months could still carry the LIHEAP-conferred HCSUA ($298 a month). 3,240 is the value for a household under the new rule all year, not an unconditional law-correct value. This supports keeping the exclusion.", + "The 3,240 alternative also rests on engine treatments that 1.755.4 and 2.15.17 share and that the settlement does not flag as conditional. (a) The $924 of long-term capital gains is excluded from SNAP income: parameters/gov/usda/snap/income/sources/unearned.yaml lists dividend and interest income but no capital gains, and gross is $301.33 = $3,616/12. Counting it would give 12 x (298 - ceil(0.3 x 169.33)) = 2,964. (b) The $2,000 subsidy is treated as federal energy assistance, which 7 U.S.C. 2014(d)(11)(A) excludes. The investigator did flag the 2014(k)(4)(A) state-law alternative. So 'the law-correct value is 3,240' should read as conditional. This does not change the action.", + "Provenance, if the record is regenerated: 3,576 is policyengine-us 2.15.17 plus latest_conventions (c_snap_hold_fy2026 holds FY2026 figures for Oct-Dec 2026). Raw 2.15.17 is 3,600. engine_version should say so, e.g. 'policyengine-us 2.15.17 + latest_c_snap_hold_fy2026', or the frozen_value reads as raw engine output.", + "Tooling nit: fixes/r30_snap_heat_and_eat_sua.py (written for 1.755.4) gates on has_usda_elderly_disabled, which is yearly and counts every SPM-unit member. In 2.15.17 the SNAP-canonical test is has_snap_elderly_disabled_member, which is monthly and excludes ineligible members (added by 0d3d6bee0d, 2026-07-14). This makes no difference for scenario_080 (one person) or for this probe, which moves only 080 and 100. Any regenerated alternative for mixed-eligibility households should use the 2.15.17 variable." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_080", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "The r30 defect is unfixed in 2.15.17. snap_state_using_standard_utility_allowance.py is byte-identical to 1.755.4 and returns p.always_standard[state]. always_standard.yaml's only change since 06665727d8 is metadata (d7fc17abfe), and PA stays true from 2015-10-01. snap_utility_allowance_type still grants the SUA on has_heating_cooling | always_sua. The only upstream fix in progress (draft PR #9610) covers California only. My trace on 2.15.17 + latest_conventions shows always_sua=True, has_heating_cooling_expense=False, is_usda_disabled=False and has_snap_elderly_disabled_member=False. The $857 SUA is capped at $744, net income is 0, and the benefit is $298 x 12 = 3,576. With r30 added it is $270 x 12 = 3,240 (net $92, contribution $28). Under 7 U.S.C. 2014(e)(6)(C)(iv)(I) as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04, before the freeze), the engine's own reading (no elderly or disabled member) should give 3,240, so 3,576 is defect-driven under that reading. With is_usda_disabled forced True, 2.15.17 gives 3,576 with or without r30, and the household stays eligible. The prompt supports both readings. It says 'is disabled' but gives no 2012(j) receipt route. Its $3,600 'financial assistance' is unexplained and could be disability-based general assistance. Its $2,000 'spm unit energy subsidy' could be LIHEAP or a state-law payment. It states no certification timing, which matters under the FNS recertification transition. Model answers split the same way: 15 near the SUA-conferred value (3,492-3,576) against 3,240, 3,228 and 2,964-2,988 on other readings. The output is therefore both defect-affected and indeterminate on the stated facts, and stays excluded. The -$20.04 move against the board (3,596.04 to 3,576) comes from the FY2026 SNAP hold convention: the FY2027 COLA memo is dated 2026-08-21, after the freeze. It does not come from r30. Scoring is unaffected." + } + ], + "what_i_checked": "HARNESS (policyengine-us 2.15.17, the brief's interpreter, run myself): full sweeps with fixes/latest_conventions.py (rv2_full_conv.csv) and with my own probe of latest_conventions + r30_snap_heat_and_eat_sua (rv2_conv_plus_r30.py, output rv2_full_conv_r30.csv). My conventions sweep matches out/latest_conventions.csv on all 1,984 outputs (29 moved vs the board). Adding r30 changes exactly two outputs: scenario_080 snap 3,576 to 3,240 and scenario_100 snap 8,556 to 6,924, both already excluded. Raw 2.15.17 on 080 and 100 (rv2_raw.csv): 3,600 and 8,637. My monthly trace (rv2_trace.py, rv2_trace.log): raw is 298 for Jan-Sep and 306 for Oct-Dec (standard deduction 209 to 217, shelter cap 744 to 769), total 3,600. With conventions, 298 every month, 3,576. With conventions + r30, 270 every month (SUA NONE, net 92, contribution 28), 3,240. With is_usda_disabled forced True, 3,576 with and without r30 (uncapped shelter 810.83, eligible). ENGINE CODE: diffed 1.755.4 and 2.15.17 site-packages. snap_state_using_standard_utility_allowance.py is identical, and always_standard.yaml differs only by 'propagate_metadata_to_children: true'. Read snap_utility_allowance_type.py, snap_excess_shelter_expense_deduction.py, has_heating_cooling_expense.py, is_usda_disabled.py, disabled_programs.yaml (1.755.4 used is_ssi_disabled, 2.15.17 uses receives_ssi/ssi/SSDI/veteran flags), has_snap_elderly_disabled_member.py, SNAP unearned.yaml (financial_assistance counted, no capital gains, no energy subsidy) and financial_assistance.py. UPSTREAM (read-only git): git log 06665727d8..upstream/main on the SUA variable and parameter shows only d7fc17abfe (metadata; git show read). Checked the dates and subjects of all 14 commits the settlement cites. d27d6c3cc3's message cites the USDA memo dated August 21, 2026. upstream/main is now d81bbf1b2c, and the two commits after 79be99f671 are not SNAP. Read-only gh: issues #7745 and #7853 (closed NOT_PLANNED) and #9374 (open, CA), PR #8132 (merged 2026-05-01, added PA to always_standard) and draft PR #9610 (CA-only gate). LAW, from primary sources fetched this session: P.L. 119-21 from govinfo (approved July 4, 2025; sec. 10103(a) and (b) text verified; on page 139 Stat. 83). 7 U.S.C. 2014 from the OLRC prelim: the amended (e)(6)(C)(iv)(I) text, the (k)(4)(A)-(B) text, (d)(11)(A), and the amendment notes naming P.L. 119-21 sec. 10103. 7 U.S.C. 2012(j) from the OLRC prelim, including the (2)(B) disability-based State general assistance route. FNS memo 'SNAP Implementation of the One Big Beautiful Bill Act of 2025 - Treatment of Energy Assistance Payments' dated August 29, 2025 (Wayback capture 2025-09-06): effective on enactment, no automatic HCSUA for households without an elderly or disabled member, ongoing households at the latest at next recertification. FNS FY2026 COLA memo (Wayback capture 2025-08-17 of fns.usda.gov shows 'DATE: August 13, 2025'; PDF read): 1-person maximum $298, standard deduction $209 for sizes 1-3, shelter cap $744. FNS FY2027 COLA memo PDF (usda.gov, via Wayback 2026-09-08): dated August 21, 2026, with $306, $217 and $769, after the 2026-07-03 freeze. PROMPT: regenerated the scenario_080 SNAP prompt with policybench.prompts.make_no_tools_prompt from the adds0928-stage2 worktree (rv2_prompt_080.txt): 'is disabled', 'financial assistance: $3,600', 'spm unit energy subsidy: $2,000', no heating or cooling costs, no SSI or SSDI, and a preface saying not to infer unlisted benefit receipt or expenses. The June 12 run's stored artifacts do not keep the prompt text, but the model explanations quote the same facts. Tabulated all 42 model answers for scenario_080 snap from predictions.csv.gz (rv2_model_preds_080.txt). INDEPENDENT ARITHMETIC: gross (3,600 + 16)/12 = 301.33; minus 209 = 92.33; ceil(0.3 x 92.33) = 28; 298 - 28 = 270; x 12 = 3,240. The SUA route gives 857 - 46.17 = 810.83 (capped at 744 for a non-E/D household), net 0, 298 x 12 = 3,576. Any SUA of at least $139 gives net 0, so 3,576 does not depend on PA's exact SUA amount. The 1.755.4 counterparts in out/c13v3_plus_upstream_snap.csv and c13v3_upstream_plus_r30.csv are 3,576 and 3,240. I did not verify PA's own sec. 10103 implementation date or the SNAP treatment of individual capital gains under 7 CFR 273.9." + } + } + ], + "reconciliations": [ + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "cluster": "md_county_tax_8888", + "cluster_review_action": "adopt_latest", + "final_action": "exclude_unlisted_input", + "cites_review": "salt_refund_gross_income_9122", + "decided_by": "developer", + "decided_on": "2026-09-29", + "reason": "The two reviews conflict on this output, and the developer follows the salt_refund_gross_income_9122 review. The Maryland review settles the county question: under the withholding proxy the SALT cap binds at every 2026 county rate, so the value does not depend on the unlisted county. Its value of 24,164.46 still counts the $1,631.91 state and local tax refund as income, as policyengine-us#9422 (issue #9122) does. Left out of income, as 26 U.S.C. 111(a) requires when the refunded tax gave no benefit in the prior year, the value is 23,772.80. The prompt does not say whether the refund is taxable, the same unlisted input as 033 and 117, so the output is excluded with those two." + }, + { + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "cluster": "excl_snap_ssi_disability", + "cluster_review_action": null, + "flag": "out_of_cluster", + "final_action": "exclude_unlisted_input", + "cites_review": "excl_snap_ssi_disability", + "decided_by": "developer", + "decided_on": "2026-09-29", + "reason": "The excl_snap_ssi_disability investigation flagged this scored output out of cluster for the lead: ca_wdp_disability_eligible reads the broad is_disabled flag, so the reference of 1 follows reading B, and under reading A the law gives 0 (42 CFR 435.540(a)). The cluster's independent review agreed: under the stated facts the engine places the head in WORKING_DISABLED_BUY_IN, and that is disability Medi-Cal only if 'is disabled' means the Social Security definition. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% adult limit, so only a disability pathway leads to Medi-Cal, and California's 250% Working Disabled Program requires the Social Security definition: meets_ssi_disability_criteria, the unlisted input that already excludes this household's SNAP. On 2.15.17 with fixes/latest_final.py (verification/probe_023_medicaid.json), the reference system gives 1 under either reading (WORKING_DISABLED_BUY_IN under reading A, SENIOR_OR_DISABLED under reading B). With the program's disability test reading meets_ssi_disability_criteria, reading A gives 0 (no Medicaid category) and reading B gives 1. Rule 4 excludes the output. The pre-merge review of the release found the flag without a recorded disposition (verification/reviews/pr182_review_023.md); final_actions.json lists the exclusion under audit_exclusions." + } + ] +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/final_actions.json b/reference_audit/2026-09-28/final_actions.json new file mode 100644 index 00000000..bdb9ff8d --- /dev/null +++ b/reference_audit/2026-09-28/final_actions.json @@ -0,0 +1,219 @@ +{ + "engine": "policyengine-us 2.15.17", + "approved": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "value": 5842.39990234375, + "cause": "nj_ctc_fy2027_budget", + "basis": "New Jersey child tax credit schedule for 2026-2028 (P.L.2026, c.26, approved June 30, 2026), encoded upstream in policyengine-us#8971 (0319635b6e)." + }, + { + "scenario_id": "scenario_013", + "variable": "snap", + "value": 240.0, + "cause": "az_snap_bbce_200", + "basis": "Arizona raised its expanded categorical eligibility gross limit from 185% to 200% of poverty from benefit month 03/2026 (DES CNAP \"What's Changed on 03/23/2026\"); encoded upstream after 1.755.4." + }, + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "value": 0.0, + "cause": "school_meals_child_support_income", + "basis": "Child support received counts as household income for school meals (7 CFR 245.6(a)(5)(ii)); policyengine-us added it to the school-meal income sources after 1.755.4." + }, + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "value": 667.0, + "cause": "ny_ctc_phaseout_rounding_9425", + "basis": "New York Empire State child credit phase-out rounding corrected upstream (policyengine-us#9425)." + } + ], + "new_exclusions": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 3818.148193, + "alternative_value": 3788.988037, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py)." + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 24164.457031, + "alternative_value": 23772.800781, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py). The frozen value also counts Maryland county income tax, at the rate policyengine-us 2.15.17 assigns an unlisted county, in the federal SALT deduction through the Maryland withholding proxy (c_md_2026); the SALT cap binds at every 2026 county rate." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax when it was deducted in the prior year (it labels already-taxable inputs 'taxable'). policyengine-us 2.15.17 counts the whole refund in gross income (policyengine-us#9422, which fixed issue #9122); under 26 U.S.C. 111(a) none of it is income if the household did not itemize, or took no tax benefit, in the year it paid the tax.", + "frozen_value": 24961.339844, + "alternative_value": 24391.796875, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "unlisted_input": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)", + "note": "Found in the 2026-09-29 engine upgrade (cluster salt_refund_gross_income_9122); investigator and an independent reviewer. Alternative computed on 2.15.17 with the refund left out of gross income (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py)." + } + ], + "excluded_rechecked": [ + { + "scenario_id": "scenario_112", + "variable": "snap", + "cluster": "snap_abawd_hours_default_9261", + "reason": "Hours are unlisted, and at 0 hours the law also depends on unlisted 3-in-36 history, so no engine version resolves this. Refresh the record: frozen_value 0 (the 2.15.17 plus conventions reference), alternative_value 288, engine_version 2.15.17, and flip the alternative_reading text. At 20 or more hours, TX BBCE (165%; $5,000 assets with one vehicle up to $22,000 excluded) is met, and the net income of about $1,269 (engine) or $1,429 (farm rent counted) leaves only the $24 one-person minimum: 12 x 24 = 288. The text should say the 0 comes from the engine's no-history shortcut (the law gives up to $72), and that the stated wages imply at most 15.7 hours a week at the federal minimum wage." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "cluster": "snap_abawd_hours_default_9261", + "reason": "Two inputs are still unlisted: hours, and whether the mortgage (and the property tax) is on the occupied home. At 0 hours the value also depends on the county and on 3-in-36 history. Refresh the record: frozen_value 95, alternative_value 1,908, engine_version 2.15.17, flipped alternative_reading. The 95 is January 2026 only, from the FNS-approved 20-county waiver (letter 2024-12-16, through 2026-01-31; NJ DFD confirms) applied to the engine's Atlantic County fallback. The amount is the NJ state minimum $95 (law signed 2023-02-08; federal formula $54). The alternative of 40 hours with the mortgage counted gives shelter capped at $744, net $461, contribution $139, so 12 x 159 = 1,908. I reproduced it with my own composition (latest_alt_snap_mortgage_residence reform plus the latest_alt_unlisted_hours_40 patch) and by hand. The one-input alternatives are 1,140 (40 hours only) and 159 (mortgage only)." + }, + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "cluster": "excl_r02_ira_219g_federal", + "reason": "r02 is unfixed in 2.15.17. gov.irs.ald.deductions (2026) still lists traditional_ira_contributions; the list and the retirement variables are identical to 1.755.4 apart from uprating attributes (44001a4c24). No 219(g) or active-participant code exists, and upstream #8388 scoped 219(g) out with no follow-up. Both spouses list 401(k) deferrals, so both are active participants under 219(g)(5)(A)(i); ira_219g_magi is $543,076, far above $149,000, so the deductible limit is $0 and the engine's $2,163.84 deduction is wrong. The +$1,327.26 move is #9122 alone: salt_refund_income=0 on 2.15.17 gives exactly 106505.8984375. The alternative on 2.15.17 is 108525.578125 with the refund in AGI; 107198.3359375 applies with both readings or under a hold. Both were reproduced by the harness and by my engine-free calculation to $0.01. It also depends on the unlisted 402(g) excess allocation (106,342.59 to 108,096.20 on latest), so it should stay excluded even after an upstream r02 fix unless that allocation is ruled. NIIT is included (r25), as in the board." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "cluster": "excl_r02_ira_219g_federal", + "reason": "r02 is unfixed in 2.15.17 (same evidence as 005). The HoH head lists a $181 traditional 401(k) deferral with $100,195 of wages, so is an active participant; ira_219g_magi is $117,661, above $91,000, so the limit is $0 but the engine deducts $8.48. The +$14.85 move is #9122 alone (67.50 at 22%); refund=0 on 2.15.17 gives exactly 9563.052734375. The alternative on 2.15.17 is 9579.767578125 with the refund in AGI, or 9564.9169921875 with both readings or under a hold. My engine-free calc reproduces these (standard deduction $24,150 + 170(p) $700; CTC $2,200; CDCC 20% of $3,000). No other unlisted dependence was found. The only non-excluded output r02 moves is 082 state, by $0.59, under the $1 tolerance." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "cluster": "excl_r02_ira_219g_federal", + "reason": "r02 is unfixed in 2.15.17. The single head (76) lists a $617 traditional 401(k) deferral with $165,597 of wages, so is an active participant; ira_219g_magi is $234,414, above $91,000, so the limit is $0 but the engine deducts $28.85. The +$394.85 move is #9122 alone. Taxable income rises $1,645.22, because the engine's ct_withheld_income_tax proxy (CT single rates on federal AGI less $15,000) raises SALT by $113.77 and the 0.5% charitable floor rises $8.75; CT's own tax is unchanged at 11917.18 because CT subtracts salt_refund_income. Refund=0 gives exactly 40021.81640625. The alternative on 2.15.17 is 40423.17578125 with the refund in AGI, or 40028.3203125 with both readings or under a hold. Beyond r02 the output also depends on the engine's missing 469 limit (+24.75) and the unlisted mortgage origination date (-224.64 if grandfathered), so it should not be un-excluded on an r02 fix alone." + }, + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r11_ca_itemized_conformity", + "reason": "Both r11 and r02 are still in 2.15.17. The CA itemized, charity and misc variables and parameters, traditional_ira_contributions and gov/irs/ald/deductions.yaml are identical over 06665727d8..79be99f671, and git grep finds no active_participant or 219(g) logic. The stored, unscored value is 2.15.17 + conventions = 41,267.011719; I reproduced it in the harness and by hand. The +215.50 gap to the board splits as +168.36 from c_ca_hold_2025 (the board kept raw v1.1 for this excluded output) and +47.14 from #9122: 23.14 through the line 29 limitation (248.86 x 9.3%) and 24.00 through the exemption credits (13 to 15 phase-out steps x $6 x 2). With #9122 reverted on the conventions, the value is 41,219.867188 = 1.755.4 + r19. The output also depends on three stated-but-unresolved or unlisted facts: whether the SALT refund was taxable (40,975.25 vs 40,920.40 after r11 + r02), who issued the tax-exempt interest (+833.41 if non-California), and how the over-limit 401(k) deferrals are cut (-491.85). The exclusion should therefore also carry an unlisted-input reason, and alternative_value should record both SALT-refund readings rather than a single 'law-correct' 40,975.25." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r11_ca_itemized_conformity", + "reason": "r11 is still present. ca_itemized_deductions_pre_limitation carries the federal charitable_deduction (0.5% floor from 2026: 14,847.67 instead of 15,393.41) and the federal misc_deduction (0 since 2018). R&TC 17076(c) (Stats. 2025 ch. 231, SB 711 Sec. 11, effective 2025-10-01) and the conformity date in 17024.5(a)(1)(Q) (IRC as of 2025-01-01) mean California allows both. The law-correct value is 1,968.81; I reproduced it by harness and by hand. Federal AGI is 109,149.16 and CA AGI is 91,524.41; itemized deductions are 8,829 + 15,393.41 + 6,167.56 = 30,389.97; CA taxable income 61,134.44 gives tax of 2,274.81, less 306 of exemption credits. That equals the Sept 22 alternative. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 2,505.870117 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. I found no unlisted-input dependence: medical stays under the floor under any reading, and the prompt lists no refund and no tax-exempt interest. This is a pure engine defect, so the output could be un-excluded once r11 is fixed upstream. The value uses the rate schedule, as for every CA output (FTB's tax table for taxable income of $100,000 or less could differ by about a dollar)." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r11_ca_itemized_conformity", + "reason": "r11 and r02 are still present. The engine's federal floor removes 797.93 of charity (-63.83). Both spouses defer into a 401(k), which makes them active participants, and MAGI of 159,729.30 exceeds the $149,000 top of the 2026 joint range (IRS Notice 2025-67, posted 2025-11-13), so the $144.26 IRA deduction is fully phased out (+11.54 after r11). Together these give 4,588.48, the Sept 22 alternative. I confirm the new educator-expense defect: FTB 2025 Schedule CA line 11 says 'California law does not conform to federal law regarding educator expenses', but 2.15.17's ca_additions adds only ca_hsa_addition. Adding back $337.50 x 8% = +27.00 gives a fully law-correct 4,615.48 (harness and hand calculation agree). The exclusion reason should cite this third defect. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 4,640.777344 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. Possible extra reading: if both premium lines are summed, the output moves -241.62." + }, + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r07_idaho_health_premiums", + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 755.7783203125, identical to v11_1755. latest_conventions gives 761.5570068359375 (+5.7787 = 0.053 x (4920.0313 - 4811)). latest_conventions + r07_v2 gives 651.3170166. The subtractions.yaml and main/single.yaml files are identical between 1.755.4 and 2.15.17, and the premium subtraction is still absent, so the r07 defect is still present. The board row is the raw 755.7783 because the c_id_hold_2025 revision has excluded_outputs_untouched: true. By hand: taxable SS 13,784.125, Idaho TI 19,180, and 0.053 x (19,180 - 4,811) = 761.557. With the premium subtraction, 0.053 x (17,100 - 4,811) = 651.317, which matches the exclusion record's alternative_value. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r07_idaho_health_premiums", + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 2435.2783203125, identical to v11_1755. latest_conventions gives 2441.05712890625. latest_conventions + r07_v2 gives 2176.05712890625 (-265.00 = 0.053 x 5,000). id_subtractions is 0 and the r07 defect is still present. By hand: 0.053 x (50,868.68 - 4,811) = 2441.057, and 0.053 x (45,868.68 - 4,811) = 2176.057. The alternative also assumes the $5,000 is not paid through a pre-tax salary reduction. The prompt leaves that unstated (the household has wages and ESI), which gives an additional unlisted-input reason to keep this output excluded. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r32_wi_capital_gain_distributions", + "reason": "Both defects are still present in policyengine-us 2.15.17, and I confirmed each directly.\n- diff -rq shows wi_capital_gain_loss_subtraction.py and wi_income_tax_before_refundable_credits.py byte-identical to 1.755.4, and git log 06665727d8..79be99f671 shows no commit touching either file.\n- My probe of 2.15.17 plus latest_conventions shows non_sch_d_capital_gains 3,753 in federal AGI (50,580.71; net_capital_gain 4,681) but wi_capital_gain_loss_subtraction 0.\n- The probe also shows wi_income_tax 0 (the min with the line-16 exclusion path, since wi_retirement_income_exclusion_amount is 24,000 and wi_retirement_income_exclusion_tax is 0) while wi_income_tax_before_refundable_credits is 464.1828.\nHarness runs on 042/091:\n- latest_conventions: 464.1828.\n- Plus r32: 408.698425.\n- Plus r32 and the r06 elected-path port: 0.0.\nAn engine-free recompute on the 2026 1-ES parameters reproduces 464.1827 (standard path, no r32), 408.6984 (with r32) and 0.0 (elected line 16). 71.05(6)(b)54m lets a filer aged 67 or older subtract up to $24,000 of qualified-plan and IRA distributions, and 54m.d forfeits the credits. On the elected path, WI income 8,608.81 is below the 13,960 standard deduction plus 950 exemptions, so tax is 0, and 0 is below the standard path's net 408.70, so the filer elects it. The 0.0 holds under every alternative reading I tried:\n- only the 19,200 IRA counts as line 16 (WI income 14,534.71 or 13,408.81, both below 14,910);\n- the farm loss is double-counted;\n- qualified_bdc_income is added.\nr06 alone suffices; r32 does not change the alternative. #9122 does not apply (salt_refund_income is 0). If the record is refreshed, engine value 464.1828 and alternative 0.0." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r32_wi_capital_gain_distributions", + "reason": "The r32 defect is still present in 2.15.17. The subtraction file is byte-identical to 1.755.4 and no commit in the range touches it. My probe on 2.15.17 plus conventions shows non_sch_d_capital_gains 1,170 in AGI 36,182.56 but wi_capital_gain_loss_subtraction 0.\nThe harness gives 895.81311 with latest_conventions and 878.515747 with r32 added; the engine-free recompute gives 895.8131 and 878.5158. 71.05(6)(b)9 subtracts 30% of long-term gain, IRC 852(b)(3)(B) treats capital gain dividends as long-term, and 2025 SB line 5 applies the 30% exclusion to distributions reported without Schedule WD.\nThere is a second, independent reason to keep this excluded. 2.15.17 does not model the WI medical care insurance subtraction: 71.05(6)(b)42, which covers an employee whose employer pays part of the cost, and 2025 SB line 6, Worksheet 2. The prompt lists $1,800 of other health insurance premiums (repeated as 'health insurance premiums excluding Medicare Part B') without saying whether they were paid pre-tax. The law-correct value is:\n- 878.52 if the premiums were pre-tax;\n- 789.81 if the $1,800 was paid after tax, the reading the prompt's unlisted-boolean-false rule and the engine's own AGI point to;\n- 701.11 if a reader sums both lines.\nThe exclusion record should list root causes r32_wi_capital_gain_distributions plus this unlisted pre-tax-premium ambiguity, and it should label 878.515747 as the r32-only correction, not the unique law-correct value." + }, + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "cluster": "excl_r01_ira_compensation", + "reason": "Root cause r01 is still present in 2.15.17. traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, above_the_line_deductions.py and gov/irs/ald/deductions.yaml have identical blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. The dependent's $18.03 IRA contribution (stated in the prompt; $0 wages) is still in the joint return's ALD (19,926.22 = 19,800 loss ALD + 108.19 + 18.03). Under 26 U.S.C. 219(a), (b)(1)(B) and (c) (LII, re-fetched this session) it is not deductible there. The review harness gives 2.15.17 + latest_conventions = 4,598.476074, which equals 1.755.4 + r19_wi_convention (re-run this session). Adding r01_v2 gives 4,599.621582, and also reproduces the 064 federal (4,441.455078) and 003 federal (22,400.654297) alternative values exactly; the only other output it moves is scenario_085 federal, by +0.089. Under the board's formula convention the defect moves this output by $1.15, beyond tolerance, so it stays excluded. Caveat: Wis. Stat. 71.05(22)(dp)2 makes DOR's $500-bracket table determinative. On that reading r01's effect is $0.96, and 4,598.476 is $0.64 from a table-law value of about 4,597.83, but the 2026 table was not published before the freeze. Exclusion is the safe call under either reading. Update frozen_value to 4598.476074 and engine_version to 2.15.17. Keep alternative_value at 4599.621582, and note in the exclusion record that it is formula-convention based." + }, + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "cluster": "excl_niit_scope_020", + "reason": "Confirmed. On 2.15.17, sweep_latest.py gives 68334.734375 both raw and with latest_conventions.py. latest_conventions plus r25_niit_excluded gives 68112.09375, exactly the exclusion record's alternative_value, so the two readings still differ by 222.64 (more than the $1 tolerance). The r25 root cause is the benchmark_specs.json output definition ('federal individual income tax after nonrefundable credits and before refundable credits'), which is unchanged. The engine formula (income_tax_before_refundable_credits.py; git diff 06665727d8..79be99f671 is empty; cmp of the installed files is identical) still adds net_investment_income_tax, and net_investment_income_tax.py has the same sha1 in both engines. The move comes only from #9616 (3f029abf5b, 2026-09-27), which fixed the column-shifted sales tax table. The IRS figure it carries, the 2025 TX size-1 $300k+ cell of 1,595, was published in December 2025, before the freeze. No hold module is needed, because raw and conventions agree." + }, + { + "scenario_id": "scenario_023", + "variable": "snap", + "cluster": "excl_snap_ssi_disability", + "reason": "Reproduced on the harness. latest_conventions gives 408 ($34/mo); raw 2.15.17 gives 462. Hand check: gross $2,120.22 (wages $1,453.55 plus 403(b) $666.67); minus 20% earned ($290.71) and the $209 standard deduction gives $1,620.51; minus the $744 cap (binds with or without the SUA) gives $876.51, rounded to $877; 30% is $263.10, rounded up to $264; $298 - $264 = $34. The unlisted input still decides the output under law. Under reading B, federal SSI is $0 (countable $1,340.94 > $994) and CA SSP is $0 (countable income over the $1,335.81 standard). She has no MAGI route (141.19% FPL > 138%). Her Medi-Cal is a disability pathway (engine: SENIOR_OR_DISABLED), and receiving it makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), both text-verified. 2.15.17 does not model that route, so flipping meets_ssi_disability_criteria alone gives 408 (rv_probe conv_B). Record the alternative as 3,576 with the engine's CA heat-and-eat SUA, and 2,832 if no LIHEAP payment is inferred (a SUA of about $206 suffices, not $384). Reword the record: in 2.15.17 SSI criteria no longer confer SNAP status (#9345); the route is Medi-Cal receipt." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "cluster": "excl_snap_ssi_disability", + "reason": "Reproduced. latest_conventions gives 2,628 ($219/mo); raw gives 2,682. Hand and engine agree. Head SSI is $994 - $391.42 = $602.58/mo; gross is $1,472.42; minus $173.33 (20% of earned) and $209 gives $1,090.08, which rounds to $1,090; 30% is $327; $546 - $327 = $219. Under reading B (rv_probe conv_B, and with the r30 port and the title XIX route), both spouses get SSI at the couple rate: $1,491 - $392.42 = $1,098.58/mo in total ($6,591.50 per person per year). Net is $1,586, 30% is $476, $546 - $476 = $70, and 840 per year, so the recorded alternative of 840 stands on 2.15.17 plus the hold. LA always_standard is false, so there is no SUA and the r30 port cannot move it. The dependence runs through actual SSI receipt, an unlisted-input ambiguity that no engine version resolves." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "cluster": "excl_snap_ssi_disability", + "reason": "Reproduced. latest_conventions gives 8,556 ($713/mo); raw gives 8,637. Pre-shelter net is $690.73 (earned $492.89 plus the engine's MT TANF of $505.41/mo, minus $98.58 and $209). With the $799 SUA (FNS FY2026 SUA table: MT HCSUA $799), net is $237, 30% rounds up to $72, and $785 - $72 = $713. r30 is confirmed still present in 2.15.17. The engine grants the always_standard SUA with has_heating_cooling_expense = 0 and no elderly-or-disabled member. My r30 port (rv_fix_r30.py, full sweep rv_r30_full.csv) moves only 080 and 100, giving 100 = 6,924: net $691, $208, $785 - $208 = $577. Keep the output excluded under both bases. Add r30_snap_heat_and_eat_sua with the stated-facts corrected value 6,924. Do not drop the unlisted-input basis. Under reading B, the working head qualifies for Montana MWD (ABD 201-6; CMA 001 $15,000 resource limit), and 42 CFR 435.404 lets her select it. Receipt makes her elderly or disabled under 7 CFR 271.2(11), giving 8,844 with the engine's SUA (the existing alternative) or 7,116 without. The mechanism text should change from 'SSI criteria' to 'MWD (title XIX) receipt', which 2.15.17 does not model." + }, + { + "scenario_id": "scenario_118", + "variable": "snap", + "cluster": "excl_snap_mortgage_residence", + "reason": "Keep excluded, now for three independent reasons.\n1. The mortgage-residence ambiguity (r15_snap_mortgage_interest), which no engine version resolves. 2.15.17 still counts only mortgage_payments, not mortgage interest, as a SNAP shelter cost.\n2. A second unlisted input, the county. The prompt gives only NY, and New York's FY2026 HCSUA is $877, $988 or $1,062 by region. That gives 2,868, 3,276 or 3,540 on 2.15.17 + conventions.\n3. An unfixed 2.15.17 defect: the NY State Supplement Program ($87 a month for an individual living alone in 2026, per the OTDA chart revised 2025-10-27; pre-freeze law) is missing from SNAP income. With it, the value would be 2,400, 2,796 or 3,072 by region.\nRecord 2,868 as the regenerated engine value, replacing the stale raw-1.755.4 2,903.94. Label it as 2.15.17 + latest_conventions under the engine's default county (Albany, Rest of State SUA) without NY SSP, not as the law-correct stated-facts value. Add the NY SSP omission as an unfixed-defect root cause and the county/SUA region as a second unlisted input. The alternative value stays 3,576 (298 x 12). It is the same across all three regions, with or without SSP, and under every 7 CFR 273.10(e) rounding option. Scoring is unaffected." + }, + { + "scenario_id": "scenario_080", + "variable": "snap", + "cluster": "excl_r30_snap_heat_and_eat_sua", + "reason": "The r30 defect is unfixed in 2.15.17. snap_state_using_standard_utility_allowance.py is byte-identical to 1.755.4 and returns p.always_standard[state]. always_standard.yaml's only change since 06665727d8 is metadata (d7fc17abfe), and PA stays true from 2015-10-01. snap_utility_allowance_type still grants the SUA on has_heating_cooling | always_sua. The only upstream fix in progress (draft PR #9610) covers California only. My trace on 2.15.17 + latest_conventions shows always_sua=True, has_heating_cooling_expense=False, is_usda_disabled=False and has_snap_elderly_disabled_member=False. The $857 SUA is capped at $744, net income is 0, and the benefit is $298 x 12 = 3,576. With r30 added it is $270 x 12 = 3,240 (net $92, contribution $28). Under 7 U.S.C. 2014(e)(6)(C)(iv)(I) as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04, before the freeze), the engine's own reading (no elderly or disabled member) should give 3,240, so 3,576 is defect-driven under that reading. With is_usda_disabled forced True, 2.15.17 gives 3,576 with or without r30, and the household stays eligible. The prompt supports both readings. It says 'is disabled' but gives no 2012(j) receipt route. Its $3,600 'financial assistance' is unexplained and could be disability-based general assistance. Its $2,000 'spm unit energy subsidy' could be LIHEAP or a state-law payment. It states no certification timing, which matters under the FNS recertification transition. Model answers split the same way: 15 near the SUA-conferred value (3,492-3,576) against 3,240, 3,228 and 2,964-2,988 on other readings. The output is therefore both defect-affected and indeterminate on the stated facts, and stays excluded. The -$20.04 move against the board (3,596.04 to 3,576) comes from the FY2026 SNAP hold convention: the FY2027 COLA memo is dated 2026-08-21, after the freeze. It does not come from r30. Scoring is unaffected." + } + ], + "audit_exclusions": [ + { + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "flagged_by": "excl_snap_ssi_disability", + "decided_on": "2026-09-29", + "decided_by": "developer", + "basis": "Rule 4: the reference turns on an input the prompt does not state. The excl_snap_ssi_disability investigation flagged the scored output out of cluster, and its independent review agreed. The head's MAGI is 141.2% of the federal poverty guideline, above the 138% adult limit, so only a disability pathway leads to Medi-Cal, and California's 250% Working Disabled Program requires the Social Security definition of disability (42 CFR 435.540(a)): meets_ssi_disability_criteria, the input that already excludes this household's SNAP. On 2.15.17 with fixes/latest_final.py and the program's disability test reading that input, reading A (the head does not meet the criteria) gives 0 and reading B gives 1; the reference system gives 1 under either reading (verification/probe_023_medicaid.json). The reference did not move, so the engine_upgrade revision does not list it.", + "evidence": [ + "clusters.json reconciliations", + "verification/reviews/pr182_review_023.md", + "verification/probe_023_medicaid.json" + ], + "exclusion": { + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "reason_code": "reference_depends_on_unlisted_input", + "unlisted_input": "meets_ssi_disability_criteria", + "alternative_reading": "The household fact 'is disabled' is read as not meeting the Social Security definition of disability (the engine input meets_ssi_disability_criteria, which the prompt does not list). The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. California's 250% Working Disabled Program requires the Social Security definition (42 CFR 435.540(a)); policyengine-us 2.15.17 tests the broad is_disabled flag instead and places the head in that program (medicaid_category WORKING_DISABLED_BUY_IN).", + "frozen_value": 1.0, + "alternative_value": 0.0, + "engine_version": "policyengine-us 2.15.17", + "decided_on": "2026-09-29", + "decided_by": "developer", + "judge_verdict": "llm_error (claude-opus-5-5, 2026-09-29)", + "note": "Flagged in the 2026-09-29 engine upgrade by the investigation of cluster excl_snap_ssi_disability and its independent review, and excluded on review of the release (reference_audit/2026-09-28/final_actions.json, audit_exclusions). Alternative computed on 2.15.17 with the publication conventions and the Working Disabled Program's disability test reading meets_ssi_disability_criteria (reference_audit/2026-09-28/scripts/probe_023_medicaid.py); unmodified, 2.15.17 gives 1 under either reading. The same unlisted input excludes this household's SNAP." + } + } + ] +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/fixes/alt_conventions_r25.py b/reference_audit/2026-09-28/fixes/alt_conventions_r25.py new file mode 100644 index 00000000..f0cae09b --- /dev/null +++ b/reference_audit/2026-09-28/fixes/alt_conventions_r25.py @@ -0,0 +1,32 @@ +"""Alternative reading for the r25 exclusion on policyengine-us 2.15.17 (not a fix module). + +latest_conventions (all nine pre-freeze-law conventions) plus r25_niit_excluded (the reading of +federal_income_tax_before_refundable_credits that leaves out the IRC 1411 NIIT). Used only to +recompute the exclusion record's alternative_value on 2.15.17. +""" +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIXES = Path("/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes") +FIX_ID = "latest_conventions_plus_r25_niit_excluded" +DESCRIPTION = "latest_conventions + r25_niit_excluded (the NIIT-excluded alternative reading) on 2.15.17" + + +def _load(name): + spec = importlib.util.spec_from_file_location(f"alt020_{name}", FIXES / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.reform + + +PARTS = (_load("latest_conventions"), _load("r25_niit_excluded")) + + +class reform(Reform): + def apply(self): + for part in PARTS: + part.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py b/reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py new file mode 100644 index 00000000..8354c071 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py @@ -0,0 +1,49 @@ +"""Alternative-value module (not a convention): latest_conventions + r02_ira_219g_v2 on 2.15.17. + +Purpose: recompute, on policyengine-us 2.15.17, the "alternative_value" recorded in +reference_exclusions.json for outputs excluded under root cause r02_ira_219g (IRC 219(g) +active-participant phase-out of the traditional IRA deduction). Those alternative values +were computed on 1.755.4 on 2026-09-22 (r02_ira_219g_v2 with every convention and upstream +fix). This module is for the excluded outputs' recorded alternative values only. It is not +part of the scored reference: the outputs stay excluded while 2.15.17 still deducts +traditional IRA contributions without the 219(g) phase-out (gov.irs.ald.deductions 2026 +list names `traditional_ira_contributions` directly; no 219(g) code anywhere in 2.15.17). + +Composition: the nine pre-freeze-law conventions (latest_conventions.py) and the verified +r02 fix (r02_ira_219g_v2.py, which reuses r02_ira_219g.py's 2026 arithmetic). The r02 part +reads `gov.irs.gross_income.sources`, so on 2.15.17 its 219(g)(3)(A) MAGI includes +`salt_refund_income`, which PR #9422 (issue #9122) added to federal gross income. + +Written for triage cluster excl_r02_ira_219g_federal, 2026-09-28. +""" + +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_alt_r02_ira_219g" +DESCRIPTION = ( + "Alternative values for r02-excluded outputs on 2.15.17: latest_conventions plus " + "r02_ira_219g_v2 (IRC 219(g) phase-out, 2026 ranges from Notice 2025-67)" +) + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"latest_alt_r02_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +CONVENTIONS = _load("latest_conventions") +R02 = _load("r02_ira_219g_v2") + + +class reform(Reform): + def apply(self): + CONVENTIONS.reform.apply(self) + R02.reform.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py b/reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py new file mode 100644 index 00000000..6402c6e8 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py @@ -0,0 +1,73 @@ +"""Alternative reading (not a reference fix, not a hold): the listed SALT refund is not income. + +Written for triage cluster salt_refund_gross_income_9122, 2026-09-28, on policyengine-us +2.15.17 plus every pre-freeze-law convention (latest_conventions.py). + +What 2.15.17 does. Upstream #9122 (commit 316e7832a1, 2026-09-08; merged as #9422, +e990b4a6f8, 2026-09-16) added `salt_refund_income` to gov.irs.gross_income.sources and +documented the input as "Taxable state and local income tax refunds, credits, or offsets +reported on Form 1040, Schedule 1, line 1". 1.755.4 (the board) never counted the input in +federal gross income. The prompt shows it as "state and local tax refund income: $X", +without the word "taxable" and with no prior-year facts. + +Why an alternative reading exists. 26 U.S.C. 111(a) (text last amended by Pub. L. 99-514, +1986) excludes a recovery "to the extent such amount did not reduce the amount of tax +imposed". The 2025 Form 1040 instructions (cover dated Feb 25, 2026), Schedule 1 line 1: +"None of your refund is taxable if, in the year you paid the tax, you either (a) didn't +itemize deductions, or (b) elected to deduct state and local general sales taxes instead of +state and local income taxes." Pub. 525 (2025), "Deductions not itemized", says the same, +and Rev. Rul. 2019-11 (2019-17 I.R.B., 2019-04-22) limits the taxable part further under the +SALT cap. Whether the household itemized, and deducted income tax, in the prior year is an +unlisted status fact; the prompt says to treat unlisted status inputs as false. Read that +way, none of the refund is gross income. + +Implementation. The patch sets every listed `salt_refund_income` to 0, so the refund is in +neither federal gross income nor any state base or subtraction (the state subtractions of +the refund that 2.15.17 applies then have nothing to remove, which is the state result of a +refund that never entered federal AGI). The reform is latest_conventions unchanged. +""" + +import copy +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_alt_salt_refund_no_prior_benefit" +DESCRIPTION = ( + "latest_conventions with every listed salt_refund_income read as not income (IRC 111: no " + "prior-year tax benefit); recomputes an alternative_value, not a reference" +) +FIELD = "salt_refund_income" + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"alt_salt_refund_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +CONVENTIONS = _load("latest_conventions").reform + + +def _zero(value): + if isinstance(value, dict): + return {period: 0.0 for period in value} + return 0.0 + + +def patch(situation, scenario): + situation = copy.deepcopy(situation) + for person in situation.get("people", {}).values(): + if FIELD in person: + person[FIELD] = _zero(person[FIELD]) + return situation + + +class reform(Reform): + def apply(self): + CONVENTIONS.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py b/reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py new file mode 100644 index 00000000..6ae64248 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py @@ -0,0 +1,135 @@ +"""Alternative reading for the excluded output scenario_118 snap, on policyengine-us 2.15.17 +plus every pre-freeze-law convention (latest_conventions.py). + +NOT a reference fix and NOT a hold convention. The output stays excluded: its root cause +(triage/root_causes.json, r15_snap_mortgage_interest) is an unlisted input, "whether the +listed home mortgage interest is on the home the SNAP household occupies". This module +only recomputes the exclusion record's alternative_value on the new stack, the way +r15_snap_mortgage_interest_v2.py (written for 1.755.4) did on the 22c board stack. + +Alternative reading (as in r15 v2): the listed mortgage interest is on the occupied home, +so it is a continuing mortgage charge counted as a shelter cost (7 CFR 273.9(d)(6)(ii)(A): +"Continuing charges for the shelter occupied by the household, ... including mortgage +payments ... and interest on such payments"). Unlisted principal is not invented; only +max(interest - mortgage_payments, 0) is added, so a reported payment is never counted +twice. First-home versus person interest is chosen per tax unit, as in r15 v2. + +Port to 2.15.17. 2.15.17's snap_excess_shelter_expense_deduction differs from 1.755.4 in +two places (diff of the installed files): actual housing costs are multiplied by +snap_expense_counted_share (7 CFR 273.11(c)(2)(iii)) with the utility allowance added +unprorated, and the uncapped branch reads has_snap_elderly_disabled_member instead of +has_usda_elderly_disabled. This module copies the 2.15.17 formula verbatim and adds the +mortgage-interest term to the prorated actual housing costs. housing_cost and +mortgage_payments are identical in both versions and still exclude mortgage interest. +""" + +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * + +FIX_ID = "latest_alt_snap_mortgage_residence" +DESCRIPTION = ( + "latest_conventions plus the r15 alternative reading (listed mortgage interest is on the " + "occupied home, a SNAP shelter cost), ported to 2.15.17; recomputes an alternative_value, " + "not a reference" +) + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"alt_mortgage_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +CONVENTIONS = _load("latest_conventions").reform + + +class snap_mortgage_interest_shelter_cost(Variable): + value_type = float + entity = SPMUnit + label = "SNAP shelter cost from listed mortgage interest beyond mortgage_payments" + unit = USD + definition_period = YEAR + reference = ( + "https://www.ecfr.gov/current/title-7/subtitle-B/chapter-II/subchapter-C/part-273/subpart-D/section-273.9#p-273.9(d)(6)(ii)(A)", + ) + + def formula(spm_unit, period, parameters): + person = spm_unit.members + head = person("is_tax_unit_head", period) + first = person.tax_unit("first_home_mortgage_interest", period) + second = person.tax_unit("second_home_mortgage_interest", period) + interest_by_person = where( + first + second > 0, + head * first, + person("home_mortgage_interest", period), + ) + occupied_home_interest = spm_unit.sum(interest_by_person) + mortgage_payments = spm_unit("mortgage_payments", period) + return max_(occupied_home_interest - mortgage_payments, 0) + + +class snap_excess_shelter_expense_deduction(Variable): + value_type = float + entity = SPMUnit + definition_period = MONTH + documentation = ( + "Excess shelter expense deduction for calculating SNAP benefit amount" + ) + label = "SNAP shelter deduction" + reference = ("United States Code, Title 7, Section 2014(e)(6)",) + unit = USD + + def formula(spm_unit, period, parameters): + # policyengine-us 2.15.17's formula, except for the mortgage-interest term. + p = parameters(period).gov.usda.snap.income.deductions.excess_shelter_expense + net_income_pre_shelter = spm_unit("snap_net_income_pre_shelter", period) + subtracted_income = p.income_share_disregard * net_income_pre_shelter + expense_share = spm_unit("snap_expense_counted_share", period) + utility_allowance = spm_unit("snap_utility_allowance", period) + mortgage_interest = ( + spm_unit("snap_mortgage_interest_shelter_cost", period.this_year) + / MONTHS_IN_YEAR + ) + housing_cost = ( + expense_share + * (add(spm_unit, period, ["housing_cost"]) + mortgage_interest) + + utility_allowance + ) + uncapped_ded = max_(housing_cost - subtracted_income, 0) + state_group = spm_unit.household("snap_region_str", period) + ded_cap = p.cap[state_group] + capped_ded = min_(uncapped_ded, ded_cap) + has_elderly_disabled = spm_unit("has_snap_elderly_disabled_member", period) + non_homeless_shelter_deduction = where( + has_elderly_disabled, uncapped_ded, capped_ded + ) + state = spm_unit.household("state_code_str", period) + homeless_deduction = p.homeless.deduction * p.homeless.available[state] + return where( + spm_unit.household("is_homeless", period) + & (housing_cost > 0) + & (homeless_deduction > non_homeless_shelter_deduction), + homeless_deduction, + non_homeless_shelter_deduction, + ) + + +class alternative_reading(Reform): + """The r15 alternative reading alone (no conventions).""" + + def apply(self): + self.update_variable(snap_mortgage_interest_shelter_cost) + self.update_variable(snap_excess_shelter_expense_deduction) + + +class reform(Reform): + def apply(self): + CONVENTIONS.apply(self) + alternative_reading.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py b/reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py new file mode 100644 index 00000000..c6799314 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py @@ -0,0 +1,56 @@ +"""Alternative reading for the unlisted-hours exclusions (scenario_056, scenario_112 snap) +on policyengine-us 2.15.17 plus every pre-freeze-law convention. + +NOT a reference fix and NOT a hold convention. Both outputs stay excluded: their root +cause (triage/root_causes.json, r14_unlisted_weekly_hours_v2) is an unlisted input, +weekly_hours_worked_before_lsr. This module only recomputes, on the new stack, the value +under the reading the 22c board used: a person whose prompt shows no weekly hours works +40 hours a week (the 1.755.4 default of weekly_hours_worked_before_lsr). + +On 2.15.17 the readings swap. Upstream 82745ca239 (PR #9261, merged a112cc5a0a on +2026-08-12) made the default 0, which is the prompt's "Treat any unlisted numeric input +as 0" reading, so the 2.15.17 reference now takes the zero-hours reading and the +40-hour reading becomes the alternative. + +Composition: latest_map_stated_hours (conventions plus stated usual weekly hours mapped +to weekly_hours_worked_before_lsr), then 40 hours for every person with no stated hours. +Diffing this sweep against out/latest_map_stated_hours.csv lists every output that +depends on the unlisted-hours reading. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +FIX_ID = "latest_alt_unlisted_hours_40" +DESCRIPTION = ( + "latest_map_stated_hours plus weekly_hours_worked_before_lsr = 40 for every person with " + "no stated weekly hours (the 22c board's reading); recomputes an alternative_value, not a " + "reference" +) +TARGET = "weekly_hours_worked_before_lsr" +ALTERNATIVE_HOURS = 40.0 + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"alt_hours40_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_mapping = _load("latest_map_stated_hours") +reform = _mapping.reform + + +def patch(situation: dict, scenario) -> dict: + situation = _mapping.patch(situation, scenario) + year = str(scenario.year) + for person in situation["people"].values(): + if TARGET not in person: + person[TARGET] = {year: ALTERNATIVE_HOURS} + return situation diff --git a/reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py b/reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py new file mode 100644 index 00000000..fdfad2c5 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py @@ -0,0 +1,130 @@ +"""c_ca_hold_2025 on policyengine-us 2.15.17: California's 2025 published indexed amounts held for 2026. + +Rule (reference sidecar, 2026-09-22): a scored reference follows from the stated facts +and from law published before the 2026-07-03 reference freeze. California's 2026 +indexed amounts rest on the June 2026 California CPI (BLS 2026-07-14; DIR 2026-08-12), +so the last amounts FTB published before the freeze are the 2025 amounts. + +Port of r19_ca_convention.py (written for 1.755.4). What 2.15.17 carries, read from its +built parameter tree (latest/port_state/p21517.json, classify.txt, proj_21517.json): + * Every amount r19 set is still a CPI projection in 2.15.17 (2026 value identical to + 1.755.4's), so all of r19's VALUES are kept. + * The CalEITC final phase-out breakpoints are now projected from the 2019 $200/$505 + bases without annual rounding: 2.15.17 gives $251/$635 for 2025 and $257/$649 for + 2026. The statutory derived amount under RTC 17052(o) with annual rounding under + 17041(h) is $252/$636 (r19 derivation), held for 2025 and 2026 as before. + * NEW in this port: 30 amounts that 1.755.4 carried at their 2025 values but 2.15.17 + now uprates by California CPI (upstream PolicyEngine/policyengine-us#9059, commit + df3482f4ef 2026-07-21, "fix inert uprating on six CA breakdown parameters"; #9429, + commit 2572674b97 2026-09-16). They are held at the 2025 amounts FTB published: + standard deduction $5,706 / $11,412 + (FTB Tax News, October 2025, "2025 Indexing" table; + https://www.ftb.ca.gov/about-ftb/newsroom/tax-news/2025/10.html) + AMT exemption $92,749 / $123,667 / $61,830; AMTI phase-out lower + $347,808 / $463,745 / $231,868 and upper $718,804 / $958,413 / $479,188 + (FTB 2025 Schedule P (540) instructions, exemption worksheet; + https://www.ftb.ca.gov/forms/2025/2025-540-p-instructions.html) + exemption-credit phase-out and itemized-deduction limitation AGI thresholds + $252,203 / $504,411 / $378,310 (FTB 2025 Form 540 booklet and 2025 Schedule CA + (540) instructions; https://www.ftb.ca.gov/forms/2025/2025-540-booklet.html, + https://www.ftb.ca.gov/forms/2025/2025-540-ca-instructions.html) + FTB's October 2025 Tax News says the complete 2025 amounts are posted in late + December 2025, i.e. before the freeze. + +Only parameters change: formulas are 2.15.17's. 2.15.17 applies a reform after its +uprating pass (policyengine_us/system.py), so a 2026 update is not re-uprated. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.periods import period +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_ca_hold_2025" +DESCRIPTION = ( + "California 2025 published indexed amounts held for 2026 on policyengine-us 2.15.17 " + "(r19 set plus the 30 amounts 2.15.17 newly projects)." +) +P = "gov.states.ca.tax.income." +STATUSES = ("SINGLE", "SEPARATE", "JOINT", "SURVIVING_SPOUSE", "HEAD_OF_HOUSEHOLD") + +# ---- r19_ca_convention.py VALUES (unchanged) ------------------------------------- +VALUES_2026 = {} +BRACKETS = { + "single": [11079, 26264, 41452, 57542, 72724, 371479, 445771, 742953], + "separate": [11079, 26264, 41452, 57542, 72724, 371479, 445771, 742953], + "joint": [22158, 52528, 82904, 115084, 145448, 742958, 891542, 1485906], + "surviving_spouse": [22158, 52528, 82904, 115084, 145448, 742958, 891542, 1485906], + "head_of_household": [22173, 52530, 67716, 83805, 98990, 505208, 606251, 1010417], +} +for _status, _values in BRACKETS.items(): + for _i, _value in enumerate(_values, 1): + VALUES_2026[P + f"rates.{_status}[{_i}].threshold"] = _value +VALUES_2026.update({ + P + "exemptions.amount": 153, + P + "exemptions.dependent_amount": 475, + P + "credits.earned_income.eligibility.max_investment_income": 4814, + P + "credits.earned_income.phase_out.final.start[0].amount": 252, + P + "credits.earned_income.phase_out.final.start[1].amount": 636, + P + "credits.foster_youth.amount[1].amount": 1189, + P + "credits.foster_youth.phase_out.start": 27425, + P + "credits.young_child.amount": 1189, + P + "credits.young_child.loss_threshold": 35640, + P + "credits.young_child.phase_out.start": 27425, +}) +for _i, _value in enumerate([4661, 6998, 9823]): + for _branch in ["earned_income_amount", "phase_out.start"]: + VALUES_2026[P + f"credits.earned_income.{_branch}[{_i}].amount"] = _value +for _status in STATUSES: + VALUES_2026[P + f"credits.renter.income_cap.{_status}"] = ( + 53994 if _status in ("SINGLE", "SEPARATE") else 107988 + ) + +# ---- new for 2.15.17: amounts it now projects, held at FTB's 2025 amounts ---------- +NEW_IN_2_15_17 = {} + + +def _by_status(path, single, joint, head, separate): + for status, value in { + "SINGLE": single, + "JOINT": joint, + "SURVIVING_SPOUSE": joint, + "HEAD_OF_HOUSEHOLD": head, + "SEPARATE": separate, + }.items(): + NEW_IN_2_15_17[P + path + "." + status] = value + + +_by_status("deductions.standard.amount", 5_706, 11_412, 11_412, 5_706) +_by_status("amt.exemption.amount", 92_749, 123_667, 92_749, 61_830) +_by_status("amt.exemption.amti.threshold.lower", 347_808, 463_745, 347_808, 231_868) +_by_status("amt.exemption.amti.threshold.upper", 718_804, 958_413, 718_804, 479_188) +_by_status("exemptions.phase_out.start", 252_203, 504_411, 378_310, 252_203) +_by_status("deductions.itemized.limit.agi_threshold", 252_203, 504_411, 378_310, 252_203) +assert len(NEW_IN_2_15_17) == 30 +VALUES_2026.update(NEW_IN_2_15_17) + +# The statutory 2025 CalEITC breakpoints also replace 2.15.17's projected 2025 entry. +VALUES_2025 = { + P + "credits.earned_income.phase_out.final.start[0].amount": 252, + P + "credits.earned_income.phase_out.final.start[1].amount": 636, +} + + +def modify(parameters): + seen = set() + for param in parameters.get_descendants(): + if not isinstance(param, Parameter): + continue + if param.name in VALUES_2026: + param.update(period=period("year:2026-01-01:1"), value=VALUES_2026[param.name]) + seen.add(param.name) + if param.name in VALUES_2025: + param.update(period=period("year:2025-01-01:1"), value=VALUES_2025[param.name]) + missing = set(VALUES_2026) - seen + assert not missing, sorted(missing) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(modify) diff --git a/reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py b/reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py new file mode 100644 index 00000000..9fc2b7fe --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py @@ -0,0 +1,63 @@ +"""c_id_hold_2025 on policyengine-us 2.15.17: Idaho's 2025 zero-rate thresholds held for 2026. + +Rule (reference sidecar, 2026-09-22): Idaho's 2026 zero-rate taxable income thresholds +hold the 2025 amounts ($4,811 single; $9,622 joint and head of household) the Tax +Commission published (2025 Form 40 instructions, rev. 2026-03-02, p. 9), the last +located before the 2026-07-03 reference freeze. The retirement-benefit caps take their +statutory 2026 values under Idaho Code 63-3022A, from SSA's 2026 maximum +full-retirement-age benefit ($4,152/month, SSA release 2025-10-24): $49,824 / $74,736. + +Port of r19_id_convention.py (default mode). What 2.15.17 carries: every value r19 set +is still an uprated projection in 2.15.17, identical to 1.755.4's (thresholds +4,920.03 / 9,840.06; caps 49,308.72 / 73,963.07), so all are kept. 2.15.17's 2025 +SEPARATE cap is already 0, so r19's 2025 repair is a no-op there and is kept only for +parity. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_id_hold_2025" +DESCRIPTION = "Idaho 2025 thresholds held for 2026; SSA-derived 2026 retirement caps (2.15.17)." + +THRESHOLDS = { + "single": 4_811, + "separate": 4_811, + "joint": 9_622, + "head_of_household": 9_622, + "surviving_spouse": 9_622, +} +CAPS_2026 = { + "SINGLE": 49_824, + "SEPARATE": 0, + "JOINT": 74_736, + "HEAD_OF_HOUSEHOLD": 49_824, + "SURVIVING_SPOUSE": 49_824, +} +VALUES = { + f"gov.states.id.tax.income.main.{status}[1].threshold": value + for status, value in THRESHOLDS.items() +} +VALUES.update( + { + f"gov.states.id.tax.income.deductions.retirement_benefits.cap.{status}": value + for status, value in CAPS_2026.items() + } +) + + +def _modify(parameters): + found = set() + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in VALUES: + parameter.update(period="2026", value=VALUES[parameter.name]) + found.add(parameter.name) + if parameter.name.endswith("retirement_benefits.cap.SEPARATE"): + parameter.update(period="2025", value=0) + assert found == set(VALUES), f"Missing Idaho parameters: {set(VALUES) - found}" + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py b/reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py new file mode 100644 index 00000000..9107480c --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py @@ -0,0 +1,52 @@ +"""c_irs_sales_tax_2025 on policyengine-us 2.15.17: the IRS 2025 optional sales tax tables held for 2026. + +Rule (reference sidecar, 2026-09-22): the optional state sales tax tables for 2026 are +the 2025 tables the IRS published in the 2025 Instructions for Schedule A (IRS directory +date 2025-12-18), the last published before the 2026-07-03 reference freeze. + +Port of r19_irs_sales_tax_convention.py. What 2.15.17 carries (upstream +PolicyEngine/policyengine-us#9616, commit 3f029abf5b, 2026-09-27, "Correct the IRS +Optional State Sales Tax Table and add 2022-2025 values"): + * 2025: the IRS 2025 table, cell for cell. All 51 x 6 x 19 = 5,814 cells of 2.15.17's + 2025 table equal r19_irs_sales_tax_2025.json (latest/port_state/p21517.json vs the + JSON; asserted again below), so no 2025 update is needed. + * 2026: "later years are uprated from 2025" by gov.irs.uprating (tax.yaml metadata), + a projection, not an IRS publication. This module holds the 2025 cells for 2026. +2.15.17 also corrected the income-bracket rows and zeroes the local estimate in the ten +worksheet jurisdictions; those are formula/metadata fixes of pre-freeze instructions and +are left as 2.15.17 has them. +""" + +import json +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_irs_sales_tax_2025" +DESCRIPTION = "IRS 2025 optional state sales tax tables held for 2026 (2.15.17 carries 2025 already)" +TABLES = json.loads(Path(__file__).with_name("r19_irs_sales_tax_2025.json").read_text()) + + +def _convention(parameters): + table = parameters.gov.irs.deductions.itemized.salt_and_real_estate.state_sales_tax_table.tax + assert len(TABLES) == 51 + set_cells = 0 + for state, sizes in TABLES.items(): + assert state in table.children, state + assert len(sizes) == 6 + for size, brackets in enumerate(sizes, 1): + assert len(brackets) == 19 + for bracket, amount in enumerate(brackets, 1): + parameter = table.children[state].children[str(size)].children[str(bracket)] + # 2.15.17 already carries the published 2025 cell; stop if that drifts. + assert abs(float(parameter("2025-01-01")) - amount) < 1e-9, ( + state, size, bracket, parameter("2025-01-01"), amount) + parameter.update(period="2026", value=amount) + set_cells += 1 + assert set_cells == 5814 + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-28/fixes/latest_c_md_2026.py b/reference_audit/2026-09-28/fixes/latest_c_md_2026.py new file mode 100644 index 00000000..d2bb2406 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_md_2026.py @@ -0,0 +1,59 @@ +"""c_md_2026 on policyengine-us 2.15.17: Maryland's published withholding allowance and 2025 holds. + +Rule (reference sidecar, 2026-09-22): Maryland's 2026 withholding allowance is the +$3,400 the Comptroller published in the 2026 Employer Withholding Guide (revised +December 2025); the 2026 flat standard deduction holds the published 2025 amounts +($3,350; $6,700), because no 2026 return amount was located before the 2026-07-03 +reference freeze (provisional); the child and dependent care credit caps hold their +2025 amounts. Basis: Md. Tax-Gen. 10-217 (Chapter 604 of 2025); Comptroller Tax Alert +rev. 2025-12-22; 2026 Employer Withholding Guide. + +Port of r19_md_convention.py (default mode, provisional holds included). What 2.15.17 +carries: every value r19 set is unchanged from 1.755.4 in 2.15.17 (withholding table +2,800/5,700 for 2026 and 2,750/5,600 for 2025; flat deduction 3,400/6,850; CDCC caps +114,600/178,250 and 62,250/93,450), so all are kept. + +Not a law convention and NOT in this module: 2.15.17 adds Maryland county income tax +to state_income_tax_before_refundable_credits and to the Maryland withholding proxy +(upstream #8888, commit 6b0bca0b9f, 2026-07-05). See latest_md_local_output_scope.py. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_md_2026" +DESCRIPTION = "Maryland published 2026 withholding allowance and provisional 2025 return/CDCC holds (2.15.17)" +STATUSES = ("SINGLE", "SEPARATE", "JOINT", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE") +DOUBLE = {"JOINT", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE"} +ROOT = "gov.states.md.tax.income." + +VALUES = {} +for _status in STATUSES: + # Legacy table; its only live 2025/26 reader in 2.15.17 is md_withheld_income_tax, + # which reads SINGLE (state and, in 2.15.17, county withholding share it). + VALUES[ROOT + "deductions.standard.max." + _status] = {2025: 3350, 2026: 3400} + VALUES[ROOT + "deductions.standard.flat_deduction.amount." + _status] = { + 2026: 6700 if _status in DOUBLE else 3350 + } + VALUES[ROOT + "credits.cdcc.eligibility.agi_cap." + _status] = { + 2026: 174300 if _status == "JOINT" else 112100 + } + VALUES[ROOT + "credits.cdcc.eligibility.refundable_agi_cap." + _status] = { + 2026: 91400 if _status == "JOINT" else 60900 + } + + +def _modify(parameters): + found = set() + for parameter in parameters.get_descendants(): + if isinstance(parameter, Parameter) and parameter.name in VALUES: + for year, value in VALUES[parameter.name].items(): + parameter.update(period=str(year), value=value) + found.add(parameter.name) + assert found == set(VALUES), sorted(set(VALUES) - found) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py b/reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py new file mode 100644 index 00000000..2f1c1bf0 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py @@ -0,0 +1,79 @@ +"""c_mi_published_2026 on policyengine-us 2.15.17: Michigan's published 2026 amounts and 2025 holds. + +Rule (reference sidecar, 2026-09-22): Michigan's 2026 personal exemption ($5,900) and +retirement limits are those Treasury published in the 2026 Form 446 (revised February +2026), before the 2026-07-03 reference freeze; amounts with no located 2026 publication +(disability exemption, senior investment limits, homestead and home heating tables) +hold their published 2025 values, which also replace the engine's projected 2025 +entries. Basis: Michigan Treasury Form 446 (2026, rev. February 2026); 2025 MI-1040 +and MI-1040CR-7 instructions. +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/SUW/TY2026/446_Withholding-Guide_2026.pdf +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040-Book.pdf +https://www.michigan.gov/taxes/-/media/Project/Websites/taxes/Forms/IIT/TY2025/MI-1040CR-7-Book.pdf + +Port of r19_mi_convention.py (default mode). What 2.15.17 carries: + * Retirement tier-one limits: 2.15.17 now has explicit 2026 entries of $67,610 / + $135,220 (upstream #9073, commit b46a672edd 2026-09-04, and caf421bbe9 2026-09-12), + equal to Form 446's. They are checked, not set. + * Personal exemption: 2.15.17 still projects $5,950 for 2026; set to $5,900. + * Every HELD_2025 amount is unchanged from 1.755.4 in 2.15.17; all are kept. + * Home heating credit percentage: 2.15.17 sets 0.60 from 2025 (TY2025 MI-1040CR-7 + book, line 45) and carries it into 2026, which is the published-2025 hold this rule + asks for; 1.755.4 had 0.52. Not set here. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_mi_published_2026" +DESCRIPTION = "Michigan 2026 Form 446 personal exemption and published-2025 holds (2.15.17)" +ROOT = "gov.states.mi.tax.income." +STATUSES = ("SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE") +PUBLISHED_2026 = {ROOT + "exemptions.personal": 5_900} +CARRIED_2026 = {ROOT + "credits.home_heating.credit_percentage": 0.6} +HELD_2025 = { + ROOT + "exemptions.disabled.amount.base": 3_400, + ROOT + "credits.homestead_property_tax.cap": 1_900, + ROOT + "credits.homestead_property_tax.household_resources_limit": 71_500, + ROOT + "credits.homestead_property_tax.property_value_limit": 165_400, + ROOT + "credits.homestead_property_tax.reduction.start": 62_500, + ROOT + "credits.home_heating.additional_exemption.amount": 212, + ROOT + "credits.home_heating.alternate.heating_costs.cap": 3_765, +} +for _status in STATUSES: + CARRIED_2026[ROOT + "deductions.retirement_benefits.tier_one.amount." + _status] = ( + 135_220 if _status == "JOINT" else 67_610 + ) + HELD_2025[ROOT + "deductions.interest_dividends_capital_gains.amount." + _status] = ( + 29_376 if _status == "JOINT" else 14_688 + ) +for _i, _amount in enumerate((604, 815, 1027, 1239, 1451, 1662)): + HELD_2025[ROOT + f"credits.home_heating.standard.base[{_i}].amount"] = _amount +for _i, _amount in enumerate((18_592, 25_018, 31_449, 34_227)): + HELD_2025[ROOT + f"credits.home_heating.alternate.household_resources.cap[{_i}].amount"] = _amount + + +def _convention(parameters): + seen, checked = set(), set() + for parameter in parameters.get_descendants(): + if not isinstance(parameter, Parameter): + continue + if parameter.name in PUBLISHED_2026: + parameter.update(period="2026", value=PUBLISHED_2026[parameter.name]) + seen.add(parameter.name) + if parameter.name in HELD_2025: + for year in (2025, 2026): + parameter.update(period=str(year), value=HELD_2025[parameter.name]) + seen.add(parameter.name) + if parameter.name in CARRIED_2026: + assert abs(float(parameter("2026-01-01")) - CARRIED_2026[parameter.name]) < 1e-9, ( + parameter.name, parameter("2026-01-01")) + checked.add(parameter.name) + missing = (set(PUBLISHED_2026) | set(HELD_2025)) - seen | (set(CARRIED_2026) - checked) + assert not missing, sorted(missing) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py b/reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py new file mode 100644 index 00000000..fcde1222 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py @@ -0,0 +1,151 @@ +"""c_mn_published_2026 on policyengine-us 2.15.17: Minnesota's published 2026 amounts. + +Rule (reference sidecar, 2026-09-22): Minnesota's 2026 inflation-adjusted amounts are +those the Department of Revenue published on 2025-12-01 (announced 2025-12-16), before +the 2026-07-03 reference freeze; the 2025 renter's credit schedule takes its published +values. The marriage credit cap, first located in an August 2026 draft, holds its 2025 +amount ($1,851). + +Sources: +https://www.revenue.state.mn.us/sites/default/files/2025-12/inflation-adjusted-amounts-2026.pdf +https://www.revenue.state.mn.us/sites/default/files/2024-12/inflation-adjusted-amounts-2025.pdf +https://www.revenue.state.mn.us/sites/default/files/2025-10/m1ma-25-grid-0.pdf + +Port of r19_mn_convention.py, values unchanged. What 2.15.17 carries: every value r19 +sets has the same 2026 (and 2025 renter) value in 2.15.17 as in 1.755.4 (still CPI +projections; 90 parameter-years compared in latest/port_state/classify.txt), so all are +kept. Where a projection happens to equal the published amount, setting it is a no-op. +Two published amounts that 1.755.4 lacked are now in 2.15.17 and are only checked here: +the $1,107,750 alternate standard/itemized reduction threshold (2026 publication p. 2) +and the $31,000 / $48,000 marriage credit minimums (2025 Schedule M1MA, carried into +2026, consistent with the cap hold). +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_mn_published_2026" +DESCRIPTION = "Minnesota amounts published before the 2026-07-03 reference freeze (2.15.17)" +PREFIX = "gov.states.mn.tax.income." + + +def _status(single, joint, head=None, separate=None): + return dict( + SINGLE=single, + JOINT=joint, + HEAD_OF_HOUSEHOLD=single if head is None else head, + SEPARATE=single if separate is None else separate, + SURVIVING_SPOUSE=joint, + ) + + +VALUES_2026 = {} + + +def _statuses(path, values): + VALUES_2026.update({PREFIX + path + "." + key: value for key, value in values.items()}) + + +_statuses("amt.fractional_income_threshold", _status(73100, 97470, separate=48740)) +_statuses("deductions.standard.base", _status(15300, 30600, 23000)) +_statuses("deductions.standard.extra", _status(2000, 1600, separate=1600)) +for deduction in ("standard", "itemized"): + _statuses(f"deductions.{deduction}.reduction.agi_threshold.low", _status(244400, 244400, separate=122200)) + _statuses(f"deductions.{deduction}.reduction.agi_threshold.high", _status(337800, 337800, separate=168900)) +_statuses("exemptions.agi_threshold", _status(244500, 366700, 305600, 183350)) +_statuses("subtractions.pension_income.cap", _status(13850, 27690)) +for subtraction in ("pension_income", "social_security"): + _statuses(f"subtractions.{subtraction}.reduction.start", _status(86410, 110780, separate=55390)) +# The source explicitly labels both alternate-subtraction tables Not Indexed. +_statuses("subtractions.social_security.alternative_amount", _status(4560, 5840, separate=2920)) +_statuses("subtractions.social_security.income_amount", _status(69250, 88630, separate=44315)) +for name, value in { + "exemptions.amount": 5300, + "credits.cdcc.phaseout_threshold": 65610, + "credits.cwfc.ctc.amount": 1800, + "credits.cwfc.phase_out.threshold.joint": 38770, + "credits.cwfc.phase_out.threshold.other": 32680, + "credits.cwfc.wfc.phase_in[1].threshold": 9690, + "credits.cwfc.wfc.additional.amount[1].amount": 1020, + "credits.cwfc.wfc.additional.amount[2].amount": 2330, + "credits.cwfc.wfc.additional.amount[3].amount": 2770, +}.items(): + VALUES_2026[PREFIX + name] = value + +# Earliest located 2026 agency form: near-final draft 2026-08-03, cap $1,894. +# Hold the $1,851 cap printed in the 2025 final draft dated 2025-10-15. +# An earlier 2026 publication was not located; see report's explicit limitation. +# https://www.revenue.state.mn.us/sites/default/files/2026-08/m1ma-26-grid.pdf +# https://www.revenue.state.mn.us/sites/default/files/2025-10/m1ma-25-grid-0.pdf +VALUES_2026[PREFIX + "credits.marriage.maximum_amount"] = 1851 + +RATE_THRESHOLDS_2026 = { + "single": (33310, 109430, 203150), + "joint": (48700, 193480, 337930), + "separate": (24350, 96740, 168965), + "head_of_household": (41010, 164800, 270060), + "surviving_spouse": (48700, 193480, 337930), +} +for status, thresholds in RATE_THRESHOLDS_2026.items(): + for index, value in enumerate(thresholds, 1): + VALUES_2026[f"{PREFIX}rates.{status}[{index}].threshold"] = value + +# Full renter schedule, compressed only where adjacent values are equal in the +# agency table; indices match the existing engine's three single-amount scales. +RENTER_VALUES = { + 2025: { + "claimant_share": (6670, 15530, 22160, 31030, 37690, 46540, 53180, 62060, 68720, 77570), + "percent_of_income": (8860, 11070, 15530, 19960, 24360, 28820, 31030, 33240, 37690, 39890, 77570), + "max_thresholds": (8860, 11070, 15530, 19960, 22160, 24360, 28820, 62060, 64260, 66480, 68720, 70920, 73140, 75350, 77570), + "max_amounts": (2720, 2640, 2580, 2500, 2440, 2380, 2300, 2240, 2040, 1830, 1550, 1360, 1220, 680, 270), + }, + 2026: { + "claimant_share": (6820, 15880, 22670, 31740, 38540, 47590, 54390, 63470, 70280, 79330), + "percent_of_income": (9060, 11320, 15880, 20410, 24920, 29470, 31740, 34000, 38540, 40800, 79330), + "max_thresholds": (9060, 11320, 15880, 20410, 22670, 24920, 29470, 63470, 65720, 68000, 70280, 72530, 74810, 77070, 79330), + "max_amounts": (2780, 2700, 2640, 2560, 2490, 2430, 2360, 2290, 2080, 1870, 1590, 1390, 1250, 690, 270), + }, +} + + +def _renters(year): + result = {} + values = RENTER_VALUES[year] + for scale in ("claimant_share", "percent_of_income"): + for index, value in enumerate(values[scale], 1): + result[f"{PREFIX}credits.renters.{scale}[{index}].threshold"] = value + for index, value in enumerate(values["max_thresholds"], 1): + result[f"{PREFIX}credits.renters.max_credit[{index}].threshold"] = value + for index, value in enumerate(values["max_amounts"]): + result[f"{PREFIX}credits.renters.max_credit[{index}].amount"] = value + return result + + +VALUES_2025 = _renters(2025) +VALUES_2026.update(_renters(2026)) + + +# Published amounts 2.15.17 already carries for 2026, consistent with the rule (checked). +CARRIED_2026 = { + # 2026 inflation-adjusted amounts (2025-12-01), p. 2; explicit 2026 entry in 2.15.17. + PREFIX + "deductions.standard.reduction.alternate.income_threshold": 1_107_750, + PREFIX + "deductions.itemized.reduction.alternate.income_threshold": 1_107_750, + # 2025 Schedule M1MA line 6 amounts (2025-12), carried into 2026 by 2.15.17. + PREFIX + "credits.marriage.minimum_individual_income": 31_000, + PREFIX + "credits.marriage.minimum_taxable_income": 48_000, +} + + +def _convention(parameters): + indexed = {p.name: p for p in parameters.get_descendants() if isinstance(p, Parameter)} + for year, values in ((2025, VALUES_2025), (2026, VALUES_2026)): + for name, value in values.items(): + indexed[name].update(period=str(year), value=value) + for name, value in CARRIED_2026.items(): + assert abs(float(indexed[name]("2026-01-01")) - value) < 1e-9, (name, indexed[name]("2026-01-01")) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py b/reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py new file mode 100644 index 00000000..105beeb9 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py @@ -0,0 +1,44 @@ +"""c_mo_published_2026 on policyengine-us 2.15.17: Missouri's published 2026 brackets. + +Rule (reference sidecar, 2026-09-22): Missouri's 2026 income tax brackets are those +the Department of Revenue published in its 2026 withholding formula (dated 2025-11-21), +before the 2026-07-03 reference freeze, not the engine's CPI projections. +https://dor.mo.gov/forms/Withholding%20Formula_2026.pdf (annual table, p. 2) +2025: https://dor.mo.gov/forms/2025%20Tax%20Chart_2025.pdf + +Port of r19_mo_convention.py. What 2.15.17 carries: 2026 thresholds are still uprated +projections (1,342.76 ... 9,399.29), identical to 1.755.4's; 2025 thresholds equal the +published 2025 chart already. All thresholds are set as in r19. + +Public-pension cap: r19 repaired the 2025 cap to $47,633 and re-pinned the 2026 cap to +the engine's own 2026 value, because 1.755.4 applied reforms before uprating and a 2025 +edit would have moved the 2026 projection. 2.15.17 applies a reform after its uprating +pass (policyengine_us/system.py), so the 2025 edit cannot move 2026; the re-pin (which +imported the baseline system) is dropped. The 2026 cap stays 2.15.17's value +($49,308.72), as in r19, which claims no 2026 convention value for it. No MO benchmark +household has public pensions. +""" + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_mo_published_2026" +DESCRIPTION = "Missouri income-tax thresholds published before 2026-07-03 (2.15.17)." + +THRESHOLDS_2025 = (0, 1313, 2626, 3939, 5252, 6565, 7878, 9191, float("inf")) +THRESHOLDS_2026 = (0, 1348, 2696, 4044, 5392, 6740, 8088, 9436, float("inf")) + + +def _convention(parameters): + scale = parameters.gov.states.mo.tax.income.rates + assert len(scale.brackets) == len(THRESHOLDS_2026) + for year, thresholds in ((2025, THRESHOLDS_2025), (2026, THRESHOLDS_2026)): + for bracket, value in zip(scale.brackets, thresholds): + bracket.threshold.update(period=str(year), value=value) + pension = parameters.gov.states.mo.tax.income.deductions.social_security_and_public_pension + pension.mo_max_social_security_benefit.update(period="2025", value=47633) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py b/reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py new file mode 100644 index 00000000..e7eb44dc --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py @@ -0,0 +1,170 @@ +"""Hold the FY2026 SNAP schedule for October-December 2026 (convention c_snap_hold_fy2026) +on policyengine-us 2.15.17. + +This is a publication convention, not a claim that FY2026 law stays in force after +September 2026. The sidecar rule: "A scored reference follows from the stated facts +and from law published before the 2026-07-03 reference freeze. SNAP October-December +2026 hold the FY2026 schedule ..., the last USDA published before the 2026-07-03 +reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the +2026 HHS guideline, published in January 2026." + +Sources, with publication dates: +- USDA FNS, "SNAP Fiscal Year 2026 Cost-of-Living Adjustments", fns.usda.gov/snap/ + allotment/cola/fy26 (memo DATE August 13, 2025; page updated August 19, 2025; + Wayback capture 20250902215244; the sidecar gives the signed memo as 2025-08-14): + 4-person maximum $994, minimum benefit $24, shelter cap $744, homeless shelter + deduction $198.99, standard deduction $209 for sizes 1-3, asset limits $3,000 and + $4,500 (at least one member 60+ or disabled). +- USDA FNA, "SNAP FY 2027 Cost-of-Living Adjustments", fna.usda.gov/snap/allotment/ + cola/fy27 (page updated August 28, 2026; memo dated 2026-08-21 per the sidecar and + policyengine-us#9623): after the freeze. + +Why the 1.755.4 module (r13_hold_fy2026_v3) cannot be reused. It held one leaf, +gov.usda.snap.uprating. policyengine-us 1.755.4 applied reforms before +uprate_parameters (system.py), so that hold also set the 339 leaves whose October- +December 2026 values the index produced (maximum allotments, deductions, shelter caps, +utility allowances). +policyengine-us 2.15.17 applies reforms after uprate_parameters (system.py, the +"issue #9075" comment), and encodes FY2027 as published values +(policyengine-us#9623, d27d6c3cc3), so r13_hold_fy2026_v3 on 2.15.17 changes only the +index leaf, which no 2.15.17 formula reads. + +What this module does. Every leaf in HOLD_NODES takes, for 2026-10-01 to 2026-12-31 +only, its own 2.15.17 value on 2026-09-30 (the FY2026 value). In 2.15.17 those are the +leaves whose value changes at 2026-10-01 as part of the FY2027 schedule: +- max_allotment.main and .additional (FY2027 memo values) +- min_allotment.published_adjustment (FY2027: +$1 in the 48 states and D.C., so that + round(8% x $298) + 1 = $25 on the held maximum; FY2026 value 0 gives the $24 USDA + published for FY2026) +- income.deductions.standard, excess_shelter_expense.cap, excess_shelter_expense. + homeless.deduction (FY2027 memo values) +- income.deductions.utility.single.* and utility.limited.main (uprated by the SNAP + index, whose 2026-10-01 entry is the June 2026 CPI-U actual, policyengine-us#9090, + f28aa73eff, 2026-07-20) +- income.deductions.utility.limited.by_household_size.amount (Arizona's FY2027 LUA, + $154 / $208 from 2026-10-01, added in 9164f90305 on 2026-08-31 from the DES manual; + a FY2027 utility allowance, which the convention holds; publication date not + verifiable here, and holding it moves no benchmark output) +- asset_test.limit (FY2027 elderly/disabled limit $4,750, FY2027 memo; FY2026 $4,500) +- gov.usda.snap.uprating (the leaf the 1.755.4 convention held; no 2.15.17 formula + reads it directly, so holding it changes nothing else) +It also lifts gov.usda.snap.max_allotment.cap for October-December 2026. 2.15.17 first +applies that cap from 2026-10-01 (snap_max_allotment.formula_2026_10_01) at amounts +first published in the FY2027 memo; the FY2026 table has none, and 2.15.17 applies +none in FY2026. The cap binds only at 18+ members, so lifting it moves no benchmark +output. + +Not held: the poverty guideline (2.15.17 already uses the 2026 HHS guideline, $15,960 ++ $5,680, from 2026-10-01, as the convention specifies); ABAWD waiver timelines +(Alaska's good-faith window ending 2026-11-01, county waiver lists), which are work +rules, not the FY2026 schedule; every other parameter and every other date. + +Each leaf's pre-October and post-December values are left exactly as 2.15.17 has them; +the module is idempotent (2.15.17 applies a reform twice while building the system). + +The SNAP engine fixes the board regenerated on 1.755.4 are already in 2.15.17 and are +not re-applied: r26 and r27 (policyengine-us#9318, merge 5d88007d90, 2026-08-25; net +income further rounded to cents before half-up by #9587, fa27adbffc), r28 and r31 +(policyengine-us#9162, merge 3c41c31457, 2026-07-28; income standards for households +over eight refined by #9587), r33 (policyengine-us#9586, d9e801df41, 2026-09-24). +Re-applying the five 1.755.4 modules on top of this one moves none of the 1,984 outputs. + +Validation (2026-09-28, sweep_latest.py -> out/latest_c_snap_hold_fy2026.csv): across the +whole 102,807-leaf parameter tree the module changes 359 leaves, only on dates from +2026-10-01 to 2026-12-31. It changes 17 outputs against raw 2.15.17, all SNAP, and +reproduces 12 of the 13 SNAP outputs the convention set on the board; scenario_066 stays +at 0 because 2.15.17's SNAP work rules read weekly_hours_worked_before_lsr (default 0 +since policyengine-us#9261), not the prompt's stated hours, which is independent of this +convention. +""" + +from policyengine_core.periods import instant +from policyengine_core.reforms import Reform + +try: # 2.15.17: policyengine_us/tools/parameters.py, used by backdate_parameters + from policyengine_us.tools.parameters import FIRST_MODELED_YEAR +except ImportError: # pragma: no cover + FIRST_MODELED_YEAR = 2015 + +FIX_ID = "latest_c_snap_hold_fy2026" +DESCRIPTION = ( + "Hold the FY2026 SNAP schedule (maximum and minimum allotments, deductions, shelter " + "caps, utility allowances, asset limits; no 18+ cap) for October-December 2026 on " + "policyengine-us 2.15.17." +) + +START = instant("2026-10-01") +STOP = instant("2026-12-31") +FY2026_LAST_DAY = "2026-09-30" + +HOLD_NODES = ( + "gov.usda.snap.uprating", + "gov.usda.snap.max_allotment.main", + "gov.usda.snap.max_allotment.additional", + "gov.usda.snap.min_allotment.published_adjustment", + "gov.usda.snap.income.deductions.standard", + "gov.usda.snap.income.deductions.excess_shelter_expense.cap", + "gov.usda.snap.income.deductions.excess_shelter_expense.homeless.deduction", + "gov.usda.snap.income.deductions.utility.single", + "gov.usda.snap.income.deductions.utility.limited.main", + "gov.usda.snap.income.deductions.utility.limited.by_household_size.amount", + "gov.usda.snap.asset_test.limit", +) +LIFT_NODES = ("gov.usda.snap.max_allotment.cap",) + + +def _node(parameters, path): + node = parameters + for part in path.split("."): + node = node.children[part] + return node + + +def _leaves(node): + if hasattr(node, "values_list"): + yield node + return + for child in node.get_descendants(): + if hasattr(child, "values_list"): + yield child + + +def _hold_leaf(leaf): + value = leaf(FY2026_LAST_DAY) + if value is None: + raise ValueError(f"{leaf.name} has no FY2026 value to hold") + leaf.update(start=START, stop=STOP, value=value) + + +def _lift_leaf(leaf): + earliest = leaf.values_list[-1] + if instant(earliest.instant_str) >= START: + # First application, before backdate_parameters: give the dates before + # October 2026 the value backdating would give them (the leaf's first + # value), so that only October-December 2026 change. + leaf.update( + start=instant(f"{FIRST_MODELED_YEAR}-01-01"), + stop=instant(FY2026_LAST_DAY), + value=earliest.value, + ) + leaf.update(start=START, stop=STOP, value=float("inf")) + + +def make_reform(hold_nodes=HOLD_NODES, lift_nodes=LIFT_NODES): + def _modify(parameters): + for path in hold_nodes: + for leaf in _leaves(_node(parameters, path)): + _hold_leaf(leaf) + for path in lift_nodes: + for leaf in _leaves(_node(parameters, path)): + _lift_leaf(leaf) + return parameters + + class _reform(Reform): + def apply(self): + self.modify_parameters(_modify) + + return _reform + + +reform = make_reform() diff --git a/reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py b/reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py new file mode 100644 index 00000000..865502d5 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py @@ -0,0 +1,72 @@ +"""c_wi_published_2026 on policyengine-us 2.15.17: Wisconsin's published 2026 standard deduction. + +Rule (reference sidecar, 2026-09-22): Wisconsin's 2026 standard deduction and tax +brackets are the amounts DOR published before the 2026-07-03 reference freeze (2026 +Form 1-ES instructions, rev. 1-26, pp. 2-3; 2026 WT-4A worksheet, rev. 11-25, p. 2). + +Port of r19_wi_convention.py. What 2.15.17 carries: + * Brackets: the 12 bracket thresholds r19 set are now explicit 2026 entries in + 2.15.17 (parameters/gov/states/wi/tax/income/rates/*.yaml cite the 2026 Form 1-ES + instructions) and equal r19's values exactly. They are not re-set; the module + asserts they still match. + * Standard deduction: still projected. 2.15.17 uprates the 2025 amounts by + gov.irs.uprating rounded to $10 (upstream commit 94e40f1471, 2026-09-01), giving + 13,870 / 25,680 / 12,200 / 17,920 maxima and 19,990 / 28,850 / 13,690 / 19,990 + phase-out starts, not DOR's published 13,960 / 25,840 / 12,280 / 18,030 and + 20,120 / 29,040 / 13,780 / 20,120 (and 58,827 for the head-of-household second + phase-out threshold). These 10 values are set here. +SURVIVING_SPOUSE keeps the engine's JOINT alias, as in r19. +""" + +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +FIX_ID = "latest_c_wi_published_2026" +DESCRIPTION = "Wisconsin 2026 standard deduction from the published 1-ES (brackets already in 2.15.17)" +ROOT = "gov.states.wi.tax.income." +VALUES = { + ROOT + "deductions.standard.max.SINGLE": 13_960, + ROOT + "deductions.standard.max.JOINT": 25_840, + ROOT + "deductions.standard.max.SEPARATE": 12_280, + ROOT + "deductions.standard.max.HEAD_OF_HOUSEHOLD": 18_030, + ROOT + "deductions.standard.max.SURVIVING_SPOUSE": 25_840, + ROOT + "deductions.standard.phase_out.single[1].threshold": 20_120, + ROOT + "deductions.standard.phase_out.joint[1].threshold": 29_040, + ROOT + "deductions.standard.phase_out.separate[1].threshold": 13_780, + ROOT + "deductions.standard.phase_out.head_of_household[1].threshold": 20_120, + ROOT + "deductions.standard.phase_out.head_of_household[2].threshold": 58_827, +} +# Published 2026 brackets that 2.15.17 already carries (checked, not set). +CARRIED = {} +for _status, _thresholds in { + "single": (15_110, 51_950, 332_720), + "head_of_household": (15_110, 51_950, 332_720), + "joint": (20_150, 69_260, 443_630), + "separate": (10_080, 34_630, 221_820), +}.items(): + for _index, _amount in enumerate(_thresholds, start=1): + CARRIED[ROOT + f"rates.{_status}[{_index}].threshold"] = _amount + + +def _convention(parameters): + seen, checked = set(), set() + for parameter in parameters.get_descendants(): + if not isinstance(parameter, Parameter): + continue + if parameter.name in VALUES: + parameter.update(period="2026", value=VALUES[parameter.name]) + seen.add(parameter.name) + elif parameter.name in CARRIED: + assert abs(float(parameter("2026-01-01")) - CARRIED[parameter.name]) < 1e-9, ( + parameter.name, parameter("2026-01-01")) + checked.add(parameter.name) + if seen != set(VALUES) or checked != set(CARRIED): + raise ValueError( + f"Missing Wisconsin parameters: {sorted((set(VALUES) - seen) | (set(CARRIED) - checked))}" + ) + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_convention) diff --git a/reference_audit/2026-09-28/fixes/latest_conventions.py b/reference_audit/2026-09-28/fixes/latest_conventions.py new file mode 100644 index 00000000..26e11325 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_conventions.py @@ -0,0 +1,73 @@ +"""Every ported pre-freeze-law convention on policyengine-us 2.15.17, applied together. + +The reference rule (reference sidecar; Max, 2026-09-28 "we should be using the latest pe +for this always!"): a scored reference follows from the stated facts and from law +published before the 2026-07-03 reference freeze. New references are therefore +policyengine-us 2.15.17 plus the conventions that hold pre-freeze law, and nothing else. + +Composes, in this order, the reforms of the nine ported convention modules: + latest_c_ca_hold_2025 (c_ca_hold_2025) + latest_c_irs_sales_tax_2025 (c_irs_sales_tax_2025) + latest_c_wi_published_2026 (c_wi_published_2026) + latest_c_id_hold_2025 (c_id_hold_2025) + latest_c_mn_published_2026 (c_mn_published_2026) + latest_c_md_2026 (c_md_2026) + latest_c_mi_published_2026 (c_mi_published_2026) + latest_c_mo_published_2026 (c_mo_published_2026) + latest_c_snap_hold_fy2026 (c_snap_hold_fy2026) +The same way the 1.755.4 combined modules (c13v3_plus_r33.py and relatives) and +latest_state_conventions.py compose: each part's Reform.apply runs against this reform. +The parts touch disjoint parameter leaves (checked on the full parameter tree by +triage/latest/compose/check_composition.py), so order does not matter. + +Not included: +- latest_md_local_output_scope.py: an output-definition adapter (keeps Maryland county + income tax out of state_income_tax_before_refundable_credits), not a pre-freeze-law + convention; the port report leaves adopting it to the lead. +- A situation-builder mapping of stated weekly hours to weekly_hours_worked_before_lsr + (scenario_066): an input mapping, not a convention. +- The 1.755.4 upstream-fix modules (r04, r09, r17, r26, r27, r28, r31, r33): the port + reports found each already in 2.15.17. +""" + +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_conventions" +DESCRIPTION = ( + "All nine pre-freeze-law conventions on 2.15.17: CA, IRS sales tax, WI, ID, MN, MD, " + "MI, MO published-law holds plus the FY2026 SNAP schedule hold" +) +PARTS = ( + "latest_c_ca_hold_2025", + "latest_c_irs_sales_tax_2025", + "latest_c_wi_published_2026", + "latest_c_id_hold_2025", + "latest_c_mn_published_2026", + "latest_c_md_2026", + "latest_c_mi_published_2026", + "latest_c_mo_published_2026", + "latest_c_snap_hold_fy2026", +) + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"latest_conventions_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULES = tuple(_load(name) for name in PARTS) +REFORMS = tuple(module.reform for module in MODULES) + + +class reform(Reform): + def apply(self): + for part in REFORMS: + part.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_final.py b/reference_audit/2026-09-28/fixes/latest_final.py new file mode 100644 index 00000000..608376bb --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_final.py @@ -0,0 +1,33 @@ +"""The reference system for the 2026-09-28 wave: policyengine-us 2.15.17 plus +latest_conventions (the nine pre-freeze-law conventions) plus +latest_md_local_output_scope (the output-definition adapter that keeps Maryland county +income tax out of the state income tax output). The stated-hours mapping lives in +policybench.scenarios (PE_INPUT_ALIASES), not here.""" + +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_final" +DESCRIPTION = "latest_conventions + latest_md_local_output_scope on policyengine-us 2.15.17" +PARTS = ("latest_conventions", "latest_md_local_output_scope") + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"latest_final_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +REFORMS = tuple(_load(name).reform for name in PARTS) + + +class reform(Reform): + def apply(self): + for part in REFORMS: + part.apply(self) diff --git a/reference_audit/2026-09-28/fixes/latest_map_stated_hours.py b/reference_audit/2026-09-28/fixes/latest_map_stated_hours.py new file mode 100644 index 00000000..de910171 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_map_stated_hours.py @@ -0,0 +1,66 @@ +"""Input mapping on policyengine-us 2.15.17: stated usual weekly hours reach the SNAP work tests. + +NOT a hold convention and NOT an engine fix. It is a situation-builder mapping, composed +with every pre-freeze-law convention (latest_conventions.py) so its sweep compares +one-for-one with out/latest_conventions.csv. + +Why. The prompt shows the input hours_worked_last_week with the label "usual weekly +hours worked" (policybench/prompts.py, US_VARIABLE_DESCRIPTIONS). The sweep builder +(Scenario.to_pe_household) passes it to the engine under that name only. In +policyengine-us the SNAP work rules never read hours_worked_last_week; they read +weekly_hours_worked_before_lsr (2.15.17: meets_snap_abawd_work_requirements, +meets_snap_general_work_requirements, is_snap_work_registration_exempt_non_age, +meets_snap_work_exception). policybench/scenarios.py lists weekly_hours_worked_before_lsr +in EXCLUDED_INPUT_VARIABLES, so no scenario sets it. In 1.755.4 its default was 40 +(variables/household/income/person/weekly_hours_worked.py), which happened to match a +stated 40; upstream 82745ca239 (PR #9261, merged a112cc5a0a on 2026-08-12, closes +#9254) changed the default to 0, so on 2.15.17 a person who states 40 usual weekly +hours fails the 20-hour ABAWD test (7 U.S.C. 2015(o)(2); 7 CFR 273.24(a)(1)). + +What it does. For each person whose prompt shows usual weekly hours +(hours_worked_last_week, a prompt-visible input), copy that value into +weekly_hours_worked_before_lsr, unless the situation already sets it. Persons with no +stated hours are untouched and keep the engine default (0 in 2.15.17, which is also the +prompt's "Treat any unlisted numeric input as 0" rule). +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +from policybench.scenarios import is_excluded_prompt_input_name + +FIX_ID = "latest_map_stated_hours" +DESCRIPTION = ( + "latest_conventions plus a situation-builder mapping: each person's stated usual weekly " + "hours (hours_worked_last_week, shown in the prompt) also set weekly_hours_worked_before_lsr, " + "the input the SNAP work tests read (its default fell from 40 to 0 in upstream #9261)" +) + +SOURCE = "hours_worked_last_week" +TARGET = "weekly_hours_worked_before_lsr" + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"map_stated_hours_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +reform = _load("latest_conventions").reform + +# The mapping is only sound while the source is a prompt-visible input and the target is not. +assert not is_excluded_prompt_input_name(SOURCE) +assert is_excluded_prompt_input_name(TARGET) + + +def patch(situation: dict, scenario) -> dict: + for person in situation["people"].values(): + if SOURCE in person and TARGET not in person: + person[TARGET] = dict(person[SOURCE]) + return situation diff --git a/reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py b/reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py new file mode 100644 index 00000000..8d1bddc2 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py @@ -0,0 +1,44 @@ +"""Candidate (not a law convention): keep Maryland county income tax out of the state output. + +PolicyBench defines state_income_tax_before_refundable_credits as "state individual +income tax after nonrefundable credits and before refundable credits, excluding local +income and payroll taxes" (policybench/benchmark_specs.json, output prompt). Its +separate local_income_tax output names only NYC, Philadelphia, Kansas City and +St. Louis. + +policyengine-us 2.15.17 adds md_local_income_tax_before_refundable_credits (Maryland +county income tax net of the local EITC and local poverty credit) to the list behind +state_income_tax_before_refundable_credits (parameters/gov/states/household/ +state_income_tax_before_refundable_credits.yaml; upstream PolicyEngine/policyengine-us +#8888, commit 6b0bca0b9f, 2026-07-05). The county is not a benchmark input; with the +default UNKNOWN county 2.15.17 falls back to Allegany County's 3.03% rate +(variables/gov/states/md/tax/income/local/md_applicable_local_tax_rate.py). On 1.755.4 +the list had no local entry, so the board's Maryland state amounts are state-only. + +This module removes that one entry from the list, so the scored output again matches +its definition. It leaves the county tax in md_withheld_income_tax, i.e. in the federal +SALT deduction (local income taxes are deductible, IRC 164(a)(3)); see the port report +for scenario_078's federal output. Only the list changes; no formula or amount changes. +""" + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_md_local_output_scope" +DESCRIPTION = "Exclude Maryland county income tax from the state income tax output (output definition)" +LOCAL = "md_local_income_tax_before_refundable_credits" + + +def _modify(parameters): + param = parameters.gov.states.household.state_income_tax_before_refundable_credits + current = list(param("2026-01-01")) + if LOCAL in current: + current.remove(LOCAL) + param.update(period="year:2015-01-01:20", value=current) + assert LOCAL not in param("2026-01-01") + assert "md_income_tax_before_refundable_credits" in param("2026-01-01") + return parameters + + +class reform(Reform): + def apply(self): + self.modify_parameters(_modify) diff --git a/reference_audit/2026-09-28/fixes/latest_state_conventions.py b/reference_audit/2026-09-28/fixes/latest_state_conventions.py new file mode 100644 index 00000000..9c93f327 --- /dev/null +++ b/reference_audit/2026-09-28/fixes/latest_state_conventions.py @@ -0,0 +1,49 @@ +"""The eight state and IRS published-law conventions on policyengine-us 2.15.17, applied together. + +Composes, in this order, the reforms of: + latest_c_ca_hold_2025, latest_c_irs_sales_tax_2025, latest_c_wi_published_2026, + latest_c_id_hold_2025, latest_c_mn_published_2026, latest_c_md_2026, + latest_c_mi_published_2026, latest_c_mo_published_2026. +Their parameter sets are disjoint (each touches only its own state's tree, or the IRS +sales tax table), so order does not matter. The SNAP convention (c_snap_hold_fy2026) +and the Maryland output-scope candidate (latest_md_local_output_scope.py) are not +included. +""" + +import importlib.util +import sys +from pathlib import Path + +from policyengine_core.reforms import Reform + +FIX_ID = "latest_state_conventions" +DESCRIPTION = "CA, IRS sales tax, WI, ID, MN, MD, MI, MO published-law conventions on 2.15.17" +PARTS = ( + "latest_c_ca_hold_2025", + "latest_c_irs_sales_tax_2025", + "latest_c_wi_published_2026", + "latest_c_id_hold_2025", + "latest_c_mn_published_2026", + "latest_c_md_2026", + "latest_c_mi_published_2026", + "latest_c_mo_published_2026", +) + + +def _load(name): + path = Path(__file__).with_name(name + ".py") + spec = importlib.util.spec_from_file_location(f"latest_state_part_{name}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULES = tuple(_load(name) for name in PARTS) +REFORMS = tuple(module.reform for module in MODULES) + + +class reform(Reform): + def apply(self): + for part in REFORMS: + part.apply(self) diff --git a/reference_audit/2026-09-28/scripts/build_references_latest.py b/reference_audit/2026-09-28/scripts/build_references_latest.py new file mode 100644 index 00000000..003e3805 --- /dev/null +++ b/reference_audit/2026-09-28/scripts/build_references_latest.py @@ -0,0 +1,419 @@ +"""Build PolicyBench's US references on the newest policyengine-us (the 2026-09-29 upgrade). + +Rulings: +- Max's ruling, 2026-09-28: "we should be using the latest pe for this always!" + References come from the newest policyengine-us release when PolicyBench begins the + reference sweep; at publication PolicyBench checks that the newest release gives the + same values. The sweep began at 01:42 UTC on 2026-09-29, when the newest release was + 2.15.17 (uploaded to PyPI at 00:23 UTC; verification/sweep_timing.json). + policyengine.py 6.1.2 is recorded for provenance only: its certified bundle is + policyengine-us 2.2.1, and `import policyengine` refuses to load next to 2.15.17. +- Max, 2026-09-22 (reference sidecar rule): a scored reference follows from the stated + facts and from law published before the 2026-07-03 reference freeze. The nine + conventions that hold pre-freeze law are re-expressed for 2.15.17 + (sweep/fixes/latest_c_*.py, composed in latest_conventions.py). + +Method: +- Every output is computed on 2.15.17 with latest_conventions plus + latest_md_local_output_scope (the output-definition adapter: the benchmark's state + income tax output excludes local tax; 2.15.17 folded Maryland county tax into it). + Households are built by policybench's own Scenario.to_pe_household, which since this + wave also passes stated usual weekly hours to weekly_hours_worked_before_lsr (upstream + #9261 changed that input's default from 40 to 0). +- A scored output takes the computed value. It may move against the board (release + 20260922c) only if it is listed in --actions as a reviewed change, and it must then + equal the listed value. A move not listed, a listed move that does not happen, or a + listed value the engine does not produce stops the build. Moves of $1 or less are + applied and recorded. +- An excluded output keeps the value its exclusion record names, as on 2026-09-22. It is + not scored. Its 2.15.17 value and the reviewed reason it stays excluded are recorded: + the reason is the full text of its cluster review's corrected_per_output entry in + clusters.json (beside --actions), and final_actions.json must carry the same text. +- An output the audit excluded on review, apart from any engine change, is listed in + --actions' audit_exclusions with its exclusion record. Its engine value must equal + both the board's and the record's frozen_value, so the revision's changed list does + not name it; the record is appended to the exclusions after the upgrade's own. + +Dates are UTC days. The wave began on the evening of 2026-09-28, US Eastern time, which +is this directory's name; the sweep began at 01:42 UTC on 2026-09-29, so the records +date the upgrade 2026-09-29: the revision's date, the derivation, and each new +exclusion's basis, which takes the exclusion's decided_on. The first build ran at 11:57 +UTC. The rebuild later that day changed only record text (the upgrade's date and +wording); its reference CSV is byte-identical, and the sidecar's regenerated_at_utc is +the rebuild's. + +The first build ran in the triage directory, whose layout this script assumes: + + cd triage && PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 \\ + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2 \\ + .venv-pepy612-us21517/bin/python build_references_latest.py \\ + --actions latest/final_actions.json --out-dir ../reference_v13 + +Later on 2026-09-29, rewrite_reference_records.py applied this script's record functions +(reviewed_reason, audit_exclusions, exclusion_derivation, dump_record) to the installed +records without recomputing any output: the rechecked outputs' full reviewed reasons, +and the one audit exclusion (scenario_023 head_medicaid_eligible). A rebuild writes the +same records. + +The rebuild ran from this directory's committed files, laid out the same way (B is a +scratch directory), and scripts/install_adds0929_references.py installed its output: + + mkdir -p $B/sweep/fixes && cp scripts/build_references_latest.py $B/ \\ + && cp scripts/sweep.py $B/sweep/ && cp fixes/*.py $B/sweep/fixes/ \\ + && cp ../2026-09-22/fixes/r19_irs_sales_tax_2025.json $B/sweep/fixes/ + (cd $B && PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ + /.venv-pepy612-us21517/bin/python build_references_latest.py \\ + --actions /final_actions.json --out-dir $B/out) +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import importlib.util +import json +import sys +from importlib.metadata import version +from pathlib import Path + +import pandas as pd + +HERE = Path(__file__).resolve().parent +FIXES = HERE / "sweep" / "fixes" +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) # the frozen v1.1 bundle (policyengine-us 1.755.4), read-only +CHECKOUT = Path("/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2") +RUN = "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +# Release dashboard-data-20260922c: main at this commit. Read from git, not from the +# checkout, whose snapshot holds whatever this script last wrote. +BASE_COMMIT = "3220a7a62b6be83032e9313c9df539c619ad8932" + + +def _board() -> Path: + import subprocess + import tempfile + + out = Path(tempfile.mkdtemp(prefix="board-20260922c-")) + for name in ("reference_outputs.csv", "reference_outputs.csv.meta.json", "reference_exclusions.json"): + raw = subprocess.run( + ["git", "-C", str(CHECKOUT), "show", f"{BASE_COMMIT}:{RUN}/{name}"], + check=True, capture_output=True, + ).stdout + (out / name).write_bytes(raw) + return out +YEAR = 2026 +DATE = "2026-09-29" +ENGINE = "2.15.17" +PARTS = ("latest_conventions", "latest_md_local_output_scope") +CONVENTION_MODULES = ( + "latest_c_ca_hold_2025", + "latest_c_irs_sales_tax_2025", + "latest_c_wi_published_2026", + "latest_c_id_hold_2025", + "latest_c_mn_published_2026", + "latest_c_md_2026", + "latest_c_mi_published_2026", + "latest_c_mo_published_2026", + "latest_c_snap_hold_fy2026", +) +RULE = ( + "A scored reference follows from the stated facts and from law published before " + "the 2026-07-03 reference freeze." +) +# Lead's ruling of 2026-09-29 on the wording: "newest" holds at a stated time. +ENGINE_RULE = ( + "References come from the newest policyengine-us release when PolicyBench begins " + "the reference sweep; at publication PolicyBench checks that the newest release " + "gives the same values." +) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +NUMBER_WORDS = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six"} + + +def dump_record(value: dict) -> str: + """The serialization of the sidecar and the exclusion record.""" + return json.dumps(value, indent=2) + "\n" + + +def reviewed_reason(clusters: dict, entry: dict) -> str: + """The full reason a rechecked excluded output stays excluded: its cluster + review's corrected_per_output entry in clusters.json, verbatim.""" + key = (entry["scenario_id"], entry["variable"]) + reasons = [ + item["reason"] + for item in clusters[entry["cluster"]]["review"]["corrected_per_output"] + if (item["scenario_id"], item["variable"]) == key + ] + if len(reasons) != 1: + raise SystemExit(f"{key}: {len(reasons)} reviewed reasons in {entry['cluster']}") + return reasons[0] + + +def audit_exclusions(actions: dict) -> dict: + """The exclusion records of the outputs the audit excluded on review, apart from + any engine change (final_actions.json audit_exclusions), by output.""" + records = {} + for item in actions.get("audit_exclusions", []): + key = (item["scenario_id"], item["variable"]) + record = item["exclusion"] + if (record["scenario_id"], record["variable"]) != key or key in records: + raise SystemExit(f"malformed audit exclusion {key}") + records[key] = record + return records + + +def exclusion_derivation(base: str, decided: int, on_review: int) -> str: + """The exclusion record's derivation: the 20260922c text, then the upgrade's. + `decided` counts the records decided on DATE, `on_review` those among them the + audit excluded on review.""" + text = ( + base + + f" On {DATE} the references moved to policyengine-us {ENGINE} with the" + " pre-freeze conventions. Every excluded output was recomputed there, and each" + " one that moved was re-reviewed and stays excluded (the reference sidecar's" + " engine_upgrade revision lists them). Records decided before that date keep" + f" the values they were decided on; the {NUMBER_WORDS[decided]} records" + f" decided that day were computed on {ENGINE}." + ) + if on_review == 1: + text += ( + " The audit excluded one of them on review of the release. Its reference" + " did not move, so the engine_upgrade revision does not list it" + " (final_actions.json audit_exclusions)." + ) + elif on_review: + text += ( + f" The audit excluded {NUMBER_WORDS[on_review]} of them on review of the" + " release. Their references did not move, so the engine_upgrade revision" + " does not list them (final_actions.json audit_exclusions)." + ) + return text + + +def load(name: str): + spec = importlib.util.spec_from_file_location(f"build_{name}", FIXES / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def moved(variable: str, a: float, b: float) -> bool: + if variable.endswith("_eligible"): + return round(a) != round(b) + return abs(a - b) > 1.0 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--actions", required=True) + parser.add_argument("--out-dir", required=True) + args = parser.parse_args() + if version("policyengine-us") != ENGINE: + raise SystemExit(f"need policyengine-us {ENGINE}, have {version('policyengine-us')}") + + sys.path.insert(0, str(HERE / "sweep")) + from sweep import build_situation # noqa: E402 (sweep.py's reference builder) + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.policyengine_runtime import policyengine_release_bundle + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + from policyengine_us import CountryTaxBenefitSystem, Simulation + + actions = json.loads(Path(args.actions).read_text()) + approved = {(a["scenario_id"], a["variable"]): a for a in actions["approved"]} + rechecked = {(a["scenario_id"], a["variable"]): a for a in actions["excluded_rechecked"]} + clusters = { + c["id"]: c + for c in json.loads(Path(args.actions).with_name("clusters.json").read_text())["clusters"] + } + audit = audit_exclusions(actions) + + BOARD = _board() + modules = {name: load(name) for name in PARTS} + system = CountryTaxBenefitSystem(reform=tuple(m.reform for m in modules.values())) + board = pd.read_csv(BOARD / "reference_outputs.csv") + board_meta = json.loads((BOARD / "reference_outputs.csv.meta.json").read_text()) + v11 = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index(["scenario_id", "variable"])["value"] + exclusion_file = json.loads((BOARD / "reference_exclusions.json").read_text()) + exclusions = exclusion_file["exclusions"] + excluded = {(e["scenario_id"], e["variable"]) for e in exclusions} + added = {(e["scenario_id"], e["variable"]): e for e in actions["new_exclusions"]} + if set(added) & excluded: + raise SystemExit("a new exclusion is already excluded") + if set(audit) & (excluded | set(added) | set(approved)): + raise SystemExit("an audit exclusion is already excluded, added or approved") + scenarios = pd.read_csv(BUNDLE / "scenarios.csv") + + computed = {} + for _, srow in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + situation = build_situation(scenario) + if scenario.id == "scenario_066": + # The builder mapping this wave depends on (stated 40 hours reach SNAP). + head = next(iter(situation["people"].values())) + assert "weekly_hours_worked_before_lsr" in head, "builder hours mapping missing" + sim = Simulation(tax_benefit_system=system, situation=situation) + for variable in expand_programs_for_scenario(board_meta["programs"], scenario): + key = (scenario.id, variable) + computed[key] = float( + _extract_person_value( + sim.calculate(_pe_variable_for_output(variable, "us"), YEAR), scenario, variable + ) + ) + + problems, changed, small, excluded_values = [], [], [], [] + new = board.copy() + for idx, row in board.iterrows(): + key = (row["scenario_id"], row["variable"]) + value, old = computed[key], float(row["value"]) + if key in excluded: + excluded_values.append((key, value, old)) + continue + if key in audit: + # Excluded on review, not for an engine change: the value must not move. + if abs(value - old) > 1e-6 or abs(value - float(audit[key]["frozen_value"])) > 1e-6: + problems.append( + f"audit exclusion {key}: engine {value}, board {old}, " + f"record {audit[key]['frozen_value']}" + ) + continue + new.loc[idx, "value"] = value + continue + if key in added: + record = added[key] + if abs(value - float(record["frozen_value"])) > 1e-3: + problems.append(f"new exclusion {key}: engine {value} != record {record['frozen_value']}") + continue + new.loc[idx, "value"] = value + changed.append({ + "scenario_id": key[0], "variable": key[1], "frozen": float(v11[key]), + "previous": old, "regenerated": value, + "cause": "excluded_reference_depends_on_unlisted_input", + "basis": ( + f"Newly excluded from scoring on {record['decided_on']} " + "(reference_exclusions.json): " + record["unlisted_input"] + ), + }) + continue + if moved(key[1], value, old): + if key not in approved: + problems.append(f"unreviewed move {key}: {old} -> {value}") + continue + if abs(value - float(approved[key]["value"])) > 1e-3: + problems.append(f"{key}: engine {value} != approved {approved[key]['value']}") + continue + changed.append({ + "scenario_id": key[0], "variable": key[1], "frozen": float(v11[key]), + "previous": old, "regenerated": value, "cause": approved[key]["cause"], + "basis": approved[key]["basis"], + }) + elif abs(value - old) > 1e-6: + small.append({ + "scenario_id": key[0], "variable": key[1], "frozen": float(v11[key]), + "previous": old, "regenerated": value, "cause": "engine_upgrade_within_1", + "basis": "Moves by $1 or less on policyengine-us 2.15.17; no score changes at the exact-match tolerance.", + }) + new.loc[idx, "value"] = value + for key in approved: + if key in excluded or not moved(key[1], computed[key], float(board.set_index(["scenario_id", "variable"]).loc[key, "value"])): + problems.append(f"approved change did not happen or is excluded: {key}") + moved_excluded = [(k, v, o) for k, v, o in excluded_values if moved(k[1], v, o)] + reasons = {} + for key, value, old in moved_excluded: + if key not in rechecked: + problems.append(f"excluded output moved without a recheck: {key}: {old} -> {value}") + continue + reasons[key] = reviewed_reason(clusters, rechecked[key]) + if rechecked[key].get("reason", reasons[key]) != reasons[key]: + problems.append(f"final_actions reason for {key} is not the reviewer's text") + if problems: + raise SystemExit("refusing to write references:\n " + "\n ".join(problems)) + + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + new.to_csv(out / "reference_outputs.csv", index=False) + exclusion_file["exclusions"] = exclusions + list(added.values()) + list(audit.values()) + exclusion_file["derivation"] = exclusion_derivation( + exclusion_file["derivation"], len(added) + len(audit), len(audit) + ) + (out / "reference_exclusions.json").write_text(dump_record(exclusion_file)) + meta = json.loads(json.dumps(board_meta)) + meta["policyengine_bundles"]["us"] = policyengine_release_bundle("us") + meta["reference_csv_sha256"] = sha256(out / "reference_outputs.csv") + meta["regenerated_at_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + meta["revisions"].append({ + "date": DATE, + "kind": "engine_upgrade", + "root_cause": "engine_upgrade_policyengine_us_2_15_17", + "outputs": "every scored output", + "rule": ( + f"{RULE} {ENGINE_RULE} Max's ruling, 2026-09-28: 'we should be using the " + "latest pe for this always!' The conventions that hold pre-freeze law are " + "re-expressed for the release the sweep uses." + ), + "engine_version": f"policyengine-us {ENGINE}", + "previous_engine_version": "policyengine-us 1.755.4", + "policyengine_py": ( + "policyengine 6.1.2, provenance only: its certified US bundle is " + "policyengine-us 2.2.1, and it refuses to import next to a newer model" + ), + "fix_modules": [ + {"module": f"{name}.py", "sha256": sha256(FIXES / f"{name}.py")} + for name in (*CONVENTION_MODULES, *PARTS) + ], + "builder": ( + "policybench.scenarios.Scenario.to_pe_household passes stated usual weekly " + "hours (hours_worked_last_week) to weekly_hours_worked_before_lsr as well; " + "upstream policyengine-us#9261 changed that input's default from 40 to 0" + ), + "excluded_outputs_untouched": True, + "excluded_outputs_rechecked": [ + { + "scenario_id": k[0], "variable": k[1], "kept_value": o, + "value_on_2_15_17": v, "reason": reasons[k], + } + for k, v, o in moved_excluded + ], + "changed": changed + small, + }) + (out / "reference_outputs.csv.meta.json").write_text(dump_record(meta)) + + # Engine traces of every changed output, for the derivation narratives + # (regen_references.py narratives reads reference_traces.json the same way). + import types + + sys.modules.setdefault("litellm", types.ModuleType("litellm")) # tracing needs no LLM + from policybench.case_reference_explanations import _find_target_tree, _render_trace + + by_id = scenarios.set_index("scenario_id") + traces = {} + for item in changed + small: + scenario = scenario_from_dict(json.loads(by_id.loc[item["scenario_id"], "scenario_json"])) + sim = Simulation(tax_benefit_system=system, situation=build_situation(scenario)) + sim.trace = True + pe_variable = _pe_variable_for_output(item["variable"], "us") + sim.calculate(pe_variable, YEAR) + tree = _find_target_tree(sim.tracer.trees, pe_variable) + traces[f"{item['scenario_id']}|{item['variable']}"] = { + "pe_variable": pe_variable, + "trace": "\n".join(_render_trace(tree)) if tree else "", + } + (out / "reference_traces.json").write_text(json.dumps(traces, indent=1)) + print(f"sha256 {meta['reference_csv_sha256']}") + print(f"{len(changed)} reviewed changes, {len(small)} within $1, {len(moved_excluded)} excluded outputs rechecked") + for c in changed + small: + print(f" {c['scenario_id']} {c['variable']:46s} {c['previous']:>11.2f} -> {c['regenerated']:>11.2f} {c['cause']}") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/narratives_latest.py b/reference_audit/2026-09-28/scripts/narratives_latest.py new file mode 100644 index 00000000..6716177f --- /dev/null +++ b/reference_audit/2026-09-28/scripts/narratives_latest.py @@ -0,0 +1,214 @@ +"""Rewrite the derivation narratives of the outputs the 2026-09-28 upgrade changed. + +Same writer as every other reference narrative (policybench.case_reference_explanations: +REFERENCE_MODEL, TEMPERATURE, MAX_TOKENS, _prompt), given the policyengine-us 2.15.17 +trace from build_references_latest.py and the reviewed basis of each change as +grounding. The narrative must state the new reference value; a draft that omits it is +retried twice, then refused. + +HAND_CORRECTED replaces narratives the writer got wrong, each checked against the +2.15.17 trace (reference_traces.json): 013 (the writer called $240 a monthly benefit +over January-October and said the household met the net income and asset tests; the +trace is $0 in January-February and $24 from March, through categorical eligibility), +008 and 082 (the writer misdescribed how the New Jersey EITC and New York CDCC derive +from the federal credits), 028 (the writer dated the child support fix to 2.15.17). +The frozen narrative of scenario_064 dependent2_chip_eligible said the 18-year-old "exceeds CHIP's age +eligibility threshold". CHIP's age limit is under 19 (gov.hhs.chip.child.max_age); on +2.15.17 with the conventions, medicaid_income_level is 3.215 against Wisconsin's 3.06 +CHIP limit, and has_chip_disqualifying_health_coverage is true (has_esi). That wrong +narrative is what the 2026-09-28 judge flag on this case repeated. + + ANTHROPIC_API_KEY=... PYTHONPATH= python narratives_latest.py \\ + --references ../reference_v13 --actions latest/final_actions.json \\ + --explanations /us_case_reference_explanations.csv --out +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +import pandas as pd + +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) +YEAR = 2026 +ENGINE_NOTE = ( + " PolicyEngine here is policyengine-us 2.15.17 with the benchmark's conventions for " + "law published before July 3, 2026." +) +HAND_CORRECTED = { + ("scenario_013", "snap"): ( + "PolicyEngine calculated SNAP benefits of $240 for 2026 for this 80-year-old " + "disabled Arizonan who lives alone: $0 in January and February and the $24 monthly " + "minimum from March through December. The household's gross income is $2,539.33 a " + "month. Under SNAP's ordinary rules it fails the net income test, with net income " + "of $2,138 a month, and the asset test, with $58,700 in the bank. From March 2026 " + "Arizona extends SNAP through expanded categorical eligibility to households with " + "gross income up to 200% of the poverty guideline ($2,608.33 a month, $2,660 from " + "October), which treats the income and asset tests as met. The ordinary benefit " + "formula, the $298 maximum less 30% of net income, gives less than nothing, so the " + "household receives the $24 minimum in each of those ten months." + ), + ("scenario_008", "state_refundable_credits"): ( + "PolicyEngine calculated New Jersey refundable credits of $5,842.40 for this " + "married couple filing jointly with six children. The New Jersey earned income " + "credit is $3,292.40: 40% of the federal earned income credit, which reaches its " + "$8,231 maximum for a family with three or more children. The New Jersey child tax " + "credit is $2,500 under the schedule New Jersey enacted on June 30, 2026 for tax " + "years 2026 to 2028, which pays more per young child at low taxable income. The " + "New Jersey property tax credit adds $50." + ), + ("scenario_082", "state_refundable_credits"): ( + "PolicyEngine calculated New York refundable credits of $667 for this head of " + "household with one child aged 1. The Empire State child credit is $307: the " + "$1,000 credit for a child aged three or younger, reduced by $16.50 for each whole " + "$1,000 of federal adjusted gross income ($117,652.65) above the $75,000 threshold, " + "42 steps or $693. New York's child and dependent care credit is $360: 60% of the " + "federal credit, which is 20% of the $3,000 of care expenses the federal credit " + "counts for one child." + ), + ("scenario_028", "reduced_price_school_meals_eligible"): ( + "PolicyEngine determined that this Pennsylvania household of four is not eligible " + "for reduced-price school meals. School meals count the household's income of " + "$61,277: $60,000 of wages, $10 of interest and $1,267 of child support received, " + "which counts as income for school meals (7 CFR 245.6(a)(5)(ii)). That is 1.86 " + "times the $33,000 poverty guideline for a household of four, above the 185% limit " + "for reduced-price meals, so the value is 0." + ), + ("scenario_064", "dependent2_chip_eligible"): ( + "PolicyEngine determined that Dependent 2 is not eligible for CHIP in Wisconsin " + "for 2026. At 18, Dependent 2 is within CHIP's age limit, which covers children " + "under 19. Two other conditions fail. The household's income is 321.5% of the " + "federal poverty guideline, above Wisconsin's CHIP limit of 306%. And Dependent 2 " + "has employer-sponsored insurance, which PolicyEngine counts as coverage that " + "rules a child out of CHIP. PolicyEngine also finds Dependent 2 not eligible for " + "Medicaid, so the value is False." + ), +} + + +def money(value: float) -> str: + return f"{value:,.2f}".removesuffix(".00") + + +def main() -> None: + import litellm + + from policybench.case_reference_explanations import ( + MAX_TOKENS, + REFERENCE_MODEL, + TEMPERATURE, + _prompt, + _scenario_summary, + ) + + parser = argparse.ArgumentParser() + parser.add_argument("--references", required=True) + parser.add_argument("--actions", required=True) + parser.add_argument("--explanations", required=True) + parser.add_argument("--out", required=True) + args = parser.parse_args() + refs = Path(args.references) + meta = json.loads((refs / "reference_outputs.csv.meta.json").read_text()) + upgrade = meta["revisions"][-1] + assert upgrade["kind"] == "engine_upgrade" + traces = json.loads((refs / "reference_traces.json").read_text()) + actions = json.loads(Path(args.actions).read_text()) + basis = {(a["scenario_id"], a["variable"]): a["basis"] for a in actions["approved"]} + basis |= { + (e["scenario_id"], e["variable"]): ( + "PolicyBench does not score this output, because the reference depends on " + "an input the prompt does not state. " + e["alternative_reading"] + ) + for e in actions["new_exclusions"] + } + scenarios = pd.read_csv(BUNDLE / "scenarios.csv").set_index( + "scenario_id", drop=False + ) + explanations = pd.read_csv(args.explanations) + + async def one(item, extra=""): + key = (item["scenario_id"], item["variable"]) + traced = traces[f"{key[0]}|{key[1]}"] + prompt = _prompt( + "us", + _scenario_summary(scenarios.loc[key[0]]), + key[1], + traced["pe_variable"], + item["regenerated"], + YEAR, + traced["trace"], + grounding=basis.get(key, item["basis"]) + ENGINE_NOTE + extra, + ) + response = await litellm.acompletion( + model=REFERENCE_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + ) + text = response.choices[0].message.content.strip() + lines = text.split("\n") + if lines and lines[0].lstrip().startswith("#"): + text = "\n".join(lines[1:]).strip() + return text, len(traced["trace"].splitlines()) + + def required(item) -> list[str]: + if item["variable"].endswith("_eligible"): + return [] + return [money(item["regenerated"])] + + results = [] + for item in upgrade["changed"]: + key = (item["scenario_id"], item["variable"]) + if key in HAND_CORRECTED: + text = HAND_CORRECTED[key] + assert not [f for f in required(item) if f not in text], key + n_lines = len(traces[f"{key[0]}|{key[1]}"]["trace"].splitlines()) + results.append((key[0], key[1], item["regenerated"], n_lines, text)) + continue + text, n_lines = asyncio.run(one(item)) + for _ in range(2): + missing = [f for f in required(item) if f not in text] + if not missing: + break + text, n_lines = asyncio.run( + one(item, f" The narrative must state the value {', '.join(missing)}.") + ) + missing = [f for f in required(item) if f not in text] + if missing: + raise SystemExit( + f"{item['scenario_id']} {item['variable']}: omits {missing}" + ) + results.append( + (item["scenario_id"], item["variable"], item["regenerated"], n_lines, text) + ) + for (scenario_id, variable), text in HAND_CORRECTED.items(): + if (scenario_id, variable) in {(r[0], r[1]) for r in results}: + continue + mask = (explanations.scenario_id == scenario_id) & ( + explanations.variable == variable + ) + value = float(explanations.loc[mask, "reference_value"].iloc[0]) + lines = int(explanations.loc[mask, "trace_lines"].iloc[0]) + results.append((scenario_id, variable, value, lines, text)) + for scenario_id, variable, value, n_lines, text in results: + mask = (explanations.scenario_id == scenario_id) & ( + explanations.variable == variable + ) + assert mask.sum() == 1, (scenario_id, variable) + explanations.loc[mask, "reference_value"] = value + explanations.loc[mask, "trace_lines"] = n_lines + explanations.loc[mask, "explanation"] = text + explanations.loc[mask, "error"] = pd.NA + print(f"--- {scenario_id} {variable} ({value:,.2f})\n{text}\n") + explanations.to_csv(args.out, index=False) + print(f"rewrote {len(results)} narratives -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/probe_023_medicaid.py b/reference_audit/2026-09-28/scripts/probe_023_medicaid.py new file mode 100644 index 00000000..d2679794 --- /dev/null +++ b/reference_audit/2026-09-28/scripts/probe_023_medicaid.py @@ -0,0 +1,194 @@ +"""Compute scenario_023 head_medicaid_eligible under both readings of "is disabled". + +The prompt states only that the California head is disabled. Her MAGI is above +the 138% adult limit, so Medi-Cal is open to her only through a disability +pathway, here the 250% Working Disabled Program. That program requires the +federal (SSA) definition of disability (42 CFR 435.540(a); the LA DPSS 250% WDP +policy the engine cites), which is policyengine-us's meets_ssi_disability_criteria, +the input that already excludes this household's SNAP. 2.15.17's +ca_wdp_disability_eligible reads the broad is_disabled flag instead. + +The script runs the reference system (fixes/latest_final.py on policyengine-us +2.15.17) and the same system with the WDP disability test following the SSA +definition, each with the stated facts, reading A (the head does not meet the +SSI/SSA disability criteria) and reading B (she does). It asserts that the +reference system reproduces the committed reference, and writes +verification/probe_023_medicaid.json. + +latest_c_irs_sales_tax_2025.py reads r19_irs_sales_tax_2025.json beside itself, +and that table is committed in ../2026-09-22/fixes, so the script loads the fix +modules from a temporary copy of fixes/ with the table added, as the builder's +docstring does. Run from the checkout: + + OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD .venv/bin/python \\ + reference_audit/2026-09-28/scripts/probe_023_medicaid.py +""" + +from __future__ import annotations + +import copy +import csv +import hashlib +import importlib.util +import json +import shutil +import sys +import tempfile +from importlib.metadata import version +from pathlib import Path + +HERE = Path(__file__).resolve().parent +AUDIT = HERE.parent +ROOT = AUDIT.parents[1] +RUN = ROOT / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +OUT = AUDIT / "verification" / "probe_023_medicaid.json" +ENGINE = "2.15.17" +YEAR = 2026 +SCENARIO = "scenario_023" +OUTPUT = "head_medicaid_eligible" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def main() -> None: + if version("policyengine-us") != ENGINE: + raise SystemExit(f"need policyengine-us {ENGINE}, have {version('policyengine-us')}") + + from policyengine_core.reforms import Reform + from policyengine_us import CountryTaxBenefitSystem, Simulation + from policyengine_us.model_api import YEAR as PERIOD_YEAR + from policyengine_us.model_api import Person, StateCode, Variable + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + + sweep = _load("probe_sweep", HERE / "sweep.py") # the references' household builder + + with tempfile.TemporaryDirectory(prefix="probe023-") as scratch: + fixes = Path(scratch) + for module in (AUDIT / "fixes").glob("*.py"): + shutil.copyfile(module, fixes / module.name) + shutil.copyfile( + AUDIT.parent / "2026-09-22" / "fixes" / "r19_irs_sales_tax_2025.json", + fixes / "r19_irs_sales_tax_2025.json", + ) + latest_final = _load("probe_latest_final", fixes / "latest_final.py") + + class ca_wdp_disability_eligible(Variable): + value_type = bool + entity = Person + label = "California 250% WDP disability eligible (SSA definition)" + definition_period = PERIOD_YEAR + defined_for = StateCode.CA + + def formula(person, period, parameters): + # 2.15.17 reads is_disabled here; the law reads the SSA definition. + return ( + person("meets_ssi_disability_criteria", period) + | person("is_blind", period) + | (person("social_security_disability", period) > 0) + ) + + class wdp_ssa_definition(Reform): + def apply(self): + latest_final.reform.apply(self) + self.update_variable(ca_wdp_disability_eligible) + + systems = { + "latest_final": CountryTaxBenefitSystem(reform=latest_final.reform), + "latest_final_wdp_ssa_definition": CountryTaxBenefitSystem( + reform=wdp_ssa_definition + ), + } + + with (RUN / "scenarios.csv").open(newline="") as source: + csv.field_size_limit(sys.maxsize) + row = next(r for r in csv.DictReader(source) if r["scenario_id"] == SCENARIO) + scenario = scenario_from_dict(json.loads(row["scenario_json"])) + situation = sweep.build_situation(scenario) + readings = { + "stated_facts": None, + "reading_a": False, # does not meet the SSI/SSA disability criteria + "reading_b": True, # meets them + } + pe_variable = _pe_variable_for_output(OUTPUT, "us") + results = {} + for system_name, system in systems.items(): + for reading, criteria in readings.items(): + household = copy.deepcopy(situation) + if criteria is not None: + for person in household["people"].values(): + person["meets_ssi_disability_criteria"] = {str(YEAR): criteria} + sim = Simulation(tax_benefit_system=system, situation=household) + value = float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, OUTPUT) + ) + head = 0 # the household's only person + results[f"{system_name}/{reading}"] = { + OUTPUT: value, + "medicaid_category": str( + sim.calculate("medicaid_category", YEAR).decode_to_str()[head] + ), + "ca_wdp_disability_eligible": bool( + sim.calculate("ca_wdp_disability_eligible", YEAR)[head] + ), + "ca_wdp_eligible": bool(sim.calculate("ca_wdp_eligible", YEAR)[head]), + "medicaid_income_level": round( + float(sim.calculate("medicaid_income_level", YEAR)[head]), 4 + ), + "ssi": float(sim.calculate("ssi", YEAR)[head]), + } + + with (RUN / "reference_outputs.csv").open(newline="") as source: + reference = next( + float(r["value"]) + for r in csv.DictReader(source) + if (r["scenario_id"], r["variable"]) == (SCENARIO, OUTPUT) + ) + stated = results["latest_final/stated_facts"][OUTPUT] + if stated != reference: + raise SystemExit(f"reference system gives {stated}, committed {reference}") + + record = { + "scenario_id": SCENARIO, + "variable": OUTPUT, + "pe_variable": pe_variable, + "engine_version": f"policyengine-us {ENGINE}", + "committed_reference": reference, + "fix_module": { + "module": "fixes/latest_final.py", + "sha256": _sha256(AUDIT / "fixes" / "latest_final.py"), + }, + "probe_sha256": _sha256(Path(__file__)), + "readings": { + "stated_facts": "the scenario as built, meets_ssi_disability_criteria unset", + "reading_a": "meets_ssi_disability_criteria false: the head does not meet " + "the SSI/SSA disability criteria", + "reading_b": "meets_ssi_disability_criteria true: the head meets them", + }, + "systems": { + "latest_final": "the reference system: policyengine-us 2.15.17 with " + "fixes/latest_final.py; ca_wdp_disability_eligible reads is_disabled", + "latest_final_wdp_ssa_definition": "the same with " + "ca_wdp_disability_eligible reading meets_ssi_disability_criteria " + "(the SSA definition, 42 CFR 435.540(a)) in place of is_disabled", + }, + "results": results, + } + OUT.write_text(json.dumps(record, indent=1) + "\n") + for key, value in results.items(): + print(key, json.dumps(value)) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/rewrite_reference_records.py b/reference_audit/2026-09-28/scripts/rewrite_reference_records.py new file mode 100644 index 00000000..1880f8e1 --- /dev/null +++ b/reference_audit/2026-09-28/scripts/rewrite_reference_records.py @@ -0,0 +1,119 @@ +"""Rewrite the 2026-09-29 reference records without recomputing any output. + +build_references_latest.py recomputes all 1,984 outputs on policyengine-us 2.15.17 +before it writes the records. For record changes that need no engine run, this +script applies the builder's own record functions to the installed files instead: + +- each rechecked excluded output's reason in the sidecar's engine_upgrade revision + becomes the full text of its cluster review's corrected_per_output entry + (clusters.json; reviewed_reason); +- the exclusion record gains the audit exclusions listed in final_actions.json + (audit_exclusions), after the upgrade's own, with the derivation the builder writes + (exclusion_derivation). + +It copies the reference CSV byte for byte, refuses an audit exclusion whose +frozen_value is not the committed reference or that the engine upgrade changed, and +writes the records with the builder's serialization (dump_record). +scripts/install_adds0929_references.py then installs the output only if the reference +CSV is byte-identical and nothing changed but record text and the listed audit +exclusions. Run from the checkout (git must hold the 20260922c base commit): + + .venv/bin/python reference_audit/2026-09-28/scripts/rewrite_reference_records.py \\ + --out-dir OUT + .venv/bin/python scripts/install_adds0929_references.py --built OUT \\ + --stage-dir results/local/adds0928-v3 \\ + --built-by reference_audit/2026-09-28/scripts/rewrite_reference_records.py +""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +AUDIT = HERE.parent +ROOT = AUDIT.parents[1] +RUN = "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +CSV = "reference_outputs.csv" +SIDECAR = "reference_outputs.csv.meta.json" +EXCLUSIONS = "reference_exclusions.json" + + +def builder(): + """build_references_latest.py, for its record functions (no engine import).""" + spec = importlib.util.spec_from_file_location( + "build_references_latest", HERE / "build_references_latest.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def rewrite(installed: Path, actions: dict, clusters: dict, base_derivation: str): + """The installed sidecar and exclusion record, rewritten as the builder writes + them. Returns (sidecar, exclusions).""" + build = builder() + records = {} + for name in (SIDECAR, EXCLUSIONS): + raw = (installed / name).read_text() + records[name] = json.loads(raw) + if build.dump_record(records[name]) != raw: + raise SystemExit(f"{name} is not in the builder's serialization") + sidecar, exclusions = records[SIDECAR], records[EXCLUSIONS] + upgrade = sidecar["revisions"][-1] + if upgrade.get("kind") != "engine_upgrade": + raise SystemExit("the sidecar's last revision is not the engine upgrade") + + rechecked = {(a["scenario_id"], a["variable"]): a for a in actions["excluded_rechecked"]} + for item in upgrade["excluded_outputs_rechecked"]: + entry = rechecked[(item["scenario_id"], item["variable"])] + item["reason"] = build.reviewed_reason(clusters, entry) + + audit = build.audit_exclusions(actions) + with (installed / CSV).open(newline="") as source: + values = {(r["scenario_id"], r["variable"]): float(r["value"]) for r in csv.DictReader(source)} + changed = {(c["scenario_id"], c["variable"]) for c in upgrade["changed"]} + for key, record in audit.items(): + if key in changed or record["frozen_value"] != values[key]: + raise SystemExit(f"audit exclusion {key} is not an unchanged reference") + kept = [e for e in exclusions["exclusions"] if (e["scenario_id"], e["variable"]) not in audit] + exclusions["exclusions"] = kept + list(audit.values()) + exclusions["derivation"] = build.exclusion_derivation( + base_derivation, len(actions["new_exclusions"]) + len(audit), len(audit) + ) + return sidecar, exclusions + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--installed", type=Path, default=ROOT / RUN) + args = parser.parse_args() + build = builder() + actions = json.loads((AUDIT / "final_actions.json").read_text()) + clusters = {c["id"]: c for c in json.loads((AUDIT / "clusters.json").read_text())["clusters"]} + base = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{build.BASE_COMMIT}:{RUN}/{EXCLUSIONS}"], + check=True, + capture_output=True, + ).stdout + sidecar, exclusions = rewrite(args.installed, actions, clusters, json.loads(base)["derivation"]) + args.out_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(args.installed / CSV, args.out_dir / CSV) + (args.out_dir / SIDECAR).write_text(build.dump_record(sidecar)) + (args.out_dir / EXCLUSIONS).write_text(build.dump_record(exclusions)) + print( + f"{len(sidecar['revisions'][-1]['excluded_outputs_rechecked'])} rechecked reasons, " + f"{len(exclusions['exclusions'])} exclusions -> {args.out_dir}" + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/summarize_rerun_sweeps.py b/reference_audit/2026-09-28/scripts/summarize_rerun_sweeps.py new file mode 100644 index 00000000..6a0c62fd --- /dev/null +++ b/reference_audit/2026-09-28/scripts/summarize_rerun_sweeps.py @@ -0,0 +1,184 @@ +"""Summarize what each sweep re-run on policyengine-us 2.15.17 moves. + +On 2.15.17 PolicyBench re-ran four of the September 22 sweeps over every output +(the IRA deduction limit fix r02, the net investment income tax definition r25, +and the readings for mortgage residence and 40 unlisted weekly hours) and ran a +new one for the state and local tax refund reading. Each sweep adds one fix or +reading to a baseline sweep: + +- latest_conventions: the nine ported publication conventions + (fixes/latest_conventions.py); +- latest_map_stated_hours: the same, with each prompt's stated usual weekly + hours also passed to weekly_hours_worked_before_lsr, as the scenario builder + now does for the references (fixes/latest_map_stated_hours.py). The 40-hour + sweep composes it (fixes/latest_alt_unlisted_hours_40.py). + +This script reads the sweeps' CSVs (sweep_latest.py output, one row per output) +from the triage directory and writes verification/rerun_sweeps.json: each +sweep's source hash and module, and every output whose value differs from its +baseline or from latest_conventions by any amount, with the three values. The +tests recompute the claims the README, the card and the paper make from it. + +Run from the checkout (the paths are those of the machine that ran the sweeps): + + .venv/bin/python reference_audit/2026-09-28/scripts/summarize_rerun_sweeps.py +""" + +from __future__ import annotations + +import csv +import hashlib +import json +from pathlib import Path + +HERE = Path(__file__).resolve().parents[1] +OUT = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/out" +) +TRIAGE = OUT.parents[1] +ENGINE = "2.15.17" +BASELINES = { + "latest_conventions": { + "csv": "latest_conventions.csv", + "module": "fixes/latest_conventions.py", + }, + "latest_map_stated_hours": { + "csv": "latest_map_stated_hours.csv", + "module": "fixes/latest_map_stated_hours.py", + }, +} +SWEEPS = [ + { + "sweep": "latest_alt_r02_ira_219g", + "csv": "latest_alt_r02_ira_219g.csv", + "module": "fixes/latest_alt_r02_ira_219g.py", + "adds": "the IRA deduction limit fix (IRC 219(g) active-participant " + "phase-out; reference_audit/2026-09-22/fixes/r02_ira_219g_v2.py)", + "september_22_root_cause": "r02_ira_219g", + "baseline": "latest_conventions", + }, + { + "sweep": "latest_conventions_plus_r25_niit_excluded", + "csv": "latest_conventions_r25_niit_excluded.csv", + "module": "fixes/alt_conventions_r25.py", + "adds": "the reading of federal income tax before refundable credits " + "without the net investment income tax " + "(reference_audit/2026-09-22/fixes/r25_niit_excluded.py)", + "september_22_root_cause": "r25_niit_in_federal_output", + "baseline": "latest_conventions", + }, + { + "sweep": "latest_alt_snap_mortgage_residence", + "csv": "latest_alt_snap_mortgage_residence.csv", + "module": "fixes/latest_alt_snap_mortgage_residence.py", + "adds": "the reading that listed mortgage interest is on the occupied " + "home, a SNAP shelter cost", + "september_22_root_cause": "r15_snap_mortgage_interest", + "baseline": "latest_conventions", + }, + { + "sweep": "latest_alt_unlisted_hours_40", + "csv": "latest_alt_unlisted_hours_40.csv", + "module": "fixes/latest_alt_unlisted_hours_40.py", + "adds": "the reading that a person with no stated weekly hours works 40", + "september_22_root_cause": "r14_unlisted_weekly_hours_v2", + "baseline": "latest_map_stated_hours", + }, + { + "sweep": "latest_alt_salt_refund_no_prior_benefit", + "csv": "latest_alt_salt_refund_no_prior_benefit.csv", + "module": "fixes/latest_alt_salt_refund_no_prior_benefit.py", + "adds": "the reading that a listed state and local tax refund is not " + "income (26 U.S.C. 111(a)); new on 2.15.17", + "september_22_root_cause": None, + "baseline": "latest_conventions", + }, +] + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _values(name: str, fix: str | None = None) -> dict[tuple[str, str], float]: + with (OUT / name).open(newline="") as source: + rows = list(csv.DictReader(source)) + if len(rows) != 1984 or {r["engine"] for r in rows} != {ENGINE}: + raise SystemExit(f"{name}: not a full sweep on policyengine-us {ENGINE}") + if fix is not None and {r["fix"] for r in rows} != {fix}: + raise SystemExit(f"{name}: not the {fix} sweep") + return {(r["scenario_id"], r["variable"]): float(r["recomputed"]) for r in rows} + + +def main() -> None: + conventions = _values(BASELINES["latest_conventions"]["csv"], "latest_conventions") + values = {"latest_conventions": conventions} + baselines = {} + for name, spec in BASELINES.items(): + values[name] = _values(spec["csv"], name) + entry = { + "csv": spec["csv"], + "csv_sha256": _sha256(OUT / spec["csv"]), + "module": spec["module"], + "module_sha256": _sha256(HERE / spec["module"]), + } + if name != "latest_conventions": + entry["differs_from_latest_conventions"] = [ + { + "scenario_id": key[0], + "variable": key[1], + "latest_conventions": conventions[key], + name: value, + } + for key, value in sorted(values[name].items()) + if value != conventions[key] + ] + baselines[name] = entry + sweeps = [] + for spec in SWEEPS: + recomputed = _values(spec["csv"], spec["sweep"]) + base = values[spec["baseline"]] + moves = [ + { + "scenario_id": key[0], + "variable": key[1], + "baseline": base[key], + "latest_conventions": conventions[key], + "recomputed": value, + } + for key, value in sorted(recomputed.items()) + if value != base[key] or value != conventions[key] + ] + sweeps.append( + { + **spec, + "csv_sha256": _sha256(OUT / spec["csv"]), + "module_sha256": _sha256(HERE / spec["module"]), + "outputs": len(recomputed), + "moves": moves, + } + ) + record = { + "note": ( + "What each sweep PolicyBench ran on policyengine-us 2.15.17 for an " + "exclusion's fix or reading moves, written by " + "scripts/summarize_rerun_sweeps.py from the sweeps' CSVs in " + f"{OUT.relative_to(TRIAGE.parent)} (sweep_latest.py output; each " + "csv_sha256 is the file read). A sweep adds its fix or reading to " + "its baseline sweep. 'moves' lists every output whose recomputed " + "value differs, by any amount, from the baseline's or from " + "latest_conventions'. Every other output of the 1,984 has the same " + "value in the sweep, its baseline and latest_conventions." + ), + "engine": ENGINE, + "baselines": baselines, + "sweeps": sweeps, + } + path = HERE / "verification" / "rerun_sweeps.json" + path.write_text(json.dumps(record, indent=2) + "\n") + for sweep in sweeps: + print(f"{sweep['sweep']}: {len(sweep['moves'])} outputs differ") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/sweep.py b/reference_audit/2026-09-28/scripts/sweep.py new file mode 100644 index 00000000..a226f1c4 --- /dev/null +++ b/reference_audit/2026-09-28/scripts/sweep.py @@ -0,0 +1,141 @@ +"""Recompute every PolicyBench reference under one fix and list the outputs that move. + +A fix is a Python module that defines either or both of: + reform a policyengine_core Reform subclass (or None) + patch(situation, scenario) returns an edited copy of the situation dict +and optionally FIX_ID and DESCRIPTION strings. + +Run with the policyengine-us 1.755.4 triage venv: + cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ + ../.venv-pe1755/bin/python sweep.py --fix fixes/r01_ira_compensation.py --out out/r01.csv + (no --fix: the baseline, which must reproduce all 1,984 frozen references) + +The output CSV has one row per reference output: scenario_id, variable, frozen, +recomputed, delta, moved. An amount output moves when |delta| > 1 (the exact-match +tolerance); a binary output moves when the flag flips. The script prints the +moved rows and a summary line. +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +from policybench.ground_truth import _extract_person_value, _pe_variable_for_output +from policybench.scenarios import scenario_from_dict +from policybench.spec import expand_programs_for_scenario + +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) +# The frozen v1.1 bundle (reference_outputs.csv sha256 b9136a15...), read-only. Not +# adds202609/publish/: the fold overwrites that dir with the regenerated references. +YEAR = 2026 +BINARY_SUFFIXES = ("_eligible",) +# The v1.1 reference run renamed this input for policyengine-us 1.755.4 +# (results/local/v1_1_runbook.md step 3; v1_1/us/scenarios_patched.csv). +RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} + + +def build_situation(scenario) -> dict: + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in RENAME.items(): + if old in person: + person[new] = person.pop(old) + return situation + + +def load_fix(path: str | None): + if not path: + return None, None, "baseline" + spec = importlib.util.spec_from_file_location("fix_module", path) + module = importlib.util.module_from_spec(spec) + sys.modules["fix_module"] = module + spec.loader.exec_module(module) + return ( + getattr(module, "reform", None), + getattr(module, "patch", None), + getattr(module, "FIX_ID", Path(path).stem), + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--fix") + parser.add_argument("--out", required=True) + parser.add_argument("--scenarios", nargs="*", help="restrict to these scenario ids") + args = parser.parse_args() + + from policyengine_us import CountryTaxBenefitSystem, Simulation + + reform, patch, fix_id = load_fix(args.fix) + # Build the reformed system once and share it across households: passing + # reform= to each Simulation re-applies it to a fresh copy every time. + system = CountryTaxBenefitSystem(reform=reform) if reform is not None else None + meta = json.loads((BUNDLE / "reference_outputs.csv.meta.json").read_text()) + programs = meta["programs"] + frozen = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index( + ["scenario_id", "variable"] + )["value"] + scenarios = pd.read_csv(BUNDLE / "scenarios.csv") + if args.scenarios: + scenarios = scenarios[scenarios["scenario_id"].isin(args.scenarios)] + + rows = [] + for _, srow in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + situation = build_situation(scenario) + if patch is not None: + situation = patch(copy.deepcopy(situation), scenario) + sim = ( + Simulation(tax_benefit_system=system, situation=situation) + if system is not None + else Simulation(situation=situation) + ) + for variable in expand_programs_for_scenario(programs, scenario): + key = (scenario.id, variable) + if key not in frozen.index: + continue + pe_variable = _pe_variable_for_output(variable, "us") + value = float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, variable) + ) + ref = float(frozen[key]) + delta = value - ref + binary = variable.endswith(BINARY_SUFFIXES) + moved = (round(value) != round(ref)) if binary else abs(delta) > 1.0 + rows.append( + { + "fix": fix_id, + "scenario_id": scenario.id, + "state": srow["state"], + "variable": variable, + "frozen": ref, + "recomputed": value, + "delta": delta, + "moved": moved, + } + ) + out = pd.DataFrame(rows) + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + out.to_csv(args.out, index=False) + moved = out[out["moved"]] + with pd.option_context("display.width", 200, "display.max_rows", 500): + print(moved[["scenario_id", "state", "variable", "frozen", "recomputed", "delta"]].to_string(index=False)) + print( + f"SUMMARY fix={fix_id} outputs={len(out)} moved={len(moved)} " + f"small_nonzero_deltas={int(((out['delta'].abs() > 1e-6) & ~out['moved']).sum())}" + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/scripts/sweep_latest.py b/reference_audit/2026-09-28/scripts/sweep_latest.py new file mode 100644 index 00000000..8c1419ba --- /dev/null +++ b/reference_audit/2026-09-28/scripts/sweep_latest.py @@ -0,0 +1,164 @@ +"""Recompute every PolicyBench reference with the newest policyengine-us (sweep.py's harness). + +Max, 2026-09-28: references always come from the latest PolicyEngine. This runs +sweep.py's exact situation builder under whatever policyengine-us the interpreter +has (the .venv-pe21517 venv: 2.15.17, newest on PyPI at 2026-09-29 00:23 UTC) and +compares each output with both the raw v1.1 references (1.755.4) and the current +board's references (release 20260922c: 1.755.4 plus the r26-r33 regenerations and +the published-law conventions). "moved" is against the current board. + +Original sweep.py docstring follows. + +Recompute every PolicyBench reference under one fix and list the outputs that move. + +A fix is a Python module that defines either or both of: + reform a policyengine_core Reform subclass (or None) + patch(situation, scenario) returns an edited copy of the situation dict +and optionally FIX_ID and DESCRIPTION strings. + +Run with the policyengine-us 1.755.4 triage venv: + cd /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep + PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/opus55 \ + ../.venv-pe1755/bin/python sweep.py --fix fixes/r01_ira_compensation.py --out out/r01.csv + (no --fix: the baseline, which must reproduce all 1,984 frozen references) + +The output CSV has one row per reference output: scenario_id, variable, frozen, +recomputed, delta, moved. An amount output moves when |delta| > 1 (the exact-match +tolerance); a binary output moves when the flag flips. The script prints the +moved rows and a summary line. +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pandas as pd + +from policybench.ground_truth import _extract_person_value, _pe_variable_for_output +from policybench.scenarios import scenario_from_dict +from policybench.spec import expand_programs_for_scenario + +BUNDLE = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/newmodels/publish/" + "us_full_run_20260612_policyengine_4_16_1_populace/us" +) +BOARD = Path( + "/Users/maxghenis/PolicyEngine/policybench-wt/adds0928-stage2/paper/snapshot/20260501/" + "runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv" +) +# The frozen v1.1 bundle (reference_outputs.csv sha256 b9136a15...), read-only. Not +# adds202609/publish/: the fold overwrites that dir with the regenerated references. +YEAR = 2026 +BINARY_SUFFIXES = ("_eligible",) +# The v1.1 reference run renamed this input for policyengine-us 1.755.4 +# (results/local/v1_1_runbook.md step 3; v1_1/us/scenarios_patched.csv). +RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} + + +def build_situation(scenario) -> dict: + situation = scenario.to_pe_household() + for person in situation["people"].values(): + for old, new in RENAME.items(): + if old in person: + person[new] = person.pop(old) + return situation + + +def load_fix(path: str | None): + if not path: + return None, None, "baseline" + spec = importlib.util.spec_from_file_location("fix_module", path) + module = importlib.util.module_from_spec(spec) + sys.modules["fix_module"] = module + spec.loader.exec_module(module) + return ( + getattr(module, "reform", None), + getattr(module, "patch", None), + getattr(module, "FIX_ID", Path(path).stem), + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--fix") + parser.add_argument("--out", required=True) + parser.add_argument("--scenarios", nargs="*", help="restrict to these scenario ids") + args = parser.parse_args() + + from policyengine_us import CountryTaxBenefitSystem, Simulation + + reform, patch, fix_id = load_fix(args.fix) + # Build the reformed system once and share it across households: passing + # reform= to each Simulation re-applies it to a fresh copy every time. + system = CountryTaxBenefitSystem(reform=reform) if reform is not None else None + meta = json.loads((BUNDLE / "reference_outputs.csv.meta.json").read_text()) + programs = meta["programs"] + frozen = pd.read_csv(BUNDLE / "reference_outputs.csv").set_index( + ["scenario_id", "variable"] + )["value"] + board = pd.read_csv(BOARD).set_index(["scenario_id", "variable"])["value"] + from importlib.metadata import version as _version + engine = _version("policyengine-us") + print(f"policyengine-us {engine}", flush=True) + scenarios = pd.read_csv(BUNDLE / "scenarios.csv") + if args.scenarios: + scenarios = scenarios[scenarios["scenario_id"].isin(args.scenarios)] + + rows = [] + for _, srow in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + situation = build_situation(scenario) + if patch is not None: + situation = patch(copy.deepcopy(situation), scenario) + sim = ( + Simulation(tax_benefit_system=system, situation=situation) + if system is not None + else Simulation(situation=situation) + ) + for variable in expand_programs_for_scenario(programs, scenario): + key = (scenario.id, variable) + if key not in frozen.index: + continue + pe_variable = _pe_variable_for_output(variable, "us") + value = float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, variable) + ) + v11 = float(frozen[key]) + ref = float(board[key]) + delta = value - ref + binary = variable.endswith(BINARY_SUFFIXES) + moved = (round(value) != round(ref)) if binary else abs(delta) > 1.0 + rows.append( + { + "fix": fix_id, + "scenario_id": scenario.id, + "state": srow["state"], + "variable": variable, + "engine": engine, + "frozen": ref, + "v11_1755": v11, + "recomputed": value, + "moved_vs_v11": (round(value) != round(v11)) if binary else abs(value - v11) > 1.0, + "delta": delta, + "moved": moved, + } + ) + out = pd.DataFrame(rows) + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + out.to_csv(args.out, index=False) + moved = out[out["moved"]] + with pd.option_context("display.width", 200, "display.max_rows", 500): + print(moved[["scenario_id", "state", "variable", "frozen", "recomputed", "delta"]].to_string(index=False)) + print( + f"SUMMARY fix={fix_id} outputs={len(out)} moved={len(moved)} " + f"small_nonzero_deltas={int(((out['delta'].abs() > 1e-6) & ~out['moved']).sum())}" + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-09-28/sweep_moves.csv b/reference_audit/2026-09-28/sweep_moves.csv new file mode 100644 index 00000000..cf4f027d --- /dev/null +++ b/reference_audit/2026-09-28/sweep_moves.csv @@ -0,0 +1,1985 @@ +scenario_id,variable,state,v11_1755,board_20260922c,raw_2_15_17,conventions,final,reference,excluded,cluster,action,moved_vs_board +scenario_000,federal_income_tax_before_refundable_credits,TX,2883.49365234375,2906.45361328125,2906.45361328125,2906.45361328125,2906.45361328125,2906.45361328125,False,,,False +scenario_000,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,head_medicare_eligible,TX,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_000,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,payroll_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_000,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,federal_income_tax_before_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,federal_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,free_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,head_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,head_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,head_medicare_eligible,VA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_001,head_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,local_income_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,payroll_tax,VA,2223.393798828125,2223.393798828125,2223.393798828125,2223.393798828125,2223.393798828125,2223.393798828125,False,,,False +scenario_001,reduced_price_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,self_employment_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,snap,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,spouse_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,spouse_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,spouse_medicare_eligible,VA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_001,spouse_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,ssi,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,state_income_tax_before_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,state_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_001,tanf,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,federal_income_tax_before_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_002,federal_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,free_school_meals_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,head_chip_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,head_medicaid_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,head_medicare_eligible,WA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_002,head_wic_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,local_income_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,payroll_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,reduced_price_school_meals_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,self_employment_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,snap,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,spouse_chip_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,spouse_medicaid_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,spouse_medicare_eligible,WA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_002,spouse_wic_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,ssi,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,state_income_tax_before_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,state_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_002,tanf,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,federal_income_tax_before_refundable_credits,TX,22154.69921875,22154.69921875,22154.69921875,22154.69921875,22154.69921875,22154.69921875,True,,,False +scenario_003,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,payroll_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,spouse_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,spouse_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,spouse_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,spouse_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_003,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,federal_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,federal_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,free_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,head_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,head_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,head_medicare_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,head_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,local_income_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,payroll_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,reduced_price_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,self_employment_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,snap,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,spouse_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,spouse_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,spouse_medicare_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,spouse_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,ssi,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,state_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,state_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_004,tanf,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,federal_income_tax_before_refundable_credits,CA,106505.8984375,106505.8984375,107833.15625,107833.15625,107833.15625,106505.8984375,True,excl_r02_ira_219g_federal,keep_excluded,True +scenario_005,federal_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,free_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,head_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,head_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,head_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,head_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,local_income_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,payroll_tax,CA,36044.0,36044.0,36044.0,36044.0,36044.0,36044.0,False,,,False +scenario_005,reduced_price_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,self_employment_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,snap,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,spouse_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,spouse_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,spouse_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,spouse_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,ssi,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,state_income_tax_before_refundable_credits,CA,41051.51171875,41051.51171875,40986.8671875,41267.01171875,41267.01171875,41051.51171875,True,excl_r11_ca_itemized_conformity,keep_excluded,True +scenario_005,state_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_005,tanf,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,federal_income_tax_before_refundable_credits,ID,3707.695068359375,3707.695068359375,3707.695068359375,3707.695068359375,3707.695068359375,3707.695068359375,False,,,False +scenario_007,federal_refundable_credits,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,free_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,head_chip_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,head_medicaid_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,head_medicare_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_007,head_wic_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,local_income_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,payroll_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,reduced_price_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,self_employment_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,snap,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,ssi,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_007,state_income_tax_before_refundable_credits,ID,755.7783203125,755.7783203125,755.7783203125,761.5570068359375,761.5570068359375,755.7783203125,True,excl_r07_idaho_health_premiums,keep_excluded,True +scenario_007,state_refundable_credits,ID,155.0,155.0,155.0,155.0,155.0,155.0,False,,,False +scenario_007,tanf,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child1_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child1_early_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child1_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child1_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child1_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child1_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child2_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child2_early_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child2_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child2_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child2_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child2_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child3_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child3_early_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child3_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child3_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child3_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child3_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child4_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child4_early_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child4_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child4_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child4_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child4_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child5_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child5_early_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child5_head_start_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child5_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child5_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child5_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child6_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child6_early_head_start_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child6_head_start_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child6_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,child6_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,child6_wic_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,federal_income_tax_before_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,federal_refundable_credits,NJ,12433.611328125,12433.611328125,12433.611328125,12433.611328125,12433.611328125,12433.611328125,False,,,False +scenario_008,free_school_meals_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,head_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,head_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,head_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,head_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,local_income_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,payroll_tax,NJ,2162.760009765625,2162.760009765625,2162.760009765625,2162.760009765625,2162.760009765625,2162.760009765625,True,,,False +scenario_008,reduced_price_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,self_employment_tax,NJ,565.1820068359375,565.1820068359375,565.1820068359375,565.1820068359375,565.1820068359375,565.1820068359375,False,,,False +scenario_008,snap,NJ,15246.9052734375,15108.0,15273.0,15108.0,15108.0,15108.0,False,,,False +scenario_008,spouse_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,spouse_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_008,spouse_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,spouse_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,ssi,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_008,state_income_tax_before_refundable_credits,NJ,264.80999755859375,264.80999755859375,264.80999755859375,264.80999755859375,264.80999755859375,264.80999755859375,False,,,False +scenario_008,state_refundable_credits,NJ,5342.39990234375,5342.39990234375,5842.39990234375,5842.39990234375,5842.39990234375,5842.39990234375,False,nj_ctc_fy2027_budget,adopt_latest,True +scenario_008,tanf,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,federal_income_tax_before_refundable_credits,NC,8348.0,8348.0,8348.0,8348.0,8348.0,8348.0,False,,,False +scenario_009,federal_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,payroll_tax,NC,4590.0,4590.0,4590.0,4590.0,4590.0,4590.0,False,,,False +scenario_009,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,self_employment_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,spouse_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,spouse_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,spouse_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,spouse_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,state_income_tax_before_refundable_credits,NC,2190.510009765625,2190.510009765625,2190.510009765625,2190.510009765625,2190.510009765625,2190.510009765625,False,,,False +scenario_009,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_009,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,child1_chip_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,child1_early_head_start_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,child1_head_start_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,child1_medicaid_eligible,MS,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_012,child1_medicare_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,child1_wic_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,federal_income_tax_before_refundable_credits,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,federal_refundable_credits,MS,6127.0,6127.0,6127.0,6127.0,6127.0,6127.0,False,,,False +scenario_012,free_school_meals_eligible,MS,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_012,head_chip_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,head_medicaid_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,head_medicare_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,head_wic_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,local_income_tax,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,payroll_tax,MS,1683.0,1683.0,1683.0,1683.0,1683.0,1683.0,False,,,False +scenario_012,reduced_price_school_meals_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,self_employment_tax,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,snap,MS,4952.08935546875,4884.0,4962.0,4884.0,4884.0,4884.0,False,,,False +scenario_012,spouse_chip_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,spouse_medicaid_eligible,MS,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_012,spouse_medicare_eligible,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,spouse_wic_eligible,MS,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_012,ssi,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,state_income_tax_before_refundable_credits,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,state_refundable_credits,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_012,tanf,MS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,federal_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,federal_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,free_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,head_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,head_medicaid_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,head_medicare_eligible,AZ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_013,head_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,local_income_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,payroll_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,reduced_price_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,self_employment_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,snap,AZ,0.0,0.0,243.0,240.0,240.0,240.0,False,az_snap_bbce_200,adopt_latest,True +scenario_013,ssi,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,state_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_013,state_refundable_credits,AZ,25.0,25.0,25.0,25.0,25.0,25.0,False,,,False +scenario_013,tanf,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,federal_income_tax_before_refundable_credits,WV,6311.31787109375,6311.31787109375,6311.31787109375,6311.31787109375,6311.31787109375,6311.31787109375,False,,,False +scenario_014,federal_refundable_credits,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,free_school_meals_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,head_chip_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,head_medicaid_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,head_medicare_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,head_wic_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,local_income_tax,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,payroll_tax,WV,6802.96484375,6802.96484375,6802.96484375,6802.96484375,6802.96484375,6802.96484375,False,,,False +scenario_014,reduced_price_school_meals_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,self_employment_tax,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,snap,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,spouse_chip_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,spouse_medicaid_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,spouse_medicare_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,spouse_wic_eligible,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,ssi,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,state_income_tax_before_refundable_credits,WV,3092.186279296875,3092.186279296875,3092.186279296875,3092.186279296875,3092.186279296875,3092.186279296875,False,,,False +scenario_014,state_refundable_credits,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_014,tanf,WV,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,federal_income_tax_before_refundable_credits,IN,2969.1259765625,2969.1259765625,2969.1259765625,2969.1259765625,2969.1259765625,2969.1259765625,False,,,False +scenario_015,federal_refundable_credits,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,free_school_meals_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,head_chip_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,head_medicaid_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,head_medicare_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,head_wic_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,local_income_tax,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,payroll_tax,IN,3442.5,3442.5,3442.5,3442.5,3442.5,3442.5,False,,,False +scenario_015,reduced_price_school_meals_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,self_employment_tax,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,snap,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,ssi,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,state_income_tax_before_refundable_credits,IN,1236.3267822265625,1236.3267822265625,1236.3267822265625,1236.3267822265625,1236.3267822265625,1236.3267822265625,False,,,False +scenario_015,state_refundable_credits,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_015,tanf,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,federal_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,head_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,payroll_tax,FL,2649.80224609375,2649.80224609375,2649.80224609375,2649.80224609375,2649.80224609375,2649.80224609375,False,,,False +scenario_016,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_016,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,federal_income_tax_before_refundable_credits,AZ,5211.05078125,5211.05078125,5211.05078125,5211.05078125,5211.05078125,5211.05078125,False,,,False +scenario_018,federal_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,free_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,head_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,head_medicaid_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,head_medicare_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,head_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,local_income_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,payroll_tax,AZ,5202.38232421875,5202.38232421875,5202.38232421875,5202.38232421875,5202.38232421875,5202.38232421875,False,,,False +scenario_018,reduced_price_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,self_employment_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,snap,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,ssi,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,state_income_tax_before_refundable_credits,AZ,1146.05224609375,1146.05224609375,1146.05224609375,1146.05224609375,1146.05224609375,1146.05224609375,False,,,False +scenario_018,state_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_018,tanf,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,federal_income_tax_before_refundable_credits,TX,68056.7109375,68056.7109375,68334.734375,68334.734375,68334.734375,68056.7109375,True,excl_niit_scope_020,keep_excluded,True +scenario_020,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,payroll_tax,TX,16689.0,16689.0,16689.0,16689.0,16689.0,16689.0,False,,,False +scenario_020,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_020,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,federal_income_tax_before_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,federal_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,free_school_meals_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,head_chip_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,head_medicaid_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,head_medicare_eligible,MO,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_021,head_wic_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,local_income_tax,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,payroll_tax,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,reduced_price_school_meals_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,self_employment_tax,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,snap,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,ssi,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,state_income_tax_before_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,state_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_021,tanf,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,federal_income_tax_before_refundable_credits,CA,11131.3271484375,11113.5703125,11133.97265625,11113.5703125,11113.5703125,11113.5703125,False,,,False +scenario_022,federal_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,free_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,head_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,head_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,head_medicare_eligible,CA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_022,head_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,local_income_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,payroll_tax,CA,4710.83251953125,4710.83251953125,4710.83251953125,4710.83251953125,4710.83251953125,4710.83251953125,False,,,False +scenario_022,reduced_price_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,self_employment_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,snap,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,ssi,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,state_income_tax_before_refundable_credits,CA,2439.650146484375,2439.650146484375,2439.650146484375,2505.8701171875,2505.8701171875,2439.650146484375,True,excl_r11_ca_itemized_conformity,keep_excluded,True +scenario_022,state_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_022,tanf,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,federal_income_tax_before_refundable_credits,CA,643.43359375,643.43359375,643.43359375,643.43359375,643.43359375,643.43359375,False,,,False +scenario_023,federal_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,free_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,head_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,head_medicaid_eligible,CA,1.0,1.0,1.0,1.0,1.0,1.0,True,excl_snap_ssi_disability,exclude_unlisted_input,False +scenario_023,head_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,head_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,local_income_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,payroll_tax,CA,1561.1168212890625,1561.1168212890625,1561.1168212890625,1561.1168212890625,1561.1168212890625,1561.1168212890625,False,,,False +scenario_023,reduced_price_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,self_employment_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,snap,CA,461.3397216796875,461.3397216796875,462.0,408.0,408.0,461.3397216796875,True,excl_snap_ssi_disability,keep_excluded,True +scenario_023,ssi,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_023,state_income_tax_before_refundable_credits,CA,6.7984771728515625,12.776718139648438,4.212188720703125,12.776718139648438,12.776718139648438,12.776718139648438,False,,,False +scenario_023,state_refundable_credits,CA,148.3108673095703,95.1540069580078,97.51927185058594,95.1540069580078,95.1540069580078,95.1540069580078,False,,,False +scenario_023,tanf,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,federal_income_tax_before_refundable_credits,OH,7031.03564453125,7031.03564453125,7031.03564453125,7031.03564453125,7031.03564453125,7031.03564453125,False,,,False +scenario_025,federal_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,free_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,head_chip_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,head_medicaid_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,head_medicare_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,head_wic_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,local_income_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,payroll_tax,OH,4798.48486328125,4798.48486328125,4798.48486328125,4798.48486328125,4798.48486328125,4798.48486328125,False,,,False +scenario_025,reduced_price_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,self_employment_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,snap,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,spouse_chip_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,spouse_medicaid_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,spouse_medicare_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,spouse_wic_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,ssi,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,state_income_tax_before_refundable_credits,OH,1921.57275390625,1921.57275390625,1921.57275390625,1921.57275390625,1921.57275390625,1921.57275390625,False,,,False +scenario_025,state_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_025,tanf,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child1_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child1_early_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child1_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child1_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_026,child1_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child1_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child2_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child2_early_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child2_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child2_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_026,child2_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child2_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child3_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child3_early_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child3_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child3_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_026,child3_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,child3_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,federal_income_tax_before_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,federal_refundable_credits,NC,2013.4892578125,2013.4892578125,2013.4892578125,2013.4892578125,2013.4892578125,2013.4892578125,False,,,False +scenario_026,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,payroll_tax,NC,6518.474609375,6518.474609375,6518.474609375,6518.474609375,6518.474609375,6518.474609375,False,,,False +scenario_026,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,self_employment_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,spouse_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,spouse_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,spouse_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,spouse_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,state_income_tax_before_refundable_credits,NC,2086.79052734375,2086.79052734375,2086.79052734375,2086.79052734375,2086.79052734375,2086.79052734375,False,,,False +scenario_026,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_026,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,federal_income_tax_before_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,federal_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,free_school_meals_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,head_chip_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,head_medicaid_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,head_medicare_eligible,CT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_027,head_wic_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,local_income_tax,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,payroll_tax,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,reduced_price_school_meals_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,self_employment_tax,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,snap,CT,287.68316650390625,288.0,291.0,288.0,288.0,288.0,False,,,False +scenario_027,spouse_chip_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,spouse_medicaid_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,spouse_medicare_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,spouse_wic_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,ssi,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,state_income_tax_before_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,state_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_027,tanf,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_early_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child1_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_early_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child2_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_early_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,child3_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,federal_income_tax_before_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,federal_refundable_credits,PA,3274.927978515625,3274.927978515625,3274.927978515625,3274.927978515625,3274.927978515625,3274.927978515625,False,,,False +scenario_028,free_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,head_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,head_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,head_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,head_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,local_income_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,payroll_tax,PA,4632.0,4632.0,4632.0,4632.0,4632.0,4632.0,False,,,False +scenario_028,reduced_price_school_meals_eligible,PA,1.0,1.0,0.0,0.0,0.0,0.0,False,school_meals_child_support_income,adopt_latest,True +scenario_028,self_employment_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,snap,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,ssi,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_028,state_income_tax_before_refundable_credits,PA,1842.3070068359375,1842.3070068359375,1842.3070068359375,1842.3070068359375,1842.3070068359375,1842.3070068359375,False,,,False +scenario_028,state_refundable_credits,PA,62.41279220581055,62.41279220581055,62.41279220581055,62.41279220581055,62.41279220581055,62.41279220581055,False,,,False +scenario_028,tanf,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,federal_income_tax_before_refundable_credits,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,federal_refundable_credits,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,free_school_meals_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,head_chip_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,head_medicaid_eligible,OK,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_029,head_medicare_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,head_wic_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,local_income_tax,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,payroll_tax,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,reduced_price_school_meals_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,self_employment_tax,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,snap,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,ssi,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,state_income_tax_before_refundable_credits,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_029,state_refundable_credits,OK,40.0,40.0,40.0,40.0,40.0,40.0,False,,,False +scenario_029,tanf,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,federal_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,head_medicaid_eligible,TX,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_030,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,head_wic_eligible,TX,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_030,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,payroll_tax,TX,994.5,994.5,994.5,994.5,994.5,994.5,False,,,False +scenario_030,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,snap,TX,287.68316650390625,288.0,291.0,288.0,288.0,288.0,False,,,False +scenario_030,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_030,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,federal_income_tax_before_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,federal_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,free_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,head_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,head_medicaid_eligible,CA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_031,head_medicare_eligible,CA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_031,head_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,local_income_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,payroll_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,reduced_price_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,self_employment_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,snap,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,ssi,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,state_income_tax_before_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,state_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_031,tanf,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,child1_chip_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,child1_early_head_start_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,child1_head_start_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,child1_medicaid_eligible,MN,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_032,child1_medicare_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,child1_wic_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,federal_income_tax_before_refundable_credits,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,federal_refundable_credits,MN,6127.0,6127.0,6127.0,6127.0,6127.0,6127.0,False,,,False +scenario_032,free_school_meals_eligible,MN,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_032,head_chip_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,head_medicaid_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,head_medicare_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,head_wic_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,local_income_tax,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,payroll_tax,MN,2346.10009765625,2346.10009765625,2346.10009765625,2346.10009765625,2346.10009765625,2346.10009765625,False,,,False +scenario_032,reduced_price_school_meals_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,self_employment_tax,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,snap,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,spouse_chip_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,spouse_medicaid_eligible,MN,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_032,spouse_medicare_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,spouse_wic_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,ssi,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,state_income_tax_before_refundable_credits,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_032,state_refundable_credits,MN,2187.60009765625,2187.60009765625,2187.60009765625,2187.60009765625,2187.60009765625,2187.60009765625,False,,,False +scenario_032,tanf,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,federal_income_tax_before_refundable_credits,SD,3788.988037109375,3788.988037109375,3818.148193359375,3818.148193359375,3818.148193359375,3818.148193359375,True,salt_refund_gross_income_9122,exclude_unlisted_input,True +scenario_033,federal_refundable_credits,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,free_school_meals_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,head_chip_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,head_medicaid_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,head_medicare_eligible,SD,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_033,head_wic_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,local_income_tax,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,payroll_tax,SD,6043.5,6043.5,6043.5,6043.5,6043.5,6043.5,False,,,False +scenario_033,reduced_price_school_meals_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,self_employment_tax,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,snap,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,spouse_chip_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,spouse_medicaid_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,spouse_medicare_eligible,SD,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_033,spouse_wic_eligible,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,ssi,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,state_income_tax_before_refundable_credits,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,state_refundable_credits,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_033,tanf,SD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,federal_income_tax_before_refundable_credits,NJ,4926.39990234375,4926.39990234375,4926.39990234375,4926.39990234375,4926.39990234375,4926.39990234375,False,,,False +scenario_036,federal_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,free_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,head_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,head_medicaid_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,head_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,head_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,local_income_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,payroll_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,reduced_price_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,self_employment_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,snap,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,ssi,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,state_income_tax_before_refundable_credits,NJ,1724.155029296875,1724.155029296875,1724.155029296875,1724.155029296875,1724.155029296875,1724.155029296875,False,,,False +scenario_036,state_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_036,tanf,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,federal_income_tax_before_refundable_credits,NC,1649.2513427734375,1649.2513427734375,1649.2513427734375,1649.2513427734375,1649.2513427734375,1649.2513427734375,False,,,False +scenario_037,federal_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,payroll_tax,NC,3208.668701171875,3208.668701171875,3208.668701171875,3208.668701171875,3208.668701171875,3208.668701171875,False,,,False +scenario_037,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,self_employment_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,state_income_tax_before_refundable_credits,NC,1148.69921875,1148.69921875,1148.69921875,1148.69921875,1148.69921875,1148.69921875,False,,,False +scenario_037,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_037,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child1_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child1_early_head_start_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child1_head_start_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child1_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_038,child1_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child1_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child2_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child2_early_head_start_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child2_head_start_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child2_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_038,child2_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,child2_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,federal_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,federal_refundable_credits,LA,10225.353515625,10225.353515625,10225.353515625,10225.353515625,10225.353515625,10225.353515625,False,,,False +scenario_038,free_school_meals_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_038,head_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,head_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_038,head_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,head_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,local_income_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,payroll_tax,LA,571.2637329101562,571.2637329101562,571.2637329101562,571.2637329101562,571.2637329101562,571.2637329101562,False,,,False +scenario_038,reduced_price_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,self_employment_tax,LA,2193.612548828125,2193.612548828125,2193.612548828125,2193.612548828125,2193.612548828125,2193.612548828125,False,,,False +scenario_038,snap,LA,7286.9443359375,7212.0,7302.0,7212.0,7212.0,7212.0,False,,,False +scenario_038,spouse_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,spouse_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_038,spouse_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,spouse_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,ssi,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,state_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_038,state_refundable_credits,LA,365.8000183105469,365.8000183105469,365.8000183105469,365.8000183105469,365.8000183105469,365.8000183105469,False,,,False +scenario_038,tanf,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,federal_income_tax_before_refundable_credits,VA,1345.510498046875,1345.510498046875,1345.510498046875,1345.510498046875,1345.510498046875,1345.510498046875,True,,,False +scenario_039,federal_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,free_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,head_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,head_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,head_medicare_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,head_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,local_income_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,payroll_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,reduced_price_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,self_employment_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,snap,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,ssi,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,state_income_tax_before_refundable_credits,VA,514.4984130859375,514.4984130859375,514.4984130859375,514.4984130859375,514.4984130859375,514.4984130859375,True,,,False +scenario_039,state_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_039,tanf,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,federal_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,federal_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,free_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,head_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,head_medicaid_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,head_medicare_eligible,AZ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_040,head_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,local_income_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,payroll_tax,AZ,2142.0,2142.0,2142.0,2142.0,2142.0,2142.0,False,,,False +scenario_040,reduced_price_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,self_employment_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,snap,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,spouse_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,spouse_medicaid_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,spouse_medicare_eligible,AZ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_040,spouse_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,ssi,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,state_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,state_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_040,tanf,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,federal_income_tax_before_refundable_credits,WI,1979.1583251953125,2361.96484375,2361.96484375,2361.96484375,2361.96484375,2361.96484375,False,,,False +scenario_042,federal_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,free_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,head_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,head_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,head_medicare_eligible,WI,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_042,head_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,local_income_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,payroll_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,reduced_price_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,self_employment_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,snap,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,ssi,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,state_income_tax_before_refundable_credits,WI,284.74090576171875,284.74090576171875,468.82916259765625,464.1828002929687,464.1828002929687,284.74090576171875,True,excl_r32_wi_capital_gain_distributions,keep_excluded,True +scenario_042,state_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_042,tanf,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,federal_income_tax_before_refundable_credits,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,federal_refundable_credits,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,free_school_meals_eligible,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,head_chip_eligible,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,head_medicaid_eligible,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,head_medicare_eligible,CO,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_043,head_wic_eligible,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,local_income_tax,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,payroll_tax,CO,330.7310791015625,330.7310791015625,330.7310791015625,330.7310791015625,330.7310791015625,330.7310791015625,False,,,False +scenario_043,reduced_price_school_meals_eligible,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,self_employment_tax,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,snap,CO,3596.039794921875,3576.0,3600.0,3576.0,3576.0,3576.0,False,,,False +scenario_043,ssi,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,state_income_tax_before_refundable_credits,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_043,state_refundable_credits,CO,19.0,19.0,19.0,19.0,19.0,19.0,False,,,False +scenario_043,tanf,CO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,federal_income_tax_before_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,federal_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,free_school_meals_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,head_chip_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,head_medicaid_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,head_medicare_eligible,KS,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_044,head_wic_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,local_income_tax,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,payroll_tax,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,reduced_price_school_meals_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,self_employment_tax,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,snap,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,spouse_chip_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,spouse_medicaid_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,spouse_medicare_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,spouse_wic_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,ssi,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,state_income_tax_before_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,state_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_044,tanf,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,federal_income_tax_before_refundable_credits,MI,2173.07080078125,2173.07080078125,2173.07080078125,2173.07080078125,2173.07080078125,2173.07080078125,False,,,False +scenario_045,federal_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,free_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,head_chip_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,head_medicaid_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,head_medicare_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,head_wic_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,local_income_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,payroll_tax,MI,2775.08251953125,2775.08251953125,2775.08251953125,2775.08251953125,2775.08251953125,2775.08251953125,False,,,False +scenario_045,reduced_price_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,self_employment_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,snap,MI,287.68316650390625,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,ssi,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_045,state_income_tax_before_refundable_credits,MI,1288.8375244140625,1290.9625244140625,1288.8375244140625,1290.9625244140625,1290.9625244140625,1290.9625244140625,False,,,False +scenario_045,state_refundable_credits,MI,760.7887573242188,760.7887573242188,760.7887573242188,760.7887573242188,760.7887573242188,760.7887573242188,False,,,False +scenario_045,tanf,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_chip_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_early_head_start_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_head_start_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_medicaid_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_medicare_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child1_wic_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_chip_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_early_head_start_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_head_start_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_medicaid_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_medicare_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,child2_wic_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,federal_income_tax_before_refundable_credits,OK,499.71484375,499.71484375,499.71484375,499.71484375,499.71484375,499.71484375,False,,,False +scenario_046,federal_refundable_credits,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,free_school_meals_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,head_chip_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,head_medicaid_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,head_medicare_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,head_wic_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,local_income_tax,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,payroll_tax,OK,7038.0,7038.0,7038.0,7038.0,7038.0,7038.0,False,,,False +scenario_046,reduced_price_school_meals_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,self_employment_tax,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,snap,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,spouse_chip_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,spouse_medicaid_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,spouse_medicare_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,spouse_wic_eligible,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,ssi,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,state_income_tax_before_refundable_credits,OK,2739.135009765625,2739.135009765625,2739.135009765625,2739.135009765625,2739.135009765625,2739.135009765625,False,,,False +scenario_046,state_refundable_credits,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_046,tanf,OK,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,federal_income_tax_before_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,federal_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,free_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,head_chip_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,head_medicaid_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,head_medicare_eligible,OH,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_048,head_wic_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,local_income_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,payroll_tax,OH,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,False,,,False +scenario_048,reduced_price_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,self_employment_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,snap,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,ssi,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,state_income_tax_before_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,state_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_048,tanf,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,federal_income_tax_before_refundable_credits,NH,30543.908203125,30543.908203125,30543.908203125,30543.908203125,30543.908203125,30543.908203125,True,,,False +scenario_049,federal_refundable_credits,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,free_school_meals_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,head_chip_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,head_medicaid_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,head_medicare_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,head_wic_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,local_income_tax,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,payroll_tax,NH,15134.5,15134.5,15134.5,15134.5,15134.5,15134.5,False,,,False +scenario_049,reduced_price_school_meals_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,self_employment_tax,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,snap,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,spouse_chip_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,spouse_medicaid_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,spouse_medicare_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,spouse_wic_eligible,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,ssi,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,state_income_tax_before_refundable_credits,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,state_refundable_credits,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_049,tanf,NH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,federal_income_tax_before_refundable_credits,LA,2620.0,2620.0,2620.0,2620.0,2620.0,2620.0,False,,,False +scenario_051,federal_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,free_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,head_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,head_medicaid_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,head_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,head_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,local_income_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,payroll_tax,LA,3060.0,3060.0,3060.0,3060.0,3060.0,3060.0,False,,,False +scenario_051,reduced_price_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,self_employment_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,snap,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,ssi,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,state_income_tax_before_refundable_credits,LA,814.9500122070312,820.3499755859375,820.3499755859375,820.3499755859375,820.3499755859375,820.3499755859375,False,,,False +scenario_051,state_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_051,tanf,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,federal_income_tax_before_refundable_credits,TX,104211.40625,104211.40625,104211.40625,104211.40625,104211.40625,104211.40625,True,,,False +scenario_052,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,payroll_tax,TX,20939.0,20939.0,20939.0,20939.0,20939.0,20939.0,False,,,False +scenario_052,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,spouse_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,spouse_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,spouse_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,spouse_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_052,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,federal_income_tax_before_refundable_credits,ID,5903.109375,5903.109375,5903.109375,5903.109375,5903.109375,5903.109375,False,,,False +scenario_053,federal_refundable_credits,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,free_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,head_chip_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,head_medicaid_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,head_medicare_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,head_wic_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,local_income_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,payroll_tax,ID,5123.10400390625,5123.10400390625,5123.10400390625,5123.10400390625,5123.10400390625,5123.10400390625,False,,,False +scenario_053,reduced_price_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,self_employment_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,snap,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,ssi,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_053,state_income_tax_before_refundable_credits,ID,2435.2783203125,2435.2783203125,2435.2783203125,2441.05712890625,2441.05712890625,2435.2783203125,True,excl_r07_idaho_health_premiums,keep_excluded,True +scenario_053,state_refundable_credits,ID,155.0,155.0,155.0,155.0,155.0,155.0,False,,,False +scenario_053,tanf,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child1_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child1_early_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child1_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child1_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,child1_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child1_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child2_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child2_early_head_start_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,child2_head_start_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child2_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,child2_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,child2_wic_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,federal_income_tax_before_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,federal_refundable_credits,NC,8365.55859375,8365.55859375,8365.55859375,8365.55859375,8365.55859375,8365.55859375,False,,,False +scenario_054,free_school_meals_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,head_medicaid_eligible,NC,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_054,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,payroll_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,self_employment_tax,NC,2416.15283203125,2416.15283203125,2416.15283203125,2416.15283203125,2416.15283203125,2416.15283203125,False,,,False +scenario_054,snap,NC,6125.68896484375,6060.0,6138.0,6060.0,6060.0,6060.0,False,,,False +scenario_054,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,state_income_tax_before_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_054,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,federal_income_tax_before_refundable_credits,FL,25238.578125,25238.578125,25238.578125,25238.578125,25238.578125,25238.578125,False,,,False +scenario_055,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,head_medicare_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_055,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,payroll_tax,FL,6398.08837890625,6398.08837890625,6398.08837890625,6398.08837890625,6398.08837890625,6398.08837890625,False,,,False +scenario_055,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_055,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,federal_income_tax_before_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,federal_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,free_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,head_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,head_medicaid_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_056,head_medicare_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,head_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,local_income_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,payroll_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,reduced_price_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,self_employment_tax,NJ,613.2224731445312,613.2224731445312,613.2224731445312,613.2224731445312,613.2224731445312,613.2224731445312,False,,,False +scenario_056,snap,NJ,1140.0,1140.0,95.0,95.0,95.0,1140.0,True,snap_abawd_hours_default_9261,keep_excluded,True +scenario_056,ssi,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,state_income_tax_before_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_056,state_refundable_credits,NJ,265.6000061035156,265.6000061035156,265.6000061035156,265.6000061035156,265.6000061035156,265.6000061035156,False,,,False +scenario_056,tanf,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,federal_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,federal_refundable_credits,LA,664.0,664.0,664.0,664.0,664.0,664.0,False,,,False +scenario_057,free_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,head_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,head_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_057,head_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,head_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,local_income_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,payroll_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,reduced_price_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,self_employment_tax,LA,1469.4732666015625,1469.4732666015625,1469.4732666015625,1469.4732666015625,1469.4732666015625,1469.4732666015625,False,,,False +scenario_057,snap,LA,2669.217041015625,2669.217041015625,2682.0,2628.0,2628.0,2669.217041015625,True,excl_snap_ssi_disability,keep_excluded,True +scenario_057,spouse_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,spouse_medicaid_eligible,LA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_057,spouse_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,spouse_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,ssi,LA,7231.00146484375,7231.00146484375,7231.00146484375,7231.00146484375,7231.00146484375,7231.00146484375,True,,,False +scenario_057,state_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_057,state_refundable_credits,LA,33.20000076293945,33.20000076293945,33.20000076293945,33.20000076293945,33.20000076293945,33.20000076293945,False,,,False +scenario_057,tanf,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,federal_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,head_medicare_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_059,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,payroll_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,spouse_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,spouse_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,spouse_medicare_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_059,spouse_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_059,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,federal_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,head_medicare_eligible,TX,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_060,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,payroll_tax,TX,229.5,229.5,229.5,229.5,229.5,229.5,False,,,False +scenario_060,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_060,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,federal_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_062,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,head_medicare_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_062,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,payroll_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_062,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_early_head_start_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_head_start_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,child1_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent1_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent1_medicaid_eligible,WI,1.0,1.0,1.0,1.0,1.0,1.0,True,,,False +scenario_064,dependent1_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent1_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent2_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent2_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_064,dependent2_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,dependent2_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,federal_income_tax_before_refundable_credits,WI,4439.28955078125,4439.28955078125,4439.28955078125,4439.28955078125,4439.28955078125,4439.28955078125,True,,,False +scenario_064,federal_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,free_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,head_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,head_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,head_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,head_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,local_income_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,payroll_tax,WI,7443.0673828125,7443.0673828125,7443.0673828125,7443.0673828125,7443.0673828125,7443.0673828125,False,,,False +scenario_064,reduced_price_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,self_employment_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,snap,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,spouse_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,spouse_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,spouse_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,spouse_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,ssi,WI,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_064,state_income_tax_before_refundable_credits,WI,4605.9970703125,4605.9970703125,4608.94775390625,4598.47607421875,4598.47607421875,4605.9970703125,True,excl_r01_ira_compensation,keep_excluded,True +scenario_064,state_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_064,tanf,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,federal_income_tax_before_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,federal_refundable_credits,VA,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,False,,,False +scenario_066,free_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,head_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,head_medicaid_eligible,VA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_066,head_medicare_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,head_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,local_income_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,payroll_tax,VA,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,39.779998779296875,False,,,False +scenario_066,reduced_price_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,self_employment_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,snap,VA,3596.039794921875,3576.0,0.0,0.0,3576.0,3576.0,False,snap_abawd_hours_default_9261,hold_output_scope,False +scenario_066,ssi,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,state_income_tax_before_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_066,state_refundable_credits,VA,7.955999851226807,7.955999851226807,7.955999851226807,7.955999851226807,7.955999851226807,7.955999851226807,False,,,False +scenario_066,tanf,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,dependent1_chip_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,dependent1_medicaid_eligible,IN,1.0,1.0,1.0,1.0,1.0,1.0,True,,,False +scenario_067,dependent1_medicare_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,dependent1_wic_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,federal_income_tax_before_refundable_credits,IN,5616.240234375,5616.240234375,5616.240234375,5616.240234375,5616.240234375,5616.240234375,False,,,False +scenario_067,federal_refundable_credits,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,free_school_meals_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,head_chip_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,head_medicaid_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,head_medicare_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,head_wic_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,local_income_tax,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,payroll_tax,IN,6300.5400390625,6300.5400390625,6300.5400390625,6300.5400390625,6300.5400390625,6300.5400390625,False,,,False +scenario_067,reduced_price_school_meals_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,self_employment_tax,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,snap,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,spouse_chip_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,spouse_medicaid_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,spouse_medicare_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,spouse_wic_eligible,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,ssi,IN,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_067,state_income_tax_before_refundable_credits,IN,2486.908935546875,2486.908935546875,2486.908935546875,2486.908935546875,2486.908935546875,2486.908935546875,False,,,False +scenario_067,state_refundable_credits,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_067,tanf,IN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,federal_income_tax_before_refundable_credits,MD,1910.0238037109375,1910.0238037109375,1910.0238037109375,1910.0238037109375,1910.0238037109375,1910.0238037109375,False,,,False +scenario_068,federal_refundable_credits,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,free_school_meals_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,head_chip_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,head_medicaid_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,head_medicare_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,head_wic_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,local_income_tax,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,payroll_tax,MD,2607.39013671875,2607.39013671875,2607.39013671875,2607.39013671875,2607.39013671875,2607.39013671875,False,,,False +scenario_068,reduced_price_school_meals_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,self_employment_tax,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,snap,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,ssi,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,state_income_tax_before_refundable_credits,MD,1252.9677734375,1255.3427734375,2085.71875,2089.60888671875,1255.3427734375,1255.3427734375,False,md_county_tax_8888,hold_output_scope,False +scenario_068,state_refundable_credits,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_068,tanf,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,federal_income_tax_before_refundable_credits,IL,1979.9605712890625,1979.9605712890625,1979.9605712890625,1979.9605712890625,1979.9605712890625,1979.9605712890625,False,,,False +scenario_070,federal_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,free_school_meals_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,head_chip_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,head_medicaid_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,head_medicare_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,head_wic_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,local_income_tax,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,payroll_tax,IL,2907.0,2907.0,2907.0,2907.0,2907.0,2907.0,False,,,False +scenario_070,reduced_price_school_meals_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,self_employment_tax,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,snap,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,ssi,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,state_income_tax_before_refundable_credits,IL,1650.561279296875,1650.561279296875,1650.561279296875,1650.561279296875,1650.561279296875,1650.561279296875,False,,,False +scenario_070,state_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_070,tanf,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,federal_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,federal_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,free_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,head_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,head_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,head_medicare_eligible,NY,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_071,head_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,local_income_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,payroll_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,reduced_price_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,self_employment_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,snap,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,ssi,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,state_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,state_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_071,tanf,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,federal_income_tax_before_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,federal_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,free_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,head_chip_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,head_medicaid_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,head_medicare_eligible,MI,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_072,head_wic_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,local_income_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,payroll_tax,MI,1530.0,1530.0,1530.0,1530.0,1530.0,1530.0,False,,,False +scenario_072,reduced_price_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,self_employment_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,snap,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,spouse_chip_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,spouse_medicaid_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,spouse_medicare_eligible,MI,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_072,spouse_wic_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,ssi,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,state_income_tax_before_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,state_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_072,tanf,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,federal_income_tax_before_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,federal_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,free_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,head_chip_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,head_medicaid_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,head_medicare_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_073,head_wic_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,local_income_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,payroll_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,reduced_price_school_meals_eligible,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,self_employment_tax,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,snap,MI,287.68316650390625,288.0,291.0,288.0,288.0,288.0,False,,,False +scenario_073,ssi,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,state_income_tax_before_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,state_refundable_credits,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_073,tanf,MI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,federal_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,federal_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,free_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,head_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,head_medicaid_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,head_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_074,head_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,local_income_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,payroll_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,reduced_price_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,self_employment_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,snap,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,ssi,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,state_income_tax_before_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,state_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_074,tanf,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,federal_income_tax_before_refundable_credits,FL,14777.1455078125,14777.1455078125,14777.1455078125,14777.1455078125,14777.1455078125,14777.1455078125,False,,,False +scenario_075,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,head_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,payroll_tax,FL,968.2120971679688,968.2120971679688,968.2120971679688,968.2120971679688,968.2120971679688,968.2120971679688,False,,,False +scenario_075,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_075,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_chip_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_early_head_start_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_head_start_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_medicaid_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_medicare_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child1_wic_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_chip_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_early_head_start_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_head_start_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_medicaid_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_medicare_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,child2_wic_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,federal_income_tax_before_refundable_credits,ID,19611.80078125,19611.80078125,19611.80078125,19611.80078125,19611.80078125,19611.80078125,False,,,False +scenario_076,federal_refundable_credits,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,free_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,head_chip_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,head_medicaid_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,head_medicare_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,head_wic_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,local_income_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,payroll_tax,ID,12240.0,12240.0,12240.0,12240.0,12240.0,12240.0,False,,,False +scenario_076,reduced_price_school_meals_eligible,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,self_employment_tax,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,snap,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,ssi,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_076,state_income_tax_before_refundable_credits,ID,6806.78662109375,6818.34423828125,6806.78662109375,6818.34423828125,6818.34423828125,6818.34423828125,False,,,False +scenario_076,state_refundable_credits,ID,465.0,465.0,465.0,465.0,465.0,465.0,False,,,False +scenario_076,tanf,ID,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,federal_income_tax_before_refundable_credits,LA,690.6175537109375,690.6175537109375,690.6175537109375,690.6175537109375,690.6175537109375,690.6175537109375,False,,,False +scenario_077,federal_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,free_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,head_chip_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,head_medicaid_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,head_medicare_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,head_wic_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,local_income_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,payroll_tax,LA,1989.472412109375,1989.472412109375,1989.472412109375,1989.472412109375,1989.472412109375,1989.472412109375,False,,,False +scenario_077,reduced_price_school_meals_eligible,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,self_employment_tax,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,snap,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,ssi,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,state_income_tax_before_refundable_credits,LA,305.13525390625,305.13525390625,305.13525390625,305.13525390625,305.13525390625,305.13525390625,False,,,False +scenario_077,state_refundable_credits,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_077,tanf,LA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,federal_income_tax_before_refundable_credits,MD,24772.693359375,24780.61328125,24164.45703125,24164.45703125,24164.45703125,24164.45703125,True,md_county_tax_8888,exclude_unlisted_input,True +scenario_078,federal_refundable_credits,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,free_school_meals_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,head_chip_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,head_medicaid_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,head_medicare_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,head_wic_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,local_income_tax,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,payroll_tax,MD,14317.322265625,14317.322265625,14317.322265625,14317.322265625,14317.322265625,14317.322265625,False,,,False +scenario_078,reduced_price_school_meals_eligible,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,self_employment_tax,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,snap,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,ssi,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,state_income_tax_before_refundable_credits,MD,6936.33740234375,6936.33740234375,11295.388671875,11295.388671875,6936.87646484375,6936.87646484375,False,md_county_tax_8888,hold_output_scope,False +scenario_078,state_refundable_credits,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_078,tanf,MD,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,federal_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,federal_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,free_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,head_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,head_medicaid_eligible,AZ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_079,head_medicare_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_079,head_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,local_income_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,payroll_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,reduced_price_school_meals_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,self_employment_tax,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,snap,AZ,2428.017333984375,2376.0,2433.0,2376.0,2376.0,2376.0,False,,,False +scenario_079,spouse_chip_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,spouse_medicaid_eligible,AZ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_079,spouse_medicare_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,spouse_wic_eligible,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,ssi,AZ,10572.0,10572.0,10572.0,10572.0,10572.0,10572.0,False,,,False +scenario_079,state_income_tax_before_refundable_credits,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_079,state_refundable_credits,AZ,50.0,50.0,50.0,50.0,50.0,50.0,False,,,False +scenario_079,tanf,AZ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,federal_income_tax_before_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,federal_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,free_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,head_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,head_medicaid_eligible,PA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_080,head_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,head_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,local_income_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,payroll_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,reduced_price_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,self_employment_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,snap,PA,3596.039794921875,3596.039794921875,3600.0,3576.0,3576.0,3596.039794921875,True,excl_r30_snap_heat_and_eat_sua,keep_excluded,True +scenario_080,ssi,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,state_income_tax_before_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,state_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_080,tanf,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,federal_income_tax_before_refundable_credits,MA,26932.2734375,26932.2734375,26932.2734375,26932.2734375,26932.2734375,26932.2734375,False,,,False +scenario_081,federal_refundable_credits,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,free_school_meals_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,head_chip_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,head_medicaid_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,head_medicare_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,head_wic_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,local_income_tax,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,payroll_tax,MA,14192.662109375,14192.662109375,14192.662109375,14192.662109375,14192.662109375,14192.662109375,False,,,False +scenario_081,reduced_price_school_meals_eligible,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,self_employment_tax,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,snap,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,ssi,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,state_income_tax_before_refundable_credits,MA,8238.40625,8238.40625,8238.40625,8238.40625,8238.40625,8238.40625,True,,,False +scenario_081,state_refundable_credits,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_081,tanf,MA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_early_head_start_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_head_start_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_medicare_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,child1_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,federal_income_tax_before_refundable_credits,NY,9563.052734375,9563.052734375,9577.90234375,9577.90234375,9577.90234375,9563.052734375,True,excl_r02_ira_219g_federal,keep_excluded,True +scenario_082,federal_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,free_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,head_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,head_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,head_medicare_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,head_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,local_income_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,payroll_tax,NY,8108.02734375,8108.02734375,8108.02734375,8108.02734375,8108.02734375,8108.02734375,False,,,False +scenario_082,reduced_price_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,self_employment_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,snap,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,ssi,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_082,state_income_tax_before_refundable_credits,NY,5598.55224609375,5598.55224609375,5598.55224609375,5598.55224609375,5598.55224609375,5598.55224609375,False,,,False +scenario_082,state_refundable_credits,NY,650.5,650.5,667.0,667.0,667.0,667.0,False,ny_ctc_phaseout_rounding_9425,adopt_latest,True +scenario_082,tanf,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,federal_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,payroll_tax,TX,539.3250122070312,539.3250122070312,539.3250122070312,539.3250122070312,539.3250122070312,539.3250122070312,False,,,False +scenario_083,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_083,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,federal_income_tax_before_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,federal_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_084,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,payroll_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,self_employment_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,state_income_tax_before_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_084,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,federal_income_tax_before_refundable_credits,PA,2322.760009765625,2322.760009765625,2322.760009765625,2322.760009765625,2322.760009765625,2322.760009765625,False,,,False +scenario_085,federal_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,free_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,head_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,head_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,head_medicare_eligible,PA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_085,head_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,local_income_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,payroll_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,reduced_price_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,self_employment_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,snap,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,ssi,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,state_income_tax_before_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,state_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_085,tanf,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_chip_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_early_head_start_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_head_start_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_medicaid_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_medicare_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,child1_wic_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,federal_income_tax_before_refundable_credits,GA,3255.5478515625,3255.5478515625,3255.5478515625,3255.5478515625,3255.5478515625,3255.5478515625,False,,,False +scenario_086,federal_refundable_credits,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,free_school_meals_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,head_chip_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,head_medicaid_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,head_medicare_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,head_wic_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,local_income_tax,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,payroll_tax,GA,4743.0,4743.0,4743.0,4743.0,4743.0,4743.0,False,,,False +scenario_086,reduced_price_school_meals_eligible,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,self_employment_tax,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,snap,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,ssi,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,state_income_tax_before_refundable_credits,GA,2078.958740234375,2078.958740234375,2078.958740234375,2078.958740234375,2078.958740234375,2078.958740234375,False,,,False +scenario_086,state_refundable_credits,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_086,tanf,GA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,federal_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,payroll_tax,TX,1421.1448974609375,1421.1448974609375,1421.1448974609375,1421.1448974609375,1421.1448974609375,1421.1448974609375,False,,,False +scenario_088,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_088,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,federal_income_tax_before_refundable_credits,NC,6819.7119140625,6819.7119140625,6819.7119140625,6819.7119140625,6819.7119140625,6819.7119140625,False,,,False +scenario_089,federal_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,payroll_tax,NC,5039.91015625,5039.91015625,5039.91015625,5039.91015625,5039.91015625,5039.91015625,False,,,False +scenario_089,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,self_employment_tax,NC,4286.24072265625,4286.24072265625,4286.24072265625,4286.24072265625,4286.24072265625,4286.24072265625,False,,,False +scenario_089,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,spouse_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,spouse_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,spouse_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,spouse_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,state_income_tax_before_refundable_credits,NC,3219.935302734375,3219.935302734375,3219.935302734375,3219.935302734375,3219.935302734375,3219.935302734375,False,,,False +scenario_089,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_089,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,federal_income_tax_before_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,federal_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,free_school_meals_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,head_chip_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,head_medicaid_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,head_medicare_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,head_wic_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,local_income_tax,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,payroll_tax,KS,535.5,535.5,535.5,535.5,535.5,535.5,False,,,False +scenario_090,reduced_price_school_meals_eligible,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,self_employment_tax,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,snap,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,ssi,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,state_income_tax_before_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,state_refundable_credits,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_090,tanf,KS,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,federal_income_tax_before_refundable_credits,WI,1349.998779296875,1349.998779296875,1349.998779296875,1349.998779296875,1349.998779296875,1349.998779296875,False,,,False +scenario_091,federal_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,free_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,head_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,head_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,head_medicare_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,head_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,local_income_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,payroll_tax,WI,2065.5,2065.5,2065.5,2065.5,2065.5,2065.5,False,,,False +scenario_091,reduced_price_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,self_employment_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,snap,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,ssi,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,state_income_tax_before_refundable_credits,WI,843.6614990234375,843.6614990234375,900.45947265625,895.8131103515625,895.8131103515625,843.6614990234375,True,excl_r32_wi_capital_gain_distributions,keep_excluded,True +scenario_091,state_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_091,tanf,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,federal_income_tax_before_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,federal_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,free_school_meals_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,head_chip_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,head_medicaid_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,head_medicare_eligible,AL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_092,head_wic_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,local_income_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,payroll_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,reduced_price_school_meals_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,self_employment_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,snap,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,ssi,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,state_income_tax_before_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,state_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_092,tanf,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,dependent1_chip_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,dependent1_medicaid_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,dependent1_medicare_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,dependent1_wic_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,federal_income_tax_before_refundable_credits,MO,8628.990234375,8628.990234375,8628.990234375,8628.990234375,8628.990234375,8628.990234375,False,,,False +scenario_093,federal_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,free_school_meals_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,head_chip_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,head_medicaid_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,head_medicare_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,head_wic_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,local_income_tax,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,payroll_tax,MO,12469.5,12469.5,12469.5,12469.5,12469.5,12469.5,False,,,False +scenario_093,reduced_price_school_meals_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,self_employment_tax,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,snap,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,spouse_chip_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,spouse_medicaid_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,spouse_medicare_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,spouse_wic_eligible,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,ssi,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,state_income_tax_before_refundable_credits,MO,3389.65087890625,3388.24560546875,3389.65087890625,3388.24560546875,3388.24560546875,3388.24560546875,False,,,False +scenario_093,state_refundable_credits,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_093,tanf,MO,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,federal_income_tax_before_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,federal_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,free_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,head_chip_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,head_medicaid_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,head_medicare_eligible,NJ,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_095,head_wic_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,local_income_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,payroll_tax,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,reduced_price_school_meals_eligible,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,self_employment_tax,NJ,593.2291259765625,593.2291259765625,593.2291259765625,593.2291259765625,593.2291259765625,593.2291259765625,False,,,False +scenario_095,snap,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,ssi,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,state_income_tax_before_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,state_refundable_credits,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_095,tanf,NJ,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,federal_income_tax_before_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,federal_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,free_school_meals_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,head_chip_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,head_medicaid_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,head_medicare_eligible,IL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_098,head_wic_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,local_income_tax,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,payroll_tax,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,reduced_price_school_meals_eligible,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,self_employment_tax,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,snap,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,ssi,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,state_income_tax_before_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,state_refundable_credits,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_098,tanf,IL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_early_head_start_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_head_start_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child1_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_early_head_start_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_head_start_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,child2_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,federal_income_tax_before_refundable_credits,CA,10679.75,10679.75,10679.75,10679.75,10679.75,10679.75,True,,,False +scenario_099,federal_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,free_school_meals_eligible,CA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_099,head_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,head_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,head_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,head_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,local_income_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,payroll_tax,CA,14767.5,14767.5,14767.5,14767.5,14767.5,14767.5,False,,,False +scenario_099,reduced_price_school_meals_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,self_employment_tax,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,snap,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,spouse_chip_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,spouse_medicaid_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,spouse_medicare_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,spouse_wic_eligible,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,ssi,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,state_income_tax_before_refundable_credits,CA,4493.7431640625,4493.7431640625,4493.7431640625,4640.77734375,4640.77734375,4493.7431640625,True,excl_r11_ca_itemized_conformity,keep_excluded,True +scenario_099,state_refundable_credits,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_099,tanf,CA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child1_chip_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child1_early_head_start_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child1_head_start_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child1_medicaid_eligible,MT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_100,child1_medicare_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child1_wic_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child2_chip_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child2_early_head_start_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child2_head_start_eligible,MT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_100,child2_medicaid_eligible,MT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_100,child2_medicare_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,child2_wic_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,federal_income_tax_before_refundable_credits,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,federal_refundable_credits,MT,2878.0966796875,2878.0966796875,2878.0966796875,2878.0966796875,2878.0966796875,2878.0966796875,True,,,False +scenario_100,free_school_meals_eligible,MT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_100,head_chip_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,head_medicaid_eligible,MT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_100,head_medicare_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,head_wic_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,local_income_tax,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,payroll_tax,MT,452.47613525390625,452.47613525390625,452.47613525390625,452.47613525390625,452.47613525390625,452.47613525390625,False,,,False +scenario_100,reduced_price_school_meals_eligible,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,self_employment_tax,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,snap,MT,8625.8896484375,8625.8896484375,8637.0,8556.0,8556.0,8625.8896484375,True,excl_snap_ssi_disability,keep_excluded,True +scenario_100,ssi,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,state_income_tax_before_refundable_credits,MT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_100,state_refundable_credits,MT,473.1777038574219,473.1777038574219,473.1777038574219,473.1777038574219,473.1777038574219,473.1777038574219,True,,,False +scenario_100,tanf,MT,6064.958984375,6064.958984375,6064.958984375,6064.958984375,6064.958984375,6064.958984375,False,,,False +scenario_101,federal_income_tax_before_refundable_credits,TX,13418.599609375,13418.599609375,13418.599609375,13418.599609375,13418.599609375,13418.599609375,False,,,False +scenario_101,federal_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,payroll_tax,TX,7879.5,7879.5,7879.5,7879.5,7879.5,7879.5,False,,,False +scenario_101,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,snap,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_101,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,federal_income_tax_before_refundable_credits,NC,6104.09619140625,6104.09619140625,6104.09619140625,6104.09619140625,6104.09619140625,6104.09619140625,False,,,False +scenario_102,federal_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,free_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,head_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,head_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,head_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,head_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,local_income_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,payroll_tax,NC,6670.7998046875,6670.7998046875,6670.7998046875,6670.7998046875,6670.7998046875,6670.7998046875,False,,,False +scenario_102,reduced_price_school_meals_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,self_employment_tax,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,snap,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,spouse_chip_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,spouse_medicaid_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,spouse_medicare_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,spouse_wic_eligible,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,ssi,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,state_income_tax_before_refundable_credits,NC,2461.869873046875,2461.869873046875,2461.869873046875,2461.869873046875,2461.869873046875,2461.869873046875,False,,,False +scenario_102,state_refundable_credits,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_102,tanf,NC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,federal_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,federal_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,free_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,head_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,head_medicaid_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,head_medicare_eligible,NY,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_104,head_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,local_income_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,payroll_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,reduced_price_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,self_employment_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,snap,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,ssi,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,state_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,state_refundable_credits,NY,375.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_104,tanf,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,federal_income_tax_before_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_107,federal_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,free_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,head_chip_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,head_medicaid_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,head_medicare_eligible,OH,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_107,head_wic_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,local_income_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,payroll_tax,OH,486.9607238769531,486.9607238769531,486.9607238769531,486.9607238769531,486.9607238769531,486.9607238769531,False,,,False +scenario_107,reduced_price_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,self_employment_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,snap,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,ssi,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,state_income_tax_before_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,state_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_107,tanf,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,federal_income_tax_before_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,federal_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,free_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,head_chip_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,head_medicaid_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,head_medicare_eligible,WI,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_108,head_wic_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,local_income_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,payroll_tax,WI,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,585.6839599609375,False,,,False +scenario_108,reduced_price_school_meals_eligible,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,self_employment_tax,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,snap,WI,287.68316650390625,288.0,291.0,288.0,288.0,288.0,False,,,False +scenario_108,ssi,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,state_income_tax_before_refundable_credits,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_108,state_refundable_credits,WI,271.3916015625,271.3916015625,271.3916015625,271.3916015625,271.3916015625,271.3916015625,True,,,False +scenario_108,tanf,WI,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child1_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child1_early_head_start_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child1_head_start_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child1_medicaid_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,child1_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child1_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child2_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child2_early_head_start_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child2_head_start_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child2_medicaid_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,child2_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child2_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child3_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child3_early_head_start_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,child3_head_start_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child3_medicaid_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,child3_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,child3_wic_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,federal_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,federal_refundable_credits,FL,12038.0859375,12038.0859375,12038.0859375,12038.0859375,12038.0859375,12038.0859375,False,,,False +scenario_109,free_school_meals_eligible,FL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_109,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,head_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,payroll_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,self_employment_tax,FL,4238.86474609375,4238.86474609375,4238.86474609375,4238.86474609375,4238.86474609375,4238.86474609375,False,,,False +scenario_109,snap,FL,8020.55419921875,7932.0,8040.0,7932.0,7932.0,7932.0,False,,,False +scenario_109,spouse_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,spouse_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,spouse_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,spouse_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_109,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,federal_income_tax_before_refundable_credits,OH,23897.443359375,23897.443359375,23897.443359375,23897.443359375,23897.443359375,23897.443359375,True,,,False +scenario_110,federal_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,free_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,head_chip_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,head_medicaid_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,head_medicare_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,head_wic_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,local_income_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,payroll_tax,OH,7650.0,7650.0,7650.0,7650.0,7650.0,7650.0,False,,,False +scenario_110,reduced_price_school_meals_eligible,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,self_employment_tax,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,snap,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,ssi,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,state_income_tax_before_refundable_credits,OH,4057.47216796875,4057.47216796875,4057.47216796875,4057.47216796875,4057.47216796875,4057.47216796875,True,,,False +scenario_110,state_refundable_credits,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_110,tanf,OH,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,federal_income_tax_before_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,federal_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,free_school_meals_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,head_chip_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,head_medicaid_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,head_medicare_eligible,WA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_111,head_wic_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,local_income_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,payroll_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,reduced_price_school_meals_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,self_employment_tax,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,snap,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,spouse_chip_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,spouse_medicaid_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,spouse_medicare_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,spouse_wic_eligible,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,ssi,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,state_income_tax_before_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,state_refundable_credits,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_111,tanf,WA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,federal_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,federal_refundable_credits,TX,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,False,,,False +scenario_112,free_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,head_chip_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,head_medicaid_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,head_medicare_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,head_wic_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,local_income_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,payroll_tax,TX,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,453.0960083007813,False,,,False +scenario_112,reduced_price_school_meals_eligible,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,self_employment_tax,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,snap,TX,287.68316650390625,287.68316650390625,0.0,0.0,0.0,287.68316650390625,True,snap_abawd_hours_default_9261,keep_excluded,True +scenario_112,ssi,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,state_income_tax_before_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,state_refundable_credits,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_112,tanf,TX,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,federal_income_tax_before_refundable_credits,VA,10729.6123046875,10729.6123046875,10729.6123046875,10729.6123046875,10729.6123046875,10729.6123046875,False,,,False +scenario_114,federal_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,free_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,head_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,head_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,head_medicare_eligible,VA,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_114,head_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,local_income_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,payroll_tax,VA,1736.550048828125,1736.550048828125,1736.550048828125,1736.550048828125,1736.550048828125,1736.550048828125,False,,,False +scenario_114,reduced_price_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,self_employment_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,snap,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,ssi,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,state_income_tax_before_refundable_credits,VA,3514.14501953125,3514.14501953125,3514.14501953125,3514.14501953125,3514.14501953125,3514.14501953125,False,,,False +scenario_114,state_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_114,tanf,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,federal_income_tax_before_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,federal_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,free_school_meals_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,head_chip_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,head_medicaid_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,head_medicare_eligible,AL,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_115,head_wic_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,local_income_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,payroll_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,reduced_price_school_meals_eligible,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,self_employment_tax,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,snap,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,ssi,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,state_income_tax_before_refundable_credits,AL,4.679999828338623,4.679999828338623,4.679999828338623,4.679999828338623,4.679999828338623,4.679999828338623,False,,,False +scenario_115,state_refundable_credits,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_115,tanf,AL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,federal_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,federal_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,free_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,head_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,head_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,head_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,head_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,local_income_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,payroll_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,reduced_price_school_meals_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,self_employment_tax,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,snap,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,spouse_chip_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,spouse_medicaid_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,spouse_medicare_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,spouse_wic_eligible,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,ssi,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,state_income_tax_before_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,state_refundable_credits,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_116,tanf,FL,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_chip_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_early_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_medicaid_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_medicare_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child1_wic_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_chip_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_early_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_medicaid_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_medicare_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child2_wic_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_chip_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_early_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_head_start_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_medicaid_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_medicare_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,child3_wic_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,federal_income_tax_before_refundable_credits,AR,24391.796875,24391.796875,24961.33984375,24961.33984375,24961.33984375,24961.33984375,True,salt_refund_gross_income_9122,exclude_unlisted_input,True +scenario_117,federal_refundable_credits,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,free_school_meals_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,head_chip_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,head_medicaid_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,head_medicare_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,head_wic_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,local_income_tax,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,payroll_tax,AR,15044.787109375,15044.787109375,15044.787109375,15044.787109375,15044.787109375,15044.787109375,False,,,False +scenario_117,reduced_price_school_meals_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,self_employment_tax,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,snap,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,spouse_chip_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,spouse_medicaid_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,spouse_medicare_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,spouse_wic_eligible,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,ssi,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,state_income_tax_before_refundable_credits,AR,7984.05419921875,7984.05419921875,7984.53369140625,7984.53369140625,7984.53369140625,7984.53369140625,False,,,False +scenario_117,state_refundable_credits,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_117,tanf,AR,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,federal_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,federal_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,free_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,head_chip_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,head_medicaid_eligible,NY,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_118,head_medicare_eligible,NY,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_118,head_wic_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,local_income_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,payroll_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,reduced_price_school_meals_eligible,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,self_employment_tax,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,snap,NY,2903.9404296875,2903.9404296875,2904.0,2868.0,2868.0,2903.9404296875,True,excl_snap_mortgage_residence,keep_excluded,True +scenario_118,ssi,NY,9368.0,9368.0,9368.0,9368.0,9368.0,9368.0,False,,,False +scenario_118,state_income_tax_before_refundable_credits,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_118,state_refundable_credits,NY,375.0,375.0,375.0,375.0,375.0,375.0,False,,,False +scenario_118,tanf,NY,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_early_head_start_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_head_start_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_medicare_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child1_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_early_head_start_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_head_start_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_medicare_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,child2_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,federal_income_tax_before_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,federal_refundable_credits,VA,2854.0087890625,2854.0087890625,2854.0087890625,2854.0087890625,2854.0087890625,2854.0087890625,True,,,False +scenario_119,free_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,head_chip_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,head_medicaid_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,head_medicare_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,head_wic_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,local_income_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,payroll_tax,VA,4207.5,4207.5,4207.5,4207.5,4207.5,4207.5,False,,,False +scenario_119,reduced_price_school_meals_eligible,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,self_employment_tax,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,snap,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,ssi,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,state_income_tax_before_refundable_credits,VA,1859.0943603515625,1859.0943603515625,1859.0943603515625,1859.0943603515625,1859.0943603515625,1859.0943603515625,True,,,False +scenario_119,state_refundable_credits,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_119,tanf,VA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,federal_income_tax_before_refundable_credits,CT,40021.81640625,40021.81640625,40416.66796875,40416.66796875,40416.66796875,40021.81640625,True,excl_r02_ira_219g_federal,keep_excluded,True +scenario_120,federal_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,free_school_meals_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,head_chip_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,head_medicaid_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,head_medicare_eligible,CT,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_120,head_wic_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,local_income_tax,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,payroll_tax,CT,13496.1552734375,13496.1552734375,13496.1552734375,13496.1552734375,13496.1552734375,13496.1552734375,False,,,False +scenario_120,reduced_price_school_meals_eligible,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,self_employment_tax,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,snap,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,ssi,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,state_income_tax_before_refundable_credits,CT,11917.18359375,11917.18359375,11917.18359375,11917.18359375,11917.18359375,11917.18359375,True,,,False +scenario_120,state_refundable_credits,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_120,tanf,CT,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,federal_income_tax_before_refundable_credits,SC,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_121,federal_refundable_credits,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,free_school_meals_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,head_chip_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,head_medicaid_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,head_medicare_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,head_wic_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,local_income_tax,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,payroll_tax,SC,1963.3724365234375,1963.3724365234375,1963.3724365234375,1963.3724365234375,1963.3724365234375,1963.3724365234375,False,,,False +scenario_121,reduced_price_school_meals_eligible,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,self_employment_tax,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,snap,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,ssi,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,state_income_tax_before_refundable_credits,SC,0.0,0.0,0.0,0.0,0.0,0.0,True,,,False +scenario_121,state_refundable_credits,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_121,tanf,SC,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,federal_income_tax_before_refundable_credits,MN,4746.66162109375,4746.66162109375,4746.66162109375,4746.66162109375,4746.66162109375,4746.66162109375,False,,,False +scenario_122,federal_refundable_credits,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,free_school_meals_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,head_chip_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,head_medicaid_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,head_medicare_eligible,MN,1.0,1.0,1.0,1.0,1.0,1.0,False,,,False +scenario_122,head_wic_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,local_income_tax,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,payroll_tax,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,reduced_price_school_meals_eligible,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,self_employment_tax,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,snap,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,ssi,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,state_income_tax_before_refundable_credits,MN,1028.216552734375,1025.54150390625,1028.216552734375,1025.54150390625,1025.54150390625,1025.54150390625,False,,,False +scenario_122,state_refundable_credits,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_122,tanf,MN,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_early_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_head_start_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,child1_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,federal_income_tax_before_refundable_credits,PA,5200.240234375,5200.240234375,5200.240234375,5200.240234375,5200.240234375,5200.240234375,False,,,False +scenario_123,federal_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,free_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,head_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,head_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,head_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,head_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,local_income_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,payroll_tax,PA,11194.0,11194.0,11194.0,11194.0,11194.0,11194.0,False,,,False +scenario_123,reduced_price_school_meals_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,self_employment_tax,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,snap,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,spouse_chip_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,spouse_medicaid_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,spouse_medicare_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,spouse_wic_eligible,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,ssi,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,state_income_tax_before_refundable_credits,PA,3070.061279296875,3070.061279296875,3070.061279296875,3070.061279296875,3070.061279296875,3070.061279296875,False,,,False +scenario_123,state_refundable_credits,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False +scenario_123,tanf,PA,0.0,0.0,0.0,0.0,0.0,0.0,False,,,False diff --git a/reference_audit/2026-09-28/verification/compose_result.json b/reference_audit/2026-09-28/verification/compose_result.json new file mode 100644 index 00000000..09f86032 --- /dev/null +++ b/reference_audit/2026-09-28/verification/compose_result.json @@ -0,0 +1,410 @@ +{ + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_conventions.py", + "sweep_csv": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/out/latest_conventions.csv", + "moved": 29, + "clusters": [ + { + "id": "md_county_tax_8888", + "title": "Maryland county income tax wired into the state output and SALT (scored)", + "hypothesis": "Upstream #8888 (6b0bca0b9f, 2026-07-05) did two things. It added md_local_income_tax_before_refundable_credits to the list behind state_income_tax_before_refundable_credits. It also put county withholding into md_withheld_income_tax, which feeds federal SALT. The county is unlisted, so 2.15.17 uses Allegany's 3.03% rate. That adds 834.27 to 068 state and 4,358.51 to 078 state. For 078 federal it pushes SALT to the $40,400 cap (itemized SALT 36,233.77 on 1.755.4 vs 40,400 on 2.15.17). 078 federal also carries #9122: the listed salt_refund_income of 1,631.91 enters AGI (198,505 -> 200,136.91). The state port found that removing both upstream changes reproduces the board exactly. The prompt excludes local tax from the state output. latest_md_local_output_scope.py, the candidate the lead has to rule on and which I left out of the composed module, gives 068 state exactly and 078 state within $1 (+0.54 through #9122). Two decisions settle this cluster: the output-scope adapter, and whether county tax belongs in federal SALT when the county is unlisted. 078 federal also depends on the #9122 ruling (cluster salt_refund_gross_income_9122).", + "outputs": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "state": "MD", + "board": 1255.342773, + "latest": 2089.608887, + "v11_1755": 1252.967773, + "excluded": false, + "exclusion_root_cause": "" + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "state": "MD", + "board": 6936.337402, + "latest": 11295.388672, + "v11_1755": 6936.337402, + "excluded": false, + "exclusion_root_cause": "" + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "state": "MD", + "board": 24780.613281, + "latest": 24164.457031, + "v11_1755": 24772.693359, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "salt_refund_gross_income_9122", + "title": "Listed SALT refund now in federal gross income (scored)", + "hypothesis": "Upstream #9122 (316e7832a1, 2026-09-08) added salt_refund_income, a listed input with no formula in 2.15.17, to gov.irs.gross_income.sources. The probe shows AGI rising by exactly the listed refund: 033 SD +243, 117 AR +2,588.82. The rule applied is IRC 111 / Schedule 1 line 1, which is pre-freeze law, so the default is to adopt. The investigator should settle whether including the whole refund is right under the IRC 111 tax-benefit rule, since prior-year itemization is not stated, and whether that ambiguity makes these exclusion candidates. 033 is in SD, which has no income tax. The same change moves 078 MD federal (cluster md_county_tax_8888) and the excluded r02 federal outputs 005, 082 and 120 (cluster excl_r02_ira_219g_federal). One ruling covers all six.", + "outputs": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "state": "SD", + "board": 3788.988037, + "latest": 3818.148193, + "v11_1755": 3788.988037, + "excluded": false, + "exclusion_root_cause": "" + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "state": "AR", + "board": 24391.796875, + "latest": 24961.339844, + "v11_1755": 24391.796875, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "snap_abawd_hours_default_9261", + "title": "SNAP work rules read weekly_hours_worked_before_lsr, default now 0 (scored 066, excluded 112 and 056)", + "hypothesis": "Upstream #9261 (82745ca239, merged a112cc5a0a on 2026-08-12) changed the default of weekly_hours_worked_before_lsr from 40 to 0, and the 2.15.17 SNAP ABAWD and general work rules read it. 066 VA is scored: the prompt states 40 hours, but the builder maps that to hours_worked_last_week, so the ABAWD test fails in every month and SNAP is 0. The SNAP port's probe maps stated hours into weekly_hours_worked_before_lsr and gets back the board's 3,576 without moving any other output. 112 TX (excluded, hours unlisted): 0 is the exclusion's alternative reading. 056 NJ (excluded, hours plus mortgage residence): 95 comes from 0 hours with a January-only NJ waiver, so the exclusion's alternative_value of 0.0, computed on 1.755.4, is stale. One decision settles all three: whether to add the stated-hours mapping to the situation builder, and whether to refresh the alternative values of the two exclusions.", + "outputs": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "state": "VA", + "board": 3576, + "latest": 0, + "v11_1755": 3596.039795, + "excluded": false, + "exclusion_root_cause": "" + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "state": "TX", + "board": 287.683167, + "latest": 0, + "v11_1755": 287.683167, + "excluded": true, + "exclusion_root_cause": "unlisted:weekly_hours_worked_before_lsr" + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "state": "NJ", + "board": 1140, + "latest": 95, + "v11_1755": 1140, + "excluded": true, + "exclusion_root_cause": "unlisted:weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies" + } + ] + }, + { + "id": "nj_ctc_fy2027_budget", + "title": "New Jersey child tax credit raised 25% for TY2026-2028 (scored)", + "hypothesis": "nj_ctc goes from 2,000 to 2,500 (two children under 6, $1,000 to $1,250 each). Upstream 0319635b6e (2026-07-08, 'Update New Jersey for the enacted FY2027 budget') encodes P.L.2026 c.26 (S-4531) in gov.states.nj.tax.income.credits.ctc.amount: +25% for tax years 2026-2028, reverting in 2029. The commit message and the parameter reference say it was enacted 2026-06-30. If the enactment and publication date holds (before the 2026-07-03 freeze), adopt 5,842.40. The investigator should confirm the signing and chaptering date from NJ Legislature records.", + "outputs": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "state": "NJ", + "board": 5342.399902, + "latest": 5842.399902, + "v11_1755": 5342.399902, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "ny_ctc_phaseout_rounding_9425", + "title": "NY Empire State child credit phase-out increment (scored)", + "hypothesis": "ny_ctc goes from 290.50 to 307.00, +16.50, which is one $16.50 phase-out step. The only 1.755.4..2.15.17 commits in NY ctc parameters and variables with a formula effect are d8f06f394c (#9425, 2026-09-10): the surviving-spouse threshold moves to $75,000 under S.3009-C, and a review change rounds phase-out increments down. The filer is a 23-year-old single parent with a child aged 1, not a surviving spouse, so the likely cause is the increment-rounding change. That rounding should be checked against Tax Law 606(c-1). A minor possibility is #9122, which added a NY subtraction for SALT refunds while federal AGI rose 67.50.", + "outputs": [ + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "state": "NY", + "board": 650.5, + "latest": 667, + "v11_1755": 650.5, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "school_meals_child_support_income", + "title": "Child support now counted in school-meal income (scored)", + "hypothesis": "Upstream fe1cb9ab45 (2026-08-14) added child_support_received to gov.usda.school_meals.income.sources, with 7 CFR 245.6(a)(5)(ii) cited in 2.15.17. The head's listed child_support_received of $1,267 raises school_meal_countable_income from 60,010 to 61,277. The FPG ratio goes from 1.818 to 1.857 and crosses the 185% reduced-price limit, so eligibility flips from 1 to 0 (school_meal_tier 1 -> 2). The rule is longstanding, so the likely call is to adopt it as an upstream correction. The investigator should confirm the regulation counts child support received.", + "outputs": [ + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "state": "PA", + "board": 1, + "latest": 0, + "v11_1755": 1, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "az_snap_bbce_200", + "title": "Arizona SNAP BBCE gross limit 200% FPL from March 2026 (scored)", + "hypothesis": "Upstream 0881eff4b8 (2026-08-06) raised AZ gross.yaml and gross_hheod.yaml from 185% to 200% FPL from 2026-03-01, quoting the CNAP manual. Gross income of $2,539.33 fails 185% but passes 200%, so BBCE waives the net and asset tests from March to December and the household gets the $24 minimum x 10 = 240. Raw 2.15.17 gives 243, because the FY2027 +$1 applies in Oct-Dec, which the hold removes. The rule is effective before the freeze. The investigator should verify DES's publication date and whether 200% applies to the elderly/disabled standard, which is upstream's reading. The manual sits behind Cloudflare.", + "outputs": [ + { + "scenario_id": "scenario_013", + "variable": "snap", + "state": "AZ", + "board": 0, + "latest": 240, + "v11_1755": 0, + "excluded": false, + "exclusion_root_cause": "" + } + ] + }, + { + "id": "excl_r02_ira_219g_federal", + "title": "Excluded r02_ira_219g federal outputs moved by #9122", + "hypothesis": "Exclusion root cause r02_ira_219g. Raw 2.15.17 already equals latest, so no convention moves these. The movement is #9122: AGI rises by exactly the listed salt_refund_income (005 +4,147.69, 082 +67.50, 120 +1,750.24). Two checks settle the cluster: is r02 (IRA 219(g) deduction limit) still unfixed in 2.15.17 (if so the exclusions stay), and do the alternative values need recomputing on 2.15.17? Scoring is not affected while they stay excluded.", + "outputs": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "state": "CA", + "board": 106505.898438, + "latest": 107833.15625, + "v11_1755": 106505.898438, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g" + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "state": "NY", + "board": 9563.052734, + "latest": 9577.902344, + "v11_1755": 9563.052734, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g" + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "state": "CT", + "board": 40021.816406, + "latest": 40416.667969, + "v11_1755": 40021.816406, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g" + } + ] + }, + { + "id": "excl_r11_ca_itemized_conformity", + "title": "Excluded CA state outputs (r11_ca_itemized_conformity, +r02) moved by c_ca_hold_2025", + "hypothesis": "Exclusion root cause r11_ca_itemized_conformity (005 and 099 also r02). The board keeps the untouched v1.1 values for excluded outputs. Raw 2.15.17 equals the board for 022 and 099, so c_ca_hold_2025 alone moves them. The state port reports these equal the 1.755.4 + r19 values (2,505.87 and 4,640.78). 005 state also moves upstream: raw 40,986.87, then +280 from the convention. The state port attributes the 005 move to #9122 (CA subtracts SALT refunds) and to the r04/r32 changes. The investigator should confirm r11 (and r02) are still unfixed in 2.15.17, so the exclusions stay. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 41051.511719, + "latest": 41267.011719, + "v11_1755": 41051.511719, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g+r11_ca_itemized_conformity" + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 2439.650146, + "latest": 2505.870117, + "v11_1755": 2439.650146, + "excluded": true, + "exclusion_root_cause": "r11_ca_itemized_conformity" + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "state": "CA", + "board": 4493.743164, + "latest": 4640.777344, + "v11_1755": 4493.743164, + "excluded": true, + "exclusion_root_cause": "r02_ira_219g+r11_ca_itemized_conformity" + } + ] + }, + { + "id": "excl_r07_idaho_health_premiums", + "title": "Excluded ID state outputs (r07_idaho_health_premiums) moved by c_id_hold_2025", + "hypothesis": "Exclusion root cause r07_idaho_health_premiums. Raw 2.15.17 equals the board (755.78 and 2,435.28). The +5.78 comes only from c_id_hold_2025, which holds the $4,811/$9,622 zero-rate thresholds, and the result equals 1.755.4 + r19 (761.56 and 2,441.06). The board keeps v1.1 for excluded outputs. The investigator should confirm r07 is still unfixed in 2.15.17. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "state": "ID", + "board": 755.77832, + "latest": 761.557007, + "v11_1755": 755.77832, + "excluded": true, + "exclusion_root_cause": "r07_idaho_health_premiums" + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "state": "ID", + "board": 2435.27832, + "latest": 2441.057129, + "v11_1755": 2435.27832, + "excluded": true, + "exclusion_root_cause": "r07_idaho_health_premiums" + } + ] + }, + { + "id": "excl_r32_wi_capital_gain_distributions", + "title": "Excluded WI outputs (r32_wi_capital_gain_distributions, 042 also r06) moved mostly upstream", + "hypothesis": "Exclusion root cause r32_wi_capital_gain_distributions (042 also r06_wi_act15_before_refundable). Both households list non_sch_d_capital_gains (3,753 and 1,170). Most of the move is already in raw 2.15.17 (042 468.83, 091 900.46). That fits r04 upstream (#8839, which puts capital gain distributions in gross income and the preferential base) flowing into WI. c_wi_published_2026 then subtracts 4.65. The state port reports the values differ from 1.755.4 + r19 because of #9122 and r04/r32. The investigator should check whether r32 (and r06) are fixed or partly fixed in 2.15.17 and whether the exclusions still hold. The exclusion alternatives are 0.0 for 042 and 878.52 for 091.", + "outputs": [ + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 284.740906, + "latest": 464.1828, + "v11_1755": 284.740906, + "excluded": true, + "exclusion_root_cause": "r06_wi_act15_before_refundable+r32_wi_capital_gain_distributions" + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 843.661499, + "latest": 895.81311, + "v11_1755": 843.661499, + "excluded": true, + "exclusion_root_cause": "r32_wi_capital_gain_distributions" + } + ] + }, + { + "id": "excl_r01_ira_compensation", + "title": "Excluded WI output (r01_ira_compensation) moved by c_wi_published_2026", + "hypothesis": "Exclusion root cause r01_ira_compensation. Raw 2.15.17 is 4,608.95 (+2.95 upstream). c_wi_published_2026, the published 2026 standard deduction, brings it to 4,598.48, which the state port reports equals 1.755.4 + r19. The board keeps v1.1 for excluded outputs. The investigator should confirm r01 is still unfixed in 2.15.17. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "state": "WI", + "board": 4605.99707, + "latest": 4598.476074, + "v11_1755": 4605.99707, + "excluded": true, + "exclusion_root_cause": "r01_ira_compensation" + } + ] + }, + { + "id": "excl_niit_scope_020", + "title": "Excluded TX federal output (NIIT-scope reading) moved by the sales tax deduction", + "hypothesis": "The exclusion is about an unlisted reading: whether federal_income_tax_before_refundable_credits includes NIIT. The move has a different cause. Raw 2.15.17 equals latest (68,334.73), so no convention is involved. state_and_local_sales_or_income_tax fell from 3,256.81 (1.755.4) to 1,914.00 (2.15.17), itemized deductions dropped from 17,090 to 15,748, and the filer stopped itemizing. The likely cause is #9616 (3f029abf5b, 2026-09-27), which corrected the IRS optional sales tax table and added the 2025 values that c_irs_sales_tax_2025 holds for 2026. The other candidate is a change in the local sales tax computation. 033 SD's sales tax also fell (1,473.57 to 1,168.80) but does not bind there. The investigator should check the move is the sales tax table, that the exclusion's NIIT alternative still differs, and recompute the alternative value on 2.15.17.", + "outputs": [ + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "state": "TX", + "board": 68056.710938, + "latest": 68334.734375, + "v11_1755": 68056.710938, + "excluded": true, + "exclusion_root_cause": "unlisted:whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say" + } + ] + }, + { + "id": "excl_snap_ssi_disability", + "title": "Excluded SNAP outputs (meets_ssi_disability_criteria unlisted): board keeps fractional v1.1", + "hypothesis": "Exclusion root cause: meets_ssi_disability_criteria is unlisted. The board keeps the untouched fractional v1.1 values for excluded outputs. The new values are the stated-facts values under the SNAP rounding fixes already in 2.15.17 (r26/r27/r28/r31 via #9318 and #9162) plus the FY2026 hold. The SNAP port reports 023 = 408 and 057 = 2,628, both equal to the 1.755.4 regeneration's stated-facts values (c13v3_upstream_plus_r33). 100 = 8,556 matches the exclusion note. Raw 2.15.17 is 462, 2,682 and 8,637. The exclusions stay valid. The investigator should check whether the alternative values (3,576, 840 and 8,844, computed on 1.755.4) need recomputing on 2.15.17 plus the hold. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_023", + "variable": "snap", + "state": "CA", + "board": 461.339722, + "latest": 408, + "v11_1755": 461.339722, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria" + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "state": "LA", + "board": 2669.217041, + "latest": 2628, + "v11_1755": 2669.217041, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria" + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "state": "MT", + "board": 8625.889648, + "latest": 8556, + "v11_1755": 8625.889648, + "excluded": true, + "exclusion_root_cause": "unlisted:meets_ssi_disability_criteria" + } + ] + }, + { + "id": "excl_snap_mortgage_residence", + "title": "Excluded NY SNAP output (mortgage-residence reading): board keeps fractional v1.1", + "hypothesis": "Exclusion root cause: it is unstated whether the listed mortgage interest is on the home the SNAP household occupies. The mechanism is the same as excl_snap_ssi_disability. Raw 2.15.17 gives 2,904, within $1 of the board. The FY2026 hold (standard deduction, shelter cap, max allotment) brings it to 2,868. The SNAP port reports 2,868 equals the 1.755.4 regeneration's stated-facts value. The exclusion stays; only the alternative value (3,576) may need recomputing. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_118", + "variable": "snap", + "state": "NY", + "board": 2903.94043, + "latest": 2868, + "v11_1755": 2903.94043, + "excluded": true, + "exclusion_root_cause": "unlisted:whether the listed home mortgage interest is on the home the SNAP household occupies" + } + ] + }, + { + "id": "excl_r30_snap_heat_and_eat_sua", + "title": "Excluded PA SNAP output (r30 heat-and-eat SUA, still unfixed)", + "hypothesis": "Exclusion root cause r30_snap_heat_and_eat_sua. The SNAP port found snap_state_using_standard_utility_allowance.py in 2.15.17 still returns p.always_standard[state] with no elderly/disabled condition (only a metadata change, d7fc17abfe). The defect is still present. Latest 3,576 is the conventions-plus-upstream-fixes value, and hold + r30 gives 3,240, the exclusion's alternative. Raw 2.15.17 is 3,600. The exclusion stays valid. Scoring is not affected.", + "outputs": [ + { + "scenario_id": "scenario_080", + "variable": "snap", + "state": "PA", + "board": 3596.039795, + "latest": 3576, + "v11_1755": 3596.039795, + "excluded": true, + "exclusion_root_cause": "r30_snap_heat_and_eat_sua" + } + ] + } + ], + "notes": "Module: /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_conventions.py (sha256 4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb). It composes the nine ported conventions in the same pattern as c13v3_plus_r33.py and latest_state_conventions.py: each part's Reform.apply runs against the composed reform. The parts are ca, irs_sales_tax, wi, id, mn, md, mi, mo and snap_hold_fy2026.\n\nTwo things are left out:\n- latest_md_local_output_scope.py. The brief says new references are 2.15.17 plus the pre-freeze-law conventions and nothing else, and the state port labels this module an output-definition adapter for the lead to decide.\n- The stated-hours to weekly_hours_worked_before_lsr mapping (scenario_066). It is an input mapping, not a convention.\nIf the lead adopts either one, the clusters md_county_tax_8888 and snap_abawd_hours_default_9261 are where they act.\n\nComposition checks. All pass; scripts and logs are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/compose/.\n1. Parameter level (check_composition.py/.json). Over the 102,807-leaf tree (values_list compared), the nine parts change disjoint leaf sets: 91 + 5,700 + 10 + 9 + 87 + 20 + 21 + 8 + 685. The composed system changes exactly their union, 6,631 leaves, and every leaf's values_list is identical to its part's system.\n2. Output level (check_outputs.py). Every output a part changes alone against raw 2.15.17 has the same value composed: 6 CA, 3 WI, 3 ID, 1 MN, 1 MD, 1 MI, 1 MO, 17 SNAP and 0 IRS. There are no unexplained composed changes and no output is touched by two parts. The composed CSV is identical to the SNAP agent's probe_snap_plus_state_conventions.csv (max diff 0.0).\n3. Reproduced outputs (check_reproduced.py). Every output either port report lists as reproduced is exact to the board. The two exceptions are the ones the reports expected: 068 MD state needs the MD scope adapter, and 066 VA SNAP needs the hours mapping.\n\nFull sweep (out/latest_conventions.csv, sha256 af37fd3e...): 1,984 outputs, 29 moved against the board (19 excluded, 10 scored). The raw 2.15.17 baseline had 41 moved.\n- The conventions fix 18: SNAP 008, 012, 027, 030, 038, 043, 054, 073, 079, 108, 109; 022 fed; 023 state and refundable; 045, 076, 093, 122 state.\n- They newly move 6, all excluded, because the board keeps v1.1 values for excluded outputs: 007, 022, 053, 099 state; 023 and 118 SNAP.\n- One small nonzero delta stays under $1: 117 AR state, +0.48.\n\nGrouping evidence (grouping only, not investigation):\n- probe_inputs.py lists each moved scenario's inputs.\n- probe_components.py compares intermediate variables on 1.755.4 (comp_1755_raw.json) and 2.15.17 + conventions (comp_21517_conv.json).\n- moved_joined.csv joins each moved output to reference_exclusions.json.\nThe commit attributions in the hypotheses come from git log/show on the read-only upstream repo, range 06665727d8..79be99f671. The enactment and publication dates are taken from commit messages and still need primary-source checks. That applies especially to NJ S-4531 (said to be signed 2026-06-30) and the AZ DES manual.\n\nCross-cluster links: #9122 (the SALT refund in gross income) moves 033, 117, 078 fed and the excluded 005, 082 and 120 federal outputs, so one ruling covers all six. #8888 and #9122 together explain 078 fed. No existing file was modified, nothing was pushed or filed, and the upstream repo was only read." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/flagged_sept22_wave.json b/reference_audit/2026-09-28/verification/flagged_sept22_wave.json new file mode 100644 index 00000000..902d64e4 --- /dev/null +++ b/reference_audit/2026-09-28/verification/flagged_sept22_wave.json @@ -0,0 +1,41 @@ +[ + "scenario_003:federal_income_tax_before_refundable_credits", + "scenario_005:state_income_tax_before_refundable_credits", + "scenario_007:state_income_tax_before_refundable_credits", + "scenario_008:payroll_tax", + "scenario_020:federal_income_tax_before_refundable_credits", + "scenario_022:state_income_tax_before_refundable_credits", + "scenario_028:child3_chip_eligible", + "scenario_030:snap", + "scenario_039:federal_income_tax_before_refundable_credits", + "scenario_039:state_income_tax_before_refundable_credits", + "scenario_042:federal_income_tax_before_refundable_credits", + "scenario_042:state_income_tax_before_refundable_credits", + "scenario_049:federal_income_tax_before_refundable_credits", + "scenario_051:state_income_tax_before_refundable_credits", + "scenario_052:federal_income_tax_before_refundable_credits", + "scenario_053:state_income_tax_before_refundable_credits", + "scenario_056:snap", + "scenario_056:state_refundable_credits", + "scenario_057:snap", + "scenario_057:ssi", + "scenario_064:federal_income_tax_before_refundable_credits", + "scenario_066:snap", + "scenario_067:dependent1_medicaid_eligible", + "scenario_067:ssi", + "scenario_076:state_income_tax_before_refundable_credits", + "scenario_081:state_income_tax_before_refundable_credits", + "scenario_086:federal_income_tax_before_refundable_credits", + "scenario_091:state_income_tax_before_refundable_credits", + "scenario_099:state_income_tax_before_refundable_credits", + "scenario_100:federal_refundable_credits", + "scenario_100:state_refundable_credits", + "scenario_104:state_refundable_credits", + "scenario_108:state_refundable_credits", + "scenario_109:snap", + "scenario_110:federal_income_tax_before_refundable_credits", + "scenario_112:snap", + "scenario_118:state_refundable_credits", + "scenario_119:federal_refundable_credits", + "scenario_119:state_income_tax_before_refundable_credits" +] \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/judge_verdicts.json b/reference_audit/2026-09-28/verification/judge_verdicts.json new file mode 100644 index 00000000..1bd88d7c --- /dev/null +++ b/reference_audit/2026-09-28/verification/judge_verdicts.json @@ -0,0 +1,1774 @@ +{ + "note": "The verdicts the adjudication record's judge dates and flags name, read by scripts/date_adds0928_judge_verdicts.py from each case's verdict.json and its sha256-bound verdict.meta.json. 'current' is the case's verdict in the release's audit tree (the stage); 'previous' is the verdict a judge_previous item dates, from the September 22c audit tree; 'published' is the judge verdict the decision's own wave release recorded (git show :annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json).", + "wave_releases": { + "2026-09-05": { + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "release": "dashboard-data-20260905c", + "pull_request": 164, + "committed_on": "2026-09-05" + }, + "2026-09-22": { + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "release": "dashboard-data-20260922", + "pull_request": 174, + "committed_on": "2026-09-23" + }, + "2026-09-29": { + "commit": null, + "release": "dashboard-data-20260929", + "pull_request": null, + "adjudications_written_on": "2026-09-29" + } + }, + "cases": { + "us__scenario_002__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T02:43:30.671664+00:00", + "verdict_sha256": "7b4c64608ce0152998d61f834b2f56e051899f304f1e234c6bc8d67f31d3a53c" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_003__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:39.973219+00:00", + "verdict_sha256": "b701c74f04ece7bac39677f594e480cfc1e51235d59027f442253bd6f261a9dd" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:05.256460+00:00", + "verdict_sha256": "137a4e4e03748edc1c4761a31b2d13e30af75231d1411e94c21499ac852c7fdf" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_005__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:40.183300+00:00", + "verdict_sha256": "df93ae6729a4c8a7b1faa94b346776e631f437a9363eb56eb7439bb430a56443" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:04.465263+00:00", + "verdict_sha256": "d822ff9bf8ee9e8b6b41a6e747fca10f1c666c95403da8af5c5d1044f5d96df1" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_005__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:40.389780+00:00", + "verdict_sha256": "6a72ba07aaef729002b373234d135afd7e1fc7a592a42589e5791f0d1420b80b" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:02.588625+00:00", + "verdict_sha256": "9ad2e12af0a84a02d6526d1ba6149ea4a27802ca4050950b2d7c76c74f802b18" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_007__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:40.505506+00:00", + "verdict_sha256": "ae4f45e4c06169ac8062ba546969e25a0e2e25569b56143c4811d70c95867032" + } + }, + "us__scenario_007__head_medicare_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:01.631618+00:00", + "verdict_sha256": "3f881ff75efd3f35dfa847af2c94cb597a6fff67eaf70f139a3eddd9a9faaf24" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_007__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:40.628719+00:00", + "verdict_sha256": "4c86652fbfe23e755f537782717de45e07b8617da10bb31967771456d9ca65bd" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:03.414547+00:00", + "verdict_sha256": "52e646c07bba4018281f0587735e79c03acc790335ab17f0661743c7ed58cc6b" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_008__payroll_tax": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T11:49:30.688104+00:00", + "verdict_sha256": "439ddfb2413ecd119f97fb1da66faa6a0fe2f432238f1d428d870e68f4b783a3" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:03.118387+00:00", + "verdict_sha256": "36c7d799d2359b26d4ca2422ce924de0310183ded2089bfea90e7de95c54e8ad" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "judged_on_utc": null + } + }, + "us__scenario_008__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T11:49:30.750595+00:00", + "verdict_sha256": "dbeadb80ba65622d9bb1ed2ff69529705926124f35e893d4223b9b35ecf2c72f" + } + }, + "us__scenario_020__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:44.395087+00:00", + "verdict_sha256": "3083467d9c44e31ba649d3b96b2b10d90fe8acc9dc0ed8beade020a516b077fd" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T01:36:48.372396+00:00", + "verdict_sha256": "a9909243e079931d2b6b9c7c6c16b531ab9bee11cba3ccdb4e47f18e667c2b13" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_022__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:44.676837+00:00", + "verdict_sha256": "feca3f8988dea0e271e35d3d6bc440557bc26e349992c189c7cc8f8aa249661e" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:07.647367+00:00", + "verdict_sha256": "ba923b2873dadb261e734f82b8e6f9cab70a2e284c2c82b8bb786150e8fc6c33" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_023__head_medicaid_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:44.868932+00:00", + "verdict_sha256": "5094b4eda857c36ce42cf847b76eee7ea7f082c61eb623fabbd7c2e324b28fce" + } + }, + "us__scenario_023__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:45.040435+00:00", + "verdict_sha256": "41a054abc7a10195a12df267168716cc1245878221aa7f702c779f0b0e343317" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:04.242090+00:00", + "verdict_sha256": "7363fda3bd1cd0590e749b03f33a576c7b6afe7e9d1184f52f25d3bdf960d49c" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "gpt-5.6-sol", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": "2026-09-01" + } + }, + "us__scenario_028__child3_chip_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "health_coverage", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:46.862324+00:00", + "verdict_sha256": "573d972c8d5cc1bc9e181e8a4f8456a3aa54f3e73f9e2bfc5a4afd7c25721c07" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "health_coverage", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:03.903183+00:00", + "verdict_sha256": "a12397b9137a2ea4a724d98e0f2d88c32c15427265179aa400008bcdf2d4ca69" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "health_coverage", + "judged_on_utc": null + } + }, + "us__scenario_030__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:48.152909+00:00", + "verdict_sha256": "1a48069de387a5858b2c09f397b611be0eac9b3d3a352185f3e15d7730ce4f83" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:01.658131+00:00", + "verdict_sha256": "14255dd73cf62e4216398ce2994295564e8e528f2b1d4b224c4e93c19d88bdde" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": null + } + }, + "us__scenario_033__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T11:49:30.939882+00:00", + "verdict_sha256": "5c1dbb482df86fbc3ae59ee58962e84a115a0640ddae8cd622fd4f0ccfb066a8" + } + }, + "us__scenario_039__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:49.842049+00:00", + "verdict_sha256": "ee6d3d2d302cdabf1814936915dd0616df025a0ced2b94f7cb6f33f3538679d1" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:07.477748+00:00", + "verdict_sha256": "4fa1d5029eaefff5d3365deb667892cf4ab0bd2ff3da5e5769ce4bfdc20e8e02" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_039__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:49.991160+00:00", + "verdict_sha256": "3aad1cbdc88205b1569d16dfa2077fc1c3665cc5e455613b100bfccad6ea6f72" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:07.997657+00:00", + "verdict_sha256": "cda74f434c086b7efc60ac6e793555469f720c567620da844a120e85da8b913f" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_042__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:50.123690+00:00", + "verdict_sha256": "5170813b5e766e78fb652be695e0f32a0093105e1fb1c410eb710bc109d0055c" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:02.575294+00:00", + "verdict_sha256": "75201e341015414768b41fe25fc4abd4fb745ee6eef48120514ee309813cc460" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_042__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:50.215128+00:00", + "verdict_sha256": "bd0484d9b12ea42cbe6d91e30b3877e5f8375e84d2491e5ef58d8d97060b223d" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:10.163154+00:00", + "verdict_sha256": "acdf9e59e439422da5c03808b14a45d6643877dc7ce5d7affa22b239cbb649aa" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_049__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:51.553485+00:00", + "verdict_sha256": "64ca2961109ec9568db4ef3e5f43263a74f73da37fab979016d549432d4ba923" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:08.916353+00:00", + "verdict_sha256": "052d5d43c259db09138b08eb5f32f20b879593421ef6cd0a9bb53bda07ddac9a" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_051__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:51.816587+00:00", + "verdict_sha256": "debe0ac227595d88e7b2ec74b9995e165182c239dd5c124d073a1e722bb6c2a5" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:01.365298+00:00", + "verdict_sha256": "f6be4503a154edd7f8a1ab2ee95fa8c23b9affae7582d0b98c2d9774990218a8" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_052__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:51.893601+00:00", + "verdict_sha256": "998abf3ea2f093a513fc7988dc2b132edbc0dcc6945611d97054d95b534304ac" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:09.662642+00:00", + "verdict_sha256": "a84b9d12a1b106b94d36b11d0ae9526e234164d92d685ea114a733d40d73ccf7" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_053__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:52.052498+00:00", + "verdict_sha256": "2f345eedc5d57116fd91a647f50bc8469fabb997154c833e6a1b12f7e3520014" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:07.824581+00:00", + "verdict_sha256": "6e578cabcc0d360f1c5e8d084a128bf780446bbde90c2a4ee6417ff88895cfc9" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_056__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:54.200208+00:00", + "verdict_sha256": "cd7f02516e974876d855fbea87b896eaee5c5c75813f3f6bfd8a835edbf31d44" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:08.539127+00:00", + "verdict_sha256": "09d69947556f4c7ce480532c7e9fd4f24044ae4d7cd95003b7f497b21bc0edec" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "judged_on_utc": null + } + }, + "us__scenario_056__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:54.278896+00:00", + "verdict_sha256": "c6c23b679da533c4f4ecb38d3bba82ddd46b8578dc10a04185740c12dbfae43d" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:08.821803+00:00", + "verdict_sha256": "92b31a4a9db0b6a8f109d759ee6e3a938e1b69ddeb92cceac1dbb0a484f18147" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "credit_phaseout", + "judged_on_utc": null + } + }, + "us__scenario_057__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:54.553048+00:00", + "verdict_sha256": "29cc6f8b354d55aa186eb9ba4f822446e05d6d5152ae1848ee3a0457e649223b" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "age_disability", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:10.452990+00:00", + "verdict_sha256": "001c151b71f904fd9997fd2ecf1e9f8b941c027f73830fe86047663e3dd3e6b1" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "gpt-5.6-sol", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": "2026-09-01" + } + }, + "us__scenario_057__ssi": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:54.717704+00:00", + "verdict_sha256": "a03c8f46a67a2683c3661c174cfcb49326a718ae46edb527813af10030059b38" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:10.263721+00:00", + "verdict_sha256": "dc476e95992ff8f486f1bd67a56a13b857b6496fd1eacfd876795b1e7c4995bf" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "gpt-5.6-sol", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-01" + } + }, + "us__scenario_062__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:54.977478+00:00", + "verdict_sha256": "d81a5a5dc1eca4ba55cdb3cdaba0a372a40a339a0e79efb01d663ec67aa49a9d" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:09.327919+00:00", + "verdict_sha256": "8ac8d4a94d9cf08fb225f2e2d0199e846bd1459ee055ab998a31fd912a882422" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_064__dependent1_medicaid_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.227471+00:00", + "verdict_sha256": "8173e4b5fd7befcff8b654e1e66a1560b53474b93424a19e3b3542350d92acb9" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T06:36:20.322020+00:00", + "verdict_sha256": "04453c09b1b90dd623091c58709968103e0fc3d789f603bea20b8e6d368687fe" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "judged_on_utc": null + } + }, + "us__scenario_064__dependent2_medicaid_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.432520+00:00", + "verdict_sha256": "6a4dc6506ad751fa451ad0d43ff52203a6f501dddaf5d2f3e00500ca6e4ecb91" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T06:40:22.257702+00:00", + "verdict_sha256": "6d48584bb1c7bc8fa0fcd71254097aa735ce6d87cbcd55e8b74e8427ec273358" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_064__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.519926+00:00", + "verdict_sha256": "8dd1c443f4d6094c8ecf3ce6022f3b297c51c7366f526f0b4887e66ded2d7151" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:08.631901+00:00", + "verdict_sha256": "8cc366565a1783e8be469dad1ff81c653813e4cdca8edc585dadf85c00c43a5a" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_064__ssi": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.609393+00:00", + "verdict_sha256": "0dc3089d4ec15ca23b7689e8b351e75b4ab7dbabf8c6290b5a8b63e59969f55e" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:08.348264+00:00", + "verdict_sha256": "a5053b0868174cf4d5594daedcdb492398b2e3ea4bff9998adde89658c735fa1" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "prompt_ambiguity", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_064__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.706215+00:00", + "verdict_sha256": "806778b3960b2cb8e5aa46de8a5ccf1ab4191f09a7eecd45f6a31d96ee0fd4fa" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:11.981428+00:00", + "verdict_sha256": "0ae103e79bef21843fef6e173d865f26761c33ed1648a8d6d9fe1877c0d11408" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_066__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:01.184130+00:00", + "verdict_sha256": "0b3f12a3795ea34ac276c9245317ee7337787caeb44b664c08c40edcd49ee827" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_067__dependent1_medicaid_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:55.891418+00:00", + "verdict_sha256": "fb827611e784dfba133a4174709eae2f660a3dde77563313e9821e2d574097be" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "household_unit_or_filing_status", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:09.016013+00:00", + "verdict_sha256": "3eb59933637d8683728e7622abdb4053132c91eb1b85b456b1531b6bc7b8e3b9" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "household_unit_or_filing_status", + "judged_on_utc": null + } + }, + "us__scenario_067__ssi": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_data_issue_fixed", + "case_failure_subtype": "age_disability", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:56.143279+00:00", + "verdict_sha256": "2158e5fc2980876918a076ee51fada9d91f4c49cf98203b43368d9f090a6b515" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "age_disability", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:09.414479+00:00", + "verdict_sha256": "f08fa22320919f5f883fc67c4435fce2c22017ed786062fd671d1a5f21d50f96" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "gpt-5.6-sol", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-01" + } + }, + "us__scenario_073__head_medicare_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:08.733030+00:00", + "verdict_sha256": "9e871421c786a853c005cbc79fe0284def24692131427b59955eea5516a6b8c4" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_074__head_medicare_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:56.967931+00:00", + "verdict_sha256": "b181ddfa64457c117d3a1797c688674737a6f2fca0298c3763a1d2d10845ea97" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T07:11:05.484455+00:00", + "verdict_sha256": "00808442cb21531f8f3de93faa45c524702a4c8c53d4a982ab90bf9aed067237" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_076__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:57.233243+00:00", + "verdict_sha256": "0ac03bfb2f41f7be5111665a1fc71dc29420482194f38a7f60131ecd70f6de57" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:52:18.926418+00:00", + "verdict_sha256": "c5aad86673b9ba0ef2aa8e4f2b0a2f4a68503ab2cf8958e2bf8fa145e07bc2d1" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_078__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T11:49:31.081978+00:00", + "verdict_sha256": "6610e1906bd9548d193559ab8b4db2b9331ce40955cbc21be8e9c97a96dd121d" + } + }, + "us__scenario_079__head_medicare_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:57.801280+00:00", + "verdict_sha256": "851ff4a1182ec096dc4d65fe3b64cd357fcd4c565f98d88e6d2cde138959bf20" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:09.970375+00:00", + "verdict_sha256": "4ad57e2ab7c4293c7f0d0ba13a50147d90a00a7ef9edd9f10631cf7499755392" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_080__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:58.270918+00:00", + "verdict_sha256": "222f30b266dcfe3ce6e7c46e65a885ad170343b2205d3b7d29f59c39ffba5d72" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T13:45:23.340904+00:00", + "verdict_sha256": "e86afe45ce1eb58b289b5ef1e7e133106930abbc151026632dcc36e3d44151f8" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": null + } + }, + "us__scenario_081__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:58.529851+00:00", + "verdict_sha256": "793d50e48f8598b85f5c1c3c0eeddc1b1fa01ff5a329d9931a079b85bd5d1876" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:12.745524+00:00", + "verdict_sha256": "91c5ce9d5d1f5ecbfaedeb73459146a5e1e9e14023cdfce207ce3f7012b91b7e" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_082__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:58.608114+00:00", + "verdict_sha256": "81aa2a674a4188d65d1bc87fbf11be80f77db5fbd028db3dcacadf9950f7bc00" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:12.319113+00:00", + "verdict_sha256": "b3ebc9aeb7b21e3b1cf88ab7937144538bbb03b1bc89aec2ce0ecb9f9c81ec1e" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_084__head_medicare_eligible": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:58.952533+00:00", + "verdict_sha256": "1aae4222e16575101c4da0ed1e8fc764050c1fe4bbbb9ed9ee589ff2da6071ff" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:11.529107+00:00", + "verdict_sha256": "37a4c55752f636b89e6df7942c92f7a58d2e861766bf5db3ccdbf2cc0ff816a4" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "age_disability", + "judged_on_utc": "2026-09-05" + } + }, + "us__scenario_086__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:49:59.141201+00:00", + "verdict_sha256": "1c97c59a94e665898894b9350df34122ce8a9fdee7e32b4610da97aaf42a14f1" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:13.186603+00:00", + "verdict_sha256": "617c4f301b69a07edff0c8241c0bab3d239ed42d64d1fdf8efc44bcd08776c2b" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_091__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:49:59.937230+00:00", + "verdict_sha256": "4016d2798ec41c954b76f1e21d2ef4e8833826bad8798daa17b5ed43eaa3c31c" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T18:09:15.309825+00:00", + "verdict_sha256": "9dc6a4f45ac86e25628448180327351d08f836ec0966c31fec025f1c5c7253bc" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_099__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:00.715416+00:00", + "verdict_sha256": "34a0352852925ca3f56e90224c39dd4abb35d043f44bf8311506c0edb4b30cfb" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:13.020436+00:00", + "verdict_sha256": "d74c175eec3c9a18eaa178ef6b29fd535e5b950fc4efc8ef5533d1ac9a383f2a" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "judged_on_utc": null + } + }, + "us__scenario_099__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:50:01.192004+00:00", + "verdict_sha256": "099ab7858bdf8764fb01c3cf1eec279c1efa5bc9c6ebea8150941e7c1072ca8d" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:15.605826+00:00", + "verdict_sha256": "4fe08d137c52f282f96a0dfe157a37295e2bdf3510ae4aa3d86996fcbf9c9d81" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_100__federal_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-22T17:55:46.739341+00:00", + "verdict_sha256": "d69995cecd526b9b9c5e882690a2e687f09bdf76ef3d380640c71c3424132127" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_100__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:01.301327+00:00", + "verdict_sha256": "7ce99fb4948a9c14cce7e3aac16f38f215049336a7ed1959f4d9742b2454045a" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:13.354385+00:00", + "verdict_sha256": "b5402d8d6b7b69c8b723f5f83954893387203c7c62eee93be83263bf30eacbef" + }, + "published": { + "release": "dashboard-data-20260905c", + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "judge_model": "gpt-5.6-sol", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": "2026-09-01" + } + }, + "us__scenario_100__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:50:02.703407+00:00", + "verdict_sha256": "97dc0c0930067a602e1243249df3d13062e54b642406fb48e069482e87f46811" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:13.946521+00:00", + "verdict_sha256": "5e2b1f92b1f5072ebbd34a6baf2edb17d9de9616d9940ff6a0b79938072233fa" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "judged_on_utc": null + } + }, + "us__scenario_104__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:01.273841+00:00", + "verdict_sha256": "48a4b28bcb0b63f57af2e47109f0cda96823db8e281ebbcf46675e98d8a0f605" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": null + } + }, + "us__scenario_107__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:12.939983+00:00", + "verdict_sha256": "f392af050b61548f936e52455de3aeefd5f32310a0f1478d8d6d6381b054ddef" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_108__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:50:03.346706+00:00", + "verdict_sha256": "bf289fe7f42a0a8c562da2a581fd0b6e871f4277c54ea2e0c58711eadf25da5a" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "state_local_rule", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:13.862252+00:00", + "verdict_sha256": "d026bfc7434f4ed273f1d6e6253f7837e31b587cc220d991a90c13c951ea1fb7" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "state_local_rule", + "judged_on_utc": null + } + }, + "us__scenario_109__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:03.542816+00:00", + "verdict_sha256": "4f37a6168f6ddd64e13273e4d5b51887a583b3cc4b2e263ba23be3c489200871" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T12:00:02.771649+00:00", + "verdict_sha256": "63db77b571e2a4f618099e34c55057c869ad801153719e6de5d76667c5c83597" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_110__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:50:03.612235+00:00", + "verdict_sha256": "2d0acb6d999128a685eb312c9b1ba2cb1c6977144841e85d910d71e48f077ea4" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:15.530125+00:00", + "verdict_sha256": "2544a8b3cfd7006804479d28fa4a9b4c8f772edf657fe5ca8eb0688668347d24" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_110__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:03.686907+00:00", + "verdict_sha256": "fdac8632534960ae57af1b45e93e1c279d50f2b9b62ed59cfb884ebab404ceef" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:14.465411+00:00", + "verdict_sha256": "21d7f02d55bbe0aeb4efe7f1cb8ec04f0b39f2b730cc496f024747abbec56b60" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "judged_on_utc": null + } + }, + "us__scenario_112__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:50:05.957419+00:00", + "verdict_sha256": "df3b27ef8a1b854dbae0c5f8e2132e226640c8a7a8aabf6b2ac719c64d2b2019" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:14.203397+00:00", + "verdict_sha256": "1e2b3969e39d1b7ca4ef7bd6e9e89996ff228fcb223d00d253215e183575c289" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": null + } + }, + "us__scenario_117__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T11:49:31.383752+00:00", + "verdict_sha256": "6fe8a0fcffaba12526ecae2c0f791fac636675aa1e1ff426f5864aab90797642" + } + }, + "us__scenario_118__snap": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:33:42.895878+00:00", + "verdict_sha256": "da8e108a324a75da56462cc5359bab742464d566c038a552458d47d949109c8b" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:14.860416+00:00", + "verdict_sha256": "c504b8728180b02fbbf264083c4571fb7a0a74398e9f5977abe5abf518f3430b" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_118__state_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": true, + "judged_at_utc": "2026-09-29T01:33:43.012626+00:00", + "verdict_sha256": "b7f923c1a512c7b94a2273e21d65d636f59069fb5a4aaf54e4fa0ffd54587487" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:14.946092+00:00", + "verdict_sha256": "3f79cd5bf04cc5bc937ed614662dd5dcc338d9b58ec814552a8af4eefe0041c1" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "categorical_eligibility", + "judged_on_utc": null + } + }, + "us__scenario_119__federal_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:06.898088+00:00", + "verdict_sha256": "a4c92a0da6ff70ba83547bf62db038219b3939471d447938ea84079c7e58143a" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:16.316723+00:00", + "verdict_sha256": "411b51e4b828144d1d38b843d6ee4d39476aede68c85d7d095105f53edb52d21" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_engine_defect", + "case_failure_subtype": "credit_phaseout", + "judged_on_utc": null + } + }, + "us__scenario_119__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:50:07.049488+00:00", + "verdict_sha256": "1128d26b6d3f4197b0ddf1f94e77f8f284fb313494b2e9fdd8ebea6a949453b3" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "credit_phaseout", + "reference_suspect": true, + "judged_at_utc": "2026-09-23T00:27:16.115766+00:00", + "verdict_sha256": "c7365a3bdfa30e2cfa84403440cd9c89abdaeb390b9dcd0e4caa785da485cd34" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "reference_model_issue_fixed", + "case_failure_subtype": "credit_phaseout", + "judged_on_utc": null + } + }, + "us__scenario_120__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:33:43.085873+00:00", + "verdict_sha256": "918306009e82bcc7ae5fd17b491f8a6c64aa8530cbe7af9c2670514f3e8225cd" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:16.218843+00:00", + "verdict_sha256": "924ba711320a2b0918d114256276f2bc531c69e3fc7bad6ce75d7eb0d5d1c340" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_120__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-29T01:33:43.224167+00:00", + "verdict_sha256": "40cf423750666c61604f52b14f3ee8bc5ab0dd09b6afbf207b0d9c18b215929e" + }, + "previous": { + "tree": "20260922c", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "reference_suspect": false, + "judged_at_utc": "2026-09-23T00:27:15.686947+00:00", + "verdict_sha256": "dffec8b3762a13011177d0be33c857e1d7537fddd332a05b259bc707616f3146" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "state_local_rule", + "judged_on_utc": null + } + }, + "us__scenario_121__federal_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T10:04:45.045684+00:00", + "verdict_sha256": "ec63cc2614dabc2731ae4cd4c2aaf51ff02a2bdfe46e944b673d4d3db7425b36" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + }, + "us__scenario_121__state_income_tax_before_refundable_credits": { + "current": { + "tree": "stage", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "reference_suspect": false, + "judged_at_utc": "2026-09-05T10:11:22.305555+00:00", + "verdict_sha256": "a8b657c3e9398e752ec8ab3cf4d875ba203398c56ba07803b1a6b0ffc7533375" + }, + "published": { + "release": "dashboard-data-20260922", + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "judge_model": "claude-opus-5", + "case_failure_source": "llm_error", + "case_failure_subtype": "taxable_income_or_deductions", + "judged_on_utc": null + } + } + } +} diff --git a/reference_audit/2026-09-28/verification/latest_final_2170.csv b/reference_audit/2026-09-28/verification/latest_final_2170.csv new file mode 100644 index 00000000..ac7f4c0e --- /dev/null +++ b/reference_audit/2026-09-28/verification/latest_final_2170.csv @@ -0,0 +1,1985 @@ +fix,scenario_id,state,variable,engine,frozen,v11_1755,recomputed,moved_vs_v11,delta,moved +latest_final,scenario_000,TX,federal_income_tax_before_refundable_credits,2.17.0,2906.45361328125,2883.49365234375,2906.45361328125,True,0.0,False +latest_final,scenario_000,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_000,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,payroll_tax,2.17.0,2223.393798828125,2223.393798828125,2223.393798828125,False,0.0,False +latest_final,scenario_001,VA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_001,VA,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_001,VA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_002,WA,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_002,WA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,federal_income_tax_before_refundable_credits,2.17.0,22154.69921875,22154.69921875,22154.69921875,False,0.0,False +latest_final,scenario_003,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,federal_income_tax_before_refundable_credits,2.17.0,106505.8984375,106505.8984375,107833.15625,True,1327.2578125,True +latest_final,scenario_005,CA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,payroll_tax,2.17.0,36044.0,36044.0,36044.0,False,0.0,False +latest_final,scenario_005,CA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,state_income_tax_before_refundable_credits,2.17.0,41051.51171875,41051.51171875,41267.01171875,True,215.5,True +latest_final,scenario_005,CA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,federal_income_tax_before_refundable_credits,2.17.0,3707.695068359375,3707.695068359375,3707.695068359375,False,0.0,False +latest_final,scenario_007,ID,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,state_income_tax_before_refundable_credits,2.17.0,755.7783203125,755.7783203125,761.5570068359375,True,5.7786865234375,True +latest_final,scenario_007,ID,state_refundable_credits,2.17.0,155.0,155.0,155.0,False,0.0,False +latest_final,scenario_007,ID,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,federal_refundable_credits,2.17.0,12433.611328125,12433.611328125,12433.611328125,False,0.0,False +latest_final,scenario_008,NJ,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,payroll_tax,2.17.0,2162.760009765625,2162.760009765625,2162.760009765625,False,0.0,False +latest_final,scenario_008,NJ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_early_head_start_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_head_start_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child6_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child3_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child4_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child5_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child6_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_wic_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,self_employment_tax,2.17.0,565.1820068359375,565.1820068359375,565.1820068359375,False,0.0,False +latest_final,scenario_008,NJ,snap,2.17.0,15108.0,15246.9052734375,15108.0,True,0.0,False +latest_final,scenario_008,NJ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,state_income_tax_before_refundable_credits,2.17.0,264.80999755859375,264.80999755859375,264.80999755859375,False,0.0,False +latest_final,scenario_008,NJ,state_refundable_credits,2.17.0,5842.39990234375,5342.39990234375,5842.39990234375,True,0.0,False +latest_final,scenario_008,NJ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,federal_income_tax_before_refundable_credits,2.17.0,8348.0,8348.0,8348.0,False,0.0,False +latest_final,scenario_009,NC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,payroll_tax,2.17.0,4590.0,4590.0,4590.0,False,0.0,False +latest_final,scenario_009,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,state_income_tax_before_refundable_credits,2.17.0,2190.510009765625,2190.510009765625,2190.510009765625,False,0.0,False +latest_final,scenario_009,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,federal_refundable_credits,2.17.0,6127.0,6127.0,6127.0,False,0.0,False +latest_final,scenario_012,MS,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,payroll_tax,2.17.0,1683.0,1683.0,1683.0,False,0.0,False +latest_final,scenario_012,MS,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_wic_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,snap,2.17.0,4884.0,4952.08935546875,4884.0,True,0.0,False +latest_final,scenario_012,MS,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_013,AZ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,snap,2.17.0,240.0,0.0,240.0,True,0.0,False +latest_final,scenario_013,AZ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,state_refundable_credits,2.17.0,25.0,25.0,25.0,False,0.0,False +latest_final,scenario_013,AZ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,federal_income_tax_before_refundable_credits,2.17.0,6311.31787109375,6311.31787109375,6311.31787109375,False,0.0,False +latest_final,scenario_014,WV,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,payroll_tax,2.17.0,6802.96484375,6802.96484375,6802.96484375,False,0.0,False +latest_final,scenario_014,WV,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,state_income_tax_before_refundable_credits,2.17.0,3092.186279296875,3092.186279296875,3092.186279296875,False,0.0,False +latest_final,scenario_014,WV,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,federal_income_tax_before_refundable_credits,2.17.0,2969.1259765625,2969.1259765625,2969.1259765625,False,0.0,False +latest_final,scenario_015,IN,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,payroll_tax,2.17.0,3442.5,3442.5,3442.5,False,0.0,False +latest_final,scenario_015,IN,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,state_income_tax_before_refundable_credits,2.17.0,1236.3267822265625,1236.3267822265625,1236.3267822265625,False,0.0,False +latest_final,scenario_015,IN,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,payroll_tax,2.17.0,2649.80224609375,2649.80224609375,2649.80224609375,False,0.0,False +latest_final,scenario_016,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,federal_income_tax_before_refundable_credits,2.17.0,5211.05078125,5211.05078125,5211.05078125,False,0.0,False +latest_final,scenario_018,AZ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,payroll_tax,2.17.0,5202.38232421875,5202.38232421875,5202.38232421875,False,0.0,False +latest_final,scenario_018,AZ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,state_income_tax_before_refundable_credits,2.17.0,1146.05224609375,1146.05224609375,1146.05224609375,False,0.0,False +latest_final,scenario_018,AZ,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,federal_income_tax_before_refundable_credits,2.17.0,68056.7109375,68056.7109375,68334.734375,True,278.0234375,True +latest_final,scenario_020,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,payroll_tax,2.17.0,16689.0,16689.0,16689.0,False,0.0,False +latest_final,scenario_020,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_021,MO,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,federal_income_tax_before_refundable_credits,2.17.0,11113.5703125,11131.3271484375,11113.5703125,True,0.0,False +latest_final,scenario_022,CA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,payroll_tax,2.17.0,4710.83251953125,4710.83251953125,4710.83251953125,False,0.0,False +latest_final,scenario_022,CA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_022,CA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,state_income_tax_before_refundable_credits,2.17.0,2439.650146484375,2439.650146484375,2505.8701171875,True,66.219970703125,True +latest_final,scenario_022,CA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,federal_income_tax_before_refundable_credits,2.17.0,643.43359375,643.43359375,643.43359375,False,0.0,False +latest_final,scenario_023,CA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,payroll_tax,2.17.0,1561.1168212890625,1561.1168212890625,1561.1168212890625,False,0.0,False +latest_final,scenario_023,CA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_023,CA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,snap,2.17.0,461.3397216796875,461.3397216796875,408.0,True,-53.3397216796875,True +latest_final,scenario_023,CA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,state_income_tax_before_refundable_credits,2.17.0,12.776718139648438,6.7984771728515625,12.776718139648438,True,0.0,False +latest_final,scenario_023,CA,state_refundable_credits,2.17.0,95.1540069580078,148.3108673095703,95.15400695800781,True,1.4210854715202004e-14,False +latest_final,scenario_023,CA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,federal_income_tax_before_refundable_credits,2.17.0,7031.03564453125,7031.03564453125,7031.03564453125,False,0.0,False +latest_final,scenario_025,OH,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,payroll_tax,2.17.0,4798.48486328125,4798.48486328125,4798.48486328125,False,0.0,False +latest_final,scenario_025,OH,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,state_income_tax_before_refundable_credits,2.17.0,1921.57275390625,1921.57275390625,1921.57275390625,False,0.0,False +latest_final,scenario_025,OH,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,federal_refundable_credits,2.17.0,2013.4892578125,2013.4892578125,2013.4892578125,False,0.0,False +latest_final,scenario_026,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,payroll_tax,2.17.0,6518.474609375,6518.474609375,6518.474609375,False,0.0,False +latest_final,scenario_026,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,child3_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,state_income_tax_before_refundable_credits,2.17.0,2086.79052734375,2086.79052734375,2086.79052734375,False,0.0,False +latest_final,scenario_026,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_027,CT,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,snap,2.17.0,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_027,CT,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,federal_refundable_credits,2.17.0,3274.927978515625,3274.927978515625,3274.927978515625,False,0.0,False +latest_final,scenario_028,PA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,payroll_tax,2.17.0,4632.0,4632.0,4632.0,False,0.0,False +latest_final,scenario_028,PA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,reduced_price_school_meals_eligible,2.17.0,0.0,1.0,0.0,True,0.0,False +latest_final,scenario_028,PA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,state_income_tax_before_refundable_credits,2.17.0,1842.3070068359375,1842.3070068359375,1842.3070068359375,False,0.0,False +latest_final,scenario_028,PA,state_refundable_credits,2.17.0,62.41279220581055,62.41279220581055,62.41279220581055,False,0.0,False +latest_final,scenario_028,PA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_029,OK,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,state_refundable_credits,2.17.0,40.0,40.0,40.0,False,0.0,False +latest_final,scenario_029,OK,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,payroll_tax,2.17.0,994.5,994.5,994.5,False,0.0,False +latest_final,scenario_030,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_030,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,head_wic_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_030,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,snap,2.17.0,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_030,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_031,CA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_031,CA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,federal_refundable_credits,2.17.0,6127.0,6127.0,6127.0,False,0.0,False +latest_final,scenario_032,MN,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,payroll_tax,2.17.0,2346.10009765625,2346.10009765625,2346.10009765625,False,0.0,False +latest_final,scenario_032,MN,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,state_refundable_credits,2.17.0,2187.60009765625,2187.60009765625,2187.60009765625,False,0.0,False +latest_final,scenario_032,MN,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,federal_income_tax_before_refundable_credits,2.17.0,3818.148193359375,3788.988037109375,3818.148193359375,True,0.0,False +latest_final,scenario_033,SD,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,payroll_tax,2.17.0,6043.5,6043.5,6043.5,False,0.0,False +latest_final,scenario_033,SD,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_033,SD,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_033,SD,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,federal_income_tax_before_refundable_credits,2.17.0,4926.39990234375,4926.39990234375,4926.39990234375,False,0.0,False +latest_final,scenario_036,NJ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,state_income_tax_before_refundable_credits,2.17.0,1724.155029296875,1724.155029296875,1724.155029296875,False,0.0,False +latest_final,scenario_036,NJ,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,federal_income_tax_before_refundable_credits,2.17.0,1649.2513427734375,1649.2513427734375,1649.2513427734375,False,0.0,False +latest_final,scenario_037,NC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,payroll_tax,2.17.0,3208.668701171875,3208.668701171875,3208.668701171875,False,0.0,False +latest_final,scenario_037,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,state_income_tax_before_refundable_credits,2.17.0,1148.69921875,1148.69921875,1148.69921875,False,0.0,False +latest_final,scenario_037,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,federal_refundable_credits,2.17.0,10225.353515625,10225.353515625,10225.353515625,False,0.0,False +latest_final,scenario_038,LA,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,payroll_tax,2.17.0,571.2637329101562,571.2637329101562,571.2637329101562,False,0.0,False +latest_final,scenario_038,LA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,self_employment_tax,2.17.0,2193.612548828125,2193.612548828125,2193.612548828125,False,0.0,False +latest_final,scenario_038,LA,snap,2.17.0,7212.0,7286.9443359375,7212.0,True,0.0,False +latest_final,scenario_038,LA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,state_refundable_credits,2.17.0,365.8000183105469,365.8000183105469,365.8000183105469,False,0.0,False +latest_final,scenario_038,LA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,federal_income_tax_before_refundable_credits,2.17.0,1345.510498046875,1345.510498046875,1345.510498046875,False,0.0,False +latest_final,scenario_039,VA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,state_income_tax_before_refundable_credits,2.17.0,514.4984130859375,514.4984130859375,514.4984130859375,False,0.0,False +latest_final,scenario_039,VA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,payroll_tax,2.17.0,2142.0,2142.0,2142.0,False,0.0,False +latest_final,scenario_040,AZ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_040,AZ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,federal_income_tax_before_refundable_credits,2.17.0,2361.96484375,1979.1583251953125,2361.96484375,True,0.0,False +latest_final,scenario_042,WI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_042,WI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,state_income_tax_before_refundable_credits,2.17.0,284.74090576171875,284.74090576171875,464.18280029296875,True,179.44189453125,True +latest_final,scenario_042,WI,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,payroll_tax,2.17.0,330.7310791015625,330.7310791015625,330.7310791015625,False,0.0,False +latest_final,scenario_043,CO,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_043,CO,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,snap,2.17.0,3576.0,3596.039794921875,3576.0,True,0.0,False +latest_final,scenario_043,CO,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,state_refundable_credits,2.17.0,19.0,19.0,19.0,False,0.0,False +latest_final,scenario_043,CO,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_044,KS,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,federal_income_tax_before_refundable_credits,2.17.0,2173.07080078125,2173.07080078125,2173.07080078125,False,0.0,False +latest_final,scenario_045,MI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,payroll_tax,2.17.0,2775.08251953125,2775.08251953125,2775.08251953125,False,0.0,False +latest_final,scenario_045,MI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,snap,2.17.0,0.0,287.68316650390625,0.0,True,0.0,False +latest_final,scenario_045,MI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,state_income_tax_before_refundable_credits,2.17.0,1290.9625244140625,1288.8375244140625,1290.9625244140625,True,0.0,False +latest_final,scenario_045,MI,state_refundable_credits,2.17.0,760.7887573242188,760.7887573242188,760.7887573242188,False,0.0,False +latest_final,scenario_045,MI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,federal_income_tax_before_refundable_credits,2.17.0,499.71484375,499.71484375,499.71484375,False,0.0,False +latest_final,scenario_046,OK,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,payroll_tax,2.17.0,7038.0,7038.0,7038.0,False,0.0,False +latest_final,scenario_046,OK,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,state_income_tax_before_refundable_credits,2.17.0,2739.135009765625,2739.135009765625,2739.135009765625,False,0.0,False +latest_final,scenario_046,OK,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,payroll_tax,2.17.0,585.6839599609375,585.6839599609375,585.6839599609375,False,0.0,False +latest_final,scenario_048,OH,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_048,OH,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,federal_income_tax_before_refundable_credits,2.17.0,30543.908203125,30543.908203125,30543.908203125,False,0.0,False +latest_final,scenario_049,NH,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,payroll_tax,2.17.0,15134.5,15134.5,15134.5,False,0.0,False +latest_final,scenario_049,NH,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,federal_income_tax_before_refundable_credits,2.17.0,2620.0,2620.0,2620.0,False,0.0,False +latest_final,scenario_051,LA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,payroll_tax,2.17.0,3060.0,3060.0,3060.0,False,0.0,False +latest_final,scenario_051,LA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,state_income_tax_before_refundable_credits,2.17.0,820.3499755859375,814.9500122070312,820.3499755859375,True,0.0,False +latest_final,scenario_051,LA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,federal_income_tax_before_refundable_credits,2.17.0,104211.40625,104211.40625,104211.40625,False,0.0,False +latest_final,scenario_052,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,payroll_tax,2.17.0,20939.0,20939.0,20939.0,False,0.0,False +latest_final,scenario_052,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,federal_income_tax_before_refundable_credits,2.17.0,5903.109375,5903.109375,5903.109375,False,0.0,False +latest_final,scenario_053,ID,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,payroll_tax,2.17.0,5123.10400390625,5123.10400390625,5123.10400390625,False,0.0,False +latest_final,scenario_053,ID,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,state_income_tax_before_refundable_credits,2.17.0,2435.2783203125,2435.2783203125,2441.05712890625,True,5.77880859375,True +latest_final,scenario_053,ID,state_refundable_credits,2.17.0,155.0,155.0,155.0,False,0.0,False +latest_final,scenario_053,ID,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,federal_refundable_credits,2.17.0,8365.55859375,8365.55859375,8365.55859375,False,0.0,False +latest_final,scenario_054,NC,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_early_head_start_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_wic_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,self_employment_tax,2.17.0,2416.15283203125,2416.15283203125,2416.15283203125,False,0.0,False +latest_final,scenario_054,NC,snap,2.17.0,6060.0,6125.68896484375,6060.0,True,0.0,False +latest_final,scenario_054,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,federal_income_tax_before_refundable_credits,2.17.0,25238.578125,25238.578125,25238.578125,False,0.0,False +latest_final,scenario_055,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,payroll_tax,2.17.0,6398.08837890625,6398.08837890625,6398.08837890625,False,0.0,False +latest_final,scenario_055,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_055,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_056,NJ,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,self_employment_tax,2.17.0,613.2224731445312,613.2224731445312,613.2224731445312,False,0.0,False +latest_final,scenario_056,NJ,snap,2.17.0,1140.0,1140.0,95.0,True,-1045.0,True +latest_final,scenario_056,NJ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,state_refundable_credits,2.17.0,265.6000061035156,265.6000061035156,265.6000061035156,False,0.0,False +latest_final,scenario_056,NJ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,federal_refundable_credits,2.17.0,664.0,664.0,664.0,False,0.0,False +latest_final,scenario_057,LA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_057,LA,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_057,LA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,self_employment_tax,2.17.0,1469.4732666015625,1469.4732666015625,1469.4732666015625,False,0.0,False +latest_final,scenario_057,LA,snap,2.17.0,2669.217041015625,2669.217041015625,2628.0,True,-41.217041015625,True +latest_final,scenario_057,LA,ssi,2.17.0,7231.00146484375,7231.00146484375,7231.00146484375,False,0.0,False +latest_final,scenario_057,LA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,state_refundable_credits,2.17.0,33.20000076293945,33.20000076293945,33.20000076293945,False,0.0,False +latest_final,scenario_057,LA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_059,FL,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_059,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,payroll_tax,2.17.0,229.5,229.5,229.5,False,0.0,False +latest_final,scenario_060,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_060,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_062,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,federal_income_tax_before_refundable_credits,2.17.0,4439.28955078125,4439.28955078125,4439.28955078125,False,0.0,False +latest_final,scenario_064,WI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,payroll_tax,2.17.0,7443.0673828125,7443.0673828125,7443.0673828125,False,0.0,False +latest_final,scenario_064,WI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,state_income_tax_before_refundable_credits,2.17.0,4605.9970703125,4605.9970703125,4598.47607421875,True,-7.52099609375,True +latest_final,scenario_064,WI,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,federal_refundable_credits,2.17.0,39.779998779296875,39.779998779296875,39.779998779296875,False,0.0,False +latest_final,scenario_066,VA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,payroll_tax,2.17.0,39.779998779296875,39.779998779296875,39.779998779296875,False,0.0,False +latest_final,scenario_066,VA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_066,VA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,snap,2.17.0,3576.0,3596.039794921875,3576.0,True,0.0,False +latest_final,scenario_066,VA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,state_refundable_credits,2.17.0,7.955999851226807,7.955999851226807,7.955999851226807,False,0.0,False +latest_final,scenario_066,VA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,federal_income_tax_before_refundable_credits,2.17.0,5616.240234375,5616.240234375,5616.240234375,False,0.0,False +latest_final,scenario_067,IN,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,payroll_tax,2.17.0,6300.5400390625,6300.5400390625,6300.5400390625,False,0.0,False +latest_final,scenario_067,IN,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_067,IN,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,state_income_tax_before_refundable_credits,2.17.0,2486.908935546875,2486.908935546875,2486.908935546875,False,0.0,False +latest_final,scenario_067,IN,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,federal_income_tax_before_refundable_credits,2.17.0,1910.0238037109375,1910.0238037109375,1910.0238037109375,False,0.0,False +latest_final,scenario_068,MD,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,payroll_tax,2.17.0,2607.39013671875,2607.39013671875,2607.39013671875,False,0.0,False +latest_final,scenario_068,MD,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,state_income_tax_before_refundable_credits,2.17.0,1255.3427734375,1252.9677734375,1255.3427734375,True,0.0,False +latest_final,scenario_068,MD,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,federal_income_tax_before_refundable_credits,2.17.0,1979.9605712890625,1979.9605712890625,1979.9605712890625,False,0.0,False +latest_final,scenario_070,IL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,payroll_tax,2.17.0,2907.0,2907.0,2907.0,False,0.0,False +latest_final,scenario_070,IL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,state_income_tax_before_refundable_credits,2.17.0,1650.561279296875,1650.561279296875,1650.561279296875,False,0.0,False +latest_final,scenario_070,IL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_071,NY,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,payroll_tax,2.17.0,1530.0,1530.0,1530.0,False,0.0,False +latest_final,scenario_072,MI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_072,MI,spouse_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_072,MI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,snap,2.17.0,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_073,MI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,federal_income_tax_before_refundable_credits,2.17.0,14777.1455078125,14777.1455078125,14777.1455078125,False,0.0,False +latest_final,scenario_075,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,payroll_tax,2.17.0,968.2120971679688,968.2120971679688,968.2120971679688,False,0.0,False +latest_final,scenario_075,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,federal_income_tax_before_refundable_credits,2.17.0,19611.80078125,19611.80078125,19611.80078125,False,0.0,False +latest_final,scenario_076,ID,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,payroll_tax,2.17.0,12240.0,12240.0,12240.0,False,0.0,False +latest_final,scenario_076,ID,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,state_income_tax_before_refundable_credits,2.17.0,6818.34423828125,6806.78662109375,6818.34423828125,True,0.0,False +latest_final,scenario_076,ID,state_refundable_credits,2.17.0,465.0,465.0,465.0,False,0.0,False +latest_final,scenario_076,ID,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,federal_income_tax_before_refundable_credits,2.17.0,690.6175537109375,690.6175537109375,690.6175537109375,False,0.0,False +latest_final,scenario_077,LA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,payroll_tax,2.17.0,1989.472412109375,1989.472412109375,1989.472412109375,False,0.0,False +latest_final,scenario_077,LA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,state_income_tax_before_refundable_credits,2.17.0,305.13525390625,305.13525390625,305.13525390625,False,0.0,False +latest_final,scenario_077,LA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,federal_income_tax_before_refundable_credits,2.17.0,24164.45703125,24772.693359375,24164.45703125,True,0.0,False +latest_final,scenario_078,MD,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,payroll_tax,2.17.0,14317.322265625,14317.322265625,14317.322265625,False,0.0,False +latest_final,scenario_078,MD,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,state_income_tax_before_refundable_credits,2.17.0,6936.87646484375,6936.33740234375,6936.87646484375,False,0.0,False +latest_final,scenario_078,MD,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_079,AZ,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,snap,2.17.0,2376.0,2428.017333984375,2376.0,True,0.0,False +latest_final,scenario_079,AZ,ssi,2.17.0,10572.0,10572.0,10572.0,False,0.0,False +latest_final,scenario_079,AZ,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,state_refundable_credits,2.17.0,50.0,50.0,50.0,False,0.0,False +latest_final,scenario_079,AZ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_080,PA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,snap,2.17.0,3596.039794921875,3596.039794921875,3576.0,True,-20.039794921875,True +latest_final,scenario_080,PA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,federal_income_tax_before_refundable_credits,2.17.0,26932.2734375,26932.2734375,26932.2734375,False,0.0,False +latest_final,scenario_081,MA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,payroll_tax,2.17.0,14192.662109375,14192.662109375,14192.662109375,False,0.0,False +latest_final,scenario_081,MA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,state_income_tax_before_refundable_credits,2.17.0,8238.40625,8238.40625,8238.40625,False,0.0,False +latest_final,scenario_081,MA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,federal_income_tax_before_refundable_credits,2.17.0,9563.052734375,9563.052734375,9577.90234375,True,14.849609375,True +latest_final,scenario_082,NY,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,payroll_tax,2.17.0,8108.02734375,8108.02734375,8108.02734375,False,0.0,False +latest_final,scenario_082,NY,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,state_income_tax_before_refundable_credits,2.17.0,5598.55224609375,5598.55224609375,5598.55224609375,False,0.0,False +latest_final,scenario_082,NY,state_refundable_credits,2.17.0,667.0,650.5,667.0,True,0.0,False +latest_final,scenario_082,NY,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,payroll_tax,2.17.0,539.3250122070312,539.3250122070312,539.3250122070312,False,0.0,False +latest_final,scenario_083,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,federal_income_tax_before_refundable_credits,2.17.0,2322.760009765625,2322.760009765625,2322.760009765625,False,0.0,False +latest_final,scenario_085,PA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_085,PA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,federal_income_tax_before_refundable_credits,2.17.0,3255.5478515625,3255.5478515625,3255.5478515625,False,0.0,False +latest_final,scenario_086,GA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,payroll_tax,2.17.0,4743.0,4743.0,4743.0,False,0.0,False +latest_final,scenario_086,GA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,state_income_tax_before_refundable_credits,2.17.0,2078.958740234375,2078.958740234375,2078.958740234375,False,0.0,False +latest_final,scenario_086,GA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,payroll_tax,2.17.0,1421.1448974609375,1421.1448974609375,1421.1448974609375,False,0.0,False +latest_final,scenario_088,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,federal_income_tax_before_refundable_credits,2.17.0,6819.7119140625,6819.7119140625,6819.7119140625,False,0.0,False +latest_final,scenario_089,NC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,payroll_tax,2.17.0,5039.91015625,5039.91015625,5039.91015625,False,0.0,False +latest_final,scenario_089,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,self_employment_tax,2.17.0,4286.24072265625,4286.24072265625,4286.24072265625,False,0.0,False +latest_final,scenario_089,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,state_income_tax_before_refundable_credits,2.17.0,3219.935302734375,3219.935302734375,3219.935302734375,False,0.0,False +latest_final,scenario_089,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,payroll_tax,2.17.0,535.5,535.5,535.5,False,0.0,False +latest_final,scenario_090,KS,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,federal_income_tax_before_refundable_credits,2.17.0,1349.998779296875,1349.998779296875,1349.998779296875,False,0.0,False +latest_final,scenario_091,WI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,payroll_tax,2.17.0,2065.5,2065.5,2065.5,False,0.0,False +latest_final,scenario_091,WI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,state_income_tax_before_refundable_credits,2.17.0,843.6614990234375,843.6614990234375,895.8131103515625,True,52.151611328125,True +latest_final,scenario_091,WI,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_092,AL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,federal_income_tax_before_refundable_credits,2.17.0,8628.990234375,8628.990234375,8628.990234375,False,0.0,False +latest_final,scenario_093,MO,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,payroll_tax,2.17.0,12469.5,12469.5,12469.5,False,0.0,False +latest_final,scenario_093,MO,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,state_income_tax_before_refundable_credits,2.17.0,3388.24560546875,3389.65087890625,3388.24560546875,True,0.0,False +latest_final,scenario_093,MO,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_095,NJ,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,self_employment_tax,2.17.0,593.2291259765625,593.2291259765625,593.2291259765625,False,0.0,False +latest_final,scenario_095,NJ,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_098,IL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,federal_income_tax_before_refundable_credits,2.17.0,10679.75,10679.75,10679.75,False,0.0,False +latest_final,scenario_099,CA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_099,CA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,payroll_tax,2.17.0,14767.5,14767.5,14767.5,False,0.0,False +latest_final,scenario_099,CA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,state_income_tax_before_refundable_credits,2.17.0,4493.7431640625,4493.7431640625,4640.77734375,True,147.0341796875,True +latest_final,scenario_099,CA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,federal_refundable_credits,2.17.0,2878.0966796875,2878.0966796875,2878.0966796875,False,0.0,False +latest_final,scenario_100,MT,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,payroll_tax,2.17.0,452.47613525390625,452.47613525390625,452.47613525390625,False,0.0,False +latest_final,scenario_100,MT,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_head_start_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,snap,2.17.0,8625.8896484375,8625.8896484375,8556.0,True,-69.8896484375,True +latest_final,scenario_100,MT,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,state_refundable_credits,2.17.0,473.1777038574219,473.1777038574219,473.1777038574219,False,0.0,False +latest_final,scenario_100,MT,tanf,2.17.0,6064.958984375,6064.958984375,6064.958984375,False,0.0,False +latest_final,scenario_101,TX,federal_income_tax_before_refundable_credits,2.17.0,13418.599609375,13418.599609375,13418.599609375,False,0.0,False +latest_final,scenario_101,TX,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,payroll_tax,2.17.0,7879.5,7879.5,7879.5,False,0.0,False +latest_final,scenario_101,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,federal_income_tax_before_refundable_credits,2.17.0,6104.09619140625,6104.09619140625,6104.09619140625,False,0.0,False +latest_final,scenario_102,NC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,payroll_tax,2.17.0,6670.7998046875,6670.7998046875,6670.7998046875,False,0.0,False +latest_final,scenario_102,NC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,state_income_tax_before_refundable_credits,2.17.0,2461.869873046875,2461.869873046875,2461.869873046875,False,0.0,False +latest_final,scenario_102,NC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_104,NY,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,state_refundable_credits,2.17.0,0.0,375.0,0.0,True,0.0,False +latest_final,scenario_104,NY,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,payroll_tax,2.17.0,486.9607238769531,486.9607238769531,486.9607238769531,False,0.0,False +latest_final,scenario_107,OH,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_107,OH,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,payroll_tax,2.17.0,585.6839599609375,585.6839599609375,585.6839599609375,False,0.0,False +latest_final,scenario_108,WI,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_108,WI,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,snap,2.17.0,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_108,WI,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,state_refundable_credits,2.17.0,271.3916015625,271.3916015625,271.3916015625,False,0.0,False +latest_final,scenario_108,WI,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,federal_refundable_credits,2.17.0,12038.0859375,12038.0859375,12038.0859375,False,0.0,False +latest_final,scenario_109,FL,free_school_meals_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_early_head_start_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child2_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child3_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_wic_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,self_employment_tax,2.17.0,4238.86474609375,4238.86474609375,4238.86474609375,False,0.0,False +latest_final,scenario_109,FL,snap,2.17.0,7932.0,8020.55419921875,7932.0,True,0.0,False +latest_final,scenario_109,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,federal_income_tax_before_refundable_credits,2.17.0,23897.443359375,23897.443359375,23897.443359375,False,0.0,False +latest_final,scenario_110,OH,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,payroll_tax,2.17.0,7650.0,7650.0,7650.0,False,0.0,False +latest_final,scenario_110,OH,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,state_income_tax_before_refundable_credits,2.17.0,4057.47216796875,4057.47216796875,4057.47216796875,False,0.0,False +latest_final,scenario_110,OH,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_111,WA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,federal_refundable_credits,2.17.0,453.0960083007813,453.0960083007813,453.09600830078125,False,-5.684341886080802e-14,False +latest_final,scenario_112,TX,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,payroll_tax,2.17.0,453.0960083007813,453.0960083007813,453.09600830078125,False,-5.684341886080802e-14,False +latest_final,scenario_112,TX,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,snap,2.17.0,287.68316650390625,287.68316650390625,0.0,True,-287.68316650390625,True +latest_final,scenario_112,TX,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,federal_income_tax_before_refundable_credits,2.17.0,10729.6123046875,10729.6123046875,10729.6123046875,False,0.0,False +latest_final,scenario_114,VA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,payroll_tax,2.17.0,1736.550048828125,1736.550048828125,1736.550048828125,False,0.0,False +latest_final,scenario_114,VA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_114,VA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,state_income_tax_before_refundable_credits,2.17.0,3514.14501953125,3514.14501953125,3514.14501953125,False,0.0,False +latest_final,scenario_114,VA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_115,AL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,state_income_tax_before_refundable_credits,2.17.0,4.679999828338623,4.679999828338623,4.679999828338623,False,0.0,False +latest_final,scenario_115,AL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,federal_income_tax_before_refundable_credits,2.17.0,24961.33984375,24391.796875,24961.33984375,True,0.0,False +latest_final,scenario_117,AR,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,payroll_tax,2.17.0,15044.787109375,15044.787109375,15044.787109375,False,0.0,False +latest_final,scenario_117,AR,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,state_income_tax_before_refundable_credits,2.17.0,7984.53369140625,7984.05419921875,7984.53369140625,False,0.0,False +latest_final,scenario_117,AR,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,head_medicaid_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_118,NY,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_118,NY,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,snap,2.17.0,2903.9404296875,2903.9404296875,2868.0,True,-35.9404296875,True +latest_final,scenario_118,NY,ssi,2.17.0,9368.0,9368.0,9368.0,False,0.0,False +latest_final,scenario_118,NY,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,state_refundable_credits,2.17.0,375.0,375.0,375.0,False,0.0,False +latest_final,scenario_118,NY,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,federal_refundable_credits,2.17.0,2854.0087890625,2854.0087890625,2854.0087890625,False,0.0,False +latest_final,scenario_119,VA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,payroll_tax,2.17.0,4207.5,4207.5,4207.5,False,0.0,False +latest_final,scenario_119,VA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,state_income_tax_before_refundable_credits,2.17.0,1859.0943603515625,1859.0943603515625,1859.0943603515625,False,0.0,False +latest_final,scenario_119,VA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,federal_income_tax_before_refundable_credits,2.17.0,40021.81640625,40021.81640625,40416.66796875,True,394.8515625,True +latest_final,scenario_120,CT,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,payroll_tax,2.17.0,13496.1552734375,13496.1552734375,13496.1552734375,False,0.0,False +latest_final,scenario_120,CT,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_120,CT,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,state_income_tax_before_refundable_credits,2.17.0,11917.18359375,11917.18359375,11917.18359375,False,0.0,False +latest_final,scenario_120,CT,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,federal_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,payroll_tax,2.17.0,1963.3724365234375,1963.3724365234375,1963.3724365234375,False,0.0,False +latest_final,scenario_121,SC,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,state_income_tax_before_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,federal_income_tax_before_refundable_credits,2.17.0,4746.66162109375,4746.66162109375,4746.66162109375,False,0.0,False +latest_final,scenario_122,MN,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,payroll_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_medicare_eligible,2.17.0,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_122,MN,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,state_income_tax_before_refundable_credits,2.17.0,1025.54150390625,1028.216552734375,1025.54150390625,True,0.0,False +latest_final,scenario_122,MN,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,federal_income_tax_before_refundable_credits,2.17.0,5200.240234375,5200.240234375,5200.240234375,False,0.0,False +latest_final,scenario_123,PA,federal_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,free_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,local_income_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,payroll_tax,2.17.0,11194.0,11194.0,11194.0,False,0.0,False +latest_final,scenario_123,PA,head_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_chip_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_early_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_head_start_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_medicaid_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_medicare_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_wic_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,reduced_price_school_meals_eligible,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,self_employment_tax,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,snap,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,ssi,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,state_income_tax_before_refundable_credits,2.17.0,3070.061279296875,3070.061279296875,3070.061279296875,False,0.0,False +latest_final,scenario_123,PA,state_refundable_credits,2.17.0,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,tanf,2.17.0,0.0,0.0,0.0,False,0.0,False diff --git a/reference_audit/2026-09-28/verification/latest_final_2170.log b/reference_audit/2026-09-28/verification/latest_final_2170.log new file mode 100644 index 00000000..00671e1f --- /dev/null +++ b/reference_audit/2026-09-28/verification/latest_final_2170.log @@ -0,0 +1,22 @@ +policyengine-us 2.17.0 + scenario_id state variable frozen recomputed delta +scenario_005 CA federal_income_tax_before_refundable_credits 106505.898438 107833.156250 1327.257812 +scenario_005 CA state_income_tax_before_refundable_credits 41051.511719 41267.011719 215.500000 +scenario_007 ID state_income_tax_before_refundable_credits 755.778320 761.557007 5.778687 +scenario_020 TX federal_income_tax_before_refundable_credits 68056.710938 68334.734375 278.023438 +scenario_022 CA state_income_tax_before_refundable_credits 2439.650146 2505.870117 66.219971 +scenario_023 CA snap 461.339722 408.000000 -53.339722 +scenario_042 WI state_income_tax_before_refundable_credits 284.740906 464.182800 179.441895 +scenario_053 ID state_income_tax_before_refundable_credits 2435.278320 2441.057129 5.778809 +scenario_056 NJ snap 1140.000000 95.000000 -1045.000000 +scenario_057 LA snap 2669.217041 2628.000000 -41.217041 +scenario_064 WI state_income_tax_before_refundable_credits 4605.997070 4598.476074 -7.520996 +scenario_080 PA snap 3596.039795 3576.000000 -20.039795 +scenario_082 NY federal_income_tax_before_refundable_credits 9563.052734 9577.902344 14.849609 +scenario_091 WI state_income_tax_before_refundable_credits 843.661499 895.813110 52.151611 +scenario_099 CA state_income_tax_before_refundable_credits 4493.743164 4640.777344 147.034180 +scenario_100 MT snap 8625.889648 8556.000000 -69.889648 +scenario_112 TX snap 287.683167 0.000000 -287.683167 +scenario_118 NY snap 2903.940430 2868.000000 -35.940430 +scenario_120 CT federal_income_tax_before_refundable_credits 40021.816406 40416.667969 394.851562 +SUMMARY fix=latest_final outputs=1984 moved=19 small_nonzero_deltas=0 diff --git a/reference_audit/2026-09-28/verification/latest_final_2171.csv b/reference_audit/2026-09-28/verification/latest_final_2171.csv new file mode 100644 index 00000000..3f72cae7 --- /dev/null +++ b/reference_audit/2026-09-28/verification/latest_final_2171.csv @@ -0,0 +1,1985 @@ +fix,scenario_id,state,variable,engine,frozen,v11_1755,recomputed,moved_vs_v11,delta,moved +latest_final,scenario_000,TX,federal_income_tax_before_refundable_credits,2.17.1,2906.45361328125,2883.49365234375,2906.45361328125,True,0.0,False +latest_final,scenario_000,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_000,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_000,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,payroll_tax,2.17.1,2223.393798828125,2223.393798828125,2223.393798828125,False,0.0,False +latest_final,scenario_001,VA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_001,VA,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_001,VA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_001,VA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_002,WA,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_002,WA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_002,WA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,federal_income_tax_before_refundable_credits,2.17.1,22154.69921875,22154.69921875,22154.69921875,False,0.0,False +latest_final,scenario_003,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_003,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_004,NY,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,federal_income_tax_before_refundable_credits,2.17.1,106505.8984375,106505.8984375,107833.15625,True,1327.2578125,True +latest_final,scenario_005,CA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,payroll_tax,2.17.1,36044.0,36044.0,36044.0,False,0.0,False +latest_final,scenario_005,CA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,state_income_tax_before_refundable_credits,2.17.1,41051.51171875,41051.51171875,41267.01171875,True,215.5,True +latest_final,scenario_005,CA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_005,CA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,federal_income_tax_before_refundable_credits,2.17.1,3707.695068359375,3707.695068359375,3707.695068359375,False,0.0,False +latest_final,scenario_007,ID,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_007,ID,state_income_tax_before_refundable_credits,2.17.1,755.7783203125,755.7783203125,761.5570068359375,True,5.7786865234375,True +latest_final,scenario_007,ID,state_refundable_credits,2.17.1,155.0,155.0,155.0,False,0.0,False +latest_final,scenario_007,ID,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,federal_refundable_credits,2.17.1,12433.611328125,12433.611328125,12433.611328125,False,0.0,False +latest_final,scenario_008,NJ,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,payroll_tax,2.17.1,2162.760009765625,2162.760009765625,2162.760009765625,False,0.0,False +latest_final,scenario_008,NJ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_early_head_start_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_head_start_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child6_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child3_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child4_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child5_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,child6_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child3_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child4_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child5_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,child6_wic_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_008,NJ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,self_employment_tax,2.17.1,565.1820068359375,565.1820068359375,565.1820068359375,False,0.0,False +latest_final,scenario_008,NJ,snap,2.17.1,15108.0,15246.9052734375,15108.0,True,0.0,False +latest_final,scenario_008,NJ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_008,NJ,state_income_tax_before_refundable_credits,2.17.1,264.80999755859375,264.80999755859375,264.80999755859375,False,0.0,False +latest_final,scenario_008,NJ,state_refundable_credits,2.17.1,5842.39990234375,5342.39990234375,5842.39990234375,True,0.0,False +latest_final,scenario_008,NJ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,federal_income_tax_before_refundable_credits,2.17.1,8348.0,8348.0,8348.0,False,0.0,False +latest_final,scenario_009,NC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,payroll_tax,2.17.1,4590.0,4590.0,4590.0,False,0.0,False +latest_final,scenario_009,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,state_income_tax_before_refundable_credits,2.17.1,2190.510009765625,2190.510009765625,2190.510009765625,False,0.0,False +latest_final,scenario_009,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_009,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,federal_refundable_credits,2.17.1,6127.0,6127.0,6127.0,False,0.0,False +latest_final,scenario_012,MS,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,payroll_tax,2.17.1,1683.0,1683.0,1683.0,False,0.0,False +latest_final,scenario_012,MS,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,spouse_wic_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_012,MS,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,snap,2.17.1,4884.0,4952.08935546875,4884.0,True,0.0,False +latest_final,scenario_012,MS,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_012,MS,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_013,AZ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,snap,2.17.1,240.0,0.0,240.0,True,0.0,False +latest_final,scenario_013,AZ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_013,AZ,state_refundable_credits,2.17.1,25.0,25.0,25.0,False,0.0,False +latest_final,scenario_013,AZ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,federal_income_tax_before_refundable_credits,2.17.1,6311.31787109375,6311.31787109375,6311.31787109375,False,0.0,False +latest_final,scenario_014,WV,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,payroll_tax,2.17.1,6802.96484375,6802.96484375,6802.96484375,False,0.0,False +latest_final,scenario_014,WV,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,state_income_tax_before_refundable_credits,2.17.1,3092.186279296875,3092.186279296875,3092.186279296875,False,0.0,False +latest_final,scenario_014,WV,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_014,WV,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,federal_income_tax_before_refundable_credits,2.17.1,2969.1259765625,2969.1259765625,2969.1259765625,False,0.0,False +latest_final,scenario_015,IN,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,payroll_tax,2.17.1,3442.5,3442.5,3442.5,False,0.0,False +latest_final,scenario_015,IN,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,state_income_tax_before_refundable_credits,2.17.1,1236.3267822265625,1236.3267822265625,1236.3267822265625,False,0.0,False +latest_final,scenario_015,IN,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_015,IN,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,payroll_tax,2.17.1,2649.80224609375,2649.80224609375,2649.80224609375,False,0.0,False +latest_final,scenario_016,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_016,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,federal_income_tax_before_refundable_credits,2.17.1,5211.05078125,5211.05078125,5211.05078125,False,0.0,False +latest_final,scenario_018,AZ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,payroll_tax,2.17.1,5202.38232421875,5202.38232421875,5202.38232421875,False,0.0,False +latest_final,scenario_018,AZ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,state_income_tax_before_refundable_credits,2.17.1,1146.05224609375,1146.05224609375,1146.05224609375,False,0.0,False +latest_final,scenario_018,AZ,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_018,AZ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,federal_income_tax_before_refundable_credits,2.17.1,68056.7109375,68056.7109375,68334.734375,True,278.0234375,True +latest_final,scenario_020,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,payroll_tax,2.17.1,16689.0,16689.0,16689.0,False,0.0,False +latest_final,scenario_020,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_020,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_021,MO,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_021,MO,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,federal_income_tax_before_refundable_credits,2.17.1,11113.5703125,11131.3271484375,11113.5703125,True,0.0,False +latest_final,scenario_022,CA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,payroll_tax,2.17.1,4710.83251953125,4710.83251953125,4710.83251953125,False,0.0,False +latest_final,scenario_022,CA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_022,CA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,state_income_tax_before_refundable_credits,2.17.1,2439.650146484375,2439.650146484375,2505.8701171875,True,66.219970703125,True +latest_final,scenario_022,CA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_022,CA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,federal_income_tax_before_refundable_credits,2.17.1,643.43359375,643.43359375,643.43359375,False,0.0,False +latest_final,scenario_023,CA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,payroll_tax,2.17.1,1561.1168212890625,1561.1168212890625,1561.1168212890625,False,0.0,False +latest_final,scenario_023,CA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_023,CA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,snap,2.17.1,461.3397216796875,461.3397216796875,408.0,True,-53.3397216796875,True +latest_final,scenario_023,CA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_023,CA,state_income_tax_before_refundable_credits,2.17.1,12.776718139648438,6.7984771728515625,12.776718139648438,True,0.0,False +latest_final,scenario_023,CA,state_refundable_credits,2.17.1,95.1540069580078,148.3108673095703,95.15400695800781,True,1.4210854715202004e-14,False +latest_final,scenario_023,CA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,federal_income_tax_before_refundable_credits,2.17.1,7031.03564453125,7031.03564453125,7031.03564453125,False,0.0,False +latest_final,scenario_025,OH,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,payroll_tax,2.17.1,4798.48486328125,4798.48486328125,4798.48486328125,False,0.0,False +latest_final,scenario_025,OH,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,state_income_tax_before_refundable_credits,2.17.1,1921.57275390625,1921.57275390625,1921.57275390625,False,0.0,False +latest_final,scenario_025,OH,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_025,OH,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,federal_refundable_credits,2.17.1,2013.4892578125,2013.4892578125,2013.4892578125,False,0.0,False +latest_final,scenario_026,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,payroll_tax,2.17.1,6518.474609375,6518.474609375,6518.474609375,False,0.0,False +latest_final,scenario_026,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,child3_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_026,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,child3_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,state_income_tax_before_refundable_credits,2.17.1,2086.79052734375,2086.79052734375,2086.79052734375,False,0.0,False +latest_final,scenario_026,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_026,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_027,CT,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,snap,2.17.1,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_027,CT,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_027,CT,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,federal_refundable_credits,2.17.1,3274.927978515625,3274.927978515625,3274.927978515625,False,0.0,False +latest_final,scenario_028,PA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,payroll_tax,2.17.1,4632.0,4632.0,4632.0,False,0.0,False +latest_final,scenario_028,PA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,child3_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,reduced_price_school_meals_eligible,2.17.1,0.0,1.0,0.0,True,0.0,False +latest_final,scenario_028,PA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_028,PA,state_income_tax_before_refundable_credits,2.17.1,1842.3070068359375,1842.3070068359375,1842.3070068359375,False,0.0,False +latest_final,scenario_028,PA,state_refundable_credits,2.17.1,62.41279220581055,62.41279220581055,62.41279220581055,False,0.0,False +latest_final,scenario_028,PA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_029,OK,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_029,OK,state_refundable_credits,2.17.1,40.0,40.0,40.0,False,0.0,False +latest_final,scenario_029,OK,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,payroll_tax,2.17.1,994.5,994.5,994.5,False,0.0,False +latest_final,scenario_030,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_030,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,head_wic_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_030,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,snap,2.17.1,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_030,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_030,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_031,CA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_031,CA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_031,CA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,federal_refundable_credits,2.17.1,6127.0,6127.0,6127.0,False,0.0,False +latest_final,scenario_032,MN,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,payroll_tax,2.17.1,2346.10009765625,2346.10009765625,2346.10009765625,False,0.0,False +latest_final,scenario_032,MN,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_032,MN,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_032,MN,state_refundable_credits,2.17.1,2187.60009765625,2187.60009765625,2187.60009765625,False,0.0,False +latest_final,scenario_032,MN,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,federal_income_tax_before_refundable_credits,2.17.1,3818.148193359375,3788.988037109375,3818.148193359375,True,0.0,False +latest_final,scenario_033,SD,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,payroll_tax,2.17.1,6043.5,6043.5,6043.5,False,0.0,False +latest_final,scenario_033,SD,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_033,SD,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_033,SD,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_033,SD,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,federal_income_tax_before_refundable_credits,2.17.1,4926.39990234375,4926.39990234375,4926.39990234375,False,0.0,False +latest_final,scenario_036,NJ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,state_income_tax_before_refundable_credits,2.17.1,1724.155029296875,1724.155029296875,1724.155029296875,False,0.0,False +latest_final,scenario_036,NJ,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_036,NJ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,federal_income_tax_before_refundable_credits,2.17.1,1649.2513427734375,1649.2513427734375,1649.2513427734375,False,0.0,False +latest_final,scenario_037,NC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,payroll_tax,2.17.1,3208.668701171875,3208.668701171875,3208.668701171875,False,0.0,False +latest_final,scenario_037,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,state_income_tax_before_refundable_credits,2.17.1,1148.69921875,1148.69921875,1148.69921875,False,0.0,False +latest_final,scenario_037,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_037,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,federal_refundable_credits,2.17.1,10225.353515625,10225.353515625,10225.353515625,False,0.0,False +latest_final,scenario_038,LA,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,payroll_tax,2.17.1,571.2637329101562,571.2637329101562,571.2637329101562,False,0.0,False +latest_final,scenario_038,LA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_038,LA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,self_employment_tax,2.17.1,2193.612548828125,2193.612548828125,2193.612548828125,False,0.0,False +latest_final,scenario_038,LA,snap,2.17.1,7212.0,7286.9443359375,7212.0,True,0.0,False +latest_final,scenario_038,LA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_038,LA,state_refundable_credits,2.17.1,365.8000183105469,365.8000183105469,365.8000183105469,False,0.0,False +latest_final,scenario_038,LA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,federal_income_tax_before_refundable_credits,2.17.1,1345.510498046875,1345.510498046875,1345.510498046875,False,0.0,False +latest_final,scenario_039,VA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,state_income_tax_before_refundable_credits,2.17.1,514.4984130859375,514.4984130859375,514.4984130859375,False,0.0,False +latest_final,scenario_039,VA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_039,VA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,payroll_tax,2.17.1,2142.0,2142.0,2142.0,False,0.0,False +latest_final,scenario_040,AZ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_040,AZ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_040,AZ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,federal_income_tax_before_refundable_credits,2.17.1,2361.96484375,1979.1583251953125,2361.96484375,True,0.0,False +latest_final,scenario_042,WI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_042,WI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,state_income_tax_before_refundable_credits,2.17.1,284.74090576171875,284.74090576171875,464.18280029296875,True,179.44189453125,True +latest_final,scenario_042,WI,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_042,WI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,payroll_tax,2.17.1,330.7310791015625,330.7310791015625,330.7310791015625,False,0.0,False +latest_final,scenario_043,CO,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_043,CO,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,snap,2.17.1,3576.0,3596.039794921875,3576.0,True,0.0,False +latest_final,scenario_043,CO,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_043,CO,state_refundable_credits,2.17.1,19.0,19.0,19.0,False,0.0,False +latest_final,scenario_043,CO,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_044,KS,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_044,KS,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,federal_income_tax_before_refundable_credits,2.17.1,2173.07080078125,2173.07080078125,2173.07080078125,False,0.0,False +latest_final,scenario_045,MI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,payroll_tax,2.17.1,2775.08251953125,2775.08251953125,2775.08251953125,False,0.0,False +latest_final,scenario_045,MI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,snap,2.17.1,0.0,287.68316650390625,0.0,True,0.0,False +latest_final,scenario_045,MI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_045,MI,state_income_tax_before_refundable_credits,2.17.1,1290.9625244140625,1288.8375244140625,1290.9625244140625,True,0.0,False +latest_final,scenario_045,MI,state_refundable_credits,2.17.1,760.7887573242188,760.7887573242188,760.7887573242188,False,0.0,False +latest_final,scenario_045,MI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,federal_income_tax_before_refundable_credits,2.17.1,499.71484375,499.71484375,499.71484375,False,0.0,False +latest_final,scenario_046,OK,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,payroll_tax,2.17.1,7038.0,7038.0,7038.0,False,0.0,False +latest_final,scenario_046,OK,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,state_income_tax_before_refundable_credits,2.17.1,2739.135009765625,2739.135009765625,2739.135009765625,False,0.0,False +latest_final,scenario_046,OK,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_046,OK,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,payroll_tax,2.17.1,585.6839599609375,585.6839599609375,585.6839599609375,False,0.0,False +latest_final,scenario_048,OH,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_048,OH,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_048,OH,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,federal_income_tax_before_refundable_credits,2.17.1,30543.908203125,30543.908203125,30543.908203125,False,0.0,False +latest_final,scenario_049,NH,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,payroll_tax,2.17.1,15134.5,15134.5,15134.5,False,0.0,False +latest_final,scenario_049,NH,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_049,NH,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,federal_income_tax_before_refundable_credits,2.17.1,2620.0,2620.0,2620.0,False,0.0,False +latest_final,scenario_051,LA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,payroll_tax,2.17.1,3060.0,3060.0,3060.0,False,0.0,False +latest_final,scenario_051,LA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,state_income_tax_before_refundable_credits,2.17.1,820.3499755859375,814.9500122070312,820.3499755859375,True,0.0,False +latest_final,scenario_051,LA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_051,LA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,federal_income_tax_before_refundable_credits,2.17.1,104211.40625,104211.40625,104211.40625,False,0.0,False +latest_final,scenario_052,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,payroll_tax,2.17.1,20939.0,20939.0,20939.0,False,0.0,False +latest_final,scenario_052,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_052,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,federal_income_tax_before_refundable_credits,2.17.1,5903.109375,5903.109375,5903.109375,False,0.0,False +latest_final,scenario_053,ID,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,payroll_tax,2.17.1,5123.10400390625,5123.10400390625,5123.10400390625,False,0.0,False +latest_final,scenario_053,ID,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_053,ID,state_income_tax_before_refundable_credits,2.17.1,2435.2783203125,2435.2783203125,2441.05712890625,True,5.77880859375,True +latest_final,scenario_053,ID,state_refundable_credits,2.17.1,155.0,155.0,155.0,False,0.0,False +latest_final,scenario_053,ID,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,federal_refundable_credits,2.17.1,8365.55859375,8365.55859375,8365.55859375,False,0.0,False +latest_final,scenario_054,NC,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_early_head_start_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,child2_wic_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_054,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,self_employment_tax,2.17.1,2416.15283203125,2416.15283203125,2416.15283203125,False,0.0,False +latest_final,scenario_054,NC,snap,2.17.1,6060.0,6125.68896484375,6060.0,True,0.0,False +latest_final,scenario_054,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_054,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,federal_income_tax_before_refundable_credits,2.17.1,25238.578125,25238.578125,25238.578125,False,0.0,False +latest_final,scenario_055,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,payroll_tax,2.17.1,6398.08837890625,6398.08837890625,6398.08837890625,False,0.0,False +latest_final,scenario_055,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_055,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_055,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_056,NJ,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,self_employment_tax,2.17.1,613.2224731445312,613.2224731445312,613.2224731445312,False,0.0,False +latest_final,scenario_056,NJ,snap,2.17.1,1140.0,1140.0,95.0,True,-1045.0,True +latest_final,scenario_056,NJ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_056,NJ,state_refundable_credits,2.17.1,265.6000061035156,265.6000061035156,265.6000061035156,False,0.0,False +latest_final,scenario_056,NJ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,federal_refundable_credits,2.17.1,664.0,664.0,664.0,False,0.0,False +latest_final,scenario_057,LA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_057,LA,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_057,LA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,self_employment_tax,2.17.1,1469.4732666015625,1469.4732666015625,1469.4732666015625,False,0.0,False +latest_final,scenario_057,LA,snap,2.17.1,2669.217041015625,2669.217041015625,2628.0,True,-41.217041015625,True +latest_final,scenario_057,LA,ssi,2.17.1,7231.00146484375,7231.00146484375,7231.00146484375,False,0.0,False +latest_final,scenario_057,LA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_057,LA,state_refundable_credits,2.17.1,33.20000076293945,33.20000076293945,33.20000076293945,False,0.0,False +latest_final,scenario_057,LA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_059,FL,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_059,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_059,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,payroll_tax,2.17.1,229.5,229.5,229.5,False,0.0,False +latest_final,scenario_060,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_060,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_060,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_062,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_062,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,federal_income_tax_before_refundable_credits,2.17.1,4439.28955078125,4439.28955078125,4439.28955078125,False,0.0,False +latest_final,scenario_064,WI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,payroll_tax,2.17.1,7443.0673828125,7443.0673828125,7443.0673828125,False,0.0,False +latest_final,scenario_064,WI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,dependent2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,state_income_tax_before_refundable_credits,2.17.1,4605.9970703125,4605.9970703125,4598.47607421875,True,-7.52099609375,True +latest_final,scenario_064,WI,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_064,WI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,federal_refundable_credits,2.17.1,39.779998779296875,39.779998779296875,39.779998779296875,False,0.0,False +latest_final,scenario_066,VA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,payroll_tax,2.17.1,39.779998779296875,39.779998779296875,39.779998779296875,False,0.0,False +latest_final,scenario_066,VA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_066,VA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,snap,2.17.1,3576.0,3596.039794921875,3576.0,True,0.0,False +latest_final,scenario_066,VA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_066,VA,state_refundable_credits,2.17.1,7.955999851226807,7.955999851226807,7.955999851226807,False,0.0,False +latest_final,scenario_066,VA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,federal_income_tax_before_refundable_credits,2.17.1,5616.240234375,5616.240234375,5616.240234375,False,0.0,False +latest_final,scenario_067,IN,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,payroll_tax,2.17.1,6300.5400390625,6300.5400390625,6300.5400390625,False,0.0,False +latest_final,scenario_067,IN,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_067,IN,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,dependent1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,state_income_tax_before_refundable_credits,2.17.1,2486.908935546875,2486.908935546875,2486.908935546875,False,0.0,False +latest_final,scenario_067,IN,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_067,IN,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,federal_income_tax_before_refundable_credits,2.17.1,1910.0238037109375,1910.0238037109375,1910.0238037109375,False,0.0,False +latest_final,scenario_068,MD,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,payroll_tax,2.17.1,2607.39013671875,2607.39013671875,2607.39013671875,False,0.0,False +latest_final,scenario_068,MD,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,state_income_tax_before_refundable_credits,2.17.1,1255.3427734375,1252.9677734375,1255.3427734375,True,0.0,False +latest_final,scenario_068,MD,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_068,MD,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,federal_income_tax_before_refundable_credits,2.17.1,1979.9605712890625,1979.9605712890625,1979.9605712890625,False,0.0,False +latest_final,scenario_070,IL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,payroll_tax,2.17.1,2907.0,2907.0,2907.0,False,0.0,False +latest_final,scenario_070,IL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,state_income_tax_before_refundable_credits,2.17.1,1650.561279296875,1650.561279296875,1650.561279296875,False,0.0,False +latest_final,scenario_070,IL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_070,IL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_071,NY,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_071,NY,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,payroll_tax,2.17.1,1530.0,1530.0,1530.0,False,0.0,False +latest_final,scenario_072,MI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_072,MI,spouse_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_072,MI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_072,MI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,snap,2.17.1,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_073,MI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_073,MI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_074,LA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,federal_income_tax_before_refundable_credits,2.17.1,14777.1455078125,14777.1455078125,14777.1455078125,False,0.0,False +latest_final,scenario_075,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,payroll_tax,2.17.1,968.2120971679688,968.2120971679688,968.2120971679688,False,0.0,False +latest_final,scenario_075,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_075,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,federal_income_tax_before_refundable_credits,2.17.1,19611.80078125,19611.80078125,19611.80078125,False,0.0,False +latest_final,scenario_076,ID,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,payroll_tax,2.17.1,12240.0,12240.0,12240.0,False,0.0,False +latest_final,scenario_076,ID,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_076,ID,state_income_tax_before_refundable_credits,2.17.1,6818.34423828125,6806.78662109375,6818.34423828125,True,0.0,False +latest_final,scenario_076,ID,state_refundable_credits,2.17.1,465.0,465.0,465.0,False,0.0,False +latest_final,scenario_076,ID,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,federal_income_tax_before_refundable_credits,2.17.1,690.6175537109375,690.6175537109375,690.6175537109375,False,0.0,False +latest_final,scenario_077,LA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,payroll_tax,2.17.1,1989.472412109375,1989.472412109375,1989.472412109375,False,0.0,False +latest_final,scenario_077,LA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,state_income_tax_before_refundable_credits,2.17.1,305.13525390625,305.13525390625,305.13525390625,False,0.0,False +latest_final,scenario_077,LA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_077,LA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,federal_income_tax_before_refundable_credits,2.17.1,24164.45703125,24772.693359375,24164.45703125,True,0.0,False +latest_final,scenario_078,MD,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,payroll_tax,2.17.1,14317.322265625,14317.322265625,14317.322265625,False,0.0,False +latest_final,scenario_078,MD,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,state_income_tax_before_refundable_credits,2.17.1,6936.87646484375,6936.33740234375,6936.87646484375,False,0.0,False +latest_final,scenario_078,MD,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_078,MD,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_079,AZ,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,snap,2.17.1,2376.0,2428.017333984375,2376.0,True,0.0,False +latest_final,scenario_079,AZ,ssi,2.17.1,10572.0,10572.0,10572.0,False,0.0,False +latest_final,scenario_079,AZ,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_079,AZ,state_refundable_credits,2.17.1,50.0,50.0,50.0,False,0.0,False +latest_final,scenario_079,AZ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_080,PA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,snap,2.17.1,3596.039794921875,3596.039794921875,3576.0,True,-20.039794921875,True +latest_final,scenario_080,PA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_080,PA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,federal_income_tax_before_refundable_credits,2.17.1,26932.2734375,26932.2734375,26932.2734375,False,0.0,False +latest_final,scenario_081,MA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,payroll_tax,2.17.1,14192.662109375,14192.662109375,14192.662109375,False,0.0,False +latest_final,scenario_081,MA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,state_income_tax_before_refundable_credits,2.17.1,8238.40625,8238.40625,8238.40625,False,0.0,False +latest_final,scenario_081,MA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_081,MA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,federal_income_tax_before_refundable_credits,2.17.1,9563.052734375,9563.052734375,9577.90234375,True,14.849609375,True +latest_final,scenario_082,NY,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,payroll_tax,2.17.1,8108.02734375,8108.02734375,8108.02734375,False,0.0,False +latest_final,scenario_082,NY,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_082,NY,state_income_tax_before_refundable_credits,2.17.1,5598.55224609375,5598.55224609375,5598.55224609375,False,0.0,False +latest_final,scenario_082,NY,state_refundable_credits,2.17.1,667.0,650.5,667.0,True,0.0,False +latest_final,scenario_082,NY,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,payroll_tax,2.17.1,539.3250122070312,539.3250122070312,539.3250122070312,False,0.0,False +latest_final,scenario_083,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_083,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_084,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,federal_income_tax_before_refundable_credits,2.17.1,2322.760009765625,2322.760009765625,2322.760009765625,False,0.0,False +latest_final,scenario_085,PA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_085,PA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_085,PA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,federal_income_tax_before_refundable_credits,2.17.1,3255.5478515625,3255.5478515625,3255.5478515625,False,0.0,False +latest_final,scenario_086,GA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,payroll_tax,2.17.1,4743.0,4743.0,4743.0,False,0.0,False +latest_final,scenario_086,GA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,state_income_tax_before_refundable_credits,2.17.1,2078.958740234375,2078.958740234375,2078.958740234375,False,0.0,False +latest_final,scenario_086,GA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_086,GA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,payroll_tax,2.17.1,1421.1448974609375,1421.1448974609375,1421.1448974609375,False,0.0,False +latest_final,scenario_088,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_088,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,federal_income_tax_before_refundable_credits,2.17.1,6819.7119140625,6819.7119140625,6819.7119140625,False,0.0,False +latest_final,scenario_089,NC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,payroll_tax,2.17.1,5039.91015625,5039.91015625,5039.91015625,False,0.0,False +latest_final,scenario_089,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,self_employment_tax,2.17.1,4286.24072265625,4286.24072265625,4286.24072265625,False,0.0,False +latest_final,scenario_089,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,state_income_tax_before_refundable_credits,2.17.1,3219.935302734375,3219.935302734375,3219.935302734375,False,0.0,False +latest_final,scenario_089,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_089,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,payroll_tax,2.17.1,535.5,535.5,535.5,False,0.0,False +latest_final,scenario_090,KS,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_090,KS,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,federal_income_tax_before_refundable_credits,2.17.1,1349.998779296875,1349.998779296875,1349.998779296875,False,0.0,False +latest_final,scenario_091,WI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,payroll_tax,2.17.1,2065.5,2065.5,2065.5,False,0.0,False +latest_final,scenario_091,WI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,state_income_tax_before_refundable_credits,2.17.1,843.6614990234375,843.6614990234375,895.8131103515625,True,52.151611328125,True +latest_final,scenario_091,WI,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_091,WI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_092,AL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_092,AL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,federal_income_tax_before_refundable_credits,2.17.1,8628.990234375,8628.990234375,8628.990234375,False,0.0,False +latest_final,scenario_093,MO,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,payroll_tax,2.17.1,12469.5,12469.5,12469.5,False,0.0,False +latest_final,scenario_093,MO,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,dependent1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,state_income_tax_before_refundable_credits,2.17.1,3388.24560546875,3389.65087890625,3388.24560546875,True,0.0,False +latest_final,scenario_093,MO,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_093,MO,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_095,NJ,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,self_employment_tax,2.17.1,593.2291259765625,593.2291259765625,593.2291259765625,False,0.0,False +latest_final,scenario_095,NJ,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_095,NJ,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_098,IL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_098,IL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,federal_income_tax_before_refundable_credits,2.17.1,10679.75,10679.75,10679.75,False,0.0,False +latest_final,scenario_099,CA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_099,CA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,payroll_tax,2.17.1,14767.5,14767.5,14767.5,False,0.0,False +latest_final,scenario_099,CA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,state_income_tax_before_refundable_credits,2.17.1,4493.7431640625,4493.7431640625,4640.77734375,True,147.0341796875,True +latest_final,scenario_099,CA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_099,CA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,federal_refundable_credits,2.17.1,2878.0966796875,2878.0966796875,2878.0966796875,False,0.0,False +latest_final,scenario_100,MT,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,payroll_tax,2.17.1,452.47613525390625,452.47613525390625,452.47613525390625,False,0.0,False +latest_final,scenario_100,MT,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_head_start_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_100,MT,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,snap,2.17.1,8625.8896484375,8625.8896484375,8556.0,True,-69.8896484375,True +latest_final,scenario_100,MT,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_100,MT,state_refundable_credits,2.17.1,473.1777038574219,473.1777038574219,473.1777038574219,False,0.0,False +latest_final,scenario_100,MT,tanf,2.17.1,6064.958984375,6064.958984375,6064.958984375,False,0.0,False +latest_final,scenario_101,TX,federal_income_tax_before_refundable_credits,2.17.1,13418.599609375,13418.599609375,13418.599609375,False,0.0,False +latest_final,scenario_101,TX,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,payroll_tax,2.17.1,7879.5,7879.5,7879.5,False,0.0,False +latest_final,scenario_101,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_101,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,federal_income_tax_before_refundable_credits,2.17.1,6104.09619140625,6104.09619140625,6104.09619140625,False,0.0,False +latest_final,scenario_102,NC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,payroll_tax,2.17.1,6670.7998046875,6670.7998046875,6670.7998046875,False,0.0,False +latest_final,scenario_102,NC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,state_income_tax_before_refundable_credits,2.17.1,2461.869873046875,2461.869873046875,2461.869873046875,False,0.0,False +latest_final,scenario_102,NC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_102,NC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_104,NY,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_104,NY,state_refundable_credits,2.17.1,0.0,375.0,0.0,True,0.0,False +latest_final,scenario_104,NY,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,payroll_tax,2.17.1,486.9607238769531,486.9607238769531,486.9607238769531,False,0.0,False +latest_final,scenario_107,OH,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_107,OH,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_107,OH,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,payroll_tax,2.17.1,585.6839599609375,585.6839599609375,585.6839599609375,False,0.0,False +latest_final,scenario_108,WI,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_108,WI,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,snap,2.17.1,288.0,287.68316650390625,288.0,False,0.0,False +latest_final,scenario_108,WI,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_108,WI,state_refundable_credits,2.17.1,271.3916015625,271.3916015625,271.3916015625,False,0.0,False +latest_final,scenario_108,WI,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,federal_refundable_credits,2.17.1,12038.0859375,12038.0859375,12038.0859375,False,0.0,False +latest_final,scenario_109,FL,free_school_meals_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_early_head_start_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child2_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,child3_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,child3_wic_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_109,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,self_employment_tax,2.17.1,4238.86474609375,4238.86474609375,4238.86474609375,False,0.0,False +latest_final,scenario_109,FL,snap,2.17.1,7932.0,8020.55419921875,7932.0,True,0.0,False +latest_final,scenario_109,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_109,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,federal_income_tax_before_refundable_credits,2.17.1,23897.443359375,23897.443359375,23897.443359375,False,0.0,False +latest_final,scenario_110,OH,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,payroll_tax,2.17.1,7650.0,7650.0,7650.0,False,0.0,False +latest_final,scenario_110,OH,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,state_income_tax_before_refundable_credits,2.17.1,4057.47216796875,4057.47216796875,4057.47216796875,False,0.0,False +latest_final,scenario_110,OH,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_110,OH,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_111,WA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_111,WA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,federal_refundable_credits,2.17.1,453.0960083007813,453.0960083007813,453.09600830078125,False,-5.684341886080802e-14,False +latest_final,scenario_112,TX,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,payroll_tax,2.17.1,453.0960083007813,453.0960083007813,453.09600830078125,False,-5.684341886080802e-14,False +latest_final,scenario_112,TX,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,snap,2.17.1,287.68316650390625,287.68316650390625,0.0,True,-287.68316650390625,True +latest_final,scenario_112,TX,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_112,TX,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,federal_income_tax_before_refundable_credits,2.17.1,10729.6123046875,10729.6123046875,10729.6123046875,False,0.0,False +latest_final,scenario_114,VA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,payroll_tax,2.17.1,1736.550048828125,1736.550048828125,1736.550048828125,False,0.0,False +latest_final,scenario_114,VA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_114,VA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,state_income_tax_before_refundable_credits,2.17.1,3514.14501953125,3514.14501953125,3514.14501953125,False,0.0,False +latest_final,scenario_114,VA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_114,VA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_115,AL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,state_income_tax_before_refundable_credits,2.17.1,4.679999828338623,4.679999828338623,4.679999828338623,False,0.0,False +latest_final,scenario_115,AL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_115,AL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_116,FL,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,federal_income_tax_before_refundable_credits,2.17.1,24961.33984375,24391.796875,24961.33984375,True,0.0,False +latest_final,scenario_117,AR,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,payroll_tax,2.17.1,15044.787109375,15044.787109375,15044.787109375,False,0.0,False +latest_final,scenario_117,AR,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,child3_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,state_income_tax_before_refundable_credits,2.17.1,7984.53369140625,7984.05419921875,7984.53369140625,False,0.0,False +latest_final,scenario_117,AR,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_117,AR,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,head_medicaid_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_118,NY,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_118,NY,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,snap,2.17.1,2903.9404296875,2903.9404296875,2868.0,True,-35.9404296875,True +latest_final,scenario_118,NY,ssi,2.17.1,9368.0,9368.0,9368.0,False,0.0,False +latest_final,scenario_118,NY,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_118,NY,state_refundable_credits,2.17.1,375.0,375.0,375.0,False,0.0,False +latest_final,scenario_118,NY,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,federal_refundable_credits,2.17.1,2854.0087890625,2854.0087890625,2854.0087890625,False,0.0,False +latest_final,scenario_119,VA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,payroll_tax,2.17.1,4207.5,4207.5,4207.5,False,0.0,False +latest_final,scenario_119,VA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,child2_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,state_income_tax_before_refundable_credits,2.17.1,1859.0943603515625,1859.0943603515625,1859.0943603515625,False,0.0,False +latest_final,scenario_119,VA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_119,VA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,federal_income_tax_before_refundable_credits,2.17.1,40021.81640625,40021.81640625,40416.66796875,True,394.8515625,True +latest_final,scenario_120,CT,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,payroll_tax,2.17.1,13496.1552734375,13496.1552734375,13496.1552734375,False,0.0,False +latest_final,scenario_120,CT,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_120,CT,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,state_income_tax_before_refundable_credits,2.17.1,11917.18359375,11917.18359375,11917.18359375,False,0.0,False +latest_final,scenario_120,CT,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_120,CT,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,federal_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,payroll_tax,2.17.1,1963.3724365234375,1963.3724365234375,1963.3724365234375,False,0.0,False +latest_final,scenario_121,SC,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,state_income_tax_before_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_121,SC,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,federal_income_tax_before_refundable_credits,2.17.1,4746.66162109375,4746.66162109375,4746.66162109375,False,0.0,False +latest_final,scenario_122,MN,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,payroll_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,head_medicare_eligible,2.17.1,1.0,1.0,1.0,False,0.0,False +latest_final,scenario_122,MN,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,state_income_tax_before_refundable_credits,2.17.1,1025.54150390625,1028.216552734375,1025.54150390625,True,0.0,False +latest_final,scenario_122,MN,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_122,MN,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,federal_income_tax_before_refundable_credits,2.17.1,5200.240234375,5200.240234375,5200.240234375,False,0.0,False +latest_final,scenario_123,PA,federal_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,free_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,local_income_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,payroll_tax,2.17.1,11194.0,11194.0,11194.0,False,0.0,False +latest_final,scenario_123,PA,head_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_chip_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_early_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_head_start_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_medicaid_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_medicare_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,head_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,spouse_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,child1_wic_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,reduced_price_school_meals_eligible,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,self_employment_tax,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,snap,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,ssi,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,state_income_tax_before_refundable_credits,2.17.1,3070.061279296875,3070.061279296875,3070.061279296875,False,0.0,False +latest_final,scenario_123,PA,state_refundable_credits,2.17.1,0.0,0.0,0.0,False,0.0,False +latest_final,scenario_123,PA,tanf,2.17.1,0.0,0.0,0.0,False,0.0,False diff --git a/reference_audit/2026-09-28/verification/port_snap_result.json b/reference_audit/2026-09-28/verification/port_snap_result.json new file mode 100644 index 00000000..87ed0ea9 --- /dev/null +++ b/reference_audit/2026-09-28/verification/port_snap_result.json @@ -0,0 +1,202 @@ +{ + "modules": [ + { + "convention": "c_snap_hold_fy2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_snap_hold_fy2026.py", + "needed": true, + "reason": "2.15.17 carries USDA's FY2027 schedule from 2026-10-01. It uses the published values from USDA's memo dated 2026-08-21 (policyengine-us#9623, d27d6c3cc3; the FNA fy27 page shows 'updated August 28, 2026'), which came after the freeze. For October-December 2026 the module puts back the FY2026 schedule USDA published before the freeze. The FNS fy26 memo (dated August 13, 2025; page updated August 19, 2025; Wayback capture 20250902215244) gives: 4-person maximum $994, minimum benefit $24, shelter cap $744, homeless deduction $198.99, standard deduction $209, asset limits $3,000 and $4,500. For 2026-10-01 to 2026-12-31 only, each held leaf takes its own 2.15.17 value from 2026-09-30. The held leaves are: max_allotment.main and .additional; min_allotment.published_adjustment (FY2027 +$1 in the 48 states and DC, which turns $24 into $25); standard deduction; excess shelter cap; homeless deduction; utility.single.* and utility.limited.main (uprated by the SNAP index, whose 2026-10-01 entry is June 2026 CPI-U, #9090); Arizona's limited utility allowance by household size (FY2027 $154/$208, added 9164f90305); asset_test.limit (FY2027 elderly/disabled limit $4,750); and the uprating index itself. The module also removes the max_allotment cap for October-December 2026 (formula_2026_10_01; the amounts were first published in the FY2027 memo, and FY2026 had no cap). The 1.755.4 module r13_hold_fy2026_v3 cannot be reused. On 2.15.17 it changes only the index leaf, which no formula reads, because 2.15.17 applies reforms after uprate_parameters (system.py, issue #9075 comment) and encodes FY2027 as literal values. Checks: a diff of the full parameter tree (102,807 leaves) shows 359 leaves changed, on dates 2026-10-01 to 2026-12-31 only. For 51 jurisdictions x household sizes 1-20, max allotment, min allotment and standard deduction in October-December equal raw September values, and January-September is identical to raw 2.15.17. The module gives the same parameters when applied 3 times, and the same result when applied after init (Simulation(reform=)). Not held: the poverty guideline (2.15.17 already uses the 2026 HHS guideline, $15,960 + $5,680, from 2026-10-01), ABAWD waiver timelines, and everything else." + } + ], + "upstream_fixes": [ + { + "root_cause": "r26_snap_contribution_rounding (30% of net income rounded up to the next dollar, 7 CFR 273.10(e)(2)(ii)(A))", + "present_in_2_15_17": true, + "evidence": "2.15.17 variables/gov/usda/snap/snap_expected_contribution.py returns np.ceil(np.round(net_income * rate, 2)) on net income that is already whole-dollar. Commit 8df8c5043a (2026-08-19), in PolicyEngine/policyengine-us#9318, merge 5d88007d90 (2026-08-25), which is an ancestor of 79be99f671. Not re-applied." + }, + { + "root_cause": "r27_snap_net_income_rounding (net income rounded to the nearest dollar, 7 CFR 273.10(e)(1)(ii))", + "present_in_2_15_17": true, + "evidence": "2.15.17 variables/gov/usda/snap/income/snap_net_income.py returns np.floor(np.round(net_income, 2) + 0.5). Commit 949546af53 in #9318 (merge 5d88007d90, 2026-08-25); rounding to cents first was added in #9587 (fa27adbffc, 2026-09-28). The r27 module (floor(net + 0.5)) can differ only on a fraction in [0.495, 0.5). Not re-applied." + }, + { + "root_cause": "r28_snap_min_allotment_rounding (minimum = 8% of the one-person maximum, rounded, 7 CFR 273.10(e)(2)(ii)(C))", + "present_in_2_15_17": true, + "evidence": "2.15.17 variables/gov/usda/snap/snap_min_allotment.py computes np.round(rate * relevant_max_allotment) + published_adjustment[region], then applies the DC/MD/NJ overrides. Commit cf165464a6 in PolicyEngine/policyengine-us#9162, merge 3c41c31457 (2026-07-28); d27d6c3cc3 (#9623) added published_adjustment, which the hold sets to 0 for October-December 2026. np.round and floor(x + 0.5) agree for every integer maximum, because 0.08*m is never exactly x.5. Not re-applied." + }, + { + "root_cause": "r31_snap_income_limit_rounding (gross and net standards rounded up to the next dollar, 7 CFR 273.9(a)(3))", + "present_in_2_15_17": true, + "evidence": "2.15.17 meets_snap_gross_income_test.py and meets_snap_net_income_test.py call snap_monthly_income_standard (income/snap_income_standard_helpers.py), which computes np.ceil(np.round(rate * fpg, 2)), with the increment rounded separately for households over 8. Commit cf165464a6 in #9162 (merge 3c41c31457, 2026-07-28), refined by #9587 (fa27adbffc). Not re-applied." + }, + { + "root_cause": "r33_snap_child_support_treatment (flag = states excluding child support paid from gross income, per the State Options Reports)", + "present_in_2_15_17": true, + "evidence": "PolicyEngine/policyengine-us#9586, d9e801df41 (2026-09-24), is an ancestor of 79be99f671. latest/port_snap/check_r33_values.py compared 636 state-months (every jurisdiction, each month of 2026) of 2.15.17's gov.usda.snap.income.deductions.child_support with the r33 module's VALUES: 0 mismatches. The later commit 3102d3db05 (#9622) adds only 2010-2017 citations and tests. The formulas read the flag the same way (true = exclude from gross; snap_child_support_deduction deducts the rest from net). Differential test: re-applying r26+r27+r28+r31+r33 (the 1.755.4 modules) on top of 2.15.17 plus this module moves 0 of 1,984 outputs (latest/port_snap/probe_hold_plus_1755_snap_fixes.csv)." + }, + { + "root_cause": "r30_snap_heat_and_eat_sua (the root cause of excluded 080 and noted on 100; not one of the board's regenerated fixes, checked because the task named 080 and 100)", + "present_in_2_15_17": false, + "evidence": "2.15.17 snap_state_using_standard_utility_allowance.py still returns p.always_standard[state] with no elderly/disabled condition. Its only change since 1.755.4 is metadata (d7fc17abfe). Hold + r30 on 2.15.17 gives 080 = 3,240 and 100 = 6,924, the values the exclusion records give. The defect is still real in 2.15.17, so 080 stays excluded." + } + ], + "reproduced": [ + { + "scenario_id": "scenario_008", + "variable": "snap", + "board": 15108, + "value": 15108, + "note": "Convention plus r26 and r27. Raw 2.15.17 gives 15,273; the held max allotment and standard deduction give 15,108." + }, + { + "scenario_id": "scenario_012", + "variable": "snap", + "board": 4884, + "value": 4884, + "note": "Convention plus r26 and r27. Raw 2.15.17 gives 4,962." + }, + { + "scenario_id": "scenario_027", + "variable": "snap", + "board": 288, + "value": 288, + "note": "Convention plus r28. Raw 2.15.17 gives 291 (FY2027 +$1 published_adjustment makes the Oct-Dec minimum $25). Holding the adjustment at 0 gives the FY2026 minimum of $24." + }, + { + "scenario_id": "scenario_030", + "variable": "snap", + "board": 288, + "value": 288, + "note": "Convention plus r28. Raw 2.15.17 gives 291 (same published_adjustment)." + }, + { + "scenario_id": "scenario_038", + "variable": "snap", + "board": 7212, + "value": 7212, + "note": "Convention plus r26 and r27. Raw 2.15.17 gives 7,302." + }, + { + "scenario_id": "scenario_043", + "variable": "snap", + "board": 3576, + "value": 3576, + "note": "Convention. Raw 2.15.17 gives 3,600 (FY2027 one-person maximum $306)." + }, + { + "scenario_id": "scenario_045", + "variable": "snap", + "board": 0, + "value": 0, + "note": "r33 (and r28). 2.15.17 already gives 0 because #9586 is in it; the hold does not change it." + }, + { + "scenario_id": "scenario_054", + "variable": "snap", + "board": 6060, + "value": 6060, + "note": "Convention plus r26. Raw 2.15.17 gives 6,138." + }, + { + "scenario_id": "scenario_066", + "variable": "snap", + "board": 3576, + "value": 0, + "note": "NOT reproduced; see residuals. The convention is not the cause. The ABAWD work test in 2.15.17 fails for all 12 months, so the hold has nothing to act on. Hold plus the stated 40 hours gives 3,576." + }, + { + "scenario_id": "scenario_073", + "variable": "snap", + "board": 288, + "value": 288, + "note": "Convention plus r28. Raw 2.15.17 gives 291." + }, + { + "scenario_id": "scenario_079", + "variable": "snap", + "board": 2376, + "value": 2376, + "note": "Convention plus r26. Raw 2.15.17 gives 2,433. Holding Arizona's FY2027 limited utility allowance does not move it." + }, + { + "scenario_id": "scenario_108", + "variable": "snap", + "board": 288, + "value": 288, + "note": "Convention plus r28. Raw 2.15.17 gives 291." + }, + { + "scenario_id": "scenario_109", + "variable": "snap", + "board": 7932, + "value": 7932, + "note": "Convention plus r26. Raw 2.15.17 gives 8,040." + } + ], + "residuals": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "board": 3576, + "value": 0, + "note": "Scored, VA. Cause: an input mapping, not the convention. The prompt states 40 'usual weekly hours worked' (policybench/prompts.py labels hours_worked_last_week that way). The SNAP work rules in 2.15.17 (meets_snap_abawd_work_requirements.py:39, meets_snap_general_work_requirements.py:37, is_snap_work_registration_exempt_non_age.py:62) read weekly_hours_worked_before_lsr, whose default changed from 40 to 0 in policyengine-us#9261 (82745ca239, merged a112cc5a0a on 2026-08-12, closes #9254). With 0 hours this childless 40-year-old in VA fails the ABAWD test every month (no waived area), so SNAP is $0. 1.755.4 matched only because its default was 40. Probe latest/port_snap/probe_hold_plus_stated_hours.py copies stated hours into weekly_hours_worked_before_lsr. It gives 3,576, equal to the board, and moves no other output of the 1,984. Suggested remedy: add that mapping to the situation builder or a patch. An exclusion does not fit, because the hours are listed." + }, + { + "scenario_id": "scenario_013", + "variable": "snap", + "board": 0, + "value": 240, + "note": "Scored, AZ. Cause: an upstream correction under pre-freeze state law, not the convention. BBCE (is_tanf_non_cash_eligible) holds from March to December 2026, which waives the net income and asset tests, so the household gets the $24 minimum x 10 months = $240. Raw 2.15.17 gives 243 because the October-December minimum is $25 until the hold removes the FY2027 +$1. Source: 0881eff4b8 (2026-08-06). It sets AZ gross.yaml and gross_hheod.yaml to 200% FPL from 2026-03-01, quoting the CNAP manual: 'Starting the benefit month of 03/2026 ... changed from 185% of the FPL to 200%'. In 1.755.4 both were 185% all year. Gross income of $2,539.33 fails 185% x 2025 FPL ($2,412.71) but passes 200% ($2,608.33). The rule took effect before the freeze, so 240 should be the new reference. Caveat: the quote covers the general standard; applying it to the elderly/disabled standard, which this household uses, is upstream's reading. I could not retrieve the manual because azdes.gov serves a Cloudflare challenge." + }, + { + "scenario_id": "scenario_023", + "variable": "snap", + "board": 461.339722, + "value": 408, + "note": "Excluded (meets_ssi_disability_criteria not listed). The board keeps the untouched v1.1 value. 408 equals the stated-facts value the board's 1.755.4 regeneration computed (convention + r26-r33, out/c13v3_upstream_plus_r33.csv). Raw 2.15.17 gives 462; the hold lowers it through the standard deduction, shelter cap and max allotment. No scoring impact." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "board": 2669.217041, + "value": 2628, + "note": "Excluded (meets_ssi_disability_criteria not listed). 2628 equals the 1.755.4 regeneration's stated-facts value; the board keeps v1.1 for excluded outputs. Raw 2.15.17 gives 2,682. No scoring impact." + }, + { + "scenario_id": "scenario_080", + "variable": "snap", + "board": 3596.039795, + "value": 3576, + "note": "Excluded (r30_snap_heat_and_eat_sua, still unfixed in 2.15.17). 3,576 is the value the exclusion note says the conventions and upstream fixes give. Hold + r30 on 2.15.17 gives 3,240, the exclusion's alternative value. The exclusion stays valid." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "board": 8625.889648, + "value": 8556, + "note": "Excluded (meets_ssi_disability_criteria not listed). 8,556 matches the exclusion note ('from 8,556.00 to 6,924.00' under r30), and hold + r30 on 2.15.17 gives 6,924. The board keeps v1.1. No scoring impact." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "board": 287.683167, + "value": 0, + "note": "Excluded (weekly_hours_worked_before_lsr not listed). The 2.15.17 default of 0 hours (#9261) is the exclusion's alternative reading, so the ABAWD test fails all year (TX, no waived area). With 1.755.4's default of 40 hours (probe_hold_plus_40h_default) it gives 288, the 1.755.4 regeneration value. No scoring impact." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "board": 1140, + "value": 95, + "note": "Excluded (weekly hours not listed; mortgage-residence reading). Zero hours per the #9261 default. 2.15.17 puts NJ in an ABAWD waived area for January 2026 only, so January pays NJ's $95 state minimum and February-December pay $0. That makes 95, where the exclusion record's alternative (computed on 1.755.4, without waiver data) is 0.0. With 40 hours it gives 1,140, equal to the board. The exclusion record's alternative_value is stale for 2.15.17, but there is no scoring impact." + }, + { + "scenario_id": "scenario_118", + "variable": "snap", + "board": 2903.94043, + "value": 2868, + "note": "Excluded (mortgage-residence reading). 2,868 equals the 1.755.4 regeneration's stated-facts value; the board keeps v1.1. Raw 2.15.17 gives 2,904. No scoring impact." + } + ], + "sweep_csv": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/out/latest_c_snap_hold_fy2026.csv", + "notes": "Module: /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_snap_hold_fy2026.py (sha256 3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd). I rewrote the draft from the interrupted attempt. That draft had a defect: it turned the cap's missing value into inf, and backdate_parameters then copied that inf to every date before 2026-10-01 (21 leaf-date values outside the window; no formula reads them). The new _lift_leaf keeps what raw backdating produces and changes only October-December. It is idempotent, and 2.15.17 applies reforms twice.\n\nFull sweep (sweep_latest.py): 1,984 outputs, 32 moved against the board (41 on raw 2.15.17). Against raw 2.15.17 the module changes 17 outputs, all SNAP. With the eight state/IRS conventions added (latest_state_conventions), the SNAP outputs are identical (probe_snap_plus_state_conventions.csv).\n\nLeave-one-out attribution (latest/port_snap/attribution_v3.csv):\n- max_allotment.main moves 12 outputs.\n- The standard deduction moves 10.\n- published_adjustment moves 5 (013, 027, 030, 073, 108).\n- The excess shelter cap moves 1 (023).\n- These are held for fidelity to the FY2026 schedule and move no benchmark output: the asset limit, homeless deduction, Arizona LUA, limited.main, utility.single, additional, the cap lift, and the index.\n\nFY2026 values match between engines for allotments, deductions and caps. The only differences from the board's effective Q4 values are 22 utility-allowance leaves (ID, NV, WI, MD, SD, VA). There, 2.15.17 corrected the FY2026 figures to the official FNS table (policyengine-us#9319, ec91251c2e, 2026-08-25). That is an upstream correction, not the convention, and it moves no benchmark output.\n\nJudgment calls for the orchestrator:\n- Arizona's FY2027 LUA is held as an FY2027 utility allowance. I could not verify when DES published it: azdes.gov is behind Cloudflare and Wayback has no 2026 capture. If it was published before 2026-07-03, remove that node; no output changes.\n- The sidecar dates the FY2026 memo as signed 2025-08-14. The FNS web memo shows DATE August 13, 2025. Both are before the freeze.\n- Scenario_066 needs a situation-builder mapping of stated hours to weekly_hours_worked_before_lsr, outside this convention. The probe shows it restores 3,576 and touches nothing else.\n- Scenario_013's 240 follows from Arizona's March 2026 BBCE increase (pre-freeze) as upstream encoded it.\n- Scenario_056's exclusion alternative_value (0.0) is stale for 2.15.17 (95).\n\nEvidence is in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/port_snap/:\n- param_diff_module_21517.json (full-tree diff)\n- test_hold_invariants.py and .log (0 failures)\n- p1755_*.json and p21517_*.json (parameter dumps)\n- held_1755_rerun.json (the 340 leaves the 1.755.4 convention held)\n- snap_table.csv\n- the probe_*.py and .csv files\n- check_post_init_apply.py\n- fns_fy26_cola_wayback_20250902215244.html and fns_fy27_cola_live_20260928.html\n\nI modified no existing file except the draft module the task allowed me to overwrite. Nothing was pushed or filed. DONE." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/port_state_result.json b/reference_audit/2026-09-28/verification/port_state_result.json new file mode 100644 index 00000000..4b64f6b0 --- /dev/null +++ b/reference_audit/2026-09-28/verification/port_state_result.json @@ -0,0 +1,206 @@ +{ + "modules": [ + { + "convention": "c_ca_hold_2025", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_ca_hold_2025.py", + "needed": true, + "reason": "Every amount r19 set is still a CA-CPI projection in 2.15.17, with 2026 values identical to 1.755.4's. The CalEITC final phase-out breakpoints are now projected without annual rounding: 251/635 for 2025 and 257/649 for 2026, not the statutory 252/636, which is held for both years. New in this port: 2.15.17 now uprates 30 amounts that 1.755.4 kept at their 2025 values (upstream #9059, df3482f4ef, 2026-07-21; #9429, 2572674b97). These are the standard deduction, AMT exemption, AMTI lower and upper thresholds, exemption-credit phase-out start, and itemized-limitation AGI threshold. They are held at FTB's 2025 amounts, which I read this session: 5,706/11,412 (FTB Tax News, Oct 2025); 92,749/123,667/61,830, 347,808/463,745/231,868 and 718,804/958,413/479,188 (2025 Schedule P (540) instructions); 252,203/504,411/378,310 (2025 540 booklet and 540 CA instructions). Attribution sweeps show both halves are needed. r19's set alone leaves 022 federal at +2.65 and 023 state at -2.59. The 30 new holds alone leave 022 federal at +17.76, 023 state at -5.98 and 023 refundable at +2.37." + }, + { + "convention": "c_irs_sales_tax_2025", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_irs_sales_tax_2025.py", + "needed": true, + "reason": "2.15.17 now carries the IRS 2025 table (upstream #9616, 3f029abf5b, 2026-09-27), and all 5,814 cells match r19_irs_sales_tax_2025.json (asserted in the module). For 2026, tax.yaml says 'later years are uprated from 2025' by gov.irs.uprating, a projection (for example AL 1/1 is 312.93 against the published 306). The module holds the 2025 cells for 2026 only. On 2.15.17 it changes no scored output. In scenario_000 the filer takes the 18,150 standard deduction under either table (itemized 17,612.52 raw, 17,587.20 held). The module reproduces the board's underlying sales tax exactly: 931.0 state, 186.2 local, SALT 17,587.20." + }, + { + "convention": "c_wi_published_2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_wi_published_2026.py", + "needed": true, + "reason": "The brackets are now carried: 2.15.17 has explicit 2026 entries citing the 2026 Form 1-ES, and they equal the published 15,110/51,950/332,720 and so on (asserted, not set). The standard deduction is still projected, now with $10 rounding (94e40f1471): 13,870/25,680/12,200/17,920 maxima and 19,990/28,850/13,690/19,990 phase-out starts. DOR publishes 13,960/25,840/12,280/18,030, 20,120/29,040/13,780/20,120 and 58,827 (2026 Form 1-ES instructions, D-101A R. 1-26, read this session). The module sets these 10 values. It changes only excluded outputs (042, 064, 091), as on the board (changed=[])." + }, + { + "convention": "c_id_hold_2025", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_id_hold_2025.py", + "needed": true, + "reason": "The zero-rate thresholds are still projected in 2.15.17 (4,920.03/9,840.06), identical to 1.755.4. The published 2025 amounts, $4,811/$9,622 (2025 Form 40 instructions rev. 2026-03-02, read this session), are held. The retirement caps are still projected (49,308.72/73,963.07); the SSA-derived 49,824/74,736 is kept as in r19." + }, + { + "convention": "c_mn_published_2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_mn_published_2026.py", + "needed": true, + "reason": "All 90 parameter-years r19 sets are unchanged from 1.755.4 in 2.15.17 (still CPI projections), so the r19 values are kept. I read the source this session: MN 2026 inflation-adjusted amounts, dated December 1, 2025 (5,300 exemption, 15,300/30,600/23,000 standard deduction, 33,310/48,700 thresholds). Two published amounts 1.755.4 lacked are now in 2.15.17 and consistent with the rule, so they are asserted but not set: the 1,107,750 alternate reduction threshold (2026 publication) and the 31,000/48,000 marriage-credit minimums (2025 M1MA, carried into 2026)." + }, + { + "convention": "c_md_2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_md_2026.py", + "needed": true, + "reason": "Every MD value r19 set is unchanged from 1.755.4 in 2.15.17: withholding table 2,800/5,700, flat deduction 3,400/6,850, CDCC caps 114,600/178,250 and 62,250/93,450. The r19 values are kept. I read the 2026 Employer Withholding Guide ('Revised December 2025'; standard deduction $3,400; 'includes local income tax rates'). The module reproduces the MD part of both households: 068 md_income_tax_before_refundable_credits is 1,255.3428, equal to the board. The scored MD outputs still move because of a separate output-scope change; see latest_md_local_output_scope and the residuals." + }, + { + "convention": "c_mi_published_2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_mi_published_2026.py", + "needed": true, + "reason": "The retirement tier-one limits are now carried: 2.15.17 has explicit 2026 values of 67,610/135,220 (#9073, b46a672edd; caf421bbe9), asserted rather than set. The personal exemption is still projected at 5,950 and is set to 5,900 (Form 446 Rev. 02-26, read this session). All HELD_2025 amounts are unchanged from 1.755.4 and are kept. The home-heating credit percentage in 2.15.17 is 0.60 from 2025 (TY2025 MI-1040CR-7 book), carried into 2026. That is the published-2025 hold the rule asks for (1.755.4 had 0.52), so it is asserted, not set." + }, + { + "convention": "c_mo_published_2026", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_c_mo_published_2026.py", + "needed": true, + "reason": "The 2026 thresholds are still projected in 2.15.17 (1,342.76 through 9,399.29), identical to 1.755.4. They are set to the published 1,348 through 9,436 (2026 Withholding Formula, read this session). r19 re-pinned the 2026 pension cap by importing the baseline system; that step is dropped, because 2.15.17 applies reforms after its uprating pass (system.py), so the 2025 repair to 47,633 cannot move 2026. Checked: the 2026 cap stays 49,308.72, same as the board. 2.15.17's new 2026 MO property-tax-credit parameters, which upstream attributes to 'H.B. 594' (1d128732f9, 2026-08-04), were not dated by me. The credit is 0 in both MO households on both engines." + }, + { + "convention": "combined: all eight conventions", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_state_conventions.py", + "needed": true, + "reason": "Composes the eight latest_c_* reforms, which touch disjoint parameter sets. Parameter-level check: on 2.15.17 it reproduces all 10,748 parameter-years the 1.755.4 board conventions changed, plus the 30 new CA holds, which equal 1.755.4's own 2025-carried values (latest/port_state/p21517_latest.json vs p1755_conv.json). It excludes SNAP and the MD output-scope candidate." + }, + { + "convention": "md_local_output_scope (candidate output-definition adapter, not a law convention)", + "module_path": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/fixes/latest_md_local_output_scope.py", + "needed": true, + "reason": "2.15.17 adds md_local_income_tax_before_refundable_credits to the list behind state_income_tax_before_refundable_credits (upstream #8888, 6b0bca0b9f, 2026-07-05). The PolicyBench prompt defines that output as 'excluding local income and payroll taxes'. The county is unlisted, so 2.15.17 falls back to Allegany's 3.03%. The module removes only that list entry and leaves county tax in the federal SALT proxy. With it, 068 state goes to 1,255.342773 (exact) and 078 state to 6,936.876 (within $1). Adopting it is the lead's call." + } + ], + "upstream_fixes": [ + { + "root_cause": "r04_capital_gain_distributions (issue #8828)", + "present_in_2_15_17": true, + "evidence": "Commit 491642087f (2026-07-05, PR #8839) is an ancestor of 79be99f671. In the 2.15.17 site-packages, non_sch_d_capital_gains is in parameters/gov/irs/gross_income/sources.yaml:12 and investment/income/sources.yaml:9, and variables/gov/irs/tax/federal_income/capital_gains/net_capital_gain.py adds it to the preferential base. Raw 2.15.17 reproduces the board exactly: 042 federal 2,361.964844 and 051 state 820.349976." + }, + { + "root_cause": "r09_ny_rptc_rent_cap (issue #9298; plus the 2025 flat tables)", + "present_in_2_15_17": true, + "evidence": "Commits ba4ee190d5 (2026-08-19, PR #9301) and 5490d04cb6 (2026-08-24, PR #9313) are ancestors of 2.15.17. ny_real_property_tax_credit.py applies 'rent <= rptc.max_rent' to rent itself (max_rent 5,400/yr) and uses the 2025+ flat amount tables. Raw 2.15.17 gives 104 state_refundable_credits = 0, equal to the board." + }, + { + "root_cause": "r17_caleitc_agi_comparison", + "present_in_2_15_17": true, + "evidence": "Commit 7d19aa3556 (2026-09-01, PR #9363) is an ancestor of 2.15.17. ca_eitc.py returns min_(credit_for(earned_income), credit_for(max_(earned_income, agi))). 023 state_refundable_credits is 95.154007 with latest_c_ca_hold_2025, equal to the board. Raw 2.15.17 gives 97.52 only because of the CA projections." + } + ], + "reproduced": [ + { + "scenario_id": "scenario_022", + "variable": "federal_income_tax_before_refundable_credits", + "board": 11113.5703125, + "value": 11113.5703125, + "note": "c_ca_hold_2025, exact. Needs the 30 new CA holds as well as r19's set." + }, + { + "scenario_id": "scenario_023", + "variable": "state_income_tax_before_refundable_credits", + "board": 12.776718139648438, + "value": 12.776718139648438, + "note": "c_ca_hold_2025, exact." + }, + { + "scenario_id": "scenario_023", + "variable": "state_refundable_credits", + "board": 95.15400695800781, + "value": 95.15400695800781, + "note": "c_ca_hold_2025 together with r17 (already in 2.15.17), exact." + }, + { + "scenario_id": "scenario_000", + "variable": "federal_income_tax_before_refundable_credits", + "board": 2906.45361328125, + "value": 2906.45361328125, + "note": "c_irs_sales_tax_2025, exact. Raw 2.15.17 already equals the board because the filer takes the standard deduction." + }, + { + "scenario_id": "scenario_076", + "variable": "state_income_tax_before_refundable_credits", + "board": 6818.34423828125, + "value": 6818.34423828125, + "note": "c_id_hold_2025, exact." + }, + { + "scenario_id": "scenario_122", + "variable": "state_income_tax_before_refundable_credits", + "board": 1025.54150390625, + "value": 1025.54150390625, + "note": "c_mn_published_2026, exact." + }, + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "board": 1255.3427734375, + "value": 1255.3427734375, + "note": "c_md_2026, exact, but only together with latest_md_local_output_scope. The conventions alone give 2,089.608887 (see residuals)." + }, + { + "scenario_id": "scenario_045", + "variable": "state_income_tax_before_refundable_credits", + "board": 1290.9625244140625, + "value": 1290.9625244140625, + "note": "c_mi_published_2026, exact." + }, + { + "scenario_id": "scenario_093", + "variable": "state_income_tax_before_refundable_credits", + "board": 3388.24560546875, + "value": 3388.24560546875, + "note": "c_mo_published_2026, exact." + }, + { + "scenario_id": "scenario_042", + "variable": "federal_income_tax_before_refundable_credits", + "board": 2361.96484375, + "value": 2361.96484375, + "note": "r04 upstream (#8839) is in 2.15.17, exact with no module." + }, + { + "scenario_id": "scenario_051", + "variable": "state_income_tax_before_refundable_credits", + "board": 820.3499755859375, + "value": 820.3499755859375, + "note": "r04 upstream, exact with no module." + }, + { + "scenario_id": "scenario_104", + "variable": "state_refundable_credits", + "board": 0, + "value": 0, + "note": "r09 upstream (#9301/#9313) is in 2.15.17, exact with no module." + } + ], + "residuals": [ + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "board": 24780.61328125, + "value": 24164.45703125, + "note": "Two upstream changes, both applying law published before the freeze. Removing both reproduces 24,780.6133 exactly. (1) #8888 (6b0bca0b9f, 2026-07-05) puts county tax into md_withheld_income_tax, so SALT reaches the $40,400 cap (-986.27). This does not depend on the unlisted county: the cap binds at any rate of 2.09% or more, and Worcester 2.25%, Dorchester 3.3%, Anne Arundel and Frederick all give 24,164.457. (2) #9122 (316e7832a1, 2026-09-08) adds the $1,631.91 salt_refund_income to federal gross income (+391.66). The MD convention no longer changes this output, because the SALT cap binds. Proposed: adopt 24,164.46 if the lead adopts #9122 everywhere; it is the same root cause as the 033 SD, 082 NY, 117 AR and 120 CT federal moves (005 CA is excluded). With #8888 only the value is 23,772.80. If the refund's taxability under IRC 111 is ruled ambiguous, this output becomes an exclusion candidate." + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "board": 6936.3374, + "value": 11295.388672, + "note": "#8888 adds Maryland county tax (4,358.51 at the Allegany 3.03% default for the unlisted county) to the state aggregate. The prompt excludes local tax. With latest_md_local_output_scope the value is 6,936.876, within $1 (+0.539). That remainder comes from #9122: AGI of 200,136.91 crosses MD's $200,000 itemized phase-out, adding 10.27 of MD taxable income." + }, + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "board": 1255.3427734375, + "value": 2089.608887, + "note": "Same cause as 078 state: county tax of 834.27 at the 3.03% default. The MD part is exactly 1,255.3428, and latest_md_local_output_scope gives 1,255.342773. The federal output (1,910.02) is county-invariant because the filer takes the standard deduction." + }, + { + "scenario_id": "scenario_073", + "variable": "snap", + "board": 288, + "value": 291, + "note": "Outside the state conventions (MI SNAP). Unverified hypothesis: 2.15.17's FY2027 minimum allotment, where 9x24 + 3x25 = 291. c_snap_hold_fy2026 is not ported in this task, so the SNAP port agent owns this." + }, + { + "scenario_id": "scenario_108", + "variable": "snap", + "board": 288, + "value": 291, + "note": "Outside the state conventions (WI SNAP). Same hypothesis and owner as scenario_073." + } + ], + "sweep_csv": "/Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/sweep/out/latest_state_conventions.csv", + "notes": "On the question \"aren't we using policyengine.py\": only for provenance, not for calculation. I tested this in the triage venv. `import policyengine` (6.1.2) fails against policyengine-us 2.15.17 with \"ValueError: Data release manifest is not certified for the runtime model version 2.15.17 in country 'us'\". The error comes from model.py:552, which builds us_latest at import time, and there is no override. policyengine 6.1.2 pins policyengine-us==2.2.1.\n\nSo `pe.us.calculate_household` cannot run on 2.15.17. It would also have taken only parameter-dict reforms, and it calls `policyengine_us.Simulation` itself anyway. PolicyBench's own reference path never calculates through policyengine.py either:\n- `ground_truth.py` calls `get_us_situation_simulation_class()`, which returns `policyengine_us.Simulation`.\n- `policyengine_runtime.py` reads policyengine.py's metadata without importing it. The provenance it records is: policyengine_version 6.1.2, bundled_model_version 2.2.1, model_version 2.15.17, model_matches_policyengine_bundle False.\n\n`sweep_latest.py` takes the same direct path. The references are therefore policyengine-us 2.15.17 computed directly, with policyengine.py 6.1.2 recorded in provenance. Running calculations through policyengine.py would mean going back to its certified us 2.2.1.\n\nPort results (all runs use `.venv-pepy612-us21517` and the `sweep_latest` harness):\n- Every one of the eight conventions is still needed. 2.15.17 carries none of them in full.\n- Partly carried: the WI brackets, the MI retirement limits, and the 2025 IRS sales tax table.\n- 2.15.17 newly projects 30 CA amounts that 1.755.4 held at their 2025 values. The CA convention now holds them.\n\nSweeps:\n- With the eight conventions combined, 38 outputs are moved against the board. Before any fix there were 41.\n- With the conventions plus `latest_md_local_output_scope`, 36 are moved.\n- In my states, the only non-excluded outputs still moved are 078 MD federal and the MI/WI SNAP rows. The SNAP rows are outside this task.\n\nPer-module sweep CSVs are in `sweep/out/`: `latest_c_ca_hold_2025.csv`, `latest_c_irs_sales_tax_2025.csv`, `latest_c_wi_published_2026.csv`, `latest_c_id_hold_2025.csv`, `latest_c_mn_published_2026.csv`, `latest_c_md_2026.csv`, `latest_c_mi_published_2026.csv`, `latest_c_mo_published_2026.csv`, `latest_state_conventions.csv` and `latest_md_local_output_scope.csv`. The combined run with the MD scope module is `latest/port_state/combo_state_plus_mdlocal.csv`. Logs are in `latest/port_state/logs/`.\n\nExcluded outputs the conventions still change (not scored):\n- Equal to 1.755.4 plus r19: 022 CA state 2,505.87, 099 CA state 4,640.78, 007 ID 761.56, 053 ID 2,441.06, 064 WI 4,598.48.\n- Different from 1.755.4 plus r19, because of upstream #9122 and the r04/r32 changes: 005 CA state 41,267.01 and federal 107,833.16, 042 WI 464.18, 091 WI 895.81.\n\nThe MD output-scope module is my proposal, not a pre-freeze law convention. Adopting it is the lead's decision.\n\nSources I read this session:\n- FTB Tax News, October 2025.\n- FTB 2025 Schedule P (540), Form 540 booklet and Schedule CA (540) instructions.\n- WI 2026 Form 1-ES instructions (R. 1-26).\n- MI Form 446 (Rev. 02-26).\n- MO 2026 Withholding Formula.\n- MN 2026 inflation-adjusted amounts (dated 2025-12-01).\n- MD 2026 Withholding Guide (revised December 2025).\n- ID 2025 Form 40 instructions (rev. 2026-03-02).\n\nScratch files, including parameter dumps, the projected-parameter inventory, attribution modules and county-sensitivity patches, are in `triage/latest/port_state/`.\n\nModule sha256:\n- ca f19a47b9\n- irs 7f7fd235\n- wi 4edf4c71\n- id 39fb9929\n- mn 02a8ccc1\n- md 8111c77f\n- mi 787cf837\n- mo 0474524c\n- state_conventions 56e62e39\n- md_local_output_scope 53a6de3c\n\nNo existing file was modified, nothing was pushed, and the policyengine-us repo was only read." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/probe_023_medicaid.json b/reference_audit/2026-09-28/verification/probe_023_medicaid.json new file mode 100644 index 00000000..62d6fa7d --- /dev/null +++ b/reference_audit/2026-09-28/verification/probe_023_medicaid.json @@ -0,0 +1,71 @@ +{ + "scenario_id": "scenario_023", + "variable": "head_medicaid_eligible", + "pe_variable": "is_medicaid_eligible", + "engine_version": "policyengine-us 2.15.17", + "committed_reference": 1.0, + "fix_module": { + "module": "fixes/latest_final.py", + "sha256": "dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e" + }, + "probe_sha256": "0f894a441509c3c358fd42bed05ad8d3d4797600ef7904d4dbbc0dacee2511ca", + "readings": { + "stated_facts": "the scenario as built, meets_ssi_disability_criteria unset", + "reading_a": "meets_ssi_disability_criteria false: the head does not meet the SSI/SSA disability criteria", + "reading_b": "meets_ssi_disability_criteria true: the head meets them" + }, + "systems": { + "latest_final": "the reference system: policyengine-us 2.15.17 with fixes/latest_final.py; ca_wdp_disability_eligible reads is_disabled", + "latest_final_wdp_ssa_definition": "the same with ca_wdp_disability_eligible reading meets_ssi_disability_criteria (the SSA definition, 42 CFR 435.540(a)) in place of is_disabled" + }, + "results": { + "latest_final/stated_facts": { + "head_medicaid_eligible": 1.0, + "medicaid_category": "WORKING_DISABLED_BUY_IN", + "ca_wdp_disability_eligible": true, + "ca_wdp_eligible": true, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + }, + "latest_final/reading_a": { + "head_medicaid_eligible": 1.0, + "medicaid_category": "WORKING_DISABLED_BUY_IN", + "ca_wdp_disability_eligible": true, + "ca_wdp_eligible": true, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + }, + "latest_final/reading_b": { + "head_medicaid_eligible": 1.0, + "medicaid_category": "SENIOR_OR_DISABLED", + "ca_wdp_disability_eligible": true, + "ca_wdp_eligible": true, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + }, + "latest_final_wdp_ssa_definition/stated_facts": { + "head_medicaid_eligible": 0.0, + "medicaid_category": "NONE", + "ca_wdp_disability_eligible": false, + "ca_wdp_eligible": false, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + }, + "latest_final_wdp_ssa_definition/reading_a": { + "head_medicaid_eligible": 0.0, + "medicaid_category": "NONE", + "ca_wdp_disability_eligible": false, + "ca_wdp_eligible": false, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + }, + "latest_final_wdp_ssa_definition/reading_b": { + "head_medicaid_eligible": 1.0, + "medicaid_category": "SENIOR_OR_DISABLED", + "ca_wdp_disability_eligible": true, + "ca_wdp_eligible": true, + "medicaid_income_level": 1.4119, + "ssi": 0.0 + } + } +} diff --git a/reference_audit/2026-09-28/verification/rerun_sweeps.json b/reference_audit/2026-09-28/verification/rerun_sweeps.json new file mode 100644 index 00000000..062979aa --- /dev/null +++ b/reference_audit/2026-09-28/verification/rerun_sweeps.json @@ -0,0 +1,303 @@ +{ + "note": "What each sweep PolicyBench ran on policyengine-us 2.15.17 for an exclusion's fix or reading moves, written by scripts/summarize_rerun_sweeps.py from the sweeps' CSVs in triage/sweep/out (sweep_latest.py output; each csv_sha256 is the file read). A sweep adds its fix or reading to its baseline sweep. 'moves' lists every output whose recomputed value differs, by any amount, from the baseline's or from latest_conventions'. Every other output of the 1,984 has the same value in the sweep, its baseline and latest_conventions.", + "engine": "2.15.17", + "baselines": { + "latest_conventions": { + "csv": "latest_conventions.csv", + "csv_sha256": "af37fd3e94b9953c7c31b5fc0fc8621c841af266f21c95efedbb2d51072a740b", + "module": "fixes/latest_conventions.py", + "module_sha256": "4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb" + }, + "latest_map_stated_hours": { + "csv": "latest_map_stated_hours.csv", + "csv_sha256": "edfd8b4562beadcdd21df09ab41e4573e91a1eaf4750daaf34d7bdb6c35fa098", + "module": "fixes/latest_map_stated_hours.py", + "module_sha256": "b210e780a31cfa0a3a0da2c49a8068ee14f00029b3ed39c00be67a1e917eda21", + "differs_from_latest_conventions": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "latest_conventions": 0.0, + "latest_map_stated_hours": 3576.0 + } + ] + } + }, + "sweeps": [ + { + "sweep": "latest_alt_r02_ira_219g", + "csv": "latest_alt_r02_ira_219g.csv", + "module": "fixes/latest_alt_r02_ira_219g.py", + "adds": "the IRA deduction limit fix (IRC 219(g) active-participant phase-out; reference_audit/2026-09-22/fixes/r02_ira_219g_v2.py)", + "september_22_root_cause": "r02_ira_219g", + "baseline": "latest_conventions", + "csv_sha256": "16ac691fcfc530206540bfda10b4865707ba390d86524bda0f3ec69e9a9b87ab", + "module_sha256": "3e71bab1db867988544e6f87f6dda4dc7c719d51320fbbb7558cd574a0374f6e", + "outputs": 1984, + "moves": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 107833.15625, + "latest_conventions": 107833.15625, + "recomputed": 108525.578125 + }, + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 41267.01171875, + "latest_conventions": 41267.01171875, + "recomputed": 41492.3203125 + }, + { + "scenario_id": "scenario_049", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 30543.908203125, + "latest_conventions": 30543.908203125, + "recomputed": 30702.58984375 + }, + { + "scenario_id": "scenario_052", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 104211.40625, + "latest_conventions": 104211.40625, + "recomputed": 104225.2578125 + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 9577.90234375, + "latest_conventions": 9577.90234375, + "recomputed": 9579.767578125 + }, + { + "scenario_id": "scenario_082", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 5598.55224609375, + "latest_conventions": 5598.55224609375, + "recomputed": 5599.1435546875 + }, + { + "scenario_id": "scenario_099", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 10679.75, + "latest_conventions": 10679.75, + "recomputed": 10711.484375 + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 4640.77734375, + "latest_conventions": 4640.77734375, + "recomputed": 4652.375 + }, + { + "scenario_id": "scenario_110", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 23897.443359375, + "latest_conventions": 23897.443359375, + "recomputed": 23941.08203125 + }, + { + "scenario_id": "scenario_110", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 4057.47216796875, + "latest_conventions": 4057.47216796875, + "recomputed": 4062.927001953125 + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 40416.66796875, + "latest_conventions": 40416.66796875, + "recomputed": 40423.17578125 + }, + { + "scenario_id": "scenario_120", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 11917.18359375, + "latest_conventions": 11917.18359375, + "recomputed": 11919.0576171875 + } + ] + }, + { + "sweep": "latest_conventions_plus_r25_niit_excluded", + "csv": "latest_conventions_r25_niit_excluded.csv", + "module": "fixes/alt_conventions_r25.py", + "adds": "the reading of federal income tax before refundable credits without the net investment income tax (reference_audit/2026-09-22/fixes/r25_niit_excluded.py)", + "september_22_root_cause": "r25_niit_in_federal_output", + "baseline": "latest_conventions", + "csv_sha256": "3ca822db83d0abdab90f7e6ce151b5f53168163bd39a79469ceb987da2eda455", + "module_sha256": "0f77fdd44e758581f7e563b0f7b70c4d4359a833b08a4ac45a614c380938775e", + "outputs": 1984, + "moves": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 107833.15625, + "latest_conventions": 107833.15625, + "recomputed": 102111.1796875 + }, + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 68334.734375, + "latest_conventions": 68334.734375, + "recomputed": 68112.09375 + }, + { + "scenario_id": "scenario_052", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 104211.40625, + "latest_conventions": 104211.40625, + "recomputed": 103992.984375 + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 40416.66796875, + "latest_conventions": 40416.66796875, + "recomputed": 40253.26953125 + } + ] + }, + { + "sweep": "latest_alt_snap_mortgage_residence", + "csv": "latest_alt_snap_mortgage_residence.csv", + "module": "fixes/latest_alt_snap_mortgage_residence.py", + "adds": "the reading that listed mortgage interest is on the occupied home, a SNAP shelter cost", + "september_22_root_cause": "r15_snap_mortgage_interest", + "baseline": "latest_conventions", + "csv_sha256": "3dea0be87a30b52d8e693fe8bf45fd2339979ec2f340ac342ee7f416eb7114ac", + "module_sha256": "00cd13886a8c3d03c4e84944d35f7e8a1448af26838e8bc0761b914b887c70cd", + "outputs": 1984, + "moves": [ + { + "scenario_id": "scenario_056", + "variable": "snap", + "baseline": 95.0, + "latest_conventions": 95.0, + "recomputed": 159.0 + }, + { + "scenario_id": "scenario_118", + "variable": "snap", + "baseline": 2868.0, + "latest_conventions": 2868.0, + "recomputed": 3576.0 + } + ] + }, + { + "sweep": "latest_alt_unlisted_hours_40", + "csv": "latest_alt_unlisted_hours_40.csv", + "module": "fixes/latest_alt_unlisted_hours_40.py", + "adds": "the reading that a person with no stated weekly hours works 40", + "september_22_root_cause": "r14_unlisted_weekly_hours_v2", + "baseline": "latest_map_stated_hours", + "csv_sha256": "466a20e7b3281606b2a9c845b37e05ad6201b295a00e6a7a3f4da6440f47c01e", + "module_sha256": "5e6538a903112243ad85051ccba26b6133bdfbd5b9e5ba3dfa81ba690c061b32", + "outputs": 1984, + "moves": [ + { + "scenario_id": "scenario_056", + "variable": "snap", + "baseline": 95.0, + "latest_conventions": 95.0, + "recomputed": 1140.0 + }, + { + "scenario_id": "scenario_066", + "variable": "snap", + "baseline": 3576.0, + "latest_conventions": 0.0, + "recomputed": 3576.0 + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "baseline": 0.0, + "latest_conventions": 0.0, + "recomputed": 288.0 + } + ] + }, + { + "sweep": "latest_alt_salt_refund_no_prior_benefit", + "csv": "latest_alt_salt_refund_no_prior_benefit.csv", + "module": "fixes/latest_alt_salt_refund_no_prior_benefit.py", + "adds": "the reading that a listed state and local tax refund is not income (26 U.S.C. 111(a)); new on 2.15.17", + "september_22_root_cause": null, + "baseline": "latest_conventions", + "csv_sha256": "a9e21a71d942fd4da9dea1e7fd741b8306235a676150949b4113afe0dfa8db11", + "module_sha256": "d8a314093466caf5fe0a226f1060acba2e4364c4b87befbd80639ebf41cc4a2c", + "outputs": 1984, + "moves": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 107833.15625, + "latest_conventions": 107833.15625, + "recomputed": 106505.8984375 + }, + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 41267.01171875, + "latest_conventions": 41267.01171875, + "recomputed": 41219.8671875 + }, + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 3818.148193359375, + "latest_conventions": 3818.148193359375, + "recomputed": 3788.988037109375 + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 24164.45703125, + "latest_conventions": 24164.45703125, + "recomputed": 23772.80078125 + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 11295.388671875, + "latest_conventions": 11295.388671875, + "recomputed": 11294.5380859375 + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 9577.90234375, + "latest_conventions": 9577.90234375, + "recomputed": 9563.052734375 + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 24961.33984375, + "latest_conventions": 24961.33984375, + "recomputed": 24391.796875 + }, + { + "scenario_id": "scenario_117", + "variable": "state_income_tax_before_refundable_credits", + "baseline": 7984.53369140625, + "latest_conventions": 7984.53369140625, + "recomputed": 7984.05419921875 + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "baseline": 40416.66796875, + "latest_conventions": 40416.66796875, + "recomputed": 40021.81640625 + } + ] + } + ] +} diff --git a/reference_audit/2026-09-28/verification/reviews/az_snap_bbce_200.md b/reference_audit/2026-09-28/verification/reviews/az_snap_bbce_200.md new file mode 100644 index 00000000..3c524a38 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/az_snap_bbce_200.md @@ -0,0 +1 @@ +{"agree": true, "problems": ["Non-blocking: the upstream_changes list omits e8326c59ed (2026-08-06, 'Return the exact BBCE gross income standard'), which replaced caccdcd30b's published-chart rounding with the exact percentage x guideline. It is immaterial here: the household has $63.00/month of headroom under 200% (limit 2,608.33 vs gross 2,539.33) and $114.67 from October (2,660), so the exact $2,608.33, the rounded-up $2,609 and CNAP's published $2,610 all give the same result.", "Non-blocking: the summary gives the regular-rules net limit as '$1,304'. 100% of the 2025 guideline is $1,304.17/month and is published as a whole dollar. Net income of $2,138 fails either way, and the $58,700 bank balance fails the $4,500 elderly/disabled asset test on its own, so Jan-Feb stay at 0.", "Observation, not a reason to hold: all 42 model predictions on the board for scenario_013 snap are 0 (several cite a 185% Arizona BBCE limit). Adopting 240 makes every model miss this output. That follows from the publication-date rule, since Arizona published the change on 2026-03-23/24 and USDA on 2026-06-29, both before the 2026-07-03 freeze. The lead should know about it anyway.", "Settlement gap, now closed: the settlement did not look for Arizona legislation that could have capped the ECE limit after March. The House FY2027 human-services BRB, HB4147, would have added A.R.S. 46-233 ('may not establish a gross income limit of more than one hundred eighty-five percent of the federal poverty level' for BBCE). Governor Hobbs vetoed it on 2026-05-05 (azleg API). The enacted BRB, HB4162 (Laws 2026 ch. 134, signed and filed 2026-06-13), has no gross-limit cap. Neither do chapters 126, 128, 138, 140 or 254. No other 2026 bill whose title or NOW title mentions SNAP, nutrition, DES or human services was chaptered. So no pre-freeze law returns Oct-Dec to 185%, and the settlement stands."], "corrected_per_output": [{"scenario_id": "scenario_013", "variable": "snap", "action": "adopt_latest", "proposed_reference": 240, "reason": "Arizona DES CNAP 'What's Changed on 03/23/2026' (Wayback 20260417195741; sha256 f8173335..., matching the investigator's copy; PDF CreationDate 2026-03-24) raised the NA Expanded Categorical Eligibility gross limit from 185% to 200% FPL 'for the benefit month of 03/2026 and ongoing', and recalled a 02/23/2026 cut to 130%. CNAP FAA5.I01B has one ECE gross standard. Elderly/disabled units get special considerations (SEC) only when they meet ECE, and ECE units are treated as meeting resources and the gross and net tests. The USDA FNA BBCE chart (live PDF sha256 3ebb8ab0... matches; CreationDate 2026-06-29; page capture 2026-06-30 22:08 UTC) lists Arizona as 'All households are eligible', no asset limit, 200%. It has no separate elderly/disabled row, whereas Arkansas has one. All of this was published before the 2026-07-03 freeze, and no enacted 2026 Arizona law overrides it (HB4147's 185% cap was vetoed 2026-05-05). The household has stated gross income of $2,539.33/month (30,472/12). That fails 185% ($2,412.71) in Jan-Feb, and the $58,700 bank balance rules out regular eligibility, so Jan-Feb pay 0. It passes 200% in Mar-Sep ($2,608.33) and Oct-Dec ($2,660; it would also pass on the FY2026 guideline). Net income is about $2,138 (at least about $2,096 even with every possible medical deduction), so 30% exceeds the $298 maximum. Under 7 CFR 273.10(e)(2)(ii)(C) an eligible 1-person household gets the minimum, round(8% x $298) = $24, held for Oct-Dec by c_snap_hold_fy2026 (raw 2.15.17 gives $25 there, from the post-freeze FY2027 memo). Total 10 x $24 = $240. The prompt states every input the result depends on and says not to infer unlisted expenses or rent. The result is also robust to other readings: a benefit above $24 needs net income under about $913, which would take more than about $2,230/month of unstated shelter costs (the prompt lists a $43,000 mortgage balance but no payment). The engine's imputed medical deduction ($192.73) doesn't matter either."}], "what_i_checked": "Harness (2.15.17 venv, sweep_latest.py) for scenario_013. With fixes/latest_conventions.py, snap = 240 (board 0, v11 0) and none of the other 15 outputs move (conv_s013.csv). With no fix, 243 (raw_s013.csv). My month-by-month trace (rv_trace.py -> trace_conv.log) shows tanf_non_cash_gross_income_limit at 2,412.71 in Jan-Feb, 2,608.33 in Mar-Sep and 2,660 in Oct-Dec. It also shows is_tanf_non_cash_hheod = 1, BBCE eligibility from March, net income 2,138, expected contribution 642, snap_min_allotment 24 in all months under the hold, and snap 0,0 then 24 x 10 = 240. My own attribution module (rv_attr_az_185.py: AZ gross and gross_hheod at 1.85 for 2026 on top of latest_conventions), swept over all 1,984 outputs, differs from out/latest_conventions.csv in exactly one output, scenario_013 snap 240 -> 0. The other 1,983 are identical. A second attribution holding only gross_hheod at 185% (rv_attr_hheod_only.py) also returns scenario_013 snap to 0, so upstream's reading that 200% applies to the elderly/disabled standard is load-bearing. I checked that reading against CNAP FAA5.I01B (OCR of the archived pre-update version) and the USDA chart. Upstream (read-only): 0881eff4b8 (AZ gross.yaml and gross_hheod.yaml 2026-03-01: 2), caccdcd30b, e8326c59ed, 6e996f17c8 and d27d6c3cc3 (min_allotment.published_adjustment CONTIGUOUS_US +1 from 2026-10-01). I read the 2.15.17 engine code for tanf_non_cash_gross_income_limit, tanf_non_cash_fpg, meets_tanf_non_cash_* (AZ net_applies false, asset_limit inf, fpg_year_start_month 10), is_snap_eligible, meets_snap_categorical_eligibility, snap_normal_allotment and snap_min_allotment. Primary sources fetched this session: the Wayback DES What's Changed 03/23/2026 PDF (pdfinfo and text; hash matches the investigator's file), the live USDA BBCE-States-Chart-June2026.pdf (hash matches; Arizona row read), the Wayback FNA page 20260630220846 ('Page updated: June 29, 2026'; AZ 200%), the investigator's saved 2026-06-01 capture (Dec 29, 2025 revision, AZ 185%) and 2026-07-13/09-13 captures (still 200%), the eCFR 7 CFR 273.10(e)(2)(ii)(C) minimum-benefit text, and the azleg.gov API and chaptered texts (HB4147 vetoed 2026-05-05; HB4162 = ch. 134 signed 2026-06-13, no BBCE cap; the 2026 session's bill index screened for SNAP/DES/human-services bills; ch. 126/128/138/140/254 have no BBCE clause). An engine-independent recompute (rv_recompute.py) gives 240, with $126.62/month over 185% and $63.00 headroom under 200%, so capital-gain treatment and CNAP's whole-dollar rounding can't change the result. I rebuilt the prompt the models saw (prompt_s013.txt). It lists every input used and says not to infer expenses, and all 42 models answered 0. I did not bypass the Cloudflare challenge on dbmefaapolicy.azdes.gov, so the live CNAP text after April 2026 is unverified, but USDA's June 29 chart and the enacted-law screen cover the period to the freeze."} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_niit_scope_020.json b/reference_audit/2026-09-28/verification/reviews/excl_niit_scope_020.json new file mode 100644 index 00000000..7ff9eb81 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_niit_scope_020.json @@ -0,0 +1,18 @@ +{ + "agree": true, + "problems": [ + "Not a problem with the verdict: the investigator's note that alimony is moot understates how close this row is to a second unlisted input. The prompt lists 'first home mortgage balance: $367,000' but no mortgage interest. On 2.15.17 the filer takes the standard deduction by only $548.47 (itemized 15,747.53 vs 16,296 standard). A reader who imputes interest over about $549 would itemize. Like the undated alimony instrument (worth +1,998.09 if it is not deductible; reviewer recompute 70,332.82), this is moot only because the row stays excluded. No other scored scenario_020 output depends on AGI or itemizing: TX state tax is 0, payroll is 16,689, and every benefit is 0.", + "A third reading exists that the exclusion text does not name. PolicyEngine's income_tax_before_refundable_credits adds NIIT but not the 0.9% Additional Medicare Tax (900; it sits in payroll_tax, reference 16,689). So the reference (68,334.73) is neither Form 1040 line 22 (68,112.09) nor 'all Schedule 2 income-type taxes' (69,234.73). This supports keeping the exclusion and does not change it.", + "Minor citation detail left unverified: 'IRB 2025-45, 2025-11-03' for Rev. Proc. 2025-32. The irs.gov PDF was created 2025-10-17 and refers to amendments 'after October 9, 2025', so it predates the 2026-07-03 freeze either way. This does not affect the outcome." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_020", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 68334.734375, + "reason": "Confirmed. On 2.15.17, sweep_latest.py gives 68334.734375 both raw and with latest_conventions.py. latest_conventions plus r25_niit_excluded gives 68112.09375, exactly the exclusion record's alternative_value, so the two readings still differ by 222.64 (more than the $1 tolerance). The r25 root cause is the benchmark_specs.json output definition ('federal individual income tax after nonrefundable credits and before refundable credits'), which is unchanged. The engine formula (income_tax_before_refundable_credits.py; git diff 06665727d8..79be99f671 is empty; cmp of the installed files is identical) still adds net_investment_income_tax, and net_investment_income_tax.py has the same sha1 in both engines. The move comes only from #9616 (3f029abf5b, 2026-09-27), which fixed the column-shifted sales tax table. The IRS figure it carries, the 2025 TX size-1 $300k+ cell of 1,595, was published in December 2025, before the freeze. No hold module is needed, because raw and conventions agree." + } + ], + "what_i_checked": "Harness: I re-ran sweep_latest.py on scenario_020 (2.15.17 venv) four ways: no fix, fixes/latest_conventions.py, the investigator's alt_conventions_r25.py, and r25_niit_excluded.py alone. Results were 68334.734375, 68334.734375, 68112.09375 and 68112.09375. The other 15 scenario_020 outputs did not move. CSVs are in latest/excl_niit_scope_020/review/sweep_*.csv. The existing out/r19_irs_sales_tax_convention.csv (1.755.4 plus r19) also gives 68334.734375.\n\nRecords: reference_exclusions.json has scenario_020 with reason reference_depends_on_unlisted_input, frozen 68056.71 and alternative 68112.09375. In meta.json, c_irs_sales_tax_2025 has excluded_outputs_untouched=true and changed only scenario_000. root_causes.json lists r25 as class unlisted_input. The prompt definition in benchmark_specs.json does not mention NIIT. I rendered the scenario_020 prompt with describe_household.\n\nEngine: I read the parameters in both venvs. 1.755.4 has TX size 1, bracket 19 at 2440 for 2022-23, and 2440 \u00d7 176.713/158.872 = 2714.01 in 2026. 2.15.17 has TX 2023 = 1992, TN 2023 = 2440, TX 2025 = 1595, and 2026 = 1631.15, uprated from 2025. The no-local list is CT, DC, IN, KY, MA, MD, ME, MI, NJ, RI, so TX is not on it.\n\nUpstream: I read the git show for 3f029abf5b (#9616, 2026-09-27, Fixes #9595) and 81fd50a98a (#9306, 2026-08-19).\n\nPrimary sources:\n- IRS 2025 Instructions for Schedule A, local copy with sha256 b0999b12\u2026 (identical to a second copy fetched separately). Created 2025-12-08, modified 2025-12-18. The table row TX '$300,000 or more' has size 1 = 1595. In the worksheet, TX is neither a line-2 local-table state nor one of the ten jurisdictions sent to -0-, so local = line 1 \u00d7 rate/6.25. Table income is line 11b.\n- IRS 2023 i1040sca, fetched from irs.gov/pub/irs-prior (created 2024-01-03). TN size 1 $300k+ = 2440 and TX = 1992, which confirms the column shift.\n- Rev. Proc. 2025-32 from irs.gov (PDF created 2025-10-17). Table 3 single: $58,448 plus 35% over $256,225. Standard deduction $16,100. Maximum 0% rate amount $49,450; 15% up to $545,500. AMT exemption $90,100.\n- 26 U.S.C. 170(p): $1,000 for non-itemizers, from uscode.house.gov. 170(b)(1)(I): the 0.5% floor.\n- 26 U.S.C. 225(b): $12,500, reduced $100 per $1,000 of MAGI over $150,000, so it is 0 here.\n- Tex. Tax Code 321.101(f): 2% combined local cap.\n\nIndependent calculation: I wrote review/indep.py (base plus rate times excess, and a QDCG worksheet). It gives 68,334.736 and 68,112.094 at local rates of 0%, 1.25% and 2%. Itemizing would need about a 3.40% local rate. The 1.755.4 projection gives 68,056.72, which reproduces the board value. AMT is clearly 0: tentative minimum tax is about 54.6k against 68.1k regular tax." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r01_ira_compensation.json b/reference_audit/2026-09-28/verification/reviews/excl_r01_ira_compensation.json new file mode 100644 index 00000000..f1158493 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r01_ira_compensation.json @@ -0,0 +1,19 @@ +{ + "agree": true, + "problems": [ + "The law claim is overstated. The investigator calls 4,599.62 the law-correct value and says 2.15.17 is $1.15 off it, beyond tolerance. That is only true under the board's formula convention. Wis. Stat. 71.05(22)(dp)2 (docs.legis.wisconsin.gov, fetched this session, review/wis_71_05_22.html) says the standard deduction \"shall be determined\" under a DOR table published in the instructional booklets. DOR's 2025 Form 1 table (booklet PDF created 2025-10-17, pp. 35-37) equals round(formula at the bracket midpoint) on all 242 MFJ $500 rows (checked in the review). Building the 2026 table the same way from the pre-freeze 1-ES parameters (25,840 less 19.778% over 29,040) puts the defective AGI (114,403.38) and the law-correct AGI (114,421.41) in the same 114,000-114,500 row, with a standard deduction of 8,987. On that reading: r01's effect on this output is $0.96, inside tolerance. The table-law value is about 4,597.83. The regenerated reference 4,598.476 is $0.64 from it. The exclusion's alternative_value 4,599.62 is $1.79 from it (review/table_variant.py and .txt). The 2026 Form 1 table was not published before the 2026-07-03 freeze, so this reading is an inference. It does not overturn keep_excluded, because the defect is unfixed and a reference that differs by reading favors exclusion. But the exclusion record should say that the $1.15 gap and the alternative_value hold under the board's formula convention (c_wi_published_2026), not under statute. Whether Wisconsin's table or formula governs is a board-wide convention question that touches every WI output. It is not specific to this cluster.", + "The hand check is described as \"engine-free ... with DOR's 2026 MFJ tables\". In fact hand_check.py takes AGI from the engine (114,403.367188), and it uses the 1-ES formula schedules, not a Form 1 table. I derived AGI from the stated facts without the engine: wages 97,295 - 401(k) 2,315.40 + partnership 19,350 + 401(k) distributions 20,000 - farm loss 16,800 - capital loss capped at 3,000 - IRA 108.192 - 18.032 = 114,403.376. It matches, so the numbers stand: formula tax 4,598.4769 with the defect and 4,599.6216 without it (review/indep_calc.py).", + "upstream_changes leaves out several commits in range that touch Wisconsin tax or ALD code: 35bbe1ba0c, 84b94fc350 and 372228938d (2026-07-05), f73485d33c (2026-09-14) and 6d60c7944a (the ALD loss-threshold change of 2026-09-28). None affects this household. AGI, WI subtractions and WI credits (both 0) are identical on the two engines, and the raw +2.9507 move breaks down fully into brackets (+2.9580) and rounding (-0.0071), which I recomputed. The omission is harmless but the list is incomplete.", + "Which value to store is a convention choice. The Sept 22 revisions set excluded_outputs_untouched=true, so the board kept raw engine values for excluded outputs (4,605.997 here). The investigator proposes 2.15.17 + conventions (4,598.476074) rather than raw 2.15.17 (4,608.947754). That matches the brief (new references = 2.15.17 + conventions), and scoring is unaffected either way, but the regeneration must treat every excluded output the same way." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_064", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 4598.476074, + "reason": "Root cause r01 is still present in 2.15.17. traditional_ira_contributions.py, ira_contribution_limit.py, ira_contribution_scale.py, above_the_line_deductions.py and gov/irs/ald/deductions.yaml have identical blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. The dependent's $18.03 IRA contribution (stated in the prompt; $0 wages) is still in the joint return's ALD (19,926.22 = 19,800 loss ALD + 108.19 + 18.03). Under 26 U.S.C. 219(a), (b)(1)(B) and (c) (LII, re-fetched this session) it is not deductible there. The review harness gives 2.15.17 + latest_conventions = 4,598.476074, which equals 1.755.4 + r19_wi_convention (re-run this session). Adding r01_v2 gives 4,599.621582, and also reproduces the 064 federal (4,441.455078) and 003 federal (22,400.654297) alternative values exactly; the only other output it moves is scenario_085 federal, by +0.089. Under the board's formula convention the defect moves this output by $1.15, beyond tolerance, so it stays excluded. Caveat: Wis. Stat. 71.05(22)(dp)2 makes DOR's $500-bracket table determinative. On that reading r01's effect is $0.96, and 4,598.476 is $0.64 from a table-law value of about 4,597.83, but the 2026 table was not published before the freeze. Exclusion is the safe call under either reading. Update frozen_value to 4598.476074 and engine_version to 2.15.17. Keep alternative_value at 4599.621582, and note in the exclusion record that it is formula-convention based." + } + ], + "what_i_checked": "What I checked:\n- Harness runs. I ran the harness (sweep_latest.py, 2.15.17 venv) three times over all 1,984 outputs: baseline (41 moved), latest_conventions (29 moved), and latest_conventions + r01_ira_compensation_v2 (31 moved). Outputs are in triage/latest/excl_r01_ira_compensation/review/rv_*.csv.\n - The results match sweep/out/latest_conventions.csv and the investigator's probe CSV to 0.0.\n - scenario_064 state: raw 4,608.947754, conventions 4,598.476074, conventions + r01_v2 4,599.621582.\n - Collateral moves from r01_v2: scenario_003 federal +245.96, scenario_064 federal +2.17, scenario_085 federal +0.089.\n - I also re-ran 1.755.4 + r19_wi_convention (sweep.py) and got 4,598.476074.\n- Upstream code. The IRA and ALD files have identical git blobs at 06665727d8 and 79be99f671, and upstream/main is 79be99f671. `git log -S ira_compensation` over the range is empty. I read the installed 2.15.17 source:\n - the IRA limit is 7,000 + 1,000 with no compensation cap;\n - the ALD list includes traditional_ira_contributions;\n - above_the_line_deductions adds over the whole tax unit.\n- Wisconsin commits. I read cd4d339667 (WI brackets 20,150 / 69,260 / 443,630, citing the 2026 1-ES) and 94e40f1471 (rounding metadata). Recomputed, the raw move splits into +2.9580 from brackets and -0.0071 from rounding.\n- Scenario and prompt. I read scenario_064's JSON and rendered its prompt; the dependent's $18 traditional IRA contribution is stated, and no wages are listed.\n- Primary sources, re-fetched this session:\n - Wisconsin 2026 1-ES instructions: sha256 identical to the investigator's copy. PDF created 2026-01-16, modified 2026-01-20, server Last-Modified 2026-02-06. The MFJ standard deduction (25,840 less 19.778% over 29,040), Schedule B and the $700 exemptions match.\n - 26 U.S.C. 219 on LII: (a), (b)(1)(B), (c) and (f)(1) text confirmed; the latest amendment listed is Pub. L. 117-328.\n - IRS Notice 2025-67: PDF created 2025-11-13; it gives $7,500 / $1,100 and a 2026 joint 219(g) range of $129k-$149k, which does not bind here.\n - Wis. Stat. 71.01(13): Wisconsin AGI is federal AGI with modifications.\n - Wis. Stat. 71.05(22)(dp)2: the standard deduction is determined under DOR's published table. This is new evidence from this review.\n- Independent calculation. I recomputed AGI from the stated facts without the engine (114,403.376), then the WI tax: 4,598.4769 with the defect and 4,599.6216 without it (formula).\n- Table check. I verified that DOR's 2025 MFJ standard-deduction table equals the formula at each bracket midpoint on 242 rows. Applied to 2026, the table reading gives a $0.96 r01 effect, and the reference is $0.64 from a table-law value of about 4,597.83.\n- Verdict. keep_excluded and 4,598.476074 are sound. The investigator's statement of the Wisconsin law needs the table caveat.\n\nFiles are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/excl_r01_ira_compensation/review/: indep_calc.py and .txt, table_variant.py and .txt, rv_baseline.csv, rv_conventions.csv, rv_conv_r01v2.csv, rv_1755_r19wi_064.csv, wi1es.pdf, lii219.html, n2567.pdf, and wis_71_05_22.html." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r02_ira_219g_federal.md b/reference_audit/2026-09-28/verification/reviews/excl_r02_ira_219g_federal.md new file mode 100644 index 00000000..da69cdab --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r02_ira_219g_federal.md @@ -0,0 +1,32 @@ +{ + "agree": true, + "problems": [ + "Alternative-value bookkeeping is framed incompletely. The settlement says the Sept 22 alternatives stand 'if #9122 is not adopted', but that holds only if a hold/convention removes the refund from the scored reference. The sibling cluster's own module (sweep/fixes/latest_alt_salt_refund_no_prior_benefit.py) is written as 'Alternative reading (not a reference fix, not a hold)', i.e. an unlisted-input exclusion; under that outcome the reference stays raw 2.15.17 with the refund in AGI. The mapping should be three-way. (a) Refund inclusion adopted: frozen_value becomes latest (107833.15625 / 9577.90234375 / 40416.66796875) and alternative_value becomes 108525.578125 / 9579.767578125 / 40423.17578125. (b) Refund reading excluded as an unlisted input: same frozen_value, and the root cause becomes r02_ira_219g plus the refund cause. On the precedent of r02_ira_219g+r11_ca_itemized_conformity (both readings applied together), the combined alternative is 107198.3359375 / 9564.9169921875 / 40028.3203125, which I reproduced exactly on 2.15.17; the r02-only alternative is the (a) value. (c) A hold keeps the refund out: frozen_value stays the board and the Sept 22 values stand. In (a) and (b) the records' frozen_value and engine_version ('policyengine-us 1.755.4') must also be updated; the settlement is silent on those fields.", + "scenario_005 federal has a second, unlisted-input dependence the settlement does not mention. Each spouse lists 'traditional 401k contributions desired: $23,154' and 'roth 401k contributions desired: $4,086' ($27,240) against the 2026 402(g) limit of $24,500 (Notice 2025-67). The engine scales both pro rata (elective_deferral_contribution_scale: traditional $20,825, Roth $3,675). The prompt does not say which designation absorbs the $2,740 excess, and a careful reader could keep the $23,154 traditional deferral and cut the Roth, or the reverse. On 2.15.17 with conventions (harness-equivalent probe, confirmed by my engine-free calc): Roth cut first 106,342.59; pro rata 107,833.16; traditional cut first 108,096.20, a $1,753.60 spread. The alternative on the same three readings is 107,035.02 / 108,525.58 / 108,788.62. scenario_005 is the only one of the 100 scenarios above the 402(g) limit. This does not change keep_excluded. It does mean 005 federal must not be un-excluded merely because r02 is later fixed upstream, and 005's alternative value is not unique. The exclusion record should carry this second cause. The investigator's indep.py hard-codes the engine's pro rata split as a 'convention reproduced on purpose' without flagging it as a prompt ambiguity.", + "The 'engine-free recomputation' is less independent than described for scenario_120. Without disclosing it, it repeats the engine's absence of IRC 469. loss_ald deducts the full $2,446.59 farm-rental loss against $2,000 of rental income, although rental activities are passive under 469(c)(2) and the 469(i) allowance is zero at MAGI of about $234k. It also applies the $750k 163(h)(3)(F) cap to $1,000,000 of listed mortgage debt whose origination date is unlisted (debt from before 2017-12-16 keeps the $1M limit). My recomputation gives 40,441.42 latest and 40,447.93 alternative with 469 applied (+24.75), and 40,192.03 and 40,198.54 with grandfathered debt (-224.64). So 'matches all 12 engine values' validates engine arithmetic, not law-correct values. These effects hit board, latest and every alternative alike, so keep_excluded is unaffected. But 120 federal carries dependences beyond r02 (the mortgage-debt date, and whether the listed farm rent is already 469-limited), and its alternative value is engine-relative." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_005", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17. gov.irs.ald.deductions (2026) still lists traditional_ira_contributions; the list and the retirement variables are identical to 1.755.4 apart from uprating attributes (44001a4c24). No 219(g) or active-participant code exists, and upstream #8388 scoped 219(g) out with no follow-up. Both spouses list 401(k) deferrals, so both are active participants under 219(g)(5)(A)(i); ira_219g_magi is $543,076, far above $149,000, so the deductible limit is $0 and the engine's $2,163.84 deduction is wrong. The +$1,327.26 move is #9122 alone: salt_refund_income=0 on 2.15.17 gives exactly 106505.8984375. The alternative on 2.15.17 is 108525.578125 with the refund in AGI; 107198.3359375 applies with both readings or under a hold. Both were reproduced by the harness and by my engine-free calculation to $0.01. It also depends on the unlisted 402(g) excess allocation (106,342.59 to 108,096.20 on latest), so it should stay excluded even after an upstream r02 fix unless that allocation is ruled. NIIT is included (r25), as in the board." + }, + { + "scenario_id": "scenario_082", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17 (same evidence as 005). The HoH head lists a $181 traditional 401(k) deferral with $100,195 of wages, so is an active participant; ira_219g_magi is $117,661, above $91,000, so the limit is $0 but the engine deducts $8.48. The +$14.85 move is #9122 alone (67.50 at 22%); refund=0 on 2.15.17 gives exactly 9563.052734375. The alternative on 2.15.17 is 9579.767578125 with the refund in AGI, or 9564.9169921875 with both readings or under a hold. My engine-free calc reproduces these (standard deduction $24,150 + 170(p) $700; CTC $2,200; CDCC 20% of $3,000). No other unlisted dependence was found. The only non-excluded output r02 moves is 082 state, by $0.59, under the $1 tolerance." + }, + { + "scenario_id": "scenario_120", + "variable": "federal_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "r02 is unfixed in 2.15.17. The single head (76) lists a $617 traditional 401(k) deferral with $165,597 of wages, so is an active participant; ira_219g_magi is $234,414, above $91,000, so the limit is $0 but the engine deducts $28.85. The +$394.85 move is #9122 alone. Taxable income rises $1,645.22, because the engine's ct_withheld_income_tax proxy (CT single rates on federal AGI less $15,000) raises SALT by $113.77 and the 0.5% charitable floor rises $8.75; CT's own tax is unchanged at 11917.18 because CT subtracts salt_refund_income. Refund=0 gives exactly 40021.81640625. The alternative on 2.15.17 is 40423.17578125 with the refund in AGI, or 40028.3203125 with both readings or under a hold. Beyond r02 the output also depends on the engine's missing 469 limit (+24.75) and the unlisted mortgage origination date (-224.64 if grandfathered), so it should not be un-excluded on an r02 fix alone." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv) on 005/082/120 with no fix, latest_conventions and latest_alt_r02_ira_219g (rv_none.csv, rv_latest_conventions.csv, rv_latest_alt_r02_ira_219g.csv). It reproduces board, latest and alternative values exactly. Full 1,984-output sweeps of both modules (rv_full_*.csv; 29 and 35 moved) match the investigator's CSVs bit for bit. r02 changes exactly the 11 r02-excluded outputs plus 082 state by $0.59. My own probe (rv_probe.py → rv_probe_21517.json) shows refund=0 reproduces the board exactly and r02 with refund=0 reproduces the Sept 22 alternatives exactly. It also shows ira_active_participant=1, ira_219g_deductible_limit=0 and ira_219g_magi of 543,076 / 117,661.125 / 234,414.203, plus the 402(g) allocation variants for 005. Engine code read in both installed packages: gov/irs/ald/deductions.yaml is identical in 1.755.4 and 2.15.17; traditional_ira_contributions = desired × ira_contribution_scale; ira_contribution_limit; loss_ald (no 469); ct_withheld_income_tax; NII sources; CT subtractions (salt_refund_income); gross_income sources diff. A grep of 2.15.17 for 219(g)/active participant/ira_deduction found only an MO TANF comment. Upstream (read-only): git log 06665727d8..79be99f671 on the ALD list and retirement dir shows only 44001a4c24 (uprating attributes); -S '219(g)', active_participant and ira_deduction return nothing. 316e7832a1, 5581f5be1a, 5d3d1312fd and e990b4a6f8 exist with the stated dates and are ancestors of 79be99f671. gh search finds only #8388 (closed 2026-05-24, 219(g) explicitly out of scope). Primary sources fetched this session: Notice 2025-67 (IRS PDF, Last-Modified 2025-11-13: 402(g) $24,500; IRA $7,500/$1,100; 219(g) $81k-$91k, $129k-$149k, $242k-$252k); Rev. Proc. 2025-32 (IRS PDF, Last-Modified 2025-10-17: Tables 1-3, CG breakpoints, standard deduction, aged $2,050, AMT, CTC $2,200); and 26 U.S.C. 219(g)(1)-(8) (last amended Pub. L. 117-328), 164(b)(6)-(7), 170(b)(1)(I), 170(p), 63(b), 21(a)(2)/(c), 24(h), 1411(b) at uscode.house.gov. All of these are pre-freeze. I could not locate the uscode 'current through' line in the fetched HTML; it is immaterial. The prompt the models saw (data.json, June bundle) lists '... 401k contributions desired' for every IRA contributor, so active participation is stated and r02 is a genuine engine defect, not an unlisted input. I wrote an independent engine-free calculator (rv_indep.py → rv_indep.json). It matches all 12 board/latest/alternative values and the 005 allocation variants to within $0.01, and quantifies the 469 and grandfathered-mortgage variants for 120. A 402(g) scan of all 100 scenarios found only 005 above the limit. The 005 wage equals $69.20 × 50 h × 52, so no overtime premium is implied." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r07_idaho_health_premiums.json b/reference_audit/2026-09-28/verification/reviews/excl_r07_idaho_health_premiums.json new file mode 100644 index 00000000..6c773e04 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r07_idaho_health_premiums.json @@ -0,0 +1,24 @@ +{ + "agree": true, + "problems": [ + "Minor gap, and it does not change the action. For scenario_053, the investigator says the full $5,000 is subtractable but never addresses the salary-reduction bar. The 2025 Form 39R line 18 instructions (EIN00046 03-02-2026, printed p. 34) say premiums 'paid through a cafeteria plan or other salary-reduction arrangement' do not qualify. The household the model sees has gross wages of $66,969 and 'has employer-sponsored insurance', and nothing says whether the $5,000 is withheld pre-tax. So the 2176.057 alternative value holds only under an unstated reading that the premium is paid after tax. Under the pre-tax reading, the engine's no-subtraction value (2441.057 with the hold) would be the correct law. The exclusion therefore also has an unlisted-input side, and the investigator's line 'their full premiums ($2,080 and $5,000) are therefore subtractable' overstates the case for 053. The pre-tax argument holds for scenario_007, which has no wages. Either way the output stays excluded.", + "Minor imprecision. The investigator calls 651.317 and 2176.057 'law-correct'. They are correct only under the stated alternative reading: after-tax premiums, no Medicare Part B enrollment for scenario_007 (the engine has medicare_enrolled = 0, and a separate exclusion covers head_medicare_eligible), and the $950 auto-loan interest in scenario_053 not qualifying for Schedule 1-A (the engine has qualified_passenger_vehicle_loan_interest = 0, consistent with the benchmark's annotation convention). These are unscored alternatives, so nothing turns on it." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 755.7783203125, identical to v11_1755. latest_conventions gives 761.5570068359375 (+5.7787 = 0.053 x (4920.0313 - 4811)). latest_conventions + r07_v2 gives 651.3170166. The subtractions.yaml and main/single.yaml files are identical between 1.755.4 and 2.15.17, and the premium subtraction is still absent, so the r07 defect is still present. The board row is the raw 755.7783 because the c_id_hold_2025 revision has excluded_outputs_untouched: true. By hand: taxable SS 13,784.125, Idaho TI 19,180, and 0.053 x (19,180 - 4,811) = 761.557. With the premium subtraction, 0.053 x (17,100 - 4,811) = 651.317, which matches the exclusion record's alternative_value. Keep excluded; scoring does not change." + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Reproduced with sweep_latest.py on 2.15.17. Raw gives 2435.2783203125, identical to v11_1755. latest_conventions gives 2441.05712890625. latest_conventions + r07_v2 gives 2176.05712890625 (-265.00 = 0.053 x 5,000). id_subtractions is 0 and the r07 defect is still present. By hand: 0.053 x (50,868.68 - 4,811) = 2441.057, and 0.053 x (45,868.68 - 4,811) = 2176.057. The alternative also assumes the $5,000 is not paid through a pre-tax salary reduction. The prompt leaves that unstated (the household has wages and ESI), which gives an additional unlisted-input reason to keep this output excluded. Keep excluded; scoring does not change." + } + ], + "what_i_checked": "**Harness runs.** I used the brief's interpreter (policyengine-us 2.15.17) with sweep_latest.py. Scratch is in triage/latest/excl_r07_idaho_health_premiums/review/.\n- Baseline and latest_conventions on scenario_007 and scenario_053: raw equals v11_1755 (755.7783 and 2435.2783). Conventions give 761.557007 and 2441.057129, both moved +5.78 vs the board.\n- Full 1,984-output sweeps of latest_conventions alone and of my own composition (rev_conv_r07v2.py = latest_conventions + r07_idaho_health_premiums_v2): diffing them, r07 moves exactly two outputs, 007 by -110.24 and 053 by -265.00.\n\n**Board and sidecar.** reference_outputs.csv holds 755.7783203125 and 2435.2783203125. In the meta.json, the c_id_hold_2025 revision (fix r19_id_convention.py) has excluded_outputs_untouched: true and changed only scenario_076. reference_exclusions.json has frozen_value and alternative_value 651.317017 / 2176.057129 for both rows.\n\n**Engine.** In 2.15.17, subtractions.yaml has no premium item. single.yaml has the 2025 value 4,811 at 5.3% with gov.irs.uprating metadata, which gives 4,920.03 in 2026. Recursive diffs of parameters/ and variables/gov/states/id/tax/income between 1.755.4 and 2.15.17 show only the grocery, PBF, aged/disabled, CDCC, 529 and seasonal-rebate files differ. main/, subtractions.yaml and deductions/ are identical. In 2.15.17, medical_expense_health_insurance_premiums is 2,080 and 5,000. employer_sponsored_insurance_premiums is documented as employer-paid. id_pbf is outside this output.\n\n**Upstream.** git log 06665727d8..79be99f671 on the id parameter and variable paths returns exactly the 7 commits listed. -S 3022P and -S health_insurance return nothing. upstream/main is 79be99f671. gh search finds no Idaho premium-subtraction issue or PR.\n\n**Primary sources, fetched 2026-09-28.**\n- Idaho Code 63-3022P: text as quoted; history 2001 ch. 386, amended 2003 ch. 10.\n- 63-3024: 5.3%, last amended 2025 ch. 13, with the CPI indexing in (3).\n- 2026 H0589 (rate bill): only introduced and printed (bill history page).\n- Governor's release dated 2025-03-06: HB 40 cut the rate to 5.3%.\n- EIN00046_03-02-2026.pdf: Last-Modified 2026-03-05, CreationDate 2026-03-02. Printed p. 9 worksheet gives $4,811 single and $9,622 MFJ/HOH/QSS at 5.3%. Printed pp. 34-35 line 18 has the cafeteria-plan bar, the standard-deduction rule, and 'enter zeros on lines 8, 12, and 13' for non-itemizers.\n- tax.idaho.gov rate schedule: 'last updated December 29, 2025', with 2025 as the newest row.\n\n**Independent hand calculation.** Scenario_007: provisional income 44,922.5 gives taxable SS 13,784.125 and Idaho TI 19,180, so tax is 761.557 held and 651.317 with r07. Scenario_053: TI 50,868.68 gives 2441.057 held and 2176.057 with r07.\n\n**Rendered prompts.** The prompt shows ESI, wages and the premium lines, with no pre-tax indicator for 053." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r11_ca_itemized_conformity.md b/reference_audit/2026-09-28/verification/reviews/excl_r11_ca_itemized_conformity.md new file mode 100644 index 00000000..4b2d4e42 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r11_ca_itemized_conformity.md @@ -0,0 +1,36 @@ +{ + "agree": true, + "problems": [ + "005 state: the settlement calls 40,975.25 'law-correct' and says the exclusion's alternative_value 'should become 40,975.25 if #9122 is adopted'. But #9122 is already in 2.15.17, so the real question is what the prompt says, not whether to adopt it. The prompt shows 'state and local tax refund income: $4,148' and gives no prior-year facts. Under IRC 111 the refund is federal gross income only to the extent the prior-year deduction produced a tax benefit. The sibling cluster salt_refund_gross_income_9122 (fixes/latest_alt_salt_refund_no_prior_benefit.py) treats this as an unlisted-input ambiguity. The stated facts therefore support two values: 40,975.25 (refund taxable, the engine's reading) and 40,920.40 (refund not income). Neither is 'the' law-correct value. The exclusion record should keep both readings and add an unlisted-input reason. It should not overwrite alternative_value with 40,975.25 alone. Both numbers reproduce exactly (my rv_c_r11_r02 and rv_c_r11_r02_rev sweeps).", + "005 state, new ambiguity: the prompt lists 'tax-exempt interest income: $8,961' without saying who issued the bonds. California taxes federally exempt interest from non-California state and municipal bonds (FTB 2025 Schedule CA (540) instructions, rev. 04/2026, Part I line 2, column C). 2.15.17's ca_additions adds back only ca_hsa_addition, so the engine treats all of it as California-exempt. Reading it as non-California interest moves the output by +833.41: harness rv_c_muni.py gives 42,100.417969 on latest_conventions vs 41,267.011719, and my hand calculation gives 8,961.35 x 9.3%. No engine fix can resolve this, so 005 state must stay excluded even after r02 and r11 are fixed upstream. The settlement does not mention it.", + "005 state: the settlement cites the Sept 22 figure 'about -$471' for the two-way 401(k) reading (cut Roth deferrals first rather than the engine's proportional scaling to the $24,500 limit). That figure was computed on the 1.755.4 basis. On the current basis (conventions + r11 + r02) the reading moves the output by -491.85, under either #9122 reading (rv_independent_calc.py). This is minor, but it adds to the list of unlisted or two-way inputs behind 005.", + "upstream_changes leaves out the California AMT rework in the range: 2dd9762506 (#8873; drops the mortgage-interest line from amti/sources.yaml) and d11f711826 (#8925; ca_pre_exemption_amti no longer adds back the full pre-limitation itemized deductions, and the new ca_itemized_deductions_limitation variable is subtracted). The rework touches the path of r11_v2's Schedule P line-5 swap, and that module's docstring still describes the 1.755.4 AMTI. It changes no number here: ca_amt = 0 for all three outputs under both latest_conventions and conventions + r11 + r02. For example, 005 AMTI is 536,354.81 with a tentative minimum tax far below the regular tax. Reverting #9122 on conventions + r11 + r02 also gives the Sept 22 alternative 40,920.402344 exactly. So the settlement's attribution holds; only the list is incomplete.", + "The full-sweep claim that r02 moves only outputs whose exclusions name r02 holds only under the $1 tolerance. My full rv_c_r02 sweep and the investigator's own full_diag_conv_r02.csv both also move scenario_082 NY state_income_tax_before_refundable_credits by +0.59, and that output is not excluded. This does not affect scoring and is outside this cluster, but the claim as worded is not exact.", + "099, possible second prompt-reading issue (low confidence, benchmark-wide): each adult lists both 'health insurance premiums excluding Medicare Part B: $2,500' and 'other health insurance premiums: $2,500'. The engine counts only the first (medical_expense_health_insurance_premiums uses health_insurance_premiums_without_medicare_part_b), so medical expenses of 10,000 fall below the 7.5% floor of 11,979.70. A reader who adds both lines gets 15,000, a 3,020.30 medical deduction and -241.62 of CA tax. It does not change the action, since 099 stays excluded. The same double listing in 005 and 022 is harmless because they stay under the floor either way.", + "Citation detail: the settlement dates FTB Tax News October 2025 '2025 Indexing' to 2025-09-12, but that date belongs to a separate Tax News Flash on the same page (market-based sourcing). The issue is still pre-freeze, so nothing changes." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_005", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 41267.011719, + "reason": "Both r11 and r02 are still in 2.15.17. The CA itemized, charity and misc variables and parameters, traditional_ira_contributions and gov/irs/ald/deductions.yaml are identical over 06665727d8..79be99f671, and git grep finds no active_participant or 219(g) logic. The stored, unscored value is 2.15.17 + conventions = 41,267.011719; I reproduced it in the harness and by hand. The +215.50 gap to the board splits as +168.36 from c_ca_hold_2025 (the board kept raw v1.1 for this excluded output) and +47.14 from #9122: 23.14 through the line 29 limitation (248.86 x 9.3%) and 24.00 through the exemption credits (13 to 15 phase-out steps x $6 x 2). With #9122 reverted on the conventions, the value is 41,219.867188 = 1.755.4 + r19. The output also depends on three stated-but-unresolved or unlisted facts: whether the SALT refund was taxable (40,975.25 vs 40,920.40 after r11 + r02), who issued the tax-exempt interest (+833.41 if non-California), and how the over-limit 401(k) deferrals are cut (-491.85). The exclusion should therefore also carry an unlisted-input reason, and alternative_value should record both SALT-refund readings rather than a single 'law-correct' 40,975.25." + }, + { + "scenario_id": "scenario_022", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 2505.870117, + "reason": "r11 is still present. ca_itemized_deductions_pre_limitation carries the federal charitable_deduction (0.5% floor from 2026: 14,847.67 instead of 15,393.41) and the federal misc_deduction (0 since 2018). R&TC 17076(c) (Stats. 2025 ch. 231, SB 711 Sec. 11, effective 2025-10-01) and the conformity date in 17024.5(a)(1)(Q) (IRC as of 2025-01-01) mean California allows both. The law-correct value is 1,968.81; I reproduced it by harness and by hand. Federal AGI is 109,149.16 and CA AGI is 91,524.41; itemized deductions are 8,829 + 15,393.41 + 6,167.56 = 30,389.97; CA taxable income 61,134.44 gives tax of 2,274.81, less 306 of exemption credits. That equals the Sept 22 alternative. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 2,505.870117 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. I found no unlisted-input dependence: medical stays under the floor under any reading, and the prompt lists no refund and no tax-exempt interest. This is a pure engine defect, so the output could be un-excluded once r11 is fixed upstream. The value uses the rate schedule, as for every CA output (FTB's tax table for taxable income of $100,000 or less could differ by about a dollar)." + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": 4640.777344, + "reason": "r11 and r02 are still present. The engine's federal floor removes 797.93 of charity (-63.83). Both spouses defer into a 401(k), which makes them active participants, and MAGI of 159,729.30 exceeds the $149,000 top of the 2026 joint range (IRS Notice 2025-67, posted 2025-11-13), so the $144.26 IRA deduction is fully phased out (+11.54 after r11). Together these give 4,588.48, the Sept 22 alternative. I confirm the new educator-expense defect: FTB 2025 Schedule CA line 11 says 'California law does not conform to federal law regarding educator expenses', but 2.15.17's ca_additions adds only ca_hsa_addition. Adding back $337.50 x 8% = +27.00 gives a fully law-correct 4,615.48 (harness and hand calculation agree). The exclusion reason should cite this third defect. Raw 2.15.17 equals the board exactly, and 2.15.17 + conventions = 4,640.777344 = 1.755.4 + r19, so c_ca_hold_2025 alone moves it. Possible extra reading: if both premium lines are summed, the output moves -241.62." + } + ], + "what_i_checked": "Every scratch file is in /Users/maxghenis/PolicyEngine/pb-refs-reviews-0929/excl_r11_ca_itemized_conformity. (1) I re-ran sweep_latest.py on 2.15.17 for the three scenarios under my own composition modules (rv_*.py, loading the fix modules by path): raw; latest_conventions; + r11_v2; + r02_v2; + r11_v2 + r02_v2; + educator add-back; my own idempotent #9122 revert on conventions and on conventions + r11 + r02; and a non-California muni-interest reading. Results: raw 40,986.87 / board / board. Conventions: 41,267.011719 / 2,505.870117 / 4,640.777344. + r11: -521.09 / -537.06 / -63.83. + r02: +225.31 / 0 / +11.60. Both: 40,975.253906 / 1,968.805420 / 4,588.483887. Educator: 099 4,615.483887. Revert: 41,219.867188 and 40,920.402344. Muni: 42,100.417969. (2) I ran full 1,984-output sweeps of conventions + r11 and conventions + r02 and diffed them against latest_conventions.csv. r11 moves only the three cluster outputs. r02 moves 11 outputs by more than $1, all with exclusions naming r02, plus 082 NY state by +0.59. (3) A probe dumped the engine's intermediate values (AGI, CA AGI, itemized deductions pre and post the limit, taxable income, exemptions, AMTI and AMT, 401(k) and IRA amounts). (4) rv_independent_calc.py is pure Python with no engine and uses only scenario inputs and FTB/IRS parameters. It reproduces all 16 values above to the cent. (5) I rendered the exact prompts with policybench.prompts.describe_household (rv_prompts.txt). (6) Upstream, read-only: git diff 06665727d8..79be99f671. charitable_deduction.py, misc_deduction.py, total_misc_deductions.py, itemized_deductions_less_salt.py, ca_itemized_deductions(_pre_limitation).py, ca_deductions.py, the charity floor and ceiling and misc applies and floor parameters, itemized_deductions.yaml, ald/deductions.yaml, traditional_ira_contributions.py and educator_expense.py are all identical. git grep finds no active_participant or 219(g) logic, and no California charity or misc code. I also read git show 316e7832a1 (#9122) and the CA AMT diffs from #8873 and #8925. (7) Primary sources fetched this session, in src/: leginfo SB 711 status and history (Chapter 231, Statutes of 2025, approved and chaptered 10/01/25, urgency) and chaptered text (SECTION 1 amends 17024.5 with (Q) January 1, 2025; SEC. 11 amends 17076 to read with (c) 'Section 67(g) ... shall not apply'); current R&TC 17076 (effective 2025-10-01) and 17024.5 (amended by Stats. 2026 ch. 236, SB 1435, effective 2026-09-14, (Q) unchanged; (h)(2)(A) makes AGI-based limits use federal AGI); FTB 2025 Schedule CA (540) instructions, rev. 04/2026 (OBBBA nonconformity; Part I line 1 refund subtracted, line 2 column C non-California bond interest, line 11 educator; Part II charity at 50% of federal AGI, lines 19-22 misc nonconformity, line 29 worksheet at $504,411/$252,203); the 2025 Form 540 booklet (AGI limitation worksheet, $6 per $2,500); FTB Tax News October 2025 (2025 exemption credits $153/$475); IRS Notice 2025-67 PDF (last-modified 2025-11-13; joint active-participant range $129,000-$149,000); the Cornell text of 26 USC 67 (current (h) suspension, P.L. 119-21 section 70110, July 4, 2025); and, from the salt_refund_gross_income_9122 folder's copy (not fetched by me), Rev. Proc. 2025-32 (2026 educator limit $350, so $337.50 is within the federal cap). Every source the calls rest on was published before 2026-07-03, except SB 1435, which does not change the result." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r30_snap_heat_and_eat_sua.md b/reference_audit/2026-09-28/verification/reviews/excl_r30_snap_heat_and_eat_sua.md new file mode 100644 index 00000000..07e1af73 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r30_snap_heat_and_eat_sua.md @@ -0,0 +1,22 @@ +{ + "agree": true, + "problems": [ + "Minor and stale: the settlement says upstream/main is at 79be99f671. It is now d81bbf1b2c (2026-09-29 03:32 UTC), two commits later: ae69e84376 (child-care court-supervision routes, #9555) and d81bbf1b2c (version bump). Neither touches SNAP, so the conclusion stands.", + "Omitted upstream context: policyengine-us issue #9374 (open) reports this exact defect (the P.L. 119-21 sec. 10103 elderly/disabled gate on the always_standard SUA) for California. Draft PR #9610 (opened 2026-09-25, still a draft) adds an is_snap_sua_hr1_in_effect gate for California only. #7745 (the OBBBA heat-and-eat tracker) and #7853 (always_standard concept) were closed NOT_PLANNED. PA's always_standard=true from 2015-10-01 came from #8132 (merged 2026-05-01, before 1.755.4). No merged or in-flight fix covers PA (or MT, scenario_100). The record's upstream field ('to be filed') should cite #9374 and #9610 and say PA is not covered.", + "Overstated: 'the prompt states no 7 U.S.C. 2012(j) route.' 2012(j)(2)(B) (OLRC text read this session) also counts receipt of 'disability-based State general assistance benefits' that meet SSI-stringent criteria. The prompt lists an unexplained 'financial assistance: $3,600' next to 'is disabled'. The engine defines financial_assistance as help from friends or relatives, but the prompt does not say that. This is one more path to the disabled-member reading (3,576), so it strengthens the indeterminacy. It does not change the action.", + "Omitted from the law section: the FNS memo of 2025-08-29 (earliest Wayback capture 2025-09-06, before the freeze) says sec. 10103 is effective on enactment for new applicants, but lets states apply it to ongoing households 'at a minimum ... at the household's next recertification.' The prompt gives no certification timing. So even under the record's reading, some 2026 months could still carry the LIHEAP-conferred HCSUA ($298 a month). 3,240 is the value for a household under the new rule all year, not an unconditional law-correct value. This supports keeping the exclusion.", + "The 3,240 alternative also rests on engine treatments that 1.755.4 and 2.15.17 share and that the settlement does not flag as conditional. (a) The $924 of long-term capital gains is excluded from SNAP income: parameters/gov/usda/snap/income/sources/unearned.yaml lists dividend and interest income but no capital gains, and gross is $301.33 = $3,616/12. Counting it would give 12 x (298 - ceil(0.3 x 169.33)) = 2,964. (b) The $2,000 subsidy is treated as federal energy assistance, which 7 U.S.C. 2014(d)(11)(A) excludes. The investigator did flag the 2014(k)(4)(A) state-law alternative. So 'the law-correct value is 3,240' should read as conditional. This does not change the action.", + "Provenance, if the record is regenerated: 3,576 is policyengine-us 2.15.17 plus latest_conventions (c_snap_hold_fy2026 holds FY2026 figures for Oct-Dec 2026). Raw 2.15.17 is 3,600. engine_version should say so, e.g. 'policyengine-us 2.15.17 + latest_c_snap_hold_fy2026', or the frozen_value reads as raw engine output.", + "Tooling nit: fixes/r30_snap_heat_and_eat_sua.py (written for 1.755.4) gates on has_usda_elderly_disabled, which is yearly and counts every SPM-unit member. In 2.15.17 the SNAP-canonical test is has_snap_elderly_disabled_member, which is monthly and excludes ineligible members (added by 0d3d6bee0d, 2026-07-14). This makes no difference for scenario_080 (one person) or for this probe, which moves only 080 and 100. Any regenerated alternative for mixed-eligibility households should use the 2.15.17 variable." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_080", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "The r30 defect is unfixed in 2.15.17. snap_state_using_standard_utility_allowance.py is byte-identical to 1.755.4 and returns p.always_standard[state]. always_standard.yaml's only change since 06665727d8 is metadata (d7fc17abfe), and PA stays true from 2015-10-01. snap_utility_allowance_type still grants the SUA on has_heating_cooling | always_sua. The only upstream fix in progress (draft PR #9610) covers California only. My trace on 2.15.17 + latest_conventions shows always_sua=True, has_heating_cooling_expense=False, is_usda_disabled=False and has_snap_elderly_disabled_member=False. The $857 SUA is capped at $744, net income is 0, and the benefit is $298 x 12 = 3,576. With r30 added it is $270 x 12 = 3,240 (net $92, contribution $28). Under 7 U.S.C. 2014(e)(6)(C)(iv)(I) as amended by P.L. 119-21 sec. 10103(a) (approved 2025-07-04, before the freeze), the engine's own reading (no elderly or disabled member) should give 3,240, so 3,576 is defect-driven under that reading. With is_usda_disabled forced True, 2.15.17 gives 3,576 with or without r30, and the household stays eligible. The prompt supports both readings. It says 'is disabled' but gives no 2012(j) receipt route. Its $3,600 'financial assistance' is unexplained and could be disability-based general assistance. Its $2,000 'spm unit energy subsidy' could be LIHEAP or a state-law payment. It states no certification timing, which matters under the FNS recertification transition. Model answers split the same way: 15 near the SUA-conferred value (3,492-3,576) against 3,240, 3,228 and 2,964-2,988 on other readings. The output is therefore both defect-affected and indeterminate on the stated facts, and stays excluded. The -$20.04 move against the board (3,596.04 to 3,576) comes from the FY2026 SNAP hold convention: the FY2027 COLA memo is dated 2026-08-21, after the freeze. It does not come from r30. Scoring is unaffected." + } + ], + "what_i_checked": "HARNESS (policyengine-us 2.15.17, the brief's interpreter, run myself): full sweeps with fixes/latest_conventions.py (rv2_full_conv.csv) and with my own probe of latest_conventions + r30_snap_heat_and_eat_sua (rv2_conv_plus_r30.py, output rv2_full_conv_r30.csv). My conventions sweep matches out/latest_conventions.csv on all 1,984 outputs (29 moved vs the board). Adding r30 changes exactly two outputs: scenario_080 snap 3,576 to 3,240 and scenario_100 snap 8,556 to 6,924, both already excluded. Raw 2.15.17 on 080 and 100 (rv2_raw.csv): 3,600 and 8,637. My monthly trace (rv2_trace.py, rv2_trace.log): raw is 298 for Jan-Sep and 306 for Oct-Dec (standard deduction 209 to 217, shelter cap 744 to 769), total 3,600. With conventions, 298 every month, 3,576. With conventions + r30, 270 every month (SUA NONE, net 92, contribution 28), 3,240. With is_usda_disabled forced True, 3,576 with and without r30 (uncapped shelter 810.83, eligible). ENGINE CODE: diffed 1.755.4 and 2.15.17 site-packages. snap_state_using_standard_utility_allowance.py is identical, and always_standard.yaml differs only by 'propagate_metadata_to_children: true'. Read snap_utility_allowance_type.py, snap_excess_shelter_expense_deduction.py, has_heating_cooling_expense.py, is_usda_disabled.py, disabled_programs.yaml (1.755.4 used is_ssi_disabled, 2.15.17 uses receives_ssi/ssi/SSDI/veteran flags), has_snap_elderly_disabled_member.py, SNAP unearned.yaml (financial_assistance counted, no capital gains, no energy subsidy) and financial_assistance.py. UPSTREAM (read-only git): git log 06665727d8..upstream/main on the SUA variable and parameter shows only d7fc17abfe (metadata; git show read). Checked the dates and subjects of all 14 commits the settlement cites. d27d6c3cc3's message cites the USDA memo dated August 21, 2026. upstream/main is now d81bbf1b2c, and the two commits after 79be99f671 are not SNAP. Read-only gh: issues #7745 and #7853 (closed NOT_PLANNED) and #9374 (open, CA), PR #8132 (merged 2026-05-01, added PA to always_standard) and draft PR #9610 (CA-only gate). LAW, from primary sources fetched this session: P.L. 119-21 from govinfo (approved July 4, 2025; sec. 10103(a) and (b) text verified; on page 139 Stat. 83). 7 U.S.C. 2014 from the OLRC prelim: the amended (e)(6)(C)(iv)(I) text, the (k)(4)(A)-(B) text, (d)(11)(A), and the amendment notes naming P.L. 119-21 sec. 10103. 7 U.S.C. 2012(j) from the OLRC prelim, including the (2)(B) disability-based State general assistance route. FNS memo 'SNAP Implementation of the One Big Beautiful Bill Act of 2025 - Treatment of Energy Assistance Payments' dated August 29, 2025 (Wayback capture 2025-09-06): effective on enactment, no automatic HCSUA for households without an elderly or disabled member, ongoing households at the latest at next recertification. FNS FY2026 COLA memo (Wayback capture 2025-08-17 of fns.usda.gov shows 'DATE: August 13, 2025'; PDF read): 1-person maximum $298, standard deduction $209 for sizes 1-3, shelter cap $744. FNS FY2027 COLA memo PDF (usda.gov, via Wayback 2026-09-08): dated August 21, 2026, with $306, $217 and $769, after the 2026-07-03 freeze. PROMPT: regenerated the scenario_080 SNAP prompt with policybench.prompts.make_no_tools_prompt from the adds0928-stage2 worktree (rv2_prompt_080.txt): 'is disabled', 'financial assistance: $3,600', 'spm unit energy subsidy: $2,000', no heating or cooling costs, no SSI or SSDI, and a preface saying not to infer unlisted benefit receipt or expenses. The June 12 run's stored artifacts do not keep the prompt text, but the model explanations quote the same facts. Tabulated all 42 model answers for scenario_080 snap from predictions.csv.gz (rv2_model_preds_080.txt). INDEPENDENT ARITHMETIC: gross (3,600 + 16)/12 = 301.33; minus 209 = 92.33; ceil(0.3 x 92.33) = 28; 298 - 28 = 270; x 12 = 3,240. The SUA route gives 857 - 46.17 = 810.83 (capped at 744 for a non-E/D household), net 0, 298 x 12 = 3,576. Any SUA of at least $139 gives net 0, so 3,576 does not depend on PA's exact SUA amount. The 1.755.4 counterparts in out/c13v3_plus_upstream_snap.csv and c13v3_upstream_plus_r30.csv are 3,576 and 3,240. I did not verify PA's own sec. 10103 implementation date or the SNAP treatment of individual capital gains under 7 CFR 273.9." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_r32_wi_capital_gain_distributions.md b/reference_audit/2026-09-28/verification/reviews/excl_r32_wi_capital_gain_distributions.md new file mode 100644 index 00000000..53d32dbe --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_r32_wi_capital_gain_distributions.md @@ -0,0 +1,25 @@ +{ + "agree": true, + "problems": [ + "scenario_091: the claim that 'the law-correct value is unchanged at 878.515747' is overstated, and so is confidence 'high' for it. Wis. Stat. 71.05(6)(b)42, which covers tax years after 2012 and so is pre-freeze, lets an employee whose employer pays part of the cost subtract 100% of the medical care insurance premiums the employee paid, less any amount already deducted in federal AGI. The 2025 Schedule SB instructions (I-0104, PDF created 2025-11-07), line 6, Worksheet 2, implement this for employees and exclude only premiums paid pre-tax or through salary reduction. The prompt lists 'other health insurance premiums: $1,800' and, as a second line, 'health insurance premiums excluding Medicare Part B: $1,800'. Whether the premiums were paid pre-tax is not stated. The prompt's rule to treat any unlisted fact or boolean as false points toward after-tax. The engine's own AGI (36,182.56) does not reduce wages for any premium, so it too treats them as after-tax. policyengine-us 2.15.17 does not model this subtraction at all: the WI subtractions sources.yaml has no medical insurance entry, and no WI variable or parameter mentions medical care insurance. By engine-free recompute on the published 2026 parameters, r32 plus a 1,800 subtraction gives 789.81 (my_recompute.out), 88.70 below 878.52, and summing both premium lines (3,600) gives 701.11. So 878.515747 is correct only if the premiums were pre-tax, which is a fact no engine version can supply. The exclusion should stay, but the record should add this unlisted-input ambiguity (and the unmodeled provision) to the root cause, and it should not present 878.515747 as the unique law-correct value. The investigator raised this only as a caveat 'on the alternative only' and misstated the condition as depending only on after-tax payment, without citing 71.05(6)(b)42.", + "Minor, no outcome effect: in upstream_changes, 94e40f1471 is described as projecting the 2026 WI standard deduction (13,870). In fact the commit only restructures the uprating-rounding metadata on the WI standard deduction phase-out thresholds and the rate bracket thresholds. deductions/standard/max.yaml is byte-identical between 1.755.4 and 2.15.17 and no commit in 06665727d8..79be99f671 touches it. The raw-engine numbers the investigator quotes (468.83 / 900.46; std ded 12,220.63 = 13,870 - 12% x (33,734.71 - 19,990)) are right. c_wi_published_2026 overrides these values anyway.", + "Minor, no outcome effect: the current text of 71.05(6)(b)54m.d (database published 9-4-26) forfeits 'any credit, including any eligible carryover of such credit, listed under s. 71.07'. The text created by 2025 Act 15 section 220 (enacted and published 2025-07-03) forfeited 'any credit listed under s. 71.10 (4)'. The provision was amended after Act 15, and I did not pin down which act did it. For scenario_042 the elected path gives tax 0 before any credit, and standard-path refundable credits are 0, so the choice of forfeited credits does not change the 0.0 alternative." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "Both defects are still present in policyengine-us 2.15.17, and I confirmed each directly.\n- diff -rq shows wi_capital_gain_loss_subtraction.py and wi_income_tax_before_refundable_credits.py byte-identical to 1.755.4, and git log 06665727d8..79be99f671 shows no commit touching either file.\n- My probe of 2.15.17 plus latest_conventions shows non_sch_d_capital_gains 3,753 in federal AGI (50,580.71; net_capital_gain 4,681) but wi_capital_gain_loss_subtraction 0.\n- The probe also shows wi_income_tax 0 (the min with the line-16 exclusion path, since wi_retirement_income_exclusion_amount is 24,000 and wi_retirement_income_exclusion_tax is 0) while wi_income_tax_before_refundable_credits is 464.1828.\nHarness runs on 042/091:\n- latest_conventions: 464.1828.\n- Plus r32: 408.698425.\n- Plus r32 and the r06 elected-path port: 0.0.\nAn engine-free recompute on the 2026 1-ES parameters reproduces 464.1827 (standard path, no r32), 408.6984 (with r32) and 0.0 (elected line 16). 71.05(6)(b)54m lets a filer aged 67 or older subtract up to $24,000 of qualified-plan and IRA distributions, and 54m.d forfeits the credits. On the elected path, WI income 8,608.81 is below the 13,960 standard deduction plus 950 exemptions, so tax is 0, and 0 is below the standard path's net 408.70, so the filer elects it. The 0.0 holds under every alternative reading I tried:\n- only the 19,200 IRA counts as line 16 (WI income 14,534.71 or 13,408.81, both below 14,910);\n- the farm loss is double-counted;\n- qualified_bdc_income is added.\nr06 alone suffices; r32 does not change the alternative. #9122 does not apply (salt_refund_income is 0). If the record is refreshed, engine value 464.1828 and alternative 0.0." + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "action": "keep_excluded", + "proposed_reference": null, + "reason": "The r32 defect is still present in 2.15.17. The subtraction file is byte-identical to 1.755.4 and no commit in the range touches it. My probe on 2.15.17 plus conventions shows non_sch_d_capital_gains 1,170 in AGI 36,182.56 but wi_capital_gain_loss_subtraction 0.\nThe harness gives 895.81311 with latest_conventions and 878.515747 with r32 added; the engine-free recompute gives 895.8131 and 878.5158. 71.05(6)(b)9 subtracts 30% of long-term gain, IRC 852(b)(3)(B) treats capital gain dividends as long-term, and 2025 SB line 5 applies the 30% exclusion to distributions reported without Schedule WD.\nThere is a second, independent reason to keep this excluded. 2.15.17 does not model the WI medical care insurance subtraction: 71.05(6)(b)42, which covers an employee whose employer pays part of the cost, and 2025 SB line 6, Worksheet 2. The prompt lists $1,800 of other health insurance premiums (repeated as 'health insurance premiums excluding Medicare Part B') without saying whether they were paid pre-tax. The law-correct value is:\n- 878.52 if the premiums were pre-tax;\n- 789.81 if the $1,800 was paid after tax, the reading the prompt's unlisted-boolean-false rule and the engine's own AGI point to;\n- 701.11 if a reader sums both lines.\nThe exclusion record should list root causes r32_wi_capital_gain_distributions plus this unlisted pre-tax-premium ambiguity, and it should label 878.515747 as the r32-only correction, not the unique law-correct value." + } + ], + "what_i_checked": "Sandbox and evidence standard from LATEST_BRIEF.md; all scratch is in this workspace.\n\n(1) Scenario JSON from the bundle, and the exact no-tools prompts rendered with policybench.prompts.make_no_tools_prompt (prompts_rendered.txt): input listing, labels, the unlisted-input rules, and the output definition.\n\n(2) Harness, sweep_latest.py on scenario_042 and scenario_091 with the 2.15.17 venv:\n- no fix: 468.829163 / 900.459473 (rv_raw.csv);\n- fixes/latest_conventions.py: 464.1828 / 895.81311 (rv_conv.csv);\n- the investigator's check_r32_only_on_21517.py: 408.698425 / 878.515747;\n- check_r32_r06_on_21517.py: 0.0 / 878.515747 (rv_conv_r32*.csv).\nOnly these 2 of the 32 outputs for these scenarios move. I also diffed the investigator's full-bundle sweep_check_conv_r32_r06.csv against sweep/out/latest_conventions.csv: exactly these 2 of 1,984 outputs differ.\n\n(3) A probe of engine intermediates on 2.15.17, raw and with conventions (my_probe.py, my_probe_*_21517.json): AGI, taxable SS, non_sch_d_capital_gains, net_capital_gain, salt_refund_income (0 in both), wi_capital_gain_loss_subtraction (0 in both), wi_agi, standard deduction, taxable income, credits, wi_income_tax (0 for 042), and the line-16 amount (24,000) and exclusion-path tax (0).\n\n(4) Engine source:\n- diff -rq of variables/gov/states/wi and parameters/gov/states/wi between the 1.755.4 and 2.15.17 venvs; read 2.15.17's wi_capital_gain_loss_subtraction, wi_income_tax_before_refundable_credits, wi_income_tax, wi_retirement_income_exclusion_tax, wi_standard_deduction, wi_retirement_income_exclusion_amount, the subtractions sources.yaml and state_income_tax_before_refundable_credits;\n- git log 06665727d8..79be99f671 on both defect files (no commits) and on WI tax code;\n- git show --name-only for 491642087f (#8839: gross income sources plus net_capital_gain), 35bbe1ba0c, 757437e4f2, 316e7832a1, 5b44285433, b45bd5d5d5, and 94e40f1471 (read its phase-out diff);\n- 491642087f is in 2.15.17 and not in 1.755.4; f3eec2d09e is not an ancestor of 79be99f671.\nThe existing 1.755.4 sweep CSVs (r04 469.69/901.32; r19 -5.51; cwi_plus_r04 464.18/895.81) support the decomposition. I did not rerun 1.755.4.\n\n(5) Primary sources fetched this session (src/):\n- Wis. Stat. 71.05(6)(b)9, 54m and 35 pages, and the chapter 71 PDF (database published 9-4-26), including (b)42;\n- 2025 Wis. Act 15 PDF: enacted and published July 3, 2025; SECTION 220 creates 54m;\n- 2025 Wis. Act 118 header: enacted 2026-03-27;\n- 2025 Schedule SB instructions I-0104 (PDF created 2025-11-07; Last-Modified 2025-11-07): line 5 exception, line 6, line 16;\n- 2026 Form 1-ES instructions D-101a (PDF created 2026-01-16): standard deduction table 13,960 less 12% over 20,120; exemptions 700 plus 250; brackets 15,110 / 51,950;\n- 26 U.S.C. 852(b)(3)(B) (Cornell LII).\nAll the operative law predates 2026-07-03.\n\n(6) Engine-free recompute (my_recompute.py/.out) of both outputs under the engine path, r32 and r06, plus alternative readings: IRA-only line 16, double-counted farm loss, qualified_bdc_income included, and the WI medical care insurance subtraction at 1,800 / 3,600 / with ESI." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_snap_mortgage_residence.md b/reference_audit/2026-09-28/verification/reviews/excl_snap_mortgage_residence.md new file mode 100644 index 00000000..bcd47258 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_snap_mortgage_residence.md @@ -0,0 +1,20 @@ +{ + "agree": false, + "problems": [ + "Scope of the disagreement: the action (keep_excluded) and both numbers reproduce. 2.15.17 + latest_conventions gives 2,868, the ported alternative gives 3,576, raw 2.15.17 gives 2,904, and the 1.755.4 board stack (c13v3_upstream_plus_r33) gives 2,868. What does not hold is the settlement's description of 2,868 as \"the stated-facts value\" and its classification unlisted_input_unchanged. Both omit a second unlisted input and an unfixed engine defect that each move this output. Neither changes scoring, because the output stays excluded.", + "Second unlisted input (county / SUA region), missed by the settlement and the exclusion record. The prompt states only \"state: NY\". county_fips is in EXCLUDED_INPUT_VARIABLES (policybench/scenarios.py), so 2.15.17 falls back to first_county_in_state (ALBANY_COUNTY_NY), which maps to SNAP utility region NY_ONY and the $877 HCSUA. FNS's FY2026 SUA table (2026-05-21-SUA-Table-FY26.xlsx, effective 2025-10-01 to 2026-09-30, pre-freeze) gives $1,062 for New York City and $988 for Nassau/Suffolk. On 2.15.17 + latest_conventions the output is 2,868 for Albany, 3,276 for Nassau County and 3,540 for New York County. I verified this in the engine (probe.txt) and by hand (hand_calc.txt). So 2,868 is the engine's value under a default county the prompt never states, not the stated-facts value. The alternative value, 3,576, is the same in all three regions.", + "Unfixed engine defect, left unresolved in the settlement: 2.15.17 omits New York's State Supplement Program (SSP). No NY SSP variable exists anywhere in the 79be99f671 tree. gov.usda.snap.income.sources.unearned lists SSI supplements for 25 other jurisdictions, and unearned_spm_unit lists California's, but New York appears in neither.\nThe law, all published before 2026-07-03:\n- OTDA's \"2026 SSI and SSP Maximum Monthly Benefit Levels Chart\" (revised 2025-10-27; Wayback capture 2026-01-02; statutory reference Chapter 56 of the Laws of 2025) gives an individual living alone $994 federal plus $87 state.\n- NY Social Services Law 209(2)(a) (Wayback capture 2025-07-17) sets the 2025 living-alone standard at $1,054, which is the $967 federal amount plus $87.\n- OTDA's SSP page (Wayback capture 2026-02-13) says SSP \"is part of the monthly benefit paid to most SSI recipients\" and that the SSI application \"serves as your application for SSP benefits\".\n- 7 CFR 273.9(b)(2)(i) counts SSI and other need-based assistance as SNAP unearned income.\nThe harness already sets takes_up_ssi_if_eligible true and the engine pays $9,368 of SSI, so the $87 a month follows from the same stated facts and take-up.\nWith $87 a month added to SNAP income, the engine gives 2,400 for the default county, 2,796 for Nassau and 3,072 for New York County (probe2.txt). A hand calculation matches.\nThe settlement says OTDA and SSA were unreachable. Both are reachable through Wayback. Under the brief, an unfixed 2.15.17 defect that moves an output belongs in the exclusion record as a root cause. It does not change the action or the alternative value, because under the alternative reading net income stays 0.", + "The engine's value comes from a hybrid reading, not from either reading of the fact the exclusion names. housing_cost adds real_estate_taxes, so the $1,633.50 of property taxes counts as a shelter cost for the occupied home (7 CFR 273.9(d)(6)(ii)(B)). It adds mortgage_payments, an input with no formula, and never reads home_mortgage_interest or first_home_mortgage_interest. The interest on the same home's first mortgage therefore does not count. If the mortgaged home is read as not the residence, and the property taxes are dropped for the same reason, the result is 2,376 (Rest of State, no SSP; hand_calc.txt). The exclusion record should describe 2,868 as \"2.15.17 + conventions under the engine's default county, without NY SSP\". It should not say this is the value under the non-residence reading.", + "Minor: the settlement paraphrases 7 CFR 273.9(d)(6)(ii)(A) as covering \"mortgage payments\". The eCFR text in force on 2026-07-01 reads \"Continuing charges for the shelter occupied by the household, including rent, mortgage, condo and association fees, or other continuing charges leading to the ownership of the shelter ..., including interest on such payments.\" The meaning is the same, but quote it verbatim in the record." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_118", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 2868, + "reason": "Keep excluded, now for three independent reasons.\n1. The mortgage-residence ambiguity (r15_snap_mortgage_interest), which no engine version resolves. 2.15.17 still counts only mortgage_payments, not mortgage interest, as a SNAP shelter cost.\n2. A second unlisted input, the county. The prompt gives only NY, and New York's FY2026 HCSUA is $877, $988 or $1,062 by region. That gives 2,868, 3,276 or 3,540 on 2.15.17 + conventions.\n3. An unfixed 2.15.17 defect: the NY State Supplement Program ($87 a month for an individual living alone in 2026, per the OTDA chart revised 2025-10-27; pre-freeze law) is missing from SNAP income. With it, the value would be 2,400, 2,796 or 3,072 by region.\nRecord 2,868 as the regenerated engine value, replacing the stale raw-1.755.4 2,903.94. Label it as 2.15.17 + latest_conventions under the engine's default county (Albany, Rest of State SUA) without NY SSP, not as the law-correct stated-facts value. Add the NY SSP omission as an unfixed-defect root cause and the county/SUA region as a second unlisted input. The alternative value stays 3,576 (298 x 12). It is the same across all three regions, with or without SSP, and under every 7 CFR 273.10(e) rounding option. Scoring is unaffected." + } + ], + "what_i_checked": "Everything below was run or read this session. Scratch files are in pb-refs-reviews-0929/excl_snap_mortgage_residence.\n\n(1) Harness (sweep_latest.py, 2.15.17) on scenario_118 and scenario_056:\n- no fix: 118 gives 2,904; 056 gives 95.\n- latest_conventions: 2,868 and 95.\n- latest_c_snap_hold_fy2026 alone: 2,868 and 95.\n- latest_alt_snap_mortgage_residence: 3,576 and 159.\nFiles: rv_*.csv.\n\n(2) 1.755.4 venv (sweep.py) on scenario_118: c13v3_upstream_plus_r33 gives 2,868, r13_hold_fy2026_v3 gives 2,877.60, and r15_snap_mortgage_interest_v2 alone gives 3,596.04 (rv1755_*.csv). The board and raw 1.755.4 are both 2,903.94. Every one of the 17 sidecar revisions has excluded_outputs_untouched true.\n\n(3) The investigator's full sweeps: latest_alt vs latest_conventions differ in exactly 2 of 1,984 outputs (056 and 118), and 29 outputs move against the board in both. The other NY snap outputs (004, 071, 082, 104) are 0 and not excluded.\n\n(4) Monthly engine trace (probe.txt).\n- Values: SS 233.33 and SSI 780.67, so gross income is 1,014. The standard deduction of 209 leaves 805. The SUA is 877 plus 136.125 of property tax. The excess shelter deduction is 610.625 (uncapped; has_snap_elderly_disabled_member true). Net income rounds to 194, the contribution is 59, and the benefit is 298 - 59 = 239 a month.\n- Raw 2.15.17 pays 251 a month in Oct-Dec (std 217, max 306).\n- The alternative gives an excess shelter deduction of 1,073.21 and net income 0, so 298 a month.\n- Variants: county, and mortgage_payments=5551 as an input, which equals the alt module at 3,576.\n- probe2.txt adds the $87 SSP.\n\n(5) Code:\n- housing_cost and mortgage_payments have no diff between 06665727d8 and 79be99f671.\n- snap_excess_shelter_expense_deduction changed only in expense_share and has_snap_elderly_disabled_member.\n- Diffs read for PR #9318 (5d88007d90: half-up net income, ceil of 30%), #9587 (fa27adbffc: round to cents first), #9162 (3c41c31457) and #9623 (d27d6c3cc3: FY2027 COLA, memo dated 2026-08-21).\n- Also read snap_utility_region (NY county mapping), county, first_county_in_state, always_standard (NY true), and the SNAP unearned-income source lists.\n\n(6) Primary sources:\n- eCFR 7 CFR 273.10 and 273.9 as of 2026-07-01 (versioner API; last amended 2025-01-17): (e)(1)(ii)(A) and (B), (e)(2)(ii)(A)(1) and (2), 273.9(d)(6)(ii)(A) and (B), 273.9(b)(2)(i).\n- FNS FY2026 COLA memo, dated August 13, 2025 (Wayback page 20250902215244 plus PDF): 1-person maximum $298, standard deduction $209 for sizes 1-3.\n- FNA FY2027 COLA memo, dated August 21, 2026 (PDF via Wayback; earliest page capture 2026-08-28): $306 and $217, post-freeze.\n- FNS FY2026 SUA xlsx (live): NY NYC 1,062, Nassau/Suffolk 988, Rest of State 877.\n- SSA OACT SSIamts (Wayback 2026-05-10): 2026 individual $994.\n- OTDA 2026 SSI/SSP chart (Wayback 2026-01-02; revised 2025-10-27): NY SSP living alone $87.\n- OTDA SSP page (Wayback 2026-02-13).\n- NY SSL 209 (Wayback 2025-07-17).\n\n(7) Engine-free recomputation (hand_calc.py): 3 readings x 3 regions x SSP on/off x 3 rounding procedures. Every engine value reproduces. The alternative is 3,576 in every case.\n\n(8) The prompt the models saw (make_no_tools_prompt) lists home mortgage interest, the first home mortgage balance and interest, and real estate taxes. It lists no county, rent or utilities, and it says to assume program take-up.\n\nNot checked: any New York-specific SNAP procedure for SSI recipients. I retrieved no OTDA source for one." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/excl_snap_ssi_disability.md b/reference_audit/2026-09-28/verification/reviews/excl_snap_ssi_disability.md new file mode 100644 index 00000000..1aedd193 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/excl_snap_ssi_disability.md @@ -0,0 +1,36 @@ +```json +{ + "agree": false, + "problems": [ + "scenario_100: the claim that the unlisted input 'no longer matters under either the engine or the law' is refuted for the law. It holds only in the engine. The PARENT category that the settlement relies on is 2.15.17's own category hierarchy, and 2.15.17 does not model Montana's buy-in: is_working_disabled_buy_in_for_medicaid covers only CA WDP, IL HBWD and MS WD. Montana runs Medicaid for Workers with Disabilities (MWD), a title XIX group under 42 USC 1396a(a)(10)(A)(ii)(XIII). Per DPHHS manual ABD 201-6 (effective 2025-05-01, pre-freeze), it needs SSA disability criteria, age 16+ and paid employment, with countable income at or below 250% FPL. Per CMA 001 (effective 2026-01-01), the MWD resource limit is $15,000, not the $2,000 ABD limit. Under reading B the 100 head qualifies: she is 46, has wages and holds $2,800. 42 CFR 435.404 lets an applicant pick among the categories she qualifies for. MWD receipt makes her an elderly-or-disabled member under 7 CFR 271.2(11) and 7 U.S.C. 2012(j)(2)(B). With that status forced on 2.15.17 plus the conventions (rv_probe2), SNAP is 8,844 with the engine's MT heat-and-eat SUA and 7,116 without it. 8,844 is exactly the existing record's alternative value. Free ACA parent/caretaker coverage against a cost share of at least $35/mo makes MWD enrollment the less natural reading, but category choice is another unlisted fact. The dependence is therefore not settled away.", + "scenario_100: the proposed record change goes too far. It re-roots the output to r30 alone, changes the alternative value to 6,924, and describes 8,844 as 'an artifact of the criteria-based definition that #9345 removed'. r30 is real, still unfixed, and confirmed: always_standard.yaml has MT true with no elderly-or-disabled gate, and no change to it or its consumers since 06665727d8 addresses P.L. 119-21 sec. 10103. My own r30 port moves only scenario_080 (3,576 to 3,240) and scenario_100 (8,556 to 6,924) across all 1,984 outputs. But it should be recorded alongside the unlisted-input basis, not in place of it.", + "scenario_023: the reason says '3,576 needs a SUA of at least $384'. That is wrong on the engine it cites. With elderly-or-disabled status under reading B, 2.15.17 plus the conventions gives an excess medical deduction of $150.00/mo and a pre-shelter net of $1,470.51. So net = 205.76 - SUA, and a SUA of about $206 suffices. The $384 figure implies a medical deduction of about $31.67.", + "scenario_023: the settlement does not disclose what the 3,576 SUA rests on. It comes from CA's always_standard heat-and-eat flag: has_heating_cooling_expense = 0, and the prompt states no utility cost and no LIHEAP payment. For an elderly-or-disabled household, 7 U.S.C. 2014(e)(6)(C)(iv)(I) confers the SUA only on receipt of a LIHEAP payment over $20. The prompt says 'Do not infer unlisted ... benefit receipt'. With the heat-and-eat SUA switched off, the reading-B value is 2,832, not 3,576 (rv_probe2 conv_B_ED_noHE). The record should give the alternative as 3,576 with that caveat, or give both values.", + "scenario_023 (supports exclusion; for the lead): under the stated facts, 2.15.17 plus the conventions puts the head in medicaid_category WORKING_DISABLED_BUY_IN, with medicaid of $9,236.48. The engine's own Medicaid module therefore already treats her as receiving disability-related Medi-Cal. Adding the 271.2(11) title XIX route to SNAP moves stated-facts SNAP to 3,576 (rv_probe conv_t19_A, and the investigator's out_title19_full.csv). So 408 is the reading-A law value only if 'is disabled' confers no disability Medi-Cal, which contradicts the scored head_medicaid_eligible = 1. This strengthens the investigator's out-of-cluster flag. It does not change the SNAP action." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_023", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 408, + "reason": "Reproduced on the harness. latest_conventions gives 408 ($34/mo); raw 2.15.17 gives 462. Hand check: gross $2,120.22 (wages $1,453.55 plus 403(b) $666.67); minus 20% earned ($290.71) and the $209 standard deduction gives $1,620.51; minus the $744 cap (binds with or without the SUA) gives $876.51, rounded to $877; 30% is $263.10, rounded up to $264; $298 - $264 = $34. The unlisted input still decides the output under law. Under reading B, federal SSI is $0 (countable $1,340.94 > $994) and CA SSP is $0 (countable income over the $1,335.81 standard). She has no MAGI route (141.19% FPL > 138%). Her Medi-Cal is a disability pathway (engine: SENIOR_OR_DISABLED), and receiving it makes her an elderly-or-disabled member under 7 U.S.C. 2012(j)(2)(B) and 7 CFR 271.2(11), both text-verified. 2.15.17 does not model that route, so flipping meets_ssi_disability_criteria alone gives 408 (rv_probe conv_B). Record the alternative as 3,576 with the engine's CA heat-and-eat SUA, and 2,832 if no LIHEAP payment is inferred (a SUA of about $206 suffices, not $384). Reword the record: in 2.15.17 SSI criteria no longer confer SNAP status (#9345); the route is Medi-Cal receipt." + }, + { + "scenario_id": "scenario_057", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 2628, + "reason": "Reproduced. latest_conventions gives 2,628 ($219/mo); raw gives 2,682. Hand and engine agree. Head SSI is $994 - $391.42 = $602.58/mo; gross is $1,472.42; minus $173.33 (20% of earned) and $209 gives $1,090.08, which rounds to $1,090; 30% is $327; $546 - $327 = $219. Under reading B (rv_probe conv_B, and with the r30 port and the title XIX route), both spouses get SSI at the couple rate: $1,491 - $392.42 = $1,098.58/mo in total ($6,591.50 per person per year). Net is $1,586, 30% is $476, $546 - $476 = $70, and 840 per year, so the recorded alternative of 840 stands on 2.15.17 plus the hold. LA always_standard is false, so there is no SUA and the r30 port cannot move it. The dependence runs through actual SSI receipt, an unlisted-input ambiguity that no engine version resolves." + }, + { + "scenario_id": "scenario_100", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 8556, + "reason": "Reproduced. latest_conventions gives 8,556 ($713/mo); raw gives 8,637. Pre-shelter net is $690.73 (earned $492.89 plus the engine's MT TANF of $505.41/mo, minus $98.58 and $209). With the $799 SUA (FNS FY2026 SUA table: MT HCSUA $799), net is $237, 30% rounds up to $72, and $785 - $72 = $713. r30 is confirmed still present in 2.15.17. The engine grants the always_standard SUA with has_heating_cooling_expense = 0 and no elderly-or-disabled member. My r30 port (rv_fix_r30.py, full sweep rv_r30_full.csv) moves only 080 and 100, giving 100 = 6,924: net $691, $208, $785 - $208 = $577. Keep the output excluded under both bases. Add r30_snap_heat_and_eat_sua with the stated-facts corrected value 6,924. Do not drop the unlisted-input basis. Under reading B, the working head qualifies for Montana MWD (ABD 201-6; CMA 001 $15,000 resource limit), and 42 CFR 435.404 lets her select it. Receipt makes her elderly or disabled under 7 CFR 271.2(11), giving 8,844 with the engine's SUA (the existing alternative) or 7,116 without. The mechanism text should change from 'SSI criteria' to 'MWD (title XIX) receipt', which 2.15.17 does not model." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv), outputs in my workspace. Raw run (rv_raw.csv): 462, 2,682, 8,637. latest_conventions (rv_conv.csv): 408, 2,628, 8,556, identical to out/latest_conventions.csv. The probe driver (rv_probe.py, my own modules, not the investigator's) ran reading B, the r30 port, the title XIX route under readings A and B, and their combinations. Reading B gives 408, 840, 8,556; r30 gives 080 = 3,240 and 100 = 6,924; the title XIX route under reading A gives 023 = 3,576. rv_probe2.py forces elderly-or-disabled status with and without the heat-and-eat SUA: 023 gives 3,576 or 2,832, 100 gives 8,844 or 7,116. A full 1,984-output sweep with my r30 port moved only 080 and 100 against latest_conventions. The investigator's out_r30_full and out_title19_full CSVs diff the same way. Engine code read at 79be99f671 and in the venv: disabled_programs.yaml and is_usda_disabled.py (#9345: receives_ssi and ssi receipt), has_snap_elderly_disabled_member, meets_snap_non_parent_student_exception, snap_net_income (half-up), snap_expected_contribution (ceil), snap_utility_allowance_type, snap_state_using_standard_utility_allowance, always_standard.yaml (MT, CA, PA true; LA false; no elderly-or-disabled gate), medicaid_category hierarchy, is_working_disabled_buy_in_for_medicaid (CA, IL and MS only), and SUA values (MT $799, CA $663). Commit dates and ancestry verified with git log and merge-base: #9318, #9162, #9587, #9623, #9090 and #9345 are all in 2.15.17. No commit in 06665727d8..79be99f671 gates the heat-and-eat SUA. Primary law read this session: eCFR 7 CFR 271.2 (2026-07-01; paragraph (11) title XIX route); 7 CFR 273.10(e) rounding options; 42 CFR 435.540(a); 42 CFR 435.404 (applicant's choice of category); uscode.house.gov 7 U.S.C. 2012(j)(2)(B) and 2014(e)(6)(C)(iv)(I) and (k)(4) as amended by P.L. 119-21 secs. 10103 and 10104 (July 4, 2025); FR 2025-19763 (published 2025-11-03: $994 and $1,491); the FNS FY2026 maximum allotments and deductions PDF; the FY26 COLA memo dated 2025-08-13 (Wayback 2025-09-02); the FY27 page updated 2026-08-28 (post-freeze, so held); the FNS FY2026 SUA table (MT 799, CA 663, LA 465, PA 857); and Montana DPHHS ABD 201-6 (effective 2025-05-01), ABD 010 (2024) and CMA 001 (effective 2026-01-01). I also rendered the exact prompts (prompts_dump.txt) to check what readers saw: the prompts say 'is disabled' with no SSI or SSDI receipt, no utility costs and no LIHEAP payment, and tell readers not to infer benefit receipt. Not independently checked: the MT TANF amount the engine feeds into 100's SNAP, and CA's Medi-Cal program income limits beyond the engine's output." +} +``` \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/md_county_tax_8888.md b/reference_audit/2026-09-28/verification/reviews/md_county_tax_8888.md new file mode 100644 index 00000000..ff24652d --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/md_county_tax_8888.md @@ -0,0 +1,35 @@ +{ + "agree": true, + "problems": [ + "hold_module is empty, but both state holds depend on latest_md_local_output_scope.py (sha256 53a6de3cd9b1...), and the composed latest_conventions.py deliberately leaves that module out. If the fold uses latest_conventions alone, it publishes 2,089.608887 (068 state) and 11,295.388672 (078 state). Both include Maryland county tax, which the output definition excludes ('excluding local income and payroll taxes', benchmark_specs.json). The settlement should name the adapter as the module, and the lead must add it to the composed module. My harness runs confirm it changes exactly 2 of 1,984 outputs (moved 29 -> 27).", + "The settlement's county-tax figures describe an engine artifact, not the law. 2.15.17 resolves the unlisted county to the alphabetically first MD county (first_county_in_state.py -> ALLEGANY_COUNTY_MD). Its gov.local.md.flat_rate still carries Allegany at 3.03% (set 2023) and Kent at 3.20% for 2026. The Comptroller's Central Payroll Bureau memo '2026 Maryland State and Local Income Tax Withholding Information' (Feb 4, 2026, pre-freeze), Attachment 1, lists Allegany at 3.20% and Kent at 3.30%, and gives 3.30% as the default for an unknown county. So 834.27 and 4,358.51 are county tax at a stale rate for an arbitrarily chosen county. This is an unfixed 2.15.17 parameter defect. It changes no proposed reference, because the adapter removes county tax from the state output and SALT caps at every 2026 rate. It would matter if the adapter were ever dropped.", + "The claim that 078 federal 'does not depend on the unlisted county' holds only under the board's withholding-proxy convention. That convention is c_md_2026: 'federal tax through the Maryland withholding proxy'. Under a liability reading, where a solver deducts 2026 MD plus county liability, SALT stays below the $40,400 cap and the answer depends on the county. Using the Comptroller's 2026 rates I get 24,705.95 to 25,068.44 with #9122, and 24,314.51 to 24,676.97 without. The proposed 24,164.457031 is 2.2% to 3.7% below that range, while the old board value (24,780.61) fell inside it. I still accept adoption. IRC 164 deducts taxes paid (withheld) in the year, the Comptroller says MD withholding 'is a combination of the State income tax ... and local taxes', and the board already scores this output through the proxy. The settlement should still state the dependence on this convention explicitly.", + "Under the Comptroller's literal formula, the federal value depends slightly on the county. The formula applies the rate to wages less the $3,400 allowance and one $3,200 exemption. At Worcester's 2.25% (verified as the 2026 rate), SALT is 40,342.64, which is $57.36 under the cap, so federal is 24,178.23 (+13.77). That misses the $1 exact band but is within 1%. The settlement discloses this (as 'about $14'). It does not affect the proxy-based reference.", + "Adopting #8888's SALT piece creates a new interpretive sensitivity in 078 state. Once SALT is capped, the Md. Tax-Gen 10-218(b)(3) addback ('the amount claimed as taxes on income paid to a state or political subdivision') has several readings, and the Comptroller 'will accept any reasonable interpretation'. Property-first gives 6,936.88; that is the engine's result and the Comptroller's own example. Pro-rata gives 7,008.19 (+71.31, 1.03%), and income-first gives 7,051.65 (+114.77). The proposed 6,936.876465 is the right reference: it is the Comptroller's example and also the value under every reading where the cap does not bind. But the example rests only on a secondary source (Thomson Reuters, 2024-02-20). The Comptroller's KB page needs authentication, so neither the investigator nor I could read the primary text. The primary 2025 resident booklet says only that line 17b 'is capped at $40,000'.", + "Minor labelling point. For the two state outputs, 'hold_by_convention' is really an output-definition mapping: the 2.15.17 variable is correct Maryland law, just outside the output's scope. It is not a hold of pre-freeze law. The sidecar revision should record it as a permanent mapping grounded in benchmark_specs.json, not as a law convention." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_068", + "variable": "state_income_tax_before_refundable_credits", + "action": "hold_by_convention", + "proposed_reference": 1255.342773, + "reason": "This is an output-scope mapping (latest_md_local_output_scope.py, which must be added to the composed module), not a hold of pre-freeze law. 2.15.17 adds md_local_income_tax_before_refundable_credits to the state list (#8888, 6b0bca0b9f; that one line is the only change to the list between 06665727d8 and 79be99f671). The output is defined as excluding local income taxes, and Maryland county tax is a local income tax (Tax-Gen 10-103, 10-106). Harness (latest_conventions + adapter) gives 1,255.342773, equal to the board. Pinning Worcester gives the same value, so with the adapter the output does not depend on the unlisted county. Engine-free: MD taxable income 34,083.53 - 3,350 - 3,200 = 27,533.53; tax 90 + 4.75% x 24,533.53 = 1,255.34." + }, + { + "scenario_id": "scenario_078", + "variable": "state_income_tax_before_refundable_credits", + "action": "hold_by_convention", + "proposed_reference": 6936.876465, + "reason": "Same output-scope mapping as 068. Harness with the adapter gives 6,936.876465 (+0.539 against the board), and the value is the same at all 11 counties I pinned. The +0.539 comes from #9122: federal AGI 200,136.91 triggers Tax-Gen 10-218(c), 7.5% x 136.91 = 10.27 less MD itemized, x 5.25%. With salt_refund_income zeroed the value is 6,936.337402 (= board), so this output should follow the salt_refund_gross_income_9122 ruling; both values are within $1. Engine-free: MD itemized 28,210 + 26,460 - 10.27 = 54,659.73, exemption $0 (10-211(c)(1)(iii)), taxable 143,845.27, tax 6,936.88. This is the property-first addback, which matches the Comptroller's example; pro-rata or income-first readings would give +71 or +115." + }, + { + "scenario_id": "scenario_078", + "variable": "federal_income_tax_before_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 24164.457031, + "reason": "#8888's county withholding in md_withheld_income_tax fixes the SALT treatment of pre-freeze law: IRC 164(a)(3) allows local income taxes, Tax-Gen 10-103(a) and 10-106(a)(1) set a 2.25% to 3.30% county tax, and the Comptroller's 2026 memo (Feb 4, 2026) says withholding combines state and local tax. Under the board's c_md_2026 withholding proxy, the cap binds at any county rate of 2.089% or more (2.152% without #9122). The lowest 2026 rate is Worcester's 2.25%, so the value holds for every county. Harness: 24,164.457031 at all 11 counties pinned. Engine-free: AGI 200,136.91 - 28,210 - 40,400 = 131,526.91; 17,966 + 24% x 25,826.91 = 24,164.46 (Rev. Proc. 2025-32 Table 3; cap per 164(b)(7)(A)(ii)); no AMT (TMT 21,275). This value assumes #9122 is adopted. If salt_refund_gross_income_9122 holds #9122 or adopts the no-prior-benefit reading, the reference is 23,772.800781 (harness with the refund zeroed; engine-free 23,772.80). Caveat: the value depends on the proxy convention; under a liability reading it would be county-dependent, 24,706 to 25,068." + } + ], + "what_i_checked": "Harness (sweep_latest.py, 2.15.17 venv): (1) latest_conventions on 068/078 reproduces all three 'latest' values exactly, with no other output of the two scenarios moving (rv_conv_068_078.csv). (2) latest_conventions + latest_md_local_output_scope, both unmodified and loaded by rv_fix.py, gives 068 state 1,255.342773 (= board), 078 state 6,936.876465 and 078 federal 24,164.457031 (rv_conv_scope_068_078.csv). (3) The same pair over the full sweep changes exactly 2 of 1,984 outputs against out/latest_conventions.csv and lowers moved from 29 to 27. My output is identical to the investigator's out/latest_md_county_tax_8888_conv_scope.csv (rv_conv_scope_full.csv). These are the only two MD scenarios. One-process variants (rv_variants.py/.csv): 11 pinned counties, including Worcester 2.25%, Talbot 2.40%, Anne Arundel, Frederick and Dorchester, plus salt_refund_income zeroed. With the adapter, federal and state are county-invariant, federal is 23,772.800781 and state 6,936.337402 with the refund zeroed, and local_income_tax and state_refundable_credits stay 0. Code read with git show, read-only: 6b0bca0b9f (#8888) diff, md_withheld_income_tax.py at 06665727d8 and 79be99f671, state_and_local_sales_or_income_tax.py, local_income_tax.py, MD local tax variables, county.py and first_county_in_state.py, the gov.local.md parameters, and commits 97137280f7, b711aa56b7 and 316e7832a1 (#9122). The list parameter has a single 0000-01-01 value, so the adapter's 2015-2034 update is safe. Prompt: I rendered the exact batch prompt with prompts.make_no_tools_batch_prompt (rendered_prompts.txt). It lists no county and no withholding. The state output text says 'excluding local income and payroll taxes', and local_income_tax names only NYC, Philadelphia, Kansas City and St. Louis. Primary sources fetched this session: Md. Tax-Gen 10-103, 10-106, 10-211 and 10-218 (mgaleg.maryland.gov, current text); 26 USC 164(a)(3) and (b)(6)-(7) ($40,400 for 2026, $505,000 threshold); Rev. Proc. 2025-32 Table 3 and the AMT exemption (IRS, as of Oct 9, 2025); and the Comptroller CPB memo on 2026 MD state and local withholding (Feb 4, 2026), including its county rate table. Secondary: Thomson Reuters, 2024-02-20, quoting the Comptroller's addback example. The Comptroller's KB API returned 'User is not authenticated'. Board provenance: c_md_2026 in reference_outputs.csv.meta.json and root_causes.json ('federal tax through the Maryland withholding proxy'). Scoring (analysis.py): exact within $1, plus 1%, 5% and 10% bands. Independent engine-free recomputation (rv_recompute.py/.txt) reproduces 1,255.34, 6,936.88/6,936.34 and 24,164.46/23,772.80. It covers every 2026 county under the proxy, the literal Comptroller formula (Worcester +13.77) and the liability reading (24,705.95 to 25,068.44), plus the addback alternatives (+71.31 pro-rata, +114.77 income-first)." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/nj_ctc_fy2027_budget.json b/reference_audit/2026-09-28/verification/reviews/nj_ctc_fy2027_budget.json new file mode 100644 index 00000000..35d569f8 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/nj_ctc_fy2027_budget.json @@ -0,0 +1,18 @@ +{ + "agree": true, + "problems": [ + "Minor, not outcome-changing: the investigator says the bill text 'was public from introduction on 6/26'. The server does not show that. Fetched today, pub.njleg.state.nj.us/Bills/2026/S5000/4531_I1.HTM has Last-Modified 2026-07-01 14:27 GMT. The pre-freeze publication evidence is instead: the committee statement 4531_S1.PDF (dated June 28, 2026, Last-Modified 2026-06-29 15:17 GMT, same $1,250/$1,000/$750/$500/$250 schedule); the NJ Legislature bill history ('Approved P.L.2026, c.26.' 6/30/2026); and the Governor's release 'Posted on 06/30/2026'. All of these, and the 7/1 date too, fall before 2026-07-03.", + "Minor omission: the investigator gave the post-freeze server date only for PL26 (2026-09-17). The AL26/26_.PDF bill-format copy also carries a post-freeze Last-Modified (2026-08-13). Neither matters, because enactment on 6/30 and the 6/29 committee statement are pre-freeze.", + "Could not corroborate with the Wayback Machine. CDX and availability queries for the Governor's release, 4531_I1.HTM and 4531_S1.PDF returned no captures up to 2026-07-10, or timed out. Dating therefore rests on official records (bill history, 'Approved June 30, 2026' in the chaptered text, the Governor's release) and server Last-Modified headers, which I judge sufficient." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_008", + "variable": "state_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 5842.4, + "reason": "Confirmed. Re-running sweep_latest.py with fixes/latest_conventions.py on 2.15.17 gives 5,842.40 against a board value of 5,342.40, and it is the only moved output for scenario_008. Decomposition in both venvs shows nj_ctc going from 2,000 to 2,500 while nj_eitc (3,292.40), nj_property_tax_credit (50), nj_cdcc (0) and nj_taxable_income (18,915) are identical. nj_refundable_credits adds [property_tax_credit, eitc, cdcc, ctc], and nj_anchor (450) is not in it. The cause is commit 0319635b6e (2026-07-08, ctc/amount.yaml 2026-01-01: 1,250/1,000/750/500/250, reverting in 2029). It is an ancestor of 79be99f671 and not of 06665727d8, and was merged via b011a7ef48 (PR #8971, 2026-07-09); 115c5a8e36 changes only the CTC reference. My own attribution reform takes a different route: an nj_ctc override that multiplies the per-child amount by 0.8 for 2026-2028, which reproduces the pre-c.26 schedule exactly. On the full 1,984-output sweep it differs from latest_conventions in exactly this one output (back to 5,342.40), and 28 moved outputs remain. My sweep CSVs match the investigator's two CSVs row for row. Law: P.L.2026 c.26 (S-4531) sets $1,250 per child under 6 at NJ taxable income of $30,000 or under for TY2026-2028, effective for tax years beginning on or after 2026-01-01. It was approved 6/30/2026, per the bill history API, the chaptered text and the Governor's release posted 06/30/2026, and the committee statement dated 6/28 was served by 6/29. All of that is before the 2026-07-03 freeze, so this is adopt, not hold. Independent calculation: NJ gross income 26,800 + 4,000 + 115 = 30,915, less exemptions 12,000 (2x1,000 + 1,000 blind/disabled + 6x1,500) = 18,915. Two children under 6 (ages 5 and 1) x $1,250 = $2,500. NJ EITC is 0.40 x 8,231 = 3,292.40: Rev. Proc. 2025-32 (IRS, Last-Modified 2025-10-17) gives the 3+ child maximum of $8,231 and the MFJ threshold of $31,160, and AGI 30,632.41 and earned income 30,517.41 are both below the threshold. Adding the property tax credit of 50 gives 5,842.40." + } + ], + "what_i_checked": "I read LATEST_BRIEF.md and the investigator's saved sources and scripts. All scratch work is in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/nj_ctc_fy2027_budget/review/; I modified no existing file.\n\n(1) Harness. I ran sweep_latest.py with fixes/latest_conventions.py on 2.15.17 for scenario_008 (review/rerun_s008_conventions.csv). Only state_refundable_credits moves (5,342.40 to 5,842.40). I also ran the full 1,984-output sweep (review/review_latest_conventions_all.csv): 29 moved. latest_conventions has no NJ component.\n\n(2) Decomposition. review/decompose_review.py, run under .venv-pe1755 (1.755.4) and .venv-pepy612-us21517 (2.15.17), shows that only nj_ctc changes (2,000 to 2,500). The CTC brackets read 1,000/800/\u2026 in 1.755.4 and 1,250/1,000/750/500/250 in 2.15.17. nj_ctc.py, nj_ctc_eligible.py and nj_refundable_credits.py are identical between the venvs (diff -rq). The NJ files that do differ (EITC investment income test, 529 deduction, exemption metadata, Stay NJ, ANCHOR, CCAP) leave every intermediate for this scenario unchanged.\n\n(3) Independent attribution. review/review_attr_nj_ctc_x08.py overrides the nj_ctc variable with 0.8x the per-child amount for 2026-2028. Full sweep (review/review_attr_nj_ctc_x08_all.csv): 28 moved, and it differs from latest_conventions only on scenario_008 state_refundable_credits (back to 5,342.40). Both of my CSVs match the investigator's verify/attribution CSVs exactly.\n\n(4) Upstream. I ran git log 06665727d8..79be99f671 on the NJ credits paths and read 0319635b6e (the ctc/amount.yaml diff and its commit message citing S-4531) and 115c5a8e36 (reference-only CTC change). merge-base confirms 0319635b6e is in 2.15.17 and not in 1.755.4; b011a7ef48 is the 2026-07-09 merge of PR #8971.\n\n(5) Primary sources, fetched fresh:\n- NJ Legislature API billHistory/S4531/2026: introduced 6/26, reported 6/28, Senate 39-1, Assembly 79-0-0, 'Approved P.L.2026, c.26.' all dated as the investigator states.\n- 4531_I1.HTM and 4531_S1.PDF are SHA-256 identical to the investigator's copies. Last-Modified is 2026-07-01 and 2026-06-29. The committee statement is dated June 28, 2026, reports the bill favorably without amendments, and gives the same amounts.\n- The chaptered PL26 HTM/PDF read 'Approved June 30, 2026'; the AL26 PDF header reads 'P.L. 2026, CHAPTER 26, approved June 30, 2026'. Section 2 makes the act effective for tax years beginning on or after 1/1/2026.\n- The Governor's release nj.gov/governor/news/2026/20260630e.shtml is 'Posted on 06/30/2026' and lists S-4531/A-5329 among the bills 'signed \u2026 into law'.\n- Wayback returned no pre-freeze captures.\n\n(6) Independent calculation. From the scenario facts and Rev. Proc. 2025-32 (fetched from irs.gov; 2026 EITC table: 3+ child maximum $8,231, MFJ threshold $31,160, investment income limit $12,200) I get 5,842.40. scenario_008 state_refundable_credits is not in reference_exclusions.json; the only scenario_008 exclusion is payroll_tax (r05), and no board revision touches this output.\n\nNo hold module is needed. The un-exclusion checks do not apply." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/ny_ctc_phaseout_rounding_9425.json b/reference_audit/2026-09-28/verification/reviews/ny_ctc_phaseout_rounding_9425.json new file mode 100644 index 00000000..674b5bab --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/ny_ctc_phaseout_rounding_9425.json @@ -0,0 +1,17 @@ +{ + "agree": true, + "problems": [ + "Minor labeling only, and it does not change the verdict: 316e7832a1's title says '(#9122)', but #9122 is the issue. The change reached main as PR #9422 (merge e990b4a6f8, 2026-09-16), and I confirmed 316e7832a1 is an ancestor of that PR's branch head fc4a2d706c. Calling it '#9122' in the upstream_changes entry is ambiguous; 'issue #9122 / PR #9422' would be exact.", + "Also minor, not relied on: the investigator's scratch tax606.txt/tax606.html is a Cloudflare 'Just a moment...' challenge page from nysenate.gov, not statute text. The legal claim instead rests on the nyassembly.gov S.3009-C text, which I re-fetched independently and which does contain the quoted (1-a)(C) language. So this does not undermine the finding." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_082", + "variable": "state_refundable_credits", + "action": "adopt_latest", + "proposed_reference": 667, + "reason": "Confirmed independently. I re-fetched the S.3009-C bill text and actions from nyassembly.gov: passed both houses 05/08/2025 and signed as chap.59 on 05/09/2025. Part C section 2 adds Tax Law 606(c-1)(1-a). (B)(i) gives $1,000 per qualifying child aged three or younger for tax years 2026-2027. (C) reduces the credit by $16.50 'for each one thousand dollars by which' federal AGI exceeds $75,000 (head of household), with no 'or fraction thereof'. Part C section 3 says the act takes effect immediately. The NY Tax Department reads it the same way. The 2025 IT-213-I (PDF CreationDate and Last-Modified 2025-10-27; my download is sha1-identical to the investigator's) has line 6 'Round down the amount from Form IT-201, line 19 to the nearest $1,000'. The 2025 IT-201-I (2025-11-20) Note 4 confirms that line 19 is federal AGI. The 2025 IT-213 fill-in form (Last-Modified 2026-06-16, sha1-identical) has line 8 = line 7 x 1.65%. All of these predate 2026-07-03. The 2025 IT-201-I page 26 that 1.755.4 cited only points to IT-213 and contains no round-up. Engine side: I diffed the installed sources. The only relevant difference is ny_ctc_post_2024_phase_out, where (excess+increment-1)//increment became excess//increment (d8f06f394c, 2026-09-10, #9425; it comes after 06665727d8 and is an ancestor of 79be99f671). The only parameter change is SURVIVING_SPOUSE 110k->75k, which does not apply to this head-of-household filer. My decomposition in both venvs: the base is $1,000 in both, the phase-out is 709.50 in 1.755.4 and 693 in 2.15.17, and ny_cdcc is 360 in both. Federal AGI is 117,585.15 in 1.755.4 and 117,652.65 in 2.15.17. I recomputed everything without the engine: federal AGI is 117,585.15 to 117,661.13 under every SALT-refund and IRA (r02) reading, which is 42 whole thousands over the threshold, so ESCC = 1000 - 693 = 307. NY CDCC = 3,000 x 0.20 x 0.600 = 360; the 0.600 factor for NYAGI 65k-150k comes from the 2025 IT-216-I (2025-10-22). Total 667. Harness with latest_conventions: 667. My full conventions sweep vs the investigator's round-up counterfactual sweep: exactly 1 of 1,984 outputs differs (this one, 667 vs 650.5), and my counterfactual CSV matches theirs to 0.0. The output is not in reference_exclusions.json; only scenario_082 federal_income_tax_before_refundable_credits is excluded (r02). No hold module is needed." + } + ], + "what_i_checked": "What I read first: LATEST_BRIEF.md, all of the investigator's scratch files, and fixes/latest_conventions.py.\n\nHarness runs: sweep_latest.py with latest_conventions on scenario_082 moves 2 outputs, state_refundable_credits 650.5->667 and the separate-cluster federal tax +14.85. I then ran full 1,984-output sweeps for (a) latest_conventions and (b) the investigator's counterfactual_ceil_rounding.py. They differ only in scenario_082 state_refundable_credits (667 vs 650.5), and (b) reproduces cf_ceil_rounding.csv exactly.\n\nEngine code: I diffed the installed NY CTC variables and parameters between the 1.755.4 and 2.15.17 venvs. The only relevant change is the increment rounding in ny_ctc_post_2024_phase_out, plus the threshold for surviving spouses, which this filer is not. d7fc17abfe touches ny_ctc_pre_2024 only, which does not apply in 2026.\n\nGit (read-only):\n- d8f06f394c: 2026-09-10, #9425, lies inside 06665727d8..79be99f671, and its diff matches the claim.\n- 316e7832a1: 2026-09-08, reached main through PR #9422 (merge e990b4a6f8, 2026-09-16).\n\nDecomposition: I wrote review/rev_decompose.py and ran it in both venvs. It shows NY refundable credits = ny_ctc + ny_cdcc (360) with every other credit 0, federal AGI and NY AGI as claimed, and the base credit $1,000 in both.\n\nPrimary sources, re-fetched myself:\n- nyassembly.gov S03009 bill text and actions, signed chap.59 on 05/09/2025. I read the (1-a)(A)-(F) text and Part C section 3 (effective immediately) myself.\n- tax.ny.gov it213i.pdf (Last-Modified and CreationDate 2025-10-27) and it213_fill_in.pdf (Last-Modified 2026-06-16, barcode 213001250094), both sha1-identical to the investigator's copies.\n- it201i.pdf (2025-11-20): line 63 defers to IT-213, and Note 4 says line 19 is federal AGI.\n- it216i.pdf (2025-10-22): 0.600 NY factor for NYAGI 65k-150k and the .20 federal decimal.\n\nIndependent recompute (review/rev_recompute.py): all four AGI readings (SALT refund in or out, IRA deducted or not) fall in 42 whole thousands over $75,000. Rounding down gives ESCC 307 and a total of 667. The round-up gives 650.5, and a continuous 1.65% reading would give about 657, but the statute and IT-213 support neither.\n\nOther checks:\n- The rendered prompt for scenario_082 lists 'employer sponsored insurance premiums: $21,208'. The engine documents that input as employer-paid, and IRC 106 excludes such premiums from income, so there is no AGI effect. This is a scenario-wide reading question outside this cluster, and it would not flip the rounding verdict.\n- reference_exclusions.json has only scenario_082's federal tax output (r02), not this output.\n\nScratch outputs are in triage/latest/ny_ctc_phaseout_rounding_9425/review/: rev_conv_082.csv, rev_conv_full.csv, rev_cf_ceil_full.csv, the scripts, and the fetched PDFs and texts. I modified no existing file." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/pr182_review_023.md b/reference_audit/2026-09-28/verification/reviews/pr182_review_023.md new file mode 100644 index 00000000..38baf265 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/pr182_review_023.md @@ -0,0 +1,44 @@ +# Pre-merge review of release dashboard-data-20260929: scenario_023 head_medicaid_eligible + +Reviewed 2026-09-29 (PolicyBench PR #182). Decision recorded 2026-09-29 by the developer: exclude the output as `reference_depends_on_unlisted_input`, with `unlisted_input` = `meets_ssi_disability_criteria`, the input and reason that already exclude this household's SNAP. `clusters.json` (`reconciliations`) records the disposition, and `final_actions.json` lists the exclusion under `audit_exclusions`. + +## The flag + +- The investigation of cluster `excl_snap_ssi_disability` ended its summary with an out-of-cluster flag: the scored scenario_023 head_medicaid_eligible (1) follows reading B, because `ca_wdp_disability_eligible` reads the broad `is_disabled` flag, and under reading A the law gives 0 (42 CFR 435.540(a)), "so the lead should review it". +- The cluster's independent review agreed (`excl_snap_ssi_disability.md`, fifth problem). Under the stated facts, 2.15.17 plus the conventions puts the head in `WORKING_DISABLED_BUY_IN`, so 408 is the reading-A SNAP value only if "is disabled" confers no disability Medi-Cal, "which contradicts the scored head_medicaid_eligible = 1". +- Nothing in the wave's records disposed of the flag: `final_actions.json` had no entry for the output, `clusters.json` reconciled only scenario_078's federal income tax, and the README was silent. The output stayed scored at 1.0 with impact weight 6,612.94. + +## The review's reasoning + +- The prompt states only `is_disabled: true`, rendered as "is disabled". +- The head's MAGI is 141.2% of the federal poverty guideline, above the 138% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal. SSI is $0 under either reading. +- California's 250% Working Disabled Program requires the federal definition of disability. The review quoted the Los Angeles County DPSS 250% WDP policy, which policyengine-us cites, as requiring the person to "Meet the federal definition of disability for Social Security disability programs". 42 CFR 435.540(a) reads: "The agency must use the same definition of disability as used under SSI" (eCFR text read 2026-09-29). policyengine-us describes `meets_ssi_disability_criteria` as the SSI disability criteria before the substantial gainful activity screen. +- policyengine-us 2.15.17's `ca_wdp_disability_eligible` returns `is_disabled | is_blind | social_security_disability > 0` and never reads `meets_ssi_disability_criteria`, so flipping that input alone leaves the engine's value at 1 (the table below). The investigator reports the same code in 1.755.4, which is why the 2026-09-05 sweep over that input did not move the output. +- The published SSI case note for the same household says the head's unlisted SSI disability status is false under the prompt instructions, which contradicts a reference of 1 through a disability pathway. +- In the frozen predictions, 22 of 45 models answer 0, 21 answer 1 and 2 give no answer. The case note published the 22 zeros and the 2 missing answers as `llm_error`. +- Rule 4 of `../../README.md` excludes an output whose reference turns on a fact the prompt does not state. + +## Computed values + +`scripts/probe_023_medicaid.py` on policyengine-us 2.15.17 with `fixes/latest_final.py` (sha256 `dbbdd228…`), the household built by the references' builder (`scripts/sweep.py`). Output: `../probe_023_medicaid.json`. + +| System | Reading | head_medicaid_eligible | medicaid_category | +|---|---|---:|---| +| Reference system (`latest_final`) | Stated facts | 1 | WORKING_DISABLED_BUY_IN | +| Reference system | Reading A: does not meet the criteria | 1 | WORKING_DISABLED_BUY_IN | +| Reference system | Reading B: meets the criteria | 1 | SENIOR_OR_DISABLED | +| WDP disability test reading `meets_ssi_disability_criteria` | Stated facts | 0 | NONE | +| WDP disability test reading `meets_ssi_disability_criteria` | Reading A | 0 | NONE | +| WDP disability test reading `meets_ssi_disability_criteria` | Reading B | 1 | SENIOR_OR_DISABLED | + +- The reference system reproduces the committed reference, 1.0. +- `medicaid_income_level` is 1.4119 in every run, and SSI is 0. +- Reading A gives 0 and reading B gives 1. The exclusion records `frozen_value` 1.0, the engine's value, and `alternative_value` 0.0. +- Under reading B the engine's category hierarchy places the head in SENIOR_OR_DISABLED ahead of the buy-in; `ca_wdp_eligible` is true there too. + +## Effect + +- The output leaves scoring for every model. It stays in the payload, marked `scored=false`. +- The exclusion record has 56 entries: 28 engine-defect and 28 unlisted-input. 1,928 of the 1,984 outputs are scored. +- The reference value does not change, so the engine_upgrade revision's `changed` list keeps exactly the engine's changes. +- The staged adjudication record gains the matching entry (`prompt_ambiguity`, `reference_verdict` `unlisted_input`). Triage then rebuilds the case note as it does for every other excluded output. diff --git a/reference_audit/2026-09-28/verification/reviews/salt_refund_gross_income_9122.md b/reference_audit/2026-09-28/verification/reviews/salt_refund_gross_income_9122.md new file mode 100644 index 00000000..6ac74dc2 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/salt_refund_gross_income_9122.md @@ -0,0 +1,29 @@ +{ + "agree": false, + "problems": [ + "The per_output action is wrong: both outputs are coded adopt_latest, but the reason text says to exclude them. If the lead applies the actions as written, both outputs get scored at the 2.15.17 value. The review standard says an output that depends on an unlisted fact is excluded, not adopted. I agree with the substance, which is exclusion. Both outputs need an exclusion record with reason_code reference_depends_on_unlisted_input, the same class as the board's r16 and r24 records (e.g. scenario_002, 062, 107 and 121 in reference_exclusions.json). Fields: frozen_value = the 2.15.17 value; alternative_value = the board value; unlisted_input = whether the prior-year deduction of the refunded tax reduced federal tax. That turns on three facts: prior-year itemization, the income-versus-sales-tax election, and SALT-cap headroom.", + "The settlement leaves out evidence that settles whether the prompt states the engine's input. PolicyBench renders the input with a fixed label map. policybench/prompts.py:130 maps salt_refund_income to 'state and local tax refund income'. Inputs that are already the taxable amount get an explicit 'taxable' in the same map: 'taxable interest income', 'taxable IRA distributions', 'taxable private pension income' and 'taxable 401(k) distributions'. In 2.15.17 the engine label is still 'State and local tax refund income'. #9122 (316e7832a1) only added documentation calling it the 'Taxable ... Form 1040, Schedule 1, line 1' amount. So the prompt does not state the engine's input contract, and a careful reader could treat the $243 and the $2,589 as gross refunds whose taxability they must work out.", + "This overstates: 'South Dakota has no income tax, so 033's $243 cannot be a line 1 income-tax refund at all.' The prompt says nothing about prior-year residence or out-of-state income. A household living in SD in 2026 can get a refund of another state's or a locality's 2025 income tax, and that refund is Schedule 1 line 1. Refunds of real-property or general sales tax are also section 111 recoveries (Pub. 525 (2025), recoveries worksheet line 2). SD itself pays sales and property tax refunds to seniors, per dor.sd.gov, but its income limits ($17,215 alone, $23,265 household) rule out this household. The conclusion is unchanged, because taxability still turns on unlisted prior-year facts. The SD fact is at most a plausibility hint.", + "The ambiguity runs both ways more than the settlement says, and no single default settles the six outputs. That is why exclusion is right and hold_by_convention on either value is not.\n- 117: if the 2026 facts also held in 2025, the household would likely have itemized in 2025. Using the engine's 2026 SALT proxy of 9,193.71 plus the full $25,900 of cash charity gives about $35,094, against the 2025 MFJ standard deduction of $31,500 (2025 Form 1040 instructions). The 0.5% charitable floor and the $2,000 170(p) non-itemizer deduction only apply to tax years beginning after 2025 (Pub. L. 119-21, sections 70424(b) and 70425(c)). The tax benefit of about $3,594 exceeds the $2,589 refund, so the whole refund is taxable, which gives the 2.15.17 value. The prompt's rule that unlisted status is false gives the board value instead.\n- 033 under the same constant-facts reading: itemizable tax of 1,168.80 is far below the standard deduction, so none of the refund is taxable, which gives the board value.\n- 078: 2.15.17's own SALT paid is 15,791.66 + 26,460 = 42,251.66, which is 1,851.66 over the $40,400 cap and more than the $1,631.91 refund. By Rev. Rul. 2019-11 Situation 2 (IRB 2019-17, 2019-04-22; refund 'not includable'), the refund would be wholly nontaxable if 2025 looked like 2026, since the 2025 cap was $40,000. That holds even though this filer itemizes.", + "The two cluster settlements conflict on 078. The md_county_tax_8888 settlement proposes adopt_latest for 078 federal (24,164.457031, or 23,772.800781 if #9122 is held). This settlement makes 078 federal an unlisted-input exclusion. The lead must reconcile them. Under this review, 078 federal is excluded with frozen_value 24,164.457031 (2.15.17 + conventions) and alternative_value 23,772.800781 (the alt module; I verified it). That alternative assumes md_county_tax_8888 keeps #8888's county-SALT piece. If md holds it, the alternative must be recomputed.", + "Outputs already excluded: I verified that the refund reading alone makes the alt module equal the board exactly for 005 federal (106,505.898438), 082 federal (9,563.052734) and 120 federal (40,021.816406). But their r02_ira_219g records carry 1.755.4 frozen values. On 2.15.17 those become 107,833.156250, 9,577.902344 and 40,416.667969. Their r02 alternative values were computed when the refund was never income, so each refreshed record has to state which refund reading its alternative uses. 005 CA state also moves under the alt: 41,267.011719 becomes 41,219.867188, a change of -47.144531. It is already excluded (r02+r11) and should name the refund input too. The excl_r02_ira_219g_federal and excl_r11 reviews own those records.", + "The model tallies are incomplete, though the ruling is unchanged.\n- 033: exactly 7 models land within $1 of the 2.15.17 value: claude-opus-5.5, gpt-5.6-luna, gpt-5.6-sol, gpt-5.6-terra, gpt-6-astra, gpt-6-luna and ox-alpha. None land within $1 of the board. Besides the three named, qwen-3.7-max explicitly excluded the refund, and gemini-3.7-flash implicitly did (its stated AGI of 83,208 leaves it out).\n- 117: no model is within $1 of either value. qwen-3.7-max's explanation stops mid-recalculation, so calling it an excluder is not supported." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_033", + "variable": "federal_income_tax_before_refundable_credits", + "action": "exclude_unfixed_defect", + "proposed_reference": 3818.148193, + "reason": "Exclude from scoring. The enum has no unlisted-input code, so this is recorded as reason_code reference_depends_on_unlisted_input; it is not an engine defect.\n\nThe record: frozen_value 3818.148193 (2.15.17 + latest_conventions; I reproduced it bit-for-bit, and it equals out/latest_conventions.csv). alternative_value 3788.988037, which equals the board, v11_1755, the alt module, and 2.15.17 + conventions with only salt_refund_income dropped from gov.irs.gross_income.sources (my rv_undo9122_federal.py). Engine-free recompute: 3,818.15 vs 3,788.99. That is AGI 83,451.24 vs 83,208.24, with taxable Social Security capped at 16,320 either way, $47,500 of standard plus senior deductions, and a difference of 0.12 x 243 = 29.16.\n\nThe law is pre-freeze: 26 USC 111(a), last amended by Pub. L. 99-514 section 1812(a)(1), 1986-10-22. The 2025 Form 1040 instructions (created 2026-02-25) say 'None of your refund is taxable if ... you ... didn't itemize'. Pub. 525 (2025) says the same.\n\nThe prompt labels the input 'state and local tax refund income', without the 'taxable' the prompt uses for already-taxable inputs, and gives no prior-year facts. So the value depends on an unlisted fact." + }, + { + "scenario_id": "scenario_117", + "variable": "federal_income_tax_before_refundable_credits", + "action": "exclude_unfixed_defect", + "proposed_reference": 24961.339844, + "reason": "Exclude from scoring, recorded as reason_code reference_depends_on_unlisted_input; it is not an engine defect.\n\nThe record: frozen_value 24961.339844 (2.15.17 + latest_conventions, reproduced). alternative_value 24391.796875, which equals the board, v11_1755, the alt module, and the undo-#9122-federal run.\n\nEngine-free recompute: AGI 250,733.35 (248,675 - 530.47 + 2,588.82). Taxable income is 191,533.35 after the $32,200 standard deduction, the $2,000 170(p) charity deduction and the $25,000 225(b) overtime deduction (MAGI is under $300,000). Tax is 31,561.34, less the $6,600 CTC, for 24,961.34. CDCC is 0 because the head has no earned income (21(d)(1)(B)). Without the refund the result is 24,391.80.\n\nThe engine probe confirms the household does not itemize under either reading: itemized deductions are 33,840.04 or 33,757.20, against 34,200 of standard plus 170(p). AR state moves only 0.48 under the alternative, below tolerance, so it stays scored.\n\nThe ambiguity runs both ways here. The 'unlisted status = false' default gives the board value. Carrying the 2026 facts into 2025 implies the household itemized in 2025 and got a full tax benefit, which gives the 2.15.17 value. The prompt resolves neither, so neither value can be scored." + } + ], + "what_i_checked": "Harness, run myself on policyengine-us 2.15.17 via sweep_latest.py. Outputs are in this dir.\n- conv_full.csv: full 1,984-output sweep with fixes/latest_conventions.py. It is 29 moved and identical to out/latest_conventions.csv to 0.0.\n- alt_full.csv: full sweep with fixes/latest_alt_salt_refund_no_prior_benefit.py. Against conventions, exactly 7 outputs move by more than $1: 005 fed -1327.26, 005 CA -47.14, 033 -29.16, 078 fed -391.66, 082 fed -14.85, 117 fed -569.54 and 120 fed -394.85. Two move by less than $1: 117 AR -0.48 and 078 MD -0.85. Only 8 scenarios list salt_refund_income (005, 033, 074, 078, 082, 088, 117, 120). 074 LA and 088 TX have zero federal tax under every reading.\n- undo_fed8.csv: my own rv_undo9122_federal.py, which is conventions minus the #9122 federal entry. 033 and 117 federal equal the board bit-for-bit. The module had to be idempotent because CountryTaxBenefitSystem applies a reform twice at init (system.py).\n- rv_probe.json: an engine probe of AGI, deductions, itemization and SALT for 033, 117 and 078 under both readings.\n- rv_recompute.py/.json: an engine-free recompute matching both readings to the cent.\n\nUpstream (read-only git): 316e7832a1 (2026-09-08), 5581f5be1a (2026-09-09), 56d7361c31 (2026-09-14) and merge e990b4a6f8 (2026-09-16) are all ancestors of 79be99f671 and absent from 06665727d8. The 316e diff adds salt_refund_income to sources.yaml and documents the input as the taxable line 1 amount. The label is unchanged. At 06665727d8 only ID, SC and WV (and UT via salt_refund_last_year) used the refund, confirming the old inconsistency. The installed 2.15.17 sources.yaml and variable match. policyengine-us-data puf.py:375 has salt_refund_income = puf.E00700.\n\nPrimary sources, fetched this session:\n- 26 USC 111(a), 151(d)(5)(C), 225(b), 170(p) and 63(b) from uscode.house.gov, with amendment and effective-date notes: 111 was last amended in 1986; the 170 amendments by 70424 and 70425 apply to tax years after 2025.\n- 2025 i1040gi: PDF CreationDate 2026-02-25, with the Schedule 1 line 1 'None of your refund is taxable' text and the 2025 MFJ standard deduction of $31,500.\n- Pub. 525 (2025): CreationDate 2026-02-25, 'Deductions not itemized' and Example 29.\n- Rev. Proc. 2025-32: CreationDate 2025-10-17. Checked the 2026 MFJ brackets, the $32,200 standard deduction, the $1,650 aged amount and the $2,200 CTC.\n- Rev. Rul. 2019-11, IRB 2019-17 (2019-04-22): Situations 1 and 2.\n- SD DOR (search results only; direct page fetches returned 404): no state income tax, and the senior sales and property tax refund program with its income limits.\nAll of these predate 2026-07-03.\n\nPrompt and precedent: I read the 033 and 117 prompts, the label map at policybench/prompts.py:130, and the board's reference_exclusions.json, including the r16/r24 unlisted-input precedents and the r02 records for 005, 082 and 120.\n\nModel answers: tabulated from predictions.csv.gz in model_readings_rv.txt.\n\nCross-cluster: I read the md_county_tax_8888 review prompt and found the conflict over 078 federal." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/school_meals_child_support_income.json b/reference_audit/2026-09-28/verification/reviews/school_meals_child_support_income.json new file mode 100644 index 00000000..7c4d3a42 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/school_meals_child_support_income.json @@ -0,0 +1,19 @@ +{ + "agree": true, + "problems": [ + "Minor, and it does not change the verdict: the list of upstream changes leaves out two commits in 06665727d8..79be99f671 that touch school-meal parameter files. f5ee39ff10 (2026-08-14) edits only the reference metadata in sources.yaml: it relabels the 273.9(b) citation as CFR and adds 7 CFR 247.9(d)(1). d7fc17abfe (2026-09-02) adds propagate_metadata_to_children to state_universal_free_meals.yaml. Neither changes any value. My own diagnostic restored every 1.755.4 school-meal list at once: sources without child_support_received and general_assistance, and categorical eligibility without receives_snap and receives_tanf. It still moves only this one output.", + "Minor imprecision: the finding says 7 CFR 245.2 'Current income' lists support payments. It does not. 'Current income' is defined as 'income, as defined in \u00a7 245.6(a)', and only the 245.2 'Documentation' definition lists 'support payments' explicitly. The legal conclusion is unaffected.", + "Minor: fe1cb9ab45 justifies its own change with the CSFP rule, 7 CFR 247.9(d) (gross income, child support not excluded), not the school-meal rule. The school-meal basis (7 CFR 245.6(a)(5)(ii) and FNS IEG item (11)) comes from the investigator's research and from the later 0b558fd335 citation. I verified that basis independently from primary sources.", + "Minor: in the Federal Register text of the SY2026-27 IEG (FR 2026-06842), the table is an embedded image. So the $61,050 reduced-price limit for a household of 4 is computed from the notice's stated rule (the 2026 guideline x 1.85, rounded up), not read from the published table. The notice's '2.6 percent increase' for a family of four matches $32,150 rising to $33,000. The SY2025-26 table (FR 2025-03821) is text and shows 59,478 for 4, as the investigator stated." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_028", + "variable": "reduced_price_school_meals_eligible", + "action": "adopt_latest", + "proposed_reference": 0, + "reason": "Reproduced on 2.15.17 with latest_conventions: school_meal_countable_income is 61,277, spm_unit_fpg is 33,000, the ratio is 1.8569, the tier is PAID and reduced_price_school_meals is 0. On 1.755.4 the income is 60,010, the tier is REDUCED and the value is 3,009.82. A full 1,984-output sweep with latest_conventions plus the 1.755.4 school-meal lists differs from the latest_conventions sweep only in this output. The investigator's drop-child-support diagnostic gives the same result when I re-run it. The law is pre-freeze. eCFR shows 7 CFR 245.6 as of 2026-06-30, with 'support payments' in (a)(5)(ii) and 245.2 'Current income' pointing to 245.6(a). Its last substantive version is 2016-12-22. The FNS IEG notices were published 2025-03-13 and 2026-04-09, and item (11) of each is 'alimony or child support payments'. The HHS guidelines were published 2025-01-17 ($32,150) and 2026-01-15 ($33,000). The independent recompute is 60,000 + 10 + 1,267 = 61,277. That is above the reduced-price limit in both school years (61,050 for SY2026-27, 59,478 for SY2025-26), so the value is 0. It stays 0 if the wage is annualized from the stated $29/hr x 40 hours as well." + } + ], + "what_i_checked": "I read LATEST_BRIEF.md, the investigator's scratch files, sweep_latest.py, benchmark_specs.json and the 2.15.17 school-meal code. The output maps to reduced_price_school_meals, where a positive value counts as 1. That value is the REDUCED tier times net subsidy. The tier compares school_meal_countable_income / spm_unit_fpg with 1.30 and 1.85, and PA is false in state_universal_free_meals.\n\nI diffed the school-meal variables and parameters between the 1.755.4 and 2.15.17 venvs. The only value changes are the two added income sources and the two added categorical-eligibility flags. With read-only git, I listed every commit in 06665727d8..79be99f671 that touches school_meals. Running git log -S child_support_received on sources.yaml returns only fe1cb9ab45 (2026-08-14). I read fe1cb9ab45, f5ee39ff10, 0b558fd335, 1de01086ae, 14596edddb, f0282be843 and d7fc17abfe.\n\nI re-ran the harness three times:\n- A full latest_conventions sweep, which matches the existing out/latest_conventions.csv exactly, with 29 moved including scenario_028.\n- The investigator's diag_drop_child_support.py.\n- My own diagnostic that restores all 1.755.4 school-meal lists.\n\nBoth diagnostics differ from latest_conventions only in scenario_028 reduced_price_school_meals_eligible. A trace of the chain on both engines through the harness situation builder confirms the investigator's numbers.\n\nI checked the board's revisions and reference_exclusions.json: none touches this output, so it was never excluded, and no input is left unlisted. The relevant facts (wages, interest, child support, household of 4, PA, no SNAP/TANF) are all stated.\n\nFrom the publishers, I fetched fresh copies of the Federal Register API metadata and full texts for FR 2025-03821, 2026-06842, 2026-00755 and 2025-01377. Publication dates, the income definition item (11), the rounding rule and the guideline tables all match the investigator's copies byte for byte. I also fetched eCFR 7 CFR 245.2 and 245.6 as of 2026-06-30 (again byte-identical) and the part 245 versions API. Everything cited was published before 2026-07-03. I wrote an independent recompute.\n\nScratch files are in /Users/maxghenis/PolicyEngine/policybench/results/local/adds202609/triage/latest/school_meals_child_support_income/review/: rev_latest_conventions_full.csv, rev_diag_revert_school_meal_params.py/.csv, rev_diag_drop_child_support_rerun.csv, rev_trace.py, rev_trace_21517_conventions.txt, rev_trace_1755.txt, rev_recompute.py/.txt, and the fresh fr_*/ecfr_* copies. I modified no existing file." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/reviews/snap_abawd_hours_default_9261.md b/reference_audit/2026-09-28/verification/reviews/snap_abawd_hours_default_9261.md new file mode 100644 index 00000000..1e2587e3 --- /dev/null +++ b/reference_audit/2026-09-28/verification/reviews/snap_abawd_hours_default_9261.md @@ -0,0 +1,37 @@ +{ + "agree": true, + "problems": [ + "066's 3,576 is not a raw 2.15.17 value. It exists only with the stated-hours mapping; latest_conventions gives 0. Labelling the action adopt_latest invites adopting 0. The adoption holds only if the mapping (hours_worked_last_week copied to weekly_hours_worked_before_lsr) ships in the builder that generates the regenerated references (policybench Scenario.to_pe_household or ground_truth) and is recorded as a revision in the sidecar. Keeping it only in triage/sweep/fixes/latest_map_stated_hours.py is not enough.", + "066's prompt contradicts itself. It says 'usual weekly hours worked: 40' and that work facts are constant all year, yet wages are $520 a year at $32/h, about 16 hours a year. I agree the stated hours govern the 7 U.S.C. 2015(o)(2) test, because the prompt states them and states that work facts are constant. In predictions.csv the models that reasoned about hours used 40: fable-5, fable-5.1, gpt-5.5, gpt-5.6-sol, gpt-6-astra, kimi-k3 and ox-alpha all answered 3,576, and fable-5.1 wrote 'Work hours (40/week) satisfy ABAWD rules'. All 10 zero answers cite assets or income; none cites hours. Record this as a residual risk, not a reason to exclude.", + "The investigator's 056 figures leave the source of the $95 unstated. It is New Jersey's state-funded minimum benefit, not a formula result: the federal amount is 298 - 244 = $54, and 2.15.17's snap_min_allotment reads 95 from gov.states.nj.snap.amount. The law section never cites NJ law, and hand_calc.py cites only the engine parameter ('board value'). Primary sources: the NJ DHS SNAP eligibility page (live 2026-09-29: 'New Jersey has a minimum monthly SNAP benefit amount of $95') and the Governor's release of 2023-02-08 (Wayback capture 2023-12-30). Both predate the freeze, and the parameter is unchanged between 06665727d8 and 79be99f671. The refreshed 056 record should cite them, because 95, 1,140 and 1,908 all rest on NJ's minimum or the federal formula floor.", + "112 at 0 hours: the frozen 0 comes from the engine's documented steady-state shortcut (meets_snap_abawd_work_requirements docstring), not from the law. Under 7 U.S.C. 2015(o)(2) and 7 CFR 273.24(b), with no prior receipt listed, the person has 3 countable months: 3 x $24 = $72. The stated $5,923 in wages also imply at most 15.7 hours a week at the $7.25 federal minimum wage, so the 40-hour alternative is itself in tension with the stated facts. Both points strengthen keep_excluded. The new alternative_reading text should not present 0 as the law's answer under the zero-hours reading.", + "112 income: 2.15.17's SNAP unearned list omits farm_rent_income ($1,920). 7 CFR 273.9(b)(2)(ii) counts rental income net of costs as unearned (as earned under (b)(1)(ii) when the owner manages the property 20+ hours a week). So 'net $1,269' is the engine's figure; the legal net is about $1,429. The benefit is the $24 one-person minimum either way (273.10(e)(2)(ii)(C)), so 288 stands. This is a wording fix only.", + "056 at 0 hours also turns on an unlisted county: the engine falls back to ATLANTIC_COUNTY_NJ, which was waived in January, while Morris would give 0. It also turns on 3-in-36 history; the law gives up to about 4 x $95 = $380 if January is waived and 3 countable months are unused. And housing_cost counts the $11,949 property tax as a shelter cost with no check on whether it is the occupied home, the same question the record raises for the mortgage. None of this changes 95, 1,140, 159 or 1,908, but the record should name the county and prior-months dependence.", + "The citation of the HHS 2026 poverty guideline ($15,960) is not the FY2026 SNAP standard. FY2026 limits come from the FNS memo table ($1,305 at 100%, $1,696 at 130%, $2,152 at 165% for one person). It does not matter here: every gross test passes with a wide margin (112: $1,737 including farm rent vs $2,152).", + "Required record edits, confirmed against policybench/reference_exclusions.py: both records need engine_version 'policyengine-us 2.15.17', the new frozen_value and alternative_value, and flipped alternative_reading text. Otherwise load_reference_exclusions rejects 112 (0 == 0, 'not excludable'), and verify_exclusions_against_reference rejects both (recorded 287.68 and 1,140 vs regenerated 0 and 95). The sibling excl_snap_mortgage_residence settlement defers 056's combined alternative to this cluster ('95 to 159; a different cluster whose recorded alternative also combines r14'), so 1,908 does not conflict with it." + ], + "corrected_per_output": [ + { + "scenario_id": "scenario_066", + "variable": "snap", + "action": "adopt_latest", + "proposed_reference": 3576, + "reason": "Adopt 2.15.17 plus the stated-hours builder mapping: 3,576 = 12 x $298. The prompt states 40 usual weekly hours. That meets 7 U.S.C. 2015(o)(2)(A) and 7 CFR 273.24(a)(1)(i) (80 hours a month) and exempts the person under 2015(d)(2)(E), (o)(3)(D) and 7 CFR 273.7(b)(1)(vii) and 273.24(c)(5). All were verified from uscode.house.gov and from eCFR as of 2026-07-01 (last amended 2025-01-17). Net income is 0: 43.33 - 8.67 - 209 < 0. VA BBCE has no asset limit (FNA chart updated 2026-06-29), and the FY2026 figures come from the FNS memo signed 2025-08-14. My sweep shows the mapping moves only this output of 1,984, and forcing 40 hours on every person moves only this output and the two excluded ones, so no other output depends on #9261. Condition: the mapping must land in the reference-generating builder and be recorded in the sidecar. If it does not, use exclude_unfixed_defect with corrected value 3,576, naming the builder defect: a prompt-visible 'usual weekly hours' fed only to hours_worked_last_week, which no SNAP rule reads." + }, + { + "scenario_id": "scenario_112", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 0, + "reason": "Hours are unlisted, and at 0 hours the law also depends on unlisted 3-in-36 history, so no engine version resolves this. Refresh the record: frozen_value 0 (the 2.15.17 plus conventions reference), alternative_value 288, engine_version 2.15.17, and flip the alternative_reading text. At 20 or more hours, TX BBCE (165%; $5,000 assets with one vehicle up to $22,000 excluded) is met, and the net income of about $1,269 (engine) or $1,429 (farm rent counted) leaves only the $24 one-person minimum: 12 x 24 = 288. The text should say the 0 comes from the engine's no-history shortcut (the law gives up to $72), and that the stated wages imply at most 15.7 hours a week at the federal minimum wage." + }, + { + "scenario_id": "scenario_056", + "variable": "snap", + "action": "keep_excluded", + "proposed_reference": 95, + "reason": "Two inputs are still unlisted: hours, and whether the mortgage (and the property tax) is on the occupied home. At 0 hours the value also depends on the county and on 3-in-36 history. Refresh the record: frozen_value 95, alternative_value 1,908, engine_version 2.15.17, flipped alternative_reading. The 95 is January 2026 only, from the FNS-approved 20-county waiver (letter 2024-12-16, through 2026-01-31; NJ DFD confirms) applied to the engine's Atlantic County fallback. The amount is the NJ state minimum $95 (law signed 2023-02-08; federal formula $54). The alternative of 40 hours with the mortgage counted gives shelter capped at $744, net $461, contribution $139, so 12 x 159 = 1,908. I reproduced it with my own composition (latest_alt_snap_mortgage_residence reform plus the latest_alt_unlisted_hours_40 patch) and by hand. The one-input alternatives are 1,140 (40 hours only) and 159 (mortgage only)." + } + ], + "what_i_checked": "Harness, all 1,984 outputs, my own runs in this folder: no-fix matches out/pepy612_us21517.csv exactly (41 moved). latest_conventions (29 moved), latest_map_stated_hours (28) and latest_alt_unlisted_hours_40 (26) each match the investigator's CSVs to 0.0. conventions -> map changes only 066 snap (0 -> 3,576). map -> alt40 changes only 056 (95 -> 1,140) and 112 (0 -> 288). My extra sweep rv_all40.py sets 40 hours for every person, undoing #9261: it changes exactly these 3 outputs versus conventions, and 0 outputs versus alt40, so stated hours below 20/30 matter nowhere. Monthly traces (trace_*.out) show the ABAWD test is the only failing rule; the county fallbacks are ACCOMACK_COUNTY_VA, ANDERSON_COUNTY_TX and ATLANTIC_COUNTY_NJ (waived January only); and snap_min_allotment is 95 in NJ. combo_alt40_mortgage.py gives 1,908 for 056 and combo_alt0_mortgage.py gives 159. Code: read git show 82745ca239 (default 40 -> 0; merged a112cc5a0a on 2026-08-12; in 79be99f671, not in 06665727d8), and checked the ancestry and dates of d325c7a9af/7193611d84, 74b0a75e5f/adea8827e5/51af6c14e5, cf165464a6/3c41c31457 and 48a10d4d43/7fc69a35b6. Read the 2.15.17 SNAP work-rule, snap_min_allotment and housing_cost variables and the NJ minimum parameters; git grep lists every reader of the hours variables. Prompt: rendered make_no_tools_prompt for all three scenarios; the hours label and the 'unlisted numeric = 0' rule have been in prompts.py since 2026-05-01, before the 2026-06-12 run. Read every model's snap answer and explanation for all three. Law, fetched this session: eCFR 7 CFR 273.24 and 273.7 as of 2026-07-01 plus the version list (last amended 2025-01-17); 7 U.S.C. 2015 from uscode.house.gov (OBBBA amendments); a Wayback copy of the FNS FY2026 COLA memo, byte-identical to the investigator's (sha256 67ecdba8), signed 2025-08-14; the NJ DHS $95 minimum page and the Governor's release of 2023-02-08. Read the investigator's copies of the NJ FNS waiver letter (2024-12-16), the NJ DFD Federal Changes page, the FNA BBCE chart (updated 2026-06-29) and 7 CFR 273.10(e)(2)(ii)(C) and 273.9(b). Checked policybench/reference_exclusions.py validation, and read the sibling mortgage-residence settlement for 056. Independent arithmetic is in hand_calc_rv.py/.out: 066 3,576; 112 288 with or without farm rent; 056 95, 1,140, 159 and 1,908." +} \ No newline at end of file diff --git a/reference_audit/2026-09-28/verification/sweep_timing.json b/reference_audit/2026-09-28/verification/sweep_timing.json new file mode 100644 index 00000000..e5c2ecb2 --- /dev/null +++ b/reference_audit/2026-09-28/verification/sweep_timing.json @@ -0,0 +1,44 @@ +{ + "note": "When PolicyBench began sweeping the references, and which policyengine-us release was the newest then and at publication. Times are UTC. The PyPI upload times were read from the PyPI JSON API; the sweep times are the filesystem birth times (stat -f %SB) of the files the sweep wrote in results/local/adds202609/triage, and the pin commit's committer date is in git. The sweep began after the script was written and before its first output.", + "pypi": { + "source": "https://pypi.org/pypi/policyengine-us/json", + "read_at_utc": "2026-09-29T14:58:39Z", + "newest_at_read": "2.17.0", + "wheel_uploaded_at_utc": { + "2.15.16": "2026-09-28T19:05:41.033997Z", + "2.15.17": "2026-09-29T00:23:56.685108Z", + "2.16.0": "2026-09-29T04:10:00.290696Z", + "2.17.0": "2026-09-29T12:21:14.049966Z" + } + }, + "reference_sweep": { + "engine": "2.15.17", + "engine_installed_at_utc": "2026-09-29T01:41:34Z", + "engine_installed_evidence": "triage/.venv-pe21517/lib/python3.13/site-packages/policyengine_us-2.15.17.dist-info", + "script_written_at_utc": "2026-09-29T01:41:52Z", + "script": "triage/sweep/sweep_latest.py (committed as reference_audit/2026-09-28/scripts/sweep_latest.py)", + "first_output_at_utc": "2026-09-29T01:42:18Z", + "first_output": "triage/sweep/out/latest_2_15_17_check.csv", + "pin_commit": { + "commit": "be90e543d1164efc2b959846ecbcfb2a3b5e819d", + "committed_at_utc": "2026-09-29T01:58:29Z", + "subject": "Pin policyengine.py 6.1.2 and policyengine-us 2.15.17" + } + }, + "publication_check": { + "engine": "2.17.0", + "engine_installed_at_utc": "2026-09-29T13:24:07Z", + "engine_installed_evidence": "triage/.venv-pe2170/lib/python3.13/site-packages/policyengine_us-2.17.0.dist-info", + "output_at_utc": "2026-09-29T13:25:31Z", + "output": "triage/sweep/out/latest_final_2170.csv (committed as verification/latest_final_2170.csv)" + }, + "later_checks": [ + { + "read_at_utc": "2026-09-29T18:28:04Z", + "source": "https://pypi.org/pypi/policyengine-us/json", + "newest_at_read": "2.17.1", + "wheel_uploaded_at_utc": "2026-09-29T17:24:17.953926Z", + "result": "policyengine-us 2.17.1 (policyengine-core 3.32.10, policyengine 6.1.2) gives the same value as 2.15.17 for all 1,984 outputs under latest_final (verification/latest_final_2171.csv, sha256 0536eff572262063c0c62f82dc5a30be48b3e59fcb6d4a315cee2d16a3c3bab3)" + } + ] +} diff --git a/scripts/bbce_household_rows.py b/scripts/bbce_household_rows.py new file mode 100644 index 00000000..f898898e --- /dev/null +++ b/scripts/bbce_household_rows.py @@ -0,0 +1,234 @@ +"""Every model's SNAP answer for the households that qualify only through BBCE. + +Writes two files, each one row per board model and household, from the +committed pathway recomputation ``notes/data/snap_pathways_20260922.csv``: + +- ``notes/data/bbce_households_20260922.csv``: the four scored SNAP outputs + whose household fails an ordinary SNAP income test, qualifies only through + broad-based categorical eligibility and gets the minimum benefit. Each row + has the reference, the model's answer, whether it is within the $1 + exact-match tolerance, whether its explanation mentions categorical + eligibility, and whether it mentions a net income limit. +- ``notes/data/bbce_asset_households_20260922.csv``: the four households that + pass SNAP's income tests all year, fail only its asset test, and qualify + through broad-based categorical eligibility. Each row has the reference, + whether that output is scored, the model's answer, and whether its + explanation mentions categorical eligibility and assets. + +``tests/test_notes.py`` regenerates the rows and compares, so the note's +counts stay tied to the frozen payload of release dashboard-data-20260922b. +That release excludes scenario_045's SNAP output (root cause r33, the SNAP +child support treatment), so the Michigan worker is no longer among the +scored households. + +The rows come from the run's committed payload (``data.json.gz`` in the run +directory). The release asset ``dashboard-data.json`` carries the same US +payload under ``countries.us`` but is a different file, so each meta records +both hashes: ``run_payload_sha256`` for the committed payload and +``release_payload_sha256`` for the asset, as the snapshot manifest pins it. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path + +from policybench.snapshot_payload import read_run_payload, run_payload_path + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper/snapshot/20260501/runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +PATHWAYS = ROOT / "notes/data/snap_pathways_20260922.csv" +OUTPUT = ROOT / "notes/data/bbce_households_20260922.csv" +META = OUTPUT.with_suffix(OUTPUT.suffix + ".meta.json") +ASSET_OUTPUT = ROOT / "notes/data/bbce_asset_households_20260922.csv" +ASSET_META = ASSET_OUTPUT.with_suffix(ASSET_OUTPUT.suffix + ".meta.json") +MANIFEST = ROOT / "paper/snapshot/20260501/manifest.json" +RELEASE = "dashboard-data-20260922b" +# The note's mention patterns, matched case-insensitively. +# Categorical eligibility: the rule by name, or a household called +# categorically eligible, with a space or a hyphen ("categorical-eligibility +# ceiling"). It leaves out "categorically ineligible" and the ABAWD +# "categorically exempt" wording. +BBCE_PATTERN = r"broad-based|\bbbce\b|categorical(?:ly)?[- ]eligib" +# Assets: the September 3 note's pattern. +ASSETS_PATTERN = r"asset|resource" +# A net income limit or test: "net income limit", "net-income test", "net +# limit", or 100% of the poverty guideline. A bare "net income" (an amount) +# does not count. +NET_LIMIT_PATTERN = ( + r"\bnet[- ](?:income[- ])?(?:limit|test|screen|threshold|ceiling|rule)" + r"|(? list[str]: + """Scored SNAP outputs that pass only through BBCE and get the minimum. + + The household fails the ordinary gross or net income test, is eligible for + the TANF-funded non-cash benefit, and its twelve monthly allotments are + the minimum allotment. + """ + households = [] + for row in pathways: + year_pathways = {row["pathway_jan_sep"], row["pathway_oct_dec"]} + if row["snap_scored"] != "True" or not year_pathways <= set( + CATEGORICAL_PATHWAYS + ): + continue + monthly = [float(value) for value in row["monthly_snap"].split()] + minimum = float(row["min_allotment_jan"]) + if minimum > 0 and all(value == minimum for value in monthly): + households.append(row["scenario_id"]) + return sorted(households) + + +def asset_test_households(pathways: list[dict[str, str]]) -> list[str]: + """Households that qualify through BBCE only because of the asset test. + + They pass SNAP's gross and net income tests but fail its asset test all + year, so the pathway recomputation labels both parts of the year + ``categorical_assets``. Scored or not: the prompt asks every model. + """ + return sorted( + row["scenario_id"] + for row in pathways + if row["pathway_jan_sep"] == row["pathway_oct_dec"] == ASSET_PATHWAY + ) + + +def household_rows( + payload: dict, + households: list[str], + mentions: dict[str, str] = MINIMUM_BENEFIT_MENTIONS, + *, + scored_column: bool = False, + exact_column: bool = True, +) -> list[dict[str, object]]: + """One row per board model and household, models and households sorted.""" + regexes = { + column: re.compile(pattern, re.IGNORECASE) + for column, pattern in mentions.items() + } + board = sorted( + row["model"] for row in payload["modelStats"] if row["condition"] == "no_tools" + ) + rows = [] + for model in board: + for scenario_id in households: + entry = payload["scenarioPredictions"][scenario_id]["snap"][model] + prediction = entry["prediction"] + explanation = entry.get("explanation") or "" + row: dict[str, object] = { + "model": model, + "scenario_id": scenario_id, + "state": payload["scenarios"][scenario_id]["state"], + "reference": entry["groundTruth"], + } + if scored_column: + row["scored"] = entry.get("scored") is not False + row["prediction"] = "" if prediction is None else prediction + if exact_column: + row["within_1_dollar"] = entry["exact"] == 100 + for column, regex in regexes.items(): + row[column] = bool(regex.search(explanation)) + rows.append(row) + return rows + + +def asset_household_rows(payload: dict, households: list[str]) -> list[dict]: + """The asset-test households' rows: whether each output is scored, and no + exact-match column, since one of the four outputs is not scored.""" + return household_rows( + payload, + households, + ASSET_TEST_MENTIONS, + scored_column=True, + exact_column=False, + ) + + +def read_pathways() -> list[dict[str, str]]: + with PATHWAYS.open(encoding="utf-8", newline="") as source: + return list(csv.DictReader(source)) + + +def release_artifact() -> dict: + """The release asset the snapshot manifest pins for this board.""" + artifact = json.loads(MANIFEST.read_text(encoding="utf-8"))[ + "published_dashboard_artifact" + ] + if artifact["tag"] != RELEASE: + raise ValueError(f"The manifest pins {artifact['tag']}, not {RELEASE}.") + return artifact + + +def _write( + rows: list[dict], + output: Path, + meta_path: Path, + households: list[str], + mentions: dict[str, str], +) -> None: + with output.open("w", encoding="utf-8", newline="") as target: + writer = csv.DictWriter(target, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + meta = { + "release": RELEASE, + "source_run": RUN_DIR.name, + "run_payload_sha256": hashlib.sha256( + run_payload_path(RUN_DIR).read_bytes() + ).hexdigest(), + "release_payload_sha256": release_artifact()["sha256"], + "households": households, + "mention_patterns": mentions, + "rows": len(rows), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "script": "scripts/bbce_household_rows.py", + } + meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {len(rows)} rows for {households} to {output.relative_to(ROOT)}") + + +def main() -> None: + payload = read_run_payload(RUN_DIR) + pathways = read_pathways() + households = bbce_households(pathways) + _write( + household_rows(payload, households), + OUTPUT, + META, + households, + MINIMUM_BENEFIT_MENTIONS, + ) + asset_households = asset_test_households(pathways) + _write( + asset_household_rows(payload, asset_households), + ASSET_OUTPUT, + ASSET_META, + asset_households, + ASSET_TEST_MENTIONS, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/bbce_households_20260929.py b/scripts/bbce_households_20260929.py new file mode 100644 index 00000000..9eede984 --- /dev/null +++ b/scripts/bbce_households_20260929.py @@ -0,0 +1,113 @@ +"""Every model's SNAP answer for the BBCE households on release 20260929. + +The September 29 release note's facts count these files' households, and +``tests/test_notes.py`` regenerates them. The October 5 note on SNAP households +that qualify only through broad-based categorical eligibility reads the +20260930 files that ``scripts/bbce_households_20260930.py`` writes instead. + +- ``notes/data/bbce_households_20260929.csv``: the households held back by + income: Connecticut, Texas, Michigan and Wisconsin, and the Arizona household + the policyengine-us 2.15.17 references add, which qualifies from March 2026, + when Arizona raised its BBCE gross limit to 200% of the poverty guideline. +- ``notes/data/bbce_asset_households_20260929.csv``: the four households held + back by savings. + +Each row has one board model's answer and the mention flags the note's +patterns give its explanation, as ``scripts/bbce_household_rows.py`` writes +them for release dashboard-data-20260922b. The households are listed here +rather than rederived: the pathway recomputation behind the September 22 +lists needs policyengine-us 1.755.4. ``tests/test_notes.py`` regenerates the +rows from the frozen payload and compares them with the committed files. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +from bbce_household_rows import ( + ASSET_TEST_MENTIONS, + MINIMUM_BENEFIT_MENTIONS, + asset_household_rows, + household_rows, +) + +from policybench.snapshot_payload import read_run_payload, run_payload_path + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper/snapshot/20260501/runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +MANIFEST = ROOT / "paper/snapshot/20260501/manifest.json" +RELEASE = "dashboard-data-20260929" +INCOME_HOUSEHOLDS = [ + "scenario_013", + "scenario_027", + "scenario_030", + "scenario_073", + "scenario_108", +] +ASSET_HOUSEHOLDS = ["scenario_008", "scenario_054", "scenario_066", "scenario_080"] +OUTPUT = ROOT / "notes/data/bbce_households_20260929.csv" +ASSET_OUTPUT = ROOT / "notes/data/bbce_asset_households_20260929.csv" + + +def meta_path(output: Path) -> Path: + return output.with_suffix(output.suffix + ".meta.json") + + +def release_payload_sha256() -> str: + artifact = json.loads(MANIFEST.read_text(encoding="utf-8"))[ + "published_dashboard_artifact" + ] + if artifact["tag"] != RELEASE: + raise ValueError(f"The manifest pins {artifact['tag']}, not {RELEASE}.") + return artifact["sha256"] + + +def build(payload: dict) -> dict[Path, tuple[list[dict], list[str], dict[str, str]]]: + """The rows, households and mention patterns of each file.""" + return { + OUTPUT: ( + household_rows(payload, INCOME_HOUSEHOLDS, MINIMUM_BENEFIT_MENTIONS), + INCOME_HOUSEHOLDS, + MINIMUM_BENEFIT_MENTIONS, + ), + ASSET_OUTPUT: ( + asset_household_rows(payload, ASSET_HOUSEHOLDS), + ASSET_HOUSEHOLDS, + ASSET_TEST_MENTIONS, + ), + } + + +def main() -> None: + payload = read_run_payload(RUN_DIR) + run_sha256 = hashlib.sha256(run_payload_path(RUN_DIR).read_bytes()).hexdigest() + for output, (rows, households, mentions) in build(payload).items(): + with output.open("w", encoding="utf-8", newline="") as target: + writer = csv.DictWriter(target, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + meta = { + "release": RELEASE, + "source_run": RUN_DIR.name, + "run_payload_sha256": run_sha256, + "release_payload_sha256": release_payload_sha256(), + "households": households, + "mention_patterns": mentions, + "rows": len(rows), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "script": "scripts/bbce_households_20260929.py", + } + meta_path(output).write_text(json.dumps(meta, indent=2) + "\n") + print(f"Wrote {len(rows)} rows for {households} to {output.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/bbce_households_20260930.py b/scripts/bbce_households_20260930.py new file mode 100644 index 00000000..fb7b2109 --- /dev/null +++ b/scripts/bbce_households_20260930.py @@ -0,0 +1,193 @@ +"""Every model's SNAP answer for the BBCE note's households on release 20260930. + +The October 5 note on SNAP households that qualify only through broad-based +categorical eligibility (BBCE) reads these files: + +- ``notes/data/bbce_households_20260930.csv``: the households held back by + income. Each scored SNAP output whose household qualifies in some month of + 2026, qualifies in every such month only through BBCE after failing an + ordinary SNAP income test, and gets the minimum benefit in each of those + months. Each row has the reference, the model's answer, whether it is within + the $1 exact-match tolerance, and whether its explanation mentions + categorical eligibility and a net income limit. +- ``notes/data/bbce_asset_households_20260930.csv``: the households held back + by savings, which pass SNAP's income tests all year, fail only its asset + test, and qualify through BBCE. Each row has the reference, whether that + output is scored, the model's answer, and whether its explanation mentions + categorical eligibility and assets. + +The households come from the pathway recomputation on the release's +references, ``notes/data/snap_pathways_20260930.csv`` +(``scripts/snap_pathways_20260930.py``), month by month: one household, in +Arizona, qualifies only from March 2026, when Arizona raised its BBCE gross +income limit. ``tests/test_notes.py`` regenerates the rows from the frozen +payload and compares them with the committed files. + +The rows come from the run's committed payload (``data.json.gz`` in the run +directory). The release asset ``dashboard-data.json`` carries the same US +payload but is a different file, so each meta records both hashes: +``run_payload_sha256`` for the committed payload and +``release_payload_sha256`` for the asset, as the snapshot manifest pins it. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from bbce_household_rows import ( # noqa: E402 + ASSETS_PATTERN, + NET_LIMIT_PATTERN, + household_rows, +) + +from policybench.snapshot_payload import ( # noqa: E402 + read_run_payload, + run_payload_path, +) + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper/snapshot/20260501/runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +MANIFEST = ROOT / "paper/snapshot/20260501/manifest.json" +RELEASE = "dashboard-data-20260930" +PATHWAYS = ROOT / "notes/data/snap_pathways_20260930.csv" +OUTPUT = ROOT / "notes/data/bbce_households_20260930.csv" +ASSET_OUTPUT = ROOT / "notes/data/bbce_asset_households_20260930.csv" +SCRIPT_PATH = "scripts/bbce_households_20260930.py" +# Categorical eligibility, matched case-insensitively: the rule by name, a +# household called categorically eligible with a space or a hyphen between +# the words, or a state's "expanded categorical" limit, another name for BBCE +# (GPT-6.1 Sol: "Michigan's expanded categorical gross-income limit"). It +# leaves out "categorically ineligible" and the ABAWD "categorically exempt" +# wording. +BBCE_PATTERN = r"broad-based|\bbbce\b|categorical(?:ly)?[- ]eligib|expanded categorical" +INCOME_MENTIONS = { + "mentions_categorical_eligibility": BBCE_PATTERN, + "mentions_net_income_limit": NET_LIMIT_PATTERN, +} +ASSET_MENTIONS = { + "mentions_categorical_eligibility": BBCE_PATTERN, + "mentions_assets": ASSETS_PATTERN, +} +CATEGORICAL_INCOME_PATHWAYS = {"categorical_income", "categorical_both"} +ASSET_PATHWAY = "categorical_assets" + + +def meta_path(output: Path) -> Path: + return output.with_suffix(output.suffix + ".meta.json") + + +def read_pathways() -> list[dict[str, str]]: + with PATHWAYS.open(encoding="utf-8", newline="") as source: + return list(csv.DictReader(source)) + + +def bbce_households(pathways: list[dict[str, str]]) -> list[str]: + """Scored SNAP outputs held back by income that BBCE brings to the minimum. + + In every month of 2026 the household qualifies, it does so only through + BBCE after failing SNAP's gross or net income test, and it gets that + month's minimum allotment; it qualifies in at least one month. + """ + households = [] + for row in pathways: + pathways_by_month = row["pathway_by_month"].split() + eligible = [p for p in pathways_by_month if p != "ineligible"] + if row["snap_scored"] != "True" or not eligible: + continue + if not set(eligible) <= CATEGORICAL_INCOME_PATHWAYS: + continue + snap = [float(v) for v in row["monthly_snap"].split()] + minimum = [float(v) for v in row["monthly_min_allotment"].split()] + if all( + (s == m > 0) if p != "ineligible" else s == 0 + for s, m, p in zip(snap, minimum, pathways_by_month, strict=True) + ): + households.append(row["scenario_id"]) + return sorted(households) + + +def asset_households(pathways: list[dict[str, str]]) -> list[str]: + """Households that qualify through BBCE only because of the asset test, + in every month. Scored or not: the prompt asks every model.""" + return sorted( + row["scenario_id"] + for row in pathways + if set(row["pathway_by_month"].split()) == {ASSET_PATHWAY} + ) + + +def build( + payload: dict, pathways: list[dict[str, str]] +) -> dict[Path, tuple[list[dict], list[str], dict[str, str]]]: + """The rows, households and mention patterns of each file.""" + income = bbce_households(pathways) + savings = asset_households(pathways) + return { + OUTPUT: ( + household_rows(payload, income, INCOME_MENTIONS), + income, + INCOME_MENTIONS, + ), + ASSET_OUTPUT: ( + household_rows( + payload, + savings, + ASSET_MENTIONS, + scored_column=True, + exact_column=False, + ), + savings, + ASSET_MENTIONS, + ), + } + + +def release_payload_sha256() -> str: + artifact = json.loads(MANIFEST.read_text(encoding="utf-8"))[ + "published_dashboard_artifact" + ] + if artifact["tag"] != RELEASE: + raise ValueError(f"The manifest pins {artifact['tag']}, not {RELEASE}.") + return artifact["sha256"] + + +def main() -> None: + payload = read_run_payload(RUN_DIR) + run_sha256 = hashlib.sha256(run_payload_path(RUN_DIR).read_bytes()).hexdigest() + for output, (rows, households, mentions) in build(payload, read_pathways()).items(): + with output.open("w", encoding="utf-8", newline="") as target: + writer = csv.DictWriter(target, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + meta = { + "release": RELEASE, + "source_run": RUN_DIR.name, + "run_payload_sha256": run_sha256, + "release_payload_sha256": release_payload_sha256(), + "pathways": PATHWAYS.relative_to(ROOT).as_posix(), + "pathways_sha256": hashlib.sha256(PATHWAYS.read_bytes()).hexdigest(), + "households": households, + "mention_patterns": mentions, + "rows": len(rows), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "script": SCRIPT_PATH, + } + meta_path(output).write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + print(f"Wrote {len(rows)} rows for {households} to {output.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/date_adds0928_judge_verdicts.py b/scripts/date_adds0928_judge_verdicts.py new file mode 100644 index 00000000..bdf9ce0e --- /dev/null +++ b/scripts/date_adds0928_judge_verdicts.py @@ -0,0 +1,432 @@ +"""Date the judge verdicts the September 29 adjudication record keeps. + +The upgrade's refresh of the staged ``us_adjudications.json`` moved each +re-judged entry's replaced judge class under ``judge_previous`` and gave it a +``judged_on`` equal to the entry's ``adjudicated_on``. For some entries that +paired claude-opus-5-5 with a date before Opus 5.5 judged anything. It also +left the entry's own ``judged_on_utc`` at the replaced verdict's date and +recorded the re-judge as ``judge_rejudged_on`` 2026-09-28, the local date of a +run whose verdicts carry 2026-09-29 UTC timestamps. + +This script takes every date from the verdict sidecars (``verdict.meta.json``, +bound to its ``verdict.json`` by sha256): + +- a re-judged entry's ``judge_rejudged_on``, and its ``judged_on_utc`` where it + has one, become the UTC day of the case's current verdict in the stage; +- an entry that was not re-judged gets ``judged_on_utc``, the UTC day of the + current verdict its judge fields name; +- each ``judge_previous`` item's ``judged_on`` becomes the UTC day of the + case's previous verdict (the September 22c audit tree) when that verdict's + judge model and classes (failure source and subtype) are the recorded ones; + otherwise the field is renamed ``adjudicated_on``, which is what it holds. + +A recorded judge flag must be the flag of the verdict its date names, or name +the earlier judge run that raised it. The September 22 wave kept the flags +every one of its judge runs raised (flagged_sept22_wave.json), and recorded +such a flag at the top level with ``judge_reference_suspect_source``. When a +later wave re-judged the case, that source moves with the flag: a dated +``judge_previous`` item whose verdict does not flag the reference gets its own +``judge_reference_suspect_source``, and a top-level source is dropped once the +current verdict raises the flag itself. A flag no run explains stops the +script. + +Where a later wave replaced the verdict a decision reviewed without keeping it +(the September 22 audit re-judged ten cases the 2026-09-05 wave had decided), +``adjudicated_verdict`` records that verdict as the decision's own release +recorded it. The record's ``date_conventions`` states how the dates relate. + +It changes nothing else and is idempotent. Run it on the stage, then rerun the +documented chain (triage, export, freeze), which copies the record into +``annotations/``. It also writes the evidence it read (each bound verdict's +judge, classes, flag, time and sha256) to --evidence-out, which the tests +check the record against. + +Usage:: + + python scripts/date_adds0928_judge_verdicts.py --stage-dir STAGE \\ + --previous-cases PATH/TO/unified_audit/audit/cases +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN = "us_full_run_20260612_policyengine_4_16_1_populace" +RECORD = Path("annotations") / RUN / "us_adjudications.json" +VERIFICATION = ROOT / "reference_audit" / "2026-09-28" / "verification" +WAVE_FLAGS = VERIFICATION / "flagged_sept22_wave.json" +EVIDENCE = VERIFICATION / "judge_verdicts.json" + +# The September 22c record's wording for a flag an earlier judge run raised +# (reference_audit/2026-09-22/scripts/build_records.py, FLAG_SOURCE_EARLIER_RUN). +FLAG_SOURCE_EARLIER_RUN = ( + "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); " + "the case's current verdict.json does not flag it" +) +ITEM_FLAG_SOURCE = ( + "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); " + "the verdict dated here does not flag it" +) + +# Each earlier wave's decisions as first published: the adjudication record +# merged with the wave's release, and the day that release was committed. +# This release's own wave has no commit yet: its entry gives the day its +# adjudications were written, and the lead fills commit and pull_request after +# the merge. +WAVE_RELEASES = { + "2026-09-05": { + "commit": "7db59dab0014ba5b92b84bd6077d21e06875e1a9", + "release": "dashboard-data-20260905c", + "pull_request": 164, + "committed_on": "2026-09-05", + }, + "2026-09-22": { + "commit": "56844e2fa795e7cedc04cd3c34cfb4e67e6f08b5", + "release": "dashboard-data-20260922", + "pull_request": 174, + "committed_on": "2026-09-23", + }, + "2026-09-29": { + "commit": None, + "release": "dashboard-data-20260929", + "pull_request": None, + "adjudications_written_on": "2026-09-29", + }, +} + +DATE_CONVENTIONS = ( + "adjudicated_on names the audit wave that made the decision " + "(2026-09-05, 2026-09-22 or 2026-09-29). It carries no time of day or " + "time zone. The 2026-09-05 and 2026-09-22 waves' decisions were written " + "up to the day each wave's release was committed (2026-09-05 and " + "2026-09-23), and the 2026-09-29 wave's decisions were written on " + "2026-09-29 UTC, after its reference sweep began. judged_on_utc, " + "judge_rejudged_on and each judge_previous item's judged_on are UTC " + "days, read from the verdict's sidecar (verdict.meta.json " + "judged_at_utc). The top-level judge fields name the case's verdict in " + "this release's audit tree, dated by judge_rejudged_on when a later " + "wave judged the case again and by judged_on_utc otherwise, and " + "judge_previous keeps the verdict that re-judge replaced. A wave's own " + "judge runs could finish after one of its decisions was first written, " + "so a recorded verdict can postdate adjudicated_on up to the last day " + "its wave's decisions were written. A flag an earlier judge run raised " + "stays recorded and names that run in judge_reference_suspect_source, " + "at the top level or on the judge_previous item it belongs to. Where a " + "later wave replaced the verdict a decision reviewed without keeping " + "it, adjudicated_verdict gives that verdict as the decision's own " + "release recorded it." +) + + +def _bound_verdict(case_dir: Path) -> tuple[dict, dict] | None: + """(verdict, sidecar) when the sidecar is bound to the verdict by sha256.""" + verdict_path = case_dir / "verdict.json" + meta_path = case_dir / "verdict.meta.json" + if not verdict_path.is_file() or not meta_path.is_file(): + return None + meta = json.loads(meta_path.read_text()) + blob = verdict_path.read_bytes() + if meta.get("verdict_sha256") != hashlib.sha256(blob).hexdigest(): + return None + return json.loads(blob), meta + + +def _judge(meta: dict) -> str: + """The judge model a sidecar names, as the adjudication record spells it.""" + requested = meta.get("judge_model_requested", "") + reported = meta.get("judge_model_reported") or [] + if requested in {"opus", "claude-opus-5"} and "claude-opus-5" in reported: + return "claude-opus-5" + return requested + + +def _classes(verdict: dict) -> tuple[str, str]: + return verdict["case_failure_source"], verdict["case_failure_subtype"] + + +def _evidence(tree: str, found: tuple[dict, dict]) -> dict: + verdict, meta = found + return { + "tree": tree, + "judge_model": _judge(meta), + "case_failure_source": verdict["case_failure_source"], + "case_failure_subtype": verdict["case_failure_subtype"], + "reference_suspect": bool(verdict.get("reference_suspect")), + "judged_at_utc": meta["judged_at_utc"], + "verdict_sha256": meta["verdict_sha256"], + } + + +def _set(entry: dict, field: str, value, case: str, changes: list[str]) -> None: + if entry.get(field) != value: + changes.append(f"{case}: {field} {entry.get(field)} -> {value}") + entry[field] = value + + +def date_entries( + entries: list[dict], + current_cases: Path, + previous_cases: Path, + wave_flags: frozenset[str] | set[str] = frozenset(), + evidence: dict | None = None, +) -> list[str]: + """Rewrite the judge dates and flag sources in place; return one line per + changed field. ``wave_flags`` holds "scenario_id:variable" keys the + September 22 wave's judge runs flagged. ``evidence``, when given, collects + each bound verdict the record's dates name, keyed by case.""" + changes: list[str] = [] + for entry in entries: + case = f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + key = f"{entry['scenario_id']}:{entry['variable']}" + current = _bound_verdict(current_cases / case) + if current is None: + raise SystemExit(f"{case}: no bound current verdict in the stage") + verdict, meta = current + if _judge(meta) != entry["judge_model"] or _classes(verdict) != ( + entry["judge_failure_source"], + entry["judge_failure_subtype"], + ): + raise SystemExit(f"{case}: the stage verdict is not the recorded one") + if evidence is not None: + evidence.setdefault(case, {})["current"] = _evidence("stage", current) + day = meta["judged_at_utc"][:10] + previous = entry.get("judge_previous") + if previous: + if "judge_rejudged_on" not in entry: + raise SystemExit(f"{case}: a judge_previous entry names no re-judge") + for field in ("judge_rejudged_on", "judged_on_utc"): + if field in entry: + _set(entry, field, day, case, changes) + else: + _set(entry, "judged_on_utc", day, case, changes) + flagged = bool(entry.get("judge_reference_suspect")) + if flagged != bool(verdict.get("reference_suspect")): + # A flag the current verdict does not raise stays only with the + # source naming the earlier run of the wave that raised it. + if not ( + flagged + and entry.get("judge_reference_suspect_source") + and key in wave_flags + ): + raise SystemExit(f"{case}: the recorded flag is not the verdict's") + elif "judge_reference_suspect_source" in entry: + # The current verdict raises the flag itself, or no flag is kept. + changes.append(f"{case}: judge_reference_suspect_source dropped") + del entry["judge_reference_suspect_source"] + + found = _bound_verdict(previous_cases / case) if previous else None + for item in previous or []: + if "judged_on" not in item: + continue + matches = found is not None and ( + _judge(found[1]) == item["judge_model"] + and _classes(found[0]) + == (item["judge_failure_source"], item["judge_failure_subtype"]) + ) + if not matches: + changes.append( + f"{case}: judge_previous {item['judge_model']} judged_on " + "renamed adjudicated_on (no matching previous verdict)" + ) + item["adjudicated_on"] = item.pop("judged_on") + continue + item_flag = bool(item.get("judge_reference_suspect")) + verdict_flag = bool(found[0].get("reference_suspect")) + if item_flag != verdict_flag: + if not (item_flag and key in wave_flags): + raise SystemExit( + f"{case}: judge_previous records flag {item_flag}, its " + f"verdict says {verdict_flag}, and no earlier run raised it" + ) + if item.get("judge_reference_suspect_source") != ITEM_FLAG_SOURCE: + changes.append( + f"{case}: judge_previous judge_reference_suspect_source added" + ) + item["judge_reference_suspect_source"] = ITEM_FLAG_SOURCE + elif "judge_reference_suspect_source" in item: + changes.append( + f"{case}: judge_previous judge_reference_suspect_source dropped" + ) + del item["judge_reference_suspect_source"] + day = found[1]["judged_at_utc"][:10] + if item["judged_on"] != day: + changes.append( + f"{case}: judge_previous {item['judge_model']} judged_on " + f"{item['judged_on']} -> {day}" + ) + item["judged_on"] = day + if evidence is not None: + evidence.setdefault(case, {})["previous"] = _evidence( + "20260922c", found + ) + return changes + + +def _recorded_verdicts(entry: dict) -> list[tuple[str, str, str]]: + """(judge, failure source, subtype) of each verdict the entry records.""" + verdicts = [ + ( + entry["judge_model"], + entry["judge_failure_source"], + entry["judge_failure_subtype"], + ) + ] + for item in entry.get("judge_previous", []): + verdicts.append( + ( + item["judge_model"], + item["judge_failure_source"], + item["judge_failure_subtype"], + ) + ) + return verdicts + + +def record_adjudicated_verdicts( + entries: list[dict], + published: dict[str, list[dict]], + evidence: dict | None = None, +) -> list[str]: + """Keep the verdict each decision reviewed where a later wave dropped it. + + ``published`` maps a wave date to the adjudication entries its release + published. An entry of that wave whose published judge verdict (model, + classes) is none of the verdicts the record now keeps gets + ``adjudicated_verdict``: that verdict, its recorded day and the release. + ``evidence``, when given, collects each published verdict, keyed by case. + """ + changes: list[str] = [] + for entry in entries: + wave = entry["adjudicated_on"] + first = { + (e["scenario_id"], e["variable"]): e for e in published.get(wave, []) + }.get((entry["scenario_id"], entry["variable"])) + if first is None: + continue + reviewed = ( + first["judge_model"], + first["judge_failure_source"], + first["judge_failure_subtype"], + ) + if evidence is not None: + case = f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + evidence.setdefault(case, {})["published"] = { + "release": WAVE_RELEASES[wave]["release"], + "commit": WAVE_RELEASES[wave]["commit"], + "judge_model": reviewed[0], + "case_failure_source": reviewed[1], + "case_failure_subtype": reviewed[2], + "judged_on_utc": first.get("judged_on_utc"), + } + if reviewed in _recorded_verdicts(entry): + if "adjudicated_verdict" in entry: + changes.append( + f"{entry['scenario_id']}:{entry['variable']}: " + "adjudicated_verdict dropped" + ) + del entry["adjudicated_verdict"] + continue + release = WAVE_RELEASES[wave] + value = { + "judge_model": reviewed[0], + "judge_failure_source": reviewed[1], + "judge_failure_subtype": reviewed[2], + "judged_on": first["judged_on_utc"], + "recorded_in": ( + f"{release['release']} (the adjudication record merged with " + f"PR #{release['pull_request']}, commit {release['commit'][:10]})" + ), + } + if entry.get("adjudicated_verdict") != value: + changes.append( + f"{entry['scenario_id']}:{entry['variable']}: adjudicated_verdict " + f"{reviewed[0]} {first['judged_on_utc']}" + ) + entry["adjudicated_verdict"] = value + return changes + + +def published_records(root: Path = ROOT) -> dict[str, list[dict]]: + """The adjudication entries each earlier wave's release published.""" + out = {} + for wave, release in WAVE_RELEASES.items(): + if release["commit"] is None: + continue + raw = subprocess.run( + ["git", "-C", str(root), "show", f"{release['commit']}:{RECORD}"], + check=True, + capture_output=True, + ).stdout + out[wave] = [ + e for e in json.loads(raw)["adjudications"] if e["adjudicated_on"] == wave + ] + return out + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + parser.add_argument("--previous-cases", type=Path, required=True) + parser.add_argument("--wave-flags", type=Path, default=WAVE_FLAGS) + parser.add_argument("--evidence-out", type=Path, default=EVIDENCE) + args = parser.parse_args() + path = args.stage_dir / "publish" / RUN / "annotations" / "us_adjudications.json" + record = json.loads(path.read_text()) + evidence: dict[str, dict] = {} + changes = date_entries( + record["adjudications"], + args.stage_dir / "audit" / "cases", + args.previous_cases, + frozenset(json.loads(args.wave_flags.read_text())), + evidence, + ) + changes += record_adjudicated_verdicts( + record["adjudications"], published_records(), evidence + ) + if record.get("date_conventions") != DATE_CONVENTIONS: + changes.append("date_conventions set") + # Keep the record's key order: the conventions follow the note. + record = { + key: value + for key, value in { + "schema_version": record["schema_version"], + "note": record["note"], + "date_conventions": DATE_CONVENTIONS, + **record, + }.items() + } + record["date_conventions"] = DATE_CONVENTIONS + path.write_text(json.dumps(record, indent=2, ensure_ascii=False) + "\n") + args.evidence_out.write_text( + json.dumps( + { + "note": ( + "The verdicts the adjudication record's judge dates and flags " + "name, read by scripts/date_adds0928_judge_verdicts.py from " + "each case's verdict.json and its sha256-bound " + "verdict.meta.json. 'current' is the case's verdict in the " + "release's audit tree (the stage); 'previous' is the verdict a " + "judge_previous item dates, from the September 22c audit tree; " + "'published' is the judge verdict the decision's own wave " + "release recorded (git show :" + str(RECORD) + ")." + ), + "wave_releases": WAVE_RELEASES, + "cases": dict(sorted(evidence.items())), + }, + indent=2, + sort_keys=False, + ) + + "\n" + ) + for line in changes: + print(line) + print(f"{len(changes)} changes in {path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/date_gpt61sol_judge_verdicts.py b/scripts/date_gpt61sol_judge_verdicts.py new file mode 100644 index 00000000..eec2064c --- /dev/null +++ b/scripts/date_gpt61sol_judge_verdicts.py @@ -0,0 +1,91 @@ +"""Record the verdicts the 2026-09-30 restatements name, for the tests. + +Release dashboard-data-20260930 restated the judge fields of every decision +on a case GPT-6.1 Sol re-opened (scripts/restate_gpt61sol_adjudications.py). +Each restated entry's judge fields now name that case's new verdict in the +exported stage. This script reads each such verdict from the stage's +verdict.json and its sha256-bound verdict.meta.json and writes +docs/gpt61sol/judge_verdicts_20260930.json, which tests/test_adjudications.py +reads on machines without the stage, as it reads +reference_audit/2026-09-28/verification/judge_verdicts.json for the +2026-09-29 wave. + + python scripts/date_gpt61sol_judge_verdicts.py --stage-dir results/local/gpt61sol-v1 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import finish_gpt61sol as driver # noqa: E402 + +OUT = ROOT / "docs/gpt61sol/judge_verdicts_20260930.json" + + +def case_id(entry: dict) -> str: + return f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + + +def evidence(stage: Path) -> dict: + """The current verdict of each committed decision on a re-opened case.""" + reopened = { + entry["case_id"] + for entry in json.loads(driver.JUDGE_PROVENANCE.read_text())["verdicts"] + } + record = json.loads((driver.ANNOTATIONS / driver.ADJUDICATIONS).read_text()) + cases = {} + for entry in record["adjudications"]: + case = case_id(entry) + if case not in reopened: + continue + folder = stage / "audit" / "cases" / case + raw = (folder / "verdict.json").read_bytes() + meta = json.loads((folder / "verdict.meta.json").read_text()) + driver.require( + meta["verdict_sha256"] == hashlib.sha256(raw).hexdigest(), + f"{case}: the sidecar does not bind this verdict", + ) + verdict = json.loads(raw) + cases[case] = { + "judge_model": meta["judge_model_requested"], + "case_failure_source": verdict["case_failure_source"], + "case_failure_subtype": verdict["case_failure_subtype"], + "reference_suspect": bool(verdict["reference_suspect"]), + "judged_at_utc": meta["judged_at_utc"], + "verdict_sha256": meta["verdict_sha256"], + } + return { + "note": ( + "The verdicts release dashboard-data-20260930's restated " + "adjudication entries name: for each decision on a case GPT-6.1 " + "Sol re-opened, the case's verdict in the exported stage " + "(results/local/gpt61sol-v1), read by " + "scripts/date_gpt61sol_judge_verdicts.py from verdict.json and its " + "sha256-bound verdict.meta.json. The verdict each restatement " + "replaced is release 20260929's entry (git show " + f"{driver.BASE_COMMIT[:12]}:annotations/{driver.RUN_NAME}/" + f"{driver.ADJUDICATIONS})." + ), + "release": driver.RELEASE_TAG, + "base_commit": driver.BASE_COMMIT, + "cases": dict(sorted(cases.items())), + } + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + args = parser.parse_args(argv) + OUT.write_text(json.dumps(evidence(args.stage_dir), indent=1) + "\n") + print(f"Wrote {OUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/finish_adds0928.py b/scripts/finish_adds0928.py new file mode 100644 index 00000000..1ea0cbb3 --- /dev/null +++ b/scripts/finish_adds0928.py @@ -0,0 +1,1004 @@ +"""Stage the September 29 additions; never publish or change tracked data. + +Adapted from the September 22 finish_opus55.py and judge_stages.py. All +outputs, including audit verdicts and adjudications, stay in --stage-dir. +See docs/adds0928/stage2_design.md for the run and release procedure. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +os.environ["OPENBLAS_NUM_THREADS"] = "1" +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +SNAPSHOT = ROOT / "paper/snapshot/20260501/runs" / RUN_NAME +ANNOTATIONS = ROOT / "annotations" / RUN_NAME +BASE_TAG = "dashboard-data-20260922c" +RELEASE_TAG = "dashboard-data-20260929" +BASE_SHA256 = "01e7e72b3a6bdd2d3178ba32625ff769d5b81dc07541af6ea8da2c852774ddcc" +MODELS = { + "sonnet55": "claude-sonnet-5.5", + "grok47": "grok-4.7", + "dsflash41": "deepseek-v4.1-flash", +} +REFERENCE_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", + "us_adjudications.json", +) +JUDGE_MODEL = "claude-opus-5-5" +# main at release dashboard-data-20260922c, whose committed references are the base. +BASE_COMMIT = "3220a7a62b6be83032e9313c9df539c619ad8932" +BASE_REFERENCE_SHA256 = { + "reference_outputs.csv": ( + "800a19bc1c225656b93c4507933d98907e8d1a939248c8b021a9f26d33e9e0c5" + ), + "reference_outputs.csv.meta.json": ( + "df94cc52e2146de8b9b167772e1d6b747ce21e90b1f64e96f94bb2ac4f237d93" + ), + "reference_exclusions.json": ( + "25c4a9fd5fee59d07666254bdc1eae409b4a7c9f4ea0381dc8878a6ac0213035" + ), +} +KEY = ["scenario_id", "variable"] +# The wave's reviewed actions. Its audit_exclusions are outputs the audit +# excluded on review, apart from any engine change (their values do not move). +AUDIT_ACTIONS = ROOT / "reference_audit/2026-09-28/final_actions.json" +NEW_EXCLUSION_CAUSE = "excluded_reference_depends_on_unlisted_input" + + +def require(condition: bool, message: str) -> None: + """Fail before an unsafe or incomplete stage can proceed.""" + if not condition: + raise SystemExit(message) + + +def digest(path: Path) -> str: + """Hash a file without retaining its contents in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def write_json(path: Path, value: object) -> None: + """Write a deterministic, finite JSON artifact.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + + +@dataclass(frozen=True) +class NewRun: + slug: str + model: str + run_dir: Path + predictions: Path + state: dict + + +def discover_new_models(runs_root: Path) -> list[NewRun]: + """Require all three completed, unstopped supervisor runs, as Opus did.""" + found = [] + skipped = [] + for slug, expected in MODELS.items(): + directory = runs_root / slug / "run" + state_path = directory / "run_state.json" + if not state_path.is_file(): + skipped.append(f"{slug}: no run_state.json") + continue + state = json.loads(state_path.read_text()) + completed, total = state.get("completed"), state.get("total") + reason = state.get("stopped_reason") + predictions = directory / "predictions.csv" + if ( + type(total) is not int + or type(completed) is not int + or total <= 0 + or completed != total + or reason is not None + ): + skipped.append(f"{slug}: {completed}/{total}, stopped_reason={reason!r}") + elif state.get("model") != expected: + skipped.append(f"{slug}: unexpected model {state.get('model')!r}") + elif not predictions.is_file(): + skipped.append(f"{slug}: no predictions.csv") + else: + found.append(NewRun(slug, expected, directory, predictions, state)) + require(not skipped, "refusing incomplete additions: " + "; ".join(skipped)) + return found + + +def validate_stage_path(stage: Path, sources: list[Path]) -> None: + """Allow only scratch outputs, disjoint from every read-only source.""" + stage = stage.resolve() + scratch = (ROOT / "results/local").resolve() + require(scratch in stage.parents, f"stage-dir must be below {scratch}") + for source in sources: + source = source.resolve() + require( + source != stage + and source not in stage.parents + and stage not in source.parents, + f"stage-dir overlaps input: {source}", + ) + if stage.exists(): + require( + not any(p.is_symlink() for p in stage.rglob("*")), "symlink in stage-dir" + ) + + +def validate_keys(frame, reference, model: str) -> None: + """Row count alone cannot detect substituted or missing output keys.""" + require(set(frame.model) == {model}, f"unexpected model rows for {model}") + require(not frame.duplicated(KEY).any(), f"duplicate prediction keys: {model}") + expected = set(reference[KEY].itertuples(index=False, name=None)) + actual = set(frame[KEY].itertuples(index=False, name=None)) + require(actual == expected, f"prediction keys differ from reference: {model}") + + +def resolve_base(args): + """Pin the live payload and its committed predictions/reference bundle.""" + import pandas as pd + + # Pandas 3 infers Arrow strings and expands repeated full raw responses + # into large buffers. Object strings retain the CSV parser's deduplication, + # as in the Pandas 2 environment used for the previous release. + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + pointer = json.loads((ROOT / "app/src/data.artifact.json").read_text()) + require( + pointer["tag"] == BASE_TAG and pointer["sha256"] == BASE_SHA256, + "base pointer changed; review the base before staging", + ) + # The 22c exclusions, plus the reviewed revision's new ones and the audit's + # listed ones, and nothing else. + check_exclusions(reference_revision(), SNAPSHOT) + opener = gzip.open if args.base_payload.suffix == ".gz" else open + with opener(args.base_payload, "rb") as stream: + raw = stream.read() + live = json.loads(raw) + if "countries" not in live: + # The compact snapshot freezes the country payload, not its wrapper. + live = {"countries": {"us": live}} + raw = json.dumps(live).encode() + require( + hashlib.sha256(raw).hexdigest() == BASE_SHA256, "base payload SHA256 mismatch" + ) + require( + len(live["countries"]["us"]["modelStats"]) == 42, "base must have 42 models" + ) + + # An explicit alternative is allowed only if it has identical CSV bytes. + def csv_digest(path): + opener = gzip.open if path.suffix == ".gz" else open + h = hashlib.sha256() + with opener(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + manifest = json.loads((ROOT / "paper/snapshot/20260501/manifest.json").read_text()) + pin = manifest["source_run_artifacts"][RUN_NAME]["files"]["predictions.csv.gz"] + require( + digest(SNAPSHOT / "predictions.csv.gz") == pin, + "committed base predictions fail their manifest hash", + ) + if args.base_predictions.resolve() != (SNAPSHOT / "predictions.csv.gz").resolve(): + require( + csv_digest(args.base_predictions) + == csv_digest(SNAPSHOT / "predictions.csv.gz"), + "base predictions differ from committed September 22c snapshot", + ) + print("Reading pinned base predictions", flush=True) + # Rehearsals do not need repeated raw provider transcripts for incumbents. + # Their full original CSV remains hash-pinned and untouched in the snapshot. + usecols = (lambda column: column != "raw_response") if args.partial else None + base = pd.read_csv(args.base_predictions, low_memory=True, usecols=usecols) + reference = pd.read_csv(SNAPSHOT / "reference_outputs.csv") + require( + len(reference) == 1984 and reference.scenario_id.nunique() == 100, + "unexpected reference universe", + ) + require(base.model.nunique() == 42, "base predictions must have 42 models") + for model, frame in base.groupby("model"): + validate_keys(frame, reference, model) + return base, reference, live + + +def base_commit_blob(path: Path) -> bytes: + """A file as committed at BASE_COMMIT, the September 22c release commit. + + The checkout must hold that commit: a shallow clone (CI's default) does + not, so the CI test job checks out full history. + """ + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{BASE_COMMIT}:{path.as_posix()}"], + capture_output=True, + ) + require( + result.returncode == 0, + f"cannot read {path} at base commit {BASE_COMMIT[:12]}; fetch full " + f"history (git fetch --unshallow): {result.stderr.decode().strip()}", + ) + return result.stdout + + +def resolve_live_base(args) -> dict: + """The September 22c payload an export compares the incumbents against. + + Before the freeze the live pointer and the committed snapshot are 22c's, + and resolve_base checks all of them. After the freeze they are this + release's own, so a re-export (after a fix to the staged annotations) + reads the 22c run payload from BASE_COMMIT instead and checks that it + rewraps to the 22c release asset (BASE_SHA256). Any other pointer is + refused, as before. + """ + pointer = json.loads((ROOT / "app/src/data.artifact.json").read_text()) + if pointer["tag"] == BASE_TAG: + return resolve_base(args)[2] + require( + pointer["tag"] == RELEASE_TAG, + "base pointer changed; review the base before staging", + ) + blob = base_commit_blob((SNAPSHOT / "data.json.gz").relative_to(ROOT)) + live = {"countries": {"us": json.loads(gzip.decompress(blob))}} + require( + hashlib.sha256(json.dumps(live).encode()).hexdigest() == BASE_SHA256, + "base payload SHA256 mismatch", + ) + require( + len(live["countries"]["us"]["modelStats"]) == 42, "base must have 42 models" + ) + check_exclusions(reference_revision(), SNAPSHOT) + return live + + +def prepare_inputs(args, runs, base, reference) -> tuple[Path, dict]: + """Copy immutable inputs and fold full or explicitly partial cohorts.""" + import pandas as pd + + from policybench.config import MODELS as registry + from policybench.config import PRICE_OVERRIDES_PER_1M + from policybench.fold_board import fold_board + + stage = args.stage_dir + frames = {} + provenance = {} + for run in runs: + frame = pd.read_csv(run.predictions, low_memory=False) + if args.partial: + frame = frame.drop(columns=["raw_response"], errors="ignore") + households = set(frame.scenario_id) + require( + len(households) == run.state["total"], + f"state/CSV household mismatch: {run.model}", + ) + require( + households <= set(reference.scenario_id), f"unknown households: {run.model}" + ) + validate_keys( + frame, reference[reference.scenario_id.isin(households)], run.model + ) + require( + run.model in registry and run.model in PRICE_OVERRIDES_PER_1M, + f"missing model registration/pricing: {run.model}", + ) + if not args.partial: + require( + run.state["total"] == 100 and households == set(reference.scenario_id), + f"full release requires 100 households: {run.model}", + ) + require( + not run.state.get("synthetic_partial"), + "synthetic run state is partial only", + ) + fingerprint = run.state.get("treatment_fingerprint") + require( + isinstance(fingerprint, dict) + and fingerprint.get("model_id") == registry[run.model], + f"missing/mismatched treatment fingerprint: {run.model}", + ) + import freeze_snapshot as freezer + from freeze_adds0928 import validate_treatment + + validate_treatment( + freezer, + run.state, + run.run_dir / "run_state.json", + SNAPSHOT / "scenarios.csv", + ) + for field in ( + "estimated_cost_usd", + "total_tokens", + "provider_resolved_model", + "provider_response_id", + ): + require( + field in frame and frame[field].notna().all(), + f"missing {field}: {run.model}", + ) + expected_provider = registry[run.model].split("/")[-1] + require( + set(frame.provider_resolved_model) == {expected_provider}, + f"unexpected provider-resolved model: {run.model}", + ) + if run.model == "deepseek-v4.1-flash": + require( + "provider_system_fingerprint" in frame + and frame.provider_system_fingerprint.fillna("").ne("").all(), + "DeepSeek moving alias lacks response fingerprint", + ) + target = stage / "inputs" / run.slug + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(run.predictions, target / "predictions.csv") + shutil.copyfile(run.run_dir / "run_state.json", target / "run_state.json") + frames[run.model] = frame + provenance[run.model] = { + "households": len(households), + "rows": len(frame), + "predictions_sha256": digest(target / "predictions.csv"), + "treatment_fingerprint": run.state.get("treatment_fingerprint"), + "provider_resolved_models": sorted( + frame.provider_resolved_model.dropna().unique() + ), + "provider_system_fingerprints": sorted( + frame.get("provider_system_fingerprint", pd.Series(dtype=str)) + .dropna() + .unique() + ), + } + shared = set.intersection(*(set(frame.scenario_id) for frame in frames.values())) + require(bool(shared), "no households completed by every addition") + selected = ( + reference[reference.scenario_id.isin(shared)] if args.partial else reference + ) + scoring = stage / "scoring" + scoring.mkdir(exist_ok=True) + for name in REFERENCE_FILES: + shutil.copyfile(SNAPSHOT / name, scoring / name) + if args.partial: + selected.to_csv(scoring / "reference_outputs.csv", index=False) + meta = json.loads((scoring / "reference_outputs.csv.meta.json").read_text()) + meta.update( + reference_csv_sha256=digest(scoring / "reference_outputs.csv"), + row_count=len(selected), + partial=True, + ) + write_json(scoring / "reference_outputs.csv.meta.json", meta) + scenarios = pd.read_csv(scoring / "scenarios.csv") + scenarios[scenarios.scenario_id.isin(shared)].to_csv( + scoring / "scenarios.csv", index=False + ) + exclusions = json.loads((scoring / "reference_exclusions.json").read_text()) + exclusions["exclusions"] = [ + e for e in exclusions["exclusions"] if e["scenario_id"] in shared + ] + write_json(scoring / "reference_exclusions.json", exclusions) + base = base[base.scenario_id.isin(shared)] + base_path = stage / "base-predictions.csv" + base.to_csv(base_path, index=False) + additions = [] + for run in runs: + path = stage / "inputs" / run.slug / "fold.csv" + frames[run.model][frames[run.model].scenario_id.isin(shared)].to_csv( + path, index=False + ) + additions.append(path) + result = fold_board(base_path, additions, scoring, stage / "fold", export=False) + require( + not result["excluded"] and result["models"] == 45, + f"fold refused additions: {result}", + ) + bundle = stage / "publish" / RUN_NAME + us = bundle / "us" + us.mkdir(parents=True, exist_ok=True) + require( + not (us / "by_model").exists(), "by_model would shadow combined predictions" + ) + shutil.copyfile(stage / "fold/us/predictions.csv", us / "predictions.csv") + for name in REFERENCE_FILES: + shutil.copyfile(scoring / name, us / name) + annotations = bundle / "annotations" + annotations.mkdir(exist_ok=True) + for name in ANNOTATION_FILES: + # Re-preparing must not overwrite a recorded stage adjudication. + if not (annotations / name).exists(): + shutil.copyfile(ANNOTATIONS / name, annotations / name) + write_json(stage / "model-provenance.json", provenance) + return bundle, frames + + +def partial_scores(args, base, reference, frames) -> None: + """Rank each addition against incumbents on its own completed households.""" + import pandas as pd + + from policybench.analysis import analyze_no_tools + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + scored, _ = split_reference(reference, load_reference_exclusions(SNAPSHOT)) + scenarios = pd.read_csv(SNAPSHOT / "scenarios.csv") + results = [] + for model, frame in frames.items(): + ids = set(frame.scenario_id) + cohort = pd.concat([base[base.scenario_id.isin(ids)], frame], ignore_index=True) + analysis = analyze_no_tools( + scored[scored.scenario_id.isin(ids)], + cohort, + scenarios=scenarios[scenarios.scenario_id.isin(ids)], + ) + stats = analysis["bounded_summary"].set_index("model") + exact = float(stats.loc[model, "weighted_exact"]) + rank = 1 + int((stats.weighted_exact > exact).sum()) + result = { + "label": "PARTIAL", + "model": model, + "households": len(ids), + "scored_outputs": int(scored.scenario_id.isin(ids).sum()), + "exact_percent": exact * 100, + "rank": rank, + "models": len(stats), + } + results.append(result) + print(json.dumps(result)) + write_json(args.stage_dir / "partial-scores.json", results) + + +def prepare_cases(args, bundle) -> None: + """Seed a private audit; changed prompts invalidate copied verdicts.""" + import pandas as pd + + from policybench.audit import prepare_audit + + require( + args.audit_seed is not None and args.grounding is not None, + "prepare needs --audit-seed and --grounding (read-only sources)", + ) + audit = args.stage_dir / "audit" + if not audit.exists(): + for source in sorted((args.audit_seed / "cases").glob("*")): + if not (source / "prompt.md").is_file(): + continue + target = audit / "cases" / source.name + target.mkdir(parents=True) + for name in ("prompt.md", "verdict.json", "verdict.meta.json", "codex.log"): + if (source / name).is_file(): + shutil.copyfile(source / name, target / name) + grounding = pd.read_csv(args.grounding) + lookup = { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + pending = validate_verdicts(audit, remove_invalid=True) + write_json(args.stage_dir / "pending.json", pending) + print(f"Prepared audit: {len(pending)} cases need Opus 5.5") + + +def validate_verdicts(audit: Path, remove_invalid: bool = False) -> list[str]: + """Validate full schema and exact model coverage, not just two JSON keys.""" + import jsonschema + + schema = json.loads((audit / "schema.json").read_text()) + manifest = { + item["case_id"]: item + for item in map(json.loads, (audit / "cases.jsonl").read_text().splitlines()) + } + pending = [] + for case_id, item in manifest.items(): + if item["parse_failure_only"]: + continue + path = audit / "cases" / case_id / "verdict.json" + try: + verdict = json.loads(path.read_text()) + jsonschema.validate(verdict, schema) + names = [m["model"] for m in verdict["models"]] + if len(names) != len(set(names)) or set(names) != set(item["wrong_models"]): + raise ValueError("wrong model coverage") + if set(names) & set(MODELS.values()): + meta = json.loads(path.with_name("verdict.meta.json").read_text()) + if ( + meta.get("verdict_sha256") != digest(path) + or meta.get("judge_model_requested") != JUDGE_MODEL + or meta.get("judge_model_reported") != [JUDGE_MODEL] + or not meta.get("judge_runner") + or not meta.get("judged_at_utc") + ): + raise ValueError("missing or mismatched Opus 5.5 provenance") + except (OSError, ValueError, jsonschema.ValidationError): + pending.append(case_id) + if remove_invalid: + path.unlink(missing_ok=True) + path.with_name("verdict.meta.json").unlink(missing_ok=True) + return sorted(pending) + + +def judge(args, bundle) -> None: + """Run one Claude CLI judge at a time and retry missing/hedged cases.""" + from policybench.audit import collect_audit + + audit = args.stage_dir / "audit" + for _ in range(3): + validate_verdicts(audit, remove_invalid=True) + subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_claude.sh"), str(audit)], + cwd=ROOT, + env={ + **os.environ, + "AUDIT_MODEL": JUDGE_MODEL, + "AUDIT_PARALLEL": "1", + "AUDIT_PYTHON": sys.executable, + }, + check=True, + ) + pending = validate_verdicts(audit, remove_invalid=True) + out = collect_audit(bundle / "us", audit) + for case_id in out["hedged"].case_id: + for name in ("verdict.json", "verdict.meta.json"): + (audit / "cases" / case_id / name).unlink(missing_ok=True) + if not pending and out["missing"].empty and out["hedged"].empty: + return + raise SystemExit( + "audit missing/hedged after three passes; resume judge after investigation" + ) + + +def triage(args, bundle) -> None: + """Collect, apply recorded decisions, and stop for unresolved flags.""" + from policybench.adjudications import ( + apply_adjudications, + excluded_case_keys, + load_adjudications, + verify_adjudications_applied, + ) + from policybench.audit import collect_audit + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + audit = args.stage_dir / "audit" + require(not validate_verdicts(audit), "missing or invalid verdicts; run judge") + out = collect_audit(bundle / "us", audit) + require( + out["missing"].empty and out["hedged"].empty, "missing/hedged audit; run judge" + ) + rows = out["row"] + cases = out["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ) + counts = rows.groupby(KEY).size() + require( + all( + int(r.wrong_model_count) == counts[(r.scenario_id, r.variable)] + for r in cases.itertuples() + ), + "case wrong_model_count disagrees with collected rows", + ) + annotations = bundle / "annotations" + decisions = load_adjudications(annotations / "us_adjudications.json") + from freeze_snapshot import verify_adjudications_keep_judge_verdicts + + verify_adjudications_keep_judge_verdicts(decisions, audit / "cases") + rows, cases, _ = apply_adjudications(rows, cases, decisions) + verify_adjudications_applied(rows, cases, decisions) + excluded = exclusion_keys(load_reference_exclusions(bundle / "us")) + require( + excluded == excluded_case_keys(decisions), + "adjudication exclusions differ from frozen scoring exclusions", + ) + flags = cases[cases.reference_suspect.astype(bool)] + scored = ~rows[KEY].apply(tuple, axis=1).isin(excluded) + unresolved = rows[ + scored + & ~rows.failure_source.isin( + [ + "llm_error", + "parse_contract_failure", + "budget_exhausted_at_ceiling", + ] + ) + ] + flags.to_csv(args.stage_dir / "reference-flags.csv", index=False) + unresolved.to_csv(args.stage_dir / "unresolved-rows.csv", index=False) + rows.to_csv(annotations / "us_audit_row_annotations.csv", index=False) + cases.to_csv(annotations / "us_case_notes.csv", index=False) + require( + flags.empty and unresolved.empty, + "triage required: inspect reference-flags.csv/unresolved-rows.csv; " + "record evidence in staged us_adjudications.json and rerun triage", + ) + print(f"Triage complete: {len(rows)} rows / {len(cases)} cases") + + +def base_reference_bytes(name: str) -> bytes: + """A September 22c reference file, read from git and checked against its pin.""" + raw = base_commit_blob(SNAPSHOT.relative_to(ROOT) / name) + require( + hashlib.sha256(raw).hexdigest() == BASE_REFERENCE_SHA256[name], + f"base reference {name} does not match its pin", + ) + return raw + + +def reference_revision() -> dict | None: + """The reviewed reference revision the snapshot carries, or None if unchanged. + + The snapshot may differ from the September 22c references only by one + committed engine_upgrade revision whose `changed` list is exactly the set of + outputs whose value differs, with those values. Every other column + (impact_weight among them) must equal the base's on every row. Anything + else is refused. The exclusion record is gated by check_exclusions. + """ + import io + + import pandas as pd + + if all( + digest(SNAPSHOT / name) == sha for name, sha in BASE_REFERENCE_SHA256.items() + ): + return None + meta = json.loads((SNAPSHOT / "reference_outputs.csv.meta.json").read_text()) + require( + meta.get("reference_csv_sha256") == digest(SNAPSHOT / "reference_outputs.csv"), + "reference sidecar does not pin the committed reference CSV", + ) + revision = meta["revisions"][-1] + require( + revision.get("kind") == "engine_upgrade", + "changed references need a committed engine_upgrade revision", + ) + base = pd.read_csv( + io.BytesIO(base_reference_bytes("reference_outputs.csv")) + ).set_index(KEY) + new = pd.read_csv(SNAPSHOT / "reference_outputs.csv").set_index(KEY) + require(base.index.equals(new.index), "reference output keys changed") + require( + list(base.columns) == list(new.columns), + f"reference columns changed: {list(base.columns)} -> {list(new.columns)}", + ) + + def differs(column: str) -> list: + before, after = base[column], new[column] + same = (before == after) | (before.isna() & after.isna()) + return list(base.index[~same]) + + for column in base.columns: + if column != "value": + unlisted = differs(column) + require( + not unlisted, + f"reference {column} changed outside the revision: {unlisted[:5]}", + ) + diff = {key: float(new.at[key, "value"]) for key in differs("value")} + listed = { + (c["scenario_id"], c["variable"]): float(c["regenerated"]) + for c in revision["changed"] + } + require( + set(diff) == set(listed) + and all(abs(diff[key] - listed[key]) < 1e-9 for key in diff), + "reference changes differ from the engine_upgrade revision's list", + ) + return revision + + +def audit_exclusion_keys() -> set[tuple[str, str]]: + """The outputs final_actions.json lists under audit_exclusions.""" + if not AUDIT_ACTIONS.exists(): + return set() + actions = json.loads(AUDIT_ACTIONS.read_text()) + return { + (item["scenario_id"], item["variable"]) + for item in actions.get("audit_exclusions", []) + } + + +def check_exclusions(revision: dict | None, directory: Path) -> set[tuple[str, str]]: + """Require the exclusion record in `directory` to be exactly the expected set. + + Expected: the September 22c exclusions (read from BASE_COMMIT), each entry + unchanged; with a reviewed revision, also the outputs it newly excludes and + the audit exclusions final_actions.json lists, and nothing else. An audit + exclusion is not an engine change, so the revision must not list it. + Returns the outputs excluded beyond 22c's. + """ + from policybench.reference_exclusions import load_reference_exclusions + + def by_key(entries): + return {(e["scenario_id"], e["variable"]): e for e in entries} + + base = by_key( + json.loads(base_reference_bytes("reference_exclusions.json"))["exclusions"] + ) + current = by_key(load_reference_exclusions(directory)) + added: set[tuple[str, str]] = set() + if revision is not None: + changed = {(c["scenario_id"], c["variable"]): c for c in revision["changed"]} + new = {k for k, c in changed.items() if c.get("cause") == NEW_EXCLUSION_CAUSE} + audit = audit_exclusion_keys() + require( + not audit & (set(changed) | set(base)), + f"an audit exclusion is an engine change or a 22c exclusion: " + f"{sorted(audit & (set(changed) | set(base)))}", + ) + added = new | audit + expected = set(base) | added + require( + set(current) == expected, + "exclusions differ from the 22c record plus the revision's and the audit's: " + f"missing {sorted(expected - set(current))}, " + f"unexpected {sorted(set(current) - expected)}", + ) + changed_base = sorted(k for k in base if current[k] != base[k]) + require(not changed_base, f"22c exclusions changed: {changed_base}") + return added + + +def replay_base_references(args, bundle, previous) -> None: + """Export again on the September 22c references; incumbents must not move. + + With the three reference files as the only inputs put back, every + incumbent's modelStats must reproduce the live payload exactly. So any + drift in the real export comes from the reference files, which differ from + 22c only by the reviewed revision (reference_revision) and the listed + audit exclusions (check_exclusions). + """ + from policybench.full_run_export import export_full_run + + replay = args.stage_dir / "replay-20260922c" / bundle.name + if replay.parent.exists(): + shutil.rmtree(replay.parent) + shutil.copytree(bundle, replay) + for name in BASE_REFERENCE_SHA256: + (replay / "us" / name).write_bytes(base_reference_bytes(name)) + stats = export_full_run(replay, countries=["us"], skip_app_data=True)["countries"][ + "us" + ]["modelStats"] + fable = next(s for s in stats if s["model"] == "claude-fable-5") + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + fable[key] = previous["claude-fable-5"][key] + drift = [ + s["model"] + for s in stats + if s["model"] in previous and s != previous[s["model"]] + ] + require(not drift, f"incumbents drift even on the 22c references: {drift}") + + +def export(args, bundle, live) -> dict: + """Export into scratch, preserve incumbent statistics, and gate release.""" + from policybench.dashboard_schema import validate_dashboard_payload + from policybench.full_run_export import export_full_run + + payload = export_full_run(bundle, countries=["us"], skip_app_data=True) + stats = payload["countries"]["us"]["modelStats"] + require(len(stats) == 45, "export did not contain all 45 models") + previous = {m["model"]: m for m in live["countries"]["us"]["modelStats"]} + if not args.partial: + fable = next(s for s in stats if s["model"] == "claude-fable-5") + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + fable[key] = previous["claude-fable-5"][key] + changed = [ + s["model"] + for s in stats + if s["model"] in previous and s != previous[s["model"]] + ] + revision = reference_revision() + beyond_22c = check_exclusions(revision, bundle / "us") + if revision is None: + require(not changed, f"incumbent modelStats drift: {changed}") + else: + replay_base_references(args, bundle, previous) + audit = sorted(beyond_22c & audit_exclusion_keys()) + report = [ + { + "model": s["model"], + "exact_20260922c": previous[s["model"]]["exact"], + "exact": s["exact"], + "score_20260922c": previous[s["model"]]["score"], + "score": s["score"], + } + for s in stats + if s["model"] in previous + ] + write_json( + args.stage_dir / "incumbent-drift.json", + { + "causes": ( + "Re-exported on the September 22c references, every " + "incumbent reproduces its 22c modelStats. The references " + "differ from 22c by the reviewed reference revision " + f"({revision['root_cause']}, {len(revision['changed'])} " + f"changed outputs) and {len(audit)} audit exclusion(s) " + "listed in reference_audit/2026-09-28/final_actions.json, " + "and by nothing else; those are the causes of the drift." + ), + "reference_revision": revision["root_cause"], + "reference_changes": len(revision["changed"]), + "audit_exclusions": [ + {"scenario_id": s, "variable": v} for s, v in audit + ], + "models": report, + }, + ) + errors = validate_dashboard_payload( + payload, require_failure_annotations=not args.early + ) + require(not errors, f"payload validation failed: {errors[:8]}") + if args.partial: + payload["stage2Status"] = ( + "PARTIAL — incomplete household cohorts, not a release" + ) + path = args.stage_dir / ( + "PARTIAL-data-board45.json" if args.partial else "data-board45.json" + ) + # freeze_snapshot reassembles these exact default-json bytes from the + # compact country payload. Pretty-printing here breaks its release hash. + path.write_text(json.dumps(payload, allow_nan=False)) + # Keep the freeze input identical to the gated payload, including Fable usage. + shutil.copyfile(path, bundle / "data.json") + ranked = sorted(stats, key=lambda s: (-s["exact"], -s["score"], s["model"])) + for index, row in enumerate(ranked, 1): + if row["model"] in MODELS.values(): + print( + f"{'PARTIAL common cohort' if args.partial else 'STAGED'} " + f"{row['model']}: exact={row['exact']:.6f}% " + f"rank={index}/45 n={row['n']}" + ) + if not args.early: + pinned = [p for p in (bundle / "annotations").iterdir() if p.is_file()] + pinned += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + pinned += [p for p in (args.stage_dir / "inputs").rglob("*") if p.is_file()] + pinned += [ + p + for p in (args.stage_dir / "audit").rglob("*") + if p.is_file() + and p.name + in { + "verdict.json", + "verdict.meta.json", + "prompt.md", + "cases.jsonl", + "schema.json", + } + ] + write_json( + args.stage_dir / "release-ready.json", + { + "release_tag": RELEASE_TAG, + "payload_sha256": digest(path), + "models": 45, + "partial": False, + "files": { + str(p.relative_to(args.stage_dir)): digest(p) for p in pinned + }, + }, + ) + return payload + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs-root", type=Path) + parser.add_argument("--stage-dir", type=Path, required=True) + parser.add_argument( + "--base-predictions", type=Path, default=SNAPSHOT / "predictions.csv.gz" + ) + parser.add_argument("--base-payload", type=Path, default=SNAPSHOT / "data.json.gz") + parser.add_argument("--audit-seed", type=Path) + parser.add_argument("--grounding", type=Path) + parser.add_argument( + "--early", action="store_true", help="fold and export without paid judging" + ) + parser.add_argument( + "--partial", + action="store_true", + help="scratch-only cohort rehearsal; requires --early", + ) + parser.add_argument( + "--step", choices=("prepare", "judge", "triage", "export"), default="prepare" + ) + args = parser.parse_args(argv) + if args.partial and not args.early: + parser.error("--partial requires --early") + if args.early and args.step != "prepare": + parser.error("--early only applies to prepare") + if args.step == "prepare" and not args.runs_root: + parser.error("prepare requires --runs-root") + args.stage_dir = args.stage_dir.resolve() + return args + + +def main(argv=None) -> None: + args = parse_args(argv) + sources = [SNAPSHOT, ANNOTATIONS, args.base_predictions, args.base_payload] + sources += [ + p for p in (args.runs_root, args.audit_seed, args.grounding) if p is not None + ] + validate_stage_path(args.stage_dir, sources) + stage = args.stage_dir + bundle = stage / "publish" / RUN_NAME + receipt_path = stage / "stage.json" + if args.step == "prepare": + runs = discover_new_models(args.runs_root) + print("All copied run states qualify; resolving September 22c base", flush=True) + base, reference, live = resolve_base(args) + print("Base verified; folding copied additions", flush=True) + stage.mkdir(parents=True, exist_ok=True) + require( + not receipt_path.exists(), + "stage already prepared; use a fresh stage-dir " + "or resume judge/triage/export", + ) + bundle, frames = prepare_inputs(args, runs, base, reference) + if args.partial: + partial_scores(args, base, reference, frames) + if args.early: + export(args, bundle, live) + else: + prepare_cases(args, bundle) + inputs = [p for p in (stage / "inputs").rglob("*") if p.is_file()] + inputs += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + write_json( + receipt_path, + { + "partial": args.partial, + "early": args.early, + "base_tag": BASE_TAG, + "files": {str(p.relative_to(stage)): digest(p) for p in inputs}, + }, + ) + else: + receipt = json.loads(receipt_path.read_text()) + require( + not receipt["partial"] and not receipt["early"], + "early/partial stage cannot become a release; prepare fresh full inputs", + ) + for name, expected in receipt["files"].items(): + require( + digest(stage / name) == expected, + f"staged input changed: {name}; prepare a new stage", + ) + (stage / "release-ready.json").unlink(missing_ok=True) + if args.step == "judge": + judge(args, bundle) + elif args.step == "triage": + triage(args, bundle) + else: + triage(args, bundle) + export(args, bundle, resolve_live_base(args)) + + +if __name__ == "__main__": + main() diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py new file mode 100644 index 00000000..11551e56 --- /dev/null +++ b/scripts/finish_gpt61sol.py @@ -0,0 +1,2191 @@ +"""Stage the GPT-6.1 Sol addition; never publish or change tracked data. + +Adapted from finish_adds0928.py. It folds one supervised run onto the 45-model +board of release dashboard-data-20260929 without any reference revision: the +five reference files are pinned, and every incumbent's modelStats must come out +byte-identical. All outputs, including audit verdicts and adjudications, stay +in --stage-dir. See docs/gpt61sol/design.md for the run and release procedure. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +os.environ["OPENBLAS_NUM_THREADS"] = "1" +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +SNAPSHOT = ROOT / "paper/snapshot/20260501/runs" / RUN_NAME +ANNOTATIONS = ROOT / "annotations" / RUN_NAME +BASE_TAG = "dashboard-data-20260929" +# The one place the new release's tag is named. The lead may change it at +# freeze time; freeze_gpt61sol.py and the design note read it from here. +RELEASE_TAG = "dashboard-data-20260930" +BASE_SHA256 = "a5cb9989d78cb18d040fec2f1f5d0775df15b9b917ae99d883d8701b7fa480a7" +BASE_MODELS = 45 +BASE_OUTPUTS = 1984 +BASE_EXCLUSIONS = 56 +BASE_SCORED = BASE_OUTPUTS - BASE_EXCLUSIONS +MODELS = {"gpt61sol": "gpt-6.1-sol"} +BOARD_MODELS = BASE_MODELS + len(MODELS) +REFERENCE_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", + "us_adjudications.json", +) +JUDGE_MODEL = "claude-opus-5-5" +ADJUDICATIONS = "us_adjudications.json" +# Stage files export binds: the cases GPT-6.1 Sol re-opened, and the +# wording-only amendments a developer lists for them. +PROMPT_CHANGES = "prompt-changes.json" +AMENDMENTS = "wording-amendments.json" +# The published record of how each new verdict (a case GPT-6.1 Sol re-opened) +# was judged. Export requires it to describe the staged verdicts and binds its +# bytes in the receipt. +JUDGE_PROVENANCE_PATH = "docs/gpt61sol/judge_provenance.json" +JUDGE_PROVENANCE = ROOT / JUDGE_PROVENANCE_PATH +# The record's keys, and each entry's: every one is checked, against the +# case's verdict, prompt or sidecar, the entries' tally, or the isolated flag. +JUDGE_PROVENANCE_KEYS = frozenset({"note", "counts", "verdicts"}) +JUDGE_PROVENANCE_ENTRY_KEYS = frozenset( + { + "case_id", + "group", + "isolated", + "judge_account_declared", + "judge_effort", + "judge_model_reported", + "judged_at_utc", + "prompt_sha256", + "verdict_sha256", + } +) +# The entry fields copied from the verdict's sidecar, an e-mail address in any +# of them withheld as WITHHELD_ADDRESS: the record is public. +JUDGE_SIDECAR_FIELDS = ( + "judge_effort", + "judge_model_reported", + "judged_at_utc", + "judge_account_declared", +) +WITHHELD_ADDRESS = "" +# The context attachments an isolated judge's transcript may carry: ATTACHMENTS +# in scripts/run_audit_claude.sh, which a test keeps equal to this set. +JUDGE_ATTACHMENTS = frozenset( + { + "environment", + "model", + "date", + "session_context", + "total_tokens_reminder", + "prompt_snapshot", + "structured_output", + "silent_turn_reminder", + } +) +# The event types an isolated judge's transcript may carry, the types the +# stage's 60 isolated transcripts carry: EVENT_TYPES in +# scripts/run_audit_claude.sh, which a test keeps equal to this set. +JUDGE_EVENT_TYPES = frozenset( + { + "queue-operation", + "user", + "attachment", + "atis-latch", + "last-prompt", + "assistant", + "cost-state", + } +) +# Account data an isolated judge's transcript may not carry outside the judged +# prompt's own user message and the judge's own words: an e-mail address in any +# key or string, or a key that names an account, at any depth. The runner's +# extract_verdict uses the same two patterns. +ACCOUNT_KEY = re.compile(r"email|account|credential|organi[sz]ation|gitstatus", re.I) +EMAIL_ADDRESS = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") +# Claude Code's own nudge to a judge that answered in text, the one user text +# message an isolated judge's transcript may carry besides its prompt: +# PROMPT_NUDGE in scripts/run_audit_claude.sh, which a test keeps equal to this. +JUDGE_PROMPT_NUDGE = ( + "[structured-output-enforce] You MUST call the StructuredOutput tool to " + "complete this request. Call this tool now." +) +# The merge of PR #182 on main, whose tree holds release 20260929. +BASE_COMMIT = "d616e67c33b6f80dabf5cb7329f069f9a1de069d" +# Release 20260929's references. There is no reference revision in this +# release: the committed and staged copies must equal these bytes throughout. +BASE_REFERENCE_SHA256 = { + "reference_outputs.csv": ( + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" + ), + "reference_outputs.csv.meta.json": ( + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" + ), + "reference_exclusions.json": ( + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" + ), + "scenarios.csv": ( + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a" + ), + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +# The grounding the 20260929 stage (adds0928-v3) rendered into its prompts: +# results/local/unified_audit/grounding.csv in the main clone. Every seed +# prompt re-renders byte-identically from it and the committed snapshot, so a +# different grounding would silently invalidate carried-over verdicts. +GROUNDING_SHA256 = "b1e4a9bc74d762f410524a147efcda7d705c3afcfa3dc27f720fa60c54a7b55c" +# The seed: every judged case of the 20260929 audit, with the sha256 of its +# prompt and of its verdict, committed at docs/gpt61sol/seed_digest.csv. This +# pins that file's bytes; prepare refuses any other seed and binds this one in +# stage.json, and a carried-over verdict must keep the seed's bytes. +SEED_DIGEST = ROOT / "docs/gpt61sol/seed_digest.csv" +SEED_DIGEST_SHA256 = "a94e96970113c4fb38dad9ec5a00fb430f026370a6b63ccffeefaae44436fe3d" +# GPT-6.1 Sol's supervised run as it finished, pinned outside the stage: the +# sha256 of the run directory's predictions.csv and run_state.json, which +# --step pin-inputs writes here. Export and the freeze read the file as +# committed at HEAD, and the stage's inputs// copies must be those bytes. +INPUT_PINS_PATH = "docs/gpt61sol/input_pins.json" +INPUT_PINS = ROOT / INPUT_PINS_PATH +PINNED_INPUTS = ("predictions.csv", "run_state.json") +# Incumbent usage the exporter cannot recompute from committed predictions: +# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, +# token or latency fields, so export_full_run reports $0 and omits the rest. +# The released values are carried over; the drift gate then checks every field. +CARRIED_USAGE = { + "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") +} +KEY = ["scenario_id", "variable"] + + +def require(condition: bool, message: str) -> None: + """Fail before an unsafe or incomplete stage can proceed.""" + if not condition: + raise SystemExit(message) + + +def digest(path: Path) -> str: + """Hash a file without retaining its contents in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def write_json(path: Path, value: object) -> None: + """Write a deterministic, finite JSON artifact.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + + +def verify_reference_pins(directory: Path, label: str) -> None: + """Every reference file in ``directory`` must equal its 20260929 pin.""" + for name, pin in BASE_REFERENCE_SHA256.items(): + path = directory / name + require( + path.is_file() and digest(path) == pin, + f"{label} {name} does not match its pin; this release has no " + "reference revision", + ) + + +@dataclass(frozen=True) +class NewRun: + slug: str + model: str + run_dir: Path + predictions: Path + state: dict + + +def discover_new_models(runs_root: Path) -> list[NewRun]: + """Require every named run to be completed and unstopped, as before.""" + found = [] + skipped = [] + for slug, expected in MODELS.items(): + directory = runs_root / slug / "run" + state_path = directory / "run_state.json" + if not state_path.is_file(): + skipped.append(f"{slug}: no run_state.json") + continue + state = json.loads(state_path.read_text()) + completed, total = state.get("completed"), state.get("total") + reason = state.get("stopped_reason") + predictions = directory / "predictions.csv" + if ( + type(total) is not int + or type(completed) is not int + or total <= 0 + or completed != total + or reason is not None + ): + skipped.append(f"{slug}: {completed}/{total}, stopped_reason={reason!r}") + elif state.get("model") != expected: + skipped.append(f"{slug}: unexpected model {state.get('model')!r}") + elif not predictions.is_file(): + skipped.append(f"{slug}: no predictions.csv") + else: + found.append(NewRun(slug, expected, directory, predictions, state)) + require(not skipped, "refusing incomplete additions: " + "; ".join(skipped)) + return found + + +def input_pins_record(runs_root: Path) -> dict: + """The record --step pin-inputs writes: the sha256 of each new model's + finished run files, read from the run directory under ``runs_root``.""" + return { + "note": ( + "The sha256 of GPT-6.1 Sol's supervised run files as the run " + "finished, written by scripts/finish_gpt61sol.py --step pin-inputs " + "from the run directory. prepare copies them into the stage's " + "inputs//; export and the freeze refuse a staged copy that " + "is not these bytes, and read this file as committed at HEAD." + ), + "inputs": { + run.slug: { + "run": f"{runs_root.name}/{run.slug}/run", + "sha256": {name: digest(run.run_dir / name) for name in PINNED_INPUTS}, + } + for run in discover_new_models(runs_root) + }, + } + + +def pin_inputs(args) -> None: + """Write INPUT_PINS from the finished run; commit it before export.""" + write_json(INPUT_PINS, input_pins_record(args.runs_root)) + print(f"Pinned {', '.join(PINNED_INPUTS)} of {sorted(MODELS)} in {INPUT_PINS_PATH}") + + +def head_blob(path: str) -> bytes: + """A repository file as committed at HEAD, never the working tree's copy, + which could be edited together with the stage it pins.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"HEAD:{path}"], capture_output=True + ) + require( + result.returncode == 0, + f"{path} is not committed at HEAD; commit it: {result.stderr.decode().strip()}", + ) + return result.stdout + + +def committed_input_pins() -> dict[str, dict[str, str]]: + """Each new model's pinned run-file sha256, as committed at HEAD. + + The working-tree file must be HEAD's too, so pins written and not yet + committed are refused rather than silently ignored. + """ + blob = head_blob(INPUT_PINS_PATH) + require( + INPUT_PINS.is_file() and INPUT_PINS.read_bytes() == blob, + f"{INPUT_PINS_PATH} differs from its HEAD commit; commit it", + ) + record = json.loads(blob) + inputs = record.get("inputs") if isinstance(record, dict) else None + require( + isinstance(inputs, dict) + and set(inputs) == set(MODELS) + and all( + isinstance(item, dict) + and isinstance(item.get("sha256"), dict) + and set(item["sha256"]) == set(PINNED_INPUTS) + for item in inputs.values() + ), + f"{INPUT_PINS_PATH} does not pin {list(PINNED_INPUTS)} for {sorted(MODELS)}", + ) + return {slug: item["sha256"] for slug, item in inputs.items()} + + +def verify_input_pins(stage: Path) -> None: + """The stage's copies of each new model's run files are the pinned bytes.""" + for slug, pins in committed_input_pins().items(): + for name, pin in pins.items(): + path = stage / "inputs" / slug / name + require( + path.is_file() and digest(path) == pin, + f"staged inputs/{slug}/{name} is not the run file " + f"{INPUT_PINS_PATH} pins; prepare a new stage from the pinned run", + ) + + +def verify_prepared_inputs(stage: Path) -> None: + """The prepare-time hashes in stage.json and model-provenance.json hold. + + stage.json records every file prepare copied or folded (each new model's + run files, the bundle's predictions and references); model-provenance.json + records each new model's predictions sha256 and treatment fingerprint. + """ + receipt = json.loads((stage / "stage.json").read_text()) + files = receipt.get("files") if isinstance(receipt, dict) else None + require(isinstance(files, dict), "stage.json records no prepared files") + us = Path("publish") / RUN_NAME / "us" + expected = {str(us / name) for name in (*REFERENCE_FILES, "predictions.csv")} + expected |= { + str(Path("inputs") / slug / name) for slug in MODELS for name in PINNED_INPUTS + } + require( + expected <= set(files), + f"stage.json does not record {sorted(expected - set(files))}", + ) + for name, pin in files.items(): + path = (stage / name).resolve() + require( + path.is_relative_to(stage.resolve()) + and path.is_file() + and digest(path) == pin, + f"staged input changed since prepare: {name}", + ) + provenance = json.loads((stage / "model-provenance.json").read_text()) + require( + isinstance(provenance, dict) and set(provenance) == set(MODELS.values()), + f"model-provenance.json does not describe exactly {sorted(MODELS.values())}", + ) + for slug, model in MODELS.items(): + inputs = stage / "inputs" / slug + state = json.loads((inputs / "run_state.json").read_text()) + entry = provenance[model] + fingerprint = state.get("treatment_fingerprint") + require( + isinstance(entry, dict) + and entry.get("predictions_sha256") == digest(inputs / "predictions.csv") + and entry.get("treatment_fingerprint") == fingerprint, + f"model-provenance.json disagrees with the staged run of {model}", + ) + + +def new_model_row_differences(stage: Path, limit: int = 5) -> list[str]: + """Where a new model's rows in the bundle's predictions are not its run's. + + Every column of the staged run file (inputs//predictions.csv, which + verify_input_pins pins) is compared cell by cell, as text, keyed by + scenario and variable: the answers and also the cost, token and latency + columns the payload's costUsd, totalTokens and latencySeconds come from. + No column is exempt. prepare_inputs copies the run file, and fold_board + reads it with read_csv and writes its concat with the base by to_csv. + That round trip could rewrite a cell (an integer column the base leaves + blank would come back as a float), and such a cell is refused, never + skipped; on the real stage it changes no cell of the run's 26 columns + (its token and cost columns are already floats). A bundle column the run + lacks (a base column the concat adds) must be empty on the new model's + rows. + """ + import pandas as pd + + text = {"dtype": str, "keep_default_na": False} + us = stage / "publish" / RUN_NAME / "us" / "predictions.csv" + header = list(pd.read_csv(us, nrows=0, **text).columns) + if "model" not in header: + return [f"{us.name} has no model column"] + parts: dict[str, list] = {} + for chunk in pd.read_csv(us, chunksize=100_000, **text): + for model in MODELS.values(): + rows = chunk[chunk["model"] == model] + if len(rows): + parts.setdefault(model, []).append(rows) + found = [] + for slug, model in MODELS.items(): + source = pd.read_csv(stage / "inputs" / slug / "predictions.csv", **text) + folded = ( + pd.concat(parts[model]) if model in parts else pd.DataFrame(columns=header) + ) + missing = [column for column in source.columns if column not in header] + if missing or not set(KEY) <= set(source.columns): + found.append(f"{model}: run columns {missing or KEY} not in the bundle") + continue + run = source.sort_values(KEY, kind="stable").reset_index(drop=True) + board = folded.sort_values(KEY, kind="stable").reset_index(drop=True) + if ( + run.duplicated(KEY).any() + or len(run) != len(board) + or not run[KEY].equals(board[KEY]) + ): + found.append( + f"{model}: the bundle holds {len(board)} rows, not the run's " + f"{len(run)} scenario and variable keys" + ) + continue + for column in header: + if column in run.columns: + differ = run[column] != board[column] + what = "differs from the run" + else: + differ = board[column] != "" + what = "is filled, and the run has no such column" + if differ.any(): + first = board.loc[differ.idxmax(), KEY] + found.append( + f"{model} {column} {what} on {int(differ.sum())} rows " + f"(first {'__'.join(first)})" + ) + return found[:limit] + + +def verify_new_model_inputs(stage: Path) -> None: + """Each new model's staged inputs are its pinned run, as prepare folded it. + + Its staged run files are the bytes INPUT_PINS (committed at HEAD) records, + the prepare-time hashes in stage.json and model-provenance.json still + hold, and its rows in the bundle's predictions are its run file's, every + column, so its published cost, tokens and latency are the run's. + """ + verify_input_pins(stage) + verify_prepared_inputs(stage) + differ = new_model_row_differences(stage) + require( + not differ, + f"{sorted(MODELS.values())} rows in the bundle's predictions are not the " + f"pinned run's: {differ}; prepare a new stage", + ) + + +def validate_stage_path(stage: Path, sources: list[Path]) -> None: + """Allow only scratch outputs, disjoint from every read-only source.""" + stage = stage.resolve() + scratch = (ROOT / "results/local").resolve() + require(scratch in stage.parents, f"stage-dir must be below {scratch}") + for source in sources: + source = source.resolve() + require( + source != stage + and source not in stage.parents + and stage not in source.parents, + f"stage-dir overlaps input: {source}", + ) + if stage.exists(): + require( + not any(p.is_symlink() for p in stage.rglob("*")), "symlink in stage-dir" + ) + + +def validate_keys(frame, reference, model: str) -> None: + """Row count alone cannot detect substituted or missing output keys.""" + require(set(frame.model) == {model}, f"unexpected model rows for {model}") + require(not frame.duplicated(KEY).any(), f"duplicate prediction keys: {model}") + expected = set(reference[KEY].itertuples(index=False, name=None)) + actual = set(frame[KEY].itertuples(index=False, name=None)) + require(actual == expected, f"prediction keys differ from reference: {model}") + + +def live_pointer() -> dict: + """The committed dashboard pointer (release 20260929 until the freeze).""" + return json.loads((ROOT / "app/src/data.artifact.json").read_text()) + + +def resolve_base(args): + """Pin the live payload and its committed predictions/reference bundle.""" + import pandas as pd + + # Pandas 3 infers Arrow strings and expands repeated full raw responses + # into large buffers. Object strings retain the CSV parser's deduplication, + # as in the Pandas 2 environment used for the previous release. + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + verify_reference_pins(SNAPSHOT, "committed reference") + pointer = live_pointer() + require( + pointer["tag"] == BASE_TAG and pointer["sha256"] == BASE_SHA256, + "base pointer changed; review the base before staging", + ) + opener = gzip.open if args.base_payload.suffix == ".gz" else open + with opener(args.base_payload, "rb") as stream: + raw = stream.read() + live = json.loads(raw) + if "countries" not in live: + # The compact snapshot freezes the country payload, not its wrapper. + live = {"countries": {"us": live}} + raw = json.dumps(live).encode() + require( + hashlib.sha256(raw).hexdigest() == BASE_SHA256, "base payload SHA256 mismatch" + ) + require( + len(live["countries"]["us"]["modelStats"]) == BASE_MODELS, + f"base must have {BASE_MODELS} models", + ) + + # An explicit alternative is allowed only if it has identical CSV bytes. + def csv_digest(path): + opener = gzip.open if path.suffix == ".gz" else open + h = hashlib.sha256() + with opener(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + verify_base_predictions(SNAPSHOT) + if args.base_predictions.resolve() != (SNAPSHOT / "predictions.csv.gz").resolve(): + require( + csv_digest(args.base_predictions) + == csv_digest(SNAPSHOT / "predictions.csv.gz"), + "base predictions differ from committed 20260929 snapshot", + ) + print("Reading pinned base predictions", flush=True) + # Rehearsals do not need repeated raw provider transcripts for incumbents. + # Their full original CSV remains hash-pinned and untouched in the snapshot. + usecols = (lambda column: column != "raw_response") if args.partial else None + base = pd.read_csv(args.base_predictions, low_memory=True, usecols=usecols) + reference = pd.read_csv(SNAPSHOT / "reference_outputs.csv") + require( + len(reference) == BASE_OUTPUTS and reference.scenario_id.nunique() == 100, + "unexpected reference universe", + ) + require( + base.model.nunique() == BASE_MODELS, + f"base predictions must have {BASE_MODELS} models", + ) + for model, frame in base.groupby("model"): + validate_keys(frame, reference, model) + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + exclusions = load_reference_exclusions(SNAPSHOT) + require( + len(exclusions) == BASE_EXCLUSIONS, f"expected {BASE_EXCLUSIONS} exclusions" + ) + require( + len(split_reference(reference, exclusions)[0]) == BASE_SCORED, + f"expected {BASE_SCORED} scored outputs", + ) + return base, reference, live + + +def base_commit_blob(path: Path) -> bytes: + """A file as committed at BASE_COMMIT, whose tree holds release 20260929. + + The checkout must hold that commit: a shallow clone (CI's default) does + not, so the CI test job checks out full history. + """ + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{BASE_COMMIT}:{path.as_posix()}"], + capture_output=True, + ) + require( + result.returncode == 0, + f"cannot read {path} at base commit {BASE_COMMIT[:12]}; fetch full " + f"history (git fetch --unshallow): {result.stderr.decode().strip()}", + ) + return result.stdout + + +def verify_base_predictions(snapshot: Path) -> None: + """The snapshot's predictions are release 20260929's. + + The pin is that release's own: the snapshot manifest as committed at + BASE_COMMIT, not the working tree's, which could be edited together with + the predictions it pins. + """ + manifest = json.loads( + base_commit_blob(Path("paper/snapshot/20260501/manifest.json")) + ) + pin = manifest["source_run_artifacts"][RUN_NAME]["files"]["predictions.csv.gz"] + require( + digest(snapshot / "predictions.csv.gz") == pin, + "committed base predictions fail release 20260929's manifest hash", + ) + + +def base_payload_from_commit() -> dict: + """Release 20260929's payload, read from BASE_COMMIT and checked.""" + blob = base_commit_blob((SNAPSHOT / "data.json.gz").relative_to(ROOT)) + live = {"countries": {"us": json.loads(gzip.decompress(blob))}} + require( + hashlib.sha256(json.dumps(live).encode()).hexdigest() == BASE_SHA256, + "base payload SHA256 mismatch", + ) + require( + len(live["countries"]["us"]["modelStats"]) == BASE_MODELS, + f"base must have {BASE_MODELS} models", + ) + return live + + +def base_adjudication_record() -> dict: + """Release 20260929's adjudication record file, read from BASE_COMMIT. + + The working-tree copy is not a baseline: the freeze overwrites it, and a + freeze that stops partway would leave the staged record in its place. + """ + path = Path("annotations") / RUN_NAME / ADJUDICATIONS + return json.loads(base_commit_blob(path)) + + +def base_adjudications() -> list[dict]: + """Release 20260929's adjudication entries, read from BASE_COMMIT.""" + from policybench.adjudications import parse_adjudications + + path = Path("annotations") / RUN_NAME / ADJUDICATIONS + return parse_adjudications(base_adjudication_record(), f"{BASE_COMMIT[:12]}:{path}") + + +def record_text(record: dict) -> str: + """An adjudication record as the committed file spells it.""" + return json.dumps(record, indent=2, ensure_ascii=False) + "\n" + + +def verify_record_form(text: str, base: dict) -> None: + """The staged record's bytes are exactly its parsed content, and its top + level other than the entries is release 20260929's. + + Duplicate keys, or any other bytes a parser drops, would ship in the + frozen file unseen by the entry gate; so would a rewritten note or date + convention. + """ + record = json.loads(text) + require( + text == record_text(record), + "the staged adjudication record is not in the committed form (duplicate " + "keys or other bytes the entry gate cannot see); write it with " + "json.dumps(indent=2, ensure_ascii=False)", + ) + require( + record_text({k: v for k, v in record.items() if k != "adjudications"}) + == record_text({k: v for k, v in base.items() if k != "adjudications"}), + "the staged adjudication record changes its note, schema or date " + "conventions; only entries may change", + ) + + +def verify_restatements( + base: list[dict], staged: list[dict], rejudged: frozenset[str], cases_dir: Path +) -> None: + """A re-opened entry's judge fields must be the restate script's. + + Where they differ from 20260929's, the entry must name the case's current + Opus 5.5 verdict (bound by its sidecar) as its judge, date it by that + sidecar's UTC day, and keep 20260929's judge_previous with exactly one + item appended: the replaced verdict, which is the one 20260929's entry + names (its judge, classes, flag and day are its seed verdict's, as the + sha256-bound sidecar records it). A new entry must name the current judge + too. + """ + from restate_gpt61sol_adjudications import JUDGE_FIELDS, _utc_day, named_item + + before = {case_id(entry): entry for entry in base} + wrong = [] + for entry in staged: + case = case_id(entry) + if case not in rejudged: + continue + original = before.get(case) + judge = {k: v for k, v in entry.items() if k in JUDGE_FIELDS} + if original is not None and judge == { + k: v for k, v in original.items() if k in JUDGE_FIELDS + }: + continue + verdict = cases_dir / case / "verdict.json" + meta_path = verdict.with_name("verdict.meta.json") + meta = json.loads(meta_path.read_text()) if meta_path.is_file() else {} + bound = verdict.is_file() and meta.get("verdict_sha256") == digest(verdict) + day = _utc_day(meta["judged_at_utc"]) if meta.get("judged_at_utc") else None + problems = [] + if not bound or entry.get("judge_model") != JUDGE_MODEL: + problems.append("judge is not the current Opus 5.5 verdict") + if original is not None: + previous = original.get("judge_previous", []) + restated = entry.get("judge_previous", []) + if len(restated) != len(previous) + 1 or restated[:-1] != previous: + problems.append("judge_previous is not 20260929's plus one item") + elif restated[-1] != named_item(original): + problems.append( + "the appended judge_previous item is not the verdict " + "20260929's entry names" + ) + if ( + entry.get("judge_rejudged_on") != day + or entry.get("judged_on_utc", day) != day + ): + problems.append(f"not dated by the current verdict's day {day}") + if problems: + wrong.append(f"{case}: {'; '.join(problems)}") + require( + not wrong, + f"re-opened adjudications not restated by the restate script: {wrong[:4]}", + ) + + +def rejudged_cases(stage: Path) -> frozenset[str]: + """The cases GPT-6.1 Sol re-opened: prompt-changes.json's changed and added. + + load_seed re-derives prompt-changes.json from the stage's prompts and the + seed stage.json binds, so the lists cannot drift from the stage. + """ + load_seed(stage) + changes = json.loads((stage / PROMPT_CHANGES).read_text()) + return frozenset(changes["changed"]) | frozenset(changes["added"]) + + +def case_id(entry: dict) -> str: + """An adjudication entry's audit case id.""" + return f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + + +# The only published wording an amendment may change, and where it lives: the +# decision's reasoning in the adjudication record, the case note, and one +# model's row annotation. None of them carries a class, an exclusion or a score. +AMENDABLE_FIELDS = { + "reasoning": "adjudication record", + "case_annotation": "case note", + "annotation": "row annotation", +} + + +def load_amendments(stage: Path, rejudged: frozenset[str]) -> list[dict]: + """The stage's wording-only amendments, each checked for shape and scope. + + Each names a case GPT-6.1 Sol re-opened, a wording field, the exact old + text, the new text and the reason; a row annotation also names its model. + An absent file lists none. + """ + path = stage / AMENDMENTS + if not path.exists(): + return [] + payload = json.loads(path.read_text()) + amendments = payload.get("amendments") if isinstance(payload, dict) else None + require(isinstance(amendments, list), f"{AMENDMENTS}: 'amendments' is not a list") + for item in amendments: + require(isinstance(item, dict), f"{AMENDMENTS}: {item!r} is not an object") + field = item.get("field") + require( + field in AMENDABLE_FIELDS, + f"{AMENDMENTS}: field {field!r} is not wording; only " + f"{sorted(AMENDABLE_FIELDS)} may be amended", + ) + keys = {"case_id", "field", "old", "new", "reason"} + if field == "annotation": + keys.add("model") + require( + set(item) == keys, + f"{AMENDMENTS}: an amendment of {field} has keys {sorted(keys)}, " + f"not {sorted(item)}", + ) + require( + item["case_id"] in rejudged, + f"{AMENDMENTS}: {item['case_id']} was not re-judged in this stage", + ) + for key in ("old", "new", "reason", *(("model",) if "model" in keys else ())): + require( + isinstance(item[key], str) and item[key].strip(), + f"{AMENDMENTS}: {key} must be non-empty text: {item!r}", + ) + require( + item["old"] != item["new"], + f"{AMENDMENTS}: an amendment changes nothing: {item!r}", + ) + return amendments + + +def amend_text(text: str, amendments: list[dict], label: str) -> str: + """``text`` with each amendment's old wording, found exactly once, replaced.""" + for item in amendments: + count = text.count(item["old"]) + require( + count == 1, + f"{label}: the old text of a wording amendment occurs {count} times, " + f"not once: {item['old'][:80]!r}", + ) + text = text.replace(item["old"], item["new"]) + return text + + +def _record_amendments(amendments: list[dict]) -> dict[str, list[dict]]: + """The amendments of the adjudication record, by case.""" + grouped: dict[str, list[dict]] = {} + for item in amendments: + if item["field"] == "reasoning": + grouped.setdefault(item["case_id"], []).append(item) + return grouped + + +def verify_adjudication_changes( + base: list[dict], + staged: list[dict], + rejudged: frozenset[str], + amendments: list[dict], +) -> int: + """A staged record differs from 20260929's only where it has a reason to. + + Only a case GPT-6.1 Sol re-opened (``rejudged``) may change, and only in + its judge fields (the restate script's JUDGE_FIELDS) and in the reasoning + wording the listed amendments change, exactly as they say. Every other + field of every committed entry keeps its value and its place, key order + included, and the committed entries keep their order. A new entry may only + decide a re-opened case, and none may be dropped. Returns how many staged + entries are new. + """ + from restate_gpt61sol_adjudications import JUDGE_FIELDS + + def serialized(entry: dict, rejudged_case: bool) -> str: + # A re-opened case may rewrite its judge fields; nothing else may move. + items = [ + [key, value] + for key, value in entry.items() + if not (rejudged_case and key in JUDGE_FIELDS) + ] + return json.dumps(items, ensure_ascii=False, allow_nan=False) + + before = {case_id(entry): entry for entry in base} + after = {case_id(entry): entry for entry in staged} + dropped = sorted(set(before) - set(after)) + require(not dropped, f"Staged adjudications drop recorded decisions: {dropped}") + new = sorted(set(after) - set(before)) + require( + set(new) <= rejudged, + "Staged adjudications add decisions on cases GPT-6.1 Sol did not " + f"re-open: {sorted(set(new) - rejudged)[:8]}", + ) + require( + [case for case in after if case in before] == list(before), + "Staged adjudications change the committed entry order", + ) + grouped = _record_amendments(amendments) + require( + set(grouped) <= set(before), + f"Wording amendments name cases with no recorded decision: " + f"{sorted(set(grouped) - set(before))}", + ) + changed = [] + for case, entry in before.items(): + if case in grouped: + entry = { + **entry, + "reasoning": amend_text( + entry["reasoning"], grouped[case], f"{case} reasoning" + ), + } + reopened = case in rejudged + if serialized(after[case], reopened) != serialized(entry, reopened): + changed.append(case) + require( + not changed, + "Staged adjudications change recorded decisions beyond the re-judged " + f"cases' judge fields and the listed wording amendments: {changed[:8]}", + ) + return len(new) + + +def stage_adjudications( + path: Path, rejudged: frozenset[str], amendments: list[dict], cases_dir: Path +) -> list[dict]: + """The staged record, with its listed reasoning amendments applied. + + Checked in memory against release 20260929's record (from git) and the + stage's verdicts; written back only when an amendment was not applied yet. + """ + from freeze_snapshot import verify_adjudications_keep_judge_verdicts + + from policybench.adjudications import parse_adjudications + + base_record = base_adjudication_record() + base = base_adjudications() + current = path.read_text() + verify_record_form(current, base_record) + record = json.loads(current) + grouped = _record_amendments(amendments) + original = {case_id(entry): entry for entry in base} + applied = False + for entry in record["adjudications"]: + case = case_id(entry) + if case in grouped and case in original: + wording = original[case]["reasoning"] + if entry["reasoning"] == wording: + entry["reasoning"] = amend_text( + wording, grouped[case], f"{case} reasoning" + ) + applied = True + text = record_text(record) + verify_record_form(text, base_record) + entries = parse_adjudications(json.loads(text), path) + verify_adjudication_changes(base, entries, rejudged, amendments) + verify_restatements(base, entries, rejudged, cases_dir) + verify_adjudications_keep_judge_verdicts(entries, cases_dir) + if applied: + pending = path.with_name(path.name + ".amending") + pending.write_text(text) + os.replace(pending, path) + return entries + + +def amend_annotations(rows, cases, amendments: list[dict]) -> None: + """Apply the listed case-note and row-annotation amendments in place. + + Each must find exactly one row, and its old text exactly once in it. + """ + for item in amendments: + if item["field"] == "reasoning": + continue + country, scenario, variable = item["case_id"].split("__", 2) + frame = cases if item["field"] == "case_annotation" else rows + mask = ( + (frame["country"].astype(str) == country) + & (frame["scenario_id"].astype(str) == scenario) + & (frame["variable"].astype(str) == variable) + ) + if item["field"] == "annotation": + mask &= frame["model"].astype(str) == item["model"] + require( + int(mask.sum()) == 1, + f"a wording amendment of {item['case_id']} {item['field']} " + f"matches {int(mask.sum())} rows, not one", + ) + index = frame.index[mask][0] + frame.loc[index, item["field"]] = amend_text( + str(frame.loc[index, item["field"]]), + [item], + f"{item['case_id']} {item['field']}", + ) + + +def annotation_csv_text(frame) -> str: + """The bytes triage writes for the row annotations or the case notes, and + the freeze rebuilds: every column, in collect_audit's row order.""" + return frame.to_csv(index=False) + + +def resolve_live_base(args) -> dict: + """The 20260929 payload an export compares the incumbents against. + + Before the freeze the live pointer and the committed snapshot are + 20260929's, and resolve_base checks all of them. After the freeze they are + this release's own, so a re-export (after a fix to the staged annotations) + reads the 20260929 run payload from BASE_COMMIT instead and checks that it + rewraps to the 20260929 release asset (BASE_SHA256). Any other pointer is + refused. + """ + pointer = live_pointer() + if pointer["tag"] == BASE_TAG: + return resolve_base(args)[2] + require( + pointer["tag"] == RELEASE_TAG, + "base pointer changed; review the base before staging", + ) + return base_payload_from_commit() + + +def prepare_inputs(args, runs, base, reference) -> tuple[Path, dict]: + """Copy immutable inputs and fold full or explicitly partial cohorts.""" + import pandas as pd + + from policybench.config import MODELS as registry + from policybench.config import PRICE_OVERRIDES_PER_1M + from policybench.fold_board import fold_board + + stage = args.stage_dir + frames = {} + provenance = {} + for run in runs: + frame = pd.read_csv(run.predictions, low_memory=False) + if args.partial: + frame = frame.drop(columns=["raw_response"], errors="ignore") + households = set(frame.scenario_id) + require( + len(households) == run.state["total"], + f"state/CSV household mismatch: {run.model}", + ) + require( + households <= set(reference.scenario_id), f"unknown households: {run.model}" + ) + validate_keys( + frame, reference[reference.scenario_id.isin(households)], run.model + ) + require( + run.model in registry and run.model in PRICE_OVERRIDES_PER_1M, + f"missing model registration/pricing: {run.model}", + ) + if not args.partial: + require( + run.state["total"] == 100 and households == set(reference.scenario_id), + f"full release requires 100 households: {run.model}", + ) + require( + not run.state.get("synthetic_partial"), + "synthetic run state is partial only", + ) + fingerprint = run.state.get("treatment_fingerprint") + require( + isinstance(fingerprint, dict) + and fingerprint.get("model_id") == registry[run.model], + f"missing/mismatched treatment fingerprint: {run.model}", + ) + import freeze_snapshot as freezer + from freeze_gpt61sol import validate_treatment + + validate_treatment( + freezer, + run.state, + run.run_dir / "run_state.json", + SNAPSHOT / "scenarios.csv", + ) + for field in ( + "estimated_cost_usd", + "total_tokens", + "provider_resolved_model", + "provider_response_id", + ): + require( + field in frame and frame[field].notna().all(), + f"missing {field}: {run.model}", + ) + expected_provider = registry[run.model].split("/")[-1] + require( + set(frame.provider_resolved_model) == {expected_provider}, + f"unexpected provider-resolved model: {run.model}", + ) + target = stage / "inputs" / run.slug + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(run.predictions, target / "predictions.csv") + shutil.copyfile(run.run_dir / "run_state.json", target / "run_state.json") + frames[run.model] = frame + provenance[run.model] = { + "households": len(households), + "rows": len(frame), + "predictions_sha256": digest(target / "predictions.csv"), + "treatment_fingerprint": run.state.get("treatment_fingerprint"), + "provider_resolved_models": sorted( + frame.provider_resolved_model.dropna().unique() + ), + "provider_system_fingerprints": sorted( + frame.get("provider_system_fingerprint", pd.Series(dtype=str)) + .dropna() + .unique() + ), + } + shared = set.intersection(*(set(frame.scenario_id) for frame in frames.values())) + require(bool(shared), "no households completed by every addition") + selected = ( + reference[reference.scenario_id.isin(shared)] if args.partial else reference + ) + scoring = stage / "scoring" + scoring.mkdir(exist_ok=True) + for name in REFERENCE_FILES: + shutil.copyfile(SNAPSHOT / name, scoring / name) + if args.partial: + selected.to_csv(scoring / "reference_outputs.csv", index=False) + meta = json.loads((scoring / "reference_outputs.csv.meta.json").read_text()) + meta.update( + reference_csv_sha256=digest(scoring / "reference_outputs.csv"), + row_count=len(selected), + partial=True, + ) + write_json(scoring / "reference_outputs.csv.meta.json", meta) + scenarios = pd.read_csv(scoring / "scenarios.csv") + scenarios[scenarios.scenario_id.isin(shared)].to_csv( + scoring / "scenarios.csv", index=False + ) + exclusions = json.loads((scoring / "reference_exclusions.json").read_text()) + exclusions["exclusions"] = [ + e for e in exclusions["exclusions"] if e["scenario_id"] in shared + ] + write_json(scoring / "reference_exclusions.json", exclusions) + base = base[base.scenario_id.isin(shared)] + else: + verify_reference_pins(scoring, "staged reference") + base_path = stage / "base-predictions.csv" + base.to_csv(base_path, index=False) + additions = [] + for run in runs: + path = stage / "inputs" / run.slug / "fold.csv" + frames[run.model][frames[run.model].scenario_id.isin(shared)].to_csv( + path, index=False + ) + additions.append(path) + result = fold_board(base_path, additions, scoring, stage / "fold", export=False) + require( + not result["excluded"] and result["models"] == BOARD_MODELS, + f"fold refused additions: {result}", + ) + bundle = stage / "publish" / RUN_NAME + us = bundle / "us" + us.mkdir(parents=True, exist_ok=True) + require( + not (us / "by_model").exists(), "by_model would shadow combined predictions" + ) + shutil.copyfile(stage / "fold/us/predictions.csv", us / "predictions.csv") + for name in REFERENCE_FILES: + shutil.copyfile(scoring / name, us / name) + annotations = bundle / "annotations" + annotations.mkdir(exist_ok=True) + for name in ANNOTATION_FILES: + # Re-preparing must not overwrite a recorded stage adjudication. + if not (annotations / name).exists(): + shutil.copyfile(ANNOTATIONS / name, annotations / name) + write_json(stage / "model-provenance.json", provenance) + return bundle, frames + + +def partial_scores(args, base, reference, frames) -> None: + """Rank each addition against incumbents on its own completed households.""" + import pandas as pd + + from policybench.analysis import analyze_no_tools + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + scored, _ = split_reference(reference, load_reference_exclusions(SNAPSHOT)) + scenarios = pd.read_csv(SNAPSHOT / "scenarios.csv") + results = [] + for model, frame in frames.items(): + ids = set(frame.scenario_id) + cohort = pd.concat([base[base.scenario_id.isin(ids)], frame], ignore_index=True) + analysis = analyze_no_tools( + scored[scored.scenario_id.isin(ids)], + cohort, + scenarios=scenarios[scenarios.scenario_id.isin(ids)], + ) + stats = analysis["bounded_summary"].set_index("model") + exact = float(stats.loc[model, "weighted_exact"]) + rank = 1 + int((stats.weighted_exact > exact).sum()) + result = { + "label": "PARTIAL", + "model": model, + "households": len(ids), + "scored_outputs": int(scored.scenario_id.isin(ids).sum()), + "exact_percent": exact * 100, + "rank": rank, + "models": len(stats), + } + results.append(result) + print(json.dumps(result)) + write_json(args.stage_dir / "partial-scores.json", results) + + +def seed_prompt_digests(audit: Path) -> dict[str, str]: + """The sha256 of every prompt a case carries, keyed by case id.""" + return { + case.name: digest(case / "prompt.md") + for case in sorted((audit / "cases").glob("*")) + if (case / "prompt.md").is_file() + } + + +def seed_digest(audit: Path) -> dict[str, dict[str, str]]: + """Each judged case's prompt and verdict sha256, keyed by case id.""" + seed = {} + for case in sorted((audit / "cases").glob("*")): + if not (case / "prompt.md").is_file(): + continue + require((case / "verdict.json").is_file(), f"seed case unjudged: {case.name}") + seed[case.name] = { + "prompt_sha256": digest(case / "prompt.md"), + "verdict_sha256": digest(case / "verdict.json"), + } + return seed + + +def seed_digest_text(seed: dict[str, dict[str, str]]) -> str: + """The seed digest as docs/gpt61sol/seed_digest.csv spells it.""" + rows = [ + f"{case},{item['prompt_sha256']},{item['verdict_sha256']}\n" + for case, item in sorted(seed.items()) + ] + return "case_id,prompt_sha256,verdict_sha256\n" + "".join(rows) + + +def verify_seed(seed: dict[str, dict[str, str]]) -> None: + """The seed must be the 20260929 audit the committed digest records.""" + require( + hashlib.sha256(seed_digest_text(seed).encode()).hexdigest() + == SEED_DIGEST_SHA256, + "the audit seed is not release 20260929's (see docs/gpt61sol/" + "seed_digest.csv): its cases, prompts or verdicts differ", + ) + + +def load_seed(stage: Path) -> dict[str, dict[str, str]]: + """The seed stage.json binds, checked against the committed digest. + + It also re-derives which cases are kept, changed and added from the + stage's prompts and that seed, and refuses unless prompt-changes.json + says the same: a kept case's prompt that drifts, a case moved between + the lists, or a manifest whose wrong models the staged predictions do not + bear out, stops every step that relies on them. + """ + receipt = json.loads((stage / "stage.json").read_text()) + require( + "seed" in receipt, + "stage.json does not bind the audit seed; run --step bind-seed " + "--audit-seed ", + ) + verify_seed(receipt["seed"]) + verify_prompt_changes(stage, receipt["seed"]) + return receipt["seed"] + + +def verify_prompt_changes(stage: Path, seed: dict[str, dict[str, str]]) -> None: + """prompt-changes.json must be what the stage's prompts say against the seed. + + A case whose prompt is the seed's is kept, one whose prompt differs is + changed and one the seed lacks is added; GPT-6.1 Sol must be among the + wrong models of every changed or added case, and no seed case may vanish + (check_prompt_changes, as prepare applies it). The staged predictions + must agree (verify_reopened_by_predictions). + """ + derived = check_prompt_changes( + stage / "audit", {case: item["prompt_sha256"] for case, item in seed.items()} + ) + recorded = json.loads((stage / PROMPT_CHANGES).read_text()) + differ = sorted( + { + case + for key in ("kept", "changed", "added") + for case in set(derived[key]) ^ set(recorded.get(key, [])) + } + ) + require( + not differ, + f"{PROMPT_CHANGES} disagrees with the stage's prompts and the bound " + f"seed on {len(differ)} cases: {differ[:8]}", + ) + verify_reopened_by_predictions(stage, derived) + + +def verify_reopened_by_predictions(stage: Path, derived: dict[str, list]) -> None: + """The manifest's claim that GPT-6.1 Sol re-opened a case must hold. + + cases.jsonl is editable, so re-score the staged predictions against the + staged references with wrong_prediction_rows, the rule prepare_audit + used to list each case's wrong models. A changed or added case must list + the new model exactly when its prediction is wrong; any other case it + gets wrong must be parse-failure-only as the manifest records it, so a + kept case needs the new model's prediction right. + """ + from policybench.audit import _case_id, _load_manifest + from policybench.case_annotations import wrong_prediction_rows + + us = stage / "publish" / RUN_NAME / "us" + wrong = wrong_prediction_rows(us) + manifest = _load_manifest(stage / "audit") + reopened = {*derived["changed"], *derived["added"]} + parse_only = {case for case, row in manifest.items() if row["parse_failure_only"]} + differ = set() + for model in MODELS.values(): + rows = wrong[wrong.model == model] + missed = { + _case_id(us.name, str(scenario), str(variable)) + for scenario, variable in zip(rows.scenario_id, rows.variable) + } + differ |= { + case + for case in reopened + if (model in manifest[case]["wrong_models"]) != (case in missed) + } + differ |= missed - reopened - parse_only + require( + not differ, + f"the audit manifest's wrong models disagree with the staged " + f"predictions of {sorted(MODELS.values())} on {len(differ)} cases: " + f"{sorted(differ)[:8]}", + ) + + +def bind_seed(args) -> None: + """Bind the seed in the stage.json of a stage prepared before prepare did. + + The seed must match the committed digest, and the stage must agree with it + case by case: every kept case keeps the seed's prompt and verdict bytes, + every changed case's prompt differs, and no added case is a seed case. + """ + receipt_path = args.stage_dir / "stage.json" + receipt = json.loads(receipt_path.read_text()) + require("seed" not in receipt, "stage.json already binds its audit seed") + require(args.audit_seed is not None, "bind-seed needs --audit-seed") + seed = seed_digest(args.audit_seed) + verify_seed(seed) + changes = json.loads((args.stage_dir / PROMPT_CHANGES).read_text()) + cases = args.stage_dir / "audit" / "cases" + differ = [ + case + for case in changes["kept"] + if case not in seed + or not (cases / case / "verdict.json").is_file() + or digest(cases / case / "prompt.md") != seed[case]["prompt_sha256"] + or digest(cases / case / "verdict.json") != seed[case]["verdict_sha256"] + ] + differ += [ + case + for case in changes["changed"] + if case not in seed + or digest(cases / case / "prompt.md") == seed[case]["prompt_sha256"] + ] + differ += [case for case in changes["added"] if case in seed] + differ += sorted(set(seed) - {*changes["kept"], *changes["changed"]}) + require( + not differ, + f"the stage disagrees with the seed on {len(differ)} cases: {differ[:8]}", + ) + receipt["seed"] = seed + write_json(receipt_path, receipt) + print(f"Bound the audit seed: {len(seed)} judged cases") + + +def check_prompt_changes(audit: Path, seeded: dict[str, str]) -> dict[str, list]: + """Only a case GPT-6.1 Sol joins may change or appear. + + The seed's prompts re-render byte-identically from the committed snapshot + and the pinned grounding, so an incumbent-only case whose prompt differs + (or that is new) means an input changed under the carried-over verdicts. + """ + manifest = [ + json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() + ] + new_models = set(MODELS.values()) + changed, added, kept, offending = [], [], [], [] + for item in manifest: + if item["parse_failure_only"]: + continue + case_id = item["case_id"] + prompt = digest(audit / "cases" / case_id / "prompt.md") + if seeded.get(case_id) == prompt: + kept.append(case_id) + continue + (changed if case_id in seeded else added).append(case_id) + if not new_models & set(item["wrong_models"]): + offending.append(case_id) + require( + not offending, + "incumbent-only case prompts changed against the seed (grounding, " + f"references or annotations differ): {offending[:8]}", + ) + # Adding a model cannot remove a wrong answer, so every judged seed case + # must still be a case. + vanished = sorted(set(seeded) - {*kept, *changed}) + require(not vanished, f"seed cases vanished from the audit: {vanished[:8]}") + return {"kept": kept, "changed": changed, "added": added} + + +def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: + """Seed a private audit; changed prompts invalidate copied verdicts. + + Returns the seed digest, which main binds in stage.json. + """ + import pandas as pd + + from policybench.audit import prepare_audit + + require( + args.audit_seed is not None and args.grounding is not None, + "prepare needs --audit-seed and --grounding (read-only sources)", + ) + require( + digest(args.grounding) == GROUNDING_SHA256, + "grounding differs from the one the 20260929 audit was rendered with", + ) + audit = args.stage_dir / "audit" + # Compare against the read-only seed, not the stage copy, so a retried + # prepare cannot mistake its own earlier rendering for the seed. + seed = seed_digest(args.audit_seed) + verify_seed(seed) + seeded = {case: item["prompt_sha256"] for case, item in seed.items()} + if not audit.exists(): + for source in sorted((args.audit_seed / "cases").glob("*")): + if not (source / "prompt.md").is_file(): + continue + target = audit / "cases" / source.name + target.mkdir(parents=True) + for name in ("prompt.md", "verdict.json", "verdict.meta.json", "codex.log"): + if (source / name).is_file(): + shutil.copyfile(source / name, target / name) + grounding = pd.read_csv(args.grounding) + lookup = { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + changes = check_prompt_changes(audit, seeded) + write_json( + args.stage_dir / "prompt-changes.json", + {key: sorted(value) for key, value in changes.items()}, + ) + pending = validate_verdicts(audit, remove_invalid=True, seed=seed) + write_json(args.stage_dir / "pending.json", pending) + print( + f"Prepared audit: {len(changes['kept'])} prompts unchanged, " + f"{len(changes['changed'])} changed and {len(changes['added'])} new; " + f"{len(pending)} cases need Opus 5.5" + ) + return seed + + +def set_aside(audit: Path, case_id: str, reason: str) -> None: + """Move a case's verdict, sidecar and judge evidence out of the audit. + + They go to /rejected-verdicts/// with the judge's + envelope, log and transcript, so a re-judge never destroys the verdict it + replaces or the record of how it was made. + """ + from datetime import datetime, timezone + + case = audit / "cases" / case_id + names = ( + "verdict.json", + "verdict.meta.json", + "claude.json", + "claude.log", + "claude.transcript.jsonl", + ) + files = [case / name for name in names] + if not any(path.exists() for path in files[:2]): + return + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + target = audit.parent / "rejected-verdicts" / case_id / stamp + target.mkdir(parents=True) + (target / "reason.txt").write_text(reason + "\n") + for path in files: + if path.exists(): + os.replace(path, target / path.name) + + +def validate_verdicts( + audit: Path, + remove_invalid: bool = False, + seed: dict[str, dict[str, str]] | None = None, +) -> list[str]: + """Validate schema, model coverage and each verdict's binding to its bytes. + + Every judged verdict's sidecar must carry the verdict's own sha256. A case + whose prompt is the seed's (``seed``, as stage.json binds it) carries its + seed verdict over: the verdict must be the seed's, byte for byte, and any + prompt_sha256 its sidecar records must match. Every other verdict is new + and must record the sha256 of the prompt it judged. A verdict naming + GPT-6.1 Sol also needs bound Opus 5.5 provenance. Returns the pending + cases; ``remove_invalid`` sets their verdicts aside. A carried-over verdict + that fails is refused outright: a re-judge cannot restore it. + """ + import jsonschema + + schema = json.loads((audit / "schema.json").read_text()) + manifest = { + item["case_id"]: item + for item in map(json.loads, (audit / "cases.jsonl").read_text().splitlines()) + } + seed = seed or {} + pending, refused = [], [] + for case_id, item in manifest.items(): + if item["parse_failure_only"]: + continue + path = audit / "cases" / case_id / "verdict.json" + meta_path = path.with_name("verdict.meta.json") + prompt = path.with_name("prompt.md") + carried = ( + case_id in seed + and prompt.is_file() + and digest(prompt) == seed[case_id]["prompt_sha256"] + ) + try: + blob = path.read_bytes() + verdict = json.loads(blob) + jsonschema.validate(verdict, schema) + names = [m["model"] for m in verdict["models"]] + if len(names) != len(set(names)) or set(names) != set(item["wrong_models"]): + raise ValueError("wrong model coverage") + meta = json.loads(meta_path.read_text()) if meta_path.is_file() else {} + if not isinstance(meta, dict): + raise ValueError("verdict provenance is not an object") + if meta.get("verdict_sha256") != hashlib.sha256(blob).hexdigest(): + raise ValueError("the sidecar is not bound to this verdict") + bound = meta.get("prompt_sha256") + if carried: + if hashlib.sha256(blob).hexdigest() != seed[case_id]["verdict_sha256"]: + raise ValueError("a carried-over verdict is not the seed's") + if bound is not None and bound != digest(prompt): + raise ValueError("verdict is bound to a different prompt") + elif bound != digest(prompt): + raise ValueError("a new verdict is not bound to this prompt") + if set(names) & set(MODELS.values()): + if ( + meta.get("judge_model_requested") != JUDGE_MODEL + or meta.get("judge_model_reported") != [JUDGE_MODEL] + or not meta.get("judge_runner") + or not meta.get("judged_at_utc") + ): + raise ValueError("missing or mismatched Opus 5.5 provenance") + except (OSError, ValueError, jsonschema.ValidationError) as error: + if carried: + refused.append(f"{case_id} ({error})") + continue + pending.append(case_id) + if remove_invalid: + set_aside(audit, case_id, f"invalid: {error}") + require( + not refused, + f"{len(refused)} carried-over verdicts differ from the seed's; a stage " + f"input changed after prepare: {refused[:4]}", + ) + return sorted(pending) + + +def account_data(value, at: str, keys: bool = True) -> list[str]: + """Where ``value`` carries an e-mail address or a key naming an account. + + Any key that matches ACCOUNT_KEY (unless ``keys`` is false), and any key + or string that holds an e-mail address (EMAIL_ADDRESS), at any depth; + ``at`` names ``value``. + """ + hits = [] + if isinstance(value, dict): + for key, item in value.items(): + where = f"{at}.{key}" + if keys and ACCOUNT_KEY.search(str(key)): + hits.append(f"key {where}") + if EMAIL_ADDRESS.search(str(key)): + hits.append(f"an e-mail address in key {where}") + hits += account_data(item, where, keys) + elif isinstance(value, list): + for index, item in enumerate(value): + hits += account_data(item, f"{at}[{index}]", keys) + elif isinstance(value, str) and EMAIL_ADDRESS.search(value): + hits.append(f"an e-mail address at {at}") + return hits + + +def outside_the_judge(event: dict, prompt: str | None) -> dict: + """``event`` without the judged prompt's own user message or the judge's + words: the text of a user event that is exactly ``prompt``, and the content + of an assistant turn. The account-data rule reads everything else.""" + message = event.get("message") + if isinstance(message, dict) and ( + event.get("type") == "assistant" + or ( + event.get("type") == "user" + and prompt is not None + and message.get("content") == prompt + ) + ): + rest = {key: value for key, value in message.items() if key != "content"} + return {**event, "message": rest} + return event + + +def transcript_problems( + path: Path, effort: str | None, prompt: bytes, verdict: object +) -> list[str]: + """Why a judge's transcript does not show an isolated judge; [] if it does. + + A Python port of the transcript checks in scripts/run_audit_claude.sh + (extract_verdict): no tool call but StructuredOutput, no event of a type + outside its EVENT_TYPES (JUDGE_EVENT_TYPES), no context attachment outside + its ATTACHMENTS (JUDGE_ATTACHMENTS; so no skill listing and no + credential_org record), a session context that is empty (no account e-mail + or git status), no account data anywhere outside the prompt's own user + message and the judge's own turns (account_data, outside_the_judge), no + working directory inside a git repository, every assistant turn at the + effort the sidecar records, when it records one (the sidecars of the + stage's first 17 isolated verdicts record none), no advisor model, and + user events that are exactly one text message, the case's ``prompt`` + (prompt.md's bytes, decoded as UTF-8), besides Claude Code's own nudge + (JUDGE_PROMPT_NUDGE) and the results of the judge's StructuredOutput calls. + It also requires exactly one accepted StructuredOutput call, whose input + is the published ``verdict`` (verdict.json, parsed): any other call must + be one the schema refused, which the judge then answered again. + """ + try: + lines = path.read_text().splitlines() + except OSError: + return ["no transcript"] + try: + text = prompt.decode("utf-8") + except UnicodeDecodeError: + text = None + problems, calls, refused = [], [], set() + # The user events: text messages (the prompt) and everything else (which + # may only answer the judge's own StructuredOutput calls, by id). + texts, others, answers = [], [], set() + + def answer(part: dict) -> bool: + return part.get("type") == "tool_use" and part.get("name") == "StructuredOutput" + + for number, line in enumerate(lines, 1): + try: + event = json.loads(line) + except ValueError: + problems.append(f"transcript line {number} is not JSON") + continue + if not isinstance(event, dict): + problems.append(f"transcript line {number} is not an event") + continue + if event.get("type") not in JUDGE_EVENT_TYPES: + problems.append(f"an event of type {event.get('type')!r}") + problems += account_data(outside_the_judge(event, text), f"line {number}") + if event.get("type") == "attachment": + attachment = event.get("attachment") or {} + kind = attachment.get("type") + if kind not in JUDGE_ATTACHMENTS: + problems.append(f"a {kind!r} attachment") + if kind == "environment" and (attachment.get("snapshot") or {}).get( + "isGitRepo" + ): + problems.append("a working directory inside a git repository") + context = attachment.get("context") + if kind == "session_context" and context != {}: + carried = sorted(context) if isinstance(context, dict) else context + problems.append(f"a session context carrying {carried!r}") + if event.get("type") == "assistant": + if effort is not None and event.get("effort") != effort: + problems.append(f"a turn at effort {event.get('effort')!r}") + if event.get("advisorModel"): + problems.append(f"advisor model {event.get('advisorModel')!r}") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + if event.get("type") == "user": + if not isinstance(content, str): + others.append(content) + elif not (event.get("isMeta") is True and content == JUDGE_PROMPT_NUDGE): + texts.append(content) + for part in content if isinstance(content, list) else []: + if not isinstance(part, dict): + continue + if str(part.get("type", "")).endswith("tool_use"): + calls.append(part) + if event.get("type") == "assistant" and answer(part): + answers.add(part.get("id")) + elif part.get("type") == "tool_result" and part.get("is_error"): + refused.add(part.get("tool_use_id")) + + structured = [part for part in calls if answer(part)] + other = [part.get("name") or part.get("type") for part in calls if not answer(part)] + if other: + problems.append(f"tool calls {other}") + accepted = [part for part in structured if part.get("id") not in refused] + if len(accepted) != 1: + problems.append(f"{len(accepted)} accepted StructuredOutput calls, not 1") + elif json.dumps(accepted[0].get("input"), sort_keys=True) != json.dumps( + verdict, sort_keys=True + ): + problems.append("its accepted StructuredOutput answer is not verdict.json") + if len(texts) != 1: + problems.append(f"{len(texts)} user text messages, not the prompt alone") + elif text is None or texts[0] != text: + problems.append("its user message is not prompt.md") + if not all( + isinstance(content, list) + and content + and all( + isinstance(part, dict) + and part.get("type") == "tool_result" + and part.get("tool_use_id") in answers + for part in content + ) + for content in others + ): + problems.append("a user event that is not a StructuredOutput call's result") + return problems + + +def withhold_addresses(value): + """``value`` with every e-mail address in it replaced by WITHHELD_ADDRESS.""" + if not isinstance(value, str): + return value + return EMAIL_ADDRESS.sub(WITHHELD_ADDRESS, value) + + +def verify_judge_provenance( + cases_dir: Path, rejudged: frozenset[str], record_path: Path +) -> None: + """The published judge provenance must describe the staged new verdicts. + + The record (JUDGE_PROVENANCE) lists every case GPT-6.1 Sol re-opened, each + once, with its verdict's and prompt's sha256, its group and whether its + judge ran isolated, and its sidecar's judge_effort, judge_model_reported, + judged_at_utc and judge_account_declared (JUDGE_SIDECAR_FIELDS), an + e-mail address in any of them withheld as WITHHELD_ADDRESS. The record and + each entry carry exactly their listed keys (JUDGE_PROVENANCE_KEYS, + JUDGE_PROVENANCE_ENTRY_KEYS), its counts are the tally of the entries' + groups, and it names no e-mail address anywhere, for it is public. Each + entry's hashes must be the staged verdict's and prompt's, and its other + fields its sidecar's. Its isolation must be its sidecar's: a sidecar + records judge_isolation only when scripts/run_audit_claude.sh's hardened + runner wrote it, and only an isolated entry's group says "isolated: ". + An isolated verdict's sidecar must record a token login + (judge_auth.method oauth_token), and its transcript, + claude.transcript.jsonl, must pass the runner's transcript checks + (transcript_problems), its prompt and verdict included: the one user + text message must be the staged prompt.md, and the one accepted + StructuredOutput answer the staged verdict.json. + """ + record = json.loads(record_path.read_text()) + entries = record.get("verdicts") if isinstance(record, dict) else None + require( + isinstance(entries, list) and all(isinstance(e, dict) for e in entries), + f"{JUDGE_PROVENANCE_PATH} lists no verdicts", + ) + require( + set(record) == JUDGE_PROVENANCE_KEYS, + f"{JUDGE_PROVENANCE_PATH} has keys {sorted(record)}, not " + f"{sorted(JUDGE_PROVENANCE_KEYS)}", + ) + addresses = account_data(record, "record", keys=False) + require( + not addresses, + f"{JUDGE_PROVENANCE_PATH} is public and names {addresses[:4]}; withhold " + f"each e-mail address as {WITHHELD_ADDRESS!r}", + ) + named = [entry.get("case_id") for entry in entries] + require( + len(named) == len(set(named)) and set(named) == set(rejudged), + f"{JUDGE_PROVENANCE_PATH} does not list each re-opened case once: " + f"{sorted(set(named) ^ set(rejudged))[:4]}", + ) + tally: dict = {} + for entry in entries: + group = entry.get("group") + tally[group] = tally.get(group, 0) + 1 + require( + record["counts"] == tally, + f"{JUDGE_PROVENANCE_PATH} counts {record['counts']!r} are not the tally " + f"of its entries' groups {tally!r}", + ) + wrong = [] + for entry in entries: + case = cases_dir / entry["case_id"] + try: + meta = json.loads((case / "verdict.meta.json").read_text()) + verdict = json.loads((case / "verdict.json").read_text()) + prompt = (case / "prompt.md").read_bytes() + problems = [] + if entry.get("verdict_sha256") != digest(case / "verdict.json"): + problems.append("the record names another verdict") + if entry.get("prompt_sha256") != hashlib.sha256(prompt).hexdigest(): + problems.append("the record names another prompt") + except (OSError, ValueError) as error: + wrong.append(f"{entry['case_id']}: {error}") + continue + if set(entry) != JUDGE_PROVENANCE_ENTRY_KEYS: + problems.append(f"the entry has keys {sorted(entry)}") + meta = meta if isinstance(meta, dict) else {} + for field in JUDGE_SIDECAR_FIELDS: + expected = withhold_addresses(meta.get(field)) + if entry.get(field) != expected: + problems.append( + f"its {field} {entry.get(field)!r} is not the sidecar's " + f"{expected!r}" + ) + isolated = "judge_isolation" in meta + group = entry.get("group") + if not isinstance(group, str) or group.startswith("isolated: ") != isolated: + problems.append(f"its group {group!r} disagrees with its isolation") + if entry.get("isolated") is not isolated: + problems.append( + f"the record says isolated={entry.get('isolated')!r} but the " + f"sidecar {'records' if isolated else 'lacks'} judge_isolation" + ) + elif isolated: + auth = meta.get("judge_auth") + if not isinstance(auth, dict) or auth.get("method") != "oauth_token": + problems.append( + "the sidecar does not record a token login (judge_auth " + f"method oauth_token): {auth!r}" + ) + problems += transcript_problems( + case / "claude.transcript.jsonl", + meta.get("judge_effort"), + prompt, + verdict, + ) + if problems: + wrong.append(f"{entry['case_id']}: {'; '.join(problems)}") + require( + not wrong, + f"{len(wrong)} new verdicts disagree with {JUDGE_PROVENANCE_PATH}: {wrong[:4]}", + ) + + +def judge(args, bundle) -> None: + """Run one Claude CLI judge at a time and retry missing/hedged cases.""" + from policybench.audit import collect_audit + + audit = args.stage_dir / "audit" + seed = load_seed(args.stage_dir) + for _ in range(3): + validate_verdicts(audit, remove_invalid=True, seed=seed) + subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_claude.sh"), str(audit)], + cwd=ROOT, + env={ + **os.environ, + "AUDIT_MODEL": JUDGE_MODEL, + "AUDIT_PARALLEL": "1", + "AUDIT_PYTHON": sys.executable, + }, + check=True, + ) + pending = validate_verdicts(audit, remove_invalid=True, seed=seed) + out = collect_audit(bundle / "us", audit) + for case_id in out["hedged"].case_id: + set_aside(audit, case_id, "hedged") + if not pending and out["missing"].empty and out["hedged"].empty: + return + raise SystemExit( + "audit missing/hedged after three passes; resume judge after investigation" + ) + + +def triage(args, bundle) -> None: + """Collect, apply recorded decisions, and stop for unresolved flags.""" + from policybench.adjudications import ( + apply_adjudications, + excluded_case_keys, + verify_adjudications_applied, + ) + from policybench.audit import collect_audit + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + audit = args.stage_dir / "audit" + require( + not validate_verdicts(audit, seed=load_seed(args.stage_dir)), + "missing or invalid verdicts; run judge", + ) + out = collect_audit(bundle / "us", audit) + require( + out["missing"].empty and out["hedged"].empty, "missing/hedged audit; run judge" + ) + rows = out["row"] + cases = out["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ) + counts = rows.groupby(KEY).size() + require( + all( + int(r.wrong_model_count) == counts[(r.scenario_id, r.variable)] + for r in cases.itertuples() + ), + "case wrong_model_count disagrees with collected rows", + ) + annotations = bundle / "annotations" + rejudged = rejudged_cases(args.stage_dir) + amendments = load_amendments(args.stage_dir, rejudged) + decisions = stage_adjudications( + annotations / ADJUDICATIONS, rejudged, amendments, audit / "cases" + ) + rows, cases, _ = apply_adjudications(rows, cases, decisions) + amend_annotations(rows, cases, amendments) + # An amendment may not touch the adjudication sentence a case note carries. + verify_adjudications_applied(rows, cases, decisions) + excluded = exclusion_keys(load_reference_exclusions(bundle / "us")) + require( + excluded == excluded_case_keys(decisions), + "adjudication exclusions differ from frozen scoring exclusions", + ) + flags = cases[cases.reference_suspect.astype(bool)] + scored = ~rows[KEY].apply(tuple, axis=1).isin(excluded) + unresolved = rows[ + scored + & ~rows.failure_source.isin( + [ + "llm_error", + "parse_contract_failure", + "budget_exhausted_at_ceiling", + ] + ) + ] + flags.to_csv(args.stage_dir / "reference-flags.csv", index=False) + unresolved.to_csv(args.stage_dir / "unresolved-rows.csv", index=False) + for name, frame in ( + ("us_audit_row_annotations.csv", rows), + ("us_case_notes.csv", cases), + ): + (annotations / name).write_text(annotation_csv_text(frame), encoding="utf-8") + require( + flags.empty and unresolved.empty, + "triage required: inspect reference-flags.csv/unresolved-rows.csv; " + "record evidence in staged us_adjudications.json and rerun triage", + ) + print(f"Triage complete: {len(rows)} rows / {len(cases)} cases") + + +def incumbent_drift(stats: list[dict], previous: dict[str, dict]) -> list[str]: + """Incumbents whose modelStats entry is not byte-identical to the base's. + + Serialized as the payload serializes it, so key order counts as well as + values. A missing incumbent counts as drift. + """ + exported = {row["model"]: row for row in stats} + return sorted( + model + for model, row in previous.items() + if model not in exported + or json.dumps(exported[model], allow_nan=False) + != json.dumps(row, allow_nan=False) + ) + + +def payload_text(payload: dict) -> str: + """The bytes export writes for a payload, and the freeze rebuilds. + + freeze_snapshot reassembles these exact default-json bytes from the + compact country payload. Pretty-printing here breaks its release hash. + """ + return json.dumps(payload, allow_nan=False) + + +def build_payload( + bundle: Path, live: dict, *, partial: bool = False, early: bool = False +) -> dict: + """The payload export writes, built from ``bundle``; the one definition. + + export_full_run's US payload must hold all 46 models. For a release, its + roster must be the base's incumbents plus the addition, Fable 5's usage is + carried from ``live`` (CARRIED_USAGE), and no incumbent may drift; then the + dashboard schema. The freeze rebuilds the staged payload with this. + export_full_run writes data.json, us/data.json and us/analysis/ into + ``bundle``. + """ + from policybench.dashboard_schema import validate_dashboard_payload + from policybench.full_run_export import export_full_run + + payload = export_full_run(bundle, countries=["us"], skip_app_data=True) + stats = payload["countries"]["us"]["modelStats"] + require( + len(stats) == BOARD_MODELS, + f"export did not contain all {BOARD_MODELS} models", + ) + previous = {m["model"]: m for m in live["countries"]["us"]["modelStats"]} + if not partial: + require( + len(previous) == BASE_MODELS and not set(previous) & set(MODELS.values()), + f"base must hold the {BASE_MODELS} incumbents only", + ) + require( + {s["model"] for s in stats} == set(previous) | set(MODELS.values()), + "export roster is not the incumbents plus the addition", + ) + for model, keys in CARRIED_USAGE.items(): + row = next(s for s in stats if s["model"] == model) + for key in keys: + row[key] = previous[model][key] + drift = incumbent_drift(stats, previous) + require(not drift, f"incumbent modelStats drift: {drift}") + errors = validate_dashboard_payload(payload, require_failure_annotations=not early) + require(not errors, f"payload validation failed: {errors[:8]}") + if partial: + payload["stage2Status"] = ( + "PARTIAL — incomplete household cohorts, not a release" + ) + return payload + + +def export(args, bundle, live) -> dict: + """Export into scratch, preserve incumbent statistics, and gate release.""" + if not args.partial: + verify_reference_pins(SNAPSHOT, "committed reference") + verify_reference_pins(bundle / "us", "staged reference") + if not args.early: + verify_new_model_inputs(args.stage_dir) + verify_judge_provenance( + args.stage_dir / "audit" / "cases", + rejudged_cases(args.stage_dir), + JUDGE_PROVENANCE, + ) + payload = build_payload(bundle, live, partial=args.partial, early=args.early) + stats = payload["countries"]["us"]["modelStats"] + path = args.stage_dir / ( + f"PARTIAL-data-board{BOARD_MODELS}.json" + if args.partial + else f"data-board{BOARD_MODELS}.json" + ) + path.write_text(payload_text(payload)) + # Keep the freeze input identical to the gated payload, including Fable usage. + shutil.copyfile(path, bundle / "data.json") + ranked = sorted(stats, key=lambda s: (-s["exact"], -s["score"], s["model"])) + for index, row in enumerate(ranked, 1): + if row["model"] in MODELS.values(): + print( + f"{'PARTIAL common cohort' if args.partial else 'STAGED'} " + f"{row['model']}: exact={row['exact']:.6f}% " + f"rank={index}/{BOARD_MODELS} n={row['n']}" + ) + if not args.early: + pinned = [p for p in (bundle / "annotations").iterdir() if p.is_file()] + pinned += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + pinned += [p for p in (args.stage_dir / "inputs").rglob("*") if p.is_file()] + # The cases GPT-6.1 Sol re-opened, the listed wording amendments and the + # seed binding decide what the adjudication and verdict gates allow; + # stage.json and model-provenance.json hold the prepare-time hashes. + pinned += [ + path + for path in ( + args.stage_dir / PROMPT_CHANGES, + args.stage_dir / AMENDMENTS, + args.stage_dir / "stage.json", + args.stage_dir / "model-provenance.json", + ) + if path.is_file() + ] + pinned += [ + p + for p in (args.stage_dir / "audit").rglob("*") + if p.is_file() + and p.name + in { + "verdict.json", + "verdict.meta.json", + "prompt.md", + "claude.transcript.jsonl", + "cases.jsonl", + "schema.json", + } + ] + write_json( + args.stage_dir / "release-ready.json", + { + "release_tag": RELEASE_TAG, + "base_tag": BASE_TAG, + "base_sha256": BASE_SHA256, + "payload_sha256": digest(path), + "models": BOARD_MODELS, + "partial": False, + "files": { + str(p.relative_to(args.stage_dir)): digest(p) for p in pinned + }, + # A repository file, so outside "files", which binds stage files. + "judge_provenance": { + "path": JUDGE_PROVENANCE_PATH, + "sha256": digest(JUDGE_PROVENANCE), + }, + }, + ) + return payload + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs-root", type=Path) + parser.add_argument("--stage-dir", type=Path) + parser.add_argument( + "--base-predictions", type=Path, default=SNAPSHOT / "predictions.csv.gz" + ) + parser.add_argument("--base-payload", type=Path, default=SNAPSHOT / "data.json.gz") + parser.add_argument("--audit-seed", type=Path) + parser.add_argument("--grounding", type=Path) + parser.add_argument( + "--early", action="store_true", help="fold and export without paid judging" + ) + parser.add_argument( + "--partial", + action="store_true", + help="scratch-only cohort rehearsal; requires --early", + ) + parser.add_argument( + "--step", + choices=("pin-inputs", "prepare", "bind-seed", "judge", "triage", "export"), + default="prepare", + help="pin-inputs writes the finished run's file hashes to " + f"{INPUT_PINS_PATH} (no stage); bind-seed binds the audit seed in the " + "stage.json of a stage prepared before prepare bound it", + ) + args = parser.parse_args(argv) + if args.partial and not args.early: + parser.error("--partial requires --early") + if args.early and args.step != "prepare": + parser.error("--early only applies to prepare") + if args.step in ("prepare", "pin-inputs") and not args.runs_root: + parser.error(f"{args.step} requires --runs-root") + if args.step != "pin-inputs": + if args.stage_dir is None: + parser.error(f"{args.step} requires --stage-dir") + args.stage_dir = args.stage_dir.resolve() + return args + + +def main(argv=None) -> None: + args = parse_args(argv) + if args.step == "pin-inputs": + pin_inputs(args) + return + sources = [SNAPSHOT, ANNOTATIONS, args.base_predictions, args.base_payload] + sources += [ + p for p in (args.runs_root, args.audit_seed, args.grounding) if p is not None + ] + validate_stage_path(args.stage_dir, sources) + stage = args.stage_dir + bundle = stage / "publish" / RUN_NAME + receipt_path = stage / "stage.json" + if args.step == "prepare": + runs = discover_new_models(args.runs_root) + print("All copied run states qualify; resolving 20260929 base", flush=True) + base, reference, live = resolve_base(args) + print("Base verified; folding copied additions", flush=True) + stage.mkdir(parents=True, exist_ok=True) + require( + not receipt_path.exists(), + "stage already prepared; use a fresh stage-dir " + "or resume judge/triage/export", + ) + bundle, frames = prepare_inputs(args, runs, base, reference) + if args.partial: + partial_scores(args, base, reference, frames) + seed = None + if args.early: + export(args, bundle, live) + else: + seed = prepare_cases(args, bundle) + inputs = [p for p in (stage / "inputs").rglob("*") if p.is_file()] + inputs += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + receipt = { + "partial": args.partial, + "early": args.early, + "base_tag": BASE_TAG, + "base_commit": BASE_COMMIT, + "files": {str(p.relative_to(stage)): digest(p) for p in inputs}, + } + if seed is not None: + receipt["seed"] = seed + write_json(receipt_path, receipt) + else: + receipt = json.loads(receipt_path.read_text()) + require( + not receipt["partial"] and not receipt["early"], + "early/partial stage cannot become a release; prepare fresh full inputs", + ) + for name, expected in receipt["files"].items(): + require( + digest(stage / name) == expected, + f"staged input changed: {name}; prepare a new stage", + ) + (stage / "release-ready.json").unlink(missing_ok=True) + if args.step == "bind-seed": + bind_seed(args) + elif args.step == "judge": + judge(args, bundle) + elif args.step == "triage": + triage(args, bundle) + else: + triage(args, bundle) + export(args, bundle, resolve_live_base(args)) + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_adds0928.py b/scripts/freeze_adds0928.py new file mode 100644 index 00000000..ac60b024 --- /dev/null +++ b/scripts/freeze_adds0928.py @@ -0,0 +1,331 @@ +"""Build the September 29 release locally from a strictly gated stage. + +This adapts the existing freezer in process; it never calls a release upload. +Run only after finish_adds0928.py --step export has written release-ready.json. +The optional --dry-run validates inputs without changing repository files. +""" + +from __future__ import annotations + +import argparse +import functools +import hashlib +import json +import os +import re +import shutil +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.environ["OPENBLAS_NUM_THREADS"] = "1" +sys.path.insert(0, str(ROOT)) + +RUN = "us_full_run_20260612_policyengine_4_16_1_populace" +NEW_MODELS = { + "sonnet55": "claude-sonnet-5.5", + "grok47": "grok-4.7", + "dsflash41": "deepseek-v4.1-flash", +} + + +def read_json(path: Path) -> dict: + """Read an object, with the filename retained in parse failures.""" + return json.loads(path.read_text()) + + +def write_json(path: Path, value: dict) -> None: + """Write reviewable deterministic JSON.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def digest(path: Path) -> str: + """Hash a file without holding its bytes in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def validate_treatment(freezer, state: dict, state_path: Path, scenarios: Path) -> None: + """Check actual treatment against the registry before changing tracked files.""" + from policybench.config import MODELS, PROGRAMS + from policybench.eval_no_tools import ( + _first_request_variables, + _initial_completion_budget_tokens, + _max_repair_rounds, + _request_timeout_seconds, + _thinking_configuration, + ) + from policybench.model_cards import ( + PROMPT_CONTRACT_VERSION, + answer_contract_for, + completion_budget_ceiling_for, + explanation_chunk_size_for, + ) + from policybench.scenarios import load_scenarios_from_manifest + from policybench.spec import expand_programs_for_scenario + + model = state["model"] + provider_id = MODELS[model] + contract = answer_contract_for(provider_id) + first = load_scenarios_from_manifest(scenarios)[0] + variables = _first_request_variables( + provider_id, + expand_programs_for_scenario(PROGRAMS, first), + include_explanations=True, + ) + expected = { + "model_id": provider_id, + "answer_contract": contract, + "chunk_size": explanation_chunk_size_for(provider_id), + "tool_choice_mode": "forced" if contract == "tool" else None, + "prompt_contract_version": PROMPT_CONTRACT_VERSION, + "completion_budget_ceiling": completion_budget_ceiling_for(provider_id), + "initial_completion_budget_tokens": _initial_completion_budget_tokens( + provider_id, variables + ), + "thinking": _thinking_configuration(provider_id), + "request_timeout_seconds": _request_timeout_seconds(provider_id, env={}), + "max_repair_rounds": _max_repair_rounds(env={}), + } + freezer._validate_treatment_fingerprint( + state["treatment_fingerprint"], expected, model=model, run_state_path=state_path + ) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + parser.add_argument("--tag", default="dashboard-data-20260929") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + match = re.fullmatch(r"dashboard-data-(\d{4})(\d{2})(\d{2})[a-z]?", args.tag) + if not match: + parser.error("--tag must be dashboard-data-YYYYMMDD[a-z]") + date = "-".join(match.groups()) + stage = args.stage_dir.resolve() + if not stage.is_relative_to(ROOT): + parser.error("--stage-dir must be inside this checkout") + + import freeze_snapshot as freezer + import pandas as pd + + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + from policybench.dashboard_schema import validate_dashboard_payload + + payload_path = stage / "data-board45.json" + receipt = read_json(stage / "release-ready.json") + payload_hash = digest(payload_path) + if ( + receipt.get("payload_sha256") != payload_hash + or receipt.get("models") != 45 + or receipt.get("partial") is not False + or receipt.get("release_tag") != args.tag + ): + raise SystemExit("Strict export receipt does not match data-board45.json") + if not receipt.get("files"): + raise SystemExit("Strict export receipt is missing staged evidence hashes") + for name, expected in receipt["files"].items(): + source = (stage / name).resolve() + if not source.is_relative_to(stage) or digest(source) != expected: + raise SystemExit(f"Staged evidence changed since strict export: {name}") + payload = read_json(payload_path) + recombined = json.dumps({"countries": {"us": payload["countries"]["us"]}}) + if recombined.encode() != payload_path.read_bytes(): + raise SystemExit("Payload does not recombine to the freeze format") + errors = validate_dashboard_payload(payload, require_failure_annotations=True) + if errors: + raise SystemExit(f"Strict dashboard gate failed: {errors[:5]}") + stats = payload["countries"]["us"]["modelStats"] + model_names = {row["model"] for row in stats} + if len(stats) != 45 or any(row["condition"] != "no_tools" for row in stats): + raise SystemExit("Release must contain exactly 45 no-tools model rows") + + snapshot = ROOT / "paper/snapshot/20260501" + frozen_run = snapshot / "runs" / RUN + serving_path = snapshot / "model_serving_config.json" + previous_serving = read_json(serving_path) + incumbents = set(previous_serving["models"]) - set(NEW_MODELS.values()) + if len(incumbents) != 42 or model_names != incumbents | set(NEW_MODELS.values()): + raise SystemExit("Frozen incumbent roster and staged 45-model roster disagree") + source_run = stage / "publish" / RUN + source_us = source_run / "us" + for name in ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", + ): + if digest(source_us / name) != digest(frozen_run / name): + raise SystemExit( + f"{name} changed: this additive freezer needs reference review" + ) + previous_rows = pd.read_csv(frozen_run / "predictions.csv.gz", low_memory=False) + staged_rows = pd.read_csv(source_us / "predictions.csv", low_memory=False) + for model in sorted(incumbents): + if freezer._model_prediction_rows_sha256(previous_rows, model) != ( + freezer._model_prediction_rows_sha256(staged_rows, model) + ): + raise SystemExit(f"Incumbent prediction evidence changed: {model}") + del previous_rows + + evidence_paths = {} + for slug, model in NEW_MODELS.items(): + state_path = stage / "inputs" / slug / "run_state.json" + state = read_json(state_path) + if ( + state.get("model") != model + or state.get("completed") != 100 + or state.get("total") != 100 + or state.get("stopped_reason") is not None + or not isinstance(state.get("treatment_fingerprint"), dict) + ): + raise SystemExit( + f"Missing completed run or treatment evidence: {state_path}" + ) + validate_treatment(freezer, state, state_path, source_us / "scenarios.csv") + evidence = freezer._run_state_prediction_evidence( + state_path, staged_rows, model + ) + if evidence["kind"] != "run_state": + raise SystemExit( + f"Run-state prediction evidence failed for {model}: {evidence}" + ) + evidence_paths[model] = str(state_path) + del staged_rows + + staged_annotations = source_run / "annotations" + adjudications = freezer.load_adjudications( + staged_annotations / "us_adjudications.json" + ) + from policybench.adjudications import verify_adjudications_applied + + verify_adjudications_applied( + pd.read_csv(staged_annotations / "us_audit_row_annotations.csv"), + pd.read_csv(staged_annotations / "us_case_notes.csv"), + adjudications, + ) + cases_dir = stage / "audit" / "cases" + freezer.verify_adjudications_keep_judge_verdicts(adjudications, cases_dir) + pointer = { + "version": 1, + "repo": "PolicyEngine/policybench", + "tag": args.tag, + "asset": "dashboard-data.json", + "url": ( + "https://github.com/PolicyEngine/policybench/releases/download/" + f"{args.tag}/dashboard-data.json" + ), + "sha256": payload_hash, + "bytes": payload_path.stat().st_size, + } + versions_path = ROOT / "app/src/data.versions.json" + versions = read_json(versions_path) + live = next( + item for item in versions["versions"] if item["id"] == versions["default"] + ) + description, changed = re.subn( + r" - \d+ models$", " - 45 models", live["description"] + ) + if changed != 1: + raise SystemExit("Live version description has an unexpected model-count shape") + live.update(description=description, snapshotLabel=f"Snapshot {date}") + if args.dry_run: + print(f"Validated local release inputs: {args.tag}, 45 models, {payload_hash}") + return + + # A reviewed reference revision (the 2026-09-29 engine upgrade) and the + # audit exclusions final_actions.json lists change the committed reference + # files the manifest pins. freeze_snapshot checks the staged references + # against those pins before it rewrites the manifest, so the pins catch up + # to the committed files first, and only when the driver's gates accept + # them: an engine_upgrade revision listing exactly the value changes + # against the 22c base, every other column unchanged, and exactly the 22c + # exclusions plus the revision's and the audit's. The staged files equal + # the committed ones (above). + from finish_adds0928 import check_exclusions, reference_revision + + revision = reference_revision() + if revision is not None: + check_exclusions(revision, frozen_run) + manifest_path = snapshot / "manifest.json" + manifest = read_json(manifest_path) + pins = manifest["source_run_artifacts"][RUN]["files"] + for name in ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + ): + pins[name] = digest(frozen_run / name) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + # Read-only input paths are all in this checkout. Override defaults that + # captured the old audit directory when freeze_snapshot was imported. + freezer.SNAPSHOT_DATE = date + freezer.MODEL_RESPONSE_DATE = f"2026-06-12 to {date}" + freezer.SOURCE_RUN = source_run + freezer.SOURCE_US = source_us + freezer.SOURCE_ANNOTATIONS = staged_annotations + freezer.REFERENCE_META_SOURCE = source_us / "reference_outputs.csv.meta.json" + freezer.PUBLISHED_DASHBOARD_SOURCE = payload_path + freezer.PUBLISHED_DASHBOARD_ARTIFACT = { + key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes") + } + freezer.RUN_STATE_EVIDENCE = evidence_paths + freezer.AUDIT_CASES_DIR = cases_dir + freezer.audit_judge_provenance = functools.partial( + freezer.audit_judge_provenance, cases_dir=cases_dir + ) + freezer.developer_adjudications_block = functools.partial( + freezer.developer_adjudications_block, cases_dir=cases_dir + ) + original_freeze_serving = freezer.freeze_serving_configuration + + def freeze_serving(destination: Path) -> None: + # The old rows already have frozen evidence. Preserve it byte-for-byte + # after checking prediction identity, instead of silently downgrading + # it when this isolated checkout lacks historical run directories. + original_freeze_serving(destination) + serving = read_json(destination) + for model in incumbents: + serving["models"][model] = previous_serving["models"][model] + for slug, model in NEW_MODELS.items(): + evidence = serving["models"][model]["evidence"] + if evidence["kind"] != "run_state": + raise SystemExit(f"Missing frozen run-state evidence for {model}") + evidence["run"] = f"adds0928/{slug}" + counts = {"run_state": 0, "registry": 0} + for treatment in serving["models"].values(): + counts[treatment["evidence"]["kind"]] += 1 + serving["evidence_summary"] = counts + serving["registry_commit"] = freezer._serving_registry_commit(serving) + write_json(destination, serving) + + freezer.freeze_serving_configuration = freeze_serving + write_json(ROOT / "app/src/data.artifact.json", pointer) + write_json(versions_path, versions) + freezer.ANNOTATIONS_DEST.mkdir(parents=True, exist_ok=True) + shutil.copyfile( + staged_annotations / "us_adjudications.json", + freezer.ANNOTATIONS_DEST / "us_adjudications.json", + ) + freezer.main() + cache = ROOT / "app/.cache" / f"dashboard-data-{payload_hash[:16]}.json" + cache.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(payload_path, cache) + freezer.gzip_deterministic( + source_us / "predictions.csv", + stage / "predictions.csv.gz", + stored_name="predictions.csv", + ) + print(f"Built {args.tag} locally; payload and predictions remain in {stage}") + print("Next: update note and paper prose, render the paper, re-pin, run tests.") + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_gpt61sol.py b/scripts/freeze_gpt61sol.py new file mode 100644 index 00000000..c2297ea4 --- /dev/null +++ b/scripts/freeze_gpt61sol.py @@ -0,0 +1,667 @@ +"""Build the GPT-6.1 Sol release locally from a strictly gated stage. + +Adapted from freeze_adds0928.py. It configures the existing freezer in process +and never calls a release upload. Run only after finish_gpt61sol.py --step +export has written release-ready.json. The optional --dry-run validates inputs +without changing repository files. Both rebuild the payload from the bound +bundle, as export builds it, and refuse a staged payload that differs. This +release has no reference revision: the references, exclusions and scenarios +must equal release 20260929's pinned bytes, and the adjudications may change +only through a staged record that triage applied and export bound. +""" + +from __future__ import annotations + +import argparse +import functools +import hashlib +import json +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.environ["OPENBLAS_NUM_THREADS"] = "1" +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "scripts")) + +import finish_gpt61sol as driver # noqa: E402 + +# The treatment check is release-independent; reuse it unchanged. +from freeze_adds0928 import validate_treatment # noqa: E402 + +RUN = driver.RUN_NAME +NEW_MODELS = dict(driver.MODELS) +BOARD_MODELS = driver.BOARD_MODELS +# Frozen serving evidence names the supervised run by its runs root and slug. +EVIDENCE_RUN_ROOT = "adds202609" +ADJUDICATIONS = "us_adjudications.json" +# The published annotations the adjudications and amendments are applied to. +ANNOTATION_CSVS = ("us_audit_row_annotations.csv", "us_case_notes.csv") + + +def read_json(path: Path) -> dict: + """Read an object, with the filename retained in parse failures.""" + return json.loads(path.read_text()) + + +def write_json(path: Path, value: dict) -> None: + """Write reviewable deterministic JSON.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def digest(path: Path) -> str: + """Hash a file without holding its bytes in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def judged_cases(audit: Path) -> list[str]: + """The cases validate_verdicts requires a verdict for, from the manifest. + + A stage without a manifest has none; the receipt then fails to bind + audit/cases.jsonl. + """ + manifest = audit / "cases.jsonl" + if not manifest.is_file(): + return [] + items = map(json.loads, manifest.read_text().splitlines()) + return [item["case_id"] for item in items if not item["parse_failure_only"]] + + +def verify_verdicts(stage: Path) -> None: + """Every judged verdict passes the driver's own gate against the bound seed. + + Export validates verdicts before it writes the receipt, but the receipt + only binds hashes: a kept verdict edited afterwards, its sidecar and + receipt entry re-hashed, must still be the seed's byte for byte, and every + new one bound to its prompt and its Opus 5.5 judge. Without + ``remove_invalid`` validate_verdicts changes nothing. + """ + pending = driver.validate_verdicts(stage / "audit", seed=driver.load_seed(stage)) + if pending: + raise SystemExit( + f"{len(pending)} staged verdicts fail validation: {pending[:8]}; " + "run judge and export again" + ) + + +def verify_receipt(stage: Path, payload_path: Path, tag: str) -> dict: + """The strict export receipt must bind this payload and all staged evidence.""" + receipt = read_json(stage / "release-ready.json") + if ( + receipt.get("payload_sha256") != digest(payload_path) + or receipt.get("models") != BOARD_MODELS + or receipt.get("partial") is not False + or receipt.get("release_tag") != tag + ): + raise SystemExit( + f"Strict export receipt does not match {payload_path.name} " + f"(release tag {tag!r}; re-export after changing RELEASE_TAG)" + ) + if ( + receipt.get("base_tag") != driver.BASE_TAG + or receipt.get("base_sha256") != driver.BASE_SHA256 + ): + raise SystemExit( + "Strict export receipt does not name release 20260929 as its base " + f"({receipt.get('base_tag')!r}, {receipt.get('base_sha256')!r})" + ) + if not receipt.get("files"): + raise SystemExit("Strict export receipt is missing staged evidence hashes") + for name, expected in receipt["files"].items(): + source = (stage / name).resolve() + if ( + not source.is_relative_to(stage) + or not source.is_file() + or digest(source) != expected + ): + raise SystemExit(f"Staged evidence changed since strict export: {name}") + bundle = Path("publish") / RUN + required = [bundle / "us" / name for name in driver.REFERENCE_FILES] + required += [bundle / "us/predictions.csv", bundle / "annotations" / ADJUDICATIONS] + required += [bundle / "annotations" / name for name in ANNOTATION_CSVS] + # Each new model's pinned run files, and the prepare-time hashes. + required += [ + Path("inputs") / slug / name + for slug in NEW_MODELS + for name in driver.PINNED_INPUTS + ] + required += [Path("model-provenance.json")] + # What the adjudication and verdict gates allow rests on these. + required += [Path(driver.PROMPT_CHANGES), Path("stage.json")] + if (stage / driver.AMENDMENTS).exists(): + required.append(Path(driver.AMENDMENTS)) + # Every judged case's audit evidence, as validate_verdicts reads it: an + # unbound verdict or sidecar could be edited and re-hashed after export. + required += [Path("audit/cases.jsonl"), Path("audit/schema.json")] + required += [ + Path("audit/cases") / case / name + for case in judged_cases(stage / "audit") + for name in ("verdict.json", "verdict.meta.json", "prompt.md") + ] + unbound = [str(p) for p in required if str(p) not in receipt["files"]] + if unbound: + raise SystemExit(f"Strict export receipt does not bind: {unbound}") + return receipt + + +def verify_judge_provenance( + stage: Path, receipt: dict, rejudged: frozenset[str] +) -> None: + """The judge provenance record is the one export bound, and still true. + + The record is a repository file (driver.JUDGE_PROVENANCE), so the receipt + binds it apart from the stage's files. Its bytes must be the ones export + hashed, and it must still describe the staged new verdicts and their + transcripts, as export checked. + """ + bound = receipt.get("judge_provenance") + if ( + not isinstance(bound, dict) + or bound.get("path") != driver.JUDGE_PROVENANCE_PATH + or not driver.JUDGE_PROVENANCE.is_file() + or bound.get("sha256") != digest(driver.JUDGE_PROVENANCE) + ): + raise SystemExit( + f"{driver.JUDGE_PROVENANCE_PATH} is not the judge provenance record " + "the strict export bound; export again" + ) + driver.verify_judge_provenance( + stage / "audit" / "cases", rejudged, driver.JUDGE_PROVENANCE + ) + + +def verify_references(source_us: Path, frozen_run: Path, manifest: dict) -> None: + """Staged, committed and manifest-pinned references all equal 20260929's.""" + pins = manifest["source_run_artifacts"][RUN]["files"] + for name, pin in driver.BASE_REFERENCE_SHA256.items(): + if digest(source_us / name) != pin: + raise SystemExit(f"Staged {name} changed: this release has no revision") + if digest(frozen_run / name) != pin: + raise SystemExit(f"Committed {name} changed: this release has no revision") + if pins.get(name) != pin: + raise SystemExit(f"Manifest pin for {name} is not release 20260929's") + + +def verify_adjudication_record( + staged: Path, + source_us: Path, + rejudged: frozenset[str], + amendments: list[dict], + cases_dir: Path | None = None, +) -> int: + """The staged record may change only where GPT-6.1 Sol re-opened a case. + + The baseline is release 20260929's record in git (BASE_COMMIT), never the + working-tree copy this freeze overwrites. A re-opened case may rewrite its + judge fields, and its reasoning exactly as the listed wording amendments + say; nothing else in any committed entry may change, and the scoring + exclusions stay release 20260929's. Returns how many staged decisions the + committed record lacks. The staged file's bytes must be exactly its + parsed content, with 20260929's note and conventions; with ``cases_dir``, + each re-opened entry must be restated as the restate script does. The + staged record, prompt-changes.json and the amendments are bound by the + receipt, which export writes only after triage applied them. + """ + from policybench.adjudications import excluded_case_keys, load_adjudications + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + driver.verify_record_form(staged.read_text(), driver.base_adjudication_record()) + after = load_adjudications(staged) + base = driver.base_adjudications() + added = driver.verify_adjudication_changes(base, after, rejudged, amendments) + if cases_dir is not None: + driver.verify_restatements(base, after, rejudged, cases_dir) + if excluded_case_keys(after) != exclusion_keys( + load_reference_exclusions(source_us) + ): + raise SystemExit( + "Staged adjudications change the scoring exclusions; this release " + "has no reference revision" + ) + return added + + +def verify_incumbent_stats(stats: list[dict], live: dict) -> None: + """Every incumbent's modelStats row is release 20260929's, byte for byte. + + Export refuses any drift before it writes the receipt, but the receipt + binds only the payload's hash: a payload edited after export, with its + receipt rehashed, would otherwise publish incumbent statistics export + refuses. ``live`` is release 20260929's payload from git (BASE_COMMIT, + driver.base_payload_from_commit), checked against BASE_SHA256, as a + re-export reads it; the comparison is export's own incumbent_drift. + """ + previous = {row["model"]: row for row in live["countries"]["us"]["modelStats"]} + if set(previous) & set(NEW_MODELS.values()): + raise SystemExit("Release 20260929's modelStats already hold the addition") + drift = driver.incumbent_drift(stats, previous) + if drift: + raise SystemExit(f"Incumbent modelStats drift from release 20260929: {drift}") + + +def payload_differences( + rebuilt, staged, path: str = "payload", limit: int = 5 +) -> list[str]: + """The first ``limit`` paths at which two parsed JSON values differ. + + Key order counts, as it does in the bytes: the paths are empty exactly + when the two values serialize to the same JSON. A list item that names a + model is labelled with it. + """ + found: list[str] = [] + + def walk(a, b, at: str) -> None: + if len(found) >= limit: + return + if isinstance(a, dict) and isinstance(b, dict): + for key in a: + if key in b: + walk(a[key], b[key], f"{at}.{key}") + else: + found.append(f"{at}.{key} (only rebuilt)") + found.extend(f"{at}.{key} (only staged)" for key in b if key not in a) + if a.keys() == b.keys() and list(a) != list(b): + found.append(f"{at} (key order)") + elif isinstance(a, list) and isinstance(b, list): + for index, (x, y) in enumerate(zip(a, b)): + name = x.get("model") if isinstance(x, dict) else None + label = f"{index} {name}" if isinstance(name, str) else index + walk(x, y, f"{at}[{label}]") + if len(a) != len(b): + found.append(f"{at} (length {len(a)} rebuilt, {len(b)} staged)") + elif json.dumps(a) != json.dumps(b): + found.append(at) + + walk(rebuilt, staged, path) + return found[:limit] + + +def rebuild_payload(stage: Path, receipt: dict, payload_path: Path, live: dict) -> None: + """The staged payload must be what export builds from the bound bundle. + + The receipt binds the payload only by a hash that sits beside it, so a + payload edited after export (the addition's scores, a case's classes) with + its receipt rehashed passes every other gate. Export's own build + (driver.build_payload, against release 20260929 as ``live``) runs again + on a scratch copy of the bundle files the receipt binds, each checked + against its receipt hash, and its bytes must equal the staged payload's. + The copy is the bundle's: export_full_run writes data.json, us/data.json + and us/analysis/ into the bundle it reads, and its data.json lacks the + carried usage, so a rebuild in place would rewrite the stage. + """ + bundle = Path("publish") / RUN + with tempfile.TemporaryDirectory(prefix="freeze-gpt61sol-") as scratch: + copy = Path(scratch) / RUN + for name, expected in receipt["files"].items(): + if not Path(name).is_relative_to(bundle): + continue + target = copy / Path(name).relative_to(bundle) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(stage / name, target) + if digest(target) != expected: + raise SystemExit(f"Staged evidence changed since strict export: {name}") + rebuilt = driver.payload_text(driver.build_payload(copy, live)) + staged = payload_path.read_bytes() + if rebuilt.encode() != staged: + differ = payload_differences(json.loads(rebuilt), json.loads(staged)) + raise SystemExit( + f"Staged {payload_path.name} is not what export builds from the " + f"bound bundle; it differs at {differ or ['its serialization']}; " + "export again" + ) + + +def base_prediction_rows(): + """Release 20260929's predictions, read from git at BASE_COMMIT. + + Never the working-tree copy, which the freezer rewrites from the stage: a + freeze that stopped partway would compare the staged rows with themselves. + """ + import io + + import pandas as pd + + path = Path("paper/snapshot/20260501/runs") / RUN / "predictions.csv.gz" + return pd.read_csv( + io.BytesIO(driver.base_commit_blob(path)), compression="gzip", low_memory=False + ) + + +def base_serving_config() -> dict: + """Release 20260929's serving configuration, read from git at BASE_COMMIT.""" + path = Path("paper/snapshot/20260501/model_serving_config.json") + return json.loads(driver.base_commit_blob(path)) + + +def annotation_differences( + rebuilt_text: str, staged: Path, key: list[str], limit: int = 4 +) -> list[str]: + """The first ``limit`` places where a staged annotation CSV is not the + rebuilt one: a column set or order, a row only one side has, or a cell + (named by its case, model and column). Empty when the two parse to the + same cells, so only the row order or the serialization differs.""" + import io + + import pandas as pd + + def read(source) -> pd.DataFrame: + return pd.read_csv(source, dtype=str, keep_default_na=False) + + try: + rebuilt, published = read(io.StringIO(rebuilt_text)), read(staged) + except ValueError: + return ["its serialization"] + if list(rebuilt.columns) != list(published.columns): + return [f"its columns {list(published.columns)}, not {list(rebuilt.columns)}"] + + def by_key(frame: pd.DataFrame) -> dict[tuple, dict]: + return {tuple(row[k] for k in key): row for row in frame.to_dict("records")} + + found = [] + if published.duplicated(key).any(): + found.append(f"duplicate {key} rows") + built, staged_rows = by_key(rebuilt), by_key(published) + for item in sorted(set(built) | set(staged_rows)): + name = "__".join(item) + if item not in staged_rows or item not in built: + side = "rebuilt" if item in built else "staged" + found.append(f"{name} (only {side})") + continue + found += [ + f"{name} {column}" + for column in rebuilt.columns + if built[item][column] != staged_rows[item][column] + ] + return found[:limit] + + +def verify_annotation_amendments( + annotations: Path, amendments: list[dict], audit: Path +) -> None: + """The staged row annotations and case notes are what triage builds from + the verdicts, the staged adjudications and the listed amendments, in every + column. + + Both frames are rebuilt as triage builds them: collect_audit folds each + verdict into its rows (the model's failure class and diagnosis) and its + case note (the case's classes, flag, hypothesis and rationale); the staged + adjudication record, whose reasoning verify_adjudication_record has + already replayed, sets each decided case's classes and appends its + sentence to the case note; then the listed amendments are replayed in + order, each old wording found exactly once when it is applied. Each staged + file must be the rebuilt frame's bytes as triage writes them + (driver.annotation_csv_text), so a failure class, flag or count edited on + a case no adjudication decides is refused as surely as unlisted wording, + even beside a listed fragment. + """ + from policybench.adjudications import apply_adjudications, load_adjudications + from policybench.audit import collect_audit + + collected = collect_audit(annotations.parent / "us", audit) + rows, cases, _ = apply_adjudications( + collected["row"], + collected["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ), + load_adjudications(annotations / ADJUDICATIONS), + ) + driver.amend_annotations(rows, cases, amendments) + for name, rebuilt, key in ( + ("us_case_notes.csv", cases, driver.KEY), + ("us_audit_row_annotations.csv", rows, [*driver.KEY, "model"]), + ): + text = driver.annotation_csv_text(rebuilt) + if text.encode() != (annotations / name).read_bytes(): + differ = annotation_differences(text, annotations / name, ["country", *key]) + raise SystemExit( + f"Staged {name} is not what triage builds from the verdicts, the " + "staged adjudications and the listed wording amendments; it " + f"differs at {differ or ['its row order or serialization']}; run " + "triage and export again" + ) + + +def freeze_amendments(stage: Path, destination: Path) -> None: + """Commit the stage's wording amendments beside the adjudication record. + + The amended case notes, row annotations and reasoning are published; the + list says what changed, from what, and why. + """ + source = stage / driver.AMENDMENTS + if source.exists() and json.loads(source.read_text())["amendments"]: + shutil.copyfile(source, destination / f"us_{driver.AMENDMENTS}") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + parser.add_argument("--tag", default=driver.RELEASE_TAG) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + match = re.fullmatch(r"dashboard-data-(\d{4})(\d{2})(\d{2})[a-z]?", args.tag) + if not match: + parser.error("--tag must be dashboard-data-YYYYMMDD[a-z]") + date = "-".join(match.groups()) + stage = args.stage_dir.resolve() + if not stage.is_relative_to(ROOT): + parser.error("--stage-dir must be inside this checkout") + + import freeze_snapshot as freezer + import pandas as pd + + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + from policybench.dashboard_schema import validate_dashboard_payload + + payload_path = stage / f"data-board{BOARD_MODELS}.json" + receipt = verify_receipt(stage, payload_path, args.tag) + # GPT-6.1 Sol's run files are the committed pins' and its bundle rows are + # its run file's, every column: its cost, tokens and latency included. + driver.verify_new_model_inputs(stage) + payload_hash = digest(payload_path) + payload = read_json(payload_path) + recombined = json.dumps({"countries": {"us": payload["countries"]["us"]}}) + if recombined.encode() != payload_path.read_bytes(): + raise SystemExit("Payload does not recombine to the freeze format") + errors = validate_dashboard_payload(payload, require_failure_annotations=True) + if errors: + raise SystemExit(f"Strict dashboard gate failed: {errors[:5]}") + stats = payload["countries"]["us"]["modelStats"] + model_names = {row["model"] for row in stats} + if len(stats) != BOARD_MODELS or any( + row["condition"] != "no_tools" for row in stats + ): + raise SystemExit( + f"Release must contain exactly {BOARD_MODELS} no-tools model rows" + ) + + snapshot = ROOT / "paper/snapshot/20260501" + frozen_run = snapshot / "runs" / RUN + # The incumbents' frozen serving evidence is release 20260929's, from git. + previous_serving = base_serving_config() + incumbents = set(previous_serving["models"]) - set(NEW_MODELS.values()) + if len(incumbents) != driver.BASE_MODELS or model_names != incumbents | set( + NEW_MODELS.values() + ): + raise SystemExit( + f"Frozen incumbent roster and staged {BOARD_MODELS}-model roster disagree" + ) + base = driver.base_payload_from_commit() + verify_incumbent_stats(stats, base) + source_run = stage / "publish" / RUN + source_us = source_run / "us" + verify_references(source_us, frozen_run, read_json(snapshot / "manifest.json")) + del payload, stats + rebuild_payload(stage, receipt, payload_path, base) + del base + staged_annotations = source_run / "annotations" + verify_verdicts(stage) + rejudged = driver.rejudged_cases(stage) + verify_judge_provenance(stage, receipt, rejudged) + amendments = driver.load_amendments(stage, rejudged) + added = verify_adjudication_record( + staged_annotations / ADJUDICATIONS, + source_us, + rejudged, + amendments, + stage / "audit" / "cases", + ) + verify_annotation_amendments(staged_annotations, amendments, stage / "audit") + previous_rows = base_prediction_rows() + staged_rows = pd.read_csv(source_us / "predictions.csv", low_memory=False) + for model in sorted(incumbents): + if freezer._model_prediction_rows_sha256(previous_rows, model) != ( + freezer._model_prediction_rows_sha256(staged_rows, model) + ): + raise SystemExit(f"Incumbent prediction evidence changed: {model}") + del previous_rows + + evidence_paths = {} + for slug, model in NEW_MODELS.items(): + state_path = stage / "inputs" / slug / "run_state.json" + state = read_json(state_path) + if ( + state.get("model") != model + or state.get("completed") != 100 + or state.get("total") != 100 + or state.get("stopped_reason") is not None + or not isinstance(state.get("treatment_fingerprint"), dict) + ): + raise SystemExit( + f"Missing completed run or treatment evidence: {state_path}" + ) + validate_treatment(freezer, state, state_path, source_us / "scenarios.csv") + evidence = freezer._run_state_prediction_evidence( + state_path, staged_rows, model + ) + if evidence["kind"] != "run_state": + raise SystemExit( + f"Run-state prediction evidence failed for {model}: {evidence}" + ) + evidence_paths[model] = str(state_path) + del staged_rows + + adjudications = freezer.load_adjudications(staged_annotations / ADJUDICATIONS) + from policybench.adjudications import verify_adjudications_applied + + verify_adjudications_applied( + pd.read_csv(staged_annotations / "us_audit_row_annotations.csv"), + pd.read_csv(staged_annotations / "us_case_notes.csv"), + adjudications, + ) + cases_dir = stage / "audit" / "cases" + freezer.verify_adjudications_keep_judge_verdicts(adjudications, cases_dir) + pointer = { + "version": 1, + "repo": "PolicyEngine/policybench", + "tag": args.tag, + "asset": "dashboard-data.json", + "url": ( + "https://github.com/PolicyEngine/policybench/releases/download/" + f"{args.tag}/dashboard-data.json" + ), + "sha256": payload_hash, + "bytes": payload_path.stat().st_size, + } + versions_path = ROOT / "app/src/data.versions.json" + versions = read_json(versions_path) + live = next( + item for item in versions["versions"] if item["id"] == versions["default"] + ) + description, changed = re.subn( + r" - \d+ models$", f" - {BOARD_MODELS} models", live["description"] + ) + if changed != 1: + raise SystemExit("Live version description has an unexpected model-count shape") + live.update(description=description, snapshotLabel=f"Snapshot {date}") + if args.dry_run: + print( + f"Validated local release inputs: {args.tag}, {BOARD_MODELS} models, " + f"{payload_hash}; references unchanged; {added} adjudications added; " + f"{len(amendments)} wording amendments" + ) + return + + # Read-only input paths are all in this checkout. Override defaults that + # captured the old audit directory when freeze_snapshot was imported. + freezer.SNAPSHOT_DATE = date + freezer.MODEL_RESPONSE_DATE = f"2026-06-12 to {date}" + freezer.SOURCE_RUN = source_run + freezer.SOURCE_US = source_us + freezer.SOURCE_ANNOTATIONS = staged_annotations + freezer.REFERENCE_META_SOURCE = source_us / "reference_outputs.csv.meta.json" + freezer.PUBLISHED_DASHBOARD_SOURCE = payload_path + freezer.PUBLISHED_DASHBOARD_ARTIFACT = { + key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes") + } + freezer.RUN_STATE_EVIDENCE = evidence_paths + freezer.AUDIT_CASES_DIR = cases_dir + freezer.audit_judge_provenance = functools.partial( + freezer.audit_judge_provenance, cases_dir=cases_dir + ) + freezer.developer_adjudications_block = functools.partial( + freezer.developer_adjudications_block, cases_dir=cases_dir + ) + original_freeze_serving = freezer.freeze_serving_configuration + + def freeze_serving(destination: Path) -> None: + # The old rows already have frozen evidence. Preserve it byte-for-byte + # after checking prediction identity, instead of silently downgrading + # it when this isolated checkout lacks historical run directories. + original_freeze_serving(destination) + serving = read_json(destination) + for model in incumbents: + serving["models"][model] = previous_serving["models"][model] + for slug, model in NEW_MODELS.items(): + evidence = serving["models"][model]["evidence"] + if evidence["kind"] != "run_state": + raise SystemExit(f"Missing frozen run-state evidence for {model}") + evidence["run"] = f"{EVIDENCE_RUN_ROOT}/{slug}" + counts = {"run_state": 0, "registry": 0} + for treatment in serving["models"].values(): + counts[treatment["evidence"]["kind"]] += 1 + serving["evidence_summary"] = counts + serving["registry_commit"] = freezer._serving_registry_commit(serving) + write_json(destination, serving) + + freezer.freeze_serving_configuration = freeze_serving + write_json(ROOT / "app/src/data.artifact.json", pointer) + write_json(versions_path, versions) + freezer.ANNOTATIONS_DEST.mkdir(parents=True, exist_ok=True) + shutil.copyfile( + staged_annotations / ADJUDICATIONS, + freezer.ANNOTATIONS_DEST / ADJUDICATIONS, + ) + freezer.main() + freeze_amendments(stage, freezer.ANNOTATIONS_DEST) + # The freezer copies the staged references byte for byte; confirm it. + driver.verify_reference_pins(frozen_run, "frozen reference") + cache = ROOT / "app/.cache" / f"dashboard-data-{payload_hash[:16]}.json" + cache.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(payload_path, cache) + freezer.gzip_deterministic( + source_us / "predictions.csv", + stage / "predictions.csv.gz", + stored_name="predictions.csv", + ) + print(f"Built {args.tag} locally; payload and predictions remain in {stage}") + print("Next: update note and paper prose, render the paper, re-pin, run tests.") + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index bf729716..ccea1910 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -13,7 +13,7 @@ * ``paper/snapshot//runs/